diff --git a/packages/backend/server/src/__tests__/event/eventbus.spec.ts b/packages/backend/server/src/__tests__/event/eventbus.spec.ts index be247227a7..afb4b23054 100644 --- a/packages/backend/server/src/__tests__/event/eventbus.spec.ts +++ b/packages/backend/server/src/__tests__/event/eventbus.spec.ts @@ -187,7 +187,7 @@ test('should generate request id for event', async t => { await eventbus.emitAsync('__test__.requestId', {}); - t.true(listeners.onRequestId.lastCall.returnValue.includes(':event/')); + t.true(listeners.onRequestId.lastCall.returnValue.includes(':event:')); }); test('should continuously use the same request id', async t => { diff --git a/packages/backend/server/src/__tests__/team.e2e.ts b/packages/backend/server/src/__tests__/team.e2e.ts index b968dc7aff..5aa05e4fd9 100644 --- a/packages/backend/server/src/__tests__/team.e2e.ts +++ b/packages/backend/server/src/__tests__/team.e2e.ts @@ -10,7 +10,7 @@ import Sinon from 'sinon'; import { AppModule } from '../app.module'; import { EventBus } from '../base'; import { AuthService } from '../core/auth'; -import { DocContentService } from '../core/doc-renderer'; +import { DocReader } from '../core/doc'; import { DocRole, PermissionService, WorkspaceRole } from '../core/permission'; import { WorkspaceType } from '../core/workspaces'; import { Models } from '../models'; @@ -53,7 +53,7 @@ test.before(async t => { module .overrideProvider(EventBus) .useValue(Sinon.createStubInstance(EventBus)); - module.overrideProvider(DocContentService).useValue({ + module.overrideProvider(DocReader).useValue({ getWorkspaceContent() { return { name: 'test', diff --git a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts index a0fe401c2c..1eeb40a3da 100644 --- a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts +++ b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts @@ -226,7 +226,7 @@ test('should generate request id', async t => { await executor.run('nightly.__test__requestId', {}); - t.true(spy.returnValues.some(v => v.includes(':job/'))); + t.true(spy.returnValues.some(v => v.includes(':job:'))); spy.restore(); }); diff --git a/packages/backend/server/src/base/utils/request.ts b/packages/backend/server/src/base/utils/request.ts index f39646d42d..ee9e0642be 100644 --- a/packages/backend/server/src/base/utils/request.ts +++ b/packages/backend/server/src/base/utils/request.ts @@ -94,7 +94,7 @@ export function parseCookies( export type RequestType = GqlContextType | 'event' | 'job'; export function genRequestId(type: RequestType) { - return `${AFFiNE.flavor.type}:${type}/${randomUUID()}`; + return `${AFFiNE.flavor.type}:${type}:${randomUUID()}`; } export function getOrGenRequestId(type: RequestType) { diff --git a/packages/backend/server/src/core/doc-renderer/__tests__/service.spec.ts b/packages/backend/server/src/core/doc-renderer/__tests__/service.spec.ts deleted file mode 100644 index f7f104773f..0000000000 --- a/packages/backend/server/src/core/doc-renderer/__tests__/service.spec.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { randomUUID } from 'node:crypto'; - -import { User, Workspace } from '@prisma/client'; -import ava, { TestFn } from 'ava'; -import { Doc as YDoc } from 'yjs'; - -import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { AppModule } from '../../../app.module'; -import { Config } from '../../../base'; -import { ConfigModule } from '../../../base/config'; -import { Models } from '../../../models'; -import { PgWorkspaceDocStorageAdapter } from '../../doc'; -import { DocContentService } from '..'; - -const test = ava as TestFn<{ - models: Models; - app: TestingApp; - docContentService: DocContentService; - config: Config; - adapter: PgWorkspaceDocStorageAdapter; -}>; - -test.before(async t => { - const app = await createTestingApp({ - imports: [ - ConfigModule.forRoot({ - flavor: { - doc: false, - }, - docService: { - endpoint: '', - }, - }), - AppModule, - ], - }); - - t.context.models = app.get(Models); - t.context.docContentService = app.get(DocContentService); - t.context.config = app.get(Config); - t.context.adapter = app.get(PgWorkspaceDocStorageAdapter); - t.context.app = app; -}); - -let user: User; -let workspace: Workspace; - -test.beforeEach(async t => { - t.context.config.docService.endpoint = t.context.app.url(); - await t.context.app.initTestingDB(); - user = await t.context.models.user.create({ - email: 'test@affine.pro', - }); - workspace = await t.context.models.workspace.create(user.id); -}); - -test.after.always(async t => { - await t.context.app.close(); -}); - -test('should get doc content from doc service rpc', async t => { - const docId = randomUUID(); - const { docContentService } = t.context; - - const doc = new YDoc(); - const text = doc.getText('content'); - const updates: Buffer[] = []; - - doc.on('update', update => { - updates.push(Buffer.from(update)); - }); - - text.insert(0, 'hello'); - text.insert(5, 'world'); - text.insert(5, ' '); - - await t.context.adapter.pushDocUpdates(workspace.id, docId, updates, user.id); - - const docContent = await docContentService.getPageContent( - workspace.id, - docId - ); - // TODO(@fengmk2): should create a test ydoc with blocks - t.is(docContent, null); -}); diff --git a/packages/backend/server/src/core/doc-renderer/controller.ts b/packages/backend/server/src/core/doc-renderer/controller.ts index 76998f9515..4bce9c34a6 100644 --- a/packages/backend/server/src/core/doc-renderer/controller.ts +++ b/packages/backend/server/src/core/doc-renderer/controller.ts @@ -8,8 +8,8 @@ import isMobile from 'is-mobile'; import { Config, metrics, URLHelper } from '../../base'; import { htmlSanitize } from '../../native'; import { Public } from '../auth'; +import { DocReader } from '../doc'; import { PermissionService } from '../permission'; -import { DocContentService } from './service'; interface RenderOptions { title: string; @@ -50,7 +50,7 @@ export class DocRendererController { private readonly mobileAssets: HtmlAssets = defaultAssets; constructor( - private readonly doc: DocContentService, + private readonly doc: DocReader, private readonly permission: PermissionService, private readonly config: Config, private readonly url: URLHelper @@ -114,7 +114,7 @@ export class DocRendererController { } if (allowUrlPreview) { - return this.doc.getPageContent(workspaceId, docId); + return this.doc.getDocContent(workspaceId, docId); } return null; diff --git a/packages/backend/server/src/core/doc-renderer/event.ts b/packages/backend/server/src/core/doc-renderer/event.ts deleted file mode 100644 index 778b104f95..0000000000 --- a/packages/backend/server/src/core/doc-renderer/event.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { FactoryProvider } from '@nestjs/common'; - -import { Config, OnEvent } from '../../base'; -import { DocContentService } from './service'; - -class DocEventsListener { - constructor(private readonly doc: DocContentService) {} - - @OnEvent('doc.snapshot.updated') - async handleDocSnapshotUpdated({ - workspaceId, - docId, - }: Events['doc.snapshot.updated']) { - await this.doc.markDocContentCacheStale(workspaceId, docId); - } -} - -export const DocEventsListenerProvider: FactoryProvider = { - provide: DocEventsListener, - useFactory: (config: Config, doc: DocContentService) => { - if (config.flavor.renderer) { - return new DocEventsListener(doc); - } - return; - }, - inject: [Config, DocContentService], -}; diff --git a/packages/backend/server/src/core/doc-renderer/index.ts b/packages/backend/server/src/core/doc-renderer/index.ts index de7c3f570c..203eea66f0 100644 --- a/packages/backend/server/src/core/doc-renderer/index.ts +++ b/packages/backend/server/src/core/doc-renderer/index.ts @@ -3,15 +3,9 @@ import { Module } from '@nestjs/common'; import { DocStorageModule } from '../doc'; import { PermissionModule } from '../permission'; import { DocRendererController } from './controller'; -import { DocEventsListenerProvider } from './event'; -import { DocContentService } from './service'; @Module({ imports: [DocStorageModule, PermissionModule], - providers: [DocContentService, DocEventsListenerProvider], controllers: [DocRendererController], - exports: [DocContentService], }) export class DocRendererModule {} - -export { DocContentService }; diff --git a/packages/backend/server/src/core/doc-renderer/service.ts b/packages/backend/server/src/core/doc-renderer/service.ts deleted file mode 100644 index 842b4d359f..0000000000 --- a/packages/backend/server/src/core/doc-renderer/service.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { applyUpdate, Doc } from 'yjs'; - -import { Cache } from '../../base'; -import { DocReader } from '../doc'; -import { - type PageDocContent, - parsePageDoc, - parseWorkspaceDoc, - type WorkspaceDocContent, -} from '../utils/blocksuite'; - -@Injectable() -export class DocContentService { - constructor( - private readonly cache: Cache, - private readonly docReader: DocReader - ) {} - - async getPageContent( - workspaceId: string, - guid: string - ): Promise { - const cacheKey = `workspace:${workspaceId}:doc:${guid}:content`; - const cachedResult = await this.cache.get(cacheKey); - - if (cachedResult) { - return cachedResult; - } - - const docRecord = await this.docReader.getDoc(workspaceId, guid); - if (!docRecord) { - return null; - } - - const doc = new Doc(); - applyUpdate(doc, docRecord.bin); - - const content = parsePageDoc(doc); - - if (content) { - await this.cache.set(cacheKey, content, { - ttl: - 7 * - 24 * - 60 * - 60 * - 1000 /* TODO(@forehalo): we need time constants helper */, - }); - } - return content; - } - - async getWorkspaceContent( - workspaceId: string - ): Promise { - const cacheKey = `workspace:${workspaceId}:content`; - const cachedResult = await this.cache.get(cacheKey); - - if (cachedResult) { - return cachedResult; - } - - const docRecord = await this.docReader.getDoc(workspaceId, workspaceId); - if (!docRecord) { - return null; - } - - const doc = new Doc(); - applyUpdate(doc, docRecord.bin); - - const content = parseWorkspaceDoc(doc); - - if (content) { - await this.cache.set(cacheKey, content); - } - - return content; - } - - async markDocContentCacheStale(workspaceId: string, docId: string) { - const key = - workspaceId === docId - ? `workspace:${workspaceId}:content` - : `workspace:${workspaceId}:doc:${docId}:content`; - await this.cache.delete(key); - } -} diff --git a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts index a4cb646a78..b56a006c8a 100644 --- a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts +++ b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { mock } from 'node:test'; import { User, Workspace } from '@prisma/client'; import ava, { TestFn } from 'ava'; @@ -8,11 +9,13 @@ import { AppModule } from '../../../app.module'; import { CryptoHelper } from '../../../base'; import { ConfigModule } from '../../../base/config'; import { Models } from '../../../models'; +import { DatabaseDocReader } from '../../doc'; const test = ava as TestFn<{ models: Models; app: TestingApp; crypto: CryptoHelper; + databaseDocReader: DatabaseDocReader; }>; test.before(async t => { @@ -23,6 +26,7 @@ test.before(async t => { t.context.models = app.get(Models); t.context.crypto = app.get(CryptoHelper); t.context.app = app; + t.context.databaseDocReader = app.get(DatabaseDocReader); }); let user: User; @@ -119,3 +123,101 @@ test('should return doc when found', async t => { t.is(res.headers['x-doc-timestamp'], timestamp.toString()); t.is(res.headers['x-doc-editor-id'], user.id); }); + +test('should 404 when doc diff not found', async t => { + const { app } = t.context; + + const workspaceId = '123'; + const docId = '123'; + await app + .POST(`/rpc/workspaces/${workspaceId}/docs/${docId}/diff`) + .set('x-access-token', t.context.crypto.sign(docId)) + .expect({ + status: 404, + code: 'Not Found', + type: 'RESOURCE_NOT_FOUND', + name: 'NOT_FOUND', + message: 'Doc not found', + }) + .expect(404); + t.pass(); +}); + +test('should 404 when doc content not found', async t => { + const { app } = t.context; + + const workspaceId = '123'; + const docId = '123'; + await app + .GET(`/rpc/workspaces/${workspaceId}/docs/${docId}/content`) + .set('x-access-token', t.context.crypto.sign(docId)) + .expect({ + status: 404, + code: 'Not Found', + type: 'RESOURCE_NOT_FOUND', + name: 'NOT_FOUND', + message: 'Doc not found', + }) + .expect(404); + t.pass(); +}); + +test('should get doc content in json format', async t => { + const { app } = t.context; + mock.method(t.context.databaseDocReader, 'getDocContent', async () => { + return { + title: 'test title', + summary: 'test summary', + }; + }); + + const docId = randomUUID(); + await app + .GET(`/rpc/workspaces/${workspace.id}/docs/${docId}/content`) + .set('x-access-token', t.context.crypto.sign(docId)) + .expect({ + title: 'test title', + summary: 'test summary', + }) + .expect(200); + t.pass(); +}); + +test('should 404 when workspace content not found', async t => { + const { app } = t.context; + + const workspaceId = '123'; + await app + .GET(`/rpc/workspaces/${workspaceId}/content`) + .set('x-access-token', t.context.crypto.sign(workspaceId)) + .expect({ + status: 404, + code: 'Not Found', + type: 'RESOURCE_NOT_FOUND', + name: 'NOT_FOUND', + message: 'Workspace not found', + }) + .expect(404); + t.pass(); +}); + +test('should get workspace content in json format', async t => { + const { app } = t.context; + mock.method(t.context.databaseDocReader, 'getWorkspaceContent', async () => { + return { + name: 'test name', + avatarKey: 'avatar key', + }; + }); + + const workspaceId = randomUUID(); + await app + .GET(`/rpc/workspaces/${workspaceId}/content`) + .set('x-access-token', t.context.crypto.sign(workspaceId)) + .expect(200) + .expect({ + name: 'test name', + avatarKey: 'avatar key', + }); + t.pass(); +}); diff --git a/packages/backend/server/src/core/doc-service/controller.ts b/packages/backend/server/src/core/doc-service/controller.ts index 1f949be967..9bf3d9aa75 100644 --- a/packages/backend/server/src/core/doc-service/controller.ts +++ b/packages/backend/server/src/core/doc-service/controller.ts @@ -70,4 +70,31 @@ export class DocRpcController { ); res.send(Buffer.concat([diff.missing, diff.state])); } + + @SkipThrottle() + @Internal() + @Get('/workspaces/:workspaceId/docs/:docId/content') + async getDocContent( + @Param('workspaceId') workspaceId: string, + @Param('docId') docId: string + ) { + const content = await this.docReader.getDocContent(workspaceId, docId); + if (!content) { + throw new NotFound('Doc not found'); + } + this.logger.log(`get doc content ${docId} from workspace ${workspaceId}`); + return content; + } + + @SkipThrottle() + @Internal() + @Get('/workspaces/:workspaceId/content') + async getWorkspaceContent(@Param('workspaceId') workspaceId: string) { + const content = await this.docReader.getWorkspaceContent(workspaceId); + if (!content) { + throw new NotFound('Workspace not found'); + } + this.logger.log(`get workspace content ${workspaceId}`); + return content; + } } diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts index 6abc8b2df9..6156cad4a8 100644 --- a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts +++ b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts @@ -8,13 +8,14 @@ import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; import { AppModule } from '../../../app.module'; import { ConfigModule } from '../../../base/config'; import { Models } from '../../../models'; -import { DocReader } from '..'; +import { DocReader, PgWorkspaceDocStorageAdapter } from '..'; import { DatabaseDocReader } from '../reader'; const test = ava as TestFn<{ models: Models; app: TestingApp; docReader: DocReader; + adapter: PgWorkspaceDocStorageAdapter; }>; test.before(async t => { @@ -24,6 +25,7 @@ test.before(async t => { t.context.models = app.get(Models); t.context.docReader = app.get(DocReader); + t.context.adapter = app.get(PgWorkspaceDocStorageAdapter); t.context.app = app; }); @@ -136,3 +138,53 @@ test('should return doc diff', async t => { applyUpdate(doc2, diff3!.missing); t.is(doc2.getText('content').toString(), 'hello world!@'); }); + +test('should get doc content', async t => { + const docId = randomUUID(); + const { docReader } = t.context; + + const doc = new YDoc(); + const text = doc.getText('content'); + const updates: Buffer[] = []; + + doc.on('update', update => { + updates.push(Buffer.from(update)); + }); + + text.insert(0, 'hello'); + text.insert(5, 'world'); + text.insert(5, ' '); + + await t.context.adapter.pushDocUpdates(workspace.id, docId, updates, user.id); + + const docContent = await docReader.getDocContent(workspace.id, docId); + // TODO(@fengmk2): should create a test ydoc with blocks + t.is(docContent, null); +}); + +test('should get workspace content', async t => { + const { docReader } = t.context; + + const doc = new YDoc(); + const text = doc.getText('content'); + const updates: Buffer[] = []; + + doc.on('update', update => { + updates.push(Buffer.from(update)); + }); + + text.insert(0, 'hello'); + text.insert(5, 'world'); + text.insert(5, ' '); + + await t.context.adapter.pushDocUpdates( + workspace.id, + workspace.id, + updates, + user.id + ); + + const workspaceContent = await docReader.getWorkspaceContent(workspace.id); + // TODO(@fengmk2): should create a test ydoc with blocks + t.is(workspaceContent, null); +}); diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts index 271f1af43c..0444e38387 100644 --- a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts +++ b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts @@ -10,7 +10,7 @@ import { AppModule } from '../../../app.module'; import { Config, UserFriendlyError } from '../../../base'; import { ConfigModule } from '../../../base/config'; import { Models } from '../../../models'; -import { DatabaseDocReader, DocReader } from '..'; +import { DatabaseDocReader, DocReader, PgWorkspaceDocStorageAdapter } from '..'; import { RpcDocReader } from '../reader'; const test = ava as TestFn<{ @@ -18,6 +18,7 @@ const test = ava as TestFn<{ app: TestingApp; docReader: DocReader; databaseDocReader: DatabaseDocReader; + adapter: PgWorkspaceDocStorageAdapter; config: Config; }>; @@ -39,6 +40,7 @@ test.before(async t => { t.context.models = app.get(Models); t.context.docReader = app.get(DocReader); t.context.databaseDocReader = app.get(DatabaseDocReader); + t.context.adapter = app.get(PgWorkspaceDocStorageAdapter); t.context.config = app.get(Config); t.context.app = app; }); @@ -71,12 +73,36 @@ test('should return null when doc not found', async t => { }); test('should throw error when doc service internal error', async t => { - const { docReader, databaseDocReader } = t.context; + const { docReader, adapter } = t.context; const docId = randomUUID(); - mock.method(databaseDocReader, 'getDoc', async () => { + mock.method(adapter, 'getDoc', async () => { throw new Error('mock doc service internal error'); }); - const err = await t.throwsAsync(docReader.getDoc(workspace.id, docId), { + let err = await t.throwsAsync(docReader.getDoc(workspace.id, docId), { + instanceOf: UserFriendlyError, + message: 'An internal error occurred.', + name: 'internal_server_error', + }); + t.is(err.type, 'internal_server_error'); + t.is(err.status, 500); + + err = await t.throwsAsync(docReader.getDocDiff(workspace.id, docId), { + instanceOf: UserFriendlyError, + message: 'An internal error occurred.', + name: 'internal_server_error', + }); + t.is(err.type, 'internal_server_error'); + t.is(err.status, 500); + + err = await t.throwsAsync(docReader.getDocContent(workspace.id, docId), { + instanceOf: UserFriendlyError, + message: 'An internal error occurred.', + name: 'internal_server_error', + }); + t.is(err.type, 'internal_server_error'); + t.is(err.status, 500); + + err = await t.throwsAsync(docReader.getWorkspaceContent(workspace.id), { instanceOf: UserFriendlyError, message: 'An internal error occurred.', name: 'internal_server_error', @@ -232,3 +258,84 @@ test('should get doc diff fallback to database doc reader when endpoint network applyUpdate(doc2, diff!.missing); t.is(doc2.getText('content').toString(), 'hello world!'); }); + +test('should get doc content', async t => { + const docId = randomUUID(); + const { docReader, databaseDocReader } = t.context; + mock.method(databaseDocReader, 'getDocContent', async () => { + return { + title: 'test title', + summary: 'test summary', + }; + }); + const docContent = await docReader.getDocContent(workspace.id, docId); + t.deepEqual(docContent, { + title: 'test title', + summary: 'test summary', + }); +}); + +test('should return null when doc content not exists', async t => { + const docId = randomUUID(); + const { docReader, adapter } = t.context; + + const doc = new YDoc(); + const text = doc.getText('content'); + const updates: Buffer[] = []; + + doc.on('update', update => { + updates.push(Buffer.from(update)); + }); + + text.insert(0, 'hello'); + text.insert(5, 'world'); + text.insert(5, ' '); + + await adapter.pushDocUpdates(workspace.id, docId, updates, user.id); + + const docContent = await docReader.getDocContent(workspace.id, docId); + t.is(docContent, null); + + const notExists = await docReader.getDocContent(workspace.id, randomUUID()); + t.is(notExists, null); +}); + +test('should get workspace content from doc service rpc', async t => { + const { docReader, databaseDocReader } = t.context; + mock.method(databaseDocReader, 'getWorkspaceContent', async () => { + return { + name: 'test name', + avatarKey: 'avatar key', + }; + }); + + const workspaceContent = await docReader.getWorkspaceContent(workspace.id); + t.deepEqual(workspaceContent, { + name: 'test name', + avatarKey: 'avatar key', + }); +}); + +test('should return null when workspace bin meta not exists', async t => { + const { docReader, adapter } = t.context; + const doc = new YDoc(); + const text = doc.getText('content'); + const updates: Buffer[] = []; + + doc.on('update', update => { + updates.push(Buffer.from(update)); + }); + + text.insert(0, 'hello'); + text.insert(5, 'world'); + text.insert(5, ' '); + + await adapter.pushDocUpdates(workspace.id, workspace.id, updates, user.id); + + const workspaceContent = await docReader.getWorkspaceContent(workspace.id); + t.is(workspaceContent, null); + + // workspace not exists + const notExists = await docReader.getWorkspaceContent(randomUUID()); + t.is(notExists, null); +}); diff --git a/packages/backend/server/src/core/doc/adapters/workspace.ts b/packages/backend/server/src/core/doc/adapters/workspace.ts index a1c35f8e67..8cd74dcf49 100644 --- a/packages/backend/server/src/core/doc/adapters/workspace.ts +++ b/packages/backend/server/src/core/doc/adapters/workspace.ts @@ -344,7 +344,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { }); if (updatedSnapshot) { - this.event.broadcast('doc.snapshot.updated', { + this.event.emit('doc.snapshot.updated', { workspaceId: snapshot.spaceId, docId: snapshot.docId, }); diff --git a/packages/backend/server/src/core/doc/event.ts b/packages/backend/server/src/core/doc/event.ts new file mode 100644 index 0000000000..1f90641737 --- /dev/null +++ b/packages/backend/server/src/core/doc/event.ts @@ -0,0 +1,17 @@ +import { Injectable } from '@nestjs/common'; + +import { OnEvent } from '../../base'; +import { DocReader } from './reader'; + +@Injectable() +export class DocEventsListener { + constructor(private readonly doc: DocReader) {} + + @OnEvent('doc.snapshot.updated') + async markDocContentCacheStale({ + workspaceId, + docId, + }: Events['doc.snapshot.updated']) { + await this.doc.markDocContentCacheStale(workspaceId, docId); + } +} diff --git a/packages/backend/server/src/core/doc/index.ts b/packages/backend/server/src/core/doc/index.ts index 3b55b1e6d0..d563380674 100644 --- a/packages/backend/server/src/core/doc/index.ts +++ b/packages/backend/server/src/core/doc/index.ts @@ -6,6 +6,7 @@ import { PermissionModule } from '../permission'; import { QuotaModule } from '../quota'; import { PgUserspaceDocStorageAdapter } from './adapters/userspace'; import { PgWorkspaceDocStorageAdapter } from './adapters/workspace'; +import { DocEventsListener } from './event'; import { DocStorageCronJob } from './job'; import { DocStorageOptions } from './options'; import { DatabaseDocReader, DocReader, DocReaderProvider } from './reader'; @@ -19,6 +20,7 @@ import { DatabaseDocReader, DocReader, DocReaderProvider } from './reader'; DocStorageCronJob, DocReaderProvider, DatabaseDocReader, + DocEventsListener, ], exports: [ DatabaseDocReader, diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.ts index 3508ea4012..e527800456 100644 --- a/packages/backend/server/src/core/doc/reader.ts +++ b/packages/backend/server/src/core/doc/reader.ts @@ -1,17 +1,33 @@ import { FactoryProvider, Injectable, Logger } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; -import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs'; +import { + applyUpdate, + diffUpdate, + Doc as YDoc, + encodeStateVectorFromUpdate, +} from 'yjs'; import { + Cache, Config, CryptoHelper, getOrGenRequestId, UserFriendlyError, } from '../../base'; +import { + type PageDocContent, + parsePageDoc, + parseWorkspaceDoc, + type WorkspaceDocContent, +} from '../utils/blocksuite'; import { PgWorkspaceDocStorageAdapter } from './adapters/workspace'; import { type DocDiff, type DocRecord } from './storage'; +const DOC_CONTENT_CACHE_7_DAYS = 7 * 24 * 60 * 60 * 1000; + export abstract class DocReader { + constructor(protected readonly cache: Cache) {} + abstract getDoc( workspaceId: string, docId: string @@ -23,6 +39,60 @@ export abstract class DocReader { stateVector?: Uint8Array ): Promise; + async getDocContent( + workspaceId: string, + docId: string + ): Promise { + const cacheKey = this.cacheKey(workspaceId, docId); + const cachedResult = await this.cache.get(cacheKey); + if (cachedResult) { + return cachedResult; + } + + const content = await this.getDocContentWithoutCache(workspaceId, docId); + if (content) { + await this.cache.set(cacheKey, content, { + ttl: DOC_CONTENT_CACHE_7_DAYS, + }); + } + return content; + } + + async getWorkspaceContent( + workspaceId: string + ): Promise { + const cacheKey = this.cacheKey(workspaceId, workspaceId); + const cachedResult = await this.cache.get(cacheKey); + if (cachedResult) { + return cachedResult; + } + + const content = await this.getWorkspaceContentWithoutCache(workspaceId); + if (content) { + await this.cache.set(cacheKey, content); + } + return content; + } + + async markDocContentCacheStale(workspaceId: string, docId: string) { + await this.cache.delete(this.cacheKey(workspaceId, docId)); + } + + private cacheKey(workspaceId: string, docId: string) { + return workspaceId === docId + ? `workspace:${workspaceId}:content` + : `workspace:${workspaceId}:doc:${docId}:content`; + } + + protected abstract getDocContentWithoutCache( + workspaceId: string, + guid: string + ): Promise; + + protected abstract getWorkspaceContentWithoutCache( + workspaceId: string + ): Promise; + protected docDiff(update: Uint8Array, stateVector?: Uint8Array) { const missing = stateVector ? diffUpdate(update, stateVector) : update; const state = encodeStateVectorFromUpdate(update); @@ -35,8 +105,11 @@ export abstract class DocReader { @Injectable() export class DatabaseDocReader extends DocReader { - constructor(protected readonly workspace: PgWorkspaceDocStorageAdapter) { - super(); + constructor( + protected override readonly cache: Cache, + protected readonly workspace: PgWorkspaceDocStorageAdapter + ) { + super(cache); } async getDoc(workspaceId: string, docId: string): Promise { @@ -57,6 +130,31 @@ export class DatabaseDocReader extends DocReader { timestamp: doc.timestamp, }; } + + protected override async getDocContentWithoutCache( + workspaceId: string, + guid: string + ): Promise { + const docRecord = await this.workspace.getDoc(workspaceId, guid); + if (!docRecord) { + return null; + } + const doc = new YDoc(); + applyUpdate(doc, docRecord.bin); + return parsePageDoc(doc); + } + + protected override async getWorkspaceContentWithoutCache( + workspaceId: string + ): Promise { + const docRecord = await this.workspace.getDoc(workspaceId, workspaceId); + if (!docRecord) { + return null; + } + const doc = new YDoc(); + applyUpdate(doc, docRecord.bin); + return parseWorkspaceDoc(doc); + } } @Injectable() @@ -66,9 +164,10 @@ export class RpcDocReader extends DatabaseDocReader { constructor( private readonly config: Config, private readonly crypto: CryptoHelper, + protected override readonly cache: Cache, protected override readonly workspace: PgWorkspaceDocStorageAdapter ) { - super(workspace); + super(cache, workspace); } private async fetch( @@ -128,11 +227,8 @@ export class RpcDocReader extends DatabaseDocReader { // other error this.logger.error( `Failed to fetch doc ${url}, fallback to database doc reader`, - err.stack + err ); - if (err.cause instanceof Error) { - this.logger.error(err.cause.stack); - } // fallback to database doc reader if the error is not user friendly, like network error return await super.getDoc(workspaceId, docId); } @@ -172,15 +268,61 @@ export class RpcDocReader extends DatabaseDocReader { const err = e as Error; this.logger.error( `Failed to fetch doc diff ${url}, fallback to database doc reader`, - err.stack + err ); - if (err.cause instanceof Error) { - this.logger.error(err.cause.stack); - } // fallback to database doc reader if the error is not user friendly, like network error return await super.getDocDiff(workspaceId, docId, stateVector); } } + + protected override async getDocContentWithoutCache( + workspaceId: string, + docId: string + ): Promise { + const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/content`; + const accessToken = this.crypto.sign(docId); + try { + const res = await this.fetch(accessToken, url, 'GET'); + if (!res) { + return null; + } + return (await res.json()) as PageDocContent; + } catch (e) { + if (e instanceof UserFriendlyError) { + throw e; + } + const err = e as Error; + this.logger.error( + `Failed to fetch doc content ${url}, fallback to database doc reader`, + err + ); + return await super.getDocContentWithoutCache(workspaceId, docId); + } + } + + protected override async getWorkspaceContentWithoutCache( + workspaceId: string + ): Promise { + const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/content`; + const accessToken = this.crypto.sign(workspaceId); + try { + const res = await this.fetch(accessToken, url, 'GET'); + if (!res) { + return null; + } + return (await res.json()) as WorkspaceDocContent; + } catch (e) { + if (e instanceof UserFriendlyError) { + throw e; + } + const err = e as Error; + this.logger.error( + `Failed to fetch workspace content ${url}, fallback to database doc reader`, + err + ); + return await super.getWorkspaceContentWithoutCache(workspaceId); + } + } } export const DocReaderProvider: FactoryProvider = { diff --git a/packages/backend/server/src/core/workspaces/resolvers/service.ts b/packages/backend/server/src/core/workspaces/resolvers/service.ts index 4955d39c82..87f5f48f0a 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/service.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/service.ts @@ -10,7 +10,7 @@ import { UserNotFound, } from '../../../base'; import { Models } from '../../../models'; -import { DocContentService } from '../../doc-renderer'; +import { DocReader } from '../../doc'; import { PermissionService, WorkspaceRole } from '../../permission'; import { WorkspaceBlobStorage } from '../../storage'; @@ -30,7 +30,7 @@ export class WorkspaceService { constructor( private readonly blobStorage: WorkspaceBlobStorage, private readonly cache: Cache, - private readonly doc: DocContentService, + private readonly doc: DocReader, private readonly mailer: MailService, private readonly permission: PermissionService, private readonly prisma: PrismaClient,