feat(server): add doc title and summary to database (#10505)

close CLOUD-152

packages/backend/server/src/core/doc/__tests__/event.spec.ts

@@ -0,0 +1,120 @@
+import { randomUUID } from 'node:crypto';
+import { mock } from 'node:test';
+
+import ava, { TestFn } from 'ava';
+import Sinon from 'sinon';
+import { Doc as YDoc } from 'yjs';
+
+import {
+  createTestingModule,
+  type TestingModule,
+} from '../../../__tests__/utils';
+import { Models, User, Workspace } from '../../../models';
+import { DocReader, PgWorkspaceDocStorageAdapter as Adapter } from '..';
+import { DocEventsListener } from '../event';
+
+interface Context {
+  module: TestingModule;
+  docReader: DocReader;
+  adapter: Adapter;
+  models: Models;
+  listener: DocEventsListener;
+}
+
+const test = ava as TestFn<Context>;
+
+test.before(async t => {
+  const module = await createTestingModule();
+  t.context.module = module;
+  t.context.models = module.get(Models);
+  t.context.docReader = module.get(DocReader);
+  t.context.adapter = module.get(Adapter);
+  t.context.listener = module.get(DocEventsListener);
+});
+
+let owner: User;
+let workspace: Workspace;
+
+test.beforeEach(async t => {
+  await t.context.module.initTestingDB();
+  owner = await t.context.models.user.create({
+    email: `${randomUUID()}@affine.pro`,
+  });
+  workspace = await t.context.models.workspace.create(owner.id);
+});
+
+test.afterEach.always(() => {
+  mock.reset();
+  Sinon.restore();
+});
+
+test.after.always(async t => {
+  await t.context.module.close();
+});
+
+test('should update doc content to database when doc is updated', async t => {
+  const { docReader, models, adapter, listener } = t.context;
+  const updates: Buffer[] = [];
+  {
+    const doc = new YDoc();
+    doc.on('update', data => {
+      updates.push(Buffer.from(data));
+    });
+
+    const text = doc.getText('content');
+    text.insert(0, 'hello');
+    text.insert(5, 'world');
+  }
+
+  const docId = randomUUID();
+  await adapter.pushDocUpdates(workspace.id, docId, updates);
+  await adapter.getDoc(workspace.id, docId);
+
+  mock.method(docReader, 'parseDocContent', () => {
+    return {
+      title: 'test title',
+      summary: 'test summary',
+    };
+  });
+
+  const spy = Sinon.spy(models.doc, 'upsertMeta');
+  await listener.markDocContentCacheStale({
+    workspaceId: workspace.id,
+    docId,
+    blob: Buffer.from([]),
+  });
+  t.is(spy.callCount, 1);
+  const content = await models.doc.getMeta(workspace.id, docId);
+  t.truthy(content);
+  t.is(content!.title, 'test title');
+  t.is(content!.summary, 'test summary');
+});
+
+test('should ignore update doc content to database when snapshot parse failed', async t => {
+  const { models, adapter, listener } = t.context;
+  const updates: Buffer[] = [];
+  {
+    const doc = new YDoc();
+    doc.on('update', data => {
+      updates.push(Buffer.from(data));
+    });
+
+    const text = doc.getText('content');
+    text.insert(0, 'hello');
+    text.insert(5, 'world');
+  }
+
+  const docId = randomUUID();
+  await adapter.pushDocUpdates(workspace.id, docId, updates);
+  const doc = await adapter.getDoc(workspace.id, docId);
+
+  const spy = Sinon.spy(models.doc, 'upsertMeta');
+  await listener.markDocContentCacheStale({
+    workspaceId: workspace.id,
+    docId,
+    blob: Buffer.from(doc!.bin),
+  });
+  t.is(spy.callCount, 0);
+  const content = await models.doc.getMeta(workspace.id, docId);
+  t.is(content, null);
+});

packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts

@@ -193,7 +193,6 @@ test('should get workspace content with default avatar', async t => {
     user.id
   );
 
-  // @ts-expect-error parseWorkspaceContent is private
   const track = mock.method(docReader, 'parseWorkspaceContent', () => ({
     name: 'Test Workspace',
     avatarKey: '',
@@ -240,7 +239,6 @@ test('should get workspace content with custom avatar', async t => {
     Buffer.from('mock avatar image data here')
   );
 
-  // @ts-expect-error parseWorkspaceContent is private
   const track = mock.method(docReader, 'parseWorkspaceContent', () => ({
     name: 'Test Workspace',
     avatarKey,

packages/backend/server/src/core/doc/adapters/workspace.ts

@@ -32,6 +32,7 @@ declare global {
     'doc.snapshot.updated': {
       workspaceId: string;
       docId: string;
+      blob: Buffer;
     };
     'doc.created': {
       workspaceId: string;
@@ -335,10 +336,11 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
     }
 
     try {
+      const blob = Buffer.from(snapshot.bin);
       const updatedSnapshot = await this.models.doc.upsert({
         spaceId: snapshot.spaceId,
         docId: snapshot.docId,
-        blob: Buffer.from(snapshot.bin),
+        blob,
         timestamp: snapshot.timestamp,
         editorId: snapshot.editor,
       });
@@ -347,6 +349,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
         this.event.emit('doc.snapshot.updated', {
           workspaceId: snapshot.spaceId,
           docId: snapshot.docId,
+          blob,
         });
       }
 

packages/backend/server/src/core/doc/event.ts

@@ -1,17 +1,31 @@
 import { Injectable } from '@nestjs/common';
 
 import { OnEvent } from '../../base';
+import { Models } from '../../models';
 import { DocReader } from './reader';
 
 @Injectable()
 export class DocEventsListener {
-  constructor(private readonly doc: DocReader) {}
+  constructor(
+    private readonly docReader: DocReader,
+    private readonly models: Models
+  ) {}
 
   @OnEvent('doc.snapshot.updated')
   async markDocContentCacheStale({
     workspaceId,
     docId,
+    blob,
   }: Events['doc.snapshot.updated']) {
-    await this.doc.markDocContentCacheStale(workspaceId, docId);
+    await this.docReader.markDocContentCacheStale(workspaceId, docId);
+    const isDoc = workspaceId !== docId;
+    // update doc content to database
+    if (isDoc) {
+      const content = this.docReader.parseDocContent(blob);
+      if (!content) {
+        return;
+      }
+      await this.models.doc.upsertMeta(workspaceId, docId, content);
+    }
   }
 }

packages/backend/server/src/core/doc/reader.ts

@@ -36,6 +36,18 @@ export interface WorkspaceDocInfo {
 export abstract class DocReader {
   constructor(protected readonly cache: Cache) {}
 
+  parseDocContent(bin: Uint8Array) {
+    const doc = new YDoc();
+    applyUpdate(doc, bin);
+    return parsePageDoc(doc);
+  }
+
+  parseWorkspaceContent(bin: Uint8Array) {
+    const doc = new YDoc();
+    applyUpdate(doc, bin);
+    return parseWorkspaceDoc(doc);
+  }
+
   abstract getDoc(
     workspaceId: string,
     docId: string
@@ -47,6 +59,7 @@ export abstract class DocReader {
     stateVector?: Uint8Array
   ): Promise<DocDiff | null>;
 
+  // TODO(@fengmk2): should remove this method after frontend support doc content update
   async getDocContent(
     workspaceId: string,
     docId: string
@@ -149,9 +162,7 @@ export class DatabaseDocReader extends DocReader {
     if (!docRecord) {
       return null;
     }
-    const doc = new YDoc();
-    applyUpdate(doc, docRecord.bin);
-    return parsePageDoc(doc);
+    return this.parseDocContent(docRecord.bin);
   }
 
   protected override async getWorkspaceContentWithoutCache(
@@ -178,12 +189,6 @@ export class DatabaseDocReader extends DocReader {
       avatarUrl,
     };
   }
-
-  private parseWorkspaceContent(bin: Uint8Array) {
-    const doc = new YDoc();
-    applyUpdate(doc, bin);
-    return parseWorkspaceDoc(doc);
-  }
 }
 
 @Injectable()