feat(server): add doc title and summary to database (#10505)

close CLOUD-152

packages/backend/server/migrations/20250303102501_add_workspace_doc_title_and_summary/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "workspace_pages" ADD COLUMN     "summary" VARCHAR,
+ADD COLUMN     "title" VARCHAR;

packages/backend/server/schema.prisma

@@ -126,6 +126,8 @@ model WorkspaceDoc {
   mode        Int     @default(0) @db.SmallInt
   // Whether the doc is blocked
   blocked     Boolean @default(false)
+  title       String?  @db.VarChar
+  summary     String?  @db.VarChar
 
   workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
 

packages/backend/server/src/__tests__/models/doc.spec.ts

@@ -532,4 +532,130 @@ test('should get public docs of a workspace', async t => {
   t.deepEqual(docs.map(d => d.docId).sort(), [docId1, docId2]);
 });
 
+test('should update title and summary', async t => {
+  const docId = randomUUID();
+  const snapshot = {
+    spaceId: workspace.id,
+    docId,
+    blob: Buffer.from('blob1'),
+    timestamp: Date.now(),
+    editorId: user.id,
+  };
+  await t.context.doc.upsert(snapshot);
+  const content = {
+    title: 'test title',
+    summary: 'test summary',
+  };
+  await t.context.doc.upsertMeta(workspace.id, docId, content);
+  const foundContent = await t.context.doc.getMeta(workspace.id, docId, {
+    select: {
+      title: true,
+      summary: true,
+    },
+  });
+  t.deepEqual(foundContent, content);
+  const updatedContent = {
+    title: 'test title 2',
+    summary: 'test summary 2',
+  };
+  await t.context.doc.upsertMeta(workspace.id, docId, updatedContent);
+  const foundUpdatedContent = await t.context.doc.getMeta(workspace.id, docId, {
+    select: {
+      title: true,
+      summary: true,
+    },
+  });
+  t.deepEqual(foundUpdatedContent, updatedContent);
+});
+
+test('should find metas by workspaceIds and docIds', async t => {
+  const docId1 = randomUUID();
+  const docId2 = randomUUID();
+  const docId3 = randomUUID();
+  const snapshot1 = {
+    spaceId: workspace.id,
+    docId: docId1,
+    blob: Buffer.from('blob1'),
+    timestamp: Date.now(),
+    editorId: user.id,
+  };
+  const snapshot2 = {
+    spaceId: workspace.id,
+    docId: docId2,
+    blob: Buffer.from('blob2'),
+    timestamp: Date.now(),
+    editorId: user.id,
+  };
+  const snapshot3 = {
+    spaceId: workspace.id,
+    docId: docId3,
+    blob: Buffer.from('blob3'),
+    timestamp: Date.now(),
+    editorId: user.id,
+  };
+  await t.context.doc.upsert(snapshot1);
+  await t.context.doc.upsert(snapshot2);
+  await t.context.doc.upsert(snapshot3);
+  const content1 = {
+    title: 'test title',
+    summary: 'test summary',
+  };
+  const content2 = {
+    title: 'test title 2',
+    summary: 'test summary 2',
+  };
+  await t.context.doc.upsertMeta(workspace.id, docId1, content1);
+  await t.context.doc.upsertMeta(workspace.id, docId2, content2);
+  let contents = await t.context.doc.findMetas([
+    { workspaceId: workspace.id, docId: docId1 },
+    { workspaceId: workspace.id, docId: randomUUID() },
+    { workspaceId: randomUUID(), docId: docId1 },
+    { workspaceId: workspace.id, docId: docId2 },
+    { workspaceId: randomUUID(), docId: randomUUID() },
+  ]);
+  t.deepEqual(
+    contents.map(c =>
+      c
+        ? {
+            title: c.title,
+            summary: c.summary,
+          }
+        : null
+    ),
+    [content1, null, null, content2, null]
+  );
+  contents = await t.context.doc.findMetas([
+    { workspaceId: workspace.id, docId: docId1 },
+    { workspaceId: workspace.id, docId: docId2 },
+  ]);
+  t.deepEqual(
+    contents.map(c =>
+      c
+        ? {
+            title: c.title,
+            summary: c.summary,
+          }
+        : null
+    ),
+    [content1, content2]
+  );
+  // docId3 don't have meta
+  contents = await t.context.doc.findMetas([
+    { workspaceId: workspace.id, docId: docId1 },
+    { workspaceId: workspace.id, docId: docId2 },
+    { workspaceId: workspace.id, docId: docId3 },
+  ]);
+  t.deepEqual(
+    contents.map(c =>
+      c
+        ? {
+            title: c.title,
+            summary: c.summary,
+          }
+        : null
+    ),
+    [content1, content2, null]
+  );
+});
+
 // #endregion

packages/backend/server/src/core/doc/__tests__/event.spec.ts

@@ -0,0 +1,120 @@
+import { randomUUID } from 'node:crypto';
+import { mock } from 'node:test';
+
+import ava, { TestFn } from 'ava';
+import Sinon from 'sinon';
+import { Doc as YDoc } from 'yjs';
+
+import {
+  createTestingModule,
+  type TestingModule,
+} from '../../../__tests__/utils';
+import { Models, User, Workspace } from '../../../models';
+import { DocReader, PgWorkspaceDocStorageAdapter as Adapter } from '..';
+import { DocEventsListener } from '../event';
+
+interface Context {
+  module: TestingModule;
+  docReader: DocReader;
+  adapter: Adapter;
+  models: Models;
+  listener: DocEventsListener;
+}
+
+const test = ava as TestFn<Context>;
+
+test.before(async t => {
+  const module = await createTestingModule();
+  t.context.module = module;
+  t.context.models = module.get(Models);
+  t.context.docReader = module.get(DocReader);
+  t.context.adapter = module.get(Adapter);
+  t.context.listener = module.get(DocEventsListener);
+});
+
+let owner: User;
+let workspace: Workspace;
+
+test.beforeEach(async t => {
+  await t.context.module.initTestingDB();
+  owner = await t.context.models.user.create({
+    email: `${randomUUID()}@affine.pro`,
+  });
+  workspace = await t.context.models.workspace.create(owner.id);
+});
+
+test.afterEach.always(() => {
+  mock.reset();
+  Sinon.restore();
+});
+
+test.after.always(async t => {
+  await t.context.module.close();
+});
+
+test('should update doc content to database when doc is updated', async t => {
+  const { docReader, models, adapter, listener } = t.context;
+  const updates: Buffer[] = [];
+  {
+    const doc = new YDoc();
+    doc.on('update', data => {
+      updates.push(Buffer.from(data));
+    });
+
+    const text = doc.getText('content');
+    text.insert(0, 'hello');
+    text.insert(5, 'world');
+  }
+
+  const docId = randomUUID();
+  await adapter.pushDocUpdates(workspace.id, docId, updates);
+  await adapter.getDoc(workspace.id, docId);
+
+  mock.method(docReader, 'parseDocContent', () => {
+    return {
+      title: 'test title',
+      summary: 'test summary',
+    };
+  });
+
+  const spy = Sinon.spy(models.doc, 'upsertMeta');
+  await listener.markDocContentCacheStale({
+    workspaceId: workspace.id,
+    docId,
+    blob: Buffer.from([]),
+  });
+  t.is(spy.callCount, 1);
+  const content = await models.doc.getMeta(workspace.id, docId);
+  t.truthy(content);
+  t.is(content!.title, 'test title');
+  t.is(content!.summary, 'test summary');
+});
+
+test('should ignore update doc content to database when snapshot parse failed', async t => {
+  const { models, adapter, listener } = t.context;
+  const updates: Buffer[] = [];
+  {
+    const doc = new YDoc();
+    doc.on('update', data => {
+      updates.push(Buffer.from(data));
+    });
+
+    const text = doc.getText('content');
+    text.insert(0, 'hello');
+    text.insert(5, 'world');
+  }
+
+  const docId = randomUUID();
+  await adapter.pushDocUpdates(workspace.id, docId, updates);
+  const doc = await adapter.getDoc(workspace.id, docId);
+
+  const spy = Sinon.spy(models.doc, 'upsertMeta');
+  await listener.markDocContentCacheStale({
+    workspaceId: workspace.id,
+    docId,
+    blob: Buffer.from(doc!.bin),
+  });
+  t.is(spy.callCount, 0);
+  const content = await models.doc.getMeta(workspace.id, docId);
+  t.is(content, null);
+});

packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts

@@ -193,7 +193,6 @@ test('should get workspace content with default avatar', async t => {
     user.id
   );
 
-  // @ts-expect-error parseWorkspaceContent is private
   const track = mock.method(docReader, 'parseWorkspaceContent', () => ({
     name: 'Test Workspace',
     avatarKey: '',
@@ -240,7 +239,6 @@ test('should get workspace content with custom avatar', async t => {
     Buffer.from('mock avatar image data here')
   );
 
-  // @ts-expect-error parseWorkspaceContent is private
   const track = mock.method(docReader, 'parseWorkspaceContent', () => ({
     name: 'Test Workspace',
     avatarKey,

packages/backend/server/src/core/doc/adapters/workspace.ts

@@ -32,6 +32,7 @@ declare global {
     'doc.snapshot.updated': {
       workspaceId: string;
       docId: string;
+      blob: Buffer;
     };
     'doc.created': {
       workspaceId: string;
@@ -335,10 +336,11 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
     }
 
     try {
+      const blob = Buffer.from(snapshot.bin);
       const updatedSnapshot = await this.models.doc.upsert({
         spaceId: snapshot.spaceId,
         docId: snapshot.docId,
-        blob: Buffer.from(snapshot.bin),
+        blob,
         timestamp: snapshot.timestamp,
         editorId: snapshot.editor,
       });
@@ -347,6 +349,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
         this.event.emit('doc.snapshot.updated', {
           workspaceId: snapshot.spaceId,
           docId: snapshot.docId,
+          blob,
         });
       }
 

packages/backend/server/src/core/doc/event.ts

@@ -1,17 +1,31 @@
 import { Injectable } from '@nestjs/common';
 
 import { OnEvent } from '../../base';
+import { Models } from '../../models';
 import { DocReader } from './reader';
 
 @Injectable()
 export class DocEventsListener {
-  constructor(private readonly doc: DocReader) {}
+  constructor(
+    private readonly docReader: DocReader,
+    private readonly models: Models
+  ) {}
 
   @OnEvent('doc.snapshot.updated')
   async markDocContentCacheStale({
     workspaceId,
     docId,
+    blob,
   }: Events['doc.snapshot.updated']) {
-    await this.doc.markDocContentCacheStale(workspaceId, docId);
+    await this.docReader.markDocContentCacheStale(workspaceId, docId);
+    const isDoc = workspaceId !== docId;
+    // update doc content to database
+    if (isDoc) {
+      const content = this.docReader.parseDocContent(blob);
+      if (!content) {
+        return;
+      }
+      await this.models.doc.upsertMeta(workspaceId, docId, content);
+    }
   }
 }

packages/backend/server/src/core/doc/reader.ts

@@ -36,6 +36,18 @@ export interface WorkspaceDocInfo {
 export abstract class DocReader {
   constructor(protected readonly cache: Cache) {}
 
+  parseDocContent(bin: Uint8Array) {
+    const doc = new YDoc();
+    applyUpdate(doc, bin);
+    return parsePageDoc(doc);
+  }
+
+  parseWorkspaceContent(bin: Uint8Array) {
+    const doc = new YDoc();
+    applyUpdate(doc, bin);
+    return parseWorkspaceDoc(doc);
+  }
+
   abstract getDoc(
     workspaceId: string,
     docId: string
@@ -47,6 +59,7 @@ export abstract class DocReader {
     stateVector?: Uint8Array
   ): Promise<DocDiff | null>;
 
+  // TODO(@fengmk2): should remove this method after frontend support doc content update
   async getDocContent(
     workspaceId: string,
     docId: string
@@ -149,9 +162,7 @@ export class DatabaseDocReader extends DocReader {
     if (!docRecord) {
       return null;
     }
-    const doc = new YDoc();
-    applyUpdate(doc, docRecord.bin);
-    return parsePageDoc(doc);
+    return this.parseDocContent(docRecord.bin);
   }
 
   protected override async getWorkspaceContentWithoutCache(
@@ -178,12 +189,6 @@ export class DatabaseDocReader extends DocReader {
       avatarUrl,
     };
   }
-
-  private parseWorkspaceContent(bin: Uint8Array) {
-    const doc = new YDoc();
-    applyUpdate(doc, bin);
-    return parseWorkspaceDoc(doc);
-  }
 }
 
 @Injectable()

packages/backend/server/src/models/doc.ts

@@ -7,8 +7,6 @@ import { DocIsNotPublic } from '../base/error';
 import { BaseModel } from './base';
 import { Doc, DocRole, PublicDocMode, publicUserSelect } from './common';
 
-export interface DocRecord extends Doc {}
-
 export type DocMetaUpsertInput = Omit<
   Prisma.WorkspaceDocUncheckedCreateInput,
   'workspaceId' | 'docId'
@@ -27,7 +25,7 @@ export type DocMetaUpsertInput = Omit<
 export class DocModel extends BaseModel {
   // #region Update
 
-  private updateToDocRecord(row: Update): DocRecord {
+  private updateToDocRecord(row: Update): Doc {
     return {
       spaceId: row.workspaceId,
       docId: row.id,
@@ -37,7 +35,7 @@ export class DocModel extends BaseModel {
     };
   }
 
-  private docRecordToUpdate(record: DocRecord): Update {
+  private docRecordToUpdate(record: Doc): Update {
     return {
       workspaceId: record.spaceId,
       id: record.docId,
@@ -48,7 +46,7 @@ export class DocModel extends BaseModel {
     };
   }
 
-  async createUpdates(updates: DocRecord[]) {
+  async createUpdates(updates: Doc[]) {
     return await this.db.update.createMany({
       data: updates.map(r => this.docRecordToUpdate(r)),
     });
@@ -57,7 +55,7 @@ export class DocModel extends BaseModel {
   /**
    * Find updates by workspaceId and docId.
    */
-  async findUpdates(workspaceId: string, docId: string): Promise<DocRecord[]> {
+  async findUpdates(workspaceId: string, docId: string): Promise<Doc[]> {
     const rows = await this.db.update.findMany({
       where: {
         workspaceId,
@@ -349,6 +347,21 @@ export class DocModel extends BaseModel {
     });
   }
 
+  async findMetas(ids: { workspaceId: string; docId: string }[]) {
+    const rows = await this.db.workspaceDoc.findMany({
+      where: {
+        workspaceId: { in: ids.map(id => id.workspaceId) },
+        docId: { in: ids.map(id => id.docId) },
+      },
+    });
+    const resultMap = new Map(
+      rows.map(row => [`${row.workspaceId}-${row.docId}`, row])
+    );
+    return ids.map(
+      id => resultMap.get(`${id.workspaceId}-${id.docId}`) ?? null
+    );
+  }
+
   /**
    * Find the workspace public doc metas.
    */

packages/backend/server/src/schema.gql

@@ -402,6 +402,7 @@ enum ErrorNames {
   MEMBER_NOT_FOUND_IN_SPACE
   MEMBER_QUOTA_EXCEEDED
   MISSING_OAUTH_QUERY_PARAMETER
+  NETWORK_ERROR
   NOT_FOUND
   NOT_IN_SPACE
   NO_COPILOT_PROVIDER_AVAILABLE