diff --git a/packages/backend/server/migrations/20250303102501_add_workspace_doc_title_and_summary/migration.sql b/packages/backend/server/migrations/20250303102501_add_workspace_doc_title_and_summary/migration.sql new file mode 100644 index 0000000000..efc4d8b92c --- /dev/null +++ b/packages/backend/server/migrations/20250303102501_add_workspace_doc_title_and_summary/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "workspace_pages" ADD COLUMN "summary" VARCHAR, +ADD COLUMN "title" VARCHAR; diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma index 8449a60e34..b69edff2d5 100644 --- a/packages/backend/server/schema.prisma +++ b/packages/backend/server/schema.prisma @@ -126,6 +126,8 @@ model WorkspaceDoc { mode Int @default(0) @db.SmallInt // Whether the doc is blocked blocked Boolean @default(false) + title String? @db.VarChar + summary String? @db.VarChar workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade) diff --git a/packages/backend/server/src/__tests__/models/doc.spec.ts b/packages/backend/server/src/__tests__/models/doc.spec.ts index ca745b1533..55a4b231a5 100644 --- a/packages/backend/server/src/__tests__/models/doc.spec.ts +++ b/packages/backend/server/src/__tests__/models/doc.spec.ts @@ -532,4 +532,130 @@ test('should get public docs of a workspace', async t => { t.deepEqual(docs.map(d => d.docId).sort(), [docId1, docId2]); }); +test('should update title and summary', async t => { + const docId = randomUUID(); + const snapshot = { + spaceId: workspace.id, + docId, + blob: Buffer.from('blob1'), + timestamp: Date.now(), + editorId: user.id, + }; + await t.context.doc.upsert(snapshot); + const content = { + title: 'test title', + summary: 'test summary', + }; + await t.context.doc.upsertMeta(workspace.id, docId, content); + const foundContent = await t.context.doc.getMeta(workspace.id, docId, { + select: { + title: true, + summary: true, + }, + }); + t.deepEqual(foundContent, content); + const updatedContent = { + title: 'test title 2', + summary: 'test summary 2', + }; + await t.context.doc.upsertMeta(workspace.id, docId, updatedContent); + const foundUpdatedContent = await t.context.doc.getMeta(workspace.id, docId, { + select: { + title: true, + summary: true, + }, + }); + t.deepEqual(foundUpdatedContent, updatedContent); +}); + +test('should find metas by workspaceIds and docIds', async t => { + const docId1 = randomUUID(); + const docId2 = randomUUID(); + const docId3 = randomUUID(); + const snapshot1 = { + spaceId: workspace.id, + docId: docId1, + blob: Buffer.from('blob1'), + timestamp: Date.now(), + editorId: user.id, + }; + const snapshot2 = { + spaceId: workspace.id, + docId: docId2, + blob: Buffer.from('blob2'), + timestamp: Date.now(), + editorId: user.id, + }; + const snapshot3 = { + spaceId: workspace.id, + docId: docId3, + blob: Buffer.from('blob3'), + timestamp: Date.now(), + editorId: user.id, + }; + await t.context.doc.upsert(snapshot1); + await t.context.doc.upsert(snapshot2); + await t.context.doc.upsert(snapshot3); + const content1 = { + title: 'test title', + summary: 'test summary', + }; + const content2 = { + title: 'test title 2', + summary: 'test summary 2', + }; + await t.context.doc.upsertMeta(workspace.id, docId1, content1); + await t.context.doc.upsertMeta(workspace.id, docId2, content2); + let contents = await t.context.doc.findMetas([ + { workspaceId: workspace.id, docId: docId1 }, + { workspaceId: workspace.id, docId: randomUUID() }, + { workspaceId: randomUUID(), docId: docId1 }, + { workspaceId: workspace.id, docId: docId2 }, + { workspaceId: randomUUID(), docId: randomUUID() }, + ]); + t.deepEqual( + contents.map(c => + c + ? { + title: c.title, + summary: c.summary, + } + : null + ), + [content1, null, null, content2, null] + ); + contents = await t.context.doc.findMetas([ + { workspaceId: workspace.id, docId: docId1 }, + { workspaceId: workspace.id, docId: docId2 }, + ]); + t.deepEqual( + contents.map(c => + c + ? { + title: c.title, + summary: c.summary, + } + : null + ), + [content1, content2] + ); + // docId3 don't have meta + contents = await t.context.doc.findMetas([ + { workspaceId: workspace.id, docId: docId1 }, + { workspaceId: workspace.id, docId: docId2 }, + { workspaceId: workspace.id, docId: docId3 }, + ]); + t.deepEqual( + contents.map(c => + c + ? { + title: c.title, + summary: c.summary, + } + : null + ), + [content1, content2, null] + ); +}); + // #endregion diff --git a/packages/backend/server/src/core/doc/__tests__/event.spec.ts b/packages/backend/server/src/core/doc/__tests__/event.spec.ts new file mode 100644 index 0000000000..9acc4d465c --- /dev/null +++ b/packages/backend/server/src/core/doc/__tests__/event.spec.ts @@ -0,0 +1,120 @@ +import { randomUUID } from 'node:crypto'; +import { mock } from 'node:test'; + +import ava, { TestFn } from 'ava'; +import Sinon from 'sinon'; +import { Doc as YDoc } from 'yjs'; + +import { + createTestingModule, + type TestingModule, +} from '../../../__tests__/utils'; +import { Models, User, Workspace } from '../../../models'; +import { DocReader, PgWorkspaceDocStorageAdapter as Adapter } from '..'; +import { DocEventsListener } from '../event'; + +interface Context { + module: TestingModule; + docReader: DocReader; + adapter: Adapter; + models: Models; + listener: DocEventsListener; +} + +const test = ava as TestFn; + +test.before(async t => { + const module = await createTestingModule(); + t.context.module = module; + t.context.models = module.get(Models); + t.context.docReader = module.get(DocReader); + t.context.adapter = module.get(Adapter); + t.context.listener = module.get(DocEventsListener); +}); + +let owner: User; +let workspace: Workspace; + +test.beforeEach(async t => { + await t.context.module.initTestingDB(); + owner = await t.context.models.user.create({ + email: `${randomUUID()}@affine.pro`, + }); + workspace = await t.context.models.workspace.create(owner.id); +}); + +test.afterEach.always(() => { + mock.reset(); + Sinon.restore(); +}); + +test.after.always(async t => { + await t.context.module.close(); +}); + +test('should update doc content to database when doc is updated', async t => { + const { docReader, models, adapter, listener } = t.context; + const updates: Buffer[] = []; + { + const doc = new YDoc(); + doc.on('update', data => { + updates.push(Buffer.from(data)); + }); + + const text = doc.getText('content'); + text.insert(0, 'hello'); + text.insert(5, 'world'); + } + + const docId = randomUUID(); + await adapter.pushDocUpdates(workspace.id, docId, updates); + await adapter.getDoc(workspace.id, docId); + + mock.method(docReader, 'parseDocContent', () => { + return { + title: 'test title', + summary: 'test summary', + }; + }); + + const spy = Sinon.spy(models.doc, 'upsertMeta'); + await listener.markDocContentCacheStale({ + workspaceId: workspace.id, + docId, + blob: Buffer.from([]), + }); + t.is(spy.callCount, 1); + const content = await models.doc.getMeta(workspace.id, docId); + t.truthy(content); + t.is(content!.title, 'test title'); + t.is(content!.summary, 'test summary'); +}); + +test('should ignore update doc content to database when snapshot parse failed', async t => { + const { models, adapter, listener } = t.context; + const updates: Buffer[] = []; + { + const doc = new YDoc(); + doc.on('update', data => { + updates.push(Buffer.from(data)); + }); + + const text = doc.getText('content'); + text.insert(0, 'hello'); + text.insert(5, 'world'); + } + + const docId = randomUUID(); + await adapter.pushDocUpdates(workspace.id, docId, updates); + const doc = await adapter.getDoc(workspace.id, docId); + + const spy = Sinon.spy(models.doc, 'upsertMeta'); + await listener.markDocContentCacheStale({ + workspaceId: workspace.id, + docId, + blob: Buffer.from(doc!.bin), + }); + t.is(spy.callCount, 0); + const content = await models.doc.getMeta(workspace.id, docId); + t.is(content, null); +}); diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts index a645cc3744..a3bb9f727d 100644 --- a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts +++ b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts @@ -193,7 +193,6 @@ test('should get workspace content with default avatar', async t => { user.id ); - // @ts-expect-error parseWorkspaceContent is private const track = mock.method(docReader, 'parseWorkspaceContent', () => ({ name: 'Test Workspace', avatarKey: '', @@ -240,7 +239,6 @@ test('should get workspace content with custom avatar', async t => { Buffer.from('mock avatar image data here') ); - // @ts-expect-error parseWorkspaceContent is private const track = mock.method(docReader, 'parseWorkspaceContent', () => ({ name: 'Test Workspace', avatarKey, diff --git a/packages/backend/server/src/core/doc/adapters/workspace.ts b/packages/backend/server/src/core/doc/adapters/workspace.ts index 9ed5ebc1bd..dbcc357083 100644 --- a/packages/backend/server/src/core/doc/adapters/workspace.ts +++ b/packages/backend/server/src/core/doc/adapters/workspace.ts @@ -32,6 +32,7 @@ declare global { 'doc.snapshot.updated': { workspaceId: string; docId: string; + blob: Buffer; }; 'doc.created': { workspaceId: string; @@ -335,10 +336,11 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { } try { + const blob = Buffer.from(snapshot.bin); const updatedSnapshot = await this.models.doc.upsert({ spaceId: snapshot.spaceId, docId: snapshot.docId, - blob: Buffer.from(snapshot.bin), + blob, timestamp: snapshot.timestamp, editorId: snapshot.editor, }); @@ -347,6 +349,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { this.event.emit('doc.snapshot.updated', { workspaceId: snapshot.spaceId, docId: snapshot.docId, + blob, }); } diff --git a/packages/backend/server/src/core/doc/event.ts b/packages/backend/server/src/core/doc/event.ts index 1f90641737..ead62833b1 100644 --- a/packages/backend/server/src/core/doc/event.ts +++ b/packages/backend/server/src/core/doc/event.ts @@ -1,17 +1,31 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '../../base'; +import { Models } from '../../models'; import { DocReader } from './reader'; @Injectable() export class DocEventsListener { - constructor(private readonly doc: DocReader) {} + constructor( + private readonly docReader: DocReader, + private readonly models: Models + ) {} @OnEvent('doc.snapshot.updated') async markDocContentCacheStale({ workspaceId, docId, + blob, }: Events['doc.snapshot.updated']) { - await this.doc.markDocContentCacheStale(workspaceId, docId); + await this.docReader.markDocContentCacheStale(workspaceId, docId); + const isDoc = workspaceId !== docId; + // update doc content to database + if (isDoc) { + const content = this.docReader.parseDocContent(blob); + if (!content) { + return; + } + await this.models.doc.upsertMeta(workspaceId, docId, content); + } } } diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.ts index d4658b7be7..a15f0bdf5a 100644 --- a/packages/backend/server/src/core/doc/reader.ts +++ b/packages/backend/server/src/core/doc/reader.ts @@ -36,6 +36,18 @@ export interface WorkspaceDocInfo { export abstract class DocReader { constructor(protected readonly cache: Cache) {} + parseDocContent(bin: Uint8Array) { + const doc = new YDoc(); + applyUpdate(doc, bin); + return parsePageDoc(doc); + } + + parseWorkspaceContent(bin: Uint8Array) { + const doc = new YDoc(); + applyUpdate(doc, bin); + return parseWorkspaceDoc(doc); + } + abstract getDoc( workspaceId: string, docId: string @@ -47,6 +59,7 @@ export abstract class DocReader { stateVector?: Uint8Array ): Promise; + // TODO(@fengmk2): should remove this method after frontend support doc content update async getDocContent( workspaceId: string, docId: string @@ -149,9 +162,7 @@ export class DatabaseDocReader extends DocReader { if (!docRecord) { return null; } - const doc = new YDoc(); - applyUpdate(doc, docRecord.bin); - return parsePageDoc(doc); + return this.parseDocContent(docRecord.bin); } protected override async getWorkspaceContentWithoutCache( @@ -178,12 +189,6 @@ export class DatabaseDocReader extends DocReader { avatarUrl, }; } - - private parseWorkspaceContent(bin: Uint8Array) { - const doc = new YDoc(); - applyUpdate(doc, bin); - return parseWorkspaceDoc(doc); - } } @Injectable() diff --git a/packages/backend/server/src/models/doc.ts b/packages/backend/server/src/models/doc.ts index 3ff93d5be9..e8700786c2 100644 --- a/packages/backend/server/src/models/doc.ts +++ b/packages/backend/server/src/models/doc.ts @@ -7,8 +7,6 @@ import { DocIsNotPublic } from '../base/error'; import { BaseModel } from './base'; import { Doc, DocRole, PublicDocMode, publicUserSelect } from './common'; -export interface DocRecord extends Doc {} - export type DocMetaUpsertInput = Omit< Prisma.WorkspaceDocUncheckedCreateInput, 'workspaceId' | 'docId' @@ -27,7 +25,7 @@ export type DocMetaUpsertInput = Omit< export class DocModel extends BaseModel { // #region Update - private updateToDocRecord(row: Update): DocRecord { + private updateToDocRecord(row: Update): Doc { return { spaceId: row.workspaceId, docId: row.id, @@ -37,7 +35,7 @@ export class DocModel extends BaseModel { }; } - private docRecordToUpdate(record: DocRecord): Update { + private docRecordToUpdate(record: Doc): Update { return { workspaceId: record.spaceId, id: record.docId, @@ -48,7 +46,7 @@ export class DocModel extends BaseModel { }; } - async createUpdates(updates: DocRecord[]) { + async createUpdates(updates: Doc[]) { return await this.db.update.createMany({ data: updates.map(r => this.docRecordToUpdate(r)), }); @@ -57,7 +55,7 @@ export class DocModel extends BaseModel { /** * Find updates by workspaceId and docId. */ - async findUpdates(workspaceId: string, docId: string): Promise { + async findUpdates(workspaceId: string, docId: string): Promise { const rows = await this.db.update.findMany({ where: { workspaceId, @@ -349,6 +347,21 @@ export class DocModel extends BaseModel { }); } + async findMetas(ids: { workspaceId: string; docId: string }[]) { + const rows = await this.db.workspaceDoc.findMany({ + where: { + workspaceId: { in: ids.map(id => id.workspaceId) }, + docId: { in: ids.map(id => id.docId) }, + }, + }); + const resultMap = new Map( + rows.map(row => [`${row.workspaceId}-${row.docId}`, row]) + ); + return ids.map( + id => resultMap.get(`${id.workspaceId}-${id.docId}`) ?? null + ); + } + /** * Find the workspace public doc metas. */ diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 44e7b4e55c..4d73971275 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -402,6 +402,7 @@ enum ErrorNames { MEMBER_NOT_FOUND_IN_SPACE MEMBER_QUOTA_EXCEEDED MISSING_OAUTH_QUERY_PARAMETER + NETWORK_ERROR NOT_FOUND NOT_IN_SPACE NO_COPILOT_PROVIDER_AVAILABLE