Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-19 19:16:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: refactor doc write in native (#14272)

Browse Source
This commit is contained in:
DarkSky
2026-01-18 16:31:12 +08:00
committed by GitHub
parent 753b11deeb
commit f373e08583
52 changed files with 7140 additions and 3559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md
+27
View File
@@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
}
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md
+27
View File
@@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
}
packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap
BIN
View File
Binary file not shown.
packages/backend/server/src/core/doc/writer.ts
+250 -14
View File
@@ -1,10 +1,14 @@
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
import { nanoid } from 'nanoid';
import { EventBus } from '../../base';
import {
addDocToRootDoc,
markdownToDocBinary,
createDocWithMarkdown,
updateDocProperties,
updateDocTitle,
updateDocWithMarkdown,
updateRootDocMetaTitle,
} from '../../native';
import { PgWorkspaceDocStorageAdapter } from './adapters/workspace';
@@ -16,22 +20,40 @@ export interface UpdateDocResult {
success: boolean;
}
declare global {
interface Events {
'doc.updates.pushed': {
spaceType: 'workspace' | 'userspace';
spaceId: string;
docId: string;
updates: Uint8Array[];
timestamp: number;
editor?: string;
};
}
}
@Injectable()
export class DocWriter {
private readonly logger = new Logger(DocWriter.name);
constructor(private readonly storage: PgWorkspaceDocStorageAdapter) {}
constructor(
private readonly storage: PgWorkspaceDocStorageAdapter,
private readonly event: EventBus
) {}
/**
* Creates a new document from markdown content.
*
* @param workspaceId - The workspace ID
* @param markdown - The markdown content
* @param title - The document title
* @param markdown - The markdown content (body only)
* @param editorId - Optional editor ID for tracking
* @returns The created document ID
*/
async createDoc(
workspaceId: string,
title: string,
markdown: string,
editorId?: string
): Promise<CreateDocResult> {
@@ -58,24 +80,50 @@ export class DocWriter {
`Creating doc ${docId} in workspace ${workspaceId} from markdown`
);
// Convert markdown to y-octo binary
const binary = markdownToDocBinary(markdown, docId);
// Extract title from markdown (first H1 heading)
const titleMatch = markdown.match(/^#\s+(.+?)(?:\s*#+)?\s*$/m);
const title = titleMatch ? titleMatch[1].trim() : undefined;
// Convert markdown to y-octo binary using the provided title
const binary = createDocWithMarkdown(title, markdown, docId);
// Prepare root doc update to register the new document
const rootDocUpdate = addDocToRootDoc(rootDocBin, docId, title);
// Push both updates together - root doc first, then the new doc
await this.storage.pushDocUpdates(
const rootTimestamp = await this.storage.pushDocUpdates(
workspaceId,
workspaceId,
[rootDocUpdate],
editorId
);
await this.storage.pushDocUpdates(workspaceId, docId, [binary], editorId);
this.emitDocUpdatesPushed({
spaceId: workspaceId,
docId: workspaceId,
updates: [rootDocUpdate],
timestamp: rootTimestamp,
editor: editorId,
});
const docTimestamp = await this.storage.pushDocUpdates(
workspaceId,
docId,
[binary],
editorId
);
this.emitDocUpdatesPushed({
spaceId: workspaceId,
docId,
updates: [binary],
timestamp: docTimestamp,
editor: editorId,
});
await this.updateDocProperties(
workspaceId,
docId,
{
createdBy: editorId,
updatedBy: editorId,
},
editorId
);
this.logger.debug(
`Created and registered doc ${docId} in workspace ${workspaceId}`
@@ -88,8 +136,10 @@ export class DocWriter {
* Updates an existing document with new markdown content.
*
* Uses structural diffing to compute minimal changes between the existing
* document and new markdown, then applies only the delta. This preserves
* document history and enables proper CRDT merging with concurrent edits.
* document and new markdown, then applies block-level replacements for
* changed blocks. This preserves document history and enables proper CRDT
* merging with concurrent edits.
* Note: this does not update the document title.
*
* @param workspaceId - The workspace ID
* @param docId - The document ID to update
@@ -124,8 +174,194 @@ export class DocWriter {
const delta = updateDocWithMarkdown(existingBinary, markdown, docId);
// Push only the delta changes
await this.storage.pushDocUpdates(workspaceId, docId, [delta], editorId);
const timestamp = await this.storage.pushDocUpdates(
workspaceId,
docId,
[delta],
editorId
);
this.emitDocUpdatesPushed({
spaceId: workspaceId,
docId,
updates: [delta],
timestamp,
editor: editorId,
});
await this.updateDocProperties(
workspaceId,
docId,
{ updatedBy: editorId },
editorId
);
return { success: true };
}
/**
* Updates document metadata (currently title only).
*
* @param workspaceId - The workspace ID
* @param docId - The document ID to update
* @param meta - Metadata updates
* @param editorId - Optional editor ID for tracking
*/
async updateDocMeta(
workspaceId: string,
docId: string,
meta: { title?: string },
editorId?: string
): Promise<UpdateDocResult> {
if (meta.title === undefined) {
throw new Error('No metadata provided');
}
this.logger.debug(`Updating doc meta ${docId} in workspace ${workspaceId}`);
const existingDoc = await this.storage.getDoc(workspaceId, docId);
if (!existingDoc?.bin) {
throw new NotFoundException(`Document ${docId} not found`);
}
const rootDoc = await this.storage.getDoc(workspaceId, workspaceId);
if (!rootDoc?.bin) {
throw new NotFoundException(
`Workspace ${workspaceId} not found or has no root document`
);
}
const existingBinary = Buffer.isBuffer(existingDoc.bin)
? existingDoc.bin
: Buffer.from(
existingDoc.bin.buffer,
existingDoc.bin.byteOffset,
existingDoc.bin.byteLength
);
const rootDocBin = Buffer.isBuffer(rootDoc.bin)
? rootDoc.bin
: Buffer.from(
rootDoc.bin.buffer,
rootDoc.bin.byteOffset,
rootDoc.bin.byteLength
);
const titleUpdate = updateDocTitle(existingBinary, meta.title, docId);
const rootMetaUpdate = updateRootDocMetaTitle(
rootDocBin,
docId,
meta.title
);
const rootTimestamp = await this.storage.pushDocUpdates(
workspaceId,
workspaceId,
[rootMetaUpdate],
editorId
);
this.emitDocUpdatesPushed({
spaceId: workspaceId,
docId: workspaceId,
updates: [rootMetaUpdate],
timestamp: rootTimestamp,
editor: editorId,
});
const docTimestamp = await this.storage.pushDocUpdates(
workspaceId,
docId,
[titleUpdate],
editorId
);
this.emitDocUpdatesPushed({
spaceId: workspaceId,
docId,
updates: [titleUpdate],
timestamp: docTimestamp,
editor: editorId,
});
await this.updateDocProperties(
workspaceId,
docId,
{ updatedBy: editorId },
editorId
);
return { success: true };
}
private emitDocUpdatesPushed(payload: {
spaceId: string;
docId: string;
updates: Uint8Array[];
timestamp: number;
editor?: string;
}) {
this.event.emit('doc.updates.pushed', {
spaceType: 'workspace',
spaceId: payload.spaceId,
docId: payload.docId,
updates: payload.updates,
timestamp: payload.timestamp,
editor: payload.editor,
});
}
private async updateDocProperties(
workspaceId: string,
docId: string,
props: { createdBy?: string; updatedBy?: string },
editorId?: string
) {
if (!editorId) {
return;
}
if (
workspaceId === docId ||
docId.startsWith('db$') ||
docId.startsWith('userdata$')
) {
return;
}
if (!props.createdBy && !props.updatedBy) {
return;
}
const propertiesDocId = `db$${workspaceId}$docProperties`;
const existingDoc = await this.storage.getDoc(workspaceId, propertiesDocId);
const existingBinary = existingDoc?.bin
? Buffer.isBuffer(existingDoc.bin)
? existingDoc.bin
: Buffer.from(
existingDoc.bin.buffer,
existingDoc.bin.byteOffset,
existingDoc.bin.byteLength
)
: Buffer.alloc(0);
const update = updateDocProperties(
existingBinary,
propertiesDocId,
docId,
props.createdBy,
props.updatedBy
);
if (this.storage.isEmptyBin(update)) {
return;
}
const timestamp = await this.storage.pushDocUpdates(
workspaceId,
propertiesDocId,
[update],
editorId
);
this.emitDocUpdatesPushed({
spaceId: workspaceId,
docId: propertiesDocId,
updates: [update],
timestamp,
editor: editorId,
});
}
}
packages/backend/server/src/core/sync/gateway.ts
+46 -1
View File
@@ -6,9 +6,10 @@ import {
OnGatewayDisconnect,
SubscribeMessage as RawSubscribeMessage,
WebSocketGateway,
WebSocketServer,
} from '@nestjs/websockets';
import { ClsInterceptor } from 'nestjs-cls';
import { Socket } from 'socket.io';
import { type Server, Socket } from 'socket.io';
import {
CallMetric,
@@ -18,6 +19,7 @@ import {
GatewayErrorWrapper,
metrics,
NotInSpace,
OnEvent,
SpaceAccessDenied,
} from '../../base';
import { Models } from '../../models';
@@ -141,6 +143,9 @@ export class SpaceSyncGateway
{
protected logger = new Logger(SpaceSyncGateway.name);
@WebSocketServer()
private readonly server!: Server;
private connectionCount = 0;
constructor(
@@ -166,6 +171,46 @@ export class SpaceSyncGateway
metrics.socketio.gauge('connections').record(this.connectionCount);
}
@OnEvent('doc.updates.pushed')
onDocUpdatesPushed({
spaceType,
spaceId,
docId,
updates,
timestamp,
editor,
}: Events['doc.updates.pushed']) {
if (!this.server || updates.length === 0) {
return;
}
const encodedUpdates = updates.map(update =>
Buffer.from(update).toString('base64')
);
this.server
.to(Room(spaceId, 'sync-019'))
.emit('space:broadcast-doc-updates', {
spaceType,
spaceId,
docId,
updates: encodedUpdates,
timestamp,
});
const room = `${spaceType}:${Room(spaceId)}`;
encodedUpdates.forEach(update => {
this.server.to(room).emit('space:broadcast-doc-update', {
spaceType,
spaceId,
docId,
update,
timestamp,
editor,
});
});
}
selectAdapter(client: Socket, spaceType: SpaceType): SyncSocketAdapter {
let adapters: Record<SpaceType, SyncSocketAdapter> = (client as any)
.affineSyncAdapters;
packages/backend/server/src/core/telemetry/service.ts
+18 -8
View File
@@ -133,14 +133,24 @@ export class TelemetryService {
};
} catch (error) {
const err = error as Error;
this.logger.error('Telemetry forwarding failed', err);
return {
ok: false,
error: {
name: err?.name ?? 'TelemetryForwardingError',
message: err?.message ?? 'Telemetry forwarding failed',
},
};
if (env.dev) {
this.logger.error('Telemetry forwarding failed', err);
return {
ok: false,
error: {
name: err?.name ?? 'TelemetryForwardingError',
message: err?.message ?? 'Telemetry forwarding failed',
},
};
} else {
return {
ok: false,
error: {
name: 'TelemetryForwardingError',
message: 'Telemetry forwarding failed',
},
};
}
}
}
packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md
+87 -8
View File
@@ -1440,6 +1440,33 @@ Generated by [AVA](https://avajs.dev).
␊
␊
␊
␊
[](Bookmark,https://affine.pro/)␊
␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
}
@@ -1449,55 +1476,74 @@ Generated by [AVA](https://avajs.dev).
> Snapshot 1
{
markdown: `<!-- block_id=RX4CG2zsBk flavour=affine:note -->␊
markdown: `<!-- block_id=FoPQcAyV_m flavour=affine:paragraph -->␊
AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
␊
<!-- block_id=oz48nn_zp8 flavour=affine:paragraph -->␊
␊
␊
<!-- block_id=g8a-D9-jXS flavour=affine:paragraph -->␊
# You own your data, with no compromises␊
␊
<!-- block_id=J8lHN1GR_5 flavour=affine:paragraph -->␊
## Local-first & Real-time collaborative␊
␊
<!-- block_id=xCuWdM0VLz flavour=affine:paragraph -->␊
We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊
␊
<!-- block_id=zElMi0tViK flavour=affine:paragraph -->␊
AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊
␊
<!-- block_id=Z4rK0OF9Wk flavour=affine:paragraph -->␊
␊
␊
<!-- block_id=S1mkc8zUoU flavour=affine:note -->␊
<!-- block_id=DQ0Ryb-SpW flavour=affine:paragraph -->␊
### Blocks that assemble your next docs, tasks kanban or whiteboard␊
␊
<!-- block_id=yGlBdshAqN flavour=affine:note -->␊
<!-- block_id=HAZC3URZp_ flavour=affine:paragraph -->␊
There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊
␊
<!-- block_id=0H87ypiuv8 flavour=affine:paragraph -->␊
We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊
␊
<!-- block_id=Sp4G1KD0Wn flavour=affine:paragraph -->␊
If you want to learn more about the product design of AFFiNE, here goes the concepts:␊
␊
<!-- block_id=RsUhDuEqXa flavour=affine:paragraph -->␊
To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊
␊
<!-- block_id=6lDiuDqZGL flavour=affine:note -->␊
<!-- block_id=Z2HibKzAr- flavour=affine:paragraph -->␊
## A true canvas for blocks in any form␊
␊
<!-- block_id=UwvWddamzM flavour=affine:paragraph -->␊
[Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊
␊
<!-- block_id=g9xKUjhJj1 flavour=affine:paragraph -->␊
␊
␊
<!-- block_id=wDTn4YJ4pm flavour=affine:paragraph -->␊
"We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊
␊
<!-- block_id=xFrrdiP3-V flavour=affine:list -->␊
* Quip & Notion with their great concept of "everything is a block"␊
<!-- block_id=Tp9xyN4Okl flavour=affine:list -->␊
* Trello with their Kanban␊
<!-- block_id=K_4hUzKZFQ flavour=affine:list -->␊
* Airtable & Miro with their no-code programable datasheets␊
<!-- block_id=QwMzON2s7x flavour=affine:list -->␊
* Miro & Whimiscal with their edgeless visual whiteboard␊
<!-- block_id=FFVmit6u1T flavour=affine:list -->␊
* Remnote & Capacities with their object-based tag system␊
<!-- block_id=cauvaHOQmh flavour=affine:note -->␊
<!-- block_id=YqnG5O6AE6 flavour=affine:paragraph -->␊
For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊
␊
<!-- block_id=sbDTmZMZcq flavour=affine:paragraph -->␊
## Self Host␊
␊
<!-- block_id=QVvitesfbj flavour=affine:paragraph -->␊
Self host AFFiNE␊
␊
<!-- block_id=2jwCeO8Yot flavour=affine:note -->␊
<!-- block_id=U_GoHFD9At flavour=affine:database -->␊
␊
### Learning From␊
||Title|Tag|␊
@@ -1510,14 +1556,47 @@ Generated by [AVA](https://avajs.dev).
|Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
|Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊
␊
<!-- block_id=c9MF_JiRgx flavour=affine:note -->␊
<!-- block_id=NyHXrMX3R1 flavour=affine:paragraph -->␊
## Affine Development␊
␊
<!-- block_id=9-K49otbCv flavour=affine:paragraph -->␊
For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊
␊
<!-- block_id=faFteK9eG- flavour=affine:paragraph -->␊
␊
␊
<!-- block_id=6x7ALjUDjj flavour=affine:surface -->␊
<!-- block_id=ECrtbvW6xx flavour=affine:bookmark -->␊
␊
[](Bookmark,https://affine.pro/)␊
␊
<!-- block_id=5W--UQLN11 flavour=affine:bookmark -->␊
␊
[](Bookmark,https://www.youtube.com/@affinepro)␊
␊
<!-- block_id=lcZphIJe63 flavour=affine:image -->␊
<img␊
src="blob://BFZk3c2ERp-sliRvA7MQ_p3NdkdCLt2Ze0DQ9i21dpA="␊
alt=""␊
width="1302"␊
height="728"␊
/>␊
␊
<!-- block_id=JlgVJdWU12 flavour=affine:image -->␊
<img␊
src="blob://HWvCItS78DzPGbwcuaGcfkpVDUvL98IvH5SIK8-AcL8="␊
alt=""␊
width="1463"␊
height="374"␊
/>␊
␊
<!-- block_id=lht7AqBqnF flavour=affine:image -->␊
<img␊
src="blob://ZRKpsBoC88qEMmeiXKXqywfA1rLvWoLa5rpEh9x9Oj0="␊
alt=""␊
width="862"␊
height="1388"␊
/>␊
␊
`,
title: 'Write, Draw, Plan all at Once.',
}
packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap
BIN
View File
Binary file not shown.