mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-01 09:30:01 +08:00
refactor: new provider (#4900)
This commit is contained in:
EYHN
@@ -1,107 +0,0 @@
|
||||
interface SyncUpdateSender {
|
||||
(
|
||||
guid: string,
|
||||
updates: Uint8Array[]
|
||||
): Promise<{
|
||||
accepted: boolean;
|
||||
retry: boolean;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* BatchSyncSender is simple wrapper with vanilla update sync with several advanced features:
|
||||
* - ACK mechanism, send updates sequentially with previous sync request correctly responds with ACK
|
||||
* - batching updates, when waiting for previous ACK, new updates will be buffered and sent in single sync request
|
||||
* - retryable, allow retry when previous sync request failed but with retry flag been set to true
|
||||
*/
|
||||
export class BatchSyncSender {
|
||||
private buffered: Uint8Array[] = [];
|
||||
private job: Promise<void> | null = null;
|
||||
private started = true;
|
||||
|
||||
constructor(
|
||||
private guid: string,
|
||||
private readonly rawSender: SyncUpdateSender
|
||||
) {}
|
||||
|
||||
send(update: Uint8Array) {
|
||||
this.buffered.push(update);
|
||||
this.next();
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.started = false;
|
||||
}
|
||||
|
||||
start() {
|
||||
this.started = true;
|
||||
this.next();
|
||||
}
|
||||
|
||||
private next() {
|
||||
if (!this.started || this.job || !this.buffered.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const lastIndex = Math.min(
|
||||
this.buffered.length - 1,
|
||||
99 /* max batch updates size */
|
||||
);
|
||||
const updates = this.buffered.slice(0, lastIndex + 1);
|
||||
|
||||
if (updates.length) {
|
||||
this.job = this.rawSender(this.guid, updates)
|
||||
.then(({ accepted, retry }) => {
|
||||
// remove pending updates if updates are accepted
|
||||
if (accepted) {
|
||||
this.buffered.splice(0, lastIndex + 1);
|
||||
}
|
||||
|
||||
// stop when previous sending failed and non-recoverable
|
||||
if (accepted || retry) {
|
||||
// avoid call stack overflow
|
||||
setTimeout(() => {
|
||||
this.next();
|
||||
}, 0);
|
||||
} else {
|
||||
this.stop();
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
this.stop();
|
||||
})
|
||||
.finally(() => {
|
||||
this.job = null;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class MultipleBatchSyncSender {
|
||||
private senders: Record<string, BatchSyncSender> = {};
|
||||
|
||||
constructor(private readonly rawSender: SyncUpdateSender) {}
|
||||
|
||||
async send(guid: string, update: Uint8Array) {
|
||||
return this.getSender(guid).send(update);
|
||||
}
|
||||
|
||||
private getSender(guid: string) {
|
||||
let sender = this.senders[guid];
|
||||
if (!sender) {
|
||||
sender = new BatchSyncSender(guid, this.rawSender);
|
||||
this.senders[guid] = sender;
|
||||
}
|
||||
|
||||
return sender;
|
||||
}
|
||||
|
||||
start() {
|
||||
Object.values(this.senders).forEach(sender => sender.start());
|
||||
}
|
||||
|
||||
stop() {
|
||||
Object.values(this.senders).forEach(sender => sender.stop());
|
||||
}
|
||||
}
|
||||
@@ -9,16 +9,10 @@ import {
|
||||
getWorkspaceQuery,
|
||||
getWorkspacesQuery,
|
||||
} from '@affine/graphql';
|
||||
import { createAffineDataSource } from '@affine/workspace/affine/index';
|
||||
import { createIndexeddbStorage, Workspace } from '@blocksuite/store';
|
||||
import { migrateLocalBlobStorage } from '@toeverything/infra/blocksuite';
|
||||
import {
|
||||
createIndexedDBProvider,
|
||||
DEFAULT_DB_NAME,
|
||||
} from '@toeverything/y-indexeddb';
|
||||
import { getSession } from 'next-auth/react';
|
||||
import { proxy } from 'valtio/vanilla';
|
||||
import { syncDataSourceFromDoc } from 'y-provider';
|
||||
|
||||
import { getOrCreateWorkspace } from '../manager';
|
||||
import { fetcher } from './gql';
|
||||
@@ -77,21 +71,6 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.AFFINE_CLOUD> = {
|
||||
})
|
||||
);
|
||||
|
||||
const datasource = createAffineDataSource(
|
||||
createWorkspace.id,
|
||||
newBlockSuiteWorkspace.doc,
|
||||
newBlockSuiteWorkspace.awarenessStore.awareness
|
||||
);
|
||||
|
||||
const disconnect = datasource.onDocUpdate(() => {});
|
||||
await syncDataSourceFromDoc(upstreamWorkspace.doc, datasource);
|
||||
disconnect();
|
||||
|
||||
const provider = createIndexedDBProvider(
|
||||
newBlockSuiteWorkspace.doc,
|
||||
DEFAULT_DB_NAME
|
||||
);
|
||||
provider.connect();
|
||||
migrateLocalBlobStorage(upstreamWorkspace.id, createWorkspace.id)
|
||||
.then(() => deleteLocalBlobStorage(upstreamWorkspace.id))
|
||||
.catch(e => {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { fetchWithTraceReport } from '@affine/graphql';
|
||||
|
||||
const hashMap = new Map<string, CloudDoc>();
|
||||
type DocPublishMode = 'edgeless' | 'page';
|
||||
|
||||
export type CloudDoc = {
|
||||
arrayBuffer: ArrayBuffer;
|
||||
publishMode: DocPublishMode;
|
||||
};
|
||||
|
||||
export async function downloadBinaryFromCloud(
|
||||
rootGuid: string,
|
||||
pageGuid: string
|
||||
): Promise<CloudDoc | null> {
|
||||
const cached = hashMap.get(`${rootGuid}/${pageGuid}`);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const response = await fetchWithTraceReport(
|
||||
runtimeConfig.serverUrlPrefix +
|
||||
`/api/workspaces/${rootGuid}/docs/${pageGuid}`,
|
||||
{
|
||||
priority: 'high',
|
||||
}
|
||||
);
|
||||
if (response.ok) {
|
||||
const publishMode = (response.headers.get('publish-mode') ||
|
||||
'page') as DocPublishMode;
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
hashMap.set(`${rootGuid}/${pageGuid}`, { arrayBuffer, publishMode });
|
||||
|
||||
// return both arrayBuffer and publish mode
|
||||
return { arrayBuffer, publishMode };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import type { Socket } from 'socket.io-client';
|
||||
import { Manager } from 'socket.io-client';
|
||||
import {
|
||||
applyAwarenessUpdate,
|
||||
type Awareness,
|
||||
encodeAwarenessUpdate,
|
||||
removeAwarenessStates,
|
||||
} from 'y-protocols/awareness';
|
||||
import type { DocDataSource } from 'y-provider';
|
||||
import type { Doc } from 'yjs';
|
||||
|
||||
import { MultipleBatchSyncSender } from './batch-sync-sender';
|
||||
import {
|
||||
type AwarenessChanges,
|
||||
base64ToUint8Array,
|
||||
uint8ArrayToBase64,
|
||||
} from './utils';
|
||||
|
||||
let ioManager: Manager | null = null;
|
||||
|
||||
// use lazy initialization to avoid global side effect
|
||||
function getIoManager(): Manager {
|
||||
if (ioManager) {
|
||||
return ioManager;
|
||||
}
|
||||
ioManager = new Manager(runtimeConfig.serverUrlPrefix + '/', {
|
||||
autoConnect: false,
|
||||
transports: ['websocket'],
|
||||
});
|
||||
return ioManager;
|
||||
}
|
||||
|
||||
const logger = new DebugLogger('affine:sync');
|
||||
|
||||
export const createAffineDataSource = (
|
||||
id: string,
|
||||
rootDoc: Doc,
|
||||
awareness: Awareness
|
||||
) => {
|
||||
if (id !== rootDoc.guid) {
|
||||
console.warn('important!! please use doc.guid as roomName');
|
||||
}
|
||||
|
||||
logger.debug('createAffineDataSource', id, rootDoc.guid);
|
||||
const socket = getIoManager().socket('/');
|
||||
const syncSender = new MultipleBatchSyncSender(async (guid, updates) => {
|
||||
const payload = await Promise.all(
|
||||
updates.map(update => uint8ArrayToBase64(update))
|
||||
);
|
||||
|
||||
return new Promise(resolve => {
|
||||
socket.emit(
|
||||
'client-update-v2',
|
||||
{
|
||||
workspaceId: rootDoc.guid,
|
||||
guid,
|
||||
updates: payload,
|
||||
},
|
||||
(response: {
|
||||
// TODO: reuse `EventError` with server
|
||||
error?: any;
|
||||
data: any;
|
||||
}) => {
|
||||
// TODO: raise error with different code to users
|
||||
if (response.error) {
|
||||
logger.error('client-update-v2 error', {
|
||||
workspaceId: rootDoc.guid,
|
||||
guid,
|
||||
response,
|
||||
});
|
||||
}
|
||||
|
||||
resolve({
|
||||
accepted: !response.error,
|
||||
// TODO: reuse `EventError` with server
|
||||
retry: response.error?.code === 'INTERNAL',
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
get socket() {
|
||||
return socket;
|
||||
},
|
||||
queryDocState: async (guid, options) => {
|
||||
const stateVector = options?.stateVector
|
||||
? await uint8ArrayToBase64(options.stateVector)
|
||||
: undefined;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
logger.debug('doc-load-v2', {
|
||||
workspaceId: rootDoc.guid,
|
||||
guid,
|
||||
stateVector,
|
||||
});
|
||||
socket.emit(
|
||||
'doc-load-v2',
|
||||
{
|
||||
workspaceId: rootDoc.guid,
|
||||
guid,
|
||||
stateVector,
|
||||
},
|
||||
(
|
||||
response: // TODO: reuse `EventError` with server
|
||||
{ error: any } | { data: { missing: string; state: string } }
|
||||
) => {
|
||||
logger.debug('doc-load callback', {
|
||||
workspaceId: rootDoc.guid,
|
||||
guid,
|
||||
stateVector,
|
||||
response,
|
||||
});
|
||||
|
||||
if ('error' in response) {
|
||||
// TODO: result `EventError` with server
|
||||
if (response.error.code === 'DOC_NOT_FOUND') {
|
||||
resolve(false);
|
||||
} else {
|
||||
reject(new Error(response.error.message));
|
||||
}
|
||||
} else {
|
||||
resolve({
|
||||
missing: base64ToUint8Array(response.data.missing),
|
||||
state: response.data.state
|
||||
? base64ToUint8Array(response.data.state)
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
},
|
||||
sendDocUpdate: async (guid: string, update: Uint8Array) => {
|
||||
logger.debug('client-update-v2', {
|
||||
workspaceId: rootDoc.guid,
|
||||
guid,
|
||||
update,
|
||||
});
|
||||
|
||||
await syncSender.send(guid, update);
|
||||
},
|
||||
onDocUpdate: callback => {
|
||||
const onUpdate = async (message: {
|
||||
workspaceId: string;
|
||||
guid: string;
|
||||
updates: string[];
|
||||
}) => {
|
||||
if (message.workspaceId === rootDoc.guid) {
|
||||
message.updates.forEach(update => {
|
||||
callback(message.guid, base64ToUint8Array(update));
|
||||
});
|
||||
}
|
||||
};
|
||||
let destroyAwareness = () => {};
|
||||
socket.on('server-updates', onUpdate);
|
||||
socket.on('connect', () => {
|
||||
socket.emit(
|
||||
'client-handshake',
|
||||
rootDoc.guid,
|
||||
(response: { error?: any }) => {
|
||||
if (!response.error) {
|
||||
syncSender.start();
|
||||
destroyAwareness = setupAffineAwareness(
|
||||
socket,
|
||||
rootDoc,
|
||||
awareness
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
socket.connect();
|
||||
return () => {
|
||||
syncSender.stop();
|
||||
socket.emit('client-leave', rootDoc.guid);
|
||||
socket.off('server-updates', onUpdate);
|
||||
destroyAwareness();
|
||||
socket.disconnect();
|
||||
};
|
||||
},
|
||||
} satisfies DocDataSource & { readonly socket: Socket };
|
||||
};
|
||||
|
||||
function setupAffineAwareness(
|
||||
conn: Socket,
|
||||
rootDoc: Doc,
|
||||
awareness: Awareness
|
||||
) {
|
||||
const awarenessBroadcast = ({
|
||||
workspaceId,
|
||||
awarenessUpdate,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
awarenessUpdate: string;
|
||||
}) => {
|
||||
if (workspaceId !== rootDoc.guid) {
|
||||
return;
|
||||
}
|
||||
applyAwarenessUpdate(
|
||||
awareness,
|
||||
base64ToUint8Array(awarenessUpdate),
|
||||
'server'
|
||||
);
|
||||
};
|
||||
|
||||
const awarenessUpdate = (changes: AwarenessChanges, origin: unknown) => {
|
||||
if (origin === 'server') {
|
||||
return;
|
||||
}
|
||||
|
||||
const changedClients = Object.values(changes).reduce((res, cur) => [
|
||||
...res,
|
||||
...cur,
|
||||
]);
|
||||
|
||||
const update = encodeAwarenessUpdate(awareness, changedClients);
|
||||
uint8ArrayToBase64(update)
|
||||
.then(encodedUpdate => {
|
||||
conn.emit('awareness-update', {
|
||||
workspaceId: rootDoc.guid,
|
||||
awarenessUpdate: encodedUpdate,
|
||||
});
|
||||
})
|
||||
.catch(err => logger.error(err));
|
||||
};
|
||||
|
||||
const newClientAwarenessInitHandler = () => {
|
||||
const awarenessUpdate = encodeAwarenessUpdate(awareness, [
|
||||
awareness.clientID,
|
||||
]);
|
||||
uint8ArrayToBase64(awarenessUpdate)
|
||||
.then(encodedAwarenessUpdate => {
|
||||
conn.emit('awareness-update', {
|
||||
guid: rootDoc.guid,
|
||||
awarenessUpdate: encodedAwarenessUpdate,
|
||||
});
|
||||
})
|
||||
.catch(err => logger.error(err));
|
||||
};
|
||||
|
||||
const windowBeforeUnloadHandler = () => {
|
||||
removeAwarenessStates(awareness, [awareness.clientID], 'window unload');
|
||||
};
|
||||
|
||||
conn.on('server-awareness-broadcast', awarenessBroadcast);
|
||||
conn.on('new-client-awareness-init', newClientAwarenessInitHandler);
|
||||
awareness.on('update', awarenessUpdate);
|
||||
|
||||
window.addEventListener('beforeunload', windowBeforeUnloadHandler);
|
||||
|
||||
conn.emit('awareness-init', rootDoc.guid);
|
||||
|
||||
return () => {
|
||||
awareness.off('update', awarenessUpdate);
|
||||
conn.off('server-awareness-broadcast', awarenessBroadcast);
|
||||
conn.off('new-client-awareness-init', newClientAwarenessInitHandler);
|
||||
window.removeEventListener('unload', windowBeforeUnloadHandler);
|
||||
};
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import { createIndexeddbStorage } from '@blocksuite/store';
|
||||
import {
|
||||
createIndexedDBDatasource,
|
||||
DEFAULT_DB_NAME,
|
||||
downloadBinary,
|
||||
} from '@toeverything/y-indexeddb';
|
||||
import { syncDataSource } from 'y-provider';
|
||||
import type { Doc } from 'yjs';
|
||||
import { applyUpdate } from 'yjs';
|
||||
|
||||
import { createCloudBlobStorage } from '../blob/cloud-blob-storage';
|
||||
import { createAffineDataSource } from '.';
|
||||
import { CRUD } from './crud';
|
||||
|
||||
const performanceLogger = new DebugLogger('performance:sync');
|
||||
let abortController: AbortController | undefined;
|
||||
|
||||
const downloadRootFromIndexedDB = async (
|
||||
rootGuid: string,
|
||||
doc: Doc,
|
||||
signal: AbortSignal
|
||||
): Promise<void> => {
|
||||
if (signal.aborted) {
|
||||
return;
|
||||
}
|
||||
const update = await downloadBinary(rootGuid);
|
||||
if (update !== false) {
|
||||
applyUpdate(doc, update);
|
||||
}
|
||||
};
|
||||
|
||||
export async function startSync() {
|
||||
performanceLogger.info('start');
|
||||
|
||||
abortController = new AbortController();
|
||||
const signal = abortController.signal;
|
||||
const workspaces = await CRUD.list();
|
||||
performanceLogger.info('CRUD list');
|
||||
|
||||
const syncDocPromises = workspaces.map(workspace =>
|
||||
downloadRootFromIndexedDB(
|
||||
workspace.id,
|
||||
workspace.blockSuiteWorkspace.doc,
|
||||
signal
|
||||
)
|
||||
);
|
||||
await Promise.all(syncDocPromises);
|
||||
performanceLogger.info('all sync promise');
|
||||
|
||||
const syncPromises = workspaces.map(workspace => {
|
||||
const remoteDataSource = createAffineDataSource(
|
||||
workspace.id,
|
||||
workspace.blockSuiteWorkspace.doc,
|
||||
workspace.blockSuiteWorkspace.awarenessStore.awareness
|
||||
);
|
||||
const indexeddbDataSource = createIndexedDBDatasource({
|
||||
dbName: DEFAULT_DB_NAME,
|
||||
});
|
||||
return syncDataSource(
|
||||
(): string[] => [
|
||||
workspace.blockSuiteWorkspace.doc.guid,
|
||||
...[...workspace.blockSuiteWorkspace.doc.subdocs].map(doc => doc.guid),
|
||||
],
|
||||
remoteDataSource,
|
||||
indexeddbDataSource
|
||||
);
|
||||
});
|
||||
|
||||
const syncBlobPromises = workspaces.map(async workspace => {
|
||||
const cloudBlobStorage = createCloudBlobStorage(workspace.id);
|
||||
const indexeddbBlobStorage = createIndexeddbStorage(workspace.id);
|
||||
return Promise.all([
|
||||
cloudBlobStorage.crud.list(),
|
||||
indexeddbBlobStorage.crud.list(),
|
||||
]).then(([cloudKeys, indexeddbKeys]) => {
|
||||
if (signal.aborted) {
|
||||
return;
|
||||
}
|
||||
const cloudKeysSet = new Set(cloudKeys);
|
||||
const indexeddbKeysSet = new Set(indexeddbKeys);
|
||||
// missing in indexeddb
|
||||
const missingLocalKeys = cloudKeys.filter(
|
||||
key => !indexeddbKeysSet.has(key)
|
||||
);
|
||||
// missing in cloud
|
||||
const missingCloudKeys = indexeddbKeys.filter(
|
||||
key => !cloudKeysSet.has(key)
|
||||
);
|
||||
return Promise.all([
|
||||
...missingLocalKeys.map(key =>
|
||||
cloudBlobStorage.crud.get(key).then(async value => {
|
||||
if (signal.aborted) {
|
||||
return;
|
||||
}
|
||||
if (value) {
|
||||
await indexeddbBlobStorage.crud.set(key, value);
|
||||
}
|
||||
})
|
||||
),
|
||||
...missingCloudKeys.map(key =>
|
||||
indexeddbBlobStorage.crud.get(key).then(async value => {
|
||||
if (signal.aborted) {
|
||||
return;
|
||||
}
|
||||
if (value) {
|
||||
await cloudBlobStorage.crud.set(key, value);
|
||||
}
|
||||
})
|
||||
),
|
||||
]);
|
||||
});
|
||||
});
|
||||
await Promise.all([...syncPromises, ...syncBlobPromises]);
|
||||
performanceLogger.info('sync done');
|
||||
}
|
||||
|
||||
export async function stopSync() {
|
||||
abortController?.abort();
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import type { Doc as YDoc } from 'yjs';
|
||||
|
||||
export type SubdocEvent = {
|
||||
loaded: Set<YDoc>;
|
||||
removed: Set<YDoc>;
|
||||
added: Set<YDoc>;
|
||||
};
|
||||
|
||||
export type UpdateHandler = (update: Uint8Array, origin: unknown) => void;
|
||||
export type SubdocsHandler = (event: SubdocEvent) => void;
|
||||
export type DestroyHandler = () => void;
|
||||
|
||||
export type AwarenessChanges = Record<
|
||||
'added' | 'updated' | 'removed',
|
||||
number[]
|
||||
>;
|
||||
|
||||
export function uint8ArrayToBase64(array: Uint8Array): Promise<string> {
|
||||
return new Promise<string>(resolve => {
|
||||
// Create a blob from the Uint8Array
|
||||
const blob = new Blob([array]);
|
||||
|
||||
const reader = new FileReader();
|
||||
reader.onload = function () {
|
||||
const dataUrl = reader.result as string | null;
|
||||
if (!dataUrl) {
|
||||
resolve('');
|
||||
return;
|
||||
}
|
||||
// The result includes the `data:` URL prefix and the MIME type. We only want the Base64 data
|
||||
const base64 = dataUrl.split(',')[1];
|
||||
resolve(base64);
|
||||
};
|
||||
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
export function base64ToUint8Array(base64: string) {
|
||||
const binaryString = atob(base64);
|
||||
const binaryArray = binaryString.split('').map(function (char) {
|
||||
return char.charCodeAt(0);
|
||||
});
|
||||
return new Uint8Array(binaryArray);
|
||||
}
|
||||
Reference in New Issue
Block a user