mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-23 13:29:02 +08:00
feat: init disk remote source
This commit is contained in:
@@ -34,6 +34,7 @@
|
||||
"@types/react": "^19.0.1",
|
||||
"@types/react-dom": "^19.0.2",
|
||||
"cross-env": "^10.1.0",
|
||||
"typescript": "^5.9.3"
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^3.2.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,6 +60,20 @@ export function setupStoreManager(framework: Framework) {
|
||||
|
||||
framework.impl(NbstoreProvider, {
|
||||
openStore(key, options) {
|
||||
try {
|
||||
// E2E/debug only: capture init options passed to the nbstore worker.
|
||||
(globalThis as any).__e2eNbstoreOpenStoreLogs =
|
||||
(globalThis as any).__e2eNbstoreOpenStoreLogs ?? [];
|
||||
(globalThis as any).__e2eNbstoreOpenStoreLogs.push({
|
||||
key,
|
||||
remotes: Object.keys(options?.remotes ?? {}),
|
||||
diskSyncFolder:
|
||||
(options as any)?.remotes?.['disk']?.doc?.opts?.syncFolder ?? null,
|
||||
});
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
const { store, dispose } = storeManagerClient.open(key, options);
|
||||
|
||||
return {
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
import type { DiskSyncEvent } from '@affine/nbstore/disk';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { createDiskSyncApis } from './disk-sync-bridge';
|
||||
|
||||
describe('createDiskSyncApis', () => {
|
||||
it('forwards handler calls and filters events by session id', async () => {
|
||||
const startSession = vi.fn(async () => {});
|
||||
const stopSession = vi.fn(async () => {});
|
||||
const applyLocalUpdate = vi.fn(async () => ({
|
||||
docId: 'doc-1',
|
||||
timestamp: new Date('2026-01-04T00:00:00.000Z'),
|
||||
}));
|
||||
|
||||
const listeners = new Set<
|
||||
(payload: { sessionId: string; event: DiskSyncEvent }) => void
|
||||
>();
|
||||
const onEvent = vi.fn(
|
||||
(
|
||||
callback: (payload: { sessionId: string; event: DiskSyncEvent }) => void
|
||||
) => {
|
||||
listeners.add(callback);
|
||||
return () => {
|
||||
listeners.delete(callback);
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
const apis = createDiskSyncApis(
|
||||
{ startSession, stopSession, applyLocalUpdate },
|
||||
{ onEvent }
|
||||
);
|
||||
|
||||
await apis.startSession('session-a', {
|
||||
workspaceId: 'workspace-a',
|
||||
syncFolder: '/tmp/sync-a',
|
||||
});
|
||||
await apis.stopSession('session-a');
|
||||
await apis.applyLocalUpdate('session-a', {
|
||||
docId: 'doc-1',
|
||||
bin: new Uint8Array([1, 2, 3]),
|
||||
});
|
||||
|
||||
expect(startSession).toHaveBeenCalledWith('session-a', {
|
||||
workspaceId: 'workspace-a',
|
||||
syncFolder: '/tmp/sync-a',
|
||||
});
|
||||
expect(stopSession).toHaveBeenCalledWith('session-a');
|
||||
expect(applyLocalUpdate).toHaveBeenCalledWith(
|
||||
'session-a',
|
||||
expect.objectContaining({ docId: 'doc-1' }),
|
||||
undefined
|
||||
);
|
||||
|
||||
const callback = vi.fn();
|
||||
const unsubscribe = apis.subscribeEvents('session-a', callback);
|
||||
|
||||
expect(onEvent).toHaveBeenCalledTimes(1);
|
||||
|
||||
const docUpdate: DiskSyncEvent = {
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: 'doc-2',
|
||||
bin: new Uint8Array([4, 5, 6]),
|
||||
timestamp: new Date('2026-01-04T00:00:01.000Z'),
|
||||
},
|
||||
};
|
||||
|
||||
for (const listener of listeners) {
|
||||
listener({ sessionId: 'session-b', event: { type: 'ready' } });
|
||||
listener({ sessionId: 'session-a', event: docUpdate });
|
||||
}
|
||||
|
||||
expect(callback).toHaveBeenCalledTimes(1);
|
||||
expect(callback).toHaveBeenCalledWith(docUpdate);
|
||||
|
||||
unsubscribe();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { DocClock, DocUpdate } from '@affine/nbstore';
|
||||
import type {
|
||||
DiskSessionOptions,
|
||||
DiskSyncApis,
|
||||
DiskSyncEvent,
|
||||
} from '@affine/nbstore/disk';
|
||||
|
||||
type DiskSyncEventPayload = {
|
||||
sessionId: string;
|
||||
event: DiskSyncEvent;
|
||||
};
|
||||
|
||||
interface DiskSyncHandlers {
|
||||
startSession: (
|
||||
sessionId: string,
|
||||
options: DiskSessionOptions
|
||||
) => Promise<void>;
|
||||
stopSession: (sessionId: string) => Promise<void>;
|
||||
applyLocalUpdate: (
|
||||
sessionId: string,
|
||||
update: DocUpdate,
|
||||
origin?: string
|
||||
) => Promise<DocClock>;
|
||||
}
|
||||
|
||||
interface DiskSyncEvents {
|
||||
onEvent: (callback: (payload: DiskSyncEventPayload) => void) => () => void;
|
||||
}
|
||||
|
||||
export function createDiskSyncApis(
|
||||
handlers: DiskSyncHandlers,
|
||||
events: DiskSyncEvents
|
||||
): DiskSyncApis {
|
||||
return {
|
||||
startSession: (sessionId, options) => {
|
||||
return handlers.startSession(sessionId, options);
|
||||
},
|
||||
stopSession: sessionId => {
|
||||
return handlers.stopSession(sessionId);
|
||||
},
|
||||
applyLocalUpdate: (sessionId, update, origin) => {
|
||||
return handlers.applyLocalUpdate(sessionId, update, origin);
|
||||
},
|
||||
subscribeEvents: (sessionId, callback) => {
|
||||
return events.onEvent(payload => {
|
||||
if (payload.sessionId === sessionId) {
|
||||
callback(payload.event);
|
||||
}
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
import '@affine/core/bootstrap/electron';
|
||||
|
||||
import { apis } from '@affine/electron-api';
|
||||
import { apis, events } from '@affine/electron-api';
|
||||
import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel';
|
||||
import { cloudStorages } from '@affine/nbstore/cloud';
|
||||
import { bindDiskSyncApis, diskStorages } from '@affine/nbstore/disk';
|
||||
import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite';
|
||||
import {
|
||||
bindNativeDBV1Apis,
|
||||
@@ -14,14 +15,19 @@ import {
|
||||
} from '@affine/nbstore/worker/consumer';
|
||||
import { OpConsumer } from '@toeverything/infra/op';
|
||||
|
||||
import { createDiskSyncApis } from './disk-sync-bridge';
|
||||
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
bindNativeDBApis(apis!.nbstore);
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
bindNativeDBV1Apis(apis!.db);
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
bindDiskSyncApis(createDiskSyncApis(apis!.diskSync, events!.diskSync));
|
||||
|
||||
const storeManager = new StoreManagerConsumer([
|
||||
...sqliteStorages,
|
||||
...sqliteV1Storages,
|
||||
...diskStorages,
|
||||
...broadcastChannelStorages,
|
||||
...cloudStorages,
|
||||
]);
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import type { DiskSyncEvent as NativeDiskSyncEvent } from '@affine/native';
|
||||
import { DiskSync } from '@affine/native';
|
||||
import type { DocClock, DocUpdate } from '@affine/nbstore';
|
||||
import type { DiskSessionOptions, DiskSyncEvent } from '@affine/nbstore/disk';
|
||||
|
||||
import { diskSyncSubjects } from './subjects';
|
||||
|
||||
interface DiskSyncSubscriber {
|
||||
unsubscribe(): Promise<void | Error> | void | Error;
|
||||
}
|
||||
|
||||
type NapiMaybe<T> = T | Error;
|
||||
|
||||
function unwrapNapiResult<T>(result: NapiMaybe<T>, action: string): T {
|
||||
if (result instanceof Error) {
|
||||
throw new Error(`[disk] ${action} failed: ${result.message}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeTimestamp(timestamp: unknown): Date | null {
|
||||
const normalized =
|
||||
timestamp instanceof Date ? timestamp : new Date(timestamp as string);
|
||||
if (Number.isNaN(normalized.getTime())) {
|
||||
return null;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeDiskSyncEvent(
|
||||
event: NativeDiskSyncEvent
|
||||
): DiskSyncEvent | null {
|
||||
switch (event.type) {
|
||||
case 'ready':
|
||||
return { type: 'ready' };
|
||||
case 'doc-update': {
|
||||
if (!event.update || !(event.update.bin instanceof Uint8Array)) {
|
||||
return null;
|
||||
}
|
||||
const timestamp = normalizeTimestamp(event.update.timestamp);
|
||||
if (!timestamp) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: 'doc-update',
|
||||
update: {
|
||||
docId: event.update.docId,
|
||||
bin: event.update.bin,
|
||||
timestamp,
|
||||
editor: event.update.editor,
|
||||
},
|
||||
origin: event.origin,
|
||||
};
|
||||
}
|
||||
case 'doc-delete': {
|
||||
if (typeof event.docId !== 'string') {
|
||||
return null;
|
||||
}
|
||||
const timestamp = normalizeTimestamp(event.timestamp);
|
||||
if (!timestamp) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: 'doc-delete',
|
||||
docId: event.docId,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
case 'error': {
|
||||
if (typeof event.message !== 'string') {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: 'error',
|
||||
message: event.message,
|
||||
};
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
type DiskSyncRuntime = InstanceType<typeof DiskSync> & {
|
||||
startSession(
|
||||
sessionId: string,
|
||||
options: DiskSessionOptions
|
||||
): Promise<NapiMaybe<void>>;
|
||||
stopSession(sessionId: string): Promise<NapiMaybe<void>>;
|
||||
applyLocalUpdate(
|
||||
sessionId: string,
|
||||
update: DocUpdate,
|
||||
origin?: string
|
||||
): Promise<NapiMaybe<DocClock>>;
|
||||
subscribeEvents(
|
||||
sessionId: string,
|
||||
callback: (err: Error | null, event: NativeDiskSyncEvent) => void
|
||||
): Promise<NapiMaybe<DiskSyncSubscriber>>;
|
||||
};
|
||||
|
||||
const diskSync = new DiskSync() as DiskSyncRuntime;
|
||||
const subscriptions = new Map<string, () => Promise<void>>();
|
||||
|
||||
function e2eLog(options: DiskSessionOptions, line: string) {
|
||||
if (process.env.AFFINE_E2E !== '1') {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const p = path.join(options.syncFolder, '.disk-e2e.log');
|
||||
fs.appendFileSync(p, `${new Date().toISOString()}\t${line}\n`, 'utf8');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
export async function startSession(
|
||||
sessionId: string,
|
||||
options: DiskSessionOptions
|
||||
): Promise<void> {
|
||||
e2eLog(
|
||||
options,
|
||||
`startSession\t${sessionId}\tworkspaceId=${options.workspaceId}\tsyncFolder=${options.syncFolder}`
|
||||
);
|
||||
unwrapNapiResult(
|
||||
await diskSync.startSession(sessionId, options),
|
||||
'startSession'
|
||||
);
|
||||
|
||||
if (subscriptions.has(sessionId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const subscriber = unwrapNapiResult(
|
||||
await diskSync.subscribeEvents(sessionId, (err, event) => {
|
||||
if (err) {
|
||||
return;
|
||||
}
|
||||
const normalizedEvent = normalizeDiskSyncEvent(event);
|
||||
if (!normalizedEvent) {
|
||||
return;
|
||||
}
|
||||
diskSyncSubjects.event$.next({ sessionId, event: normalizedEvent });
|
||||
}),
|
||||
'subscribeEvents'
|
||||
);
|
||||
subscriptions.set(sessionId, async () => {
|
||||
unwrapNapiResult(await subscriber.unsubscribe(), 'unsubscribe');
|
||||
});
|
||||
}
|
||||
|
||||
export async function stopSession(sessionId: string): Promise<void> {
|
||||
await subscriptions.get(sessionId)?.();
|
||||
subscriptions.delete(sessionId);
|
||||
unwrapNapiResult(await diskSync.stopSession(sessionId), 'stopSession');
|
||||
}
|
||||
|
||||
export async function applyLocalUpdate(
|
||||
sessionId: string,
|
||||
update: DocUpdate,
|
||||
origin?: string
|
||||
): Promise<DocClock> {
|
||||
// syncFolder isn't directly available here; we log per session start only.
|
||||
return unwrapNapiResult(
|
||||
await diskSync.applyLocalUpdate(sessionId, update, origin),
|
||||
'applyLocalUpdate'
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { MainEventRegister } from '../type';
|
||||
import { applyLocalUpdate, startSession, stopSession } from './handlers';
|
||||
import { diskSyncSubjects } from './subjects';
|
||||
|
||||
export const diskSyncHandlers = {
|
||||
startSession,
|
||||
stopSession,
|
||||
applyLocalUpdate,
|
||||
};
|
||||
|
||||
export const diskSyncEvents = {
|
||||
onEvent: ((callback: (...args: any[]) => void) => {
|
||||
const subscription = diskSyncSubjects.event$.subscribe(payload => {
|
||||
callback(payload);
|
||||
});
|
||||
return () => {
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}) satisfies MainEventRegister,
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
import type { DiskSyncEvent } from '@affine/nbstore/disk';
|
||||
import { Subject } from 'rxjs';
|
||||
|
||||
export interface DiskSyncSessionEvent {
|
||||
sessionId: string;
|
||||
event: DiskSyncEvent;
|
||||
}
|
||||
|
||||
export const diskSyncSubjects = {
|
||||
event$: new Subject<DiskSyncSessionEvent>(),
|
||||
};
|
||||
@@ -1,4 +1,5 @@
|
||||
import { dialogHandlers } from './dialog';
|
||||
import { diskSyncEvents, diskSyncHandlers } from './disk-sync';
|
||||
import { dbEventsV1, dbHandlersV1, nbstoreHandlers } from './nbstore';
|
||||
import { provideExposed } from './provide';
|
||||
import { workspaceEvents, workspaceHandlers } from './workspace';
|
||||
@@ -6,12 +7,14 @@ import { workspaceEvents, workspaceHandlers } from './workspace';
|
||||
export const handlers = {
|
||||
db: dbHandlersV1,
|
||||
nbstore: nbstoreHandlers,
|
||||
diskSync: diskSyncHandlers,
|
||||
workspace: workspaceHandlers,
|
||||
dialog: dialogHandlers,
|
||||
};
|
||||
|
||||
export const events = {
|
||||
db: dbEventsV1,
|
||||
diskSync: diskSyncEvents,
|
||||
workspace: workspaceEvents,
|
||||
};
|
||||
|
||||
|
||||
@@ -82,30 +82,33 @@ function createSharedStorageApi(
|
||||
}
|
||||
});
|
||||
|
||||
const initPromise = (async () => {
|
||||
try {
|
||||
memory.setAll(init);
|
||||
const latest = await ipcRenderer.invoke(
|
||||
AFFINE_API_CHANNEL_NAME,
|
||||
event === 'onGlobalStateChanged'
|
||||
? 'sharedStorage:getAllGlobalState'
|
||||
: 'sharedStorage:getAllGlobalCache'
|
||||
);
|
||||
if (latest && typeof latest === 'object') {
|
||||
memory.setAll(latest);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load initial shared storage', err);
|
||||
} finally {
|
||||
loaded = true;
|
||||
while (updateQueue.length) {
|
||||
const updates = updateQueue.shift();
|
||||
if (updates) {
|
||||
applyUpdates(updates);
|
||||
}
|
||||
// Load initial state synchronously so consumers can read values during early
|
||||
// bootstrap without awaiting `ready`. This prevents feature config races
|
||||
// (e.g. folder sync remote options) on first app load.
|
||||
try {
|
||||
memory.setAll(init);
|
||||
const latest = ipcRenderer.sendSync(
|
||||
AFFINE_API_CHANNEL_NAME,
|
||||
event === 'onGlobalStateChanged'
|
||||
? 'sharedStorage:getAllGlobalState'
|
||||
: 'sharedStorage:getAllGlobalCache'
|
||||
);
|
||||
if (latest && typeof latest === 'object') {
|
||||
memory.setAll(latest);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load initial shared storage (sync)', err);
|
||||
} finally {
|
||||
loaded = true;
|
||||
while (updateQueue.length) {
|
||||
const updates = updateQueue.shift();
|
||||
if (updates) {
|
||||
applyUpdates(updates);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
const initPromise = Promise.resolve();
|
||||
|
||||
return {
|
||||
ready: initPromise,
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { DiskSyncEvent } from '@affine/nbstore/disk';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const diskSyncMocks = vi.hoisted(() => {
|
||||
return {
|
||||
startSession: vi.fn(async () => {}),
|
||||
stopSession: vi.fn(async () => {}),
|
||||
applyLocalUpdate: vi.fn(async () => ({
|
||||
docId: 'doc-1',
|
||||
timestamp: new Date('2026-01-06T00:00:00.000Z'),
|
||||
})),
|
||||
subscribeEvents: vi.fn(
|
||||
(
|
||||
_sessionId: string,
|
||||
_callback: (err: Error | null, event: DiskSyncEvent) => void
|
||||
) => {
|
||||
return Promise.resolve({
|
||||
unsubscribe: () => {},
|
||||
});
|
||||
}
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('@affine/native', () => {
|
||||
class DiskSyncMock {
|
||||
subscribeEvents(
|
||||
sessionId: string,
|
||||
callback: (err: Error | null, event: DiskSyncEvent) => void
|
||||
) {
|
||||
return diskSyncMocks.subscribeEvents(sessionId, callback);
|
||||
}
|
||||
|
||||
startSession(
|
||||
sessionId: string,
|
||||
options: { workspaceId: string; syncFolder: string }
|
||||
) {
|
||||
return diskSyncMocks.startSession(sessionId, options);
|
||||
}
|
||||
|
||||
stopSession(sessionId: string) {
|
||||
return diskSyncMocks.stopSession(sessionId);
|
||||
}
|
||||
|
||||
applyLocalUpdate(
|
||||
sessionId: string,
|
||||
update: { docId: string; bin: Uint8Array },
|
||||
origin?: string
|
||||
) {
|
||||
return diskSyncMocks.applyLocalUpdate(sessionId, update, origin);
|
||||
}
|
||||
}
|
||||
|
||||
return { DiskSync: DiskSyncMock };
|
||||
});
|
||||
|
||||
import {
|
||||
applyLocalUpdate,
|
||||
startSession,
|
||||
stopSession,
|
||||
} from '../../src/helper/disk-sync/handlers';
|
||||
import { diskSyncSubjects } from '../../src/helper/disk-sync/subjects';
|
||||
|
||||
describe('disk helper handlers', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('forwards subscribeEvents payload and unsubscribes on stop', async () => {
|
||||
const unsubscribe = vi.fn();
|
||||
diskSyncMocks.subscribeEvents.mockImplementation(
|
||||
(
|
||||
_sessionId: string,
|
||||
callback: (err: Error | null, event: DiskSyncEvent) => void
|
||||
) => {
|
||||
callback(null, {
|
||||
type: 'ready',
|
||||
} as DiskSyncEvent);
|
||||
return Promise.resolve({
|
||||
unsubscribe,
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
const seen: string[] = [];
|
||||
const subscription = diskSyncSubjects.event$.subscribe(payload => {
|
||||
seen.push(payload.event.type);
|
||||
});
|
||||
|
||||
await startSession('session-subscribe', {
|
||||
workspaceId: 'workspace-subscribe',
|
||||
syncFolder: '/tmp/disk-sync',
|
||||
});
|
||||
|
||||
expect(seen).toContain('ready');
|
||||
expect(diskSyncMocks.subscribeEvents).toHaveBeenCalledWith(
|
||||
'session-subscribe',
|
||||
expect.any(Function)
|
||||
);
|
||||
|
||||
await stopSession('session-subscribe');
|
||||
expect(unsubscribe).toHaveBeenCalledTimes(1);
|
||||
subscription.unsubscribe();
|
||||
});
|
||||
|
||||
it('throws when native applyLocalUpdate returns Error payload', async () => {
|
||||
diskSyncMocks.applyLocalUpdate.mockResolvedValueOnce(
|
||||
new Error('invalid_binary')
|
||||
);
|
||||
|
||||
await expect(
|
||||
applyLocalUpdate('session-subscribe', {
|
||||
docId: 'doc-failed',
|
||||
bin: new Uint8Array([1, 2, 3]),
|
||||
})
|
||||
).rejects.toThrow('[disk] applyLocalUpdate failed: invalid_binary');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user