feat: init disk remote source

This commit is contained in:
DarkSky
2026-02-27 02:39:53 +08:00
parent 895e774569
commit 6557e5d01d
51 changed files with 6793 additions and 158 deletions
@@ -34,6 +34,7 @@
"@types/react": "^19.0.1",
"@types/react-dom": "^19.0.2",
"cross-env": "^10.1.0",
"typescript": "^5.9.3"
"typescript": "^5.9.3",
"vitest": "^3.2.4"
}
}
@@ -60,6 +60,20 @@ export function setupStoreManager(framework: Framework) {
framework.impl(NbstoreProvider, {
openStore(key, options) {
try {
// E2E/debug only: capture init options passed to the nbstore worker.
(globalThis as any).__e2eNbstoreOpenStoreLogs =
(globalThis as any).__e2eNbstoreOpenStoreLogs ?? [];
(globalThis as any).__e2eNbstoreOpenStoreLogs.push({
key,
remotes: Object.keys(options?.remotes ?? {}),
diskSyncFolder:
(options as any)?.remotes?.['disk']?.doc?.opts?.syncFolder ?? null,
});
} catch {
// ignore
}
const { store, dispose } = storeManagerClient.open(key, options);
return {
@@ -0,0 +1,79 @@
import type { DiskSyncEvent } from '@affine/nbstore/disk';
import { describe, expect, it, vi } from 'vitest';
import { createDiskSyncApis } from './disk-sync-bridge';
describe('createDiskSyncApis', () => {
it('forwards handler calls and filters events by session id', async () => {
const startSession = vi.fn(async () => {});
const stopSession = vi.fn(async () => {});
const applyLocalUpdate = vi.fn(async () => ({
docId: 'doc-1',
timestamp: new Date('2026-01-04T00:00:00.000Z'),
}));
const listeners = new Set<
(payload: { sessionId: string; event: DiskSyncEvent }) => void
>();
const onEvent = vi.fn(
(
callback: (payload: { sessionId: string; event: DiskSyncEvent }) => void
) => {
listeners.add(callback);
return () => {
listeners.delete(callback);
};
}
);
const apis = createDiskSyncApis(
{ startSession, stopSession, applyLocalUpdate },
{ onEvent }
);
await apis.startSession('session-a', {
workspaceId: 'workspace-a',
syncFolder: '/tmp/sync-a',
});
await apis.stopSession('session-a');
await apis.applyLocalUpdate('session-a', {
docId: 'doc-1',
bin: new Uint8Array([1, 2, 3]),
});
expect(startSession).toHaveBeenCalledWith('session-a', {
workspaceId: 'workspace-a',
syncFolder: '/tmp/sync-a',
});
expect(stopSession).toHaveBeenCalledWith('session-a');
expect(applyLocalUpdate).toHaveBeenCalledWith(
'session-a',
expect.objectContaining({ docId: 'doc-1' }),
undefined
);
const callback = vi.fn();
const unsubscribe = apis.subscribeEvents('session-a', callback);
expect(onEvent).toHaveBeenCalledTimes(1);
const docUpdate: DiskSyncEvent = {
type: 'doc-update',
update: {
docId: 'doc-2',
bin: new Uint8Array([4, 5, 6]),
timestamp: new Date('2026-01-04T00:00:01.000Z'),
},
};
for (const listener of listeners) {
listener({ sessionId: 'session-b', event: { type: 'ready' } });
listener({ sessionId: 'session-a', event: docUpdate });
}
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith(docUpdate);
unsubscribe();
});
});
@@ -0,0 +1,52 @@
import type { DocClock, DocUpdate } from '@affine/nbstore';
import type {
DiskSessionOptions,
DiskSyncApis,
DiskSyncEvent,
} from '@affine/nbstore/disk';
type DiskSyncEventPayload = {
sessionId: string;
event: DiskSyncEvent;
};
interface DiskSyncHandlers {
startSession: (
sessionId: string,
options: DiskSessionOptions
) => Promise<void>;
stopSession: (sessionId: string) => Promise<void>;
applyLocalUpdate: (
sessionId: string,
update: DocUpdate,
origin?: string
) => Promise<DocClock>;
}
interface DiskSyncEvents {
onEvent: (callback: (payload: DiskSyncEventPayload) => void) => () => void;
}
export function createDiskSyncApis(
handlers: DiskSyncHandlers,
events: DiskSyncEvents
): DiskSyncApis {
return {
startSession: (sessionId, options) => {
return handlers.startSession(sessionId, options);
},
stopSession: sessionId => {
return handlers.stopSession(sessionId);
},
applyLocalUpdate: (sessionId, update, origin) => {
return handlers.applyLocalUpdate(sessionId, update, origin);
},
subscribeEvents: (sessionId, callback) => {
return events.onEvent(payload => {
if (payload.sessionId === sessionId) {
callback(payload.event);
}
});
},
};
}
@@ -1,8 +1,9 @@
import '@affine/core/bootstrap/electron';
import { apis } from '@affine/electron-api';
import { apis, events } from '@affine/electron-api';
import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel';
import { cloudStorages } from '@affine/nbstore/cloud';
import { bindDiskSyncApis, diskStorages } from '@affine/nbstore/disk';
import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite';
import {
bindNativeDBV1Apis,
@@ -14,14 +15,19 @@ import {
} from '@affine/nbstore/worker/consumer';
import { OpConsumer } from '@toeverything/infra/op';
import { createDiskSyncApis } from './disk-sync-bridge';
// oxlint-disable-next-line no-non-null-assertion
bindNativeDBApis(apis!.nbstore);
// oxlint-disable-next-line no-non-null-assertion
bindNativeDBV1Apis(apis!.db);
// oxlint-disable-next-line no-non-null-assertion
bindDiskSyncApis(createDiskSyncApis(apis!.diskSync, events!.diskSync));
const storeManager = new StoreManagerConsumer([
...sqliteStorages,
...sqliteV1Storages,
...diskStorages,
...broadcastChannelStorages,
...cloudStorages,
]);
@@ -0,0 +1,169 @@
import fs from 'node:fs';
import path from 'node:path';
import type { DiskSyncEvent as NativeDiskSyncEvent } from '@affine/native';
import { DiskSync } from '@affine/native';
import type { DocClock, DocUpdate } from '@affine/nbstore';
import type { DiskSessionOptions, DiskSyncEvent } from '@affine/nbstore/disk';
import { diskSyncSubjects } from './subjects';
interface DiskSyncSubscriber {
unsubscribe(): Promise<void | Error> | void | Error;
}
type NapiMaybe<T> = T | Error;
function unwrapNapiResult<T>(result: NapiMaybe<T>, action: string): T {
if (result instanceof Error) {
throw new Error(`[disk] ${action} failed: ${result.message}`);
}
return result;
}
function normalizeTimestamp(timestamp: unknown): Date | null {
const normalized =
timestamp instanceof Date ? timestamp : new Date(timestamp as string);
if (Number.isNaN(normalized.getTime())) {
return null;
}
return normalized;
}
function normalizeDiskSyncEvent(
event: NativeDiskSyncEvent
): DiskSyncEvent | null {
switch (event.type) {
case 'ready':
return { type: 'ready' };
case 'doc-update': {
if (!event.update || !(event.update.bin instanceof Uint8Array)) {
return null;
}
const timestamp = normalizeTimestamp(event.update.timestamp);
if (!timestamp) {
return null;
}
return {
type: 'doc-update',
update: {
docId: event.update.docId,
bin: event.update.bin,
timestamp,
editor: event.update.editor,
},
origin: event.origin,
};
}
case 'doc-delete': {
if (typeof event.docId !== 'string') {
return null;
}
const timestamp = normalizeTimestamp(event.timestamp);
if (!timestamp) {
return null;
}
return {
type: 'doc-delete',
docId: event.docId,
timestamp,
};
}
case 'error': {
if (typeof event.message !== 'string') {
return null;
}
return {
type: 'error',
message: event.message,
};
}
default:
return null;
}
}
type DiskSyncRuntime = InstanceType<typeof DiskSync> & {
startSession(
sessionId: string,
options: DiskSessionOptions
): Promise<NapiMaybe<void>>;
stopSession(sessionId: string): Promise<NapiMaybe<void>>;
applyLocalUpdate(
sessionId: string,
update: DocUpdate,
origin?: string
): Promise<NapiMaybe<DocClock>>;
subscribeEvents(
sessionId: string,
callback: (err: Error | null, event: NativeDiskSyncEvent) => void
): Promise<NapiMaybe<DiskSyncSubscriber>>;
};
const diskSync = new DiskSync() as DiskSyncRuntime;
const subscriptions = new Map<string, () => Promise<void>>();
function e2eLog(options: DiskSessionOptions, line: string) {
if (process.env.AFFINE_E2E !== '1') {
return;
}
try {
const p = path.join(options.syncFolder, '.disk-e2e.log');
fs.appendFileSync(p, `${new Date().toISOString()}\t${line}\n`, 'utf8');
} catch {
// ignore
}
}
export async function startSession(
sessionId: string,
options: DiskSessionOptions
): Promise<void> {
e2eLog(
options,
`startSession\t${sessionId}\tworkspaceId=${options.workspaceId}\tsyncFolder=${options.syncFolder}`
);
unwrapNapiResult(
await diskSync.startSession(sessionId, options),
'startSession'
);
if (subscriptions.has(sessionId)) {
return;
}
const subscriber = unwrapNapiResult(
await diskSync.subscribeEvents(sessionId, (err, event) => {
if (err) {
return;
}
const normalizedEvent = normalizeDiskSyncEvent(event);
if (!normalizedEvent) {
return;
}
diskSyncSubjects.event$.next({ sessionId, event: normalizedEvent });
}),
'subscribeEvents'
);
subscriptions.set(sessionId, async () => {
unwrapNapiResult(await subscriber.unsubscribe(), 'unsubscribe');
});
}
export async function stopSession(sessionId: string): Promise<void> {
await subscriptions.get(sessionId)?.();
subscriptions.delete(sessionId);
unwrapNapiResult(await diskSync.stopSession(sessionId), 'stopSession');
}
export async function applyLocalUpdate(
sessionId: string,
update: DocUpdate,
origin?: string
): Promise<DocClock> {
// syncFolder isn't directly available here; we log per session start only.
return unwrapNapiResult(
await diskSync.applyLocalUpdate(sessionId, update, origin),
'applyLocalUpdate'
);
}
@@ -0,0 +1,20 @@
import type { MainEventRegister } from '../type';
import { applyLocalUpdate, startSession, stopSession } from './handlers';
import { diskSyncSubjects } from './subjects';
export const diskSyncHandlers = {
startSession,
stopSession,
applyLocalUpdate,
};
export const diskSyncEvents = {
onEvent: ((callback: (...args: any[]) => void) => {
const subscription = diskSyncSubjects.event$.subscribe(payload => {
callback(payload);
});
return () => {
subscription.unsubscribe();
};
}) satisfies MainEventRegister,
};
@@ -0,0 +1,11 @@
import type { DiskSyncEvent } from '@affine/nbstore/disk';
import { Subject } from 'rxjs';
export interface DiskSyncSessionEvent {
sessionId: string;
event: DiskSyncEvent;
}
export const diskSyncSubjects = {
event$: new Subject<DiskSyncSessionEvent>(),
};
@@ -1,4 +1,5 @@
import { dialogHandlers } from './dialog';
import { diskSyncEvents, diskSyncHandlers } from './disk-sync';
import { dbEventsV1, dbHandlersV1, nbstoreHandlers } from './nbstore';
import { provideExposed } from './provide';
import { workspaceEvents, workspaceHandlers } from './workspace';
@@ -6,12 +7,14 @@ import { workspaceEvents, workspaceHandlers } from './workspace';
export const handlers = {
db: dbHandlersV1,
nbstore: nbstoreHandlers,
diskSync: diskSyncHandlers,
workspace: workspaceHandlers,
dialog: dialogHandlers,
};
export const events = {
db: dbEventsV1,
diskSync: diskSyncEvents,
workspace: workspaceEvents,
};
@@ -82,30 +82,33 @@ function createSharedStorageApi(
}
});
const initPromise = (async () => {
try {
memory.setAll(init);
const latest = await ipcRenderer.invoke(
AFFINE_API_CHANNEL_NAME,
event === 'onGlobalStateChanged'
? 'sharedStorage:getAllGlobalState'
: 'sharedStorage:getAllGlobalCache'
);
if (latest && typeof latest === 'object') {
memory.setAll(latest);
}
} catch (err) {
console.error('Failed to load initial shared storage', err);
} finally {
loaded = true;
while (updateQueue.length) {
const updates = updateQueue.shift();
if (updates) {
applyUpdates(updates);
}
// Load initial state synchronously so consumers can read values during early
// bootstrap without awaiting `ready`. This prevents feature config races
// (e.g. folder sync remote options) on first app load.
try {
memory.setAll(init);
const latest = ipcRenderer.sendSync(
AFFINE_API_CHANNEL_NAME,
event === 'onGlobalStateChanged'
? 'sharedStorage:getAllGlobalState'
: 'sharedStorage:getAllGlobalCache'
);
if (latest && typeof latest === 'object') {
memory.setAll(latest);
}
} catch (err) {
console.error('Failed to load initial shared storage (sync)', err);
} finally {
loaded = true;
while (updateQueue.length) {
const updates = updateQueue.shift();
if (updates) {
applyUpdates(updates);
}
}
})();
}
const initPromise = Promise.resolve();
return {
ready: initPromise,
@@ -0,0 +1,118 @@
import type { DiskSyncEvent } from '@affine/nbstore/disk';
import { beforeEach, describe, expect, it, vi } from 'vitest';
const diskSyncMocks = vi.hoisted(() => {
return {
startSession: vi.fn(async () => {}),
stopSession: vi.fn(async () => {}),
applyLocalUpdate: vi.fn(async () => ({
docId: 'doc-1',
timestamp: new Date('2026-01-06T00:00:00.000Z'),
})),
subscribeEvents: vi.fn(
(
_sessionId: string,
_callback: (err: Error | null, event: DiskSyncEvent) => void
) => {
return Promise.resolve({
unsubscribe: () => {},
});
}
),
};
});
vi.mock('@affine/native', () => {
class DiskSyncMock {
subscribeEvents(
sessionId: string,
callback: (err: Error | null, event: DiskSyncEvent) => void
) {
return diskSyncMocks.subscribeEvents(sessionId, callback);
}
startSession(
sessionId: string,
options: { workspaceId: string; syncFolder: string }
) {
return diskSyncMocks.startSession(sessionId, options);
}
stopSession(sessionId: string) {
return diskSyncMocks.stopSession(sessionId);
}
applyLocalUpdate(
sessionId: string,
update: { docId: string; bin: Uint8Array },
origin?: string
) {
return diskSyncMocks.applyLocalUpdate(sessionId, update, origin);
}
}
return { DiskSync: DiskSyncMock };
});
import {
applyLocalUpdate,
startSession,
stopSession,
} from '../../src/helper/disk-sync/handlers';
import { diskSyncSubjects } from '../../src/helper/disk-sync/subjects';
describe('disk helper handlers', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('forwards subscribeEvents payload and unsubscribes on stop', async () => {
const unsubscribe = vi.fn();
diskSyncMocks.subscribeEvents.mockImplementation(
(
_sessionId: string,
callback: (err: Error | null, event: DiskSyncEvent) => void
) => {
callback(null, {
type: 'ready',
} as DiskSyncEvent);
return Promise.resolve({
unsubscribe,
});
}
);
const seen: string[] = [];
const subscription = diskSyncSubjects.event$.subscribe(payload => {
seen.push(payload.event.type);
});
await startSession('session-subscribe', {
workspaceId: 'workspace-subscribe',
syncFolder: '/tmp/disk-sync',
});
expect(seen).toContain('ready');
expect(diskSyncMocks.subscribeEvents).toHaveBeenCalledWith(
'session-subscribe',
expect.any(Function)
);
await stopSession('session-subscribe');
expect(unsubscribe).toHaveBeenCalledTimes(1);
subscription.unsubscribe();
});
it('throws when native applyLocalUpdate returns Error payload', async () => {
diskSyncMocks.applyLocalUpdate.mockResolvedValueOnce(
new Error('invalid_binary')
);
await expect(
applyLocalUpdate('session-subscribe', {
docId: 'doc-failed',
bin: new Uint8Array([1, 2, 3]),
})
).rejects.toThrow('[disk] applyLocalUpdate failed: invalid_binary');
});
});