feat: init disk remote source

This commit is contained in:
DarkSky
2026-02-27 02:39:53 +08:00
parent 895e774569
commit 6557e5d01d
51 changed files with 6793 additions and 158 deletions
+1
View File
@@ -12,6 +12,7 @@
"./broadcast-channel": "./src/impls/broadcast-channel/index.ts",
"./idb/v1": "./src/impls/idb/v1/index.ts",
"./cloud": "./src/impls/cloud/index.ts",
"./disk": "./src/impls/disk/index.ts",
"./sqlite": "./src/impls/sqlite/index.ts",
"./sqlite/v1": "./src/impls/sqlite/v1/index.ts",
"./sync": "./src/sync/index.ts",
@@ -0,0 +1,127 @@
import { AutoReconnectConnection } from '../../connection';
import type { DocClock, DocUpdate } from '../../storage';
import { type SpaceType, universalId } from '../../utils/universal-id';
export interface DiskSessionOptions {
workspaceId: string;
syncFolder: string;
}
export type DiskSyncEvent =
| { type: 'ready' }
| {
type: 'doc-update';
update: {
docId: string;
bin: Uint8Array;
timestamp: Date;
editor?: string;
};
origin?: string;
}
| { type: 'doc-delete'; docId: string; timestamp: Date }
| { type: 'error'; message: string };
export interface DiskSyncApis {
startSession: (
sessionId: string,
options: DiskSessionOptions
) => Promise<void>;
stopSession: (sessionId: string) => Promise<void>;
applyLocalUpdate: (
sessionId: string,
update: DocUpdate,
origin?: string
) => Promise<DocClock>;
subscribeEvents: (
sessionId: string,
callback: (event: DiskSyncEvent) => void
) => () => void;
}
interface DiskSyncOptions {
readonly flavour: string;
readonly type: SpaceType;
readonly id: string;
readonly syncFolder: string;
}
interface DiskSyncApisWrapper {
startSession: (options: DiskSessionOptions) => Promise<void>;
stopSession: () => Promise<void>;
applyLocalUpdate: (update: DocUpdate, origin?: string) => Promise<DocClock>;
subscribeEvents: (callback: (event: DiskSyncEvent) => void) => () => void;
}
let apis: DiskSyncApis | null = null;
export function bindDiskSyncApis(a: DiskSyncApis) {
apis = a;
}
export class DiskSyncConnection extends AutoReconnectConnection<{
unsubscribe: () => void;
}> {
readonly apis: DiskSyncApisWrapper;
readonly sessionId: string;
readonly flavour = this.options.flavour;
readonly type = this.options.type;
readonly id = this.options.id;
constructor(
private readonly options: DiskSyncOptions,
private readonly onEvent: (event: DiskSyncEvent) => void
) {
super();
if (!apis) {
throw new Error('Not in native context.');
}
this.sessionId = universalId({
peer: this.flavour,
type: this.type,
id: this.id,
});
this.apis = this.wrapApis(apis);
}
override get shareId(): string {
return `disk:${this.sessionId}:${this.options.syncFolder}`;
}
private wrapApis(originalApis: DiskSyncApis): DiskSyncApisWrapper {
const sessionId = this.sessionId;
return new Proxy(
{},
{
get: (_target, key: keyof DiskSyncApisWrapper) => {
const method = originalApis[key];
return (...args: unknown[]) => {
// oxlint-disable-next-line @typescript-eslint/no-explicit-any
return (method as any)(sessionId, ...args);
};
},
}
) as DiskSyncApisWrapper;
}
override async doConnect() {
await this.apis.startSession({
workspaceId: this.id,
syncFolder: this.options.syncFolder,
});
const unsubscribe = this.apis.subscribeEvents(this.onEvent);
return { unsubscribe };
}
override doDisconnect(conn: { unsubscribe: () => void }) {
try {
conn.unsubscribe();
} catch (error) {
console.error('DiskSyncConnection unsubscribe failed', error);
}
this.apis.stopSession().catch(error => {
console.error('DiskSyncConnection stopSession failed', error);
});
}
}
@@ -0,0 +1,45 @@
import fs from 'node:fs';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
const PROJECT_ROOT = path.resolve(__dirname, '../../../../../../');
const JS_BOUNDARY_FILES = [
path.join(PROJECT_ROOT, 'packages/common/nbstore/src/impls/disk/doc.ts'),
path.join(
PROJECT_ROOT,
'packages/frontend/apps/electron/src/helper/disk-sync/handlers.ts'
),
];
const FORBIDDEN_PATTERNS = [
/frontmatter/i,
/gray-matter/i,
/MarkdownAdapter/,
/markdownToSnapshot/,
/fromMarkdown/,
/toMarkdown/,
];
describe('disk boundary', () => {
it('keeps markdown/frontmatter parsing out of JS adapter layer', () => {
for (const file of JS_BOUNDARY_FILES) {
const content = fs.readFileSync(file, 'utf-8');
for (const pattern of FORBIDDEN_PATTERNS) {
expect(content).not.toMatch(pattern);
}
}
});
it('keeps JS layer focused on session orchestration APIs', () => {
const adapter = fs.readFileSync(JS_BOUNDARY_FILES[0], 'utf-8');
expect(adapter).toMatch(/applyLocalUpdate/);
const helper = fs.readFileSync(JS_BOUNDARY_FILES[1], 'utf-8');
expect(helper).toMatch(/startSession/);
expect(helper).toMatch(/stopSession/);
expect(helper).toMatch(/applyLocalUpdate/);
});
});
@@ -0,0 +1,451 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
applyUpdate,
Array as YArray,
Doc as YDoc,
encodeStateAsUpdate,
Map as YMap,
} from 'yjs';
import { universalId } from '../../utils/universal-id';
import { bindDiskSyncApis, type DiskSyncApis, type DiskSyncEvent } from './api';
import { DiskDocStorage } from './doc';
function createUpdate(text: string): Uint8Array {
const doc = new YDoc();
doc.getText('content').insert(0, text);
return encodeStateAsUpdate(doc);
}
function createMapUpdate(entries: Record<string, string>): Uint8Array {
const doc = new YDoc();
const map = doc.getMap('test');
for (const [key, value] of Object.entries(entries)) {
map.set(key, value);
}
return encodeStateAsUpdate(doc);
}
function createRootMetaUpdate(docIds: string[]): Uint8Array {
const doc = new YDoc();
const meta = doc.getMap('meta');
const pages = new YArray<YMap<unknown>>();
for (const docId of docIds) {
const page = new YMap<unknown>();
page.set('id', docId);
pages.push([page]);
}
meta.set('pages', pages);
return encodeStateAsUpdate(doc);
}
describe('DiskDocStorage', () => {
const sessionId = universalId({
peer: 'local',
type: 'workspace',
id: 'workspace-test',
});
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
const startSession = vi.fn(
async (_sessionId: string, _options: { workspaceId: string }) => {}
);
const stopSession = vi.fn(async (_sessionId: string) => {});
const applyLocalUpdate = vi.fn(
async (_sessionId: string, update: { docId: string }) => {
return {
docId: update.docId,
timestamp: new Date('2026-01-02T00:00:00.000Z'),
};
}
);
const subscribeEvents = vi.fn(
(currentSessionId: string, callback: (event: DiskSyncEvent) => void) => {
let set = listeners.get(currentSessionId);
if (!set) {
set = new Set();
listeners.set(currentSessionId, set);
}
set.add(callback);
return () => {
set?.delete(callback);
};
}
);
const apis: DiskSyncApis = {
startSession,
stopSession,
applyLocalUpdate,
subscribeEvents,
};
function emit(event: DiskSyncEvent) {
const callbacks = listeners.get(sessionId);
for (const callback of callbacks ?? []) {
callback(event);
}
}
function createStorage() {
return new DiskDocStorage({
flavour: 'local',
type: 'workspace',
id: 'workspace-test',
syncFolder: '/tmp/sync',
});
}
beforeEach(() => {
bindDiskSyncApis(apis);
listeners.clear();
vi.clearAllMocks();
});
afterEach(() => {
listeners.clear();
});
it('starts and stops disk session with connection lifecycle', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
expect(startSession).toHaveBeenCalledWith(sessionId, {
workspaceId: 'workspace-test',
syncFolder: '/tmp/sync',
});
storage.connection.disconnect();
await vi.waitFor(() => {
expect(stopSession).toHaveBeenCalledWith(sessionId);
});
});
it('forwards local updates and emits doc update events', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
const seen: Array<{ docId: string; origin?: string }> = [];
const unsubscribe = storage.subscribeDocUpdate((update, origin) => {
seen.push({ docId: update.docId, origin });
});
const bin = createUpdate('local');
await storage.pushDocUpdate({ docId: 'doc-local', bin }, 'origin:local');
expect(applyLocalUpdate).toHaveBeenCalledWith(
sessionId,
expect.objectContaining({
docId: 'doc-local',
}),
'origin:local'
);
expect(seen).toEqual([{ docId: 'doc-local', origin: 'origin:local' }]);
const snapshot = await storage.getDoc('doc-local');
expect(snapshot?.docId).toBe('doc-local');
expect(snapshot?.timestamp.toISOString()).toBe('2026-01-02T00:00:00.000Z');
unsubscribe();
storage.connection.disconnect();
});
it('applies remote events into local snapshots and handles delete events', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
emit({
type: 'doc-update',
update: {
docId: 'doc-remote',
bin: createUpdate('remote'),
timestamp: new Date('2026-01-03T00:00:00.000Z'),
},
});
await vi.waitFor(async () => {
const snapshot = await storage.getDoc('doc-remote');
expect(snapshot?.docId).toBe('doc-remote');
});
const timestamps = await storage.getDocTimestamps();
expect(timestamps['doc-remote']?.toISOString()).toBe(
'2026-01-03T00:00:00.000Z'
);
emit({
type: 'doc-delete',
docId: 'doc-remote',
timestamp: new Date('2026-01-03T00:00:01.000Z'),
});
await vi.waitFor(async () => {
expect(await storage.getDoc('doc-remote')).toBeNull();
});
storage.connection.disconnect();
});
it('serializes concurrent remote doc-update merges for the same doc', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
const originalMergeUpdates = (
storage as unknown as {
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
}
).mergeUpdates.bind(storage);
let mergeCall = 0;
vi.spyOn(
storage as unknown as {
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
},
'mergeUpdates'
).mockImplementation(async updates => {
mergeCall += 1;
// Force two in-flight merge operations to overlap and complete out-of-order.
if (mergeCall === 1) {
await new Promise(resolve => setTimeout(resolve, 20));
}
return originalMergeUpdates(updates);
});
emit({
type: 'doc-update',
update: {
docId: 'doc-race',
bin: createMapUpdate({ first: '1' }),
timestamp: new Date('2026-01-03T00:00:00.000Z'),
},
});
emit({
type: 'doc-update',
update: {
docId: 'doc-race',
bin: createMapUpdate({ second: '2' }),
timestamp: new Date('2026-01-03T00:00:00.001Z'),
},
});
await vi.waitFor(async () => {
const snapshot = await storage.getDoc('doc-race');
expect(snapshot).not.toBeNull();
expect(snapshot!.timestamp.toISOString()).toBe(
'2026-01-03T00:00:00.001Z'
);
const doc = new YDoc();
applyUpdate(doc, snapshot!.bin);
expect(doc.getMap('test').toJSON()).toEqual({
first: '1',
second: '2',
});
});
storage.connection.disconnect();
});
it('does not block follow-up updates when snapshot merge fails once', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
const originalMergeUpdates = (
storage as unknown as {
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
}
).mergeUpdates.bind(storage);
let mergeCall = 0;
vi.spyOn(
storage as unknown as {
mergeUpdates: (updates: Uint8Array[]) => Promise<Uint8Array>;
},
'mergeUpdates'
).mockImplementation(async updates => {
mergeCall += 1;
if (mergeCall === 1) {
throw new Error('merge failed once');
}
return originalMergeUpdates(updates);
});
await expect(
storage.pushDocUpdate({
docId: 'doc-merge-fallback',
bin: createMapUpdate({ a: '1' }),
})
).resolves.toEqual({
docId: 'doc-merge-fallback',
timestamp: new Date('2026-01-02T00:00:00.000Z'),
});
// This update triggers the mocked merge failure, but should still resolve.
await expect(
storage.pushDocUpdate({
docId: 'doc-merge-fallback',
bin: createMapUpdate({ b: '2' }),
})
).resolves.toEqual({
docId: 'doc-merge-fallback',
timestamp: new Date('2026-01-02T00:00:00.000Z'),
});
// Follow-up update should continue to work without requiring reconnect/reload.
await expect(
storage.pushDocUpdate({
docId: 'doc-merge-fallback',
bin: createMapUpdate({ c: '3' }),
})
).resolves.toEqual({
docId: 'doc-merge-fallback',
timestamp: new Date('2026-01-02T00:00:00.000Z'),
});
const snapshot = await storage.getDoc('doc-merge-fallback');
expect(snapshot).not.toBeNull();
const doc = new YDoc();
applyUpdate(doc, snapshot!.bin);
const data = doc.getMap('test').toJSON();
expect(data).toMatchObject({
b: '2',
c: '3',
});
storage.connection.disconnect();
});
it('accepts remote doc-update bins as number[] (from native binding)', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
const original = createUpdate('remote-array');
const bin = Array.from(original) as unknown as Uint8Array;
emit({
type: 'doc-update',
update: {
docId: 'doc-remote-array',
bin,
timestamp: new Date('2026-01-03T00:00:00.000Z'),
},
});
await vi.waitFor(async () => {
const snapshot = await storage.getDoc('doc-remote-array');
expect(snapshot).not.toBeNull();
const doc = new YDoc();
applyUpdate(doc, snapshot!.bin);
expect(doc.getText('content').toString()).toBe('remote-array');
});
storage.connection.disconnect();
});
it('throws when applyLocalUpdate returns invalid timestamp', async () => {
applyLocalUpdate.mockResolvedValueOnce({
docId: 'doc-invalid-clock',
timestamp: new Date('invalid'),
});
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
await expect(
storage.pushDocUpdate({
docId: 'doc-invalid-clock',
bin: createUpdate('invalid'),
})
).rejects.toThrow('[disk] invalid timestamp');
storage.connection.disconnect();
});
it('skips remote doc-update with invalid timestamp', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
emit({
type: 'doc-update',
update: {
docId: 'doc-invalid-remote-clock',
bin: createUpdate('remote-invalid'),
timestamp: new Date('invalid') as unknown as Date,
},
});
await vi.waitFor(async () => {
expect(await storage.getDoc('doc-invalid-remote-clock')).toBeNull();
});
storage.connection.disconnect();
});
it('discovers doc ids from root meta and emits connect-driving updates once', async () => {
const storage = createStorage();
storage.connection.connect();
await storage.connection.waitForConnected();
const seen: Array<{ docId: string; origin?: string; size: number }> = [];
const unsubscribe = storage.subscribeDocUpdate((update, origin) => {
seen.push({
docId: update.docId,
origin,
size: update.bin.byteLength,
});
});
const rootUpdate = createRootMetaUpdate(['doc-a', 'doc-b']);
await storage.pushDocUpdate(
{
docId: 'workspace-test',
bin: rootUpdate,
},
'origin:root'
);
await vi.waitFor(() => {
const discovered = seen.filter(
item => item.origin === 'disk:root-meta-discovery'
);
expect(discovered).toHaveLength(2);
});
const discoveredDocIds = seen
.filter(item => item.origin === 'disk:root-meta-discovery')
.map(item => item.docId)
.sort();
expect(discoveredDocIds).toEqual(['doc-a', 'doc-b']);
expect(
seen
.filter(item => item.origin === 'disk:root-meta-discovery')
.every(item => item.size === 0)
).toBe(true);
await storage.pushDocUpdate(
{
docId: 'workspace-test',
bin: rootUpdate,
},
'origin:root'
);
const discoveryCountAfterSecondPush = seen.filter(
item => item.origin === 'disk:root-meta-discovery'
).length;
expect(discoveryCountAfterSecondPush).toBe(2);
unsubscribe();
storage.connection.disconnect();
});
});
@@ -0,0 +1,291 @@
import { applyUpdate, Doc as YDoc } from 'yjs';
import {
type DocClock,
type DocClocks,
type DocRecord,
DocStorageBase,
type DocUpdate,
} from '../../storage';
import { type SpaceType } from '../../utils/universal-id';
import { DiskSyncConnection, type DiskSyncEvent } from './api';
export interface DiskDocStorageOptions {
readonly flavour: string;
readonly type: SpaceType;
readonly id: string;
readonly syncFolder: string;
}
export class DiskDocStorage extends DocStorageBase<DiskDocStorageOptions> {
static readonly identifier = 'DiskDocStorage';
readonly connection: DiskSyncConnection;
private readonly snapshots = new Map<string, DocRecord>();
private readonly pendingUpdates = new Map<string, DocRecord[]>();
private readonly discoveredRootDocs = new Set<string>();
constructor(options: DiskDocStorageOptions) {
super(options);
this.connection = new DiskSyncConnection(options, this.handleDiskEvent);
}
override async pushDocUpdate(update: DocUpdate, origin?: string) {
const { timestamp } = await this.connection.apis.applyLocalUpdate(
update,
origin
);
const clock = normalizeDate(timestamp);
const next: DocRecord = {
docId: update.docId,
bin: update.bin,
timestamp: clock,
editor: update.editor,
};
await this.applySnapshotUpdate(next, origin);
return { docId: update.docId, timestamp: clock };
}
override async getDocTimestamp(docId: string): Promise<DocClock | null> {
const snapshot = this.snapshots.get(docId);
if (!snapshot) {
return null;
}
return {
docId,
timestamp: snapshot.timestamp,
};
}
override async getDocTimestamps(after?: Date): Promise<DocClocks> {
const timestamps: DocClocks = {};
for (const [docId, snapshot] of this.snapshots.entries()) {
if (after && snapshot.timestamp.getTime() <= after.getTime()) {
continue;
}
timestamps[docId] = snapshot.timestamp;
}
return timestamps;
}
override async deleteDoc(docId: string): Promise<void> {
this.snapshots.delete(docId);
this.pendingUpdates.delete(docId);
}
protected override async getDocSnapshot(docId: string) {
return this.snapshots.get(docId) ?? null;
}
protected override async setDocSnapshot(
snapshot: DocRecord
): Promise<boolean> {
const existing = this.snapshots.get(snapshot.docId);
if (
existing &&
existing.timestamp.getTime() > snapshot.timestamp.getTime()
) {
return false;
}
this.snapshots.set(snapshot.docId, snapshot);
return true;
}
protected override async getDocUpdates(docId: string): Promise<DocRecord[]> {
return this.pendingUpdates.get(docId) ?? [];
}
protected override async markUpdatesMerged(
docId: string,
updates: DocRecord[]
): Promise<number> {
if (updates.length) {
this.pendingUpdates.delete(docId);
}
return updates.length;
}
private readonly handleDiskEvent = (event: DiskSyncEvent) => {
switch (event.type) {
case 'doc-update': {
let timestamp: Date;
try {
timestamp = normalizeDate(event.update.timestamp);
} catch (error) {
console.warn(
'[disk] invalid doc-update timestamp, skip event',
error
);
return;
}
let bin: Uint8Array;
try {
bin = normalizeBin(event.update.bin);
} catch (error) {
console.warn('[disk] invalid doc-update bin, skip event', error);
return;
}
const update: DocRecord = {
docId: event.update.docId,
bin,
timestamp,
editor: event.update.editor,
};
void this.applySnapshotUpdate(update, event.origin).catch(error => {
console.warn(
'[disk] failed to apply remote doc-update, skip event',
error
);
});
return;
}
case 'doc-delete': {
this.snapshots.delete(event.docId);
this.pendingUpdates.delete(event.docId);
return;
}
case 'error': {
console.warn('[disk] session error', event.message);
return;
}
default: {
return;
}
}
};
private async applySnapshotUpdate(update: DocRecord, origin?: string) {
await using _lock = await this.lockDocForUpdate(update.docId);
try {
await this.mergeIntoSnapshot(update);
} catch (error) {
// Snapshot cache is best-effort. A merge failure must not block upstream sync
// forever (it can otherwise require a full app reload to recover).
console.warn(
'[disk] snapshot merge failed, reset in-memory snapshot cache',
error
);
this.snapshots.set(update.docId, update);
}
this.emit('update', update, origin);
if (update.docId === this.spaceId) {
this.emitRootMetaDiscoveryUpdates();
}
}
private async mergeIntoSnapshot(update: DocRecord) {
const current = this.snapshots.get(update.docId);
if (!current) {
this.snapshots.set(update.docId, update);
return;
}
const merged = await this.mergeUpdates([current.bin, update.bin]);
this.snapshots.set(update.docId, {
...update,
bin: merged,
timestamp:
current.timestamp.getTime() > update.timestamp.getTime()
? current.timestamp
: update.timestamp,
editor: update.editor ?? current.editor,
});
}
private emitRootMetaDiscoveryUpdates() {
const rootSnapshot = this.snapshots.get(this.spaceId);
if (!rootSnapshot) {
return;
}
const docIds = extractRootMetaDocIds(rootSnapshot.bin);
// These discovery events are only meant to "introduce" doc ids to the sync
// peer, so it can connect/pull/push them. They should NOT be treated as a
// remote clock; otherwise switching sync folders (remote empty) can be
// incorrectly seen as "remote newer than local" and skip the initial push.
const discoveryTimestamp = new Date(0);
for (const docId of docIds) {
if (docId === this.spaceId || this.discoveredRootDocs.has(docId)) {
continue;
}
this.discoveredRootDocs.add(docId);
this.emit(
'update',
{
docId,
bin: new Uint8Array(),
timestamp: discoveryTimestamp,
},
'disk:root-meta-discovery'
);
}
}
}
function normalizeDate(date: Date | string | number): Date {
const normalized = date instanceof Date ? date : new Date(date);
if (Number.isNaN(normalized.getTime())) {
throw new Error(`[disk] invalid timestamp: ${String(date)}`);
}
return normalized;
}
function extractRootMetaDocIds(rootBin: Uint8Array): string[] {
const doc = new YDoc();
try {
applyUpdate(doc, rootBin);
} catch {
return [];
}
const meta = doc.getMap<unknown>('meta');
const pages = meta.get('pages');
const pagesJson =
typeof pages === 'object' &&
pages !== null &&
'toJSON' in pages &&
typeof pages.toJSON === 'function'
? pages.toJSON()
: pages;
if (!Array.isArray(pagesJson)) {
return [];
}
const docIds: string[] = [];
for (const page of pagesJson) {
if (!page || typeof page !== 'object') {
continue;
}
const id = (page as { id?: unknown }).id;
if (typeof id === 'string' && id.length > 0) {
docIds.push(id);
}
}
return docIds;
}
function normalizeBin(bin: unknown): Uint8Array {
// Native NAPI binding may send `number[]` for `Vec<u8>` fields.
if (bin instanceof Uint8Array) {
return bin;
}
if (Array.isArray(bin)) {
return Uint8Array.from(bin);
}
// Some transports may serialize Buffer as `{ type: 'Buffer', data: number[] }`.
if (
bin &&
typeof bin === 'object' &&
'data' in bin &&
Array.isArray((bin as { data?: unknown }).data)
) {
return Uint8Array.from((bin as { data: number[] }).data);
}
throw new Error(
`[disk] invalid update bin type: ${Object.prototype.toString.call(bin)}`
);
}
@@ -0,0 +1,7 @@
import type { StorageConstructor } from '..';
import { DiskDocStorage } from './doc';
export * from './api';
export * from './doc';
export const diskStorages = [DiskDocStorage] satisfies StorageConstructor[];
@@ -0,0 +1,378 @@
import 'fake-indexeddb/auto';
import { expect, test, vi } from 'vitest';
import { Doc as YDoc, encodeStateAsUpdate } from 'yjs';
import { expectYjsEqual } from '../../__tests__/utils';
import { SpaceStorage } from '../../storage';
import { Sync } from '../../sync';
import { universalId } from '../../utils/universal-id';
import { IndexedDBDocStorage, IndexedDBDocSyncStorage } from '../idb';
import { bindDiskSyncApis, type DiskSyncApis, type DiskSyncEvent } from './api';
import { DiskDocStorage } from './doc';
test('sync local <-> disk remote updates through DocSyncPeer', async () => {
const workspaceId = 'ws-disk-integration';
const sessionId = universalId({
peer: 'local',
type: 'workspace',
id: workspaceId,
});
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
const remoteDocs = new Map<string, { timestamp: Date; bin: Uint8Array }>();
const apis: DiskSyncApis = {
startSession: async currentSessionId => {
if (!listeners.has(currentSessionId)) {
listeners.set(currentSessionId, new Set());
}
},
stopSession: async currentSessionId => {
listeners.delete(currentSessionId);
},
applyLocalUpdate: async (currentSessionId, update) => {
const timestamp = new Date();
remoteDocs.set(update.docId, { timestamp, bin: update.bin });
for (const callback of listeners.get(currentSessionId) ?? []) {
callback({
type: 'doc-update',
update: {
docId: update.docId,
bin: update.bin,
timestamp,
},
origin: 'sync:disk-mock',
});
}
return {
docId: update.docId,
timestamp,
};
},
subscribeEvents: (currentSessionId, callback) => {
let set = listeners.get(currentSessionId);
if (!set) {
set = new Set();
listeners.set(currentSessionId, set);
}
set.add(callback);
return () => {
set?.delete(callback);
};
},
};
bindDiskSyncApis(apis);
const localDoc = new IndexedDBDocStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
});
const localDocSync = new IndexedDBDocSyncStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
});
const remoteDoc = new DiskDocStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
syncFolder: '/tmp/disk-sync',
});
const local = new SpaceStorage({
doc: localDoc,
docSync: localDocSync,
});
const remote = new SpaceStorage({
doc: remoteDoc,
});
local.connect();
remote.connect();
await local.waitForConnected();
await remote.waitForConnected();
const sync = new Sync({
local,
remotes: {
disk: remote,
},
});
sync.start();
const localSource = new YDoc();
localSource.getMap('test').set('origin', 'local');
await localDoc.pushDocUpdate({
docId: 'doc-local',
bin: encodeStateAsUpdate(localSource),
});
await vi.waitFor(() => {
expect(remoteDocs.has('doc-local')).toBe(true);
});
const remoteSource = new YDoc();
remoteSource.getMap('test').set('origin', 'remote');
remoteSource.getMap('test').set('synced', 'yes');
const remoteUpdate = encodeStateAsUpdate(remoteSource);
const remoteTimestamp = new Date('2026-01-05T00:00:00.000Z');
for (const callback of listeners.get(sessionId) ?? []) {
callback({
type: 'doc-update',
update: {
docId: 'doc-remote',
bin: remoteUpdate,
timestamp: remoteTimestamp,
},
});
}
await vi.waitFor(async () => {
const doc = await localDoc.getDoc('doc-remote');
expect(doc).not.toBeNull();
expectYjsEqual(doc!.bin, {
test: {
origin: 'remote',
synced: 'yes',
},
});
});
sync.stop();
// Intentionally keep IndexedDB connections open in tests. Disconnecting can
// abort in-flight IDB transactions in fake-indexeddb and surface as unhandled
// rejections, which makes Vitest fail the run.
remote.disconnect();
});
test('forces initial push when disk has stale pushed clocks but remote is empty', async () => {
const workspaceId = 'ws-disk-stale-push';
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
const remoteDocs = new Map<string, { timestamp: Date; bin: Uint8Array }>();
const apis: DiskSyncApis = {
startSession: async currentSessionId => {
if (!listeners.has(currentSessionId)) {
listeners.set(currentSessionId, new Set());
}
},
stopSession: async currentSessionId => {
listeners.delete(currentSessionId);
},
applyLocalUpdate: async (currentSessionId, update) => {
const timestamp = new Date();
remoteDocs.set(update.docId, { timestamp, bin: update.bin });
for (const callback of listeners.get(currentSessionId) ?? []) {
callback({
type: 'doc-update',
update: {
docId: update.docId,
bin: update.bin,
timestamp,
},
origin: 'sync:disk-mock',
});
}
return {
docId: update.docId,
timestamp,
};
},
subscribeEvents: (currentSessionId, callback) => {
let set = listeners.get(currentSessionId);
if (!set) {
set = new Set();
listeners.set(currentSessionId, set);
}
set.add(callback);
return () => {
set?.delete(callback);
};
},
};
bindDiskSyncApis(apis);
const localDoc = new IndexedDBDocStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
});
const localDocSync = new IndexedDBDocSyncStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
});
const remoteDoc = new DiskDocStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
syncFolder: '/tmp/disk-sync-stale',
});
const local = new SpaceStorage({
doc: localDoc,
docSync: localDocSync,
});
const remote = new SpaceStorage({
doc: remoteDoc,
});
local.connect();
remote.connect();
await local.waitForConnected();
await remote.waitForConnected();
const source = new YDoc();
source.getMap('test').set('value', 'local');
await localDoc.pushDocUpdate({
docId: 'doc-local-stale',
bin: encodeStateAsUpdate(source),
});
await localDocSync.setPeerPushedClock('disk', {
docId: 'doc-local-stale',
timestamp: new Date('2099-01-01T00:00:00.000Z'),
});
const sync = new Sync({
local,
remotes: {
disk: remote,
},
});
sync.start();
await vi.waitFor(() => {
expect(remoteDocs.has('doc-local-stale')).toBe(true);
});
sync.stop();
remote.disconnect();
});
test('root-meta discovery must not block pushing page docs when switching disk folders', async () => {
const workspaceId = 'ws-disk-discovery-nonblocking';
const pageDocId = 'page-doc-1';
const listeners = new Map<string, Set<(event: DiskSyncEvent) => void>>();
const remoteDocs = new Map<string, { timestamp: Date; bin: Uint8Array }>();
const apis: DiskSyncApis = {
startSession: async currentSessionId => {
if (!listeners.has(currentSessionId)) {
listeners.set(currentSessionId, new Set());
}
},
stopSession: async currentSessionId => {
listeners.delete(currentSessionId);
},
applyLocalUpdate: async (currentSessionId, update) => {
const timestamp = new Date();
remoteDocs.set(update.docId, { timestamp, bin: update.bin });
for (const callback of listeners.get(currentSessionId) ?? []) {
callback({
type: 'doc-update',
update: {
docId: update.docId,
bin: update.bin,
timestamp,
},
origin: 'sync:disk-mock',
});
}
return {
docId: update.docId,
timestamp,
};
},
subscribeEvents: (currentSessionId, callback) => {
let set = listeners.get(currentSessionId);
if (!set) {
set = new Set();
listeners.set(currentSessionId, set);
}
set.add(callback);
return () => {
set?.delete(callback);
};
},
};
bindDiskSyncApis(apis);
const localDoc = new IndexedDBDocStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
});
const localDocSync = new IndexedDBDocSyncStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
});
const remoteDoc = new DiskDocStorage({
id: workspaceId,
flavour: 'local',
type: 'workspace',
syncFolder: '/tmp/disk-sync-discovery',
});
const local = new SpaceStorage({
doc: localDoc,
docSync: localDocSync,
});
const remote = new SpaceStorage({
doc: remoteDoc,
});
local.connect();
remote.connect();
await local.waitForConnected();
await remote.waitForConnected();
// Seed local root meta so disk can discover the page doc id from it.
const root = new YDoc();
const meta = root.getMap('meta');
meta.set('pages', [{ id: pageDocId }]);
await localDoc.pushDocUpdate({
docId: workspaceId,
bin: encodeStateAsUpdate(root),
});
// Seed the page doc itself.
const page = new YDoc();
page.getMap('test').set('value', 'local');
const { timestamp: pageClock } = await localDoc.pushDocUpdate({
docId: pageDocId,
bin: encodeStateAsUpdate(page),
});
// Simulate "already pushed" clocks from a previous disk folder.
await localDocSync.setPeerPushedClock('disk', {
docId: pageDocId,
timestamp: pageClock,
});
const sync = new Sync({
local,
remotes: {
disk: remote,
},
});
// Match workspace engine behavior: sync root doc first.
sync.doc.addPriority(workspaceId, 100);
sync.start();
await vi.waitFor(() => {
expect(remoteDocs.has(pageDocId)).toBe(true);
});
sync.stop();
remote.disconnect();
});
@@ -1,6 +1,7 @@
import type { Storage } from '../storage';
import type { broadcastChannelStorages } from './broadcast-channel';
import type { cloudStorages } from './cloud';
import type { diskStorages } from './disk';
import type { idbStorages } from './idb';
import type { idbV1Storages } from './idb/v1';
import type { sqliteStorages } from './sqlite';
@@ -15,6 +16,7 @@ type Storages =
| typeof cloudStorages
| typeof idbV1Storages
| typeof idbStorages
| typeof diskStorages
| typeof sqliteStorages
| typeof sqliteV1Storages
| typeof broadcastChannelStorages;
+16 -7
View File
@@ -241,13 +241,16 @@ export class DocSyncPeer {
(await this.syncMetadata.getPeerPushedClock(this.peerId, docId))
?.timestamp ?? null;
const clock = await this.local.getDocTimestamp(docId);
const remoteClock = this.status.remoteClocks.get(docId) ?? null;
throwIfAborted(signal);
if (
!this.remote.isReadonly &&
clock &&
(pushedClock === null ||
pushedClock.getTime() < clock.timestamp.getTime())
pushedClock.getTime() < clock.timestamp.getTime() ||
remoteClock === null ||
remoteClock.getTime() < clock.timestamp.getTime())
) {
await this.jobs.pullAndPush(docId, signal);
} else {
@@ -255,7 +258,6 @@ export class DocSyncPeer {
const pulled =
(await this.syncMetadata.getPeerPulledRemoteClock(this.peerId, docId))
?.timestamp ?? null;
const remoteClock = this.status.remoteClocks.get(docId);
if (
remoteClock &&
(pulled === null || pulled.getTime() < remoteClock.getTime())
@@ -676,10 +678,12 @@ export class DocSyncPeer {
this.actions.addDoc(docId);
}
const forceFullRemoteClockRefresh = this.peerId === 'disk';
// get cached clocks from metadata
const cachedClocks = await this.syncMetadata.getPeerRemoteClocks(
this.peerId
);
const cachedClocks = forceFullRemoteClockRefresh
? {}
: await this.syncMetadata.getPeerRemoteClocks(this.peerId);
this.status.remoteClocks.clear();
throwIfAborted(signal);
for (const [id, v] of Object.entries(cachedClocks)) {
@@ -687,9 +691,14 @@ export class DocSyncPeer {
}
this.statusUpdatedSubject$.next(true);
// get new clocks from server
const maxClockValue = this.status.remoteClocks.max;
// get clocks from server
const maxClockValue = forceFullRemoteClockRefresh
? undefined
: this.status.remoteClocks.max;
const newClocks = await this.remote.getDocTimestamps(maxClockValue);
if (forceFullRemoteClockRefresh) {
this.status.remoteClocks.clear();
}
for (const [id, v] of Object.entries(newClocks)) {
this.status.remoteClocks.set(id, v);
}
+112 -26
View File
@@ -1,4 +1,5 @@
import { OpConsumer } from '@toeverything/infra/op';
import { isEqual } from 'lodash-es';
import { Observable } from 'rxjs';
import { type StorageConstructor } from '../impls';
@@ -13,8 +14,9 @@ import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops';
export type { WorkerManagerOps };
class StoreConsumer {
private readonly storages: PeerStorageOptions<SpaceStorage>;
private readonly sync: Sync;
private storages: PeerStorageOptions<SpaceStorage> | null = null;
private sync: Sync | null = null;
private initOptions: StoreInitOptions;
get ensureLocal() {
if (!this.storages) {
@@ -70,20 +72,29 @@ class StoreConsumer {
private readonly availableStorageImplementations: StorageConstructor[],
init: StoreInitOptions
) {
this.initOptions = init;
this.initWithOptions(init);
}
private createStorage(opt: any): any {
if (opt === undefined) {
return undefined;
}
const Storage = this.availableStorageImplementations.find(
impl => impl.identifier === opt.name
);
if (!Storage) {
throw new Error(`Storage implementation ${opt.name} not found`);
}
return new Storage(opt.opts as any);
}
private initWithOptions(init: StoreInitOptions) {
this.storages = {
local: new SpaceStorage(
Object.fromEntries(
Object.entries(init.local).map(([type, opt]) => {
if (opt === undefined) {
return [type, undefined];
}
const Storage = this.availableStorageImplementations.find(
impl => impl.identifier === opt.name
);
if (!Storage) {
throw new Error(`Storage implementation ${opt.name} not found`);
}
return [type, new Storage(opt.opts as any)];
return [type, this.createStorage(opt)];
})
)
),
@@ -94,18 +105,7 @@ class StoreConsumer {
new SpaceStorage(
Object.fromEntries(
Object.entries(opts).map(([type, opt]) => {
if (opt === undefined) {
return [type, undefined];
}
const Storage = this.availableStorageImplementations.find(
impl => impl.identifier === opt.name
);
if (!Storage) {
throw new Error(
`Storage implementation ${opt.name} not found`
);
}
return [type, new Storage(opt.opts as any)];
return [type, this.createStorage(opt)];
})
)
),
@@ -125,6 +125,69 @@ class StoreConsumer {
this.registerHandlers(consumer);
}
async reconfigure(init: StoreInitOptions) {
if (isEqual(this.initOptions, init)) {
return;
}
// If local storage config changes, fall back to full teardown/rebuild.
// (Remote-only changes are expected, like enabling folder sync.)
if (
!this.storages ||
!this.sync ||
!isEqual(this.initOptions.local, init.local)
) {
await this.destroy();
this.initOptions = init;
this.initWithOptions(init);
return;
}
// Remote-only change: rebuild sync graph and remote storages in-place so
// existing OpConsumers keep working.
const prevInit = this.initOptions;
const storages = this.storages;
this.sync.stop();
// Destroy removed or changed remote peers.
for (const [peerId, prevPeerOpts] of Object.entries(prevInit.remotes)) {
const nextPeerOpts = init.remotes[peerId];
const changed = !nextPeerOpts || !isEqual(prevPeerOpts, nextPeerOpts);
if (!changed) {
continue;
}
const remote = storages.remotes[peerId];
if (remote) {
delete storages.remotes[peerId];
remote.disconnect();
await remote.destroy();
}
}
// Create added or changed remote peers.
for (const [peerId, nextPeerOpts] of Object.entries(init.remotes)) {
const prevPeerOpts = prevInit.remotes[peerId];
const changed = !prevPeerOpts || !isEqual(prevPeerOpts, nextPeerOpts);
if (!changed) {
continue;
}
const remote = new SpaceStorage(
Object.fromEntries(
Object.entries(nextPeerOpts).map(([type, opt]) => {
return [type, this.createStorage(opt)];
})
)
);
storages.remotes[peerId] = remote;
remote.connect();
}
this.sync = new Sync(storages);
this.sync.start();
this.initOptions = init;
}
async destroy() {
this.sync?.stop();
this.storages?.local.disconnect();
@@ -133,6 +196,9 @@ class StoreConsumer {
remote.disconnect();
await remote.destroy();
}
this.sync = null;
this.storages = null;
}
private readonly ENABLE_BATTERY_SAVE_MODE_DELAY = 1000;
@@ -337,7 +403,12 @@ export class StoreManagerConsumer {
private readonly storeDisposers = new Map<string, () => void>();
private readonly storePool = new Map<
string,
{ store: StoreConsumer; refCount: number }
{
store: StoreConsumer;
refCount: number;
options: StoreInitOptions;
reconfiguring?: Promise<void>;
}
>();
private readonly telemetry = new TelemetryManager();
@@ -360,7 +431,22 @@ export class StoreManagerConsumer {
this.availableStorageImplementations,
options
);
storeRef = { store, refCount: 0 };
storeRef = { store, refCount: 0, options };
} else if (!isEqual(storeRef.options, options)) {
const currentStoreRef = storeRef;
// Options can change across renderer reloads (or when features like
// folder sync are enabled). Reconfigure the shared store in-place
// so existing consumers keep working with the latest remotes.
currentStoreRef.reconfiguring = (
currentStoreRef.reconfiguring ?? Promise.resolve()
)
.then(async () => {
await currentStoreRef.store.reconfigure(options);
currentStoreRef.options = options;
})
.catch(error => {
console.error('failed to reconfigure store', key, error);
});
}
storeRef.refCount++;