feat(infra): new doc sync engine (#6205)

https://github.com/toeverything/AFFiNE/blob/eyhn/feat/new-sync/packages/common/infra/src/workspace/engine/doc/README.md

packages/frontend/workspace-impl/src/local/__tests__/engine.spec.ts

@@ -1,195 +0,0 @@
-import 'fake-indexeddb/auto';
-
-import { AffineSchemas } from '@blocksuite/blocks/schemas';
-import { DocCollection, Schema } from '@blocksuite/store';
-import { SyncEngine, SyncEngineStep, SyncPeerStep } from '@toeverything/infra';
-import { beforeEach, describe, expect, test, vi } from 'vitest';
-import { Doc } from 'yjs';
-
-import { IndexedDBSyncStorage } from '..';
-import { createTestStorage } from './test-storage';
-
-const schema = new Schema();
-
-schema.register(AffineSchemas);
-
-beforeEach(() => {
-  vi.useFakeTimers({ toFake: ['requestIdleCallback'] });
-});
-
-describe('SyncEngine', () => {
-  test('basic - indexeddb', async () => {
-    let prev: any;
-    {
-      const docCollection = new DocCollection({
-        id: 'test - syncengine - indexeddb',
-
-        schema,
-      });
-
-      const syncEngine = new SyncEngine(
-        docCollection.doc,
-        new IndexedDBSyncStorage(docCollection.doc.guid),
-        [
-          new IndexedDBSyncStorage(docCollection.doc.guid + '1'),
-          new IndexedDBSyncStorage(docCollection.doc.guid + '2'),
-        ]
-      );
-      syncEngine.start();
-
-      const page = docCollection.createDoc({
-        id: 'page0',
-      });
-      page.load();
-      const pageBlockId = page.addBlock(
-        'affine:page' as keyof BlockSuite.BlockModels,
-        {
-          title: new page.Text(''),
-        }
-      );
-      page.addBlock(
-        'affine:surface' as keyof BlockSuite.BlockModels,
-        {},
-        pageBlockId
-      );
-      const frameId = page.addBlock(
-        'affine:note' as keyof BlockSuite.BlockModels,
-        {},
-        pageBlockId
-      );
-      page.addBlock(
-        'affine:paragraph' as keyof BlockSuite.BlockModels,
-        {},
-        frameId
-      );
-      await syncEngine.waitForSynced();
-      syncEngine.forceStop();
-      prev = docCollection.doc.toJSON();
-    }
-
-    {
-      const docCollection = new DocCollection({
-        id: 'test - syncengine - indexeddb',
-
-        schema,
-      });
-      const syncEngine = new SyncEngine(
-        docCollection.doc,
-        new IndexedDBSyncStorage(docCollection.doc.guid),
-        []
-      );
-      syncEngine.start();
-      await syncEngine.waitForSynced();
-      expect(docCollection.doc.toJSON()).toEqual({
-        ...prev,
-      });
-      syncEngine.forceStop();
-    }
-
-    {
-      const docCollection = new DocCollection({
-        id: 'test - syncengine - indexeddb',
-
-        schema,
-      });
-      const syncEngine = new SyncEngine(
-        docCollection.doc,
-        new IndexedDBSyncStorage(docCollection.doc.guid + '1'),
-        []
-      );
-      syncEngine.start();
-      await syncEngine.waitForSynced();
-      expect(docCollection.doc.toJSON()).toEqual({
-        ...prev,
-      });
-      syncEngine.forceStop();
-    }
-
-    {
-      const docCollection = new DocCollection({
-        id: 'test - syncengine - indexeddb',
-
-        schema,
-      });
-      const syncEngine = new SyncEngine(
-        docCollection.doc,
-        new IndexedDBSyncStorage(docCollection.doc.guid + '2'),
-        []
-      );
-      syncEngine.start();
-      await syncEngine.waitForSynced();
-      expect(docCollection.doc.toJSON()).toEqual({
-        ...prev,
-      });
-      syncEngine.forceStop();
-    }
-  });
-
-  test('status', async () => {
-    const ydoc = new Doc({ guid: 'test - syncengine - status' });
-
-    const localStorage = createTestStorage(new IndexedDBSyncStorage(ydoc.guid));
-    const remoteStorage = createTestStorage(
-      new IndexedDBSyncStorage(ydoc.guid + '1')
-    );
-
-    localStorage.pausePull();
-    localStorage.pausePush();
-    remoteStorage.pausePull();
-    remoteStorage.pausePush();
-
-    const syncEngine = new SyncEngine(ydoc, localStorage, [remoteStorage]);
-    expect(syncEngine.status.step).toEqual(SyncEngineStep.Stopped);
-
-    syncEngine.start();
-
-    await vi.waitFor(() => {
-      expect(syncEngine.status.step).toEqual(SyncEngineStep.Syncing);
-      expect(syncEngine.status.local?.step).toEqual(
-        SyncPeerStep.LoadingRootDoc
-      );
-    });
-
-    localStorage.resumePull();
-
-    await vi.waitFor(() => {
-      expect(syncEngine.status.step).toEqual(SyncEngineStep.Syncing);
-      expect(syncEngine.status.local?.step).toEqual(SyncPeerStep.Synced);
-      expect(syncEngine.status.remotes[0]?.step).toEqual(
-        SyncPeerStep.LoadingRootDoc
-      );
-    });
-
-    remoteStorage.resumePull();
-
-    await vi.waitFor(() => {
-      expect(syncEngine.status.step).toEqual(SyncEngineStep.Synced);
-      expect(syncEngine.status.remotes[0]?.step).toEqual(SyncPeerStep.Synced);
-      expect(syncEngine.status.local?.step).toEqual(SyncPeerStep.Synced);
-    });
-
-    ydoc.getArray('test').insert(0, [1, 2, 3]);
-
-    await vi.waitFor(() => {
-      expect(syncEngine.status.step).toEqual(SyncEngineStep.Syncing);
-      expect(syncEngine.status.local?.step).toEqual(SyncPeerStep.Syncing);
-      expect(syncEngine.status.remotes[0]?.step).toEqual(SyncPeerStep.Syncing);
-    });
-
-    localStorage.resumePush();
-
-    await vi.waitFor(() => {
-      expect(syncEngine.status.step).toEqual(SyncEngineStep.Syncing);
-      expect(syncEngine.status.local?.step).toEqual(SyncPeerStep.Synced);
-      expect(syncEngine.status.remotes[0]?.step).toEqual(SyncPeerStep.Syncing);
-    });
-
-    remoteStorage.resumePush();
-
-    await vi.waitFor(() => {
-      expect(syncEngine.status.step).toEqual(SyncEngineStep.Synced);
-      expect(syncEngine.status.local?.step).toEqual(SyncPeerStep.Synced);
-      expect(syncEngine.status.remotes[0]?.step).toEqual(SyncPeerStep.Synced);
-    });
-  });
-});

packages/frontend/workspace-impl/src/local/__tests__/peer.spec.ts

@@ -1,109 +0,0 @@
-import 'fake-indexeddb/auto';
-
-import { AffineSchemas } from '@blocksuite/blocks/schemas';
-import { DocCollection, Schema } from '@blocksuite/store';
-import { SyncPeer, SyncPeerStep } from '@toeverything/infra';
-import { beforeEach, describe, expect, test, vi } from 'vitest';
-
-import { IndexedDBSyncStorage } from '..';
-
-const schema = new Schema();
-
-schema.register(AffineSchemas);
-
-beforeEach(() => {
-  vi.useFakeTimers({ toFake: ['requestIdleCallback'] });
-});
-
-describe('SyncPeer', () => {
-  test('basic - indexeddb', async () => {
-    let prev: any;
-    {
-      const docCollection = new DocCollection({
-        id: 'test - syncpeer - indexeddb',
-
-        schema,
-      });
-
-      const syncPeer = new SyncPeer(
-        docCollection.doc,
-        new IndexedDBSyncStorage(docCollection.doc.guid)
-      );
-      await syncPeer.waitForLoaded();
-
-      const page = docCollection.createDoc({
-        id: 'page0',
-      });
-      page.load();
-      const pageBlockId = page.addBlock(
-        'affine:page' as keyof BlockSuite.BlockModels,
-        {
-          title: new page.Text(''),
-        }
-      );
-      page.addBlock(
-        'affine:surface' as keyof BlockSuite.BlockModels,
-        {},
-        pageBlockId
-      );
-      const frameId = page.addBlock(
-        'affine:note' as keyof BlockSuite.BlockModels,
-        {},
-        pageBlockId
-      );
-      page.addBlock(
-        'affine:paragraph' as keyof BlockSuite.BlockModels,
-        {},
-        frameId
-      );
-      await syncPeer.waitForSynced();
-      syncPeer.stop();
-      prev = docCollection.doc.toJSON();
-    }
-
-    {
-      const docCollection = new DocCollection({
-        id: 'test - syncpeer - indexeddb',
-
-        schema,
-      });
-      const syncPeer = new SyncPeer(
-        docCollection.doc,
-        new IndexedDBSyncStorage(docCollection.doc.guid)
-      );
-      await syncPeer.waitForSynced();
-      expect(docCollection.doc.toJSON()).toEqual({
-        ...prev,
-      });
-      syncPeer.stop();
-    }
-  });
-
-  test('status', async () => {
-    const docCollection = new DocCollection({
-      id: 'test - syncpeer - status',
-
-      schema,
-    });
-
-    const syncPeer = new SyncPeer(
-      docCollection.doc,
-      new IndexedDBSyncStorage(docCollection.doc.guid)
-    );
-    expect(syncPeer.status.step).toBe(SyncPeerStep.LoadingRootDoc);
-    await syncPeer.waitForSynced();
-    expect(syncPeer.status.step).toBe(SyncPeerStep.Synced);
-
-    const page = docCollection.createDoc({
-      id: 'page0',
-    });
-    expect(syncPeer.status.step).toBe(SyncPeerStep.LoadingSubDoc);
-    page.load();
-    await syncPeer.waitForSynced();
-    page.addBlock('affine:page' as keyof BlockSuite.BlockModels, {
-      title: new page.Text(''),
-    });
-    expect(syncPeer.status.step).toBe(SyncPeerStep.Syncing);
-    syncPeer.stop();
-  });
-});

packages/frontend/workspace-impl/src/local/__tests__/test-storage.ts

@@ -1,42 +0,0 @@
-import type { SyncStorage } from '@toeverything/infra';
-
-export function createTestStorage(origin: SyncStorage) {
-  const controler = {
-    pausedPull: Promise.resolve(),
-    resumePull: () => {},
-    pausedPush: Promise.resolve(),
-    resumePush: () => {},
-  };
-
-  return {
-    name: `${origin.name}(testing)`,
-    pull(docId: string, state: Uint8Array) {
-      return controler.pausedPull.then(() => origin.pull(docId, state));
-    },
-    push(docId: string, data: Uint8Array) {
-      return controler.pausedPush.then(() => origin.push(docId, data));
-    },
-    subscribe(
-      cb: (docId: string, data: Uint8Array) => void,
-      disconnect: (reason: string) => void
-    ) {
-      return origin.subscribe(cb, disconnect);
-    },
-    pausePull() {
-      controler.pausedPull = new Promise(resolve => {
-        controler.resumePull = resolve;
-      });
-    },
-    resumePull() {
-      controler.resumePull?.();
-    },
-    pausePush() {
-      controler.pausedPush = new Promise(resolve => {
-        controler.resumePush = resolve;
-      });
-    },
-    resumePush() {
-      controler.resumePush?.();
-    },
-  };
-}

packages/frontend/workspace-impl/src/local/doc-broadcast-channel.ts

@@ -0,0 +1,60 @@
+import type { DocEvent, DocEventBus } from '@toeverything/infra';
+
+type LegacyChannelMessage = {
+  type: 'db-updated';
+  payload: {
+    docId: string;
+    update: Uint8Array;
+  };
+  __from_new_doc_engine?: boolean;
+};
+
+export class BroadcastChannelDocEventBus implements DocEventBus {
+  legacyChannel = new BroadcastChannel('indexeddb:' + this.workspaceId);
+  senderChannel = new BroadcastChannel('doc:' + this.workspaceId);
+  constructor(private readonly workspaceId: string) {
+    this.legacyChannel.addEventListener(
+      'message',
+      (event: MessageEvent<LegacyChannelMessage>) => {
+        if (event.data.__from_new_doc_engine) {
+          return;
+        }
+        if (event.data.type === 'db-updated') {
+          this.emit({
+            type: 'LegacyClientUpdateCommitted',
+            docId: event.data.payload.docId,
+            update: event.data.payload.update,
+          });
+        }
+      }
+    );
+  }
+  emit(event: DocEvent): void {
+    if (
+      event.type === 'ClientUpdateCommitted' ||
+      event.type === 'ServerUpdateCommitted'
+    ) {
+      this.legacyChannel.postMessage({
+        type: 'db-updated',
+        payload: {
+          docId: event.docId,
+          update: event.update,
+        },
+        __from_new_doc_engine: true,
+      } satisfies LegacyChannelMessage);
+    }
+    this.senderChannel.postMessage(event);
+  }
+
+  on(cb: (event: DocEvent) => void): () => void {
+    const listener = (event: MessageEvent<DocEvent>) => {
+      cb(event.data);
+    };
+    const channel = new BroadcastChannel('doc:' + this.workspaceId);
+    channel.addEventListener('message', listener);
+    return () => {
+      channel.removeEventListener('message', listener);
+      channel.close();
+    };
+  }
+}

packages/frontend/workspace-impl/src/local/doc-indexeddb.ts

@@ -0,0 +1,246 @@
+import type { ByteKV, ByteKVBehavior, DocStorage } from '@toeverything/infra';
+import {
+  type DBSchema,
+  type IDBPDatabase,
+  type IDBPObjectStore,
+  openDB,
+} from 'idb';
+import { mergeUpdates } from 'yjs';
+
+import { BroadcastChannelDocEventBus } from './doc-broadcast-channel';
+
+function isEmptyUpdate(binary: Uint8Array) {
+  return (
+    binary.byteLength === 0 ||
+    (binary.byteLength === 2 && binary[0] === 0 && binary[1] === 0)
+  );
+}
+
+export class IndexedDBDocStorage implements DocStorage {
+  constructor(private readonly workspaceId: string) {}
+  eventBus = new BroadcastChannelDocEventBus(this.workspaceId);
+  readonly doc = new Doc();
+  readonly syncMetadata = new KV(`${this.workspaceId}:sync-metadata`);
+  readonly serverClock = new KV(`${this.workspaceId}:server-clock`);
+}
+
+interface DocDBSchema extends DBSchema {
+  workspace: {
+    key: string;
+    value: {
+      id: string;
+      updates: {
+        timestamp: number;
+        update: Uint8Array;
+      }[];
+    };
+  };
+}
+
+type DocType = DocStorage['doc'];
+class Doc implements DocType {
+  dbName = 'affine-local';
+  dbPromise: Promise<IDBPDatabase<DocDBSchema>> | null = null;
+  dbVersion = 1;
+
+  constructor() {}
+
+  upgradeDB(db: IDBPDatabase<DocDBSchema>) {
+    db.createObjectStore('workspace', { keyPath: 'id' });
+  }
+
+  getDb() {
+    if (this.dbPromise === null) {
+      this.dbPromise = openDB<DocDBSchema>(this.dbName, this.dbVersion, {
+        upgrade: db => this.upgradeDB(db),
+      });
+    }
+    return this.dbPromise;
+  }
+
+  async get(docId: string): Promise<Uint8Array | null> {
+    const db = await this.getDb();
+    const store = db
+      .transaction('workspace', 'readonly')
+      .objectStore('workspace');
+    const data = await store.get(docId);
+
+    if (!data) {
+      return null;
+    }
+
+    const updates = data.updates
+      .map(({ update }) => update)
+      .filter(update => !isEmptyUpdate(update));
+    const update = updates.length > 0 ? mergeUpdates(updates) : null;
+
+    return update;
+  }
+
+  async set(docId: string, data: Uint8Array) {
+    const db = await this.getDb();
+    const store = db
+      .transaction('workspace', 'readwrite')
+      .objectStore('workspace');
+
+    const rows = [{ timestamp: Date.now(), update: data }];
+    await store.put({
+      id: docId,
+      updates: rows,
+    });
+  }
+
+  async keys() {
+    const db = await this.getDb();
+    const store = db
+      .transaction('workspace', 'readonly')
+      .objectStore('workspace');
+
+    return store.getAllKeys();
+  }
+
+  clear(): void | Promise<void> {
+    return;
+  }
+
+  del(_key: string): void | Promise<void> {
+    return;
+  }
+
+  async transaction<T>(
+    cb: (transaction: ByteKVBehavior) => Promise<T>
+  ): Promise<T> {
+    const db = await this.getDb();
+    const store = db
+      .transaction('workspace', 'readwrite')
+      .objectStore('workspace');
+    return await cb({
+      async get(docId) {
+        const data = await store.get(docId);
+
+        if (!data) {
+          return null;
+        }
+
+        const { updates } = data;
+        const update = mergeUpdates(updates.map(({ update }) => update));
+
+        return update;
+      },
+      keys() {
+        return store.getAllKeys();
+      },
+      async set(docId, data) {
+        const rows = [{ timestamp: Date.now(), update: data }];
+        await store.put({
+          id: docId,
+          updates: rows,
+        });
+      },
+      async clear() {
+        return await store.clear();
+      },
+      async del(key) {
+        return store.delete(key);
+      },
+    });
+  }
+}
+
+interface KvDBSchema extends DBSchema {
+  kv: {
+    key: string;
+    value: { key: string; val: Uint8Array };
+  };
+}
+
+class KV implements ByteKV {
+  constructor(private readonly dbName: string) {}
+
+  dbPromise: Promise<IDBPDatabase<KvDBSchema>> | null = null;
+  dbVersion = 1;
+
+  upgradeDB(db: IDBPDatabase<KvDBSchema>) {
+    db.createObjectStore('kv', { keyPath: 'key' });
+  }
+
+  getDb() {
+    if (this.dbPromise === null) {
+      this.dbPromise = openDB<KvDBSchema>(this.dbName, this.dbVersion, {
+        upgrade: db => this.upgradeDB(db),
+      });
+    }
+    return this.dbPromise;
+  }
+
+  async transaction<T>(
+    cb: (transaction: ByteKVBehavior) => Promise<T>
+  ): Promise<T> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+
+    const behavior = new KVBehavior(store);
+    return await cb(behavior);
+  }
+
+  async get(key: string): Promise<Uint8Array | null> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readonly').objectStore('kv');
+    return new KVBehavior(store).get(key);
+  }
+  async set(key: string, value: Uint8Array): Promise<void> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).set(key, value);
+  }
+  async keys(): Promise<string[]> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).keys();
+  }
+  async clear() {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).clear();
+  }
+  async del(key: string) {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).del(key);
+  }
+}
+
+class KVBehavior implements ByteKVBehavior {
+  constructor(
+    private readonly store: IDBPObjectStore<KvDBSchema, ['kv'], 'kv', any>
+  ) {}
+  async get(key: string): Promise<Uint8Array | null> {
+    const value = await this.store.get(key);
+    return value?.val ?? null;
+  }
+  async set(key: string, value: Uint8Array): Promise<void> {
+    if (this.store.put === undefined) {
+      throw new Error('Cannot set in a readonly transaction');
+    }
+    await this.store.put({
+      key: key,
+      val: value,
+    });
+  }
+  async keys(): Promise<string[]> {
+    return await this.store.getAllKeys();
+  }
+  async del(key: string) {
+    if (this.store.delete === undefined) {
+      throw new Error('Cannot set in a readonly transaction');
+    }
+    return await this.store.delete(key);
+  }
+
+  async clear() {
+    if (this.store.clear === undefined) {
+      throw new Error('Cannot set in a readonly transaction');
+    }
+    return await this.store.clear();
+  }
+}

packages/frontend/workspace-impl/src/local/doc-sqlite.ts

@@ -0,0 +1,186 @@
+import { apis } from '@affine/electron-api';
+import {
+  AsyncLock,
+  type ByteKV,
+  type ByteKVBehavior,
+  type DocStorage,
+  MemoryDocEventBus,
+} from '@toeverything/infra';
+import {
+  type DBSchema,
+  type IDBPDatabase,
+  type IDBPObjectStore,
+  openDB,
+} from 'idb';
+
+export class SqliteDocStorage implements DocStorage {
+  constructor(private readonly workspaceId: string) {}
+  eventBus = new MemoryDocEventBus();
+  readonly doc = new Doc(this.workspaceId);
+  readonly syncMetadata = new KV(`${this.workspaceId}:sync-metadata`);
+  readonly serverClock = new KV(`${this.workspaceId}:server-clock`);
+}
+
+type DocType = DocStorage['doc'];
+
+class Doc implements DocType {
+  lock = new AsyncLock();
+  constructor(private readonly workspaceId: string) {
+    if (!apis?.db) {
+      throw new Error('sqlite datasource is not available');
+    }
+  }
+
+  async transaction<T>(
+    cb: (transaction: ByteKVBehavior) => Promise<T>
+  ): Promise<T> {
+    using _lock = await this.lock.acquire();
+    return await cb(this);
+  }
+
+  keys(): string[] | Promise<string[]> {
+    return [];
+  }
+
+  async get(docId: string) {
+    if (!apis?.db) {
+      throw new Error('sqlite datasource is not available');
+    }
+    const update = await apis.db.getDocAsUpdates(
+      this.workspaceId,
+      this.workspaceId === docId ? undefined : docId
+    );
+
+    if (update) {
+      if (
+        update.byteLength === 0 ||
+        (update.byteLength === 2 && update[0] === 0 && update[1] === 0)
+      ) {
+        return null;
+      }
+
+      return update;
+    }
+
+    return null;
+  }
+
+  async set(docId: string, data: Uint8Array) {
+    if (!apis?.db) {
+      throw new Error('sqlite datasource is not available');
+    }
+    await apis.db.applyDocUpdate(
+      this.workspaceId,
+      data,
+      this.workspaceId === docId ? undefined : docId
+    );
+  }
+
+  clear(): void | Promise<void> {
+    return;
+  }
+
+  del(): void | Promise<void> {
+    return;
+  }
+}
+
+interface KvDBSchema extends DBSchema {
+  kv: {
+    key: string;
+    value: { key: string; val: Uint8Array };
+  };
+}
+
+class KV implements ByteKV {
+  constructor(private readonly dbName: string) {}
+
+  dbPromise: Promise<IDBPDatabase<KvDBSchema>> | null = null;
+  dbVersion = 1;
+
+  upgradeDB(db: IDBPDatabase<KvDBSchema>) {
+    db.createObjectStore('kv', { keyPath: 'key' });
+  }
+
+  getDb() {
+    if (this.dbPromise === null) {
+      this.dbPromise = openDB<KvDBSchema>(this.dbName, this.dbVersion, {
+        upgrade: db => this.upgradeDB(db),
+      });
+    }
+    return this.dbPromise;
+  }
+
+  async transaction<T>(
+    cb: (transaction: ByteKVBehavior) => Promise<T>
+  ): Promise<T> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+
+    const behavior = new KVBehavior(store);
+    return await cb(behavior);
+  }
+
+  async get(key: string): Promise<Uint8Array | null> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readonly').objectStore('kv');
+    return new KVBehavior(store).get(key);
+  }
+  async set(key: string, value: Uint8Array): Promise<void> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).set(key, value);
+  }
+  async keys(): Promise<string[]> {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).keys();
+  }
+  async clear() {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).clear();
+  }
+  async del(key: string) {
+    const db = await this.getDb();
+    const store = db.transaction('kv', 'readwrite').objectStore('kv');
+    return new KVBehavior(store).del(key);
+  }
+}
+
+class KVBehavior implements ByteKVBehavior {
+  constructor(
+    private readonly store: IDBPObjectStore<KvDBSchema, ['kv'], 'kv', any>
+  ) {}
+
+  async get(key: string): Promise<Uint8Array | null> {
+    const value = await this.store.get(key);
+    return value?.val ?? null;
+  }
+  async set(key: string, value: Uint8Array): Promise<void> {
+    if (this.store.put === undefined) {
+      throw new Error('Cannot set in a readonly transaction');
+    }
+    await this.store.put({
+      key: key,
+      val: value,
+    });
+  }
+  async keys(): Promise<string[]> {
+    return await this.store.getAllKeys();
+  }
+
+  async del(key: string) {
+    if (this.store.delete === undefined) {
+      throw new Error('Cannot set in a readonly transaction');
+    }
+    return await this.store.delete(key);
+  }
+
+  async clear() {
+    if (this.store.clear === undefined) {
+      throw new Error('Cannot set in a readonly transaction');
+    }
+    return await this.store.clear();
+  }
+}

packages/frontend/workspace-impl/src/local/index.ts

@@ -1,9 +1,3 @@
-export * from './awareness';
-export * from './blob-indexeddb';
-export * from './blob-sqlite';
-export * from './blob-static';
-export * from './consts';
+export { StaticBlobStorage } from './blob-static';
 export * from './list';
-export * from './sync-indexeddb';
-export * from './sync-sqlite';
 export * from './workspace-factory';

packages/frontend/workspace-impl/src/local/list.ts

@@ -18,8 +18,8 @@ import {
   LOCAL_WORKSPACE_CREATED_BROADCAST_CHANNEL_KEY,
   LOCAL_WORKSPACE_LOCAL_STORAGE_KEY,
 } from './consts';
-import { IndexedDBSyncStorage } from './sync-indexeddb';
-import { SQLiteSyncStorage } from './sync-sqlite';
+import { IndexedDBDocStorage } from './doc-indexeddb';
+import { SqliteDocStorage } from './doc-sqlite';
 
 export class LocalWorkspaceListProvider implements WorkspaceListProvider {
   name = WorkspaceFlavour.LOCAL;
@@ -62,9 +62,9 @@ export class LocalWorkspaceListProvider implements WorkspaceListProvider {
     const blobStorage = environment.isDesktop
       ? new SQLiteBlobStorage(id)
       : new IndexedDBBlobStorage(id);
-    const syncStorage = environment.isDesktop
-      ? new SQLiteSyncStorage(id)
-      : new IndexedDBSyncStorage(id);
+    const docStorage = environment.isDesktop
+      ? new SqliteDocStorage(id)
+      : new IndexedDBDocStorage(id);
 
     const workspace = new DocCollection({
       id: id,
@@ -76,9 +76,9 @@ export class LocalWorkspaceListProvider implements WorkspaceListProvider {
     await initial(workspace, blobStorage);
 
     // save workspace to local storage
-    await syncStorage.push(id, encodeStateAsUpdate(workspace.doc));
+    await docStorage.doc.set(id, encodeStateAsUpdate(workspace.doc));
     for (const subdocs of workspace.doc.getSubdocs()) {
-      await syncStorage.push(subdocs.guid, encodeStateAsUpdate(subdocs));
+      await docStorage.doc.set(subdocs.guid, encodeStateAsUpdate(subdocs));
     }
 
     // save workspace id to local storage
@@ -128,9 +128,9 @@ export class LocalWorkspaceListProvider implements WorkspaceListProvider {
   async getInformation(id: string): Promise<WorkspaceInfo | undefined> {
     // get information from root doc
     const storage = environment.isDesktop
-      ? new SQLiteSyncStorage(id)
-      : new IndexedDBSyncStorage(id);
-    const data = await storage.pull(id, new Uint8Array([]));
+      ? new SqliteDocStorage(id)
+      : new IndexedDBDocStorage(id);
+    const data = await storage.doc.get(id);
 
     if (!data) {
       return;
@@ -141,7 +141,7 @@ export class LocalWorkspaceListProvider implements WorkspaceListProvider {
       schema: globalBlockSuiteSchema,
     });
 
-    applyUpdate(bs.doc, data.data);
+    applyUpdate(bs.doc, data);
 
     return {
       name: bs.meta.name,

packages/frontend/workspace-impl/src/local/sync-indexeddb.ts

@@ -1,118 +0,0 @@
-import { mergeUpdates, type SyncStorage } from '@toeverything/infra';
-import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
-import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';
-
-export const dbVersion = 1;
-export const DEFAULT_DB_NAME = 'affine-local';
-
-type UpdateMessage = {
-  timestamp: number;
-  update: Uint8Array;
-};
-
-type WorkspacePersist = {
-  id: string;
-  updates: UpdateMessage[];
-};
-
-interface BlockSuiteBinaryDB extends DBSchema {
-  workspace: {
-    key: string;
-    value: WorkspacePersist;
-  };
-  milestone: {
-    key: string;
-    value: unknown;
-  };
-}
-
-export function upgradeDB(db: IDBPDatabase<BlockSuiteBinaryDB>) {
-  db.createObjectStore('workspace', { keyPath: 'id' });
-  db.createObjectStore('milestone', { keyPath: 'id' });
-}
-
-type ChannelMessage = {
-  type: 'db-updated';
-  payload: { docId: string; update: Uint8Array };
-};
-
-export class IndexedDBSyncStorage implements SyncStorage {
-  name = 'indexeddb';
-  dbName = DEFAULT_DB_NAME;
-  mergeCount = 1;
-  dbPromise: Promise<IDBPDatabase<BlockSuiteBinaryDB>> | null = null;
-  // indexeddb could be shared between tabs, so we use broadcast channel to notify other tabs
-  channel = new BroadcastChannel('indexeddb:' + this.workspaceId);
-
-  constructor(private readonly workspaceId: string) {}
-
-  getDb() {
-    if (this.dbPromise === null) {
-      this.dbPromise = openDB<BlockSuiteBinaryDB>(this.dbName, dbVersion, {
-        upgrade: upgradeDB,
-      });
-    }
-    return this.dbPromise;
-  }
-
-  async pull(
-    docId: string,
-    state: Uint8Array
-  ): Promise<{ data: Uint8Array; state?: Uint8Array | undefined } | null> {
-    const db = await this.getDb();
-    const store = db
-      .transaction('workspace', 'readonly')
-      .objectStore('workspace');
-    const data = await store.get(docId);
-
-    if (!data) {
-      return null;
-    }
-
-    const { updates } = data;
-    const update = mergeUpdates(updates.map(({ update }) => update));
-
-    const diff = state.length ? diffUpdate(update, state) : update;
-
-    return { data: diff, state: encodeStateVectorFromUpdate(update) };
-  }
-
-  async push(docId: string, data: Uint8Array): Promise<void> {
-    const db = await this.getDb();
-    const store = db
-      .transaction('workspace', 'readwrite')
-      .objectStore('workspace');
-
-    // TODO: maybe we do not need to get data every time
-    const { updates } = (await store.get(docId)) ?? { updates: [] };
-    let rows: UpdateMessage[] = [
-      ...updates,
-      { timestamp: Date.now(), update: data },
-    ];
-    if (this.mergeCount && rows.length >= this.mergeCount) {
-      const merged = mergeUpdates(rows.map(({ update }) => update));
-      rows = [{ timestamp: Date.now(), update: merged }];
-    }
-    await store.put({
-      id: docId,
-      updates: rows,
-    });
-    this.channel.postMessage({
-      type: 'db-updated',
-      payload: { docId, update: data },
-    } satisfies ChannelMessage);
-  }
-  async subscribe(cb: (docId: string, data: Uint8Array) => void) {
-    function onMessage(event: MessageEvent<ChannelMessage>) {
-      const { type, payload } = event.data;
-      if (type === 'db-updated') {
-        const { docId, update } = payload;
-        cb(docId, update);
-      }
-    }
-    this.channel.addEventListener('message', onMessage);
-    return () => {
-      this.channel.removeEventListener('message', onMessage);
-    };
-  }
-}

packages/frontend/workspace-impl/src/local/sync-sqlite.ts

@@ -1,53 +0,0 @@
-import { apis } from '@affine/electron-api';
-import { type SyncStorage } from '@toeverything/infra';
-import { encodeStateVectorFromUpdate } from 'yjs';
-
-export class SQLiteSyncStorage implements SyncStorage {
-  name = 'sqlite';
-  constructor(private readonly workspaceId: string) {
-    if (!apis?.db) {
-      throw new Error('sqlite datasource is not available');
-    }
-  }
-
-  async pull(docId: string, _state: Uint8Array) {
-    if (!apis?.db) {
-      throw new Error('sqlite datasource is not available');
-    }
-    const update = await apis.db.getDocAsUpdates(
-      this.workspaceId,
-      this.workspaceId === docId ? undefined : docId
-    );
-
-    if (update) {
-      if (
-        update.byteLength === 0 ||
-        (update.byteLength === 2 && update[0] === 0 && update[1] === 0)
-      ) {
-        return null;
-      }
-
-      return {
-        data: update,
-        state: encodeStateVectorFromUpdate(update),
-      };
-    }
-
-    return null;
-  }
-
-  async push(docId: string, data: Uint8Array) {
-    if (!apis?.db) {
-      throw new Error('sqlite datasource is not available');
-    }
-    return apis.db.applyDocUpdate(
-      this.workspaceId,
-      data,
-      this.workspaceId === docId ? undefined : docId
-    );
-  }
-
-  async subscribe() {
-    return () => {};
-  }
-}

packages/frontend/workspace-impl/src/local/workspace-factory.ts

@@ -2,8 +2,8 @@ import type { ServiceCollection, WorkspaceFactory } from '@toeverything/infra';
 import {
   AwarenessContext,
   AwarenessProvider,
+  DocStorageImpl,
   LocalBlobStorage,
-  LocalSyncStorage,
   RemoteBlobStorage,
   WorkspaceIdContext,
   WorkspaceScope,
@@ -13,8 +13,8 @@ import { BroadcastChannelAwarenessProvider } from './awareness';
 import { IndexedDBBlobStorage } from './blob-indexeddb';
 import { SQLiteBlobStorage } from './blob-sqlite';
 import { StaticBlobStorage } from './blob-static';
-import { IndexedDBSyncStorage } from './sync-indexeddb';
-import { SQLiteSyncStorage } from './sync-sqlite';
+import { IndexedDBDocStorage } from './doc-indexeddb';
+import { SqliteDocStorage } from './doc-sqlite';
 
 export class LocalWorkspaceFactory implements WorkspaceFactory {
   name = 'local';
@@ -23,12 +23,12 @@ export class LocalWorkspaceFactory implements WorkspaceFactory {
       services
         .scope(WorkspaceScope)
         .addImpl(LocalBlobStorage, SQLiteBlobStorage, [WorkspaceIdContext])
-        .addImpl(LocalSyncStorage, SQLiteSyncStorage, [WorkspaceIdContext]);
+        .addImpl(DocStorageImpl, SqliteDocStorage, [WorkspaceIdContext]);
     } else {
       services
         .scope(WorkspaceScope)
         .addImpl(LocalBlobStorage, IndexedDBBlobStorage, [WorkspaceIdContext])
-        .addImpl(LocalSyncStorage, IndexedDBSyncStorage, [WorkspaceIdContext]);
+        .addImpl(DocStorageImpl, IndexedDBDocStorage, [WorkspaceIdContext]);
     }
 
     services