feat: store local data to local db (#2037)

packages/i18n/src/resources/en.json

@@ -8,7 +8,7 @@
   "It takes up more space on your device": {
     "": "It takes up more space on your device."
   },
-  "Export AFFiNE backup file": "Export AFFiNE backup file (coming soon)",
+  "Export AFFiNE backup file": "Export AFFiNE backup file",
   "Saved then enable AFFiNE Cloud": "All changes are saved locally, click to enable AFFiNE Cloud.",
   "Not now": "Not now",
   "Export Description": "You can export the entire Workspace data for backup, and the exported data can be re-imported.",

packages/i18n/src/resources/zh-Hans.json

@@ -178,7 +178,7 @@
   "Owner": "所有者",
   "Published to Web": "公开到互联网",
   "Data sync mode": "数据同步模式",
-  "Export AFFiNE backup file": "导出 AFFiNE 备份文件（即将到来）",
+  "Export AFFiNE backup file": "导出 AFFiNE 备份文件",
   "Export Description": "您可以导出整个工作区数据进行备份，导出的数据可以重新被导入。",
   "It takes up little space on your device": {
     "": "此操作会在你的设备上占用少许空间。"

packages/workspace/src/atom.ts

@@ -1,9 +1,10 @@
 import { atomWithSyncStorage } from '@affine/jotai';
-import type { WorkspaceFlavour } from '@affine/workspace/type';
 import type { EditorContainer } from '@blocksuite/editor';
 import { atom, createStore } from 'jotai';
 import { atomWithStorage, createJSONStorage } from 'jotai/utils';
 
+import type { WorkspaceFlavour } from './type';
+
 export type RootWorkspaceMetadata = {
   id: string;
   flavour: WorkspaceFlavour;
@@ -45,5 +46,13 @@ export const rootCurrentEditorAtom = atom<Readonly<EditorContainer> | null>(
 );
 //#endregion
 
+const getStorage = () => createJSONStorage(() => localStorage);
+
+export const getStoredWorkspaceMeta = () => {
+  const storage = getStorage();
+  const data = storage.getItem('jotai-workspaces') as RootWorkspaceMetadata[];
+  return data;
+};
+
 // global store
 export const rootStore = createStore();

packages/workspace/src/blob/sqlite-blob-storage.ts

@@ -0,0 +1,25 @@
+import type { BlobStorage } from '@blocksuite/store';
+
+export const createSQLiteStorage = (workspaceId: string): BlobStorage => {
+  return {
+    crud: {
+      get: async (key: string) => {
+        const buffer = await window.apis.db.getBlob(workspaceId, key);
+        return buffer ? new Blob([buffer]) : null;
+      },
+      set: async (key: string, value: Blob) => {
+        return window.apis.db.addBlob(
+          workspaceId,
+          key,
+          new Uint8Array(await value.arrayBuffer())
+        );
+      },
+      delete: async (key: string) => {
+        return window.apis.db.deleteBlob(workspaceId, key);
+      },
+      list: async () => {
+        return window.apis.db.getPersistedBlobs(workspaceId);
+      },
+    },
+  };
+};

packages/workspace/src/local/crud.ts

@@ -85,18 +85,32 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.LOCAL> = {
     }
     data.splice(idx, 1);
     storage.setItem(kStoreKey, [...data]);
+    // flywire
+    if (window.apis && environment.isDesktop) {
+      await window.apis.workspace.delete(workspace.id);
+    }
   },
   list: async () => {
     logger.debug('list');
     const storage = getStorage();
-    !Array.isArray(storage.getItem(kStoreKey)) &&
-      storage.setItem(kStoreKey, []);
-    return (
-      await Promise.all(
-        (storage.getItem(kStoreKey) as z.infer<typeof schema>).map(id =>
-          CRUD.get(id)
-        )
-      )
+    let allWorkspaceIDs: string[] = Array.isArray(storage.getItem(kStoreKey))
+      ? (storage.getItem(kStoreKey) as z.infer<typeof schema>)
+      : [];
+
+    // workspaces in desktop
+    if (window.apis && environment.isDesktop) {
+      const desktopIds = await window.apis.workspace.list();
+      // the ids maybe a subset of the local storage
+      const moreWorkspaces = desktopIds.filter(
+        id => !allWorkspaceIDs.includes(id)
+      );
+      allWorkspaceIDs = [...allWorkspaceIDs, ...moreWorkspaces];
+      storage.setItem(kStoreKey, allWorkspaceIDs);
+    }
+    const workspaces = (
+      await Promise.all(allWorkspaceIDs.map(id => CRUD.get(id)))
     ).filter(item => item !== null) as LocalWorkspace[];
+
+    return workspaces;
   },
 };

packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts

@@ -0,0 +1,71 @@
+import type { SQLiteProvider } from '@affine/workspace/type';
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import type { Y as YType } from '@blocksuite/store';
+import { uuidv4, Workspace } from '@blocksuite/store';
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+
+import { createSQLiteProvider } from '../index';
+
+const Y = Workspace.Y;
+
+let id: string;
+let workspace: Workspace;
+let provider: SQLiteProvider;
+
+let offlineYdoc: YType.Doc;
+
+vi.stubGlobal('window', {
+  apis: {
+    db: {
+      getDoc: async (id: string) => {
+        return Y.encodeStateAsUpdate(offlineYdoc);
+      },
+      applyDocUpdate: async (id: string, update: Uint8Array) => {
+        Y.applyUpdate(offlineYdoc, update, 'sqlite');
+      },
+      getPersistedBlobs: async (id: string) => {
+        return [];
+      },
+    } satisfies Partial<typeof window.apis.db>,
+  },
+});
+
+vi.stubGlobal('environment', {
+  isDesktop: true,
+});
+
+beforeEach(() => {
+  id = uuidv4();
+  workspace = new Workspace({
+    id,
+    isSSR: true,
+  });
+  workspace.register(AffineSchemas).register(__unstableSchemas);
+  provider = createSQLiteProvider(workspace);
+  offlineYdoc = new Y.Doc();
+  offlineYdoc.getText('text').insert(0, '');
+});
+
+describe('SQLite provider', () => {
+  test('connect', async () => {
+    // on connect, the updates from sqlite should be sync'ed to the existing ydoc
+    // and ydoc should be sync'ed back to sqlite
+    // Workspace.Y.applyUpdate(workspace.doc);
+    workspace.doc.getText('text').insert(0, 'mem-hello');
+
+    expect(offlineYdoc.getText('text').toString()).toBe('');
+
+    await provider.connect();
+
+    expect(offlineYdoc.getText('text').toString()).toBe('mem-hello');
+    expect(workspace.doc.getText('text').toString()).toBe('mem-hello');
+
+    workspace.doc.getText('text').insert(0, 'world');
+
+    // check if the data are sync'ed
+    expect(offlineYdoc.getText('text').toString()).toBe('worldmem-hello');
+  });
+
+  // todo: test disconnect
+  // todo: test blob sync
+});

packages/workspace/src/providers/index.ts

@@ -4,15 +4,13 @@ import {
   getLoginStorage,
   storageChangeSlot,
 } from '@affine/workspace/affine/login';
-import type { Provider } from '@affine/workspace/type';
+import type { Provider, SQLiteProvider } from '@affine/workspace/type';
 import type {
   AffineWebSocketProvider,
   LocalIndexedDBProvider,
 } from '@affine/workspace/type';
-import type {
-  Disposable,
-  Workspace as BlockSuiteWorkspace,
-} from '@blocksuite/store';
+import type { BlobManager, Disposable } from '@blocksuite/store';
+import { Workspace as BlockSuiteWorkspace } from '@blocksuite/store';
 import { assertExists } from '@blocksuite/store';
 import {
   createIndexedDBProvider as create,
@@ -20,7 +18,9 @@ import {
 } from '@toeverything/y-indexeddb';
 
 import { createBroadCastChannelProvider } from './broad-cast-channel';
-import { localProviderLogger } from './logger';
+import { localProviderLogger as logger } from './logger';
+
+const Y = BlockSuiteWorkspace.Y;
 
 const createAffineWebSocketProvider = (
   blockSuiteWorkspace: BlockSuiteWorkspace
@@ -54,12 +54,12 @@ const createAffineWebSocketProvider = (
           connect: false,
         }
       );
-      localProviderLogger.info('connect', webSocketProvider.url);
+      logger.info('connect', webSocketProvider.url);
       webSocketProvider.connect();
     },
     disconnect: () => {
       assertExists(webSocketProvider);
-      localProviderLogger.info('disconnect', webSocketProvider.url);
+      logger.info('disconnect', webSocketProvider.url);
       webSocketProvider.destroy();
       webSocketProvider = null;
       dispose?.dispose();
@@ -119,10 +119,7 @@ const createIndexedDBProvider = (
       // todo: cleanup data
     },
     connect: () => {
-      localProviderLogger.info(
-        'connect indexeddb provider',
-        blockSuiteWorkspace.id
-      );
+      logger.info('connect indexeddb provider', blockSuiteWorkspace.id);
       indexeddbProvider.connect();
       indexeddbProvider.whenSynced
         .then(() => {
@@ -139,20 +136,94 @@ const createIndexedDBProvider = (
     },
     disconnect: () => {
       assertExists(indexeddbProvider);
-      localProviderLogger.info(
-        'disconnect indexeddb provider',
-        blockSuiteWorkspace.id
-      );
+      logger.info('disconnect indexeddb provider', blockSuiteWorkspace.id);
       indexeddbProvider.disconnect();
       callbacks.ready = false;
     },
   };
 };
 
+const createSQLiteProvider = (
+  blockSuiteWorkspace: BlockSuiteWorkspace
+): SQLiteProvider => {
+  const sqliteOrigin = Symbol('sqlite-provider-origin');
+  // make sure it is being used in Electron with APIs
+  assertExists(environment.isDesktop && window.apis);
+
+  function handleUpdate(update: Uint8Array, origin: unknown) {
+    if (origin === sqliteOrigin) {
+      return;
+    }
+    window.apis.db.applyDocUpdate(blockSuiteWorkspace.id, update);
+  }
+
+  async function syncBlobIntoSQLite(bs: BlobManager) {
+    const persistedKeys = await window.apis.db.getPersistedBlobs(
+      blockSuiteWorkspace.id
+    );
+
+    const allKeys = await bs.list();
+    const keysToPersist = allKeys.filter(k => !persistedKeys.includes(k));
+
+    logger.info('persisting blobs', keysToPersist, 'to sqlite');
+    keysToPersist.forEach(async k => {
+      const blob = await bs.get(k);
+      if (!blob) {
+        logger.warn('blob url not found', k);
+        return;
+      }
+      window.apis.db.addBlob(
+        blockSuiteWorkspace.id,
+        k,
+        new Uint8Array(await blob.arrayBuffer())
+      );
+    });
+  }
+
+  const provider = {
+    flavour: 'sqlite',
+    background: true,
+    cleanup: () => {
+      throw new Error('Method not implemented.');
+    },
+    connect: async () => {
+      logger.info('connecting sqlite provider', blockSuiteWorkspace.id);
+      const updates = await window.apis.db.getDoc(blockSuiteWorkspace.id);
+
+      if (updates) {
+        Y.applyUpdate(blockSuiteWorkspace.doc, updates, sqliteOrigin);
+      }
+
+      const mergeUpdates = Y.encodeStateAsUpdate(blockSuiteWorkspace.doc);
+
+      // also apply updates to sqlite
+      window.apis.db.applyDocUpdate(blockSuiteWorkspace.id, mergeUpdates);
+
+      blockSuiteWorkspace.doc.on('update', handleUpdate);
+
+      const bs = blockSuiteWorkspace.blobs;
+
+      if (bs) {
+        // this can be non-blocking
+        syncBlobIntoSQLite(bs);
+      }
+
+      // blockSuiteWorkspace.doc.on('destroy', ...);
+      logger.info('connecting sqlite done', blockSuiteWorkspace.id);
+    },
+    disconnect: () => {
+      // todo: not implemented
+    },
+  } satisfies SQLiteProvider;
+
+  return provider;
+};
+
 export {
   createAffineWebSocketProvider,
   createBroadCastChannelProvider,
   createIndexedDBProvider,
+  createSQLiteProvider,
 };
 
 export const createLocalProviders = (
@@ -163,6 +234,7 @@ export const createLocalProviders = (
       config.enableBroadCastChannelProvider &&
         createBroadCastChannelProvider(blockSuiteWorkspace),
       createIndexedDBProvider(blockSuiteWorkspace),
+      environment.isDesktop && createSQLiteProvider(blockSuiteWorkspace),
     ] as any[]
   ).filter(v => Boolean(v));
 };

packages/workspace/src/type.ts

@@ -35,6 +35,10 @@ export interface LocalIndexedDBProvider extends BackgroundProvider {
   whenSynced: Promise<void>;
 }
 
+export interface SQLiteProvider extends BaseProvider {
+  flavour: 'sqlite';
+}
+
 export interface AffineWebSocketProvider extends BaseProvider {
   flavour: 'affine-websocket';
 }

packages/workspace/src/utils.ts

@@ -1,9 +1,10 @@
 import type { createWorkspaceApis } from '@affine/workspace/affine/api';
 import { createAffineBlobStorage } from '@affine/workspace/blob';
 import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
-import type { Generator } from '@blocksuite/store';
+import type { Generator, StoreOptions } from '@blocksuite/store';
 import { createIndexeddbStorage, Workspace } from '@blocksuite/store';
 
+import { createSQLiteStorage } from './blob/sqlite-blob-storage';
 import { WorkspaceFlavour } from './type';
 
 const hashMap = new Map<string, Workspace>();
@@ -48,15 +49,26 @@ export function createEmptyBlockSuiteWorkspace(
     return hashMap.get(cacheKey) as Workspace;
   }
   const idGenerator = config?.idGenerator;
+
+  const blobStorages: StoreOptions['blobStorages'] = [];
+
+  if (flavour === WorkspaceFlavour.AFFINE) {
+    blobStorages.push(id =>
+      createAffineBlobStorage(id, config!.workspaceApis!)
+    );
+  } else {
+    if (typeof window !== 'undefined') {
+      blobStorages.push(createIndexeddbStorage);
+      if (environment.isDesktop) {
+        blobStorages.push(createSQLiteStorage);
+      }
+    }
+  }
+
   const workspace = new Workspace({
     id,
     isSSR: typeof window === 'undefined',
-    blobStorages:
-      flavour === WorkspaceFlavour.AFFINE
-        ? [id => createAffineBlobStorage(id, config!.workspaceApis!)]
-        : typeof window === 'undefined'
-        ? []
-        : [createIndexeddbStorage],
+    blobStorages: blobStorages,
     idGenerator,
   })
     .register(AffineSchemas)