refactor(infra): directory structure (#4615)

packages/frontend/electron/src/helper/db/__tests__/.gitignore

@@ -0,0 +1 @@
+tmp

packages/frontend/electron/src/helper/db/__tests__/ensure-db.spec.ts

@@ -0,0 +1,131 @@
+import path from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+
+import { removeWithRetry } from '@affine-test/kit/utils/utils';
+import { v4 } from 'uuid';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+
+const tmpDir = path.join(__dirname, 'tmp');
+const appDataPath = path.join(tmpDir, 'app-data');
+
+vi.doMock('../../main-rpc', () => ({
+  mainRPC: {
+    getPath: async () => appDataPath,
+  },
+}));
+
+const constructorStub = vi.fn();
+const destroyStub = vi.fn();
+destroyStub.mockReturnValue(Promise.resolve());
+
+function existProcess() {
+  process.emit('beforeExit', 0);
+}
+
+vi.doMock('../secondary-db', () => {
+  return {
+    SecondaryWorkspaceSQLiteDB: class {
+      constructor(...args: any[]) {
+        constructorStub(...args);
+      }
+
+      connectIfNeeded = () => Promise.resolve();
+
+      pull = () => Promise.resolve();
+
+      destroy = destroyStub;
+    },
+  };
+});
+
+beforeEach(() => {
+  vi.useFakeTimers({ shouldAdvanceTime: true });
+});
+
+afterEach(async () => {
+  existProcess();
+  await removeWithRetry(tmpDir);
+  vi.useRealTimers();
+});
+
+test('can get a valid WorkspaceSQLiteDB', async () => {
+  const { ensureSQLiteDB } = await import('../ensure-db');
+  const workspaceId = v4();
+  const db0 = await ensureSQLiteDB(workspaceId);
+  expect(db0).toBeDefined();
+  expect(db0.workspaceId).toBe(workspaceId);
+
+  const db1 = await ensureSQLiteDB(v4());
+  expect(db1).not.toBe(db0);
+  expect(db1.workspaceId).not.toBe(db0.workspaceId);
+
+  // ensure that the db is cached
+  expect(await ensureSQLiteDB(workspaceId)).toBe(db0);
+});
+
+test('db should be destroyed when app quits', async () => {
+  const { ensureSQLiteDB } = await import('../ensure-db');
+  const workspaceId = v4();
+  const db0 = await ensureSQLiteDB(workspaceId);
+  const db1 = await ensureSQLiteDB(v4());
+
+  expect(db0.db).not.toBeNull();
+  expect(db1.db).not.toBeNull();
+
+  existProcess();
+
+  // wait the async `db.destroy()` to be called
+  await setTimeout(100);
+
+  expect(db0.db).toBeNull();
+  expect(db1.db).toBeNull();
+});
+
+test('db should be removed in db$Map after destroyed', async () => {
+  const { ensureSQLiteDB, db$Map } = await import('../ensure-db');
+  const workspaceId = v4();
+  const db = await ensureSQLiteDB(workspaceId);
+  await db.destroy();
+  await setTimeout(100);
+  expect(db$Map.has(workspaceId)).toBe(false);
+});
+
+// we have removed secondary db feature
+test.skip('if db has a secondary db path, we should also poll that', async () => {
+  const { ensureSQLiteDB } = await import('../ensure-db');
+  const { storeWorkspaceMeta } = await import('../../workspace');
+  const workspaceId = v4();
+  await storeWorkspaceMeta(workspaceId, {
+    secondaryDBPath: path.join(tmpDir, 'secondary.db'),
+  });
+
+  const db = await ensureSQLiteDB(workspaceId);
+
+  await setTimeout(10);
+
+  expect(constructorStub).toBeCalledTimes(1);
+  expect(constructorStub).toBeCalledWith(path.join(tmpDir, 'secondary.db'), db);
+
+  // if secondary meta is changed
+  await storeWorkspaceMeta(workspaceId, {
+    secondaryDBPath: path.join(tmpDir, 'secondary2.db'),
+  });
+
+  // wait the async `db.destroy()` to be called
+  await setTimeout(100);
+  expect(constructorStub).toBeCalledTimes(2);
+  expect(destroyStub).toBeCalledTimes(1);
+
+  // if secondary meta is changed (but another workspace)
+  await storeWorkspaceMeta(v4(), {
+    secondaryDBPath: path.join(tmpDir, 'secondary3.db'),
+  });
+  await vi.advanceTimersByTimeAsync(1500);
+  expect(constructorStub).toBeCalledTimes(2);
+  expect(destroyStub).toBeCalledTimes(1);
+
+  // if primary is destroyed, secondary should also be destroyed
+  await db.destroy();
+  await setTimeout(100);
+  expect(destroyStub).toBeCalledTimes(2);
+});

packages/frontend/electron/src/helper/db/__tests__/migration.spec.ts

@@ -0,0 +1,67 @@
+import path from 'node:path';
+
+import { SqliteConnection } from '@affine/native';
+import { removeWithRetry } from '@affine-test/kit/utils/utils';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { applyUpdate, Doc as YDoc } from 'yjs';
+
+import { copyToTemp, migrateToSubdocAndReplaceDatabase } from '../migration';
+
+const tmpDir = path.join(__dirname, 'tmp');
+const testDBFilePath = path.resolve(__dirname, 'old-db.affine');
+
+const appDataPath = path.join(tmpDir, 'app-data');
+
+vi.mock('../../main-rpc', () => ({
+  mainRPC: {
+    getPath: async () => appDataPath,
+  },
+}));
+
+afterEach(async () => {
+  await removeWithRetry(tmpDir);
+});
+
+describe('migrateToSubdocAndReplaceDatabase', () => {
+  it('should migrate and replace the database', async () => {
+    const copiedDbFilePath = await copyToTemp(testDBFilePath);
+    await migrateToSubdocAndReplaceDatabase(copiedDbFilePath);
+
+    const db = new SqliteConnection(copiedDbFilePath);
+    await db.connect();
+
+    // check if db has two rows, one for root doc and one for subdoc
+    const rows = await db.getAllUpdates();
+    expect(rows.length).toBe(2);
+
+    const rootUpdate = rows.find(row => row.docId === undefined)!.data;
+    const subdocUpdate = rows.find(row => row.docId !== undefined)!.data;
+
+    expect(rootUpdate).toBeDefined();
+    expect(subdocUpdate).toBeDefined();
+
+    // apply updates
+    const rootDoc = new YDoc();
+    applyUpdate(rootDoc, rootUpdate);
+
+    // check if root doc has one subdoc
+    expect(rootDoc.subdocs.size).toBe(1);
+
+    // populates subdoc
+    applyUpdate(rootDoc.subdocs.values().next().value, subdocUpdate);
+
+    // check if root doc's meta is correct
+    const meta = rootDoc.getMap('meta').toJSON();
+    expect(meta.workspaceVersion).toBe(1);
+    expect(meta.name).toBe('hiw');
+    expect(meta.pages.length).toBe(1);
+    const pageMeta = meta.pages[0];
+    expect(pageMeta.title).toBe('Welcome to AFFiNEd');
+
+    // get the subdoc through id
+    const subDoc = rootDoc.getMap('spaces').get(pageMeta.id) as YDoc;
+    expect(subDoc).toEqual(rootDoc.subdocs.values().next().value);
+
+    await db.close();
+  });
+});

packages/frontend/electron/src/helper/db/__tests__/workspace-db-adapter.spec.ts

@@ -0,0 +1,142 @@
+import path from 'node:path';
+
+import { removeWithRetry } from '@affine-test/kit/utils/utils';
+import fs from 'fs-extra';
+import { v4 } from 'uuid';
+import { afterEach, expect, test, vi } from 'vitest';
+import { Doc as YDoc, encodeStateAsUpdate } from 'yjs';
+
+import { dbSubjects } from '../subjects';
+
+const tmpDir = path.join(__dirname, 'tmp');
+const appDataPath = path.join(tmpDir, 'app-data');
+
+vi.doMock('../../main-rpc', () => ({
+  mainRPC: {
+    getPath: async () => appDataPath,
+  },
+}));
+
+afterEach(async () => {
+  await removeWithRetry(tmpDir);
+});
+
+let testYDoc: YDoc;
+let testYSubDoc: YDoc;
+
+function getTestUpdates() {
+  testYDoc = new YDoc();
+  const yText = testYDoc.getText('test');
+  yText.insert(0, 'hello');
+
+  testYSubDoc = new YDoc();
+  testYDoc.getMap('subdocs').set('test-subdoc', testYSubDoc);
+
+  const updates = encodeStateAsUpdate(testYDoc);
+
+  return updates;
+}
+
+function getTestSubDocUpdates() {
+  const yText = testYSubDoc.getText('test');
+  yText.insert(0, 'hello');
+
+  const updates = encodeStateAsUpdate(testYSubDoc);
+
+  return updates;
+}
+
+test('can create new db file if not exists', async () => {
+  const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
+  const workspaceId = v4();
+  const db = await openWorkspaceDatabase(workspaceId);
+  const dbPath = path.join(
+    appDataPath,
+    `workspaces/${workspaceId}`,
+    `storage.db`
+  );
+  expect(await fs.exists(dbPath)).toBe(true);
+  await db.destroy();
+});
+
+test('on applyUpdate (from self), will not trigger update', async () => {
+  const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
+  const workspaceId = v4();
+  const onUpdate = vi.fn();
+
+  const db = await openWorkspaceDatabase(workspaceId);
+  db.update$.subscribe(onUpdate);
+  db.applyUpdate(getTestUpdates(), 'self');
+  expect(onUpdate).not.toHaveBeenCalled();
+  await db.destroy();
+});
+
+test('on applyUpdate (from renderer), will trigger update', async () => {
+  const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
+  const workspaceId = v4();
+  const onUpdate = vi.fn();
+  const onExternalUpdate = vi.fn();
+
+  const db = await openWorkspaceDatabase(workspaceId);
+  db.update$.subscribe(onUpdate);
+  const sub = dbSubjects.externalUpdate.subscribe(onExternalUpdate);
+  db.applyUpdate(getTestUpdates(), 'renderer');
+  expect(onUpdate).toHaveBeenCalled();
+  sub.unsubscribe();
+  await db.destroy();
+});
+
+test('on applyUpdate (from renderer, subdoc), will trigger update', async () => {
+  const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
+  const workspaceId = v4();
+  const onUpdate = vi.fn();
+  const insertUpdates = vi.fn();
+
+  const db = await openWorkspaceDatabase(workspaceId);
+  db.applyUpdate(getTestUpdates(), 'renderer');
+
+  db.db!.insertUpdates = insertUpdates;
+  db.update$.subscribe(onUpdate);
+
+  const subdocUpdates = getTestSubDocUpdates();
+  db.applyUpdate(subdocUpdates, 'renderer', testYSubDoc.guid);
+
+  expect(onUpdate).toHaveBeenCalled();
+  expect(insertUpdates).toHaveBeenCalledWith([
+    {
+      docId: testYSubDoc.guid,
+      data: subdocUpdates,
+    },
+  ]);
+  await db.destroy();
+});
+
+test('on applyUpdate (from external), will trigger update & send external update event', async () => {
+  const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
+  const workspaceId = v4();
+  const onUpdate = vi.fn();
+  const onExternalUpdate = vi.fn();
+
+  const db = await openWorkspaceDatabase(workspaceId);
+  db.update$.subscribe(onUpdate);
+  const sub = dbSubjects.externalUpdate.subscribe(onExternalUpdate);
+  db.applyUpdate(getTestUpdates(), 'external');
+  expect(onUpdate).toHaveBeenCalled();
+  expect(onExternalUpdate).toHaveBeenCalled();
+  sub.unsubscribe();
+  await db.destroy();
+});
+
+test('on destroy, check if resources have been released', async () => {
+  const { openWorkspaceDatabase } = await import('../workspace-db-adapter');
+  const workspaceId = v4();
+  const db = await openWorkspaceDatabase(workspaceId);
+  const updateSub = {
+    complete: vi.fn(),
+    next: vi.fn(),
+  };
+  db.update$ = updateSub as any;
+  await db.destroy();
+  expect(db.db).toBe(null);
+  expect(updateSub.complete).toHaveBeenCalled();
+});

packages/frontend/electron/src/helper/db/base-db-adapter.ts

@@ -0,0 +1,145 @@
+import {
+  type InsertRow,
+  SqliteConnection,
+  ValidationResult,
+} from '@affine/native';
+import { WorkspaceVersion } from '@toeverything/infra/blocksuite';
+
+import { migrateToLatest } from '../db/migration';
+import { logger } from '../logger';
+
+/**
+ * A base class for SQLite DB adapter that provides basic methods around updates & blobs
+ */
+export abstract class BaseSQLiteAdapter {
+  db: SqliteConnection | null = null;
+  abstract role: string;
+
+  constructor(public readonly path: string) {}
+
+  async connectIfNeeded() {
+    if (!this.db) {
+      const validation = await SqliteConnection.validate(this.path);
+      if (validation === ValidationResult.MissingVersionColumn) {
+        await migrateToLatest(this.path, WorkspaceVersion.SubDoc);
+      }
+      this.db = new SqliteConnection(this.path);
+      await this.db.connect();
+      const maxVersion = await this.db.getMaxVersion();
+      if (maxVersion !== WorkspaceVersion.Surface) {
+        await migrateToLatest(this.path, WorkspaceVersion.Surface);
+      }
+      logger.info(`[SQLiteAdapter:${this.role}]`, 'connected:', this.path);
+    }
+    return this.db;
+  }
+
+  async destroy() {
+    const { db } = this;
+    this.db = null;
+    // log after close will sometimes crash the app when quitting
+    logger.info(`[SQLiteAdapter:${this.role}]`, 'destroyed:', this.path);
+    await db?.close();
+  }
+
+  async addBlob(key: string, data: Uint8Array) {
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return;
+      }
+      await this.db.addBlob(key, data);
+    } catch (error) {
+      logger.error('addBlob', error);
+    }
+  }
+
+  async getBlob(key: string) {
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return null;
+      }
+      const blob = await this.db.getBlob(key);
+      return blob?.data ?? null;
+    } catch (error) {
+      logger.error('getBlob', error);
+      return null;
+    }
+  }
+
+  async deleteBlob(key: string) {
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return;
+      }
+      await this.db.deleteBlob(key);
+    } catch (error) {
+      logger.error(`${this.path} delete blob failed`, error);
+    }
+  }
+
+  async getBlobKeys() {
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return [];
+      }
+      return await this.db.getBlobKeys();
+    } catch (error) {
+      logger.error(`getBlobKeys failed`, error);
+      return [];
+    }
+  }
+
+  async getUpdates(docId?: string) {
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return [];
+      }
+      return await this.db.getUpdates(docId);
+    } catch (error) {
+      logger.error('getUpdates', error);
+      return [];
+    }
+  }
+
+  async getAllUpdates() {
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return [];
+      }
+      return await this.db.getAllUpdates();
+    } catch (error) {
+      logger.error('getAllUpdates', error);
+      return [];
+    }
+  }
+
+  // add a single update to SQLite
+  async addUpdateToSQLite(updates: InsertRow[]) {
+    // batch write instead write per key stroke?
+    try {
+      if (!this.db) {
+        logger.warn(`${this.path} is not connected`);
+        return;
+      }
+      const start = performance.now();
+      await this.db.insertUpdates(updates);
+      logger.debug(
+        `[SQLiteAdapter][${this.role}] addUpdateToSQLite`,
+        'length:',
+        updates.length,
+        'docids',
+        updates.map(u => u.docId),
+        performance.now() - start,
+        'ms'
+      );
+    } catch (error) {
+      logger.error('addUpdateToSQLite', this.path, error);
+    }
+  }
+}

packages/frontend/electron/src/helper/db/ensure-db.ts

@@ -0,0 +1,145 @@
+import type { Subject } from 'rxjs';
+import { Observable } from 'rxjs';
+import {
+  concat,
+  defer,
+  from,
+  fromEvent,
+  interval,
+  lastValueFrom,
+  merge,
+} from 'rxjs';
+import {
+  concatMap,
+  distinctUntilChanged,
+  filter,
+  ignoreElements,
+  last,
+  map,
+  shareReplay,
+  startWith,
+  switchMap,
+  take,
+  takeUntil,
+  tap,
+} from 'rxjs/operators';
+
+import { logger } from '../logger';
+import { getWorkspaceMeta } from '../workspace/meta';
+import { workspaceSubjects } from '../workspace/subjects';
+import { SecondaryWorkspaceSQLiteDB } from './secondary-db';
+import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
+import { openWorkspaceDatabase } from './workspace-db-adapter';
+
+// export for testing
+export const db$Map = new Map<string, Observable<WorkspaceSQLiteDB>>();
+
+// use defer to prevent `app` is undefined while running tests
+const beforeQuit$ = defer(() => fromEvent(process, 'beforeExit'));
+
+// return a stream that emit a single event when the subject completes
+function completed<T>(subject: Subject<T>) {
+  return new Observable(subscriber => {
+    const sub = subject.subscribe({
+      complete: () => {
+        subscriber.next();
+        subscriber.complete();
+      },
+    });
+    return () => sub.unsubscribe();
+  });
+}
+
+function getWorkspaceDB$(id: string) {
+  if (!db$Map.has(id)) {
+    db$Map.set(
+      id,
+      from(openWorkspaceDatabase(id)).pipe(
+        tap({
+          next: db => {
+            logger.info(
+              '[ensureSQLiteDB] db connection established',
+              db.workspaceId
+            );
+          },
+        }),
+        switchMap(db =>
+          // takeUntil the polling stream, and then destroy the db
+          concat(
+            startPollingSecondaryDB(db).pipe(
+              ignoreElements(),
+              startWith(db),
+              takeUntil(merge(beforeQuit$, completed(db.update$))),
+              last(),
+              tap({
+                next() {
+                  logger.info(
+                    '[ensureSQLiteDB] polling secondary db complete',
+                    db.workspaceId
+                  );
+                },
+              })
+            ),
+            defer(async () => {
+              try {
+                await db.destroy();
+                db$Map.delete(id);
+                return db;
+              } catch (err) {
+                logger.error('[ensureSQLiteDB] destroy db failed', err);
+                throw err;
+              }
+            })
+          ).pipe(startWith(db))
+        ),
+        shareReplay(1)
+      )
+    );
+  }
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  return db$Map.get(id)!;
+}
+
+function startPollingSecondaryDB(db: WorkspaceSQLiteDB) {
+  return merge(
+    getWorkspaceMeta(db.workspaceId),
+    workspaceSubjects.meta.pipe(
+      map(({ meta }) => meta),
+      filter(meta => meta.id === db.workspaceId)
+    )
+  ).pipe(
+    map(meta => meta?.secondaryDBPath),
+    filter((p): p is string => !!p),
+    distinctUntilChanged(),
+    switchMap(path => {
+      // on secondary db path change, destroy the old db and create a new one
+      const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db);
+      return new Observable<SecondaryWorkspaceSQLiteDB>(subscriber => {
+        subscriber.next(secondaryDB);
+        return () => {
+          secondaryDB.destroy().catch(err => {
+            subscriber.error(err);
+          });
+        };
+      });
+    }),
+    switchMap(secondaryDB => {
+      return interval(300000).pipe(
+        startWith(0),
+        concatMap(() => secondaryDB.pull()),
+        tap({
+          error: err => {
+            logger.error(`[ensureSQLiteDB] polling secondary db error`, err);
+          },
+          complete: () => {
+            logger.info('[ensureSQLiteDB] polling secondary db complete');
+          },
+        })
+      );
+    })
+  );
+}
+
+export function ensureSQLiteDB(id: string) {
+  return lastValueFrom(getWorkspaceDB$(id).pipe(take(1)));
+}

packages/frontend/electron/src/helper/db/index.ts

@@ -0,0 +1,56 @@
+import { mainRPC } from '../main-rpc';
+import type { MainEventRegister } from '../type';
+import { ensureSQLiteDB } from './ensure-db';
+import { dbSubjects } from './subjects';
+
+export * from './ensure-db';
+export * from './subjects';
+
+export const dbHandlers = {
+  getDocAsUpdates: async (workspaceId: string, subdocId?: string) => {
+    const workspaceDB = await ensureSQLiteDB(workspaceId);
+    return workspaceDB.getDocAsUpdates(subdocId);
+  },
+  applyDocUpdate: async (
+    workspaceId: string,
+    update: Uint8Array,
+    subdocId?: string
+  ) => {
+    const workspaceDB = await ensureSQLiteDB(workspaceId);
+    return workspaceDB.applyUpdate(update, 'renderer', subdocId);
+  },
+  addBlob: async (workspaceId: string, key: string, data: Uint8Array) => {
+    const workspaceDB = await ensureSQLiteDB(workspaceId);
+    return workspaceDB.addBlob(key, data);
+  },
+  getBlob: async (workspaceId: string, key: string) => {
+    const workspaceDB = await ensureSQLiteDB(workspaceId);
+    return workspaceDB.getBlob(key);
+  },
+  deleteBlob: async (workspaceId: string, key: string) => {
+    const workspaceDB = await ensureSQLiteDB(workspaceId);
+    return workspaceDB.deleteBlob(key);
+  },
+  getBlobKeys: async (workspaceId: string) => {
+    const workspaceDB = await ensureSQLiteDB(workspaceId);
+    return workspaceDB.getBlobKeys();
+  },
+  getDefaultStorageLocation: async () => {
+    return await mainRPC.getPath('sessionData');
+  },
+};
+
+export const dbEvents = {
+  onExternalUpdate: (
+    fn: (update: {
+      workspaceId: string;
+      update: Uint8Array;
+      docId?: string;
+    }) => void
+  ) => {
+    const sub = dbSubjects.externalUpdate.subscribe(fn);
+    return () => {
+      sub.unsubscribe();
+    };
+  },
+} satisfies Record<string, MainEventRegister>;

packages/frontend/electron/src/helper/db/merge-update.ts

@@ -0,0 +1,11 @@
+import { applyUpdate, Doc as YDoc, encodeStateAsUpdate, transact } from 'yjs';
+
+export function mergeUpdate(updates: Uint8Array[]) {
+  const yDoc = new YDoc();
+  transact(yDoc, () => {
+    for (const update of updates) {
+      applyUpdate(yDoc, update);
+    }
+  });
+  return encodeStateAsUpdate(yDoc);
+}

packages/frontend/electron/src/helper/db/migration.ts

@@ -0,0 +1,121 @@
+import { equal } from 'node:assert';
+import { resolve } from 'node:path';
+
+import { SqliteConnection } from '@affine/native';
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import { Schema } from '@blocksuite/store';
+import {
+  forceUpgradePages,
+  migrateToSubdoc,
+  WorkspaceVersion,
+} from '@toeverything/infra/blocksuite';
+import fs from 'fs-extra';
+import { nanoid } from 'nanoid';
+import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
+
+import { mainRPC } from '../main-rpc';
+
+export const migrateToSubdocAndReplaceDatabase = async (path: string) => {
+  const db = new SqliteConnection(path);
+  await db.connect();
+
+  const rows = await db.getAllUpdates();
+  const originalDoc = new YDoc();
+
+  // 1. apply all updates to the root doc
+  rows.forEach(row => {
+    applyUpdate(originalDoc, row.data);
+  });
+
+  // 2. migrate using migrateToSubdoc
+  const migratedDoc = migrateToSubdoc(originalDoc);
+
+  // 3. replace db rows with the migrated doc
+  await replaceRows(db, migratedDoc, true);
+
+  // 4. close db
+  await db.close();
+};
+
+// v1 v2 -> v3
+// v3 -> v4
+export const migrateToLatest = async (
+  path: string,
+  version: WorkspaceVersion
+) => {
+  const connection = new SqliteConnection(path);
+  await connection.connect();
+  if (version === WorkspaceVersion.SubDoc) {
+    await connection.initVersion();
+  } else {
+    await connection.setVersion(version);
+  }
+  const schema = new Schema();
+  schema.register(AffineSchemas).register(__unstableSchemas);
+  const rootDoc = new YDoc();
+  const downloadBinary = async (doc: YDoc, isRoot: boolean): Promise<void> => {
+    const update = (
+      await connection.getUpdates(isRoot ? undefined : doc.guid)
+    ).map(update => update.data);
+    // Buffer[] -> Uint8Array[]
+    const data = update.map(update => new Uint8Array(update));
+    data.forEach(data => {
+      applyUpdate(doc, data);
+    });
+    // trigger data manually
+    if (isRoot) {
+      doc.getMap('meta');
+      doc.getMap('spaces');
+    } else {
+      doc.getMap('blocks');
+    }
+    await Promise.all(
+      [...doc.subdocs].map(subdoc => {
+        return downloadBinary(subdoc, false);
+      })
+    );
+  };
+  await downloadBinary(rootDoc, true);
+  const result = await forceUpgradePages({
+    getSchema: () => schema,
+    getCurrentRootDoc: () => Promise.resolve(rootDoc),
+  });
+  equal(result, true, 'migrateWorkspace should return boolean value');
+  const uploadBinary = async (doc: YDoc, isRoot: boolean) => {
+    await connection.replaceUpdates(doc.guid, [
+      { docId: isRoot ? undefined : doc.guid, data: encodeStateAsUpdate(doc) },
+    ]);
+    // connection..applyUpdate(encodeStateAsUpdate(doc), 'self', doc.guid)
+    await Promise.all(
+      [...doc.subdocs].map(subdoc => {
+        return uploadBinary(subdoc, false);
+      })
+    );
+  };
+  await uploadBinary(rootDoc, true);
+  await connection.close();
+};
+
+export const copyToTemp = async (path: string) => {
+  const tmpDirPath = resolve(await mainRPC.getPath('sessionData'), 'tmp');
+  const tmpFilePath = resolve(tmpDirPath, nanoid());
+  await fs.ensureDir(tmpDirPath);
+  await fs.copyFile(path, tmpFilePath);
+  return tmpFilePath;
+};
+
+async function replaceRows(
+  db: SqliteConnection,
+  doc: YDoc,
+  isRoot: boolean
+): Promise<void> {
+  const migratedUpdates = encodeStateAsUpdate(doc);
+  const docId = isRoot ? undefined : doc.guid;
+  const rows = [{ data: migratedUpdates, docId: docId }];
+  await db.replaceUpdates(docId, rows);
+  await Promise.all(
+    [...doc.subdocs].map(async subdoc => {
+      await replaceRows(db, subdoc, false);
+    })
+  );
+}

packages/frontend/electron/src/helper/db/secondary-db.ts

@@ -0,0 +1,296 @@
+import assert from 'node:assert';
+
+import type { InsertRow } from '@affine/native';
+import { debounce } from 'lodash-es';
+import { applyUpdate, Doc as YDoc } from 'yjs';
+
+import { logger } from '../logger';
+import type { YOrigin } from '../type';
+import { getWorkspaceMeta } from '../workspace/meta';
+import { BaseSQLiteAdapter } from './base-db-adapter';
+import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
+
+const FLUSH_WAIT_TIME = 5000;
+const FLUSH_MAX_WAIT_TIME = 10000;
+
+// todo: trim db when it is too big
+export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
+  role = 'secondary';
+  yDoc = new YDoc();
+  firstConnected = false;
+  destroyed = false;
+
+  updateQueue: { data: Uint8Array; docId?: string }[] = [];
+
+  unsubscribers = new Set<() => void>();
+
+  constructor(
+    public override path: string,
+    public upstream: WorkspaceSQLiteDB
+  ) {
+    super(path);
+    this.init();
+    logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId);
+  }
+
+  getDoc(docId?: string) {
+    if (!docId) {
+      return this.yDoc;
+    }
+    // this should be pretty fast and we don't need to cache it
+    for (const subdoc of this.yDoc.subdocs) {
+      if (subdoc.guid === docId) {
+        return subdoc;
+      }
+    }
+    return null;
+  }
+
+  override async destroy() {
+    await this.flushUpdateQueue();
+    this.unsubscribers.forEach(unsub => unsub());
+    this.yDoc.destroy();
+    await super.destroy();
+    this.destroyed = true;
+  }
+
+  get workspaceId() {
+    return this.upstream.workspaceId;
+  }
+
+  // do not update db immediately, instead, push to a queue
+  // and flush the queue in a future time
+  async addUpdateToUpdateQueue(update: InsertRow) {
+    this.updateQueue.push(update);
+    await this.debouncedFlush();
+  }
+
+  async flushUpdateQueue() {
+    if (this.destroyed) {
+      return;
+    }
+    logger.debug(
+      'flushUpdateQueue',
+      this.workspaceId,
+      'queue',
+      this.updateQueue.length
+    );
+    const updates = [...this.updateQueue];
+    this.updateQueue = [];
+    await this.run(async () => {
+      await this.addUpdateToSQLite(updates);
+    });
+  }
+
+  // flush after 5s, but will not wait for more than 10s
+  debouncedFlush = debounce(this.flushUpdateQueue, FLUSH_WAIT_TIME, {
+    maxWait: FLUSH_MAX_WAIT_TIME,
+  });
+
+  runCounter = 0;
+
+  // wrap the fn with connect and close
+  async run<T extends (...args: any[]) => any>(
+    fn: T
+  ): Promise<
+    (T extends (...args: any[]) => infer U ? Awaited<U> : unknown) | undefined
+  > {
+    try {
+      if (this.destroyed) {
+        return;
+      }
+      await this.connectIfNeeded();
+      this.runCounter++;
+      return await fn();
+    } catch (err) {
+      logger.error(err);
+      throw err;
+    } finally {
+      this.runCounter--;
+      if (this.runCounter === 0) {
+        // just close db, but not the yDoc
+        await super.destroy();
+      }
+    }
+  }
+
+  setupListener(docId?: string) {
+    logger.debug(
+      'SecondaryWorkspaceSQLiteDB:setupListener',
+      this.workspaceId,
+      docId
+    );
+    const doc = this.getDoc(docId);
+    const upstreamDoc = this.upstream.getDoc(docId);
+    if (!doc || !upstreamDoc) {
+      logger.warn(
+        '[SecondaryWorkspaceSQLiteDB] setupListener: doc not found',
+        docId
+      );
+      return;
+    }
+
+    const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
+      logger.debug(
+        'SecondaryWorkspaceSQLiteDB:onUpstreamUpdate',
+        origin,
+        this.workspaceId,
+        docId,
+        update.length
+      );
+      if (origin === 'renderer' || origin === 'self') {
+        // update to upstream yDoc should be replicated to self yDoc
+        this.applyUpdate(update, 'upstream', docId);
+      }
+    };
+
+    const onSelfUpdate = async (update: Uint8Array, origin: YOrigin) => {
+      logger.debug(
+        'SecondaryWorkspaceSQLiteDB:onSelfUpdate',
+        origin,
+        this.workspaceId,
+        docId,
+        update.length
+      );
+      // for self update from upstream, we need to push it to external DB
+      if (origin === 'upstream') {
+        await this.addUpdateToUpdateQueue({
+          data: update,
+          docId,
+        });
+      }
+
+      if (origin === 'self') {
+        this.upstream.applyUpdate(update, 'external', docId);
+      }
+    };
+
+    const onSubdocs = ({ added }: { added: Set<YDoc> }) => {
+      added.forEach(subdoc => {
+        this.setupListener(subdoc.guid);
+      });
+    };
+
+    doc.subdocs.forEach(subdoc => {
+      this.setupListener(subdoc.guid);
+    });
+
+    // listen to upstream update
+    this.upstream.yDoc.on('update', onUpstreamUpdate);
+    doc.on('update', onSelfUpdate);
+    doc.on('subdocs', onSubdocs);
+
+    this.unsubscribers.add(() => {
+      this.upstream.yDoc.off('update', onUpstreamUpdate);
+      doc.off('update', onSelfUpdate);
+      doc.off('subdocs', onSubdocs);
+    });
+  }
+
+  init() {
+    if (this.firstConnected) {
+      return;
+    }
+    this.firstConnected = true;
+    this.setupListener();
+    // apply all updates from upstream
+    // we assume here that the upstream ydoc is already sync'ed
+    const syncUpstreamDoc = (docId?: string) => {
+      const update = this.upstream.getDocAsUpdates(docId);
+      if (update) {
+        this.applyUpdate(update, 'upstream');
+      }
+    };
+    syncUpstreamDoc();
+    this.upstream.yDoc.subdocs.forEach(subdoc => {
+      syncUpstreamDoc(subdoc.guid);
+    });
+  }
+
+  applyUpdate = (
+    data: Uint8Array,
+    origin: YOrigin = 'upstream',
+    docId?: string
+  ) => {
+    const doc = this.getDoc(docId);
+    if (doc) {
+      applyUpdate(this.yDoc, data, origin);
+    } else {
+      logger.warn(
+        '[SecondaryWorkspaceSQLiteDB] applyUpdate: doc not found',
+        docId
+      );
+    }
+  };
+
+  // TODO: have a better solution to handle blobs
+  async syncBlobs() {
+    await this.run(async () => {
+      // skip if upstream db is not connected (maybe it is already closed)
+      const blobsKeys = await this.getBlobKeys();
+      if (!this.upstream.db || this.upstream.db?.isClose) {
+        return;
+      }
+      const upstreamBlobsKeys = await this.upstream.getBlobKeys();
+      // put every missing blob to upstream
+      for (const key of blobsKeys) {
+        if (!upstreamBlobsKeys.includes(key)) {
+          const blob = await this.getBlob(key);
+          if (blob) {
+            await this.upstream.addBlob(key, blob);
+            logger.debug('syncBlobs', this.workspaceId, key);
+          }
+        }
+      }
+    });
+  }
+
+  /**
+   * pull from external DB file and apply to embedded yDoc
+   * workflow:
+   * - connect to external db
+   * - get updates
+   * - apply updates to local yDoc
+   * - get blobs and put new blobs to upstream
+   * - disconnect
+   */
+  async pull() {
+    const start = performance.now();
+    assert(this.upstream.db, 'upstream db should be connected');
+    const rows = await this.run(async () => {
+      // TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
+      await this.syncBlobs();
+      return await this.getAllUpdates();
+    });
+
+    if (!rows || this.destroyed) {
+      return;
+    }
+
+    // apply root doc first
+    rows.forEach(row => {
+      if (!row.docId) {
+        this.applyUpdate(row.data, 'self');
+      }
+    });
+
+    rows.forEach(row => {
+      if (row.docId) {
+        this.applyUpdate(row.data, 'self', row.docId);
+      }
+    });
+
+    logger.debug(
+      'pull external updates',
+      this.path,
+      rows.length,
+      (performance.now() - start).toFixed(2),
+      'ms'
+    );
+  }
+}
+
+export async function getSecondaryWorkspaceDBPath(workspaceId: string) {
+  const meta = await getWorkspaceMeta(workspaceId);
+  return meta?.secondaryDBPath;
+}

packages/frontend/electron/src/helper/db/subjects.ts

@@ -0,0 +1,9 @@
+import { Subject } from 'rxjs';
+
+export const dbSubjects = {
+  externalUpdate: new Subject<{
+    workspaceId: string;
+    update: Uint8Array;
+    docId?: string;
+  }>(),
+};

packages/frontend/electron/src/helper/db/workspace-db-adapter.ts

@@ -0,0 +1,196 @@
+import type { InsertRow } from '@affine/native';
+import { debounce } from 'lodash-es';
+import { Subject } from 'rxjs';
+import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
+
+import { logger } from '../logger';
+import type { YOrigin } from '../type';
+import { getWorkspaceMeta } from '../workspace/meta';
+import { BaseSQLiteAdapter } from './base-db-adapter';
+import { dbSubjects } from './subjects';
+
+const TRIM_SIZE = 500;
+
+export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
+  role = 'primary';
+  yDoc = new YDoc();
+  firstConnected = false;
+
+  update$ = new Subject<void>();
+
+  constructor(
+    public override path: string,
+    public workspaceId: string
+  ) {
+    super(path);
+  }
+
+  override async destroy() {
+    await super.destroy();
+    this.yDoc.destroy();
+
+    // when db is closed, we can safely remove it from ensure-db list
+    this.update$.complete();
+    this.firstConnected = false;
+  }
+
+  getDoc(docId?: string) {
+    if (!docId) {
+      return this.yDoc;
+    }
+    // this should be pretty fast and we don't need to cache it
+    for (const subdoc of this.yDoc.subdocs) {
+      if (subdoc.guid === docId) {
+        return subdoc;
+      }
+    }
+    return null;
+  }
+
+  getWorkspaceName = () => {
+    return this.yDoc.getMap('meta').get('name') as string;
+  };
+
+  setupListener(docId?: string) {
+    logger.debug('WorkspaceSQLiteDB:setupListener', this.workspaceId, docId);
+    const doc = this.getDoc(docId);
+    if (doc) {
+      const onUpdate = async (update: Uint8Array, origin: YOrigin) => {
+        logger.debug(
+          'WorkspaceSQLiteDB:onUpdate',
+          this.workspaceId,
+          docId,
+          update.length
+        );
+        const insertRows = [{ data: update, docId }];
+        if (origin === 'renderer') {
+          await this.addUpdateToSQLite(insertRows);
+        } else if (origin === 'external') {
+          dbSubjects.externalUpdate.next({
+            workspaceId: this.workspaceId,
+            update,
+            docId,
+          });
+          await this.addUpdateToSQLite(insertRows);
+          logger.debug('external update', this.workspaceId);
+        }
+      };
+      doc.subdocs.forEach(subdoc => {
+        this.setupListener(subdoc.guid);
+      });
+      const onSubdocs = ({ added }: { added: Set<YDoc> }) => {
+        logger.info('onSubdocs', this.workspaceId, docId, added);
+        added.forEach(subdoc => {
+          this.setupListener(subdoc.guid);
+        });
+      };
+
+      doc.on('update', onUpdate);
+      doc.on('subdocs', onSubdocs);
+    } else {
+      logger.error('setupListener: doc not found', docId);
+    }
+  }
+
+  async init() {
+    const db = await super.connectIfNeeded();
+
+    if (!this.firstConnected) {
+      this.setupListener();
+    }
+
+    const updates = await this.getAllUpdates();
+
+    // apply root first (without ID).
+    // subdoc will be available after root is applied
+    updates.forEach(update => {
+      if (!update.docId) {
+        this.applyUpdate(update.data, 'self');
+      }
+    });
+
+    // then, for all subdocs, apply the updates
+    updates.forEach(update => {
+      if (update.docId) {
+        this.applyUpdate(update.data, 'self', update.docId);
+      }
+    });
+
+    this.firstConnected = true;
+    this.update$.next();
+
+    return db;
+  }
+
+  // unlike getUpdates, this will return updates in yDoc
+  getDocAsUpdates = (docId?: string) => {
+    const doc = docId ? this.getDoc(docId) : this.yDoc;
+    if (doc) {
+      return encodeStateAsUpdate(doc);
+    }
+    return false;
+  };
+
+  // non-blocking and use yDoc to validate the update
+  // after that, the update is added to the db
+  applyUpdate = (
+    data: Uint8Array,
+    origin: YOrigin = 'renderer',
+    docId?: string
+  ) => {
+    // todo: trim the updates when the number of records is too large
+    // 1. store the current ydoc state in the db
+    // 2. then delete the old updates
+    // yjs-idb will always trim the db for the first time after DB is loaded
+    const doc = this.getDoc(docId);
+    if (doc) {
+      applyUpdate(doc, data, origin);
+    } else {
+      logger.warn('[WorkspaceSQLiteDB] applyUpdate: doc not found', docId);
+    }
+  };
+
+  override async addBlob(key: string, value: Uint8Array) {
+    this.update$.next();
+    const res = await super.addBlob(key, value);
+    return res;
+  }
+
+  override async deleteBlob(key: string) {
+    this.update$.next();
+    await super.deleteBlob(key);
+  }
+
+  override async addUpdateToSQLite(data: InsertRow[]) {
+    this.update$.next();
+    data.forEach(row => {
+      this.trimWhenNecessary(row.docId)?.catch(err => {
+        logger.error('trimWhenNecessary failed', err);
+      });
+    });
+    await super.addUpdateToSQLite(data);
+  }
+
+  trimWhenNecessary = debounce(async (docId?: string) => {
+    if (this.firstConnected) {
+      const count = (await this.db?.getUpdatesCount(docId)) ?? 0;
+      if (count > TRIM_SIZE) {
+        logger.debug(`trim ${this.workspaceId}:${docId} ${count}`);
+        const update = this.getDocAsUpdates(docId);
+        if (update) {
+          const insertRows = [{ data: update, docId }];
+          await this.db?.replaceUpdates(docId, insertRows);
+          logger.debug(`trim ${this.workspaceId}:${docId} successfully`);
+        }
+      }
+    }
+  }, 1000);
+}
+
+export async function openWorkspaceDatabase(workspaceId: string) {
+  const meta = await getWorkspaceMeta(workspaceId);
+  const db = new WorkspaceSQLiteDB(meta.mainDBPath, workspaceId);
+  await db.init();
+  logger.info(`openWorkspaceDatabase [${workspaceId}]`);
+  return db;
+}

packages/frontend/electron/src/helper/dialog/dialog.ts

@@ -0,0 +1,361 @@
+import path from 'node:path';
+
+import { ValidationResult } from '@affine/native';
+import { WorkspaceVersion } from '@toeverything/infra/blocksuite';
+import type {
+  FakeDialogResult,
+  LoadDBFileResult,
+  MoveDBFileResult,
+  SaveDBFileResult,
+  SelectDBFileLocationResult,
+} from '@toeverything/infra/type';
+import fs from 'fs-extra';
+import { nanoid } from 'nanoid';
+
+import { ensureSQLiteDB } from '../db/ensure-db';
+import {
+  copyToTemp,
+  migrateToLatest,
+  migrateToSubdocAndReplaceDatabase,
+} from '../db/migration';
+import type { WorkspaceSQLiteDB } from '../db/workspace-db-adapter';
+import { logger } from '../logger';
+import { mainRPC } from '../main-rpc';
+import { listWorkspaces, storeWorkspaceMeta } from '../workspace';
+import {
+  getWorkspaceDBPath,
+  getWorkspaceMeta,
+  getWorkspacesBasePath,
+} from '../workspace/meta';
+
+// NOTE:
+// we are using native dialogs because HTML dialogs do not give full file paths
+
+export async function revealDBFile(workspaceId: string) {
+  const meta = await getWorkspaceMeta(workspaceId);
+  if (!meta) {
+    return;
+  }
+  await mainRPC.showItemInFolder(meta.secondaryDBPath ?? meta.mainDBPath);
+}
+
+// result will be used in the next call to showOpenDialog
+// if it is being read once, it will be reset to undefined
+let fakeDialogResult: FakeDialogResult | undefined = undefined;
+
+function getFakedResult() {
+  const result = fakeDialogResult;
+  fakeDialogResult = undefined;
+  return result;
+}
+
+export function setFakeDialogResult(result: FakeDialogResult | undefined) {
+  fakeDialogResult = result;
+  // for convenience, we will fill filePaths with filePath if it is not set
+  if (result?.filePaths === undefined && result?.filePath !== undefined) {
+    result.filePaths = [result.filePath];
+  }
+}
+
+const extension = 'affine';
+
+function getDefaultDBFileName(name: string, id: string) {
+  const fileName = `${name}_${id}.${extension}`;
+  // make sure fileName is a valid file name
+  return fileName.replace(/[/\\?%*:|"<>]/g, '-');
+}
+
+/**
+ * This function is called when the user clicks the "Save" button in the "Save Workspace" dialog.
+ *
+ * It will just copy the file to the given path
+ */
+export async function saveDBFileAs(
+  workspaceId: string
+): Promise<SaveDBFileResult> {
+  try {
+    const db = await ensureSQLiteDB(workspaceId);
+    const ret =
+      getFakedResult() ??
+      (await mainRPC.showSaveDialog({
+        properties: ['showOverwriteConfirmation'],
+        title: 'Save Workspace',
+        showsTagField: false,
+        buttonLabel: 'Save',
+        filters: [
+          {
+            extensions: [extension],
+            name: '',
+          },
+        ],
+        defaultPath: getDefaultDBFileName(db.getWorkspaceName(), workspaceId),
+        message: 'Save Workspace as a SQLite Database file',
+      }));
+    const filePath = ret.filePath;
+    if (ret.canceled || !filePath) {
+      return {
+        canceled: true,
+      };
+    }
+
+    await fs.copyFile(db.path, filePath);
+    logger.log('saved', filePath);
+    mainRPC.showItemInFolder(filePath).catch(err => {
+      console.error(err);
+    });
+    return { filePath };
+  } catch (err) {
+    logger.error('saveDBFileAs', err);
+    return {
+      error: 'UNKNOWN_ERROR',
+    };
+  }
+}
+
+export async function selectDBFileLocation(): Promise<SelectDBFileLocationResult> {
+  try {
+    const ret =
+      getFakedResult() ??
+      (await mainRPC.showOpenDialog({
+        properties: ['openDirectory'],
+        title: 'Set Workspace Storage Location',
+        buttonLabel: 'Select',
+        defaultPath: await mainRPC.getPath('documents'),
+        message: "Select a location to store the workspace's database file",
+      }));
+    const dir = ret.filePaths?.[0];
+    if (ret.canceled || !dir) {
+      return {
+        canceled: true,
+      };
+    }
+    return { filePath: dir };
+  } catch (err) {
+    logger.error('selectDBFileLocation', err);
+    return {
+      error: (err as any).message,
+    };
+  }
+}
+
+/**
+ * This function is called when the user clicks the "Load" button in the "Load Workspace" dialog.
+ *
+ * It will
+ * - symlink the source db file to a new workspace id to app-data
+ * - return the new workspace id
+ *
+ * eg, it will create a new folder in app-data:
+ * <app-data>/<app-name>/workspaces/<workspace-id>/storage.db
+ *
+ * On the renderer side, after the UI got a new workspace id, it will
+ * update the local workspace id list and then connect to it.
+ *
+ */
+export async function loadDBFile(): Promise<LoadDBFileResult> {
+  try {
+    const ret =
+      getFakedResult() ??
+      (await mainRPC.showOpenDialog({
+        properties: ['openFile'],
+        title: 'Load Workspace',
+        buttonLabel: 'Load',
+        filters: [
+          {
+            name: 'SQLite Database',
+            // do we want to support other file format?
+            extensions: ['db', 'affine'],
+          },
+        ],
+        message: 'Load Workspace from a AFFiNE file',
+      }));
+    let originalPath = ret.filePaths?.[0];
+    if (ret.canceled || !originalPath) {
+      logger.info('loadDBFile canceled');
+      return { canceled: true };
+    }
+
+    // the imported file should not be in app data dir
+    if (originalPath.startsWith(await getWorkspacesBasePath())) {
+      logger.warn('loadDBFile: db file in app data dir');
+      return { error: 'DB_FILE_PATH_INVALID' };
+    }
+
+    if (await dbFileAlreadyLoaded(originalPath)) {
+      logger.warn('loadDBFile: db file already loaded');
+      return { error: 'DB_FILE_ALREADY_LOADED' };
+    }
+
+    const { SqliteConnection } = await import('@affine/native');
+
+    const validationResult = await SqliteConnection.validate(originalPath);
+
+    if (validationResult === ValidationResult.MissingDocIdColumn) {
+      try {
+        const tmpDBPath = await copyToTemp(originalPath);
+        await migrateToSubdocAndReplaceDatabase(tmpDBPath);
+        originalPath = tmpDBPath;
+      } catch (error) {
+        logger.warn(`loadDBFile, migration failed: ${originalPath}`, error);
+        return { error: 'DB_FILE_MIGRATION_FAILED' };
+      }
+    }
+
+    if (validationResult === ValidationResult.MissingVersionColumn) {
+      try {
+        const tmpDBPath = await copyToTemp(originalPath);
+        await migrateToLatest(tmpDBPath, WorkspaceVersion.SubDoc);
+        originalPath = tmpDBPath;
+      } catch (error) {
+        logger.warn(
+          `loadDBFile, migration version column failed: ${originalPath}`,
+          error
+        );
+        return { error: 'DB_FILE_MIGRATION_FAILED' };
+      }
+    }
+
+    if (
+      validationResult !== ValidationResult.MissingVersionColumn &&
+      validationResult !== ValidationResult.MissingDocIdColumn &&
+      validationResult !== ValidationResult.Valid
+    ) {
+      return { error: 'DB_FILE_INVALID' }; // invalid db file
+    }
+
+    const db = new SqliteConnection(originalPath);
+    try {
+      await db.connect();
+      if ((await db.getMaxVersion()) === WorkspaceVersion.DatabaseV3) {
+        const tmpDBPath = await copyToTemp(originalPath);
+        await migrateToLatest(tmpDBPath, WorkspaceVersion.SubDoc);
+        originalPath = tmpDBPath;
+      }
+    } catch (error) {
+      logger.warn(
+        `loadDBFile, migration version column failed: ${originalPath}`,
+        error
+      );
+      return { error: 'DB_FILE_MIGRATION_FAILED' };
+    } finally {
+      await db.close();
+    }
+
+    // copy the db file to a new workspace id
+    const workspaceId = nanoid(10);
+    const internalFilePath = await getWorkspaceDBPath(workspaceId);
+
+    await fs.ensureDir(await getWorkspacesBasePath());
+    await fs.copy(originalPath, internalFilePath);
+    logger.info(`loadDBFile, copy: ${originalPath} -> ${internalFilePath}`);
+
+    await storeWorkspaceMeta(workspaceId, {
+      id: workspaceId,
+      mainDBPath: internalFilePath,
+    });
+
+    return { workspaceId };
+  } catch (err) {
+    logger.error('loadDBFile', err);
+    return {
+      error: 'UNKNOWN_ERROR',
+    };
+  }
+}
+
+/**
+ * This function is called when the user clicks the "Move" button in the "Move Workspace Storage" setting.
+ *
+ * It will
+ * - copy the source db file to a new location
+ * - remove the old db external file
+ * - update the external db file path in the workspace meta
+ * - return the new file path
+ */
+export async function moveDBFile(
+  workspaceId: string,
+  dbFileDir?: string
+): Promise<MoveDBFileResult> {
+  let db: WorkspaceSQLiteDB | null = null;
+  try {
+    db = await ensureSQLiteDB(workspaceId);
+    const meta = await getWorkspaceMeta(workspaceId);
+
+    const oldDir = meta.secondaryDBPath
+      ? path.dirname(meta.secondaryDBPath)
+      : null;
+    const defaultDir = oldDir ?? (await mainRPC.getPath('documents'));
+
+    const newName = getDefaultDBFileName(db.getWorkspaceName(), workspaceId);
+
+    const newDirPath =
+      dbFileDir ??
+      (
+        getFakedResult() ??
+        (await mainRPC.showOpenDialog({
+          properties: ['openDirectory'],
+          title: 'Move Workspace Storage',
+          buttonLabel: 'Move',
+          defaultPath: defaultDir,
+          message: 'Move Workspace storage file',
+        }))
+      ).filePaths?.[0];
+
+    // skips if
+    // - user canceled the dialog
+    // - user selected the same dir
+    if (!newDirPath || newDirPath === oldDir) {
+      return {
+        canceled: true,
+      };
+    }
+
+    const newFilePath = path.join(newDirPath, newName);
+
+    if (await fs.pathExists(newFilePath)) {
+      return {
+        error: 'FILE_ALREADY_EXISTS',
+      };
+    }
+
+    logger.info(`[moveDBFile] copy ${meta.mainDBPath} -> ${newFilePath}`);
+
+    await fs.copy(meta.mainDBPath, newFilePath);
+
+    // remove the old db file, but we don't care if it fails
+    if (meta.secondaryDBPath) {
+      await fs
+        .remove(meta.secondaryDBPath)
+        .then(() => {
+          logger.info(`[moveDBFile] removed ${meta.secondaryDBPath}`);
+        })
+        .catch(err => {
+          logger.error(
+            `[moveDBFile] remove ${meta.secondaryDBPath} failed`,
+            err
+          );
+        });
+    }
+
+    // update meta
+    await storeWorkspaceMeta(workspaceId, {
+      secondaryDBPath: newFilePath,
+    });
+
+    return {
+      filePath: newFilePath,
+    };
+  } catch (err) {
+    await db?.destroy();
+    logger.error('[moveDBFile]', err);
+    return {
+      error: 'UNKNOWN_ERROR',
+    };
+  }
+}
+
+async function dbFileAlreadyLoaded(path: string) {
+  const meta = await listWorkspaces();
+  const paths = meta.map(m => m[1].secondaryDBPath);
+  return paths.includes(path);
+}

packages/frontend/electron/src/helper/dialog/index.ts

@@ -0,0 +1,31 @@
+import {
+  loadDBFile,
+  moveDBFile,
+  revealDBFile,
+  saveDBFileAs,
+  selectDBFileLocation,
+  setFakeDialogResult,
+} from './dialog';
+
+export const dialogHandlers = {
+  revealDBFile: async (workspaceId: string) => {
+    return revealDBFile(workspaceId);
+  },
+  loadDBFile: async () => {
+    return loadDBFile();
+  },
+  saveDBFileAs: async (workspaceId: string) => {
+    return saveDBFileAs(workspaceId);
+  },
+  moveDBFile: (workspaceId: string, dbFileLocation?: string) => {
+    return moveDBFile(workspaceId, dbFileLocation);
+  },
+  selectDBFileLocation: async () => {
+    return selectDBFileLocation();
+  },
+  setFakeDialogResult: async (
+    result: Parameters<typeof setFakeDialogResult>[0]
+  ) => {
+    return setFakeDialogResult(result);
+  },
+};

packages/frontend/electron/src/helper/exposed.ts

@@ -0,0 +1,48 @@
+import type {
+  DBHandlers,
+  DialogHandlers,
+  WorkspaceHandlers,
+} from '@toeverything/infra/type';
+
+import { dbEvents, dbHandlers } from './db';
+import { dialogHandlers } from './dialog';
+import { provideExposed } from './provide';
+import { workspaceEvents, workspaceHandlers } from './workspace';
+
+type AllHandlers = {
+  db: DBHandlers;
+  workspace: WorkspaceHandlers;
+  dialog: DialogHandlers;
+};
+
+export const handlers = {
+  db: dbHandlers,
+  workspace: workspaceHandlers,
+  dialog: dialogHandlers,
+} satisfies AllHandlers;
+
+export const events = {
+  db: dbEvents,
+  workspace: workspaceEvents,
+};
+
+const getExposedMeta = () => {
+  const handlersMeta = Object.entries(handlers).map(
+    ([namespace, namespaceHandlers]) => {
+      return [namespace, Object.keys(namespaceHandlers)] as [string, string[]];
+    }
+  );
+
+  const eventsMeta = Object.entries(events).map(
+    ([namespace, namespaceHandlers]) => {
+      return [namespace, Object.keys(namespaceHandlers)] as [string, string[]];
+    }
+  );
+
+  return {
+    handlers: handlersMeta,
+    events: eventsMeta,
+  };
+};
+
+provideExposed(getExposedMeta());

packages/frontend/electron/src/helper/index.ts

@@ -0,0 +1,82 @@
+import type { RendererToHelper } from '@toeverything/infra/preload/electron';
+import { AsyncCall } from 'async-call-rpc';
+
+import { events, handlers } from './exposed';
+import { logger } from './logger';
+
+function setupRendererConnection(rendererPort: Electron.MessagePortMain) {
+  const flattenedHandlers = Object.entries(handlers).flatMap(
+    ([namespace, namespaceHandlers]) => {
+      return Object.entries(namespaceHandlers).map(([name, handler]) => {
+        const handlerWithLog = async (...args: any[]) => {
+          try {
+            const start = performance.now();
+            const result = await handler(...args);
+            logger.info(
+              '[async-api]',
+              `${namespace}.${name}`,
+              args.filter(
+                arg => typeof arg !== 'function' && typeof arg !== 'object'
+              ),
+              '-',
+              (performance.now() - start).toFixed(2),
+              'ms'
+            );
+            return result;
+          } catch (error) {
+            logger.error('[async-api]', `${namespace}.${name}`, error);
+          }
+        };
+        return [`${namespace}:${name}`, handlerWithLog];
+      });
+    }
+  );
+  const rpc = AsyncCall<RendererToHelper>(
+    Object.fromEntries(flattenedHandlers),
+    {
+      channel: {
+        on(listener) {
+          const f = (e: Electron.MessageEvent) => {
+            listener(e.data);
+          };
+          rendererPort.on('message', f);
+          // MUST start the connection to receive messages
+          rendererPort.start();
+          return () => {
+            rendererPort.off('message', f);
+          };
+        },
+        send(data) {
+          rendererPort.postMessage(data);
+        },
+      },
+      log: false,
+    }
+  );
+
+  for (const [namespace, namespaceEvents] of Object.entries(events)) {
+    for (const [key, eventRegister] of Object.entries(namespaceEvents)) {
+      const unsub = eventRegister((...args: any[]) => {
+        const chan = `${namespace}:${key}`;
+        rpc.postEvent(chan, ...args).catch(err => {
+          console.error(err);
+        });
+      });
+      process.on('exit', () => {
+        unsub();
+      });
+    }
+  }
+}
+
+function main() {
+  process.parentPort.on('message', e => {
+    if (e.data.channel === 'renderer-connect' && e.ports.length === 1) {
+      const rendererPort = e.ports[0];
+      setupRendererConnection(rendererPort);
+      logger.info('[helper] renderer connected');
+    }
+  });
+}
+
+main();

packages/frontend/electron/src/helper/logger.ts

@@ -0,0 +1,3 @@
+import log from 'electron-log';
+
+export const logger = log.scope('helper');

packages/frontend/electron/src/helper/main-rpc.ts

@@ -0,0 +1,35 @@
+import { assertExists } from '@blocksuite/global/utils';
+import type {
+  HelperToMain,
+  MainToHelper,
+} from '@toeverything/infra/preload/electron';
+import { AsyncCall } from 'async-call-rpc';
+
+import { exposed } from './provide';
+
+const helperToMainServer: HelperToMain = {
+  getMeta: () => {
+    assertExists(exposed);
+    return exposed;
+  },
+};
+
+export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {
+  strict: {
+    unknownMessage: false,
+  },
+  channel: {
+    on(listener) {
+      const f = (e: Electron.MessageEvent) => {
+        listener(e.data);
+      };
+      process.parentPort.on('message', f);
+      return () => {
+        process.parentPort.off('message', f);
+      };
+    },
+    send(data) {
+      process.parentPort.postMessage(data);
+    },
+  },
+});

packages/frontend/electron/src/helper/provide.ts

@@ -0,0 +1,11 @@
+import type { ExposedMeta } from '@toeverything/infra/preload/electron';
+
+/**
+ * A naive DI implementation to get rid of circular dependency.
+ */
+
+export let exposed: ExposedMeta | undefined;
+
+export const provideExposed = (exposedMeta: ExposedMeta) => {
+  exposed = exposedMeta;
+};

packages/frontend/electron/src/helper/type.ts

@@ -0,0 +1,9 @@
+export interface WorkspaceMeta {
+  id: string;
+  mainDBPath: string;
+  secondaryDBPath?: string; // assume there will be only one
+}
+
+export type YOrigin = 'self' | 'external' | 'upstream' | 'renderer';
+
+export type MainEventRegister = (...args: any[]) => () => void;

packages/frontend/electron/src/helper/workspace/__tests__/.gitignore

@@ -0,0 +1 @@
+tmp

packages/frontend/electron/src/helper/workspace/__tests__/handlers.spec.ts

@@ -0,0 +1,143 @@
+import path from 'node:path';
+
+import { removeWithRetry } from '@affine-test/kit/utils/utils';
+import fs from 'fs-extra';
+import { v4 } from 'uuid';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+
+const tmpDir = path.join(__dirname, 'tmp');
+const appDataPath = path.join(tmpDir, 'app-data');
+
+vi.doMock('../../db/ensure-db', () => ({
+  ensureSQLiteDB: async () => ({
+    destroy: () => {},
+  }),
+}));
+
+vi.doMock('../../main-rpc', () => ({
+  mainRPC: {
+    getPath: async () => appDataPath,
+  },
+}));
+
+afterEach(async () => {
+  await removeWithRetry(tmpDir);
+});
+
+describe('list workspaces', () => {
+  test('listWorkspaces (valid)', async () => {
+    const { listWorkspaces } = await import('../handlers');
+    const workspaceId = v4();
+    const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+    const meta = {
+      id: workspaceId,
+    };
+    await fs.ensureDir(workspacePath);
+    await fs.writeJSON(path.join(workspacePath, 'meta.json'), meta);
+    const workspaces = await listWorkspaces();
+    expect(workspaces).toEqual([[workspaceId, meta]]);
+  });
+
+  test('listWorkspaces (without meta json file)', async () => {
+    const { listWorkspaces } = await import('../handlers');
+    const workspaceId = v4();
+    const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+    await fs.ensureDir(workspacePath);
+    const workspaces = await listWorkspaces();
+    expect(workspaces).toEqual([
+      [
+        workspaceId,
+        // meta file will be created automatically
+        { id: workspaceId, mainDBPath: path.join(workspacePath, 'storage.db') },
+      ],
+    ]);
+  });
+});
+
+describe('delete workspace', () => {
+  test('deleteWorkspace', async () => {
+    const { deleteWorkspace } = await import('../handlers');
+    const workspaceId = v4();
+    const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+    await fs.ensureDir(workspacePath);
+    await deleteWorkspace(workspaceId);
+    expect(await fs.pathExists(workspacePath)).toBe(false);
+    // removed workspace will be moved to deleted-workspaces
+    expect(
+      await fs.pathExists(
+        path.join(appDataPath, 'deleted-workspaces', workspaceId)
+      )
+    ).toBe(true);
+  });
+});
+
+describe('getWorkspaceMeta', () => {
+  test('can get meta', async () => {
+    const { getWorkspaceMeta } = await import('../meta');
+    const workspaceId = v4();
+    const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+    const meta = {
+      id: workspaceId,
+    };
+    await fs.ensureDir(workspacePath);
+    await fs.writeJSON(path.join(workspacePath, 'meta.json'), meta);
+    expect(await getWorkspaceMeta(workspaceId)).toEqual(meta);
+  });
+
+  test('can create meta if not exists', async () => {
+    const { getWorkspaceMeta } = await import('../meta');
+    const workspaceId = v4();
+    const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+    await fs.ensureDir(workspacePath);
+    expect(await getWorkspaceMeta(workspaceId)).toEqual({
+      id: workspaceId,
+      mainDBPath: path.join(workspacePath, 'storage.db'),
+    });
+    expect(
+      await fs.pathExists(path.join(workspacePath, 'meta.json'))
+    ).toBeTruthy();
+  });
+
+  test('can migrate meta if db file is a link', async () => {
+    const { getWorkspaceMeta } = await import('../meta');
+    const workspaceId = v4();
+    const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+    await fs.ensureDir(workspacePath);
+    const sourcePath = path.join(tmpDir, 'source.db');
+    await fs.writeFile(sourcePath, 'test');
+
+    await fs.ensureSymlink(sourcePath, path.join(workspacePath, 'storage.db'));
+
+    expect(await getWorkspaceMeta(workspaceId)).toEqual({
+      id: workspaceId,
+      mainDBPath: path.join(workspacePath, 'storage.db'),
+      secondaryDBPath: sourcePath,
+    });
+
+    expect(
+      await fs.pathExists(path.join(workspacePath, 'meta.json'))
+    ).toBeTruthy();
+  });
+});
+
+test('storeWorkspaceMeta', async () => {
+  const { storeWorkspaceMeta } = await import('../handlers');
+  const workspaceId = v4();
+  const workspacePath = path.join(appDataPath, 'workspaces', workspaceId);
+  await fs.ensureDir(workspacePath);
+  const meta = {
+    id: workspaceId,
+    mainDBPath: path.join(workspacePath, 'storage.db'),
+  };
+  await storeWorkspaceMeta(workspaceId, meta);
+  expect(await fs.readJSON(path.join(workspacePath, 'meta.json'))).toEqual(
+    meta
+  );
+  await storeWorkspaceMeta(workspaceId, {
+    secondaryDBPath: path.join(tmpDir, 'test.db'),
+  });
+  expect(await fs.readJSON(path.join(workspacePath, 'meta.json'))).toEqual({
+    ...meta,
+    secondaryDBPath: path.join(tmpDir, 'test.db'),
+  });
+});

packages/frontend/electron/src/helper/workspace/handlers.ts

@@ -0,0 +1,77 @@
+import path from 'node:path';
+
+import fs from 'fs-extra';
+
+import { ensureSQLiteDB } from '../db/ensure-db';
+import { logger } from '../logger';
+import type { WorkspaceMeta } from '../type';
+import {
+  getDeletedWorkspacesBasePath,
+  getWorkspaceBasePath,
+  getWorkspaceMeta,
+  getWorkspacesBasePath,
+} from './meta';
+import { workspaceSubjects } from './subjects';
+
+export async function listWorkspaces(): Promise<
+  [workspaceId: string, meta: WorkspaceMeta][]
+> {
+  const basePath = await getWorkspacesBasePath();
+  try {
+    await fs.ensureDir(basePath);
+    const dirs = (
+      await fs.readdir(basePath, {
+        withFileTypes: true,
+      })
+    ).filter(d => d.isDirectory());
+    const metaList = (
+      await Promise.all(
+        dirs.map(async dir => {
+          // ? shall we put all meta in a single file instead of one file per workspace?
+          return await getWorkspaceMeta(dir.name);
+        })
+      )
+    ).filter((w): w is WorkspaceMeta => !!w);
+    return metaList.map(meta => [meta.id, meta]);
+  } catch (error) {
+    logger.error('listWorkspaces', error);
+    return [];
+  }
+}
+
+export async function deleteWorkspace(id: string) {
+  const basePath = await getWorkspaceBasePath(id);
+  const movedPath = path.join(await getDeletedWorkspacesBasePath(), `${id}`);
+  try {
+    const db = await ensureSQLiteDB(id);
+    await db.destroy();
+    return await fs.move(basePath, movedPath, {
+      overwrite: true,
+    });
+  } catch (error) {
+    logger.error('deleteWorkspace', error);
+  }
+}
+
+export async function storeWorkspaceMeta(
+  workspaceId: string,
+  meta: Partial<WorkspaceMeta>
+) {
+  try {
+    const basePath = await getWorkspaceBasePath(workspaceId);
+    await fs.ensureDir(basePath);
+    const metaPath = path.join(basePath, 'meta.json');
+    const currentMeta = await getWorkspaceMeta(workspaceId);
+    const newMeta = {
+      ...currentMeta,
+      ...meta,
+    };
+    await fs.writeJSON(metaPath, newMeta);
+    workspaceSubjects.meta.next({
+      workspaceId,
+      meta: newMeta,
+    });
+  } catch (err) {
+    logger.error('storeWorkspaceMeta failed', err);
+  }
+}

packages/frontend/electron/src/helper/workspace/index.ts

@@ -0,0 +1,26 @@
+import type { MainEventRegister, WorkspaceMeta } from '../type';
+import { deleteWorkspace, listWorkspaces } from './handlers';
+import { getWorkspaceMeta } from './meta';
+import { workspaceSubjects } from './subjects';
+
+export * from './handlers';
+export * from './subjects';
+
+export const workspaceEvents = {
+  onMetaChange: (
+    fn: (meta: { workspaceId: string; meta: WorkspaceMeta }) => void
+  ) => {
+    const sub = workspaceSubjects.meta.subscribe(fn);
+    return () => {
+      sub.unsubscribe();
+    };
+  },
+} satisfies Record<string, MainEventRegister>;
+
+export const workspaceHandlers = {
+  list: async () => listWorkspaces(),
+  delete: async (id: string) => deleteWorkspace(id),
+  getMeta: async (id: string) => {
+    return getWorkspaceMeta(id);
+  },
+};

packages/frontend/electron/src/helper/workspace/meta.ts

@@ -0,0 +1,86 @@
+import path from 'node:path';
+
+import fs from 'fs-extra';
+
+import { logger } from '../logger';
+import { mainRPC } from '../main-rpc';
+import type { WorkspaceMeta } from '../type';
+
+let _appDataPath = '';
+
+export async function getAppDataPath() {
+  if (_appDataPath) {
+    return _appDataPath;
+  }
+  _appDataPath = await mainRPC.getPath('sessionData');
+  return _appDataPath;
+}
+
+export async function getWorkspacesBasePath() {
+  return path.join(await getAppDataPath(), 'workspaces');
+}
+
+export async function getWorkspaceBasePath(workspaceId: string) {
+  return path.join(await getAppDataPath(), 'workspaces', workspaceId);
+}
+
+export async function getDeletedWorkspacesBasePath() {
+  return path.join(await getAppDataPath(), 'deleted-workspaces');
+}
+
+export async function getWorkspaceDBPath(workspaceId: string) {
+  return path.join(await getWorkspaceBasePath(workspaceId), 'storage.db');
+}
+
+export async function getWorkspaceMetaPath(workspaceId: string) {
+  return path.join(await getWorkspaceBasePath(workspaceId), 'meta.json');
+}
+
+/**
+ * Get workspace meta, create one if not exists
+ * This function will also migrate the workspace if needed
+ */
+export async function getWorkspaceMeta(
+  workspaceId: string
+): Promise<WorkspaceMeta> {
+  try {
+    const basePath = await getWorkspaceBasePath(workspaceId);
+    const metaPath = await getWorkspaceMetaPath(workspaceId);
+    if (
+      !(await fs
+        .access(metaPath)
+        .then(() => true)
+        .catch(() => false))
+    ) {
+      // since not meta is found, we will migrate symlinked db file if needed
+      await fs.ensureDir(basePath);
+      const dbPath = await getWorkspaceDBPath(workspaceId);
+
+      // todo: remove this after migration (in stable version)
+      const realDBPath = (await fs
+        .access(dbPath)
+        .then(() => true)
+        .catch(() => false))
+        ? await fs.realpath(dbPath)
+        : dbPath;
+      const isLink = realDBPath !== dbPath;
+      if (isLink) {
+        await fs.copy(realDBPath, dbPath);
+      }
+      // create one if not exists
+      const meta = {
+        id: workspaceId,
+        mainDBPath: dbPath,
+        secondaryDBPath: isLink ? realDBPath : undefined,
+      };
+      await fs.writeJSON(metaPath, meta);
+      return meta;
+    } else {
+      const meta = await fs.readJSON(metaPath);
+      return meta;
+    }
+  } catch (err) {
+    logger.error('getWorkspaceMeta failed', err);
+    throw err;
+  }
+}

packages/frontend/electron/src/helper/workspace/subjects.ts

@@ -0,0 +1,7 @@
+import { Subject } from 'rxjs';
+
+import type { WorkspaceMeta } from '../type';
+
+export const workspaceSubjects = {
+  meta: new Subject<{ workspaceId: string; meta: WorkspaceMeta }>(),
+};