fix: some improvements to electron app (#2089)

.prettierignore

@@ -1,2 +1 @@
 pnpm-lock.yaml
-apps/electron/layers/preload/preload.d.ts

apps/electron/README.md

@@ -14,6 +14,24 @@ yarn generate-assets
 yarn dev # or yarn prod for production build
 ```
 
+## Troubleshooting
+
+### better-sqlite3 error
+
+When running tests or starting electron, you may encounter the following error:
+
+> Error: The module 'apps/electron/node_modules/better-sqlite3/build/Release/better_sqlite3.node'
+
+This is due to the fact that the `better-sqlite3` package is built for the Node.js version in Electron & in your machine. To fix this, run the following command based on different cases:
+
+```sh
+# for running unit tests, we are not using Electron's node:
+yarn rebuild better-sqlite3
+
+# for running Electron, we are using Electron's node:
+yarn postinstall
+```
+
 ## Credits
 
 Most of the boilerplate code is generously borrowed from the following

apps/electron/layers/main-events.ts

@@ -0,0 +1,5 @@
+// This file contains the main process events
+// It will guide preload and main process on the correct event types and payloads
+export interface MainEventMap {
+  'main:on-db-update': (workspaceId: string) => void;
+}

apps/electron/layers/main/src/__tests__/.gitignore

@@ -0,0 +1 @@
+tmp

apps/electron/layers/main/src/__tests__/handlers.spec.ts

@@ -0,0 +1,226 @@
+import assert from 'node:assert';
+import path from 'node:path';
+
+import fs from 'fs-extra';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import * as Y from 'yjs';
+
+const registeredHandlers = new Map<string, (...args: any[]) => any>();
+
+// common mock dispatcher for ipcMain.handle and app.on
+async function dispatch(key: string, ...args: any[]) {
+  const handler = registeredHandlers.get(key);
+  assert(handler);
+  return await handler(null, ...args);
+}
+
+const APP_PATH = path.join(__dirname, './tmp');
+
+const browserWindow = {
+  isDestroyed: () => {
+    return false;
+  },
+  setWindowButtonVisibility: (v: boolean) => {
+    // will be stubbed later
+  },
+  webContents: {
+    send: (type: string, ...args: any[]) => {
+      // ...
+    },
+  },
+};
+
+const ipcMain = {
+  handle: (key: string, callback: (...args: any[]) => any) => {
+    registeredHandlers.set(key, callback);
+  },
+};
+
+const nativeTheme = {
+  themeSource: 'light',
+};
+
+function compareBuffer(a: Uint8Array, b: Uint8Array) {
+  if (a.length !== b.length) {
+    return false;
+  }
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// dynamically import handlers so that we can inject local variables to mocks
+vi.doMock('electron', () => {
+  return {
+    app: {
+      getPath: (name: string) => {
+        assert(name === 'appData');
+        return APP_PATH;
+      },
+      name: 'affine-test',
+      on: (name: string, callback: (...args: any[]) => any) => {
+        registeredHandlers.set(name, callback);
+      },
+    },
+    BrowserWindow: {
+      getAllWindows: () => {
+        return [browserWindow];
+      },
+    },
+    nativeTheme: nativeTheme,
+    ipcMain,
+  };
+});
+
+beforeEach(async () => {
+  // clean up tmp folder
+  const { registerHandlers } = await import('../handlers');
+  registerHandlers();
+});
+
+afterEach(async () => {
+  const { cleanupWorkspaceDBs } = await import('../handlers');
+  cleanupWorkspaceDBs();
+  await fs.remove(APP_PATH);
+});
+
+describe('ensureWorkspaceDB', () => {
+  test('should create db file on connection if it does not exist', async () => {
+    const id = 'test-workspace-id';
+    const { ensureWorkspaceDB } = await import('../handlers');
+    const workspaceDB = await ensureWorkspaceDB(id);
+    const file = workspaceDB.path;
+    const fileExists = await fs.pathExists(file);
+    expect(fileExists).toBe(true);
+  });
+});
+
+describe('workspace handlers', () => {
+  test('list all workspace ids', async () => {
+    const ids = ['test-workspace-id', 'test-workspace-id-2'];
+    const { ensureWorkspaceDB } = await import('../handlers');
+    await Promise.all(ids.map(id => ensureWorkspaceDB(id)));
+    const list = await dispatch('workspace:list');
+    expect(list).toEqual(ids);
+  });
+
+  test('delete workspace', async () => {
+    const ids = ['test-workspace-id', 'test-workspace-id-2'];
+    const { ensureWorkspaceDB } = await import('../handlers');
+    await Promise.all(ids.map(id => ensureWorkspaceDB(id)));
+    await dispatch('workspace:delete', 'test-workspace-id-2');
+    const list = await dispatch('workspace:list');
+    expect(list).toEqual(['test-workspace-id']);
+  });
+});
+
+describe('UI handlers', () => {
+  test('theme-change', async () => {
+    await dispatch('ui:theme-change', 'dark');
+    expect(nativeTheme.themeSource).toBe('dark');
+    await dispatch('ui:theme-change', 'light');
+    expect(nativeTheme.themeSource).toBe('light');
+  });
+
+  test('sidebar-visibility-change (macOS)', async () => {
+    vi.stubGlobal('process', { platform: 'darwin' });
+    const setWindowButtonVisibility = vi.fn();
+    browserWindow.setWindowButtonVisibility = setWindowButtonVisibility;
+    await dispatch('ui:sidebar-visibility-change', true);
+    expect(setWindowButtonVisibility).toBeCalledWith(true);
+    await dispatch('ui:sidebar-visibility-change', false);
+    expect(setWindowButtonVisibility).toBeCalledWith(false);
+    vi.unstubAllGlobals();
+  });
+
+  test('sidebar-visibility-change (non-macOS)', async () => {
+    vi.stubGlobal('process', { platform: 'linux' });
+    const setWindowButtonVisibility = vi.fn();
+    browserWindow.setWindowButtonVisibility = setWindowButtonVisibility;
+    await dispatch('ui:sidebar-visibility-change', true);
+    expect(setWindowButtonVisibility).not.toBeCalled();
+    vi.unstubAllGlobals();
+  });
+});
+
+describe('db handlers', () => {
+  test('will reconnect on activate', async () => {
+    const { ensureWorkspaceDB } = await import('../handlers');
+    const workspaceDB = await ensureWorkspaceDB('test-workspace-id');
+    const instance = vi.spyOn(workspaceDB, 'reconnectDB');
+    await dispatch('activate');
+    expect(instance).toBeCalled();
+  });
+
+  test('apply doc and get doc updates', async () => {
+    const workspaceId = 'test-workspace-id';
+    const bin = await dispatch('db:get-doc', workspaceId);
+    // ? is this a good test?
+    expect(bin.every((byte: number) => byte === 0)).toBe(true);
+
+    const ydoc = new Y.Doc();
+    const ytext = ydoc.getText('test');
+    ytext.insert(0, 'hello world');
+    const bin2 = Y.encodeStateAsUpdate(ydoc);
+
+    await dispatch('db:apply-doc-update', workspaceId, bin2);
+
+    const bin3 = await dispatch('db:get-doc', workspaceId);
+    const ydoc2 = new Y.Doc();
+    Y.applyUpdate(ydoc2, bin3);
+    const ytext2 = ydoc2.getText('test');
+    expect(ytext2.toString()).toBe('hello world');
+  });
+
+  test('get non existent doc', async () => {
+    const workspaceId = 'test-workspace-id';
+    const bin = await dispatch('db:get-blob', workspaceId, 'non-existent-id');
+    expect(bin).toBeNull();
+  });
+
+  test('list blobs (empty)', async () => {
+    const workspaceId = 'test-workspace-id';
+    const list = await dispatch('db:get-persisted-blobs', workspaceId);
+    expect(list).toEqual([]);
+  });
+
+  test('CRUD blobs', async () => {
+    const testBin = new Uint8Array([1, 2, 3, 4, 5]);
+    const testBin2 = new Uint8Array([6, 7, 8, 9, 10]);
+    const workspaceId = 'test-workspace-id';
+
+    // add blob
+    await dispatch('db:add-blob', workspaceId, 'testBin', testBin);
+
+    // get blob
+    expect(
+      compareBuffer(
+        await dispatch('db:get-blob', workspaceId, 'testBin'),
+        testBin
+      )
+    ).toBe(true);
+
+    // add another blob
+    await dispatch('db:add-blob', workspaceId, 'testBin2', testBin2);
+    expect(
+      compareBuffer(
+        await dispatch('db:get-blob', workspaceId, 'testBin2'),
+        testBin2
+      )
+    ).toBe(true);
+
+    // list blobs
+    let lists = await dispatch('db:get-persisted-blobs', workspaceId);
+    expect(lists).toHaveLength(2);
+    expect(lists).toContain('testBin');
+    expect(lists).toContain('testBin2');
+
+    // delete blob
+    await dispatch('db:delete-blob', workspaceId, 'testBin');
+    lists = await dispatch('db:get-persisted-blobs', workspaceId);
+    expect(lists).toEqual(['testBin2']);
+  });
+});

apps/electron/layers/main/src/data/fs-watch.ts

@@ -0,0 +1,7 @@
+import type { WatchListener } from 'fs-extra';
+import fs from 'fs-extra';
+
+export function watchFile(path: string, callback: WatchListener<string>) {
+  const watcher = fs.watch(path, callback);
+  return () => watcher.close();
+}

apps/electron/layers/main/src/data/sqlite.ts

@@ -36,24 +36,10 @@ interface BlobRow {
 export class WorkspaceDatabase {
   sqliteDB: Database;
   ydoc = new Y.Doc();
-
+  firstConnect = false;
-  ready: Promise<Uint8Array>;
 
   constructor(public path: string) {
     this.sqliteDB = this.reconnectDB();
-    logger.log('open db', path);
-
-    this.ydoc.on('update', update => {
-      this.addUpdateToSQLite(update);
-    });
-
-    this.ready = (async () => {
-      const updates = await this.getUpdates();
-      updates.forEach(update => {
-        Y.applyUpdate(this.ydoc, update.data);
-      });
-      return this.getEncodedDocUpdates();
-    })();
   }
 
   // release resources
@@ -67,15 +53,22 @@ export class WorkspaceDatabase {
     if (this.sqliteDB) {
       this.sqliteDB.close();
     }
+
     // use cached version?
-    const db = sqlite(this.path);
+    const db = (this.sqliteDB = sqlite(this.path));
-    // const db = new sqlite.Database(this.path, error => {
-    //   if (error) {
-    //     logger.error('open db error', error);
-    //   }
-    // });
-    this.sqliteDB = db;
     db.exec(schemas.join(';'));
+
+    if (!this.firstConnect) {
+      this.ydoc.on('update', this.addUpdateToSQLite);
+    }
+
+    const updates = this.getUpdates();
+    updates.forEach(update => {
+      Y.applyUpdate(this.ydoc, update.data);
+    });
+
+    this.firstConnect = true;
+
     return db;
   };
 
@@ -153,12 +146,15 @@ export class WorkspaceDatabase {
     }
   };
 
+  // batch write instead write per key stroke?
   private addUpdateToSQLite = (data: Uint8Array) => {
     try {
+      const start = performance.now();
       const statement = this.sqliteDB.prepare(
         'INSERT INTO updates (data) VALUES (?)'
       );
       statement.run(data);
+      logger.debug('addUpdateToSQLite', performance.now() - start, 'ms');
     } catch (error) {
       logger.error('addUpdateToSQLite', error);
     }

apps/electron/layers/main/src/handlers.ts

@@ -12,26 +12,69 @@ import { logger } from '../../logger';
 import { isMacOS } from '../../utils';
 import { appContext } from './context';
 import { exportDatabase } from './data/export';
+import { watchFile } from './data/fs-watch';
 import type { WorkspaceDatabase } from './data/sqlite';
 import { openWorkspaceDatabase } from './data/sqlite';
 import { deleteWorkspace, listWorkspaces } from './data/workspace';
 import { getExchangeTokenParams, oauthEndpoint } from './google-auth';
+import { sendMainEvent } from './send-main-event';
 
 let currentWorkspaceId = '';
 
 const dbMapping = new Map<string, WorkspaceDatabase>();
+const dbWatchers = new Map<string, () => void>();
+const dBLastUse = new Map<string, number>();
 
-async function ensureWorkspaceDB(id: string) {
+export async function ensureWorkspaceDB(id: string) {
   let workspaceDB = dbMapping.get(id);
   if (!workspaceDB) {
     // hmm... potential race condition?
     workspaceDB = await openWorkspaceDatabase(appContext, id);
     dbMapping.set(id, workspaceDB);
+
+    logger.info('watch db file', workspaceDB.path);
+
+    dbWatchers.set(
+      id,
+      watchFile(workspaceDB.path, (event, filename) => {
+        const minTime = 1000;
+        logger.debug(
+          'db file changed',
+          event,
+          filename,
+          Date.now() - dBLastUse.get(id)!
+        );
+
+        if (Date.now() - dBLastUse.get(id)! < minTime || !filename) {
+          logger.debug('skip db update');
+          return;
+        }
+
+        sendMainEvent('main:on-db-update', id);
+
+        // handle DB file update by other process
+        dbWatchers.get(id)?.();
+        dbMapping.delete(id);
+        dbWatchers.delete(id);
+        ensureWorkspaceDB(id);
+      })
+    );
   }
-  await workspaceDB.ready;
+  dBLastUse.set(id, Date.now());
   return workspaceDB;
 }
 
+export async function cleanupWorkspaceDBs() {
+  for (const [id, db] of dbMapping) {
+    logger.info('close db connection', id);
+    db.destroy();
+    dbWatchers.get(id)?.();
+  }
+  dbMapping.clear();
+  dbWatchers.clear();
+  dBLastUse.clear();
+}
+
 function registerWorkspaceHandlers() {
   ipcMain.handle('workspace:list', async _ => {
     logger.info('list workspaces');

apps/electron/layers/main/src/send-main-event.ts

@@ -0,0 +1,14 @@
+import { BrowserWindow } from 'electron';
+
+import type { MainEventMap } from '../../main-events';
+
+function getActiveWindows() {
+  return BrowserWindow.getAllWindows().filter(win => !win.isDestroyed());
+}
+
+export function sendMainEvent<T extends keyof MainEventMap>(
+  type: T,
+  ...args: Parameters<MainEventMap[T]>
+) {
+  getActiveWindows().forEach(win => win.webContents.send(type, ...args));
+}

apps/electron/layers/preload/preload.d.ts

@@ -1,12 +1,6 @@
+/* eslint-disable @typescript-eslint/consistent-type-imports */
+
 interface Window {
-    /**
+  apis: typeof import('./src/affine-apis').apis;
-     * After analyzing the `exposeInMainWorld` calls,
+  appInfo: typeof import('./src/affine-apis').appInfo;
-     * `packages/preload/exposedInMainWorld.d.ts` file will be generated.
-     * It contains all interfaces.
-     * `packages/preload/exposedInMainWorld.d.ts` file is required for TS is `renderer`
-     *
-     * @see https://github.com/cawa-93/dts-for-context-bridge
-     */
-    readonly apis: { db: { getDoc: (id: string) => Promise<Uint8Array>; applyDocUpdate: (id: string, update: Uint8Array) => Promise<any>; addBlob: (workspaceId: string, key: string, data: Uint8Array) => Promise<any>; getBlob: (workspaceId: string, key: string) => Promise<Uint8Array>; deleteBlob: (workspaceId: string, key: string) => Promise<any>; getPersistedBlobs: (workspaceId: string) => Promise<string[]>; }; workspace: { list: () => Promise<string[]>; delete: (id: string) => Promise<void>; }; openLoadDBFileDialog: () => Promise<any>; openSaveDBFileDialog: () => Promise<any>; onThemeChange: (theme: string) => Promise<any>; onSidebarVisibilityChange: (visible: boolean) => Promise<any>; onWorkspaceChange: (workspaceId: string) => Promise<any>; openDBFolder: () => Promise<any>; getGoogleOauthCode: () => Promise<{ requestInit: RequestInit; url: string; }>; updateEnv: (env: string, value: string) => void; };
-    readonly appInfo: { electron: boolean; isMacOS: boolean; };
 }

apps/electron/layers/preload/src/affine-apis.ts

@@ -0,0 +1,81 @@
+// NOTE: we will generate preload types from this file
+
+import { ipcRenderer } from 'electron';
+
+import type { MainEventMap } from '../../main-events';
+
+// main -> renderer
+function onMainEvent<T extends keyof MainEventMap>(
+  eventName: T,
+  callback: MainEventMap[T]
+): () => void {
+  // @ts-expect-error fix me later
+  const fn = (_, ...args) => callback(...args);
+  ipcRenderer.on(eventName, fn);
+  return () => ipcRenderer.off(eventName, fn);
+}
+
+const apis = {
+  db: {
+    // workspace providers
+    getDoc: (id: string): Promise<Uint8Array | null> =>
+      ipcRenderer.invoke('db:get-doc', id),
+    applyDocUpdate: (id: string, update: Uint8Array) =>
+      ipcRenderer.invoke('db:apply-doc-update', id, update),
+    addBlob: (workspaceId: string, key: string, data: Uint8Array) =>
+      ipcRenderer.invoke('db:add-blob', workspaceId, key, data),
+    getBlob: (workspaceId: string, key: string): Promise<Uint8Array | null> =>
+      ipcRenderer.invoke('db:get-blob', workspaceId, key),
+    deleteBlob: (workspaceId: string, key: string) =>
+      ipcRenderer.invoke('db:delete-blob', workspaceId, key),
+    getPersistedBlobs: (workspaceId: string): Promise<string[]> =>
+      ipcRenderer.invoke('db:get-persisted-blobs', workspaceId),
+
+    // listeners
+    onDBUpdate: (callback: (workspaceId: string) => void) => {
+      return onMainEvent('main:on-db-update', callback);
+    },
+  },
+
+  workspace: {
+    list: (): Promise<string[]> => ipcRenderer.invoke('workspace:list'),
+    delete: (id: string): Promise<void> =>
+      ipcRenderer.invoke('workspace:delete', id),
+    // create will be implicitly called by db functions
+  },
+
+  openLoadDBFileDialog: () => ipcRenderer.invoke('ui:open-load-db-file-dialog'),
+  openSaveDBFileDialog: () => ipcRenderer.invoke('ui:open-save-db-file-dialog'),
+
+  // ui
+  onThemeChange: (theme: string) =>
+    ipcRenderer.invoke('ui:theme-change', theme),
+
+  onSidebarVisibilityChange: (visible: boolean) =>
+    ipcRenderer.invoke('ui:sidebar-visibility-change', visible),
+
+  onWorkspaceChange: (workspaceId: string) =>
+    ipcRenderer.invoke('ui:workspace-change', workspaceId),
+
+  openDBFolder: () => ipcRenderer.invoke('ui:open-db-folder'),
+
+  /**
+   * Try sign in using Google and return a request object to exchange the code for a token
+   * Not exchange in Node side because it is easier to do it in the renderer with VPN
+   */
+  getGoogleOauthCode: (): Promise<{ requestInit: RequestInit; url: string }> =>
+    ipcRenderer.invoke('ui:get-google-oauth-code'),
+
+  /**
+   * Secret backdoor to update environment variables in main process
+   */
+  updateEnv: (env: string, value: string) => {
+    ipcRenderer.invoke('main:env-update', env, value);
+  },
+};
+
+const appInfo = {
+  electron: true,
+};
+
+export { apis, appInfo };

apps/electron/layers/preload/src/index.ts

@@ -2,9 +2,9 @@
  * @module preload
  */
 
-import { contextBridge, ipcRenderer } from 'electron';
+import { contextBridge } from 'electron';
 
-import { isMacOS } from '../../utils';
+import * as affineApis from './affine-apis';
 
 /**
  * The "Main World" is the JavaScript context that your main renderer code runs in.
@@ -13,69 +13,5 @@ import { isMacOS } from '../../utils';
  * @see https://www.electronjs.org/docs/api/context-bridge
  */
 
-/**
+contextBridge.exposeInMainWorld('apis', affineApis.apis);
- * After analyzing the `exposeInMainWorld` calls,
+contextBridge.exposeInMainWorld('appInfo', affineApis.appInfo);
- * `packages/preload/exposedInMainWorld.d.ts` file will be generated.
- * It contains all interfaces.
- * `packages/preload/exposedInMainWorld.d.ts` file is required for TS is `renderer`
- *
- * @see https://github.com/cawa-93/dts-for-context-bridge
- */
-contextBridge.exposeInMainWorld('apis', {
-  db: {
-    // workspace providers
-    getDoc: (id: string): Promise<Uint8Array | null> =>
-      ipcRenderer.invoke('db:get-doc', id),
-    applyDocUpdate: (id: string, update: Uint8Array) =>
-      ipcRenderer.invoke('db:apply-doc-update', id, update),
-    addBlob: (workspaceId: string, key: string, data: Uint8Array) =>
-      ipcRenderer.invoke('db:add-blob', workspaceId, key, data),
-    getBlob: (workspaceId: string, key: string): Promise<Uint8Array | null> =>
-      ipcRenderer.invoke('db:get-blob', workspaceId, key),
-    deleteBlob: (workspaceId: string, key: string) =>
-      ipcRenderer.invoke('db:delete-blob', workspaceId, key),
-    getPersistedBlobs: (workspaceId: string): Promise<string[]> =>
-      ipcRenderer.invoke('db:get-persisted-blobs', workspaceId),
-  },
-
-  workspace: {
-    list: (): Promise<string[]> => ipcRenderer.invoke('workspace:list'),
-    delete: (id: string): Promise<void> =>
-      ipcRenderer.invoke('workspace:delete', id),
-    // create will be implicitly called by db functions
-  },
-
-  openLoadDBFileDialog: () => ipcRenderer.invoke('ui:open-load-db-file-dialog'),
-  openSaveDBFileDialog: () => ipcRenderer.invoke('ui:open-save-db-file-dialog'),
-
-  // ui
-  onThemeChange: (theme: string) =>
-    ipcRenderer.invoke('ui:theme-change', theme),
-
-  onSidebarVisibilityChange: (visible: boolean) =>
-    ipcRenderer.invoke('ui:sidebar-visibility-change', visible),
-
-  onWorkspaceChange: (workspaceId: string) =>
-    ipcRenderer.invoke('ui:workspace-change', workspaceId),
-
-  openDBFolder: () => ipcRenderer.invoke('ui:open-db-folder'),
-
-  /**
-   * Try sign in using Google and return a request object to exchange the code for a token
-   * Not exchange in Node side because it is easier to do it in the renderer with VPN
-   */
-  getGoogleOauthCode: (): Promise<{ requestInit: RequestInit; url: string }> =>
-    ipcRenderer.invoke('ui:get-google-oauth-code'),
-
-  /**
-   * Secret backdoor to update environment variables in main process
-   */
-  updateEnv: (env: string, value: string) => {
-    ipcRenderer.invoke('main:env-update', env, value);
-  },
-});
-
-contextBridge.exposeInMainWorld('appInfo', {
-  electron: true,
-  isMacOS: isMacOS(),
-});

apps/electron/package.json

@@ -6,8 +6,8 @@
   "description": "AFFiNE App",
   "homepage": "https://github.com/toeverything/AFFiNE",
   "scripts": {
-    "dev": "cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs",
+    "dev": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs",
-    "prod": "node scripts/dev.mjs",
+    "prod": "yarn electron-rebuild && yarn node scripts/dev.mjs",
     "generate-assets": "zx scripts/generate-assets.mjs",
     "package": "electron-forge package",
     "make": "electron-forge make",
@@ -15,7 +15,8 @@
     "make-macos-x64": "electron-forge make --platform=darwin --arch=x64",
     "make-windows-x64": "electron-forge make --platform=win32 --arch=x64",
     "make-linux-x64": "electron-forge make --platform=linux --arch=x64",
-    "electron-rebuild": "electron-rebuild"
+    "rebuild:for-test": "yarn rebuild better-sqlite3",
+    "rebuild:for-electron": "yarn electron-rebuild"
   },
   "config": {
     "forge": "./forge.config.js"
@@ -35,7 +36,6 @@
     "@types/better-sqlite3": "^7.6.4",
     "@types/fs-extra": "^11.0.1",
     "cross-env": "7.0.3",
-    "dts-for-context-bridge": "^0.7.1",
     "electron": "24.1.2",
     "electron-log": "^5.0.0-beta.23",
     "electron-squirrel-startup": "1.0.0",

apps/electron/scripts/dev.mjs

@@ -3,7 +3,6 @@ import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-import { generateAsync } from 'dts-for-context-bridge';
 import electronPath from 'electron';
 import * as esbuild from 'esbuild';
 
@@ -81,10 +80,6 @@ async function main() {
           setup(build) {
             let initialBuild = false;
             build.onEnd(() => {
-              generateAsync({
-                input: 'layers/preload/src/**/*.ts',
-                output: 'layers/preload/preload.d.ts',
-              });
               if (initialBuild) {
                 console.log(`[preload] has changed`);
                 spawnOrReloadElectron();

packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts

@@ -14,6 +14,10 @@ let provider: SQLiteProvider;
 
 let offlineYdoc: YType.Doc;
 
+let triggerDBUpdate: ((_: string) => void) | null = null;
+
+const mockedAddBlob = vi.fn();
+
 vi.stubGlobal('window', {
   apis: {
     db: {
@@ -24,8 +28,16 @@ vi.stubGlobal('window', {
         Y.applyUpdate(offlineYdoc, update, 'sqlite');
       },
       getPersistedBlobs: async (id: string) => {
+        // todo: may need to hack the way to get hash keys of blobs
         return [];
       },
+      onDBUpdate: (fn: (id: string) => void) => {
+        triggerDBUpdate = fn;
+        return () => {
+          triggerDBUpdate = null;
+        };
+      },
+      addBlob: mockedAddBlob,
     } satisfies Partial<typeof window.apis.db>,
   },
 });
@@ -43,7 +55,7 @@ beforeEach(() => {
   workspace.register(AffineSchemas).register(__unstableSchemas);
   provider = createSQLiteProvider(workspace);
   offlineYdoc = new Y.Doc();
-  offlineYdoc.getText('text').insert(0, '');
+  offlineYdoc.getText('text').insert(0, 'sqlite-hello');
 });
 
 describe('SQLite provider', () => {
@@ -53,19 +65,71 @@ describe('SQLite provider', () => {
     // Workspace.Y.applyUpdate(workspace.doc);
     workspace.doc.getText('text').insert(0, 'mem-hello');
 
-    expect(offlineYdoc.getText('text').toString()).toBe('');
+    expect(offlineYdoc.getText('text').toString()).toBe('sqlite-hello');
 
     await provider.connect();
 
-    expect(offlineYdoc.getText('text').toString()).toBe('mem-hello');
+    // depending on the nature of the sync, the data can be sync'ed in either direction
-    expect(workspace.doc.getText('text').toString()).toBe('mem-hello');
+    const options = ['mem-hellosqlite-hello', 'sqlite-hellomem-hello'];
+    const synced = options.filter(
+      o => o === offlineYdoc.getText('text').toString()
+    );
+    expect(synced.length).toBe(1);
+    expect(workspace.doc.getText('text').toString()).toBe(synced[0]);
 
     workspace.doc.getText('text').insert(0, 'world');
 
     // check if the data are sync'ed
-    expect(offlineYdoc.getText('text').toString()).toBe('worldmem-hello');
+    expect(offlineYdoc.getText('text').toString()).toBe('world' + synced[0]);
   });
 
-  // todo: test disconnect
+  test('blobs will be synced to sqlite on connect', async () => {
-  // todo: test blob sync
+    // mock bs.list
+    const bin = new Uint8Array([1, 2, 3]);
+    const blob = new Blob([bin]);
+    workspace.blobs.list = vi.fn(async () => ['blob1']);
+    workspace.blobs.get = vi.fn(async (key: string) => {
+      return blob;
+    });
+
+    await provider.connect();
+    await new Promise(resolve => setTimeout(resolve, 100));
+
+    expect(mockedAddBlob).toBeCalledWith(id, 'blob1', bin);
+  });
+
+  test('on db update', async () => {
+    vi.useFakeTimers();
+    await provider.connect();
+
+    offlineYdoc.getText('text').insert(0, 'sqlite-world');
+
+    triggerDBUpdate?.(id);
+
+    // not yet updated
+    expect(workspace.doc.getText('text').toString()).toBe('sqlite-hello');
+
+    // wait for the update to be sync'ed
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(workspace.doc.getText('text').toString()).toBe(
+      'sqlite-worldsqlite-hello'
+    );
+    vi.useRealTimers();
+  });
+
+  test('disconnect handlers', async () => {
+    const offHandler = vi.fn();
+    let handleUpdate = () => {};
+    workspace.doc.on = (_: string, fn: () => void) => {
+      handleUpdate = fn;
+    };
+    workspace.doc.off = offHandler;
+    await provider.connect();
+
+    provider.disconnect();
+
+    expect(triggerDBUpdate).toBe(null);
+    expect(offHandler).toBeCalledWith('update', handleUpdate);
+  });
 });

packages/workspace/src/providers/index.ts

@@ -169,7 +169,7 @@ const createSQLiteProvider = (
     keysToPersist.forEach(async k => {
       const blob = await bs.get(k);
       if (!blob) {
-        logger.warn('blob url not found', k);
+        logger.warn('blob not found for', k);
         return;
       }
       window.apis.db.addBlob(
@@ -180,6 +180,29 @@ const createSQLiteProvider = (
     });
   }
 
+  async function syncUpdates() {
+    logger.info('syncing updates from sqlite', blockSuiteWorkspace.id);
+    const updates = await window.apis.db.getDoc(blockSuiteWorkspace.id);
+
+    if (updates) {
+      Y.applyUpdate(blockSuiteWorkspace.doc, updates, sqliteOrigin);
+    }
+
+    const mergeUpdates = Y.encodeStateAsUpdate(blockSuiteWorkspace.doc);
+
+    // also apply updates to sqlite
+    window.apis.db.applyDocUpdate(blockSuiteWorkspace.id, mergeUpdates);
+
+    const bs = blockSuiteWorkspace.blobs;
+
+    if (bs) {
+      // this can be non-blocking
+      syncBlobIntoSQLite(bs);
+    }
+  }
+
+  let unsubscribe = () => {};
+
   const provider = {
     flavour: 'sqlite',
     background: true,
@@ -188,31 +211,32 @@ const createSQLiteProvider = (
     },
     connect: async () => {
       logger.info('connecting sqlite provider', blockSuiteWorkspace.id);
-      const updates = await window.apis.db.getDoc(blockSuiteWorkspace.id);
+      await syncUpdates();
-
-      if (updates) {
-        Y.applyUpdate(blockSuiteWorkspace.doc, updates, sqliteOrigin);
-      }
-
-      const mergeUpdates = Y.encodeStateAsUpdate(blockSuiteWorkspace.doc);
-
-      // also apply updates to sqlite
-      window.apis.db.applyDocUpdate(blockSuiteWorkspace.id, mergeUpdates);
 
       blockSuiteWorkspace.doc.on('update', handleUpdate);
 
-      const bs = blockSuiteWorkspace.blobs;
+      let timer = 0;
-
+      unsubscribe = window.apis.db.onDBUpdate(workspaceId => {
-      if (bs) {
+        if (workspaceId === blockSuiteWorkspace.id) {
-        // this can be non-blocking
+          // throttle
-        syncBlobIntoSQLite(bs);
+          logger.debug('on db update', workspaceId);
-      }
+          if (timer) {
+            clearTimeout(timer);
+          }
+          // @ts-expect-error ignore the type
+          timer = setTimeout(() => {
+            syncUpdates();
+            timer = 0;
+          }, 1000);
+        }
+      });
 
       // blockSuiteWorkspace.doc.on('destroy', ...);
       logger.info('connecting sqlite done', blockSuiteWorkspace.id);
     },
     disconnect: () => {
-      // todo: not implemented
+      unsubscribe();
+      blockSuiteWorkspace.doc.off('update', handleUpdate);
     },
   } satisfies SQLiteProvider;
 

vitest.config.ts

@@ -26,6 +26,7 @@ export default defineConfig({
       'packages/**/*.spec.tsx',
       'apps/web/**/*.spec.ts',
       'apps/web/**/*.spec.tsx',
+      'apps/electron/**/*.spec.ts',
       'tests/unit/**/*.spec.ts',
       'tests/unit/**/*.spec.tsx',
     ],

yarn.lock

@@ -126,7 +126,6 @@ __metadata:
     "@types/fs-extra": ^11.0.1
     better-sqlite3: ^8.3.0
     cross-env: 7.0.3
-    dts-for-context-bridge: ^0.7.1
     electron: 24.1.2
     electron-log: ^5.0.0-beta.23
     electron-squirrel-startup: 1.0.0
@@ -7398,18 +7397,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@ts-morph/common@npm:~0.12.2":
-  version: 0.12.3
-  resolution: "@ts-morph/common@npm:0.12.3"
-  dependencies:
-    fast-glob: ^3.2.7
-    minimatch: ^3.0.4
-    mkdirp: ^1.0.4
-    path-browserify: ^1.0.1
-  checksum: d96ea9805d4f0300cc05c47daa9454438903b86ffb7116f5181a1eba71e881012a1adc2a867b3afbe4429ef29e3e0d6204175cbaf33ecdd7a7d09b5d8a37f12d
-  languageName: node
-  linkType: hard
-
 "@ts-morph/common@npm:~0.19.0":
   version: 0.19.0
   resolution: "@ts-morph/common@npm:0.19.0"
@@ -7959,13 +7946,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/q@npm:^1.5.1":
-  version: 1.5.5
-  resolution: "@types/q@npm:1.5.5"
-  checksum: 3bd386fb97a0e5f1ce1ed7a14e39b60e469b5ca9d920a7f69e0cdb58d22c0f5bdd16637d8c3a5bfeda76663c023564dd47a65389ee9aaabd65aee54803d5ba45
-  languageName: node
-  linkType: hard
-
 "@types/qs@npm:*, @types/qs@npm:^6.9.5":
   version: 6.9.7
   resolution: "@types/qs@npm:6.9.7"
@@ -10155,7 +10135,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"chalk@npm:^2.0.0, chalk@npm:^2.4.1, chalk@npm:^2.4.2":
+"chalk@npm:^2.0.0, chalk@npm:^2.4.2":
   version: 2.4.2
   resolution: "chalk@npm:2.4.2"
   dependencies:
@@ -10486,24 +10466,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"coa@npm:2.0.2":
-  version: 2.0.2
-  resolution: "coa@npm:2.0.2"
-  dependencies:
-    "@types/q": ^1.5.1
-    chalk: ^2.4.1
-    q: ^1.1.2
-  checksum: 44736914aac2160d3d840ed64432a90a3bb72285a0cd6a688eb5cabdf15d15a85eee0915b3f6f2a4659d5075817b1cb577340d3c9cbb47d636d59ab69f819552
-  languageName: node
-  linkType: hard
-
-"code-block-writer@npm:^11.0.0":
-  version: 11.0.3
-  resolution: "code-block-writer@npm:11.0.3"
-  checksum: f0a2605f19963d7087267c9b0fd0b05a6638a50e7b29b70f97aa01a514f59475b0626f8aa092188df853ee6d96745426dfa132d6a677795df462c6ce32c21639
-  languageName: node
-  linkType: hard
-
 "code-block-writer@npm:^12.0.0":
   version: 12.0.0
   resolution: "code-block-writer@npm:12.0.0"
@@ -11678,18 +11640,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"dts-for-context-bridge@npm:^0.7.1":
-  version: 0.7.1
-  resolution: "dts-for-context-bridge@npm:0.7.1"
-  dependencies:
-    coa: 2.0.2
-    ts-morph: 13.0.2
-  bin:
-    dts-cb: bin/index.js
-  checksum: cb910eecb82ac1b3d506990a244083389c74c90ab599382e174a980d247e905a1d49500e85806ede75f00b3848c14197a129ecaad436e7a5bfe0f3dcb8cd1859
-  languageName: node
-  linkType: hard
-
 "duplexer2@npm:^0.1.2":
   version: 0.1.4
   resolution: "duplexer2@npm:0.1.4"
@@ -19390,7 +19340,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"q@npm:^1.1.2, q@npm:^1.5.1":
+"q@npm:^1.5.1":
   version: 1.5.1
   resolution: "q@npm:1.5.1"
   checksum: 147baa93c805bc1200ed698bdf9c72e9e42c05f96d007e33a558b5fdfd63e5ea130e99313f28efc1783e90e6bdb4e48b67a36fcc026b7b09202437ae88a1fb12
@@ -22141,16 +22091,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ts-morph@npm:13.0.2":
-  version: 13.0.2
-  resolution: "ts-morph@npm:13.0.2"
-  dependencies:
-    "@ts-morph/common": ~0.12.2
-    code-block-writer: ^11.0.0
-  checksum: 325e850b99f96a71b57ced55e2bf687bdc6dc477070a946a47ca925633c62f201d2a706c6972e7bb6a37d7d9e0414b040fd46800d2fd86423185818fefe284a6
-  languageName: node
-  linkType: hard
-
 "ts-morph@npm:18.0.0":
   version: 18.0.0
   resolution: "ts-morph@npm:18.0.0"