feat(core): new worker workspace engine (#9257)

packages/frontend/apps/electron-renderer/package.json

@@ -12,11 +12,13 @@
     "@affine/core": "workspace:*",
     "@affine/electron-api": "workspace:*",
     "@affine/i18n": "workspace:*",
+    "@affine/nbstore": "workspace:*",
     "@emotion/react": "^11.14.0",
     "@sentry/react": "^8.44.0",
     "@toeverything/infra": "workspace:*",
     "@toeverything/theme": "^1.1.3",
     "@vanilla-extract/css": "^1.16.1",
+    "async-call-rpc": "^6.4.2",
     "next-themes": "^0.4.4",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",

packages/frontend/apps/electron-renderer/src/app.tsx

@@ -19,25 +19,26 @@ import { configureFindInPageModule } from '@affine/core/modules/find-in-page';
 import { GlobalContextService } from '@affine/core/modules/global-context';
 import { I18nProvider } from '@affine/core/modules/i18n';
 import { LifecycleService } from '@affine/core/modules/lifecycle';
-import { configureElectronStateStorageImpls } from '@affine/core/modules/storage';
+import {
+  configureElectronStateStorageImpls,
+  NbstoreProvider,
+} from '@affine/core/modules/storage';
 import {
   ClientSchemeProvider,
   PopupWindowProvider,
 } from '@affine/core/modules/url';
-import { configureSqliteUserspaceStorageProvider } from '@affine/core/modules/userspace';
 import {
   configureDesktopWorkbenchModule,
   WorkbenchService,
 } from '@affine/core/modules/workbench';
 import { WorkspacesService } from '@affine/core/modules/workspace';
-import {
-  configureBrowserWorkspaceFlavours,
-  configureSqliteWorkspaceEngineStorageProvider,
-} from '@affine/core/modules/workspace-engine';
+import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
 import createEmotionCache from '@affine/core/utils/create-emotion-cache';
 import { apis, events } from '@affine/electron-api';
+import { WorkerClient } from '@affine/nbstore/worker/client';
 import { CacheProvider } from '@emotion/react';
 import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
+import { OpClient } from '@toeverything/infra/op';
 import { Suspense } from 'react';
 import { RouterProvider } from 'react-router-dom';
 
@@ -71,14 +72,61 @@ const framework = new Framework();
 configureCommonModules(framework);
 configureElectronStateStorageImpls(framework);
 configureBrowserWorkspaceFlavours(framework);
-configureSqliteWorkspaceEngineStorageProvider(framework);
-configureSqliteUserspaceStorageProvider(framework);
 configureDesktopWorkbenchModule(framework);
 configureAppTabsHeaderModule(framework);
 configureFindInPageModule(framework);
 configureDesktopApiModule(framework);
 configureSpellCheckSettingModule(framework);
+framework.impl(NbstoreProvider, {
+  openStore(key, options) {
+    const { port1: portForOpClient, port2: portForWorker } =
+      new MessageChannel();
+    let portFromWorker: MessagePort | null = null;
+    let portId = crypto.randomUUID();
 
+    const handleMessage = (ev: MessageEvent) => {
+      if (
+        ev.data.type === 'electron:worker-connect' &&
+        ev.data.portId === portId
+      ) {
+        portFromWorker = ev.ports[0];
+        // connect portForWorker and portFromWorker
+        portFromWorker.addEventListener('message', ev => {
+          portForWorker.postMessage(ev.data);
+        });
+        portForWorker.addEventListener('message', ev => {
+          // oxlint-disable-next-line no-non-null-assertion
+          portFromWorker!.postMessage(ev.data);
+        });
+        portForWorker.start();
+        portFromWorker.start();
+      }
+    };
+
+    window.addEventListener('message', handleMessage);
+
+    // oxlint-disable-next-line no-non-null-assertion
+    apis!.worker.connectWorker(key, portId).catch(err => {
+      console.error('failed to connect worker', err);
+    });
+
+    const store = new WorkerClient(new OpClient(portForOpClient), options);
+    portForOpClient.start();
+    return {
+      store,
+      dispose: () => {
+        window.removeEventListener('message', handleMessage);
+        portForOpClient.close();
+        portForWorker.close();
+        portFromWorker?.close();
+        // oxlint-disable-next-line no-non-null-assertion
+        apis!.worker.disconnectWorker(key, portId).catch(err => {
+          console.error('failed to disconnect worker', err);
+        });
+      },
+    };
+  },
+});
 framework.impl(PopupWindowProvider, p => {
   const apis = p.get(DesktopApiService).api;
   return {

packages/frontend/apps/electron-renderer/src/background-worker/index.ts

@@ -0,0 +1,36 @@
+import '@affine/core/bootstrap/electron';
+
+import { apis } from '@affine/electron-api';
+import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel';
+import { cloudStorages } from '@affine/nbstore/cloud';
+import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite';
+import {
+  bindNativeDBV1Apis,
+  sqliteV1Storages,
+} from '@affine/nbstore/sqlite/v1';
+import {
+  WorkerConsumer,
+  type WorkerOps,
+} from '@affine/nbstore/worker/consumer';
+import { OpConsumer } from '@toeverything/infra/op';
+
+// oxlint-disable-next-line no-non-null-assertion
+bindNativeDBApis(apis!.nbstore);
+// oxlint-disable-next-line no-non-null-assertion
+bindNativeDBV1Apis(apis!.db);
+
+const worker = new WorkerConsumer([
+  ...sqliteStorages,
+  ...sqliteV1Storages,
+  ...broadcastChannelStorages,
+  ...cloudStorages,
+]);
+
+window.addEventListener('message', ev => {
+  if (ev.data.type === 'electron:worker-connect') {
+    const port = ev.ports[0];
+
+    const consumer = new OpConsumer<WorkerOps>(port);
+    worker.bindConsumer(consumer);
+  }
+});

packages/frontend/apps/electron-renderer/src/nbstore.ts

@@ -0,0 +1,96 @@
+import '@affine/core/bootstrap/electron';
+
+import type { ClientHandler } from '@affine/electron-api';
+import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel';
+import { cloudStorages } from '@affine/nbstore/cloud';
+import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite';
+import {
+  bindNativeDBV1Apis,
+  sqliteV1Storages,
+} from '@affine/nbstore/sqlite/v1';
+import {
+  WorkerConsumer,
+  type WorkerOps,
+} from '@affine/nbstore/worker/consumer';
+import { OpConsumer } from '@toeverything/infra/op';
+import { AsyncCall } from 'async-call-rpc';
+
+const worker = new WorkerConsumer([
+  ...sqliteStorages,
+  ...sqliteV1Storages,
+  ...broadcastChannelStorages,
+  ...cloudStorages,
+]);
+
+let activeConnectionCount = 0;
+let electronAPIsInitialized = false;
+
+function connectElectronAPIs(port: MessagePort) {
+  if (electronAPIsInitialized) {
+    return;
+  }
+  electronAPIsInitialized = true;
+  port.postMessage({ type: '__electron-apis-init__' });
+
+  const { promise, resolve } = Promise.withResolvers<MessagePort>();
+  port.addEventListener('message', event => {
+    if (event.data.type === '__electron-apis__') {
+      const [port] = event.ports;
+      resolve(port);
+    }
+  });
+
+  const rpc = AsyncCall<Record<string, any>>(null, {
+    channel: promise.then(p => ({
+      on(listener) {
+        p.onmessage = e => {
+          listener(e.data);
+        };
+        p.start();
+        return () => {
+          p.onmessage = null;
+          try {
+            p.close();
+          } catch (err) {
+            console.error('close port error', err);
+          }
+        };
+      },
+      send(data) {
+        p.postMessage(data);
+      },
+    })),
+    log: false,
+  });
+
+  const electronAPIs = new Proxy<ClientHandler>(rpc as any, {
+    get(_, namespace: string) {
+      return new Proxy(rpc as any, {
+        get(_, method: string) {
+          return rpc[`${namespace}:${method}`];
+        },
+      });
+    },
+  });
+
+  bindNativeDBApis(electronAPIs.nbstore);
+  bindNativeDBV1Apis(electronAPIs.db);
+}
+
+(globalThis as any).onconnect = (event: MessageEvent) => {
+  activeConnectionCount++;
+  const port = event.ports[0];
+  port.addEventListener('message', (event: MessageEvent) => {
+    if (event.data.type === '__close__') {
+      activeConnectionCount--;
+      if (activeConnectionCount === 0) {
+        globalThis.close();
+      }
+    }
+  });
+
+  connectElectronAPIs(port);
+
+  const consumer = new OpConsumer<WorkerOps>(port);
+  worker.bindConsumer(consumer);
+};

packages/frontend/apps/electron-renderer/src/setup.ts

@@ -1,3 +1,12 @@
 import '@affine/core/bootstrap/electron';
 import '@affine/component/theme';
 import './global.css';
+
+import { apis } from '@affine/electron-api';
+import { bindNativeDBApis } from '@affine/nbstore/sqlite';
+import { bindNativeDBV1Apis } from '@affine/nbstore/sqlite/v1';
+
+// oxlint-disable-next-line no-non-null-assertion
+bindNativeDBApis(apis!.nbstore);
+// oxlint-disable-next-line no-non-null-assertion
+bindNativeDBV1Apis(apis!.db);

packages/frontend/apps/electron-renderer/src/shell/app.tsx

@@ -11,7 +11,7 @@ import { configureDesktopApiModule } from '@affine/core/modules/desktop-api';
 import { configureI18nModule, I18nProvider } from '@affine/core/modules/i18n';
 import {
   configureElectronStateStorageImpls,
-  configureGlobalStorageModule,
+  configureStorageModule,
 } from '@affine/core/modules/storage';
 import { configureAppThemeModule } from '@affine/core/modules/theme';
 import { Framework, FrameworkRoot } from '@toeverything/infra';
@@ -19,7 +19,7 @@ import { Framework, FrameworkRoot } from '@toeverything/infra';
 import * as styles from './app.css';
 
 const framework = new Framework();
-configureGlobalStorageModule(framework);
+configureStorageModule(framework);
 configureElectronStateStorageImpls(framework);
 configureAppTabsHeaderModule(framework);
 configureAppSidebarModule(framework);

packages/frontend/apps/electron-renderer/tsconfig.json

@@ -12,6 +12,7 @@
     { "path": "../../core" },
     { "path": "../../electron-api" },
     { "path": "../../i18n" },
+    { "path": "../../../common/nbstore" },
     { "path": "../../../common/infra" },
     { "path": "../../../../tools/utils" }
   ]

packages/frontend/apps/electron-renderer/webpack.config.ts

@@ -2,5 +2,6 @@ export const config = {
   entry: {
     app: './src/index.tsx',
     shell: './src/shell/index.tsx',
+    backgroundWorker: './src/background-worker/index.ts',
   },
 };