feat: plugin system with isolated bundles (#2660)

.gitignore

@@ -66,6 +66,7 @@ i18n-generated.ts
 # Cache
 .eslintcache
 next-env.d.ts
+.rollup.cache
 
 # Rust
 target

apps/electron/layers/main/src/ui/index.ts

@@ -1,10 +1,17 @@
+import { join } from 'node:path';
+
 import { app, BrowserWindow, nativeTheme } from 'electron';
 
 import type { NamespaceHandlers } from '../type';
 import { isMacOS } from '../utils';
-import { getMetaData } from './get-meta-data';
 import { getGoogleOauthCode } from './google-auth';
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const handlers = require(join(
+  process.env.PLUGIN_DIR ?? '../../plugins',
+  './bookmark-block/server'
+)) as NamespaceHandlers;
+
 export const uiHandlers = {
   handleThemeChange: async (_, theme: (typeof nativeTheme)['themeSource']) => {
     nativeTheme.themeSource = theme;
@@ -40,11 +47,5 @@ export const uiHandlers = {
   getGoogleOauthCode: async () => {
     return getGoogleOauthCode();
   },
-  getBookmarkDataByLink: async (_, url: string) => {
+  ...handlers,
-    return getMetaData(url, {
-      shouldReGetHTML: metaData => {
-        return !metaData.title && !metaData.description;
-      },
-    });
-  },
 } satisfies NamespaceHandlers;

apps/electron/package.json

@@ -56,7 +56,6 @@
   },
   "dependencies": {
     "better-sqlite3": "^8.4.0",
-    "cheerio": "^1.0.0-rc.12",
     "chokidar": "^3.5.3",
     "electron-updater": "^5.3.0",
     "lodash-es": "^4.17.21",

apps/electron/scripts/build-layers.mjs

@@ -1,9 +1,12 @@
 #!/usr/bin/env zx
 import 'zx/globals';
 
+import { spawnSync } from 'node:child_process';
+import { resolve } from 'node:path';
+
 import * as esbuild from 'esbuild';
 
-import { config } from './common.mjs';
+import { config, rootDir } from './common.mjs';
 
 const NODE_ENV =
   process.env.NODE_ENV === 'development' ? 'development' : 'production';
@@ -17,6 +20,12 @@ async function buildLayers() {
   const common = config();
   await esbuild.build(common.preload);
 
+  console.log('build plugins');
+  spawnSync('yarn', ['build'], {
+    cwd: resolve(rootDir, './plugins/bookmark-block'),
+    stdio: 'inherit',
+  });
+
   await esbuild.build({
     ...common.main,
     define: {

apps/electron/scripts/common.mjs

@@ -2,7 +2,10 @@ import { resolve } from 'node:path';
 
 import { fileURLToPath } from 'url';
 
-export const root = fileURLToPath(new URL('..', import.meta.url));
+export const electronDir = fileURLToPath(new URL('..', import.meta.url));
+
+export const rootDir = resolve(electronDir, '..', '..');
+
 export const NODE_MAJOR_VERSION = 18;
 
 // hard-coded for now:
@@ -33,10 +36,13 @@ export const config = () => {
   return {
     main: {
       entryPoints: [
-        resolve(root, './layers/main/src/index.ts'),
+        resolve(electronDir, './layers/main/src/index.ts'),
-        resolve(root, './layers/main/src/workers/merge-update.worker.ts'),
+        resolve(
+          electronDir,
+          './layers/main/src/workers/merge-update.worker.ts'
+        ),
       ],
-      outdir: resolve(root, './dist/layers/main'),
+      outdir: resolve(electronDir, './dist/layers/main'),
       bundle: true,
       target: `node${NODE_MAJOR_VERSION}`,
       platform: 'node',
@@ -50,8 +56,8 @@ export const config = () => {
       treeShaking: true,
     },
     preload: {
-      entryPoints: [resolve(root, './layers/preload/src/index.ts')],
+      entryPoints: [resolve(electronDir, './layers/preload/src/index.ts')],
-      outdir: resolve(root, './dist/layers/preload'),
+      outdir: resolve(electronDir, './dist/layers/preload'),
       bundle: true,
       target: `node${NODE_MAJOR_VERSION}`,
       platform: 'node',

apps/electron/scripts/dev.mjs

@@ -1,12 +1,12 @@
 /* eslint-disable no-async-promise-executor */
 import { spawn } from 'node:child_process';
 import { readFileSync } from 'node:fs';
-import path from 'node:path';
+import path, { resolve } from 'node:path';
 
 import electronPath from 'electron';
 import * as esbuild from 'esbuild';
 
-import { config, root } from './common.mjs';
+import { config, electronDir, rootDir } from './common.mjs';
 
 // this means we don't spawn electron windows, mainly for testing
 const watchMode = process.argv.includes('--watch');
@@ -21,7 +21,10 @@ const stderrFilterPatterns = [
 
 // these are set before calling `config`, so we have a chance to override them
 try {
-  const devJson = readFileSync(path.resolve(root, './dev.json'), 'utf-8');
+  const devJson = readFileSync(
+    path.resolve(electronDir, './dev.json'),
+    'utf-8'
+  );
   const devEnv = JSON.parse(devJson);
   Object.assign(process.env, devEnv);
 } catch (err) {
@@ -65,7 +68,18 @@ function spawnOrReloadElectron() {
 
 const common = config();
 
-function watchPreload() {
+function watchPlugins() {
+  const cp = spawn('yarn', ['dev'], {
+    cwd: resolve(rootDir, './plugins/bookmark-block'),
+    stdio: 'inherit',
+  });
+
+  process.once('beforeExit', () => {
+    cp.kill();
+  });
+}
+
+async function watchPreload() {
   return new Promise(async resolve => {
     let initialBuild = false;
     const preloadBuild = await esbuild.context({
@@ -122,6 +136,7 @@ async function watchMain() {
 }
 
 async function main() {
+  watchPlugins();
   await watchMain();
   await watchPreload();
 

plugins/bookmark-block/package.json

@@ -4,10 +4,16 @@
   "main": "./src/index.ts",
   "module": "./src/index.ts",
   "exports": {
-    ".": "./src/index.ts"
+    ".": "./src/index.ts",
+    "./server": "./src/server.ts"
+  },
+  "scripts": {
+    "build": "vite build",
+    "dev": "vite build --watch"
   },
   "dependencies": {
-    "@toeverything/plugin-infra": "workspace:*"
+    "@toeverything/plugin-infra": "workspace:*",
+    "cheerio": "^1.0.0-rc.12"
   },
   "devDependencies": {
     "react": "18.3.0-canary-16d053d59-20230506",

plugins/bookmark-block/src/server.ts

@@ -0,0 +1,11 @@
+import { getMetaData } from './server/get-meta-data';
+
+export default {
+  getBookmarkDataByLink: async (_: unknown, url: string) => {
+    return getMetaData(url, {
+      shouldReGetHTML: metaData => {
+        return !metaData.title && !metaData.description;
+      },
+    });
+  },
+};

plugins/bookmark-block/src/server/get-meta-data/utils.ts

@@ -1,10 +1,10 @@
-import urlparse from 'url';
+import { parse, resolve } from 'node:url';
 
 export function makeUrlAbsolute(base: string, relative: string): string {
-  const relativeParsed = urlparse.parse(relative);
+  const relativeParsed = parse(relative);
 
   if (relativeParsed.host === null) {
-    return urlparse.resolve(base, relative);
+    return resolve(base, relative);
   }
 
   return relative;
@@ -15,7 +15,7 @@ export function makeUrlSecure(url: string): string {
 }
 
 export function parseUrl(url: string): string {
-  return urlparse.parse(url).hostname || '';
+  return parse(url).hostname || '';
 }
 
 export function getProvider(host: string): string {

plugins/bookmark-block/tsconfig.json

@@ -1,4 +1,13 @@
 {
   "extends": "../../tsconfig.json",
-  "include": ["./src"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "types": ["electron"]
+  },
+  "include": ["**.ts", "**.tsx"],
+  "references": [
+    {
+      "path": "./tsconfig.node.json"
+    }
+  ]
 }

plugins/bookmark-block/tsconfig.node.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "allowSyntheticDefaultImports": true,
+    "outDir": "lib"
+  },
+  "include": ["vite.config.ts", "scripts"]
+}

plugins/bookmark-block/vite.config.ts

@@ -0,0 +1,33 @@
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { defineConfig } from 'vite';
+
+const rootDir = fileURLToPath(new URL('.', import.meta.url));
+
+export default defineConfig({
+  build: {
+    minify: false,
+    lib: {
+      entry: resolve(__dirname, 'src/server.ts'),
+      fileName: 'server',
+      formats: ['cjs'],
+    },
+    emptyOutDir: true,
+    rollupOptions: {
+      external: ['cheerio', 'electron', 'node:url'],
+      output: {
+        dir: resolve(
+          rootDir,
+          '..',
+          '..',
+          'apps',
+          'electron',
+          'dist',
+          'plugins',
+          'bookmark-block'
+        ),
+      },
+    },
+  },
+});

scripts/setup/build-plugins.ts

@@ -0,0 +1,14 @@
+import { resolve } from 'node:path';
+
+import { fileURLToPath } from 'url';
+import { build } from 'vite';
+import { beforeAll } from 'vitest';
+
+export const rootDir = fileURLToPath(new URL('../..', import.meta.url));
+
+beforeAll(async () => {
+  const { default: config } = await import(
+    resolve(rootDir, './plugins/bookmark-block/vite.config.ts')
+  );
+  await build(config);
+});

vitest.config.ts

@@ -6,6 +6,7 @@ import react from '@vitejs/plugin-react';
 import { defineConfig } from 'vitest/config';
 
 const rootDir = fileURLToPath(new URL('.', import.meta.url));
+const pluginOutputDir = resolve(rootDir, './apps/electron/dist/plugins');
 
 export default defineConfig({
   plugins: [react(), vanillaExtractPlugin()],
@@ -16,8 +17,12 @@ export default defineConfig({
       'next/config': resolve(rootDir, './scripts/vitest/next-config-mock.ts'),
     },
   },
+  define: {
+    'process.env.PLUGIN_DIR': JSON.stringify(pluginOutputDir),
+  },
   test: {
     setupFiles: [
+      resolve(rootDir, './scripts/setup/build-plugins.ts'),
       resolve(rootDir, './scripts/setup/lit.ts'),
       resolve(rootDir, './scripts/setup/i18n.ts'),
       resolve(rootDir, './scripts/setup/search.ts'),

yarn.lock

@@ -35,6 +35,7 @@ __metadata:
   resolution: "@affine/bookmark-block@workspace:plugins/bookmark-block"
   dependencies:
     "@toeverything/plugin-infra": "workspace:*"
+    cheerio: ^1.0.0-rc.12
     react: 18.3.0-canary-16d053d59-20230506
     react-dom: 18.3.0-canary-16d053d59-20230506
   peerDependencies:
@@ -183,7 +184,6 @@ __metadata:
     "@types/fs-extra": ^11.0.1
     "@types/uuid": ^9.0.1
     better-sqlite3: ^8.4.0
-    cheerio: ^1.0.0-rc.12
     chokidar: ^3.5.3
     cross-env: 7.0.3
     electron: 25.0.0