feat(plugin-infra): esm simulation in browser (#3464)

apps/core/src/bootstrap/plugins/setup.ts

@@ -1,12 +1,28 @@
 import * as AFFiNEComponent from '@affine/component';
 import { DebugLogger } from '@affine/debug';
+import { FormatQuickBar } from '@blocksuite/blocks';
 import * as BlockSuiteBlocksStd from '@blocksuite/blocks/std';
 import * as BlockSuiteGlobalUtils from '@blocksuite/global/utils';
+import { assertExists } from '@blocksuite/global/utils';
+import { DisposableGroup } from '@blocksuite/global/utils';
 import * as Icons from '@blocksuite/icons';
 import * as Atom from '@toeverything/plugin-infra/atom';
+import {
+  editorItemsAtom,
+  headerItemsAtom,
+  rootStore,
+  settingItemsAtom,
+  windowItemsAtom,
+} from '@toeverything/plugin-infra/atom';
+import type {
+  CallbackMap,
+  PluginContext,
+} from '@toeverything/plugin-infra/entry';
 import * as Jotai from 'jotai/index';
+import { Provider } from 'jotai/react';
 import * as JotaiUtils from 'jotai/utils';
 import * as React from 'react';
+import { createElement, type PropsWithChildren } from 'react';
 import * as ReactJSXRuntime from 'react/jsx-runtime';
 import * as ReactDom from 'react-dom';
 import * as ReactDomClient from 'react-dom/client';
@@ -15,35 +31,94 @@ import * as SWR from 'swr';
 import { createFetch } from './endowments/fercher';
 import { createTimers } from './endowments/timer';
 
-const logger = new DebugLogger('plugins:permission');
+const dynamicImportKey = '$h‍_import';
 
-const setupImportsMap = () => {
+const permissionLogger = new DebugLogger('plugins:permission');
-  importsMap.set('react', new Map(Object.entries(React)));
+const importLogger = new DebugLogger('plugins:import');
-  importsMap.set('react/jsx-runtime', new Map(Object.entries(ReactJSXRuntime)));
+
-  importsMap.set('react-dom', new Map(Object.entries(ReactDom)));
+const setupRootImportsMap = () => {
-  importsMap.set('react-dom/client', new Map(Object.entries(ReactDomClient)));
+  rootImportsMap.set('react', new Map(Object.entries(React)));
-  importsMap.set('@blocksuite/icons', new Map(Object.entries(Icons)));
+  rootImportsMap.set(
-  importsMap.set('@affine/component', new Map(Object.entries(AFFiNEComponent)));
+    'react/jsx-runtime',
-  importsMap.set(
+    new Map(Object.entries(ReactJSXRuntime))
+  );
+  rootImportsMap.set('react-dom', new Map(Object.entries(ReactDom)));
+  rootImportsMap.set(
+    'react-dom/client',
+    new Map(Object.entries(ReactDomClient))
+  );
+  rootImportsMap.set('@blocksuite/icons', new Map(Object.entries(Icons)));
+  rootImportsMap.set(
+    '@affine/component',
+    new Map(Object.entries(AFFiNEComponent))
+  );
+  rootImportsMap.set(
     '@blocksuite/blocks/std',
     new Map(Object.entries(BlockSuiteBlocksStd))
   );
-  importsMap.set(
+  rootImportsMap.set(
     '@blocksuite/global/utils',
     new Map(Object.entries(BlockSuiteGlobalUtils))
   );
-  importsMap.set('jotai', new Map(Object.entries(Jotai)));
+  rootImportsMap.set('jotai', new Map(Object.entries(Jotai)));
-  importsMap.set('jotai/utils', new Map(Object.entries(JotaiUtils)));
+  rootImportsMap.set('jotai/utils', new Map(Object.entries(JotaiUtils)));
-  importsMap.set(
+  rootImportsMap.set(
     '@toeverything/plugin-infra/atom',
     new Map(Object.entries(Atom))
   );
-  importsMap.set('swr', new Map(Object.entries(SWR)));
+  rootImportsMap.set('swr', new Map(Object.entries(SWR)));
 };
 
-const importsMap = new Map<string, Map<string, any>>();
+// module -> importName -> updater[]
-setupImportsMap();
+const rootImportsMap = new Map<string, Map<string, any>>();
-export { importsMap };
+setupRootImportsMap();
+// pluginName -> module -> importName -> updater[]
+const pluginNestedImportsMap = new Map<string, Map<string, Map<string, any>>>();
+
+const pluginImportsFunctionMap = new Map<string, (imports: any) => void>();
+export const createImports = (pluginName: string) => {
+  if (pluginImportsFunctionMap.has(pluginName)) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return pluginImportsFunctionMap.get(pluginName)!;
+  }
+  const imports = (
+    newUpdaters: [string, [string, ((val: any) => void)[]][]][]
+  ) => {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const currentImportMap = pluginNestedImportsMap.get(pluginName)!;
+    console.log('currentImportMap', pluginName, currentImportMap);
+
+    for (const [module, moduleUpdaters] of newUpdaters) {
+      console.log('imports module', module, moduleUpdaters);
+      let moduleImports = rootImportsMap.get(module);
+      if (!moduleImports) {
+        moduleImports = currentImportMap.get(module);
+      }
+      if (moduleImports) {
+        for (const [importName, importUpdaters] of moduleUpdaters) {
+          const updateImport = (value: any) => {
+            for (const importUpdater of importUpdaters) {
+              importUpdater(value);
+            }
+          };
+          if (moduleImports.has(importName)) {
+            const val = moduleImports.get(importName);
+            updateImport(val);
+          }
+        }
+      } else {
+        console.log(
+          'cannot find module in plugin import map',
+          module,
+          currentImportMap,
+          pluginNestedImportsMap
+        );
+      }
+    }
+  };
+  pluginImportsFunctionMap.set(pluginName, imports);
+  return imports;
+};
 
 const abortController = new AbortController();
 
@@ -54,28 +129,100 @@ const sharedGlobalThis = Object.assign(Object.create(null), timer, {
   fetch: pluginFetch,
 });
 
-export const createGlobalThis = (name: string) => {
+const dynamicImportMap = new Map<
-  return Object.assign(Object.create(null), sharedGlobalThis, {
+  string,
+  (moduleName: string) => Promise<any>
+>();
+
+export const createOrGetDynamicImport = (
+  baseUrl: string,
+  pluginName: string
+) => {
+  if (dynamicImportMap.has(pluginName)) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return dynamicImportMap.get(pluginName)!;
+  }
+  const dynamicImport = async (moduleName: string): Promise<any> => {
+    const codeUrl = `${baseUrl}/${moduleName}`;
+    const analysisUrl = `${baseUrl}/${moduleName}.json`;
+    const response = await fetch(codeUrl);
+    const analysisResponse = await fetch(analysisUrl);
+    const analysis = await analysisResponse.json();
+    const exports = analysis.exports as string[];
+    const code = await response.text();
+    const moduleCompartment = new Compartment(
+      createOrGetGlobalThis(
+        pluginName,
+        // use singleton here to avoid infinite loop
+        createOrGetDynamicImport(pluginName, baseUrl)
+      )
+    );
+    const entryPoint = moduleCompartment.evaluate(code, {
+      __evadeHtmlCommentTest__: true,
+    });
+    const moduleExports = {} as Record<string, any>;
+    const setVarProxy = new Proxy(
+      {},
+      {
+        get(_, p: string): any {
+          return (newValue: any) => {
+            moduleExports[p] = newValue;
+          };
+        },
+      }
+    );
+    entryPoint({
+      imports: createImports(pluginName),
+      liveVar: setVarProxy,
+      onceVar: setVarProxy,
+    });
+    importLogger.debug('import', moduleName, exports, moduleExports);
+    return moduleExports;
+  };
+  dynamicImportMap.set(pluginName, dynamicImport);
+  return dynamicImport;
+};
+
+const globalThisMap = new Map<string, any>();
+
+export const createOrGetGlobalThis = (
+  pluginName: string,
+  dynamicImport: (moduleName: string) => Promise<any>
+) => {
+  if (globalThisMap.has(pluginName)) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    return globalThisMap.get(pluginName)!;
+  }
+  const pluginGlobalThis = Object.assign(
+    Object.create(null),
+    sharedGlobalThis,
+    {
       process: Object.freeze({
         env: {
           NODE_ENV: process.env.NODE_ENV,
         },
       }),
+      // dynamic import function
+      [dynamicImportKey]: dynamicImport,
       // UNSAFE: React will read `window` and `document`
       window: new Proxy(
         {},
         {
           get(_, key) {
-          logger.debug(`${name} is accessing window`, key);
+            permissionLogger.debug(`${pluginName} is accessing window`, key);
             if (sharedGlobalThis[key]) return sharedGlobalThis[key];
             const result = Reflect.get(window, key);
             if (typeof result === 'function') {
               return function (...args: any[]) {
-              logger.debug(`${name} is calling window`, key, args);
+                permissionLogger.debug(
+                  `${pluginName} is calling window`,
+                  key,
+                  args
+                );
                 return result.apply(window, args);
               };
             }
-          logger.debug('window', key, result);
+            permissionLogger.debug('window', key, result);
             return result;
           },
         }
@@ -84,16 +231,20 @@ export const createGlobalThis = (name: string) => {
         {},
         {
           get(_, key) {
-          logger.debug(`${name} is accessing document`, key);
+            permissionLogger.debug(`${pluginName} is accessing document`, key);
             if (sharedGlobalThis[key]) return sharedGlobalThis[key];
             const result = Reflect.get(document, key);
             if (typeof result === 'function') {
               return function (...args: any[]) {
-              logger.debug(`${name} is calling window`, key, args);
+                permissionLogger.debug(
+                  `${pluginName} is calling window`,
+                  key,
+                  args
+                );
                 return result.apply(document, args);
               };
             }
-          logger.debug('document', key, result);
+            permissionLogger.debug('document', key, result);
             return result;
           },
         }
@@ -137,5 +288,156 @@ export const createGlobalThis = (name: string) => {
       IDBIndex: globalThis.IDBIndex,
       IDBCursor: globalThis.IDBCursor,
       IDBVersionChangeEvent: globalThis.IDBVersionChangeEvent,
-  });
+    }
+  );
+  globalThisMap.set(pluginName, pluginGlobalThis);
+  return pluginGlobalThis;
+};
+
+export const setupPluginCode = async (
+  baseUrl: string,
+  pluginName: string,
+  filename: string
+) => {
+  if (!pluginNestedImportsMap.has(pluginName)) {
+    pluginNestedImportsMap.set(pluginName, new Map());
+  }
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const currentImportMap = pluginNestedImportsMap.get(pluginName)!;
+  const isMissingPackage = (name: string) =>
+    rootImportsMap.has(name) && !currentImportMap.has(name);
+
+  const bundleAnalysis = await fetch(`${baseUrl}/${filename}.json`).then(res =>
+    res.json()
+  );
+  const moduleExports = bundleAnalysis.exports as Record<string, [string]>;
+  const moduleImports = bundleAnalysis.imports as string[];
+  const moduleReexports = bundleAnalysis.reexports as Record<
+    string,
+    [localName: string, exportedName: string][]
+  >;
+  await Promise.all(
+    moduleImports.map(name => {
+      if (isMissingPackage(name)) {
+        return Promise.resolve();
+      } else {
+        console.log('missing package', name);
+        return setupPluginCode(baseUrl, pluginName, name);
+      }
+    })
+  );
+  const code = await fetch(`${baseUrl}/${filename.replace(/^\.\//, '')}`).then(
+    res => res.text()
+  );
+  console.log('evaluating', filename);
+  const moduleCompartment = new Compartment(
+    createOrGetGlobalThis(
+      pluginName,
+      // use singleton here to avoid infinite loop
+      createOrGetDynamicImport(baseUrl, pluginName)
+    )
+  );
+  const entryPoint = moduleCompartment.evaluate(code, {
+    __evadeHtmlCommentTest__: true,
+  });
+  const moduleExportsMap = new Map<string, any>();
+  const setVarProxy = new Proxy(
+    {},
+    {
+      get(_, p: string): any {
+        return (newValue: any) => {
+          moduleExportsMap.set(p, newValue);
+        };
+      },
+    }
+  );
+  currentImportMap.set(filename, moduleExportsMap);
+  entryPoint({
+    imports: createImports(pluginName),
+    liveVar: setVarProxy,
+    onceVar: setVarProxy,
+  });
+
+  console.log('module exports alias', moduleExports);
+  for (const [newExport, [originalExport]] of Object.entries(moduleExports)) {
+    if (newExport === originalExport) continue;
+    const value = moduleExportsMap.get(originalExport);
+    moduleExportsMap.set(newExport, value);
+    moduleExportsMap.delete(originalExport);
+  }
+
+  console.log('module re-exports', moduleReexports);
+  for (const [name, reexports] of Object.entries(moduleReexports)) {
+    const targetExports = currentImportMap.get(filename);
+    const moduleExports = currentImportMap.get(name);
+    assertExists(targetExports);
+    assertExists(moduleExports);
+    for (const [exportedName, localName] of reexports) {
+      const exportedValue: any = moduleExports.get(exportedName);
+      console.log('re-export', name, localName, exportedName, exportedValue);
+      assertExists(exportedValue);
+      targetExports.set(localName, exportedValue);
+    }
+  }
+};
+
+const PluginProvider = ({ children }: PropsWithChildren) =>
+  createElement(
+    Provider,
+    {
+      store: rootStore,
+    },
+    children
+  );
+
+const group = new DisposableGroup();
+const entryLogger = new DebugLogger('plugin:entry');
+
+export const evaluatePluginEntry = (pluginName: string) => {
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const currentImportMap = pluginNestedImportsMap.get(pluginName)!;
+  const pluginExports = currentImportMap.get('index.mjs');
+  assertExists(pluginExports);
+  const entryFunction = pluginExports.get('entry');
+  const cleanup = entryFunction(<PluginContext>{
+    register: (part, callback) => {
+      entryLogger.info(`Registering ${pluginName} to ${part}`);
+      if (part === 'headerItem') {
+        rootStore.set(headerItemsAtom, items => ({
+          ...items,
+          [pluginName]: callback as CallbackMap['headerItem'],
+        }));
+      } else if (part === 'editor') {
+        rootStore.set(editorItemsAtom, items => ({
+          ...items,
+          [pluginName]: callback as CallbackMap['editor'],
+        }));
+      } else if (part === 'window') {
+        rootStore.set(windowItemsAtom, items => ({
+          ...items,
+          [pluginName]: callback as CallbackMap['window'],
+        }));
+      } else if (part === 'setting') {
+        rootStore.set(settingItemsAtom, items => ({
+          ...items,
+          [pluginName]: callback as CallbackMap['setting'],
+        }));
+      } else if (part === 'formatBar') {
+        FormatQuickBar.customElements.push((page, getBlockRange) => {
+          const div = document.createElement('div');
+          (callback as CallbackMap['formatBar'])(div, page, getBlockRange);
+          return div;
+        });
+      } else {
+        throw new Error(`Unknown part: ${part}`);
+      }
+    },
+    utils: {
+      PluginProvider,
+    },
+  });
+  if (typeof cleanup !== 'function') {
+    throw new Error('Plugin entry must return a function');
+  }
+  group.add(cleanup);
 };

apps/core/src/bootstrap/register-plugins.ts

@@ -2,25 +2,12 @@
 import 'ses';
 
 import { DebugLogger } from '@affine/debug';
-import { FormatQuickBar } from '@blocksuite/blocks';
-import { DisposableGroup } from '@blocksuite/global/utils';
 import {
-  editorItemsAtom,
-  headerItemsAtom,
   registeredPluginAtom,
   rootStore,
-  settingItemsAtom,
-  windowItemsAtom,
 } from '@toeverything/plugin-infra/atom';
-import type {
-  CallbackMap,
-  PluginContext,
-} from '@toeverything/plugin-infra/entry';
-import { Provider } from 'jotai/react';
-import type { PropsWithChildren } from 'react';
-import { createElement } from 'react';
 
-import { createGlobalThis, importsMap } from './plugins/setup';
+import { evaluatePluginEntry, setupPluginCode } from './plugins/setup';
 
 if (!process.env.COVERAGE) {
   lockdown({
@@ -33,29 +20,6 @@ if (!process.env.COVERAGE) {
   });
 }
 
-const imports = (
-  newUpdaters: [string, [string, ((val: any) => void)[]][]][]
-) => {
-  for (const [module, moduleUpdaters] of newUpdaters) {
-    const moduleImports = importsMap.get(module);
-    if (moduleImports) {
-      for (const [importName, importUpdaters] of moduleUpdaters) {
-        const updateImport = (value: any) => {
-          for (const importUpdater of importUpdaters) {
-            importUpdater(value);
-          }
-        };
-        if (moduleImports.has(importName)) {
-          const val = moduleImports.get(importName);
-          updateImport(val);
-        } else {
-          console.log('import not found', importName, module);
-        }
-      }
-    }
-  }
-};
-
 const builtinPluginUrl = new Set([
   '/plugins/bookmark',
   '/plugins/copilot',
@@ -65,17 +29,6 @@ const builtinPluginUrl = new Set([
 
 const logger = new DebugLogger('register-plugins');
 
-const PluginProvider = ({ children }: PropsWithChildren) =>
-  createElement(
-    Provider,
-    {
-      store: rootStore,
-    },
-    children
-  );
-
-const group = new DisposableGroup();
-
 declare global {
   // eslint-disable-next-line no-var
   var __pluginPackageJson__: unknown[];
@@ -83,7 +36,7 @@ declare global {
 
 globalThis.__pluginPackageJson__ = [];
 
-await Promise.all(
+Promise.all(
   [...builtinPluginUrl].map(url => {
     return fetch(`${url}/package.json`)
       .then(async res => {
@@ -102,11 +55,12 @@ await Promise.all(
         if (!release && process.env.NODE_ENV === 'production') {
           return Promise.resolve();
         }
-        const pluginCompartment = new Compartment(createGlobalThis(pluginName));
         const baseURL = url;
         const entryURL = `${baseURL}/${core}`;
         rootStore.set(registeredPluginAtom, prev => [...prev, pluginName]);
-        await fetch(entryURL).then(async res => {
+        await setupPluginCode(baseURL, pluginName, core);
+        console.log(`prepareImports for ${pluginName} done`);
+        await fetch(entryURL).then(async () => {
           if (assets.length > 0) {
             await Promise.all(
               assets.map(async (asset: string) => {
@@ -128,70 +82,7 @@ await Promise.all(
               })
             );
           }
-          const codeText = await res.text();
+          evaluatePluginEntry(pluginName);
-          try {
-            const entryPoint = pluginCompartment.evaluate(codeText, {
-              __evadeHtmlCommentTest__: true,
-            });
-            entryPoint({
-              imports,
-              onceVar: {
-                entry: (
-                  entryFunction: (context: PluginContext) => () => void
-                ) => {
-                  const cleanup = entryFunction({
-                    register: (part, callback) => {
-                      logger.info(`Registering ${pluginName} to ${part}`);
-                      if (part === 'headerItem') {
-                        rootStore.set(headerItemsAtom, items => ({
-                          ...items,
-                          [pluginName]: callback as CallbackMap['headerItem'],
-                        }));
-                      } else if (part === 'editor') {
-                        rootStore.set(editorItemsAtom, items => ({
-                          ...items,
-                          [pluginName]: callback as CallbackMap['editor'],
-                        }));
-                      } else if (part === 'window') {
-                        rootStore.set(windowItemsAtom, items => ({
-                          ...items,
-                          [pluginName]: callback as CallbackMap['window'],
-                        }));
-                      } else if (part === 'setting') {
-                        rootStore.set(settingItemsAtom, items => ({
-                          ...items,
-                          [pluginName]: callback as CallbackMap['setting'],
-                        }));
-                      } else if (part === 'formatBar') {
-                        FormatQuickBar.customElements.push(
-                          (page, getBlockRange) => {
-                            const div = document.createElement('div');
-                            (callback as CallbackMap['formatBar'])(
-                              div,
-                              page,
-                              getBlockRange
-                            );
-                            return div;
-                          }
-                        );
-                      } else {
-                        throw new Error(`Unknown part: ${part}`);
-                      }
-                    },
-                    utils: {
-                      PluginProvider,
-                    },
-                  });
-                  if (typeof cleanup !== 'function') {
-                    throw new Error('Plugin entry must return a function');
-                  }
-                  group.add(cleanup);
-                },
-              },
-            });
-          } catch (e) {
-            console.error(pluginName, e);
-          }
         });
       })
       .catch(e => {

packages/cli/src/bin/dev-plugin.ts

@@ -1,4 +1,5 @@
 import { ok } from 'node:assert';
+import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { parseArgs } from 'node:util';
@@ -155,7 +156,21 @@ await build({
             throw new Error('no name');
           }
         },
-        manualChunks: () => 'plugin',
+        chunkFileNames: chunkInfo => {
+          if (chunkInfo.name) {
+            const hash = createHash('md5')
+              .update(
+                Object.values(chunkInfo.moduleIds)
+                  .map(m => m)
+                  .join()
+              )
+              .digest('hex')
+              .substring(0, 6);
+            return `${chunkInfo.name}-${hash}.mjs`;
+          } else {
+            throw new Error('no name');
+          }
+        },
       },
       external,
     },
@@ -166,15 +181,27 @@ await build({
     {
       name: 'parse-bundle',
       renderChunk(code, chunk) {
-        if (chunk.fileName.endsWith('.mjs')) {
+        if (chunk.fileName.endsWith('js')) {
           const record = new StaticModuleRecord(code, chunk.fileName);
+          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+          // @ts-expect-error
+          const reexports = record.__reexportMap__ as Record<
+            string,
+            [localName: string, exportedName: string][]
+          >;
+          const exports = Object.assign(
+            {},
+            record.__fixedExportMap__,
+            record.__liveExportMap__
+          );
           this.emitFile({
             type: 'asset',
-            fileName: 'analysis.json',
+            fileName: `${chunk.fileName}.json`,
             source: JSON.stringify(
               {
-                exports: record.exports,
+                exports: exports,
                 imports: record.imports,
+                reexports: reexports,
               },
               null,
               2

plugins/copilot/src/index.ts

@@ -7,6 +7,7 @@ import { DetailContent } from './UI/detail-content';
 import { HeaderItem } from './UI/header-item';
 
 export const entry = (context: PluginContext) => {
+  console.log('copilot entry');
   context.register('headerItem', div => {
     const root = createRoot(div);
     root.render(

plugins/hello-world/src/index.ts

@@ -4,9 +4,12 @@ import {
 } from '@toeverything/plugin-infra/atom';
 import type { PluginContext } from '@toeverything/plugin-infra/entry';
 import { createElement } from 'react';
+import { lazy } from 'react';
 import { createRoot } from 'react-dom/client';
 
-import { HeaderItem } from './app';
+const HeaderItem = lazy(() =>
+  import('./app').then(({ HeaderItem }) => ({ default: HeaderItem }))
+);
 
 export const entry = (context: PluginContext) => {
   console.log('register');