feat: init doc monitor (#3320)

apps/core/.webpack/config.ts

@@ -262,6 +262,7 @@ export const createConfiguration: (
       new VanillaExtractPlugin(),
       new webpack.DefinePlugin({
         'process.env': JSON.stringify({}),
+        'process.env.NODE_ENV': JSON.stringify(buildFlags.mode),
         runtimeConfig: JSON.stringify(runtimeConfig),
       }),
       new CopyPlugin({

apps/core/.webpack/utils.ts

@@ -6,5 +6,6 @@ export function computeCacheKey(buildFlags: BuildFlags) {
     'node' + process.version,
     buildFlags.mode,
     buildFlags.distribution,
+    buildFlags.channel,
   ].join('-');
 }

apps/core/project.json

@@ -22,19 +22,30 @@
         },
         {
           "env": "PERFSEE_TOKEN"
+        },
+        {
+          "env": "SENTRY_ORG"
+        },
+        {
+          "env": "SENTRY_PROJECT"
+        },
+        {
+          "env": "SENTRY_AUTH_TOKEN"
+        },
+        {
+          "env": "NEXT_PUBLIC_SENTRY_DSN"
+        },
+        {
+          "env": "DISTRIBUTION"
+        },
+        {
+          "env": "COVERAGE"
         }
       ],
       "options": {
         "script": "build"
       },
       "outputs": ["{projectRoot}/dist"]
-    },
-    "dev": {
-      "executor": "nx:run-script",
-      "options": {
-        "script": "dev"
-      },
-      "outputs": ["{projectRoot}/dist"]
     }
   }
 }

apps/core/src/adapters/local/index.tsx

@@ -16,8 +16,8 @@ import {
   CRUD,
   saveWorkspaceToLocalStorage,
 } from '@affine/workspace/local/crud';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import { createIndexedDBDownloadProvider } from '@affine/workspace/providers';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
 import { nanoid } from '@blocksuite/store';
 import { useStaticBlockSuiteWorkspace } from '@toeverything/plugin-infra/__internal__/react';
 
@@ -36,7 +36,7 @@ export const LocalAdapter: WorkspaceAdapter<WorkspaceFlavour.LOCAL> = {
   loadPriority: LoadPriority.LOW,
   Events: {
     'app:init': () => {
-      const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+      const blockSuiteWorkspace = getOrCreateWorkspace(
         nanoid(),
         WorkspaceFlavour.LOCAL
       );

apps/core/src/components/__debug__/client/editor.tsx

@@ -1,6 +1,6 @@
 import { initEmptyPage } from '@affine/env/blocksuite';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import type { EditorContainer } from '@blocksuite/editor';
 import type { Page } from '@blocksuite/store';
 import type React from 'react';
@@ -8,7 +8,7 @@ import { useCallback } from 'react';
 
 import { BlockSuiteEditor } from '../../blocksuite/block-suite-editor';
 
-const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+const blockSuiteWorkspace = getOrCreateWorkspace(
   'test',
   WorkspaceFlavour.LOCAL
 );

apps/core/src/hooks/use-workspaces.ts

@@ -4,7 +4,7 @@ import { DEFAULT_HELLO_WORLD_PAGE_ID_SUFFIX } from '@affine/env/constant';
 import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import { assertEquals } from '@blocksuite/global/utils';
 import { nanoid } from '@blocksuite/store';
 import { getWorkspace } from '@toeverything/plugin-infra/__internal__/workspace';
@@ -25,7 +25,7 @@ export function useAppHelper() {
   return {
     addLocalWorkspace: useCallback(
       async (workspaceId: string): Promise<string> => {
-        createEmptyBlockSuiteWorkspace(workspaceId, WorkspaceFlavour.LOCAL);
+        getOrCreateWorkspace(workspaceId, WorkspaceFlavour.LOCAL);
         saveWorkspaceToLocalStorage(workspaceId);
         await set(workspaces => [
           ...workspaces,
@@ -42,7 +42,7 @@ export function useAppHelper() {
     ),
     createLocalWorkspace: useCallback(
       async (name: string): Promise<string> => {
-        const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+        const blockSuiteWorkspace = getOrCreateWorkspace(
           nanoid(),
           WorkspaceFlavour.LOCAL
         );
@@ -50,7 +50,7 @@ export function useAppHelper() {
         const id = await LocalAdapter.CRUD.create(blockSuiteWorkspace);
         {
           // this is hack, because CRUD doesn't return the workspace
-          const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+          const blockSuiteWorkspace = getOrCreateWorkspace(
             id,
             WorkspaceFlavour.LOCAL
           );

apps/electron/scripts/generate-assets.mjs

@@ -41,25 +41,7 @@ cd(repoRootDir);
 
 // step 1: build web (nextjs) dist
 if (!process.env.SKIP_WEB_BUILD) {
-  process.env.ENABLE_LEGACY_PROVIDER = 'false';
+  await $`DISTRIBUTION=desktop yarn nx build @affine/core`;
-  await $`yarn nx build @affine/core`;
-
-  // step 1.5: amend sourceMappingURL to allow debugging in devtools
-  await glob('**/*.{js,css}', { cwd: affineCoreOutDir }).then(files => {
-    return files.map(async file => {
-      const dir = path.dirname(file);
-      const fullpath = path.join(affineCoreOutDir, file);
-      let content = await fs.readFile(fullpath, 'utf-8');
-      // replace # sourceMappingURL=76-6370cd185962bc89.js.map
-      // to      # sourceMappingURL=assets://./{dir}/76-6370cd185962bc89.js.map
-      content = content.replace(/# sourceMappingURL=(.*)\.map/g, (_, p1) => {
-        return `# sourceMappingURL=assets://./${dir}/${p1}.map`;
-      });
-      await fs.writeFile(fullpath, content);
-    });
-  });
-
-  await fs.move(affineCoreOutDir, publicAffineOutDir, { overwrite: true });
 }
 
 // step 2: update app-updater.yml content with build type in resources folder

apps/storybook/src/stories/card.stories.tsx

@@ -2,7 +2,7 @@ import { toast, Tooltip } from '@affine/component';
 import { BlockCard } from '@affine/component/card/block-card';
 import { WorkspaceCard } from '@affine/component/card/workspace-card';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import {
   EdgelessIcon,
   ExportToHtmlIcon,
@@ -15,7 +15,7 @@ export default {
   component: WorkspaceCard,
 };
 
-const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+const blockSuiteWorkspace = getOrCreateWorkspace(
   'blocksuite-local',
   WorkspaceFlavour.LOCAL
 );

apps/storybook/src/stories/image-preview-modal.stories.tsx

@@ -4,7 +4,7 @@ import { ImagePreviewModal } from '@affine/component/image-preview-modal';
 import { initEmptyPage } from '@affine/env/blocksuite';
 import { WorkspaceFlavour } from '@affine/env/workspace';
 import { rootBlockHubAtom } from '@affine/workspace/atom';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import type { Meta } from '@storybook/react';
 
 export default {
@@ -12,10 +12,7 @@ export default {
   component: ImagePreviewModal,
 } satisfies Meta;
 
-const workspace = createEmptyBlockSuiteWorkspace(
+const workspace = getOrCreateWorkspace('test', WorkspaceFlavour.LOCAL);
-  'test',
-  WorkspaceFlavour.LOCAL
-);
 const page = workspace.createPage('page0');
 initEmptyPage(page);
 fetch(new URL('@affine-test/fixtures/large-image.png', import.meta.url))

apps/storybook/src/stories/share-menu.stories.tsx

@@ -9,7 +9,7 @@ import type {
   LocalWorkspace,
 } from '@affine/env/workspace';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import type { Page } from '@blocksuite/store';
 import { expect } from '@storybook/jest';
 import type { StoryFn } from '@storybook/react';
@@ -39,7 +39,7 @@ async function initPage(page: Page) {
   page.resetHistory();
 }
 
-const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+const blockSuiteWorkspace = getOrCreateWorkspace(
   'test-workspace',
   WorkspaceFlavour.LOCAL
 );

apps/storybook/src/stories/workspace-list.stories.tsx

@@ -1,7 +1,7 @@
 import type { WorkspaceListProps } from '@affine/component/workspace-list';
 import { WorkspaceList } from '@affine/component/workspace-list';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import { arrayMove } from '@dnd-kit/sortable';
 import type { Meta } from '@storybook/react';
 import { useState } from 'react';
@@ -17,26 +17,17 @@ export const Default = () => {
       {
         id: '1',
         flavour: WorkspaceFlavour.LOCAL,
-        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace(
+        blockSuiteWorkspace: getOrCreateWorkspace('1', WorkspaceFlavour.LOCAL),
-          '1',
-          WorkspaceFlavour.LOCAL
-        ),
       },
       {
         id: '2',
         flavour: WorkspaceFlavour.LOCAL,
-        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace(
+        blockSuiteWorkspace: getOrCreateWorkspace('2', WorkspaceFlavour.LOCAL),
-          '2',
-          WorkspaceFlavour.LOCAL
-        ),
       },
       {
         id: '3',
         flavour: WorkspaceFlavour.LOCAL,
-        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace(
+        blockSuiteWorkspace: getOrCreateWorkspace('3', WorkspaceFlavour.LOCAL),
-          '3',
-          WorkspaceFlavour.LOCAL
-        ),
       },
     ] satisfies WorkspaceListProps['items'];
 

nx.json

@@ -41,6 +41,27 @@
         },
         {
           "env": "BUILD_TYPE"
+        },
+        {
+          "env": "PERFSEE_TOKEN"
+        },
+        {
+          "env": "SENTRY_ORG"
+        },
+        {
+          "env": "SENTRY_PROJECT"
+        },
+        {
+          "env": "SENTRY_AUTH_TOKEN"
+        },
+        {
+          "env": "NEXT_PUBLIC_SENTRY_DSN"
+        },
+        {
+          "env": "DISTRIBUTION"
+        },
+        {
+          "env": "COVERAGE"
         }
       ]
     },

packages/cli/src/bin/build-core.ts

@@ -21,8 +21,23 @@ const getChannel = () => {
   }
 };
 
+const getDistribution = () => {
+  switch (process.env.DISTRIBUTION) {
+    case 'browser':
+    case 'desktop':
+      return process.env.DISTRIBUTION;
+    case undefined: {
+      console.log('DISTRIBUTION is not set, defaulting to browser');
+      return 'browser';
+    }
+    default: {
+      throw new Error('DISTRIBUTION must be one of browser, desktop');
+    }
+  }
+};
+
 const flags = {
-  distribution: 'browser',
+  distribution: getDistribution(),
   mode: 'production',
   channel: getChannel(),
   coverage: process.env.COVERAGE === 'true',

packages/workspace/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "exports": {
     "./atom": "./src/atom.ts",
-    "./utils": "./src/utils.ts",
+    "./manager": "./src/manager/index.ts",
     "./type": "./src/type.ts",
     "./migration": "./src/migration/index.ts",
     "./local/crud": "./src/local/crud.ts",

packages/workspace/src/atom.ts

@@ -1,6 +1,6 @@
 import type { WorkspaceAdapter } from '@affine/env/workspace';
 import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import type { BlockHub } from '@blocksuite/blocks';
 import { assertExists } from '@blocksuite/global/utils';
 import { atom } from 'jotai';
@@ -145,7 +145,7 @@ const rootWorkspacesMetadataPromiseAtom = atom<
         meta.flavour === WorkspaceFlavour.AFFINE_CLOUD ||
         meta.flavour === WorkspaceFlavour.LOCAL
       ) {
-        createEmptyBlockSuiteWorkspace(id, meta.flavour);
+        getOrCreateWorkspace(id, meta.flavour);
       } else {
         throw new Error(`unknown flavour ${meta.flavour}`);
       }

packages/workspace/src/local/crud.ts

@@ -6,7 +6,7 @@ import { createIndexedDBProvider } from '@toeverything/y-indexeddb';
 import { createJSONStorage } from 'jotai/utils';
 import { z } from 'zod';
 
-import { createEmptyBlockSuiteWorkspace } from '../utils';
+import { getOrCreateWorkspace } from '../manager';
 
 const getStorage = () => createJSONStorage(() => localStorage);
 
@@ -41,7 +41,7 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.LOCAL> = {
     if (!id) {
       return null;
     }
-    const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+    const blockSuiteWorkspace = getOrCreateWorkspace(
       id,
       WorkspaceFlavour.LOCAL
     );
@@ -59,7 +59,7 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.LOCAL> = {
       storage.setItem(kStoreKey, []);
     const binary = BlockSuiteWorkspace.Y.encodeStateAsUpdate(doc);
     const id = nanoid();
-    const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+    const blockSuiteWorkspace = getOrCreateWorkspace(
       id,
       WorkspaceFlavour.LOCAL
     );

packages/workspace/src/manager/index.ts

@@ -13,9 +13,11 @@ import {
   Workspace,
 } from '@blocksuite/store';
 import { INTERNAL_BLOCKSUITE_HASH_MAP } from '@toeverything/plugin-infra/__internal__/workspace';
+import type { Doc } from 'yjs';
+import type { Transaction } from 'yjs';
 
-import { createStaticStorage } from './blob/local-static-storage';
+import { createStaticStorage } from '../blob/local-static-storage';
-import { createSQLiteStorage } from './blob/sqlite-blob-storage';
+import { createSQLiteStorage } from '../blob/sqlite-blob-storage';
 
 function setEditorFlags(workspace: Workspace) {
   Object.entries(runtimeConfig.editorFlags).forEach(([key, value]) => {
@@ -30,11 +32,57 @@ function setEditorFlags(workspace: Workspace) {
   );
 }
 
-export function createEmptyBlockSuiteWorkspace(
+type UpdateCallback = (
-  id: string,
+  update: Uint8Array,
-  flavour: WorkspaceFlavour.AFFINE_CLOUD | WorkspaceFlavour.LOCAL
+  origin: string | number | null,
-): Workspace;
+  doc: Doc,
-export function createEmptyBlockSuiteWorkspace(
+  transaction: Transaction
+) => void;
+
+type SubdocEvent = {
+  loaded: Set<Doc>;
+  removed: Set<Doc>;
+  added: Set<Doc>;
+};
+
+const docUpdateCallbackWeakMap = new WeakMap<Doc, UpdateCallback>();
+
+const createMonitor = (doc: Doc) => {
+  const onUpdate: UpdateCallback = (update, origin) => {
+    if (process.env.NODE_ENV === 'development') {
+      if (typeof origin !== 'string' && typeof origin !== 'number') {
+        console.warn(
+          'origin is not a string or number, this will cause problems in the future',
+          origin
+        );
+      }
+    } else {
+      // todo: add monitor in the future
+    }
+  };
+  docUpdateCallbackWeakMap.set(doc, onUpdate);
+  doc.on('update', onUpdate);
+  const onSubdocs = (event: SubdocEvent) => {
+    event.added.forEach(subdoc => {
+      if (!docUpdateCallbackWeakMap.has(subdoc)) {
+        createMonitor(subdoc);
+      }
+    });
+    event.removed.forEach(subdoc => {
+      if (docUpdateCallbackWeakMap.has(subdoc)) {
+        docUpdateCallbackWeakMap.delete(subdoc);
+      }
+    });
+  };
+  doc.on('subdocs', onSubdocs);
+  doc.on('destroy', () => {
+    docUpdateCallbackWeakMap.delete(doc);
+    doc.off('update', onSubdocs);
+  });
+};
+
+// if not exist, create a new workspace
+export function getOrCreateWorkspace(
   id: string,
   flavour: WorkspaceFlavour
 ): Workspace {
@@ -76,6 +124,7 @@ export function createEmptyBlockSuiteWorkspace(
   })
     .register(AffineSchemas)
     .register(__unstableSchemas);
+  createMonitor(workspace.doc);
   setEditorFlags(workspace);
   INTERNAL_BLOCKSUITE_HASH_MAP.set(id, workspace);
   return workspace;

packages/workspace/src/migration/index.ts

@@ -1,7 +1,7 @@
 import { migrateToSubdoc } from '@affine/env/blocksuite';
 import type { LocalWorkspace } from '@affine/env/workspace';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
+import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import { nanoid, Workspace } from '@blocksuite/store';
 import { createIndexeddbStorage } from '@blocksuite/store';
 const Y = Workspace.Y;
@@ -14,7 +14,7 @@ export function upgradeV1ToV2(oldWorkspace: LocalWorkspace): LocalWorkspace {
     return oldWorkspace;
   } else {
     const id = nanoid();
-    const newBlockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+    const newBlockSuiteWorkspace = getOrCreateWorkspace(
       id,
       WorkspaceFlavour.LOCAL
     );

packages/workspace/src/providers/index.ts

@@ -62,6 +62,7 @@ const createIndexedDBBackgroundProvider: DocProviderCreator = (
 };
 
 const cache: WeakMap<Doc, Uint8Array> = new WeakMap();
+const indexedDBDownloadOrigin = 'indexeddb-download-provider';
 
 const createIndexedDBDownloadProvider: DocProviderCreator = (
   id,
@@ -76,11 +77,11 @@ const createIndexedDBDownloadProvider: DocProviderCreator = (
   async function downloadBinaryRecursively(doc: Doc) {
     if (cache.has(doc)) {
       const binary = cache.get(doc) as Uint8Array;
-      Y.applyUpdate(doc, binary);
+      Y.applyUpdate(doc, binary, indexedDBDownloadOrigin);
     } else {
       const binary = await downloadBinary(doc.guid);
       if (binary) {
-        Y.applyUpdate(doc, binary);
+        Y.applyUpdate(doc, binary, indexedDBDownloadOrigin);
         cache.set(doc, binary);
       }
     }

packages/y-indexeddb/src/index.ts

@@ -309,7 +309,7 @@ export const createIndexedDBProvider = (
       const fakeDoc = new Doc();
       fakeDoc.transact(() => {
         updates.forEach(update => {
-          applyUpdate(fakeDoc, update);
+          applyUpdate(fakeDoc, update, indexeddbOrigin);
         });
       }, indexeddbOrigin);
       const newUpdate = diffUpdate(
@@ -330,7 +330,7 @@ export const createIndexedDBProvider = (
       );
       doc.transact(() => {
         updates.forEach(update => {
-          applyUpdate(doc, update);
+          applyUpdate(doc, update, indexeddbOrigin);
         });
       }, indexeddbOrigin);
     }