feat: support migration (#2852)

apps/web/src/atoms/__tests__/atom.spec.ts

@@ -5,7 +5,7 @@ import 'fake-indexeddb/auto';
 
 import { initEmptyPage } from '@affine/env/blocksuite';
 import type { LocalIndexedDBBackgroundProvider } from '@affine/env/workspace';
-import { WorkspaceFlavour } from '@affine/env/workspace';
+import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
 import {
   rootCurrentWorkspaceIdAtom,
   rootWorkspacesMetadataAtom,
@@ -96,6 +96,7 @@ describe('currentWorkspace atom', () => {
         {
           id: workspaceId,
           flavour: WorkspaceFlavour.LOCAL,
+          version: WorkspaceVersion.SubDoc,
         },
       ]);
       _cleanupBlockSuiteWorkspaceCache();

apps/web/src/atoms/index.ts

@@ -1,6 +1,6 @@
 import { DebugLogger } from '@affine/debug';
-import { WorkspaceFlavour } from '@affine/env/workspace';
-import type { RootWorkspaceMetadata } from '@affine/workspace/atom';
+import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import type { RootWorkspaceMetadataV2 } from '@affine/workspace/atom';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { atom } from 'jotai';
 import { atomFamily, atomWithStorage } from 'jotai/utils';
@@ -13,7 +13,7 @@ const logger = new DebugLogger('web:atoms');
 // workspace necessary atoms
 // todo(himself65): move this to the workspace package
 rootWorkspacesMetadataAtom.onMount = setAtom => {
-  function createFirst(): RootWorkspaceMetadata[] {
+  function createFirst(): RootWorkspaceMetadataV2[] {
     const Plugins = Object.values(WorkspaceAdapters).sort(
       (a, b) => a.loadPriority - b.loadPriority
     );
@@ -24,29 +24,33 @@ rootWorkspacesMetadataAtom.onMount = setAtom => {
           ({
             id,
             flavour: Plugin.flavour,
-          } satisfies RootWorkspaceMetadata)
+            // new workspace should all support sub-doc feature
+            version: WorkspaceVersion.SubDoc,
+          } satisfies RootWorkspaceMetadataV2)
       );
-    }).filter((ids): ids is RootWorkspaceMetadata => !!ids);
+    }).filter((ids): ids is RootWorkspaceMetadataV2 => !!ids);
   }
 
   const abortController = new AbortController();
 
-  // next tick to make sure the hydration is correct
-  const id = setTimeout(() => {
-    setAtom(metadata => {
-      if (abortController.signal.aborted) return metadata;
-      if (
-        metadata.length === 0 &&
-        localStorage.getItem('is-first-open') === null
-      ) {
-        localStorage.setItem('is-first-open', 'false');
-        const newMetadata = createFirst();
-        logger.info('create first workspace', newMetadata);
-        return newMetadata;
-      }
-      return metadata;
-    });
-  }, 0);
+  if (!environment.isServer) {
+    // next tick to make sure the hydration is correct
+    setTimeout(() => {
+      setAtom(metadata => {
+        if (abortController.signal.aborted) return metadata;
+        if (
+          metadata.length === 0 &&
+          localStorage.getItem('is-first-open') === null
+        ) {
+          localStorage.setItem('is-first-open', 'false');
+          const newMetadata = createFirst();
+          logger.info('create first workspace', newMetadata);
+          return newMetadata;
+        }
+        return metadata;
+      });
+    }, 0);
+  }
 
   if (environment.isDesktop) {
     window.apis?.workspace
@@ -56,6 +60,7 @@ rootWorkspacesMetadataAtom.onMount = setAtom => {
         const newMetadata = workspaceIDs.map(w => ({
           id: w[0],
           flavour: WorkspaceFlavour.LOCAL,
+          version: undefined,
         }));
         setAtom(metadata => {
           return [
@@ -70,7 +75,6 @@ rootWorkspacesMetadataAtom.onMount = setAtom => {
   }
 
   return () => {
-    clearTimeout(id);
     abortController.abort();
   };
 };

apps/web/src/atoms/root.ts

@@ -18,60 +18,73 @@ const logger = new DebugLogger('web:atoms:root');
 /**
  * Fetch all workspaces from the Plugin CRUD
  */
-export const workspacesAtom = atom<Promise<AllWorkspace[]>>(async get => {
-  const { WorkspaceAdapters } = await import('../adapters/workspace');
-  const flavours: string[] = Object.values(WorkspaceAdapters).map(
-    plugin => plugin.flavour
-  );
-  const jotaiWorkspaces = get(rootWorkspacesMetadataAtom)
-    .filter(
-      workspace => flavours.includes(workspace.flavour)
-      // TODO: remove this when we remove the legacy cloud
-    )
-    .filter(workspace =>
-      !config.enableLegacyCloud
-        ? workspace.flavour !== WorkspaceFlavour.AFFINE
-        : true
+export const workspacesAtom = atom<Promise<AllWorkspace[]>>(
+  async (get, { signal }) => {
+    const { WorkspaceAdapters } = await import('../adapters/workspace');
+    const flavours: string[] = Object.values(WorkspaceAdapters).map(
+      plugin => plugin.flavour
     );
-  const workspaces = await Promise.all(
-    jotaiWorkspaces.map(workspace => {
-      const plugin =
-        WorkspaceAdapters[workspace.flavour as keyof typeof WorkspaceAdapters];
-      assertExists(plugin);
-      const { CRUD } = plugin;
-      return CRUD.get(workspace.id).then(workspace => {
-        if (workspace === null) {
-          console.warn(
-            'workspace is null. this should not happen. If you see this error, please report it to the developer.'
-          );
-        }
-        return workspace;
+    const jotaiWorkspaces = get(rootWorkspacesMetadataAtom)
+      .filter(
+        workspace => flavours.includes(workspace.flavour)
+        // TODO: remove this when we remove the legacy cloud
+      )
+      .filter(workspace =>
+        !config.enableLegacyCloud
+          ? workspace.flavour !== WorkspaceFlavour.AFFINE
+          : true
+      );
+    if (jotaiWorkspaces.some(meta => meta.version === undefined)) {
+      // wait until all workspaces have migrated to v2
+      await new Promise((resolve, reject) => {
+        signal.addEventListener('abort', reject);
+        setTimeout(resolve, 1000);
+      }).catch(() => {
+        // do nothing
       });
-    })
-  ).then(workspaces =>
-    workspaces.filter(
-      (workspace): workspace is WorkspaceRegistry['affine' | 'local'] =>
-        workspace !== null
-    )
-  );
-  const workspaceProviders = workspaces.map(workspace =>
-    workspace.blockSuiteWorkspace.providers.filter(
-      (provider): provider is ActiveDocProvider =>
-        'active' in provider && provider.active
-    )
-  );
-  const promises: Promise<void>[] = [];
-  for (const providers of workspaceProviders) {
-    for (const provider of providers) {
-      provider.sync();
-      promises.push(provider.whenReady);
     }
+    const workspaces = await Promise.all(
+      jotaiWorkspaces.map(workspace => {
+        const plugin =
+          WorkspaceAdapters[
+            workspace.flavour as keyof typeof WorkspaceAdapters
+          ];
+        assertExists(plugin);
+        const { CRUD } = plugin;
+        return CRUD.get(workspace.id).then(workspace => {
+          if (workspace === null) {
+            console.warn(
+              'workspace is null. this should not happen. If you see this error, please report it to the developer.'
+            );
+          }
+          return workspace;
+        });
+      })
+    ).then(workspaces =>
+      workspaces.filter(
+        (workspace): workspace is WorkspaceRegistry['affine' | 'local'] =>
+          workspace !== null
+      )
+    );
+    const workspaceProviders = workspaces.map(workspace =>
+      workspace.blockSuiteWorkspace.providers.filter(
+        (provider): provider is ActiveDocProvider =>
+          'active' in provider && provider.active
+      )
+    );
+    const promises: Promise<void>[] = [];
+    for (const providers of workspaceProviders) {
+      for (const provider of providers) {
+        provider.sync();
+        promises.push(provider.whenReady);
+      }
+    }
+    // we will wait for all the necessary providers to be ready
+    await Promise.all(promises);
+    logger.info('workspaces', workspaces);
+    return workspaces;
   }
-  // we will wait for all the necessary providers to be ready
-  await Promise.all(promises);
-  logger.info('workspaces', workspaces);
-  return workspaces;
-});
+);
 
 /**
  * This will throw an error if the workspace is not found,
@@ -79,7 +92,7 @@ export const workspacesAtom = atom<Promise<AllWorkspace[]>>(async get => {
  * use `rootCurrentWorkspaceIdAtom` instead
  */
 export const rootCurrentWorkspaceAtom = atom<Promise<AllWorkspace>>(
-  async get => {
+  async (get, { signal }) => {
     const { WorkspaceAdapters } = await import('../adapters/workspace');
     const metadata = get(rootWorkspacesMetadataAtom);
     const targetId = get(rootCurrentWorkspaceIdAtom);
@@ -92,6 +105,17 @@ export const rootCurrentWorkspaceAtom = atom<Promise<AllWorkspace>>(
     if (!targetWorkspace) {
       throw new Error(`cannot find the workspace with id ${targetId}.`);
     }
+
+    if (!targetWorkspace.version) {
+      // wait until the workspace has migrated to v2
+      await new Promise((resolve, reject) => {
+        signal.addEventListener('abort', reject);
+        setTimeout(resolve, 1000);
+      }).catch(() => {
+        // do nothing
+      });
+    }
+
     const workspace = await WorkspaceAdapters[targetWorkspace.flavour].CRUD.get(
       targetWorkspace.id
     );

apps/web/src/bootstrap/index.ts

@@ -1,4 +1,18 @@
+import { migrateToSubdoc } from '@affine/env/blocksuite';
 import { config, setupGlobal } from '@affine/env/config';
+import type { LocalIndexedDBDownloadProvider } from '@affine/env/workspace';
+import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import type { RootWorkspaceMetadata } from '@affine/workspace/atom';
+import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
+import {
+  migrateLocalBlobStorage,
+  upgradeV1ToV2,
+} from '@affine/workspace/migration';
+import { createIndexedDBDownloadProvider } from '@affine/workspace/providers';
+import { assertExists } from '@blocksuite/global/utils';
+import { rootStore } from '@toeverything/plugin-infra/manager';
+
+import { WorkspaceAdapters } from '../adapters/workspace';
 
 setupGlobal();
 
@@ -32,3 +46,66 @@ if (!environment.isDesktop && !environment.isServer) {
     writable: false,
   });
 }
+
+rootStore.sub(rootWorkspacesMetadataAtom, () => {
+  const metadata = rootStore.get(rootWorkspacesMetadataAtom);
+  metadata.forEach(oldMeta => {
+    if (!oldMeta.version) {
+      const adapter = WorkspaceAdapters[oldMeta.flavour];
+      assertExists(adapter);
+      const upgrade = async () => {
+        const workspace = await adapter.CRUD.get(oldMeta.id);
+        if (!workspace) {
+          console.warn('cannot find workspace', oldMeta.id);
+          return;
+        }
+        if (workspace.flavour !== WorkspaceFlavour.LOCAL) {
+          console.warn('not supported');
+          return;
+        }
+        const doc = workspace.blockSuiteWorkspace.doc;
+        const provider = createIndexedDBDownloadProvider(workspace.id, doc, {
+          awareness: workspace.blockSuiteWorkspace.awarenessStore.awareness,
+        }) as LocalIndexedDBDownloadProvider;
+        provider.sync();
+        await provider.whenReady;
+        const newDoc = migrateToSubdoc(doc);
+        if (doc === newDoc) {
+          console.log('doc not changed');
+          rootStore.set(rootWorkspacesMetadataAtom, metadata =>
+            metadata.map(newMeta =>
+              newMeta.id === oldMeta.id
+                ? {
+                    ...newMeta,
+                    version: WorkspaceVersion.SubDoc,
+                  }
+                : newMeta
+            )
+          );
+          return;
+        }
+        const newWorkspace = upgradeV1ToV2(workspace);
+
+        const newId = await adapter.CRUD.create(
+          newWorkspace.blockSuiteWorkspace
+        );
+
+        await adapter.CRUD.delete(workspace as any);
+        await migrateLocalBlobStorage(workspace.id, newId);
+        rootStore.set(rootWorkspacesMetadataAtom, metadata => [
+          ...metadata
+            .map(newMeta => (newMeta.id === oldMeta.id ? null : newMeta))
+            .filter((meta): meta is RootWorkspaceMetadata => !!meta),
+          {
+            id: newId,
+            flavour: oldMeta.flavour,
+            version: WorkspaceVersion.SubDoc,
+          },
+        ]);
+      };
+
+      // create a new workspace and push it to metadata
+      upgrade().catch(console.error);
+    }
+  });
+});

apps/web/src/hooks/use-transform-workspace.ts

@@ -1,5 +1,6 @@
 import type { WorkspaceFlavour } from '@affine/env/workspace';
 import type { WorkspaceRegistry } from '@affine/env/workspace';
+import { WorkspaceVersion } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { useSetAtom } from 'jotai';
 import { useCallback } from 'react';
@@ -7,7 +8,7 @@ import { useCallback } from 'react';
 import { WorkspaceAdapters } from '../adapters/workspace';
 
 /**
- * Transform workspace from one flavour to another
+ * Transform workspace from one flavor to another
  *
  * The logic here is to delete the old workspace and create a new one.
  */
@@ -29,6 +30,7 @@ export function useTransformWorkspace() {
         workspaces.splice(idx, 1, {
           id: newId,
           flavour: to,
+          version: WorkspaceVersion.SubDoc,
         });
         return [...workspaces];
       });

apps/web/src/hooks/use-workspaces.ts

@@ -1,5 +1,5 @@
 import { DebugLogger } from '@affine/debug';
-import { WorkspaceFlavour } from '@affine/env/workspace';
+import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud';
 import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
@@ -34,6 +34,7 @@ export function useAppHelper() {
           {
             id: workspaceId,
             flavour: WorkspaceFlavour.LOCAL,
+            version: WorkspaceVersion.SubDoc,
           },
         ]);
         logger.debug('imported local workspace', workspaceId);
@@ -54,6 +55,7 @@ export function useAppHelper() {
           {
             id,
             flavour: WorkspaceFlavour.LOCAL,
+            version: WorkspaceVersion.SubDoc,
           },
         ]);
         logger.debug('created local workspace', id);

apps/web/src/layouts/workspace-layout.tsx

@@ -267,7 +267,13 @@ export const WorkspaceLayout: FC<PropsWithChildren> =
                 );
               });
             }
-            items.push(...item.map(x => ({ id: x.id, flavour: x.flavour })));
+            items.push(
+              ...item.map(x => ({
+                id: x.id,
+                flavour: x.flavour,
+                version: undefined,
+              }))
+            );
           } catch (e) {
             logger.error('list data error:', e);
           }

apps/web/src/pages/_debug/migration.tsx

@@ -1,58 +1,119 @@
 import { BlockSuiteEditor } from '@affine/component/block-suite-editor';
-import { migrateToSubdoc } from '@affine/env/blocksuite';
+import type {
+  LocalIndexedDBDownloadProvider,
+  LocalWorkspace,
+} from '@affine/env/workspace';
+import { WorkspaceFlavour } from '@affine/env/workspace';
+import {
+  migrateLocalBlobStorage,
+  upgradeV1ToV2,
+} from '@affine/workspace/migration';
+import { createIndexedDBDownloadProvider } from '@affine/workspace/providers';
 import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
 import { Workspace } from '@blocksuite/store';
 import { NoSsr } from '@mui/material';
+import {
+  atom,
+  createStore,
+  Provider,
+  useAtom,
+  useAtomValue,
+  useSetAtom,
+} from 'jotai';
 import type { ReactElement } from 'react';
-import { use, useCallback } from 'react';
-import * as Y from 'yjs';
-const { default: json } = await import('@affine-test/fixtures/output.json');
+import { Suspense, use, useCallback } from 'react';
 
-const workspace = new Workspace({
-  id: 'test-migration',
-  isSSR: typeof window === 'undefined',
+const store = createStore();
+
+const workspaceIdsAtom = atom<Promise<string[]>>(async () => {
+  if (typeof window === 'undefined') {
+    return [];
+  } else {
+    const idb = await import('idb');
+    const db = await idb.openDB('affine-local', 1);
+    return (await db
+      .transaction('workspace')
+      .objectStore('workspace')
+      .getAllKeys()) as string[];
+  }
 });
 
-const finalWorkspace = new Workspace({
-  id: 'test-migration-final',
-  isSSR: typeof window === 'undefined',
-});
+const targetIdAtom = atom<string | null>(null);
 
-finalWorkspace.register(AffineSchemas).register(__unstableSchemas);
-workspace.register(AffineSchemas).register(__unstableSchemas);
-
-if (typeof window !== 'undefined') {
-  const length = Object.keys(json).length;
-  const binary = new Uint8Array(length);
-  for (let i = 0; i < length; i++) {
-    binary[i] = (json as any)[i];
+const workspaceAtom = atom<Promise<Workspace>>(async get => {
+  const id = get(targetIdAtom);
+  if (!id) {
+    throw new Error('no id');
   }
-  Y.applyUpdate(workspace.doc, binary);
-  {
-    // invoke data
-    workspace.doc.getMap('space:hello-world');
-    workspace.doc.getMap('space:meta');
-  }
-  const newDoc = migrateToSubdoc(workspace.doc);
-  Y.applyUpdate(finalWorkspace.doc, Y.encodeStateAsUpdate(newDoc));
-  finalWorkspace.doc.subdocs.forEach(finalSubdoc => {
-    newDoc.subdocs.forEach(subdoc => {
-      if (subdoc.guid === finalSubdoc.guid) {
-        Y.applyUpdate(finalSubdoc, Y.encodeStateAsUpdate(subdoc));
-      }
-    });
+  const workspace = new Workspace({
+    id,
+    isSSR: typeof window === 'undefined',
   });
-}
 
-const MigrationInner = () => {
-  const page = finalWorkspace.getPage('hello-world');
+  workspace.register(AffineSchemas).register(__unstableSchemas);
+  const provider = createIndexedDBDownloadProvider(
+    workspace.id,
+    workspace.doc,
+    {
+      awareness: workspace.awarenessStore.awareness,
+    }
+  ) as LocalIndexedDBDownloadProvider;
+  provider.sync();
+  await provider.whenReady;
+  const localWorkspace = {
+    id: workspace.id,
+    blockSuiteWorkspace: workspace,
+    flavour: WorkspaceFlavour.LOCAL,
+  } satisfies LocalWorkspace;
+  const newWorkspace = upgradeV1ToV2(localWorkspace);
+  await migrateLocalBlobStorage(localWorkspace.id, newWorkspace.id);
+  newWorkspace.blockSuiteWorkspace;
+  return newWorkspace.blockSuiteWorkspace;
+});
+
+const pageIdAtom = atom('hello-world');
+
+const PageListSelect = () => {
+  const workspace = useAtomValue(workspaceAtom);
+  const setPageId = useSetAtom(pageIdAtom);
+  return (
+    <ul>
+      {workspace.meta.pageMetas.map(meta => (
+        <li
+          key={meta.id}
+          onClick={() => {
+            setPageId(meta.id);
+          }}
+        >
+          {meta.id}
+        </li>
+      ))}
+    </ul>
+  );
+};
+
+const WorkspaceInner = () => {
+  const workspace = useAtomValue(workspaceAtom);
+  const pageId = useAtomValue(pageIdAtom);
+  const page = workspace.getPage(pageId);
   const onInit = useCallback(() => {}, []);
   if (!page) {
-    return <>loading...</>;
+    return <PageListSelect />;
   }
   if (!page.loaded) {
     use(page.waitForLoaded());
   }
+  return (
+    <>
+      <PageListSelect />
+      <BlockSuiteEditor page={page} mode="page" onInit={onInit} />;
+    </>
+  );
+};
+
+const MigrationInner = () => {
+  const ids = useAtomValue(workspaceIdsAtom);
+  const [id, setId] = useAtom(targetIdAtom);
   return (
     <div
       style={{
@@ -60,15 +121,31 @@ const MigrationInner = () => {
         height: '100vh',
       }}
     >
-      <BlockSuiteEditor page={page} mode="page" onInit={onInit} />
+      <ul>
+        {ids.map(id => (
+          <li
+            onClick={() => {
+              setId(id);
+            }}
+            key={id}
+          >
+            {id}
+          </li>
+        ))}
+      </ul>
+      <Suspense fallback="loading...">{id && <WorkspaceInner />}</Suspense>
     </div>
   );
 };
 
 export default function MigrationPage(): ReactElement {
   return (
-    <NoSsr>
-      <MigrationInner />
-    </NoSsr>
+    <Provider store={store}>
+      <NoSsr>
+        <Suspense>
+          <MigrationInner />
+        </Suspense>
+      </NoSsr>
+    </Provider>
   );
 }