feat!: unified migration logic in server electron, and browser (#4079)

Co-authored-by: Mirone <Saul-Mirone@outlook.com>

apps/core/src/adapters/local/index.tsx

@@ -15,7 +15,10 @@ import {
   CRUD,
   saveWorkspaceToLocalStorage,
 } from '@affine/workspace/local/crud';
-import { getOrCreateWorkspace } from '@affine/workspace/manager';
+import {
+  getOrCreateWorkspace,
+  globalBlockSuiteSchema,
+} from '@affine/workspace/manager';
 import { createIndexedDBDownloadProvider } from '@affine/workspace/providers';
 import { nanoid } from '@blocksuite/store';
 import { useStaticBlockSuiteWorkspace } from '@toeverything/infra/__internal__/react';
@@ -49,6 +52,7 @@ export const LocalAdapter: WorkspaceAdapter<WorkspaceFlavour.LOCAL> = {
       blockSuiteWorkspace.meta.setName(DEFAULT_WORKSPACE_NAME);
       if (runtimeConfig.enablePreloading) {
         buildShowcaseWorkspace(blockSuiteWorkspace, {
+          schema: globalBlockSuiteSchema,
           store: getCurrentStore(),
           atoms: {
             pageMode: setPageModeAtom,

apps/core/src/bootstrap/setup.ts

@@ -18,10 +18,9 @@ import {
   migrateWorkspace,
   WorkspaceVersion,
 } from '@toeverything/infra/blocksuite';
-import { downloadBinary } from '@toeverything/y-indexeddb';
+import { downloadBinary, overwriteBinary } from '@toeverything/y-indexeddb';
 import type { createStore } from 'jotai/vanilla';
-import { Doc } from 'yjs';
-import { applyUpdate } from 'yjs';
+import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
 
 import { WorkspaceAdapters } from '../adapters/workspace';
 
@@ -34,37 +33,39 @@ async function tryMigration() {
       const newMetadata = [...metadata];
       metadata.forEach(oldMeta => {
         if (oldMeta.flavour === WorkspaceFlavour.LOCAL) {
+          let doc: YDoc;
+          const options = {
+            getCurrentRootDoc: async () => {
+              doc = new YDoc({
+                guid: oldMeta.id,
+              });
+              const downloadWorkspace = async (doc: YDoc): Promise<void> => {
+                const binary = await downloadBinary(doc.guid);
+                if (binary) {
+                  applyUpdate(doc, binary);
+                }
+                await Promise.all(
+                  [...doc.subdocs.values()].map(subdoc =>
+                    downloadWorkspace(subdoc)
+                  )
+                );
+              };
+              await downloadWorkspace(doc);
+              return doc;
+            },
+            createWorkspace: async () =>
+              getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL),
+            getSchema: () => globalBlockSuiteSchema,
+          };
           promises.push(
             migrateWorkspace(
               'version' in oldMeta ? oldMeta.version : undefined,
-              {
-                getCurrentRootDoc: async () => {
-                  const doc = new Doc({
-                    guid: oldMeta.id,
-                  });
-                  const downloadWorkspace = async (doc: Doc): Promise<void> => {
-                    const binary = await downloadBinary(doc.guid);
-                    if (binary) {
-                      applyUpdate(doc, binary);
-                    }
-                    return Promise.all(
-                      [...doc.subdocs.values()].map(subdoc =>
-                        downloadWorkspace(subdoc)
-                      )
-                    ).then();
-                  };
-                  await downloadWorkspace(doc);
-                  return doc;
-                },
-                createWorkspace: async () =>
-                  getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL),
-                getSchema: () => globalBlockSuiteSchema,
-              }
-            ).then(async workspace => {
-              if (typeof workspace !== 'boolean') {
+              options
+            ).then(async status => {
+              if (typeof status !== 'boolean') {
                 const adapter = WorkspaceAdapters[oldMeta.flavour];
                 const oldWorkspace = await adapter.CRUD.get(oldMeta.id);
-                const newId = await adapter.CRUD.create(workspace);
+                const newId = await adapter.CRUD.create(status);
                 assertExists(
                   oldWorkspace,
                   'workspace should exist after migrate'
@@ -76,11 +77,25 @@ async function tryMigration() {
                 newMetadata[index] = {
                   ...oldMeta,
                   id: newId,
-                  version: WorkspaceVersion.DatabaseV3,
+                  version: WorkspaceVersion.Surface,
                 };
-                await migrateLocalBlobStorage(workspace.id, newId);
+                await migrateLocalBlobStorage(status.id, newId);
                 console.log('workspace migrated', oldMeta.id, newId);
-              } else if (workspace) {
+              } else if (status) {
+                const index = newMetadata.findIndex(
+                  meta => meta.id === oldMeta.id
+                );
+                newMetadata[index] = {
+                  ...oldMeta,
+                  version: WorkspaceVersion.Surface,
+                };
+                const overWrite = async (doc: YDoc): Promise<void> => {
+                  await overwriteBinary(doc.guid, encodeStateAsUpdate(doc));
+                  return Promise.all(
+                    [...doc.subdocs.values()].map(subdoc => overWrite(subdoc))
+                  ).then();
+                };
+                await overWrite(doc);
                 console.log('workspace migrated', oldMeta.id);
               }
             })
@@ -106,7 +121,7 @@ async function tryMigration() {
   }
 }
 
-function createFirstAppData(store: ReturnType<typeof createStore>) {
+export function createFirstAppData(store: ReturnType<typeof createStore>) {
   const createFirst = (): RootWorkspaceMetadataV2[] => {
     const Plugins = Object.values(WorkspaceAdapters).sort(
       (a, b) => a.loadPriority - b.loadPriority
@@ -144,8 +159,8 @@ export async function setup(store: ReturnType<typeof createStore>) {
   console.log('setup global');
   setupGlobal();
 
-  createFirstAppData(store);
   await tryMigration();
+  // do not read `rootWorkspacesMetadataAtom` before migration
   await store.get(rootWorkspacesMetadataAtom);
   console.log('setup done');
 }

apps/core/src/components/migration-fallback.tsx

@@ -0,0 +1,111 @@
+import type {
+  AffineSocketIOProvider,
+  LocalIndexedDBBackgroundProvider,
+  SQLiteProvider,
+} from '@affine/env/workspace';
+import { assertExists } from '@blocksuite/global/utils';
+import { Button } from '@toeverything/components/button';
+import { forceUpgradePages } from '@toeverything/infra/blocksuite';
+import { useCallback, useMemo, useState } from 'react';
+import type { Doc as YDoc } from 'yjs';
+import { applyUpdate, encodeStateAsUpdate, encodeStateVector } from 'yjs';
+
+import { useCurrentWorkspace } from '../hooks/current/use-current-workspace';
+
+export const MigrationFallback = function MigrationFallback() {
+  const [done, setDone] = useState(false);
+  const [workspace] = useCurrentWorkspace();
+  const providers = workspace.blockSuiteWorkspace.providers;
+  const remoteProvider: AffineSocketIOProvider | undefined = useMemo(() => {
+    return providers.find(
+      (provider): provider is AffineSocketIOProvider =>
+        provider.flavour === 'affine-socket-io'
+    );
+  }, [providers]);
+  const localProvider = useMemo(() => {
+    const sqliteProvider = providers.find(
+      (provider): provider is SQLiteProvider => provider.flavour === 'sqlite'
+    );
+    const indexedDbProvider = providers.find(
+      (provider): provider is LocalIndexedDBBackgroundProvider =>
+        provider.flavour === 'local-indexeddb-background'
+    );
+    const provider = sqliteProvider || indexedDbProvider;
+    assertExists(provider, 'no local provider');
+    return provider;
+  }, [providers]);
+  const handleClick = useCallback(async () => {
+    setDone(false);
+    const downloadRecursively = async (doc: YDoc) => {
+      {
+        const docState = await localProvider.datasource.queryDocState(
+          doc.guid,
+          {
+            stateVector: encodeStateVector(doc),
+          }
+        );
+        console.log('download indexeddb', doc.guid);
+        if (docState) {
+          applyUpdate(doc, docState.missing, 'migration');
+        }
+      }
+      if (remoteProvider) {
+        {
+          const docState = await remoteProvider.datasource.queryDocState(
+            doc.guid,
+            {
+              stateVector: encodeStateVector(doc),
+            }
+          );
+          console.log('download remote', doc.guid);
+          if (docState) {
+            applyUpdate(doc, docState.missing, 'migration');
+          }
+        }
+      }
+      await Promise.all(
+        [...doc.subdocs].map(async subdoc => {
+          await downloadRecursively(subdoc);
+        })
+      );
+      {
+        await localProvider.datasource.sendDocUpdate(
+          doc.guid,
+          encodeStateAsUpdate(doc)
+        );
+        console.log('upload indexeddb', doc.guid);
+        if (remoteProvider) {
+          await remoteProvider.datasource.sendDocUpdate(
+            doc.guid,
+            encodeStateAsUpdate(doc)
+          );
+          console.log('upload remote', doc.guid);
+        }
+      }
+    };
+
+    await downloadRecursively(workspace.blockSuiteWorkspace.doc);
+    console.log('download done');
+
+    console.log('start migration');
+    await forceUpgradePages({
+      getCurrentRootDoc: async () => workspace.blockSuiteWorkspace.doc,
+      getSchema: () => workspace.blockSuiteWorkspace.schema,
+    });
+    console.log('migration done');
+    setDone(true);
+  }, [
+    localProvider.datasource,
+    remoteProvider,
+    workspace.blockSuiteWorkspace.doc,
+    workspace.blockSuiteWorkspace.schema,
+  ]);
+  if (done) {
+    return <div>Done, please refresh the page.</div>;
+  }
+  return (
+    <Button data-testid="upgrade-workspace" onClick={handleClick}>
+      Upgrade Workspace
+    </Button>
+  );
+};

apps/core/src/hooks/use-workspaces.ts

@@ -2,7 +2,10 @@ import { DebugLogger } from '@affine/debug';
 import { WorkspaceFlavour } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud';
-import { getOrCreateWorkspace } from '@affine/workspace/manager';
+import {
+  getOrCreateWorkspace,
+  globalBlockSuiteSchema,
+} from '@affine/workspace/manager';
 import { nanoid } from '@blocksuite/store';
 import { getWorkspace } from '@toeverything/infra/__internal__/workspace';
 import { getCurrentStore } from '@toeverything/infra/atom';
@@ -73,6 +76,7 @@ export function useAppHelper() {
             WorkspaceFlavour.LOCAL
           );
           await buildShowcaseWorkspace(blockSuiteWorkspace, {
+            schema: globalBlockSuiteSchema,
             store: getCurrentStore(),
             atoms: {
               pageMode: setPageModeAtom,

apps/core/src/layouts/workspace-layout.tsx

@@ -47,6 +47,7 @@ import { useAppSetting } from '../atoms/settings';
 import { AdapterProviderWrapper } from '../components/adapter-worksapce-wrapper';
 import { AppContainer } from '../components/affine/app-container';
 import { usePageHelper } from '../components/blocksuite/block-suite-page-list/utils';
+import { MigrationFallback } from '../components/migration-fallback';
 import type { IslandItemNames } from '../components/pure/help-island';
 import { HelpIsland } from '../components/pure/help-island';
 import { processCollectionsDrag } from '../components/pure/workspace-slider-bar/collections';
@@ -112,9 +113,14 @@ export const CurrentWorkspaceContext = ({
   return <>{children}</>;
 };
 
+type WorkspaceLayoutProps = {
+  incompatible?: boolean;
+};
+
 export const WorkspaceLayout = function WorkspacesSuspense({
   children,
-}: PropsWithChildren) {
+  incompatible = false,
+}: PropsWithChildren<WorkspaceLayoutProps>) {
   return (
     <AdapterProviderWrapper>
       <CurrentWorkspaceContext>
@@ -124,14 +130,19 @@ export const WorkspaceLayout = function WorkspacesSuspense({
           <CurrentWorkspaceModals />
         </Suspense>
         <Suspense fallback={<WorkspaceFallback />}>
-          <WorkspaceLayoutInner>{children}</WorkspaceLayoutInner>
+          <WorkspaceLayoutInner incompatible={incompatible}>
+            {children}
+          </WorkspaceLayoutInner>
         </Suspense>
       </CurrentWorkspaceContext>
     </AdapterProviderWrapper>
   );
 };
 
-export const WorkspaceLayoutInner = ({ children }: PropsWithChildren) => {
+export const WorkspaceLayoutInner = ({
+  children,
+  incompatible = false,
+}: PropsWithChildren<WorkspaceLayoutProps>) => {
   const [currentWorkspace] = useCurrentWorkspace();
   const { openPage } = useNavigateHelper();
 
@@ -263,7 +274,7 @@ export const WorkspaceLayoutInner = ({ children }: PropsWithChildren) => {
               ref={setMainContainer}
               padding={appSetting.clientBorder}
             >
-              {children}
+              {incompatible ? <MigrationFallback /> : children}
               <ToolContainer>
                 <BlockHubWrapper blockHubAtom={rootBlockHubAtom} />
                 <HelpIsland showList={pageId ? undefined : showList} />

apps/core/src/pages/index.tsx

@@ -17,6 +17,8 @@ const logger = new DebugLogger('index-page');
 
 export const loader: LoaderFunction = async () => {
   const rootStore = getCurrentStore();
+  const { createFirstAppData } = await import('../bootstrap/setup');
+  createFirstAppData(rootStore);
   const meta = await rootStore.get(rootWorkspacesMetadataAtom);
   const lastId = localStorage.getItem('last_workspace_id');
   const lastPageId = localStorage.getItem('last_page_id');
@@ -52,9 +54,5 @@ export const loader: LoaderFunction = async () => {
 };
 
 export const Component = () => {
-  return (
-    <>
-      <AllWorkspaceModals />
-    </>
-  );
+  return <AllWorkspaceModals />;
 };

apps/core/src/pages/workspace/index.tsx

@@ -1,18 +1,27 @@
+import { WorkspaceFlavour } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
+import { assertExists } from '@blocksuite/global/utils';
+import { getActiveBlockSuiteWorkspaceAtom } from '@toeverything/infra/__internal__/workspace';
 import {
   currentPageIdAtom,
   currentWorkspaceIdAtom,
   getCurrentStore,
 } from '@toeverything/infra/atom';
 import type { ReactElement } from 'react';
-import { type LoaderFunction, Outlet, redirect } from 'react-router-dom';
+import {
+  type LoaderFunction,
+  Outlet,
+  redirect,
+  useLoaderData,
+} from 'react-router-dom';
 
 import { WorkspaceLayout } from '../../layouts/workspace-layout';
 
 export const loader: LoaderFunction = async args => {
   const rootStore = getCurrentStore();
   const meta = await rootStore.get(rootWorkspacesMetadataAtom);
-  if (!meta.some(({ id }) => id === args.params.workspaceId)) {
+  const currentMetadata = meta.find(({ id }) => id === args.params.workspaceId);
+  if (!currentMetadata) {
     return redirect('/404');
   }
   if (args.params.workspaceId) {
@@ -22,12 +31,27 @@ export const loader: LoaderFunction = async args => {
   if (!args.params.pageId) {
     rootStore.set(currentPageIdAtom, null);
   }
+  if (currentMetadata.flavour === WorkspaceFlavour.AFFINE_CLOUD) {
+    const workspaceAtom = getActiveBlockSuiteWorkspaceAtom(currentMetadata.id);
+    const workspace = await rootStore.get(workspaceAtom);
+    return (() => {
+      const blockVersions = workspace.meta.blockVersions;
+      assertExists(blockVersions, 'blockVersions should not be null');
+      for (const [flavour, schema] of workspace.schema.flavourSchemaMap) {
+        if (blockVersions[flavour] !== schema.version) {
+          return true;
+        }
+      }
+      return false;
+    })();
+  }
   return null;
 };
 
 export const Component = (): ReactElement => {
+  const incompatible = useLoaderData();
   return (
-    <WorkspaceLayout>
+    <WorkspaceLayout incompatible={!!incompatible}>
       <Outlet />
     </WorkspaceLayout>
   );