refactor: migration logic (#3973)

apps/core/src/bootstrap/setup.ts

@@ -1,27 +1,27 @@
-import {
-  migrateDatabaseBlockTo3,
-  migrateToSubdoc,
-} from '@affine/env/blocksuite';
 import { setupGlobal } from '@affine/env/global';
-import type {
+import type { WorkspaceAdapter } from '@affine/env/workspace';
-  LocalIndexedDBDownloadProvider,
+import { WorkspaceFlavour } from '@affine/env/workspace';
-  WorkspaceAdapter,
-} from '@affine/env/workspace';
-import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
 import type { RootWorkspaceMetadata } from '@affine/workspace/atom';
 import {
   type RootWorkspaceMetadataV2,
   rootWorkspacesMetadataAtom,
   workspaceAdaptersAtom,
 } from '@affine/workspace/atom';
-import { globalBlockSuiteSchema } from '@affine/workspace/manager';
+import {
+  getOrCreateWorkspace,
+  globalBlockSuiteSchema,
+} from '@affine/workspace/manager';
+import { assertExists } from '@blocksuite/global/utils';
+import { nanoid } from '@blocksuite/store';
 import {
   migrateLocalBlobStorage,
-  upgradeV1ToV2,
+  migrateWorkspace,
-} from '@affine/workspace/migration';
+  WorkspaceVersion,
-import { createIndexedDBDownloadProvider } from '@affine/workspace/providers';
+} from '@toeverything/infra/blocksuite';
-import { assertExists } from '@blocksuite/global/utils';
+import { downloadBinary } from '@toeverything/y-indexeddb';
 import type { createStore } from 'jotai/vanilla';
+import { Doc } from 'yjs';
+import { applyUpdate } from 'yjs';
 
 import { WorkspaceAdapters } from '../adapters/workspace';
 
@@ -33,94 +33,57 @@ async function tryMigration() {
       const promises: Promise<void>[] = [];
       const newMetadata = [...metadata];
       metadata.forEach(oldMeta => {
-        if (!('version' in oldMeta)) {
+        if (oldMeta.flavour === WorkspaceFlavour.LOCAL) {
-          const adapter = WorkspaceAdapters[oldMeta.flavour];
-          assertExists(adapter);
-          const upgrade = async () => {
-            if (oldMeta.flavour !== WorkspaceFlavour.LOCAL) {
-              console.warn('not supported');
-              return;
-            }
-            const workspace = await adapter.CRUD.get(oldMeta.id);
-            if (!workspace) {
-              console.warn('cannot find workspace', oldMeta.id);
-              return;
-            }
-            const doc = workspace.blockSuiteWorkspace.doc;
-            const provider = createIndexedDBDownloadProvider(
-              workspace.id,
-              doc,
-              {
-                awareness:
-                  workspace.blockSuiteWorkspace.awarenessStore.awareness,
-              }
-            ) as LocalIndexedDBDownloadProvider;
-            provider.sync();
-            await provider.whenReady;
-            const newDoc = migrateToSubdoc(doc);
-            if (doc === newDoc) {
-              console.log('doc not changed');
-              return;
-            }
-            const newWorkspace = upgradeV1ToV2(workspace);
-            await migrateDatabaseBlockTo3(
-              newWorkspace.blockSuiteWorkspace.doc,
-              globalBlockSuiteSchema
-            );
-
-            const newId = await adapter.CRUD.create(
-              newWorkspace.blockSuiteWorkspace
-            );
-
-            await adapter.CRUD.delete(workspace as any);
-            console.log('migrated', oldMeta.id, newId);
-            const index = newMetadata.findIndex(meta => meta.id === oldMeta.id);
-            newMetadata[index] = {
-              ...oldMeta,
-              id: newId,
-              version: WorkspaceVersion.DatabaseV3,
-            };
-            await migrateLocalBlobStorage(workspace.id, newId);
-            console.log('migrate to v2');
-          };
-
-          // create a new workspace and push it to metadata
-          promises.push(upgrade());
-        } else if (oldMeta.version < WorkspaceVersion.DatabaseV3) {
-          const adapter = WorkspaceAdapters[oldMeta.flavour];
-          assertExists(adapter);
           promises.push(
-            (async () => {
+            migrateWorkspace(
-              if (oldMeta.flavour !== WorkspaceFlavour.LOCAL) {
+              'version' in oldMeta ? oldMeta.version : undefined,
-                console.warn('not supported');
+              {
-                return;
+                getCurrentRootDoc: async () => {
+                  const doc = new Doc({
+                    guid: oldMeta.id,
+                  });
+                  const downloadWorkspace = async (doc: Doc): Promise<void> => {
+                    const binary = await downloadBinary(doc.guid);
+                    if (binary) {
+                      applyUpdate(doc, binary);
+                    }
+                    return Promise.all(
+                      [...doc.subdocs.values()].map(subdoc =>
+                        downloadWorkspace(subdoc)
+                      )
+                    ).then();
+                  };
+                  await downloadWorkspace(doc);
+                  return doc;
+                },
+                createWorkspace: async () =>
+                  getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL),
+                getSchema: () => globalBlockSuiteSchema,
               }
-              const workspace = await adapter.CRUD.get(oldMeta.id);
+            ).then(async workspace => {
-              if (workspace) {
+              if (typeof workspace !== 'boolean') {
-                const provider = createIndexedDBDownloadProvider(
+                const adapter = WorkspaceAdapters[oldMeta.flavour];
-                  workspace.id,
+                const oldWorkspace = await adapter.CRUD.get(oldMeta.id);
-                  workspace.blockSuiteWorkspace.doc,
+                const newId = await adapter.CRUD.create(workspace);
-                  {
+                assertExists(
-                    awareness:
+                  oldWorkspace,
-                      workspace.blockSuiteWorkspace.awarenessStore.awareness,
+                  'workspace should exist after migrate'
-                  }
-                ) as LocalIndexedDBDownloadProvider;
-                provider.sync();
-                await provider.whenReady;
-                await migrateDatabaseBlockTo3(
-                  workspace.blockSuiteWorkspace.doc,
-                  globalBlockSuiteSchema
                 );
+                await adapter.CRUD.delete(oldWorkspace.blockSuiteWorkspace);
+                const index = newMetadata.findIndex(
+                  meta => meta.id === oldMeta.id
+                );
+                newMetadata[index] = {
+                  ...oldMeta,
+                  id: newId,
+                  version: WorkspaceVersion.DatabaseV3,
+                };
+                await migrateLocalBlobStorage(workspace.id, newId);
+                console.log('workspace migrated', oldMeta.id, newId);
+              } else if (workspace) {
+                console.log('workspace migrated', oldMeta.id);
               }
-              const index = newMetadata.findIndex(
+            })
-                meta => meta.id === oldMeta.id
-              );
-              newMetadata[index] = {
-                ...oldMeta,
-                version: WorkspaceVersion.DatabaseV3,
-              };
-              console.log('migrate to v3');
-            })()
           );
         }
       });

apps/core/src/hooks/use-transform-workspace.ts

@@ -1,7 +1,7 @@
 import type { WorkspaceFlavour } from '@affine/env/workspace';
 import type { WorkspaceRegistry } from '@affine/env/workspace';
-import { WorkspaceVersion } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
+import { WorkspaceVersion } from '@toeverything/infra/blocksuite';
 import { useSetAtom } from 'jotai';
 import { useCallback } from 'react';
 

apps/core/src/hooks/use-workspaces.ts

@@ -1,12 +1,15 @@
 import { DebugLogger } from '@affine/debug';
-import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import { WorkspaceFlavour } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud';
 import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import { nanoid } from '@blocksuite/store';
 import { getWorkspace } from '@toeverything/infra/__internal__/workspace';
 import { getCurrentStore } from '@toeverything/infra/atom';
-import { buildShowcaseWorkspace } from '@toeverything/infra/blocksuite';
+import {
+  buildShowcaseWorkspace,
+  WorkspaceVersion,
+} from '@toeverything/infra/blocksuite';
 import { useAtomValue, useSetAtom } from 'jotai';
 import { useCallback } from 'react';
 

apps/electron/src/helper/db/migration.ts

@@ -1,7 +1,7 @@
 import { resolve } from 'node:path';
 
-import { migrateToSubdoc } from '@affine/env/blocksuite';
 import { SqliteConnection } from '@affine/native';
+import { migrateToSubdoc } from '@toeverything/infra/blocksuite';
 import fs from 'fs-extra';
 import { nanoid } from 'nanoid';
 import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';

packages/env/src/blocksuite/index.ts

@@ -12,5 +12,3 @@ export async function initEmptyPage(page: Page, title?: string) {
   const noteBlockId = page.addBlock('affine:note', {}, pageBlockId);
   page.addBlock('affine:paragraph', {}, noteBlockId);
 }
-
-export * from './subdoc-migration.js';

packages/env/src/blocksuite/subdoc-migration.ts

@@ -1,267 +0,0 @@
-import type { Schema } from '@blocksuite/store';
-import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs';
-
-type XYWH = [number, number, number, number];
-
-function deserializeXYWH(xywh: string): XYWH {
-  return JSON.parse(xywh) as XYWH;
-}
-
-function migrateDatabase(data: YMap<unknown>) {
-  data.delete('prop:mode');
-  data.set('prop:views', new YArray());
-  const columns = (data.get('prop:columns') as YArray<unknown>).toJSON() as {
-    id: string;
-    name: string;
-    hide: boolean;
-    type: string;
-    width: number;
-    selection?: unknown[];
-  }[];
-  const views = [
-    {
-      id: 'default',
-      name: 'Table',
-      columns: columns.map(col => ({
-        id: col.id,
-        width: col.width,
-        hide: col.hide,
-      })),
-      filter: { type: 'group', op: 'and', conditions: [] },
-      mode: 'table',
-    },
-  ];
-  const cells = (data.get('prop:cells') as YMap<unknown>).toJSON() as Record<
-    string,
-    Record<
-      string,
-      {
-        id: string;
-        value: unknown;
-      }
-    >
-  >;
-  const convertColumn = (
-    id: string,
-    update: (cell: { id: string; value: unknown }) => void
-  ) => {
-    Object.values(cells).forEach(row => {
-      if (row[id] != null) {
-        update(row[id]);
-      }
-    });
-  };
-  const newColumns = columns.map(v => {
-    let data: Record<string, unknown> = {};
-    if (v.type === 'select' || v.type === 'multi-select') {
-      data = { options: v.selection };
-      if (v.type === 'select') {
-        convertColumn(v.id, cell => {
-          if (Array.isArray(cell.value)) {
-            cell.value = cell.value[0]?.id;
-          }
-        });
-      } else {
-        convertColumn(v.id, cell => {
-          if (Array.isArray(cell.value)) {
-            cell.value = cell.value.map(v => v.id);
-          }
-        });
-      }
-    }
-    if (v.type === 'number') {
-      convertColumn(v.id, cell => {
-        if (typeof cell.value === 'string') {
-          cell.value = Number.parseFloat(cell.value.toString());
-        }
-      });
-    }
-    return {
-      id: v.id,
-      type: v.type,
-      name: v.name,
-      data,
-    };
-  });
-  data.set('prop:columns', newColumns);
-  data.set('prop:views', views);
-  data.set('prop:cells', cells);
-}
-
-function runBlockMigration(
-  flavour: string,
-  data: YMap<unknown>,
-  version: number
-) {
-  if (flavour === 'affine:frame') {
-    data.set('sys:flavour', 'affine:note');
-    return;
-  }
-  if (flavour === 'affine:surface' && version <= 3) {
-    if (data.has('elements')) {
-      const elements = data.get('elements') as YMap<unknown>;
-      migrateSurface(elements);
-      data.set('prop:elements', elements.clone());
-      data.delete('elements');
-    } else {
-      data.set('prop:elements', new YMap());
-    }
-  }
-  if (flavour === 'affine:embed') {
-    data.set('sys:flavour', 'affine:image');
-    data.delete('prop:type');
-  }
-  if (flavour === 'affine:database' && version < 2) {
-    migrateDatabase(data);
-  }
-}
-
-function migrateSurface(data: YMap<unknown>) {
-  for (const [, value] of <IterableIterator<[string, YMap<unknown>]>>(
-    data.entries()
-  )) {
-    if (value.get('type') === 'connector') {
-      migrateSurfaceConnector(value);
-    }
-  }
-}
-
-function migrateSurfaceConnector(data: YMap<any>) {
-  let id = data.get('startElement')?.id;
-  const controllers = data.get('controllers');
-  const length = controllers.length;
-  const xywh = deserializeXYWH(data.get('xywh'));
-  if (id) {
-    data.set('source', { id });
-  } else {
-    data.set('source', {
-      position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]],
-    });
-  }
-
-  id = data.get('endElement')?.id;
-  if (id) {
-    data.set('target', { id });
-  } else {
-    data.set('target', {
-      position: [
-        controllers[length - 1].x + xywh[0],
-        controllers[length - 1].y + xywh[1],
-      ],
-    });
-  }
-
-  const width = data.get('lineWidth') ?? 4;
-  data.set('strokeWidth', width);
-  const color = data.get('color');
-  data.set('stroke', color);
-
-  data.delete('startElement');
-  data.delete('endElement');
-  data.delete('controllers');
-  data.delete('lineWidth');
-  data.delete('color');
-  data.delete('xywh');
-}
-
-function updateBlockVersions(versions: YMap<number>) {
-  const frameVersion = versions.get('affine:frame');
-  if (frameVersion !== undefined) {
-    versions.set('affine:note', frameVersion);
-    versions.delete('affine:frame');
-  }
-  const embedVersion = versions.get('affine:embed');
-  if (embedVersion !== undefined) {
-    versions.set('affine:image', embedVersion);
-    versions.delete('affine:embed');
-  }
-  const databaseVersion = versions.get('affine:database');
-  if (databaseVersion !== undefined && databaseVersion < 2) {
-    versions.set('affine:database', 2);
-  }
-}
-
-function migrateMeta(oldDoc: YDoc, newDoc: YDoc) {
-  const originalMeta = oldDoc.getMap('space:meta');
-  const originalVersions = originalMeta.get('versions') as YMap<number>;
-  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
-  const meta = newDoc.getMap('meta');
-  const pages = new YArray();
-  const blockVersions = originalVersions.clone();
-
-  meta.set('workspaceVersion', 1);
-  meta.set('blockVersions', blockVersions);
-  meta.set('pages', pages);
-  meta.set('name', originalMeta.get('name') as string);
-
-  updateBlockVersions(blockVersions);
-  const mapList = originalPages.map(page => {
-    const map = new YMap();
-    Array.from(page.entries())
-      .filter(([key]) => key !== 'subpageIds')
-      .forEach(([key, value]) => {
-        map.set(key, value);
-      });
-    return map;
-  });
-  pages.push(mapList);
-}
-
-function migrateBlocks(oldDoc: YDoc, newDoc: YDoc) {
-  const spaces = newDoc.getMap('spaces');
-  const originalMeta = oldDoc.getMap('space:meta');
-  const originalVersions = originalMeta.get('versions') as YMap<number>;
-  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
-  originalPages.forEach(page => {
-    const id = page.get('id') as string;
-    const spaceId = id.startsWith('space:') ? id : `space:${id}`;
-    const originalBlocks = oldDoc.getMap(spaceId) as YMap<unknown>;
-    const subdoc = new YDoc();
-    spaces.set(spaceId, subdoc);
-    const blocks = subdoc.getMap('blocks');
-    Array.from(originalBlocks.entries()).forEach(([key, value]) => {
-      const blockData = value.clone();
-      blocks.set(key, blockData);
-      const flavour = blockData.get('sys:flavour') as string;
-      const version = originalVersions.get(flavour);
-      if (version !== undefined) {
-        runBlockMigration(flavour, blockData, version);
-      }
-    });
-  });
-}
-
-export function migrateToSubdoc(doc: YDoc): YDoc {
-  const needMigration = Array.from(doc.getMap('space:meta').keys()).length > 0;
-  if (!needMigration) {
-    return doc;
-  }
-  const output = new YDoc();
-  migrateMeta(doc, output);
-  migrateBlocks(doc, output);
-  return output;
-}
-
-export async function migrateDatabaseBlockTo3(rootDoc: YDoc, schema: Schema) {
-  const spaces = rootDoc.getMap('spaces') as YMap<any>;
-  spaces.forEach(space => {
-    schema.upgradePage(
-      {
-        'affine:note': 1,
-        'affine:bookmark': 1,
-        'affine:database': 2,
-        'affine:divider': 1,
-        'affine:image': 1,
-        'affine:list': 1,
-        'affine:code': 1,
-        'affine:page': 2,
-        'affine:paragraph': 1,
-        'affine:surface': 3,
-      },
-      space
-    );
-  });
-  const meta = rootDoc.getMap('meta') as YMap<unknown>;
-  const versions = meta.get('blockVersions') as YMap<number>;
-  versions.set('affine:database', 3);
-}

packages/env/src/workspace.ts

@@ -10,11 +10,6 @@ import type { PropsWithChildren, ReactNode } from 'react';
 
 import type { Collection } from './filter.js';
 
-export enum WorkspaceVersion {
-  SubDoc = 2,
-  DatabaseV3 = 3,
-}
-
 export enum WorkspaceSubPath {
   ALL = 'all',
   SETTING = 'setting',

packages/infra/package.json

@@ -63,7 +63,8 @@
     "electron": "link:../../apps/electron/node_modules/electron",
     "react": "^18.2.0",
     "vite": "^4.4.9",
-    "vite-plugin-dts": "3.5.2"
+    "vite-plugin-dts": "3.5.2",
+    "yjs": "^13.6.7"
   },
   "peerDependencies": {
     "@affine/templates": "*",
@@ -71,7 +72,8 @@
     "@blocksuite/lit": "*",
     "async-call-rpc": "*",
     "electron": "*",
-    "react": "*"
+    "react": "*",
+    "yjs": "^13"
   },
   "peerDependenciesMeta": {
     "@affine/templates": {
@@ -91,6 +93,9 @@
     },
     "react": {
       "optional": true
+    },
+    "yjs": {
+      "optional": true
     }
   },
   "version": "0.9.0-canary.0"

packages/infra/src/__tests__/migration.spec.ts

@@ -16,8 +16,8 @@ const doc = new Doc();
 applyUpdate(doc, new Uint8Array(yDocBuffer));
 const migratedDoc = migrateToSubdoc(doc);
 
-describe('subdoc', () => {
+describe('migration', () => {
-  test('Migration to subdoc', async () => {
+  test('migration to subdoc', async () => {
     const { default: json } = await import('@affine-test/fixtures/output.json');
     const length = Object.keys(json).length;
     const binary = new Uint8Array(length);
@@ -49,13 +49,13 @@ describe('subdoc', () => {
     });
   });
 
-  test('Test fixture should be set correctly', () => {
+  test('test fixture should be set correctly', () => {
     const meta = doc.getMap('space:meta');
     const versions = meta.get('versions') as YMap<unknown>;
     expect(versions.get('affine:code')).toBeTypeOf('number');
   });
 
-  test('Meta data should be migrated correctly', () => {
+  test('metadata should be migrated correctly', () => {
     const originalMeta = doc.getMap('space:meta');
     const originalVersions = originalMeta.get('versions') as YMap<unknown>;
 

packages/infra/src/blocksuite/index.ts

@@ -1,4 +1,5 @@
 import type { PageMeta, Workspace } from '@blocksuite/store';
+import { createIndexeddbStorage } from '@blocksuite/store';
 import type { createStore, WritableAtom } from 'jotai/vanilla';
 
 export async function buildShowcaseWorkspace(
@@ -194,3 +195,369 @@ export async function buildShowcaseWorkspace(
     })
   );
 }
+
+import { applyUpdate, encodeStateAsUpdate } from 'yjs';
+
+const migrationOrigin = 'affine-migration';
+
+import type { Schema } from '@blocksuite/store';
+import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs';
+
+type XYWH = [number, number, number, number];
+
+function deserializeXYWH(xywh: string): XYWH {
+  return JSON.parse(xywh) as XYWH;
+}
+
+function migrateDatabase(data: YMap<unknown>) {
+  data.delete('prop:mode');
+  data.set('prop:views', new YArray());
+  const columns = (data.get('prop:columns') as YArray<unknown>).toJSON() as {
+    id: string;
+    name: string;
+    hide: boolean;
+    type: string;
+    width: number;
+    selection?: unknown[];
+  }[];
+  const views = [
+    {
+      id: 'default',
+      name: 'Table',
+      columns: columns.map(col => ({
+        id: col.id,
+        width: col.width,
+        hide: col.hide,
+      })),
+      filter: { type: 'group', op: 'and', conditions: [] },
+      mode: 'table',
+    },
+  ];
+  const cells = (data.get('prop:cells') as YMap<unknown>).toJSON() as Record<
+    string,
+    Record<
+      string,
+      {
+        id: string;
+        value: unknown;
+      }
+    >
+  >;
+  const convertColumn = (
+    id: string,
+    update: (cell: { id: string; value: unknown }) => void
+  ) => {
+    Object.values(cells).forEach(row => {
+      if (row[id] != null) {
+        update(row[id]);
+      }
+    });
+  };
+  const newColumns = columns.map(v => {
+    let data: Record<string, unknown> = {};
+    if (v.type === 'select' || v.type === 'multi-select') {
+      data = { options: v.selection };
+      if (v.type === 'select') {
+        convertColumn(v.id, cell => {
+          if (Array.isArray(cell.value)) {
+            cell.value = cell.value[0]?.id;
+          }
+        });
+      } else {
+        convertColumn(v.id, cell => {
+          if (Array.isArray(cell.value)) {
+            cell.value = cell.value.map(v => v.id);
+          }
+        });
+      }
+    }
+    if (v.type === 'number') {
+      convertColumn(v.id, cell => {
+        if (typeof cell.value === 'string') {
+          cell.value = Number.parseFloat(cell.value.toString());
+        }
+      });
+    }
+    return {
+      id: v.id,
+      type: v.type,
+      name: v.name,
+      data,
+    };
+  });
+  data.set('prop:columns', newColumns);
+  data.set('prop:views', views);
+  data.set('prop:cells', cells);
+}
+
+function runBlockMigration(
+  flavour: string,
+  data: YMap<unknown>,
+  version: number
+) {
+  if (flavour === 'affine:frame') {
+    data.set('sys:flavour', 'affine:note');
+    return;
+  }
+  if (flavour === 'affine:surface' && version <= 3) {
+    if (data.has('elements')) {
+      const elements = data.get('elements') as YMap<unknown>;
+      migrateSurface(elements);
+      data.set('prop:elements', elements.clone());
+      data.delete('elements');
+    } else {
+      data.set('prop:elements', new YMap());
+    }
+  }
+  if (flavour === 'affine:embed') {
+    data.set('sys:flavour', 'affine:image');
+    data.delete('prop:type');
+  }
+  if (flavour === 'affine:database' && version < 2) {
+    migrateDatabase(data);
+  }
+}
+
+function migrateSurface(data: YMap<unknown>) {
+  for (const [, value] of <IterableIterator<[string, YMap<unknown>]>>(
+    data.entries()
+  )) {
+    if (value.get('type') === 'connector') {
+      migrateSurfaceConnector(value);
+    }
+  }
+}
+
+function migrateSurfaceConnector(data: YMap<any>) {
+  let id = data.get('startElement')?.id;
+  const controllers = data.get('controllers');
+  const length = controllers.length;
+  const xywh = deserializeXYWH(data.get('xywh'));
+  if (id) {
+    data.set('source', { id });
+  } else {
+    data.set('source', {
+      position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]],
+    });
+  }
+
+  id = data.get('endElement')?.id;
+  if (id) {
+    data.set('target', { id });
+  } else {
+    data.set('target', {
+      position: [
+        controllers[length - 1].x + xywh[0],
+        controllers[length - 1].y + xywh[1],
+      ],
+    });
+  }
+
+  const width = data.get('lineWidth') ?? 4;
+  data.set('strokeWidth', width);
+  const color = data.get('color');
+  data.set('stroke', color);
+
+  data.delete('startElement');
+  data.delete('endElement');
+  data.delete('controllers');
+  data.delete('lineWidth');
+  data.delete('color');
+  data.delete('xywh');
+}
+
+function updateBlockVersions(versions: YMap<number>) {
+  const frameVersion = versions.get('affine:frame');
+  if (frameVersion !== undefined) {
+    versions.set('affine:note', frameVersion);
+    versions.delete('affine:frame');
+  }
+  const embedVersion = versions.get('affine:embed');
+  if (embedVersion !== undefined) {
+    versions.set('affine:image', embedVersion);
+    versions.delete('affine:embed');
+  }
+  const databaseVersion = versions.get('affine:database');
+  if (databaseVersion !== undefined && databaseVersion < 2) {
+    versions.set('affine:database', 2);
+  }
+}
+
+function migrateMeta(oldDoc: YDoc, newDoc: YDoc) {
+  const originalMeta = oldDoc.getMap('space:meta');
+  const originalVersions = originalMeta.get('versions') as YMap<number>;
+  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
+  const meta = newDoc.getMap('meta');
+  const pages = new YArray();
+  const blockVersions = originalVersions.clone();
+
+  meta.set('workspaceVersion', 1);
+  meta.set('blockVersions', blockVersions);
+  meta.set('pages', pages);
+  meta.set('name', originalMeta.get('name') as string);
+
+  updateBlockVersions(blockVersions);
+  const mapList = originalPages.map(page => {
+    const map = new YMap();
+    Array.from(page.entries())
+      .filter(([key]) => key !== 'subpageIds')
+      .forEach(([key, value]) => {
+        map.set(key, value);
+      });
+    return map;
+  });
+  pages.push(mapList);
+}
+
+function migrateBlocks(oldDoc: YDoc, newDoc: YDoc) {
+  const spaces = newDoc.getMap('spaces');
+  const originalMeta = oldDoc.getMap('space:meta');
+  const originalVersions = originalMeta.get('versions') as YMap<number>;
+  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
+  originalPages.forEach(page => {
+    const id = page.get('id') as string;
+    const spaceId = id.startsWith('space:') ? id : `space:${id}`;
+    const originalBlocks = oldDoc.getMap(spaceId) as YMap<unknown>;
+    const subdoc = new YDoc();
+    spaces.set(spaceId, subdoc);
+    const blocks = subdoc.getMap('blocks');
+    Array.from(originalBlocks.entries()).forEach(([key, value]) => {
+      const blockData = value.clone();
+      blocks.set(key, blockData);
+      const flavour = blockData.get('sys:flavour') as string;
+      const version = originalVersions.get(flavour);
+      if (version !== undefined) {
+        runBlockMigration(flavour, blockData, version);
+      }
+    });
+  });
+}
+
+export function migrateToSubdoc(oldDoc: YDoc): YDoc {
+  const needMigration =
+    Array.from(oldDoc.getMap('space:meta').keys()).length > 0;
+  if (!needMigration) {
+    return oldDoc;
+  }
+  const newDoc = new YDoc();
+  migrateMeta(oldDoc, newDoc);
+  migrateBlocks(oldDoc, newDoc);
+  return newDoc;
+}
+
+export async function migrateDatabaseBlockTo3(rootDoc: YDoc, schema: Schema) {
+  const spaces = rootDoc.getMap('spaces') as YMap<any>;
+  spaces.forEach(space => {
+    schema.upgradePage(
+      {
+        'affine:note': 1,
+        'affine:bookmark': 1,
+        'affine:database': 2,
+        'affine:divider': 1,
+        'affine:image': 1,
+        'affine:list': 1,
+        'affine:code': 1,
+        'affine:page': 2,
+        'affine:paragraph': 1,
+        'affine:surface': 3,
+      },
+      space
+    );
+  });
+  const meta = rootDoc.getMap('meta') as YMap<unknown>;
+  const versions = meta.get('blockVersions') as YMap<number>;
+  versions.set('affine:database', 3);
+}
+
+export type UpgradeOptions = {
+  getCurrentRootDoc: () => Promise<YDoc>;
+  createWorkspace: () => Promise<Workspace>;
+  getSchema: () => Schema;
+};
+
+const upgradeV1ToV2 = async (options: UpgradeOptions) => {
+  const oldDoc = await options.getCurrentRootDoc();
+  const newDoc = migrateToSubdoc(oldDoc);
+  const newWorkspace = await options.createWorkspace();
+  applyUpdate(newWorkspace.doc, encodeStateAsUpdate(newDoc), migrationOrigin);
+  newDoc.getSubdocs().forEach(subdoc => {
+    newWorkspace.doc.getSubdocs().forEach(newDoc => {
+      if (subdoc.guid === newDoc.guid) {
+        applyUpdate(newDoc, encodeStateAsUpdate(subdoc), migrationOrigin);
+      }
+    });
+  });
+  return newWorkspace;
+};
+
+const upgradeV2ToV3 = async (options: UpgradeOptions): Promise<true> => {
+  const rootDoc = await options.getCurrentRootDoc();
+  const spaces = rootDoc.getMap('spaces') as YMap<any>;
+  const schema = options.getSchema();
+  spaces.forEach(space => {
+    schema.upgradePage(
+      {
+        'affine:note': 1,
+        'affine:bookmark': 1,
+        'affine:database': 2,
+        'affine:divider': 1,
+        'affine:image': 1,
+        'affine:list': 1,
+        'affine:code': 1,
+        'affine:page': 2,
+        'affine:paragraph': 1,
+        'affine:surface': 3,
+      },
+      space
+    );
+  });
+  const meta = rootDoc.getMap('meta') as YMap<unknown>;
+  const versions = meta.get('blockVersions') as YMap<number>;
+  versions.set('affine:database', 3);
+  return true;
+};
+
+export enum WorkspaceVersion {
+  // v1 is treated as undefined
+  SubDoc = 2,
+  DatabaseV3 = 3,
+}
+
+/**
+ * If returns false, it means no migration is needed.
+ * If returns true, it means migration is done.
+ * If returns Workspace, it means new workspace is created,
+ *  and the old workspace should be deleted.
+ */
+export async function migrateWorkspace(
+  currentVersion: WorkspaceVersion | undefined,
+  options: UpgradeOptions
+): Promise<Workspace | boolean> {
+  if (currentVersion === undefined) {
+    const workspace = await upgradeV1ToV2(options);
+    await upgradeV2ToV3({
+      ...options,
+      getCurrentRootDoc: () => Promise.resolve(workspace.doc),
+    });
+    return workspace;
+  }
+  if (currentVersion === WorkspaceVersion.SubDoc) {
+    return upgradeV2ToV3(options);
+  } else {
+    return false;
+  }
+}
+
+export async function migrateLocalBlobStorage(from: string, to: string) {
+  const fromStorage = createIndexeddbStorage(from);
+  const toStorage = createIndexeddbStorage(to);
+  const keys = await fromStorage.crud.list();
+  for (const key of keys) {
+    const value = await fromStorage.crud.get(key);
+    if (!value) {
+      console.warn('cannot find blob:', key);
+      continue;
+    }
+    await toStorage.crud.set(key, value);
+  }
+}

packages/infra/tsconfig.node.json

@@ -8,5 +8,10 @@
     "outDir": "lib",
     "noEmit": false
   },
-  "include": ["vite.config.ts"]
+  "include": ["vite.config.ts"],
+  "references": [
+    {
+      "path": "../../tests/fixtures"
+    }
+  ]
 }

packages/infra/vite.config.ts

@@ -34,6 +34,7 @@ export default defineConfig({
         'rxjs',
         'zod',
         'react',
+        'yjs',
         /^jotai/,
         /^@blocksuite/,
         /^@affine\/templates/,

packages/workspace/src/atom.ts

@@ -1,7 +1,8 @@
 import type { WorkspaceAdapter } from '@affine/env/workspace';
-import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import { WorkspaceFlavour } from '@affine/env/workspace';
 import type { BlockHub } from '@blocksuite/blocks';
 import { assertExists } from '@blocksuite/global/utils';
+import { WorkspaceVersion } from '@toeverything/infra/blocksuite';
 import { atom } from 'jotai';
 import { z } from 'zod';
 

packages/workspace/src/migration/index.ts

@@ -1,51 +0,0 @@
-import { migrateToSubdoc } from '@affine/env/blocksuite';
-import type { LocalWorkspace } from '@affine/env/workspace';
-import { WorkspaceFlavour } from '@affine/env/workspace';
-import { getOrCreateWorkspace } from '@affine/workspace/manager';
-import { nanoid, Workspace } from '@blocksuite/store';
-import { createIndexeddbStorage } from '@blocksuite/store';
-const Y = Workspace.Y;
-
-export function upgradeV1ToV2(oldWorkspace: LocalWorkspace): LocalWorkspace {
-  const oldDoc = oldWorkspace.blockSuiteWorkspace.doc;
-  const newDoc = migrateToSubdoc(oldDoc);
-  if (newDoc === oldDoc) {
-    console.warn('do not need update');
-    return oldWorkspace;
-  } else {
-    const id = nanoid();
-    const newBlockSuiteWorkspace = getOrCreateWorkspace(
-      id,
-      WorkspaceFlavour.LOCAL
-    );
-    Y.applyUpdate(newBlockSuiteWorkspace.doc, Y.encodeStateAsUpdate(newDoc));
-    newDoc.getSubdocs().forEach(subdoc => {
-      newBlockSuiteWorkspace.doc.getSubdocs().forEach(newDoc => {
-        if (subdoc.guid === newDoc.guid) {
-          Y.applyUpdate(newDoc, Y.encodeStateAsUpdate(subdoc));
-        }
-      });
-    });
-    console.log('migration result', newBlockSuiteWorkspace.doc.toJSON());
-
-    return {
-      blockSuiteWorkspace: newBlockSuiteWorkspace,
-      flavour: WorkspaceFlavour.LOCAL,
-      id,
-    };
-  }
-}
-
-export async function migrateLocalBlobStorage(from: string, to: string) {
-  const fromStorage = createIndexeddbStorage(from);
-  const toStorage = createIndexeddbStorage(to);
-  const keys = await fromStorage.crud.list();
-  for (const key of keys) {
-    const value = await fromStorage.crud.get(key);
-    if (!value) {
-      console.warn('cannot find blob:', key);
-      continue;
-    }
-    await toStorage.crud.set(key, value);
-  }
-}

yarn.lock

@@ -11771,6 +11771,7 @@ __metadata:
     react: ^18.2.0
     vite: ^4.4.9
     vite-plugin-dts: 3.5.2
+    yjs: ^13.6.7
     zod: ^3.22.2
   peerDependencies:
     "@affine/templates": "*"
@@ -11779,6 +11780,7 @@ __metadata:
     async-call-rpc: "*"
     electron: "*"
     react: "*"
+    yjs: ^13
   peerDependenciesMeta:
     "@affine/templates":
       optional: true
@@ -11792,6 +11794,8 @@ __metadata:
       optional: true
     react:
       optional: true
+    yjs:
+      optional: true
   languageName: unknown
   linkType: soft