From b9c4b88a6bbf9c6f8bd5ddb8175a3dbdd8cffb06 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Mon, 28 Aug 2023 00:31:56 -0500 Subject: [PATCH] refactor: migration logic (#3973) --- apps/core/src/bootstrap/setup.ts | 159 +++----- .../core/src/hooks/use-transform-workspace.ts | 2 +- apps/core/src/hooks/use-workspaces.ts | 7 +- apps/electron/src/helper/db/migration.ts | 2 +- packages/env/src/blocksuite/index.ts | 2 - .../env/src/blocksuite/subdoc-migration.ts | 267 ------------- packages/env/src/workspace.ts | 5 - packages/infra/package.json | 9 +- .../src/__tests__/migration.spec.ts} | 8 +- .../src/__tests__/workspace.ydoc | Bin packages/infra/src/blocksuite/index.ts | 367 ++++++++++++++++++ packages/infra/tsconfig.node.json | 7 +- packages/infra/vite.config.ts | 1 + packages/workspace/src/atom.ts | 3 +- packages/workspace/src/migration/index.ts | 51 --- yarn.lock | 4 + 16 files changed, 459 insertions(+), 435 deletions(-) delete mode 100644 packages/env/src/blocksuite/subdoc-migration.ts rename packages/{env/src/__tests__/subdoc-migration.spec.ts => infra/src/__tests__/migration.spec.ts} (93%) rename packages/{env => infra}/src/__tests__/workspace.ydoc (100%) delete mode 100644 packages/workspace/src/migration/index.ts diff --git a/apps/core/src/bootstrap/setup.ts b/apps/core/src/bootstrap/setup.ts index 0dd5119452..94f6f024bc 100644 --- a/apps/core/src/bootstrap/setup.ts +++ b/apps/core/src/bootstrap/setup.ts @@ -1,27 +1,27 @@ -import { - migrateDatabaseBlockTo3, - migrateToSubdoc, -} from '@affine/env/blocksuite'; import { setupGlobal } from '@affine/env/global'; -import type { - LocalIndexedDBDownloadProvider, - WorkspaceAdapter, -} from '@affine/env/workspace'; -import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace'; +import type { WorkspaceAdapter } from '@affine/env/workspace'; +import { WorkspaceFlavour } from '@affine/env/workspace'; import type { RootWorkspaceMetadata } from '@affine/workspace/atom'; import { type RootWorkspaceMetadataV2, rootWorkspacesMetadataAtom, workspaceAdaptersAtom, } from '@affine/workspace/atom'; -import { globalBlockSuiteSchema } from '@affine/workspace/manager'; +import { + getOrCreateWorkspace, + globalBlockSuiteSchema, +} from '@affine/workspace/manager'; +import { assertExists } from '@blocksuite/global/utils'; +import { nanoid } from '@blocksuite/store'; import { migrateLocalBlobStorage, - upgradeV1ToV2, -} from '@affine/workspace/migration'; -import { createIndexedDBDownloadProvider } from '@affine/workspace/providers'; -import { assertExists } from '@blocksuite/global/utils'; + migrateWorkspace, + WorkspaceVersion, +} from '@toeverything/infra/blocksuite'; +import { downloadBinary } from '@toeverything/y-indexeddb'; import type { createStore } from 'jotai/vanilla'; +import { Doc } from 'yjs'; +import { applyUpdate } from 'yjs'; import { WorkspaceAdapters } from '../adapters/workspace'; @@ -33,94 +33,57 @@ async function tryMigration() { const promises: Promise[] = []; const newMetadata = [...metadata]; metadata.forEach(oldMeta => { - if (!('version' in oldMeta)) { - const adapter = WorkspaceAdapters[oldMeta.flavour]; - assertExists(adapter); - const upgrade = async () => { - if (oldMeta.flavour !== WorkspaceFlavour.LOCAL) { - console.warn('not supported'); - return; - } - const workspace = await adapter.CRUD.get(oldMeta.id); - if (!workspace) { - console.warn('cannot find workspace', oldMeta.id); - return; - } - const doc = workspace.blockSuiteWorkspace.doc; - const provider = createIndexedDBDownloadProvider( - workspace.id, - doc, - { - awareness: - workspace.blockSuiteWorkspace.awarenessStore.awareness, - } - ) as LocalIndexedDBDownloadProvider; - provider.sync(); - await provider.whenReady; - const newDoc = migrateToSubdoc(doc); - if (doc === newDoc) { - console.log('doc not changed'); - return; - } - const newWorkspace = upgradeV1ToV2(workspace); - await migrateDatabaseBlockTo3( - newWorkspace.blockSuiteWorkspace.doc, - globalBlockSuiteSchema - ); - - const newId = await adapter.CRUD.create( - newWorkspace.blockSuiteWorkspace - ); - - await adapter.CRUD.delete(workspace as any); - console.log('migrated', oldMeta.id, newId); - const index = newMetadata.findIndex(meta => meta.id === oldMeta.id); - newMetadata[index] = { - ...oldMeta, - id: newId, - version: WorkspaceVersion.DatabaseV3, - }; - await migrateLocalBlobStorage(workspace.id, newId); - console.log('migrate to v2'); - }; - - // create a new workspace and push it to metadata - promises.push(upgrade()); - } else if (oldMeta.version < WorkspaceVersion.DatabaseV3) { - const adapter = WorkspaceAdapters[oldMeta.flavour]; - assertExists(adapter); + if (oldMeta.flavour === WorkspaceFlavour.LOCAL) { promises.push( - (async () => { - if (oldMeta.flavour !== WorkspaceFlavour.LOCAL) { - console.warn('not supported'); - return; + migrateWorkspace( + 'version' in oldMeta ? oldMeta.version : undefined, + { + getCurrentRootDoc: async () => { + const doc = new Doc({ + guid: oldMeta.id, + }); + const downloadWorkspace = async (doc: Doc): Promise => { + const binary = await downloadBinary(doc.guid); + if (binary) { + applyUpdate(doc, binary); + } + return Promise.all( + [...doc.subdocs.values()].map(subdoc => + downloadWorkspace(subdoc) + ) + ).then(); + }; + await downloadWorkspace(doc); + return doc; + }, + createWorkspace: async () => + getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL), + getSchema: () => globalBlockSuiteSchema, } - const workspace = await adapter.CRUD.get(oldMeta.id); - if (workspace) { - const provider = createIndexedDBDownloadProvider( - workspace.id, - workspace.blockSuiteWorkspace.doc, - { - awareness: - workspace.blockSuiteWorkspace.awarenessStore.awareness, - } - ) as LocalIndexedDBDownloadProvider; - provider.sync(); - await provider.whenReady; - await migrateDatabaseBlockTo3( - workspace.blockSuiteWorkspace.doc, - globalBlockSuiteSchema + ).then(async workspace => { + if (typeof workspace !== 'boolean') { + const adapter = WorkspaceAdapters[oldMeta.flavour]; + const oldWorkspace = await adapter.CRUD.get(oldMeta.id); + const newId = await adapter.CRUD.create(workspace); + assertExists( + oldWorkspace, + 'workspace should exist after migrate' ); + await adapter.CRUD.delete(oldWorkspace.blockSuiteWorkspace); + const index = newMetadata.findIndex( + meta => meta.id === oldMeta.id + ); + newMetadata[index] = { + ...oldMeta, + id: newId, + version: WorkspaceVersion.DatabaseV3, + }; + await migrateLocalBlobStorage(workspace.id, newId); + console.log('workspace migrated', oldMeta.id, newId); + } else if (workspace) { + console.log('workspace migrated', oldMeta.id); } - const index = newMetadata.findIndex( - meta => meta.id === oldMeta.id - ); - newMetadata[index] = { - ...oldMeta, - version: WorkspaceVersion.DatabaseV3, - }; - console.log('migrate to v3'); - })() + }) ); } }); diff --git a/apps/core/src/hooks/use-transform-workspace.ts b/apps/core/src/hooks/use-transform-workspace.ts index 81ae8dcc77..dcb996ddb4 100644 --- a/apps/core/src/hooks/use-transform-workspace.ts +++ b/apps/core/src/hooks/use-transform-workspace.ts @@ -1,7 +1,7 @@ import type { WorkspaceFlavour } from '@affine/env/workspace'; import type { WorkspaceRegistry } from '@affine/env/workspace'; -import { WorkspaceVersion } from '@affine/env/workspace'; import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom'; +import { WorkspaceVersion } from '@toeverything/infra/blocksuite'; import { useSetAtom } from 'jotai'; import { useCallback } from 'react'; diff --git a/apps/core/src/hooks/use-workspaces.ts b/apps/core/src/hooks/use-workspaces.ts index 8b3a78af28..bb122720d1 100644 --- a/apps/core/src/hooks/use-workspaces.ts +++ b/apps/core/src/hooks/use-workspaces.ts @@ -1,12 +1,15 @@ import { DebugLogger } from '@affine/debug'; -import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace'; +import { WorkspaceFlavour } from '@affine/env/workspace'; import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom'; import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud'; import { getOrCreateWorkspace } from '@affine/workspace/manager'; import { nanoid } from '@blocksuite/store'; import { getWorkspace } from '@toeverything/infra/__internal__/workspace'; import { getCurrentStore } from '@toeverything/infra/atom'; -import { buildShowcaseWorkspace } from '@toeverything/infra/blocksuite'; +import { + buildShowcaseWorkspace, + WorkspaceVersion, +} from '@toeverything/infra/blocksuite'; import { useAtomValue, useSetAtom } from 'jotai'; import { useCallback } from 'react'; diff --git a/apps/electron/src/helper/db/migration.ts b/apps/electron/src/helper/db/migration.ts index fe07c524c2..2323130d53 100644 --- a/apps/electron/src/helper/db/migration.ts +++ b/apps/electron/src/helper/db/migration.ts @@ -1,7 +1,7 @@ import { resolve } from 'node:path'; -import { migrateToSubdoc } from '@affine/env/blocksuite'; import { SqliteConnection } from '@affine/native'; +import { migrateToSubdoc } from '@toeverything/infra/blocksuite'; import fs from 'fs-extra'; import { nanoid } from 'nanoid'; import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs'; diff --git a/packages/env/src/blocksuite/index.ts b/packages/env/src/blocksuite/index.ts index bc04bef52b..e95e658291 100644 --- a/packages/env/src/blocksuite/index.ts +++ b/packages/env/src/blocksuite/index.ts @@ -12,5 +12,3 @@ export async function initEmptyPage(page: Page, title?: string) { const noteBlockId = page.addBlock('affine:note', {}, pageBlockId); page.addBlock('affine:paragraph', {}, noteBlockId); } - -export * from './subdoc-migration.js'; diff --git a/packages/env/src/blocksuite/subdoc-migration.ts b/packages/env/src/blocksuite/subdoc-migration.ts deleted file mode 100644 index f3feb871d7..0000000000 --- a/packages/env/src/blocksuite/subdoc-migration.ts +++ /dev/null @@ -1,267 +0,0 @@ -import type { Schema } from '@blocksuite/store'; -import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs'; - -type XYWH = [number, number, number, number]; - -function deserializeXYWH(xywh: string): XYWH { - return JSON.parse(xywh) as XYWH; -} - -function migrateDatabase(data: YMap) { - data.delete('prop:mode'); - data.set('prop:views', new YArray()); - const columns = (data.get('prop:columns') as YArray).toJSON() as { - id: string; - name: string; - hide: boolean; - type: string; - width: number; - selection?: unknown[]; - }[]; - const views = [ - { - id: 'default', - name: 'Table', - columns: columns.map(col => ({ - id: col.id, - width: col.width, - hide: col.hide, - })), - filter: { type: 'group', op: 'and', conditions: [] }, - mode: 'table', - }, - ]; - const cells = (data.get('prop:cells') as YMap).toJSON() as Record< - string, - Record< - string, - { - id: string; - value: unknown; - } - > - >; - const convertColumn = ( - id: string, - update: (cell: { id: string; value: unknown }) => void - ) => { - Object.values(cells).forEach(row => { - if (row[id] != null) { - update(row[id]); - } - }); - }; - const newColumns = columns.map(v => { - let data: Record = {}; - if (v.type === 'select' || v.type === 'multi-select') { - data = { options: v.selection }; - if (v.type === 'select') { - convertColumn(v.id, cell => { - if (Array.isArray(cell.value)) { - cell.value = cell.value[0]?.id; - } - }); - } else { - convertColumn(v.id, cell => { - if (Array.isArray(cell.value)) { - cell.value = cell.value.map(v => v.id); - } - }); - } - } - if (v.type === 'number') { - convertColumn(v.id, cell => { - if (typeof cell.value === 'string') { - cell.value = Number.parseFloat(cell.value.toString()); - } - }); - } - return { - id: v.id, - type: v.type, - name: v.name, - data, - }; - }); - data.set('prop:columns', newColumns); - data.set('prop:views', views); - data.set('prop:cells', cells); -} - -function runBlockMigration( - flavour: string, - data: YMap, - version: number -) { - if (flavour === 'affine:frame') { - data.set('sys:flavour', 'affine:note'); - return; - } - if (flavour === 'affine:surface' && version <= 3) { - if (data.has('elements')) { - const elements = data.get('elements') as YMap; - migrateSurface(elements); - data.set('prop:elements', elements.clone()); - data.delete('elements'); - } else { - data.set('prop:elements', new YMap()); - } - } - if (flavour === 'affine:embed') { - data.set('sys:flavour', 'affine:image'); - data.delete('prop:type'); - } - if (flavour === 'affine:database' && version < 2) { - migrateDatabase(data); - } -} - -function migrateSurface(data: YMap) { - for (const [, value] of ]>>( - data.entries() - )) { - if (value.get('type') === 'connector') { - migrateSurfaceConnector(value); - } - } -} - -function migrateSurfaceConnector(data: YMap) { - let id = data.get('startElement')?.id; - const controllers = data.get('controllers'); - const length = controllers.length; - const xywh = deserializeXYWH(data.get('xywh')); - if (id) { - data.set('source', { id }); - } else { - data.set('source', { - position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]], - }); - } - - id = data.get('endElement')?.id; - if (id) { - data.set('target', { id }); - } else { - data.set('target', { - position: [ - controllers[length - 1].x + xywh[0], - controllers[length - 1].y + xywh[1], - ], - }); - } - - const width = data.get('lineWidth') ?? 4; - data.set('strokeWidth', width); - const color = data.get('color'); - data.set('stroke', color); - - data.delete('startElement'); - data.delete('endElement'); - data.delete('controllers'); - data.delete('lineWidth'); - data.delete('color'); - data.delete('xywh'); -} - -function updateBlockVersions(versions: YMap) { - const frameVersion = versions.get('affine:frame'); - if (frameVersion !== undefined) { - versions.set('affine:note', frameVersion); - versions.delete('affine:frame'); - } - const embedVersion = versions.get('affine:embed'); - if (embedVersion !== undefined) { - versions.set('affine:image', embedVersion); - versions.delete('affine:embed'); - } - const databaseVersion = versions.get('affine:database'); - if (databaseVersion !== undefined && databaseVersion < 2) { - versions.set('affine:database', 2); - } -} - -function migrateMeta(oldDoc: YDoc, newDoc: YDoc) { - const originalMeta = oldDoc.getMap('space:meta'); - const originalVersions = originalMeta.get('versions') as YMap; - const originalPages = originalMeta.get('pages') as YArray>; - const meta = newDoc.getMap('meta'); - const pages = new YArray(); - const blockVersions = originalVersions.clone(); - - meta.set('workspaceVersion', 1); - meta.set('blockVersions', blockVersions); - meta.set('pages', pages); - meta.set('name', originalMeta.get('name') as string); - - updateBlockVersions(blockVersions); - const mapList = originalPages.map(page => { - const map = new YMap(); - Array.from(page.entries()) - .filter(([key]) => key !== 'subpageIds') - .forEach(([key, value]) => { - map.set(key, value); - }); - return map; - }); - pages.push(mapList); -} - -function migrateBlocks(oldDoc: YDoc, newDoc: YDoc) { - const spaces = newDoc.getMap('spaces'); - const originalMeta = oldDoc.getMap('space:meta'); - const originalVersions = originalMeta.get('versions') as YMap; - const originalPages = originalMeta.get('pages') as YArray>; - originalPages.forEach(page => { - const id = page.get('id') as string; - const spaceId = id.startsWith('space:') ? id : `space:${id}`; - const originalBlocks = oldDoc.getMap(spaceId) as YMap; - const subdoc = new YDoc(); - spaces.set(spaceId, subdoc); - const blocks = subdoc.getMap('blocks'); - Array.from(originalBlocks.entries()).forEach(([key, value]) => { - const blockData = value.clone(); - blocks.set(key, blockData); - const flavour = blockData.get('sys:flavour') as string; - const version = originalVersions.get(flavour); - if (version !== undefined) { - runBlockMigration(flavour, blockData, version); - } - }); - }); -} - -export function migrateToSubdoc(doc: YDoc): YDoc { - const needMigration = Array.from(doc.getMap('space:meta').keys()).length > 0; - if (!needMigration) { - return doc; - } - const output = new YDoc(); - migrateMeta(doc, output); - migrateBlocks(doc, output); - return output; -} - -export async function migrateDatabaseBlockTo3(rootDoc: YDoc, schema: Schema) { - const spaces = rootDoc.getMap('spaces') as YMap; - spaces.forEach(space => { - schema.upgradePage( - { - 'affine:note': 1, - 'affine:bookmark': 1, - 'affine:database': 2, - 'affine:divider': 1, - 'affine:image': 1, - 'affine:list': 1, - 'affine:code': 1, - 'affine:page': 2, - 'affine:paragraph': 1, - 'affine:surface': 3, - }, - space - ); - }); - const meta = rootDoc.getMap('meta') as YMap; - const versions = meta.get('blockVersions') as YMap; - versions.set('affine:database', 3); -} diff --git a/packages/env/src/workspace.ts b/packages/env/src/workspace.ts index 90766ced9e..395a3d6311 100644 --- a/packages/env/src/workspace.ts +++ b/packages/env/src/workspace.ts @@ -10,11 +10,6 @@ import type { PropsWithChildren, ReactNode } from 'react'; import type { Collection } from './filter.js'; -export enum WorkspaceVersion { - SubDoc = 2, - DatabaseV3 = 3, -} - export enum WorkspaceSubPath { ALL = 'all', SETTING = 'setting', diff --git a/packages/infra/package.json b/packages/infra/package.json index b528ec6717..316b07937f 100644 --- a/packages/infra/package.json +++ b/packages/infra/package.json @@ -63,7 +63,8 @@ "electron": "link:../../apps/electron/node_modules/electron", "react": "^18.2.0", "vite": "^4.4.9", - "vite-plugin-dts": "3.5.2" + "vite-plugin-dts": "3.5.2", + "yjs": "^13.6.7" }, "peerDependencies": { "@affine/templates": "*", @@ -71,7 +72,8 @@ "@blocksuite/lit": "*", "async-call-rpc": "*", "electron": "*", - "react": "*" + "react": "*", + "yjs": "^13" }, "peerDependenciesMeta": { "@affine/templates": { @@ -91,6 +93,9 @@ }, "react": { "optional": true + }, + "yjs": { + "optional": true } }, "version": "0.9.0-canary.0" diff --git a/packages/env/src/__tests__/subdoc-migration.spec.ts b/packages/infra/src/__tests__/migration.spec.ts similarity index 93% rename from packages/env/src/__tests__/subdoc-migration.spec.ts rename to packages/infra/src/__tests__/migration.spec.ts index 882ee57bb5..12cfa238fe 100644 --- a/packages/env/src/__tests__/subdoc-migration.spec.ts +++ b/packages/infra/src/__tests__/migration.spec.ts @@ -16,8 +16,8 @@ const doc = new Doc(); applyUpdate(doc, new Uint8Array(yDocBuffer)); const migratedDoc = migrateToSubdoc(doc); -describe('subdoc', () => { - test('Migration to subdoc', async () => { +describe('migration', () => { + test('migration to subdoc', async () => { const { default: json } = await import('@affine-test/fixtures/output.json'); const length = Object.keys(json).length; const binary = new Uint8Array(length); @@ -49,13 +49,13 @@ describe('subdoc', () => { }); }); - test('Test fixture should be set correctly', () => { + test('test fixture should be set correctly', () => { const meta = doc.getMap('space:meta'); const versions = meta.get('versions') as YMap; expect(versions.get('affine:code')).toBeTypeOf('number'); }); - test('Meta data should be migrated correctly', () => { + test('metadata should be migrated correctly', () => { const originalMeta = doc.getMap('space:meta'); const originalVersions = originalMeta.get('versions') as YMap; diff --git a/packages/env/src/__tests__/workspace.ydoc b/packages/infra/src/__tests__/workspace.ydoc similarity index 100% rename from packages/env/src/__tests__/workspace.ydoc rename to packages/infra/src/__tests__/workspace.ydoc diff --git a/packages/infra/src/blocksuite/index.ts b/packages/infra/src/blocksuite/index.ts index 22e071371c..0eedc14e67 100644 --- a/packages/infra/src/blocksuite/index.ts +++ b/packages/infra/src/blocksuite/index.ts @@ -1,4 +1,5 @@ import type { PageMeta, Workspace } from '@blocksuite/store'; +import { createIndexeddbStorage } from '@blocksuite/store'; import type { createStore, WritableAtom } from 'jotai/vanilla'; export async function buildShowcaseWorkspace( @@ -194,3 +195,369 @@ export async function buildShowcaseWorkspace( }) ); } + +import { applyUpdate, encodeStateAsUpdate } from 'yjs'; + +const migrationOrigin = 'affine-migration'; + +import type { Schema } from '@blocksuite/store'; +import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs'; + +type XYWH = [number, number, number, number]; + +function deserializeXYWH(xywh: string): XYWH { + return JSON.parse(xywh) as XYWH; +} + +function migrateDatabase(data: YMap) { + data.delete('prop:mode'); + data.set('prop:views', new YArray()); + const columns = (data.get('prop:columns') as YArray).toJSON() as { + id: string; + name: string; + hide: boolean; + type: string; + width: number; + selection?: unknown[]; + }[]; + const views = [ + { + id: 'default', + name: 'Table', + columns: columns.map(col => ({ + id: col.id, + width: col.width, + hide: col.hide, + })), + filter: { type: 'group', op: 'and', conditions: [] }, + mode: 'table', + }, + ]; + const cells = (data.get('prop:cells') as YMap).toJSON() as Record< + string, + Record< + string, + { + id: string; + value: unknown; + } + > + >; + const convertColumn = ( + id: string, + update: (cell: { id: string; value: unknown }) => void + ) => { + Object.values(cells).forEach(row => { + if (row[id] != null) { + update(row[id]); + } + }); + }; + const newColumns = columns.map(v => { + let data: Record = {}; + if (v.type === 'select' || v.type === 'multi-select') { + data = { options: v.selection }; + if (v.type === 'select') { + convertColumn(v.id, cell => { + if (Array.isArray(cell.value)) { + cell.value = cell.value[0]?.id; + } + }); + } else { + convertColumn(v.id, cell => { + if (Array.isArray(cell.value)) { + cell.value = cell.value.map(v => v.id); + } + }); + } + } + if (v.type === 'number') { + convertColumn(v.id, cell => { + if (typeof cell.value === 'string') { + cell.value = Number.parseFloat(cell.value.toString()); + } + }); + } + return { + id: v.id, + type: v.type, + name: v.name, + data, + }; + }); + data.set('prop:columns', newColumns); + data.set('prop:views', views); + data.set('prop:cells', cells); +} + +function runBlockMigration( + flavour: string, + data: YMap, + version: number +) { + if (flavour === 'affine:frame') { + data.set('sys:flavour', 'affine:note'); + return; + } + if (flavour === 'affine:surface' && version <= 3) { + if (data.has('elements')) { + const elements = data.get('elements') as YMap; + migrateSurface(elements); + data.set('prop:elements', elements.clone()); + data.delete('elements'); + } else { + data.set('prop:elements', new YMap()); + } + } + if (flavour === 'affine:embed') { + data.set('sys:flavour', 'affine:image'); + data.delete('prop:type'); + } + if (flavour === 'affine:database' && version < 2) { + migrateDatabase(data); + } +} + +function migrateSurface(data: YMap) { + for (const [, value] of ]>>( + data.entries() + )) { + if (value.get('type') === 'connector') { + migrateSurfaceConnector(value); + } + } +} + +function migrateSurfaceConnector(data: YMap) { + let id = data.get('startElement')?.id; + const controllers = data.get('controllers'); + const length = controllers.length; + const xywh = deserializeXYWH(data.get('xywh')); + if (id) { + data.set('source', { id }); + } else { + data.set('source', { + position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]], + }); + } + + id = data.get('endElement')?.id; + if (id) { + data.set('target', { id }); + } else { + data.set('target', { + position: [ + controllers[length - 1].x + xywh[0], + controllers[length - 1].y + xywh[1], + ], + }); + } + + const width = data.get('lineWidth') ?? 4; + data.set('strokeWidth', width); + const color = data.get('color'); + data.set('stroke', color); + + data.delete('startElement'); + data.delete('endElement'); + data.delete('controllers'); + data.delete('lineWidth'); + data.delete('color'); + data.delete('xywh'); +} + +function updateBlockVersions(versions: YMap) { + const frameVersion = versions.get('affine:frame'); + if (frameVersion !== undefined) { + versions.set('affine:note', frameVersion); + versions.delete('affine:frame'); + } + const embedVersion = versions.get('affine:embed'); + if (embedVersion !== undefined) { + versions.set('affine:image', embedVersion); + versions.delete('affine:embed'); + } + const databaseVersion = versions.get('affine:database'); + if (databaseVersion !== undefined && databaseVersion < 2) { + versions.set('affine:database', 2); + } +} + +function migrateMeta(oldDoc: YDoc, newDoc: YDoc) { + const originalMeta = oldDoc.getMap('space:meta'); + const originalVersions = originalMeta.get('versions') as YMap; + const originalPages = originalMeta.get('pages') as YArray>; + const meta = newDoc.getMap('meta'); + const pages = new YArray(); + const blockVersions = originalVersions.clone(); + + meta.set('workspaceVersion', 1); + meta.set('blockVersions', blockVersions); + meta.set('pages', pages); + meta.set('name', originalMeta.get('name') as string); + + updateBlockVersions(blockVersions); + const mapList = originalPages.map(page => { + const map = new YMap(); + Array.from(page.entries()) + .filter(([key]) => key !== 'subpageIds') + .forEach(([key, value]) => { + map.set(key, value); + }); + return map; + }); + pages.push(mapList); +} + +function migrateBlocks(oldDoc: YDoc, newDoc: YDoc) { + const spaces = newDoc.getMap('spaces'); + const originalMeta = oldDoc.getMap('space:meta'); + const originalVersions = originalMeta.get('versions') as YMap; + const originalPages = originalMeta.get('pages') as YArray>; + originalPages.forEach(page => { + const id = page.get('id') as string; + const spaceId = id.startsWith('space:') ? id : `space:${id}`; + const originalBlocks = oldDoc.getMap(spaceId) as YMap; + const subdoc = new YDoc(); + spaces.set(spaceId, subdoc); + const blocks = subdoc.getMap('blocks'); + Array.from(originalBlocks.entries()).forEach(([key, value]) => { + const blockData = value.clone(); + blocks.set(key, blockData); + const flavour = blockData.get('sys:flavour') as string; + const version = originalVersions.get(flavour); + if (version !== undefined) { + runBlockMigration(flavour, blockData, version); + } + }); + }); +} + +export function migrateToSubdoc(oldDoc: YDoc): YDoc { + const needMigration = + Array.from(oldDoc.getMap('space:meta').keys()).length > 0; + if (!needMigration) { + return oldDoc; + } + const newDoc = new YDoc(); + migrateMeta(oldDoc, newDoc); + migrateBlocks(oldDoc, newDoc); + return newDoc; +} + +export async function migrateDatabaseBlockTo3(rootDoc: YDoc, schema: Schema) { + const spaces = rootDoc.getMap('spaces') as YMap; + spaces.forEach(space => { + schema.upgradePage( + { + 'affine:note': 1, + 'affine:bookmark': 1, + 'affine:database': 2, + 'affine:divider': 1, + 'affine:image': 1, + 'affine:list': 1, + 'affine:code': 1, + 'affine:page': 2, + 'affine:paragraph': 1, + 'affine:surface': 3, + }, + space + ); + }); + const meta = rootDoc.getMap('meta') as YMap; + const versions = meta.get('blockVersions') as YMap; + versions.set('affine:database', 3); +} + +export type UpgradeOptions = { + getCurrentRootDoc: () => Promise; + createWorkspace: () => Promise; + getSchema: () => Schema; +}; + +const upgradeV1ToV2 = async (options: UpgradeOptions) => { + const oldDoc = await options.getCurrentRootDoc(); + const newDoc = migrateToSubdoc(oldDoc); + const newWorkspace = await options.createWorkspace(); + applyUpdate(newWorkspace.doc, encodeStateAsUpdate(newDoc), migrationOrigin); + newDoc.getSubdocs().forEach(subdoc => { + newWorkspace.doc.getSubdocs().forEach(newDoc => { + if (subdoc.guid === newDoc.guid) { + applyUpdate(newDoc, encodeStateAsUpdate(subdoc), migrationOrigin); + } + }); + }); + return newWorkspace; +}; + +const upgradeV2ToV3 = async (options: UpgradeOptions): Promise => { + const rootDoc = await options.getCurrentRootDoc(); + const spaces = rootDoc.getMap('spaces') as YMap; + const schema = options.getSchema(); + spaces.forEach(space => { + schema.upgradePage( + { + 'affine:note': 1, + 'affine:bookmark': 1, + 'affine:database': 2, + 'affine:divider': 1, + 'affine:image': 1, + 'affine:list': 1, + 'affine:code': 1, + 'affine:page': 2, + 'affine:paragraph': 1, + 'affine:surface': 3, + }, + space + ); + }); + const meta = rootDoc.getMap('meta') as YMap; + const versions = meta.get('blockVersions') as YMap; + versions.set('affine:database', 3); + return true; +}; + +export enum WorkspaceVersion { + // v1 is treated as undefined + SubDoc = 2, + DatabaseV3 = 3, +} + +/** + * If returns false, it means no migration is needed. + * If returns true, it means migration is done. + * If returns Workspace, it means new workspace is created, + * and the old workspace should be deleted. + */ +export async function migrateWorkspace( + currentVersion: WorkspaceVersion | undefined, + options: UpgradeOptions +): Promise { + if (currentVersion === undefined) { + const workspace = await upgradeV1ToV2(options); + await upgradeV2ToV3({ + ...options, + getCurrentRootDoc: () => Promise.resolve(workspace.doc), + }); + return workspace; + } + if (currentVersion === WorkspaceVersion.SubDoc) { + return upgradeV2ToV3(options); + } else { + return false; + } +} + +export async function migrateLocalBlobStorage(from: string, to: string) { + const fromStorage = createIndexeddbStorage(from); + const toStorage = createIndexeddbStorage(to); + const keys = await fromStorage.crud.list(); + for (const key of keys) { + const value = await fromStorage.crud.get(key); + if (!value) { + console.warn('cannot find blob:', key); + continue; + } + await toStorage.crud.set(key, value); + } +} diff --git a/packages/infra/tsconfig.node.json b/packages/infra/tsconfig.node.json index ff5308300e..5cdd09a00b 100644 --- a/packages/infra/tsconfig.node.json +++ b/packages/infra/tsconfig.node.json @@ -8,5 +8,10 @@ "outDir": "lib", "noEmit": false }, - "include": ["vite.config.ts"] + "include": ["vite.config.ts"], + "references": [ + { + "path": "../../tests/fixtures" + } + ] } diff --git a/packages/infra/vite.config.ts b/packages/infra/vite.config.ts index f912e25e9d..f1e6c63050 100644 --- a/packages/infra/vite.config.ts +++ b/packages/infra/vite.config.ts @@ -34,6 +34,7 @@ export default defineConfig({ 'rxjs', 'zod', 'react', + 'yjs', /^jotai/, /^@blocksuite/, /^@affine\/templates/, diff --git a/packages/workspace/src/atom.ts b/packages/workspace/src/atom.ts index f8b785222e..76320ba498 100644 --- a/packages/workspace/src/atom.ts +++ b/packages/workspace/src/atom.ts @@ -1,7 +1,8 @@ import type { WorkspaceAdapter } from '@affine/env/workspace'; -import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace'; +import { WorkspaceFlavour } from '@affine/env/workspace'; import type { BlockHub } from '@blocksuite/blocks'; import { assertExists } from '@blocksuite/global/utils'; +import { WorkspaceVersion } from '@toeverything/infra/blocksuite'; import { atom } from 'jotai'; import { z } from 'zod'; diff --git a/packages/workspace/src/migration/index.ts b/packages/workspace/src/migration/index.ts deleted file mode 100644 index 5022ad71fa..0000000000 --- a/packages/workspace/src/migration/index.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { migrateToSubdoc } from '@affine/env/blocksuite'; -import type { LocalWorkspace } from '@affine/env/workspace'; -import { WorkspaceFlavour } from '@affine/env/workspace'; -import { getOrCreateWorkspace } from '@affine/workspace/manager'; -import { nanoid, Workspace } from '@blocksuite/store'; -import { createIndexeddbStorage } from '@blocksuite/store'; -const Y = Workspace.Y; - -export function upgradeV1ToV2(oldWorkspace: LocalWorkspace): LocalWorkspace { - const oldDoc = oldWorkspace.blockSuiteWorkspace.doc; - const newDoc = migrateToSubdoc(oldDoc); - if (newDoc === oldDoc) { - console.warn('do not need update'); - return oldWorkspace; - } else { - const id = nanoid(); - const newBlockSuiteWorkspace = getOrCreateWorkspace( - id, - WorkspaceFlavour.LOCAL - ); - Y.applyUpdate(newBlockSuiteWorkspace.doc, Y.encodeStateAsUpdate(newDoc)); - newDoc.getSubdocs().forEach(subdoc => { - newBlockSuiteWorkspace.doc.getSubdocs().forEach(newDoc => { - if (subdoc.guid === newDoc.guid) { - Y.applyUpdate(newDoc, Y.encodeStateAsUpdate(subdoc)); - } - }); - }); - console.log('migration result', newBlockSuiteWorkspace.doc.toJSON()); - - return { - blockSuiteWorkspace: newBlockSuiteWorkspace, - flavour: WorkspaceFlavour.LOCAL, - id, - }; - } -} - -export async function migrateLocalBlobStorage(from: string, to: string) { - const fromStorage = createIndexeddbStorage(from); - const toStorage = createIndexeddbStorage(to); - const keys = await fromStorage.crud.list(); - for (const key of keys) { - const value = await fromStorage.crud.get(key); - if (!value) { - console.warn('cannot find blob:', key); - continue; - } - await toStorage.crud.set(key, value); - } -} diff --git a/yarn.lock b/yarn.lock index 61e0f8ba50..04ab54761c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11771,6 +11771,7 @@ __metadata: react: ^18.2.0 vite: ^4.4.9 vite-plugin-dts: 3.5.2 + yjs: ^13.6.7 zod: ^3.22.2 peerDependencies: "@affine/templates": "*" @@ -11779,6 +11780,7 @@ __metadata: async-call-rpc: "*" electron: "*" react: "*" + yjs: ^13 peerDependenciesMeta: "@affine/templates": optional: true @@ -11792,6 +11794,8 @@ __metadata: optional: true react: optional: true + yjs: + optional: true languageName: unknown linkType: soft