From bd9f66fbc7b1cbaa0cf228383b8b87bc91454ea5 Mon Sep 17 00:00:00 2001 From: Peng Xiao Date: Mon, 13 Nov 2023 17:57:56 +0800 Subject: [PATCH] fix(infra): compatibility fix for space prefix (#4912) It seems there are some cases that [this upstream PR](https://github.com/toeverything/blocksuite/pull/4747) will cause data loss. Because of some historical reasons, the page id could be different with its doc id. It might be caused by subdoc migration in the following (not 100% sure if all white screen issue is caused by it) https://github.com/toeverything/AFFiNE/blob/0714c12703ff8840acc180633e5489c755ddfba3/packages/common/infra/src/blocksuite/index.ts#L538-L540 In version 0.10, page id in spaces no longer has prefix "space:" The data flow for fetching a doc's updates is: - page id in `meta.pages` -> find `${page-id}` in `doc.spaces` -> `doc` -> `doc.guid` if `doc` is not found in `doc.spaces`, a new doc will be created and its `doc.guid` is the same with its pageId - because of guid logic change, the doc that previously prefixed with `space:` will not be found in `doc.spaces` - when fetching the rows of this doc using the doc id === page id, it will return EMPTY since there is no updates associated with the page id The provided fix in the PR will patch the `spaces` field of the root doc so that after 0.10 the page doc can still be found in the `spaces` map. It shall apply to both of the idb & sqlite datasources. Special thanks to @lawvs 's db file for investigation! --- packages/common/infra/src/blocksuite/index.ts | 38 +++++++++++++- .../common/y-provider/src/lazy-provider.ts | 31 +++--------- packages/frontend/electron/package.json | 2 +- packages/frontend/electron/scripts/dev.ts | 1 - .../electron/src/helper/db/base-db-adapter.ts | 3 +- .../electron/src/helper/db/migration.ts | 19 +++++++ packages/frontend/electron/src/main/logger.ts | 5 ++ .../src/main/updater/electron-updater.ts | 9 ++-- .../hooks/src/__tests__/index.spec.ts | 17 ++++++- .../hooks/src/use-block-suite-page-preview.ts | 49 +++++++++++++++---- .../src/use-block-suite-workspace-page.ts | 2 +- yarn.lock | 10 ++-- 12 files changed, 136 insertions(+), 50 deletions(-) diff --git a/packages/common/infra/src/blocksuite/index.ts b/packages/common/infra/src/blocksuite/index.ts index 5e3abe6a9d..335d137331 100644 --- a/packages/common/infra/src/blocksuite/index.ts +++ b/packages/common/infra/src/blocksuite/index.ts @@ -2,7 +2,7 @@ import type { Page, PageMeta, Workspace } from '@blocksuite/store'; import { createIndexeddbStorage } from '@blocksuite/store'; import type { createStore, WritableAtom } from 'jotai/vanilla'; import type { Doc } from 'yjs'; -import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs'; +import { Array as YArray, Doc as YDoc, Map as YMap, transact } from 'yjs'; export async function initEmptyPage(page: Page, title?: string) { await page.waitForLoaded(); @@ -537,6 +537,7 @@ function migrateBlocks( const originalBlocks = oldDoc.getMap(spaceId) as YMap; const subdoc = new YDoc(); spaces.set(newId, subdoc); + subdoc.guid = id; const blocks = subdoc.getMap('blocks'); Array.from(originalBlocks.entries()).forEach(([key, value]) => { const blockData = value.clone(); @@ -602,6 +603,7 @@ export async function forceUpgradePages( const versions = meta.get('blockVersions') as YMap; const schema = options.getSchema(); const oldVersions = versions.toJSON(); + guidCompatibilityFix(rootDoc); spaces.forEach((space: Doc) => { try { schema.upgradePage(0, oldVersions, space); @@ -623,6 +625,7 @@ async function upgradeV2ToV3(options: UpgradeOptions): Promise { const meta = rootDoc.getMap('meta') as YMap; const versions = meta.get('blockVersions') as YMap; const schema = options.getSchema(); + guidCompatibilityFix(rootDoc); spaces.forEach((space: Doc) => { schema.upgradePage( 0, @@ -654,6 +657,39 @@ async function upgradeV2ToV3(options: UpgradeOptions): Promise { return true; } +// patch root doc's space guid compatibility issue +// +// in version 0.10, page id in spaces no longer has prefix "space:" +// The data flow for fetching a doc's updates is: +// - page id in `meta.pages` -> find `${page-id}` in `doc.spaces` -> `doc` -> `doc.guid` +// if `doc` is not found in `doc.spaces`, a new doc will be created and its `doc.guid` is the same with its pageId +// - because of guid logic change, the doc that previously prefixed with "space:" will not be found in `doc.spaces` +// - when fetching the rows of this doc using the doc id === page id, +// it will return empty since there is no updates associated with the page id +export function guidCompatibilityFix(rootDoc: YDoc) { + let changed = false; + transact(rootDoc, () => { + const spaces = rootDoc.getMap('spaces') as YMap; + spaces.forEach((doc: YDoc, pageId: string) => { + if (pageId.includes(':')) { + const newPageId = pageId.split(':').at(-1) ?? pageId; + const newDoc = new YDoc(); + // clone the original doc. yjs is not happy to use the same doc instance + applyUpdate(newDoc, encodeStateAsUpdate(doc)); + newDoc.guid = doc.guid; + spaces.set(newPageId, newDoc); + // should remove the old doc, otherwise we will do it again in the next run + spaces.delete(pageId); + changed = true; + console.debug( + `fixed space id ${pageId} -> ${newPageId}, doc id: ${doc.guid}` + ); + } + }); + }); + return changed; +} + export enum WorkspaceVersion { // v1 is treated as undefined SubDoc = 2, diff --git a/packages/common/y-provider/src/lazy-provider.ts b/packages/common/y-provider/src/lazy-provider.ts index ee59b4c01e..f1c1a2ec2c 100644 --- a/packages/common/y-provider/src/lazy-provider.ts +++ b/packages/common/y-provider/src/lazy-provider.ts @@ -98,38 +98,21 @@ export const createLazyProvider = ( async function syncDoc(doc: Doc) { const guid = doc.guid; { - // backport from `@blocksuite/store` - const prefixId = guid.startsWith('space:') ? guid.slice(6) : guid; - const possible1 = `${rootDoc.guid}:space:${prefixId}`; - const possible2 = `space:${prefixId}`; - const update1 = await datasource.queryDocState(possible1); - const update2 = await datasource.queryDocState(possible2); + const update = await datasource.queryDocState(guid); let hasUpdate = false; if ( - update1 && - update1.missing.length !== 2 && - update1.missing[0] !== 0 && - update1.missing[1] !== 0 + update && + update.missing.length !== 2 && + update.missing[0] !== 0 && + update.missing[1] !== 0 ) { - applyUpdate(doc, update1.missing, origin); - hasUpdate = true; - } - if ( - update2 && - update2.missing.length !== 2 && - update2.missing[0] !== 0 && - update2.missing[1] !== 0 - ) { - applyUpdate(doc, update2.missing, origin); + applyUpdate(doc, update.missing, origin); hasUpdate = true; } if (hasUpdate) { await datasource.sendDocUpdate( guid, - encodeStateAsUpdate( - doc, - update1 ? update1.state : update2 ? update2.state : undefined - ) + encodeStateAsUpdate(doc, update ? update.state : undefined) ); } } diff --git a/packages/frontend/electron/package.json b/packages/frontend/electron/package.json index d93617e921..d5690a26d8 100644 --- a/packages/frontend/electron/package.json +++ b/packages/frontend/electron/package.json @@ -51,7 +51,7 @@ "builder-util-runtime": "^9.2.1", "cross-env": "^7.0.3", "electron": "^27.0.0", - "electron-log": "^5.0.0-rc.1", + "electron-log": "^5.0.0", "electron-squirrel-startup": "1.0.0", "electron-window-state": "^5.0.3", "esbuild": "^0.19.4", diff --git a/packages/frontend/electron/scripts/dev.ts b/packages/frontend/electron/scripts/dev.ts index 6b84ab7e7e..03796db9e7 100644 --- a/packages/frontend/electron/scripts/dev.ts +++ b/packages/frontend/electron/scripts/dev.ts @@ -54,7 +54,6 @@ function spawnOrReloadElectron() { if (code && code !== 0) { console.log(`Electron exited with code ${code}`); } - process.exit(code ?? 0); }); } diff --git a/packages/frontend/electron/src/helper/db/base-db-adapter.ts b/packages/frontend/electron/src/helper/db/base-db-adapter.ts index 32acd64bdc..30d3fe4737 100644 --- a/packages/frontend/electron/src/helper/db/base-db-adapter.ts +++ b/packages/frontend/electron/src/helper/db/base-db-adapter.ts @@ -5,7 +5,7 @@ import { } from '@affine/native'; import { WorkspaceVersion } from '@toeverything/infra/blocksuite'; -import { migrateToLatest } from '../db/migration'; +import { applyGuidCompatibilityFix, migrateToLatest } from '../db/migration'; import { logger } from '../logger'; /** @@ -29,6 +29,7 @@ export abstract class BaseSQLiteAdapter { if (maxVersion !== WorkspaceVersion.Surface) { await migrateToLatest(this.path, WorkspaceVersion.Surface); } + await applyGuidCompatibilityFix(this.db); logger.info(`[SQLiteAdapter:${this.role}]`, 'connected:', this.path); } return this.db; diff --git a/packages/frontend/electron/src/helper/db/migration.ts b/packages/frontend/electron/src/helper/db/migration.ts index 12a0a7aaf5..8673a5d869 100644 --- a/packages/frontend/electron/src/helper/db/migration.ts +++ b/packages/frontend/electron/src/helper/db/migration.ts @@ -6,6 +6,7 @@ import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models'; import { Schema } from '@blocksuite/store'; import { forceUpgradePages, + guidCompatibilityFix, migrateToSubdoc, WorkspaceVersion, } from '@toeverything/infra/blocksuite'; @@ -119,3 +120,21 @@ async function replaceRows( }) ); } + +export const applyGuidCompatibilityFix = async (db: SqliteConnection) => { + const oldRows = await db.getUpdates(undefined); + + const rootDoc = new YDoc(); + oldRows.forEach(row => applyUpdate(rootDoc, row.data)); + + // see comments of guidCompatibilityFix + guidCompatibilityFix(rootDoc); + + // todo: backup? + await db.replaceUpdates(undefined, [ + { + docId: undefined, + data: encodeStateAsUpdate(rootDoc), + }, + ]); +}; diff --git a/packages/frontend/electron/src/main/logger.ts b/packages/frontend/electron/src/main/logger.ts index 9f6faef8df..bc3ab3d24b 100644 --- a/packages/frontend/electron/src/main/logger.ts +++ b/packages/frontend/electron/src/main/logger.ts @@ -1,4 +1,5 @@ import { shell } from 'electron'; +import { app } from 'electron'; import log from 'electron-log'; export const logger = log.scope('main'); @@ -12,3 +13,7 @@ export async function revealLogFile() { const filePath = getLogFilePath(); return await shell.openPath(filePath); } + +app.on('before-quit', () => { + log.transports.console.level = false; +}); diff --git a/packages/frontend/electron/src/main/updater/electron-updater.ts b/packages/frontend/electron/src/main/updater/electron-updater.ts index dc91ab7537..74559e1a8d 100644 --- a/packages/frontend/electron/src/main/updater/electron-updater.ts +++ b/packages/frontend/electron/src/main/updater/electron-updater.ts @@ -10,6 +10,9 @@ import { updaterSubjects } from './event'; const mode = process.env.NODE_ENV; const isDev = mode === 'development'; +// skip auto update in dev mode & internal +const disabled = buildType === 'internal' || isDev; + export const quitAndInstall = async () => { autoUpdater.quitAndInstall(); }; @@ -17,7 +20,7 @@ export const quitAndInstall = async () => { let lastCheckTime = 0; export const checkForUpdates = async (force = true) => { // check every 30 minutes (1800 seconds) at most - if (force || lastCheckTime + 1000 * 1800 < Date.now()) { + if (!disabled && (force || lastCheckTime + 1000 * 1800 < Date.now())) { lastCheckTime = Date.now(); return await autoUpdater.checkForUpdates(); } @@ -25,8 +28,7 @@ export const checkForUpdates = async (force = true) => { }; export const registerUpdater = async () => { - // skip auto update in dev mode & internal - if (buildType === 'internal' || isDev) { + if (disabled) { return; } @@ -43,7 +45,6 @@ export const registerUpdater = async () => { channel: buildType, // hack for custom provider provider: 'custom' as 'github', - // @ts-expect-error - just ignore for now repo: buildType !== 'internal' ? 'AFFiNE' : 'AFFiNE-Releases', owner: 'toeverything', releaseType: buildType === 'stable' ? 'release' : 'prerelease', diff --git a/packages/frontend/hooks/src/__tests__/index.spec.ts b/packages/frontend/hooks/src/__tests__/index.spec.ts index 612f67eea1..88ecbfb611 100644 --- a/packages/frontend/hooks/src/__tests__/index.spec.ts +++ b/packages/frontend/hooks/src/__tests__/index.spec.ts @@ -76,12 +76,25 @@ describe('useBlockSuitePagePreview', () => { page.getBlockByFlavour('affine:note')[0].id ); const hook = renderHook(() => useAtomValue(useBlockSuitePagePreview(page))); - expect(hook.result.current).toBe('\nHello, world!'); + expect(hook.result.current).toBe('Hello, world!'); page.transact(() => { page.getBlockById(id)!.text!.insert('Test', 0); }); await new Promise(resolve => setTimeout(resolve, 100)); hook.rerender(); - expect(hook.result.current).toBe('\nTestHello, world!'); + expect(hook.result.current).toBe('TestHello, world!'); + + // Insert before + page.addBlock( + 'affine:paragraph', + { + text: new page.Text('First block!'), + }, + page.getBlockByFlavour('affine:note')[0].id, + 0 + ); + await new Promise(resolve => setTimeout(resolve, 100)); + hook.rerender(); + expect(hook.result.current).toBe('First block! TestHello, world!'); }); }); diff --git a/packages/frontend/hooks/src/use-block-suite-page-preview.ts b/packages/frontend/hooks/src/use-block-suite-page-preview.ts index 3947839f67..869e56b555 100644 --- a/packages/frontend/hooks/src/use-block-suite-page-preview.ts +++ b/packages/frontend/hooks/src/use-block-suite-page-preview.ts @@ -1,20 +1,49 @@ -import type { ParagraphBlockModel } from '@blocksuite/blocks/models'; import type { Page } from '@blocksuite/store'; import type { Atom } from 'jotai'; import { atom } from 'jotai'; +const MAX_PREVIEW_LENGTH = 150; +const MAX_SEARCH_BLOCK_COUNT = 30; + const weakMap = new WeakMap>(); export const getPagePreviewText = (page: Page) => { - // TODO this is incorrect, since the order of blocks is not guaranteed - const paragraphBlocks = page.getBlockByFlavour( - 'affine:paragraph' - ) as ParagraphBlockModel[]; - const text = paragraphBlocks - .slice(0, 10) - .map(block => block.text.toString()) - .join('\n'); - return text.slice(0, 300); + const pageRoot = page.root; + if (!pageRoot) { + return ''; + } + const preview: string[] = []; + // DFS + const queue = [pageRoot]; + let previewLenNeeded = MAX_PREVIEW_LENGTH; + let count = MAX_SEARCH_BLOCK_COUNT; + while (queue.length && previewLenNeeded > 0 && count-- > 0) { + const block = queue.shift(); + if (!block) { + console.error('Unexpected empty block'); + break; + } + if (block.children) { + queue.unshift(...block.children); + } + if (block.role !== 'content') { + continue; + } + if (block.text) { + const text = block.text.toString(); + if (!text.length) { + continue; + } + previewLenNeeded -= text.length; + preview.push(text); + } else { + // image/attachment/bookmark + const type = block.flavour.split('affine:')[1] ?? null; + previewLenNeeded -= type.length + 2; + type && preview.push(`[${type}]`); + } + } + return preview.join(' '); }; const emptyAtom = atom(''); diff --git a/packages/frontend/hooks/src/use-block-suite-workspace-page.ts b/packages/frontend/hooks/src/use-block-suite-workspace-page.ts index 8b0cb27e42..4df7360604 100644 --- a/packages/frontend/hooks/src/use-block-suite-workspace-page.ts +++ b/packages/frontend/hooks/src/use-block-suite-workspace-page.ts @@ -85,7 +85,7 @@ export function loadPage(page: Page, priority = 0) { logger.debug('page loaded', page.id); // we do not know how long it takes to load a page here // so that we just use 300ms timeout as the default page processing time - await awaitForTimeout(1000); + await awaitForTimeout(300); } else { // do nothing if it is already loaded } diff --git a/yarn.lock b/yarn.lock index 2f84cb0787..ed763c9e86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -427,7 +427,7 @@ __metadata: builder-util-runtime: "npm:^9.2.1" cross-env: "npm:^7.0.3" electron: "npm:^27.0.0" - electron-log: "npm:^5.0.0-rc.1" + electron-log: "npm:^5.0.0" electron-squirrel-startup: "npm:1.0.0" electron-updater: "npm:^6.1.5" electron-window-state: "npm:^5.0.3" @@ -19145,10 +19145,10 @@ __metadata: languageName: node linkType: hard -"electron-log@npm:^5.0.0-rc.1": - version: 5.0.0-rc.1 - resolution: "electron-log@npm:5.0.0-rc.1" - checksum: f4ec437197ec5801a325e062c19f182a14eba960ee683034bfea5854efe452cfa91985b7f7ab159599c8264fde869f7101d4e82303231865a6b7c8e621815f87 +"electron-log@npm:^5.0.0": + version: 5.0.0 + resolution: "electron-log@npm:5.0.0" + checksum: 23b14119a5753be24880e7466ee80ae1386f9df4123ed59bc8f4426a814c728875b07de13bf0729cba7202888fcd6230375e2b5302cee0d0c5f25584d9db3334 languageName: node linkType: hard