v0.8.2

README.md

@@ -146,14 +146,42 @@ Thanks a lot to the community for providing such powerful and simple libraries,
 
 # Contributors
 
+## Current Core members
+
+Team members who are currently maintaining the project:
+
+- [JimmFly](https://github.com/JimmFly) - Jinfei Yang <yangjinfei001@gmail.com> (he/him)
+- [pengx17](https://github.com/pengx17) - Peng Xiao <pengxiao@outlook.com> (he/him)
+- [QiShaoXuan](https://github.com/QiSHaoXuan) - Shaoxuan Qi <qishaoxuan777@gmail.com> (he/him)
+- [himself65](https://github.com/himself65) - Zeyu "Alex" Yang <himself65@outlook.com> (he/him)
+
+## All Contributors
+
 We would like to express our gratitude to all the individuals who have already contributed to AFFiNE! If you have any AFFiNE-related project, documentation, tool or template, please feel free to contribute it by submitting a pull request to our curated list on GitHub: [awesome-affine](https://github.com/toeverything/awesome-affine).
 
 <a href="https://github.com/toeverything/affine/graphs/contributors">
   <img alt="contributors" src="https://opencollective.com/affine/contributors.svg?width=890&button=false" />
 </a>
 
+## Data Compatibility
+
+Data compatibility is a very important issue for us. We will try our best to ensure that the data is compatible with the previous version.
+
+If you encounter any problems when upgrading the version, please feel free to [contact us](mailto:developer@toeverything.info).
+
+| AFFiNE Version | Export/Import workspace | Data auto migration |
+| -------------- | ----------------------- | ------------------- |
+| <= 0.5.4       | ❌️                     | ❌                  |
+| 0.6.x          | ✅️                     | ✅                  |
+| 0.7.x          | ✅️                     | ✅                  |
+| 0.8.x          | ✅                      | ✅                  |
+
 ## Self-Host
 
+> We know that the self-host version has been out of date for a long time.
+>
+> We are working hard to get this updated to the latest version, you can try our desktop version first.
+
 Get started with Docker and deploy your own feature-rich, restriction-free deployment of AFFiNE.
 We are working hard to get this updated to the latest version, you can keep an eye on the [latest packages].
 

apps/core/package.json

@@ -2,7 +2,7 @@
   "name": "@affine/core",
   "type": "module",
   "private": true,
-  "version": "0.8.0",
+  "version": "0.8.2",
   "scripts": {
     "build": "yarn -T run build-core",
     "dev": "yarn -T run dev-core",

apps/core/src/bootstrap/setup.ts

@@ -1,27 +1,27 @@
-import {
-  migrateDatabaseBlockTo3,
-  migrateToSubdoc,
-} from '@affine/env/blocksuite';
 import { setupGlobal } from '@affine/env/global';
-import type {
-  LocalIndexedDBDownloadProvider,
-  WorkspaceAdapter,
-} from '@affine/env/workspace';
-import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import type { WorkspaceAdapter } from '@affine/env/workspace';
+import { WorkspaceFlavour } from '@affine/env/workspace';
 import type { RootWorkspaceMetadata } from '@affine/workspace/atom';
 import {
   type RootWorkspaceMetadataV2,
   rootWorkspacesMetadataAtom,
   workspaceAdaptersAtom,
 } from '@affine/workspace/atom';
-import { globalBlockSuiteSchema } from '@affine/workspace/manager';
+import {
+  getOrCreateWorkspace,
+  globalBlockSuiteSchema,
+} from '@affine/workspace/manager';
+import { assertExists } from '@blocksuite/global/utils';
+import { nanoid } from '@blocksuite/store';
 import {
   migrateLocalBlobStorage,
-  upgradeV1ToV2,
-} from '@affine/workspace/migration';
-import { createIndexedDBDownloadProvider } from '@affine/workspace/providers';
-import { assertExists } from '@blocksuite/global/utils';
+  migrateWorkspace,
+  WorkspaceVersion,
+} from '@toeverything/infra/blocksuite';
+import { downloadBinary } from '@toeverything/y-indexeddb';
 import type { createStore } from 'jotai/vanilla';
+import { Doc } from 'yjs';
+import { applyUpdate } from 'yjs';
 
 import { WorkspaceAdapters } from '../adapters/workspace';
 
@@ -33,94 +33,57 @@ async function tryMigration() {
       const promises: Promise<void>[] = [];
       const newMetadata = [...metadata];
       metadata.forEach(oldMeta => {
-        if (!('version' in oldMeta)) {
-          const adapter = WorkspaceAdapters[oldMeta.flavour];
-          assertExists(adapter);
-          const upgrade = async () => {
-            if (oldMeta.flavour !== WorkspaceFlavour.LOCAL) {
-              console.warn('not supported');
-              return;
-            }
-            const workspace = await adapter.CRUD.get(oldMeta.id);
-            if (!workspace) {
-              console.warn('cannot find workspace', oldMeta.id);
-              return;
-            }
-            const doc = workspace.blockSuiteWorkspace.doc;
-            const provider = createIndexedDBDownloadProvider(
-              workspace.id,
-              doc,
-              {
-                awareness:
-                  workspace.blockSuiteWorkspace.awarenessStore.awareness,
-              }
-            ) as LocalIndexedDBDownloadProvider;
-            provider.sync();
-            await provider.whenReady;
-            const newDoc = migrateToSubdoc(doc);
-            if (doc === newDoc) {
-              console.log('doc not changed');
-              return;
-            }
-            const newWorkspace = upgradeV1ToV2(workspace);
-            await migrateDatabaseBlockTo3(
-              newWorkspace.blockSuiteWorkspace.doc,
-              globalBlockSuiteSchema
-            );
-
-            const newId = await adapter.CRUD.create(
-              newWorkspace.blockSuiteWorkspace
-            );
-
-            await adapter.CRUD.delete(workspace as any);
-            console.log('migrated', oldMeta.id, newId);
-            const index = newMetadata.findIndex(meta => meta.id === oldMeta.id);
-            newMetadata[index] = {
-              ...oldMeta,
-              id: newId,
-              version: WorkspaceVersion.DatabaseV3,
-            };
-            await migrateLocalBlobStorage(workspace.id, newId);
-            console.log('migrate to v2');
-          };
-
-          // create a new workspace and push it to metadata
-          promises.push(upgrade());
-        } else if (oldMeta.version < WorkspaceVersion.DatabaseV3) {
-          const adapter = WorkspaceAdapters[oldMeta.flavour];
-          assertExists(adapter);
+        if (oldMeta.flavour === WorkspaceFlavour.LOCAL) {
           promises.push(
-            (async () => {
-              if (oldMeta.flavour !== WorkspaceFlavour.LOCAL) {
-                console.warn('not supported');
-                return;
+            migrateWorkspace(
+              'version' in oldMeta ? oldMeta.version : undefined,
+              {
+                getCurrentRootDoc: async () => {
+                  const doc = new Doc({
+                    guid: oldMeta.id,
+                  });
+                  const downloadWorkspace = async (doc: Doc): Promise<void> => {
+                    const binary = await downloadBinary(doc.guid);
+                    if (binary) {
+                      applyUpdate(doc, binary);
+                    }
+                    return Promise.all(
+                      [...doc.subdocs.values()].map(subdoc =>
+                        downloadWorkspace(subdoc)
+                      )
+                    ).then();
+                  };
+                  await downloadWorkspace(doc);
+                  return doc;
+                },
+                createWorkspace: async () =>
+                  getOrCreateWorkspace(nanoid(), WorkspaceFlavour.LOCAL),
+                getSchema: () => globalBlockSuiteSchema,
               }
-              const workspace = await adapter.CRUD.get(oldMeta.id);
-              if (workspace) {
-                const provider = createIndexedDBDownloadProvider(
-                  workspace.id,
-                  workspace.blockSuiteWorkspace.doc,
-                  {
-                    awareness:
-                      workspace.blockSuiteWorkspace.awarenessStore.awareness,
-                  }
-                ) as LocalIndexedDBDownloadProvider;
-                provider.sync();
-                await provider.whenReady;
-                await migrateDatabaseBlockTo3(
-                  workspace.blockSuiteWorkspace.doc,
-                  globalBlockSuiteSchema
+            ).then(async workspace => {
+              if (typeof workspace !== 'boolean') {
+                const adapter = WorkspaceAdapters[oldMeta.flavour];
+                const oldWorkspace = await adapter.CRUD.get(oldMeta.id);
+                const newId = await adapter.CRUD.create(workspace);
+                assertExists(
+                  oldWorkspace,
+                  'workspace should exist after migrate'
                 );
+                await adapter.CRUD.delete(oldWorkspace.blockSuiteWorkspace);
+                const index = newMetadata.findIndex(
+                  meta => meta.id === oldMeta.id
+                );
+                newMetadata[index] = {
+                  ...oldMeta,
+                  id: newId,
+                  version: WorkspaceVersion.DatabaseV3,
+                };
+                await migrateLocalBlobStorage(workspace.id, newId);
+                console.log('workspace migrated', oldMeta.id, newId);
+              } else if (workspace) {
+                console.log('workspace migrated', oldMeta.id);
               }
-              const index = newMetadata.findIndex(
-                meta => meta.id === oldMeta.id
-              );
-              newMetadata[index] = {
-                ...oldMeta,
-                version: WorkspaceVersion.DatabaseV3,
-              };
-              console.log('migrate to v3');
-            })()
+            })
           );
         }
       });

apps/core/src/hooks/use-transform-workspace.ts

@@ -1,7 +1,7 @@
 import type { WorkspaceFlavour } from '@affine/env/workspace';
 import type { WorkspaceRegistry } from '@affine/env/workspace';
-import { WorkspaceVersion } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
+import { WorkspaceVersion } from '@toeverything/infra/blocksuite';
 import { useSetAtom } from 'jotai';
 import { useCallback } from 'react';
 

apps/core/src/hooks/use-workspaces.ts

@@ -1,12 +1,15 @@
 import { DebugLogger } from '@affine/debug';
-import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import { WorkspaceFlavour } from '@affine/env/workspace';
 import { rootWorkspacesMetadataAtom } from '@affine/workspace/atom';
 import { saveWorkspaceToLocalStorage } from '@affine/workspace/local/crud';
 import { getOrCreateWorkspace } from '@affine/workspace/manager';
 import { nanoid } from '@blocksuite/store';
 import { getWorkspace } from '@toeverything/infra/__internal__/workspace';
 import { getCurrentStore } from '@toeverything/infra/atom';
-import { buildShowcaseWorkspace } from '@toeverything/infra/blocksuite';
+import {
+  buildShowcaseWorkspace,
+  WorkspaceVersion,
+} from '@toeverything/infra/blocksuite';
 import { useAtomValue, useSetAtom } from 'jotai';
 import { useCallback } from 'react';
 

apps/core/src/layouts/workspace-layout.tsx

@@ -33,8 +33,9 @@ import { usePassiveWorkspaceEffect } from '@toeverything/infra/__internal__/reac
 import { currentWorkspaceIdAtom } from '@toeverything/infra/atom';
 import { useAtom, useAtomValue, useSetAtom } from 'jotai';
 import type { PropsWithChildren, ReactElement } from 'react';
-import { lazy, Suspense, useCallback, useMemo } from 'react';
+import { lazy, Suspense, useCallback, useEffect, useMemo } from 'react';
 import { useLocation, useParams } from 'react-router-dom';
+import { Map as YMap } from 'yjs';
 
 import { WorkspaceAdapters } from '../adapters/workspace';
 import {
@@ -150,6 +151,23 @@ export const WorkspaceLayoutInner = ({ children }: PropsWithChildren) => {
   const [currentWorkspace] = useCurrentWorkspace();
   const { openPage } = useNavigateHelper();
 
+  useEffect(() => {
+    // hotfix for blockVersions
+    // this is a mistake in the
+    //    0.8.0 ~ 0.8.1
+    //    0.8.0-beta.0 ~ 0.8.0-beta.3
+    //    0.9.0-canary.0 ~ 0.9.0-canary.3
+    const meta = currentWorkspace.blockSuiteWorkspace.doc.getMap('meta');
+    const blockVersions = meta.get('blockVersions');
+    if (!(blockVersions instanceof YMap)) {
+      console.log('fixing blockVersions');
+      meta.set(
+        'blockVersions',
+        new YMap(Object.entries(blockVersions as Record<string, number>))
+      );
+    }
+  }, [currentWorkspace.blockSuiteWorkspace.doc]);
+
   usePassiveWorkspaceEffect(currentWorkspace.blockSuiteWorkspace);
 
   const [, setOpenWorkspacesModal] = useAtom(openWorkspacesModalAtom);

apps/core/src/pages/index.tsx

@@ -23,6 +23,7 @@ export const loader: LoaderFunction = async () => {
   const target = (lastId && meta.find(({ id }) => id === lastId)) || meta.at(0);
   if (target) {
     const targetWorkspace = getWorkspace(target.id);
+
     const nonTrashPages = targetWorkspace.meta.pageMetas.filter(
       ({ trash }) => !trash
     );

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/docs",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "type": "module",
   "private": true,
   "scripts": {

apps/electron/dev-app-update.yml

@@ -1,4 +1,4 @@
 owner: toeverything
 repo: AFFiNE
-provider: github
+provider: custom
 private: false

apps/electron/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/electron",
   "private": true,
-  "version": "0.8.0",
+  "version": "0.8.2",
   "author": "affine",
   "repository": {
     "url": "https://github.com/toeverything/AFFiNE",
@@ -42,6 +42,7 @@
     "@electron-forge/shared-types": "^6.4.0",
     "@electron/remote": "2.0.10",
     "@reforged/maker-appimage": "^3.3.1",
+    "@toeverything/infra": "workspace:*",
     "@types/fs-extra": "^11.0.1",
     "@types/uuid": "^9.0.2",
     "cross-env": "7.0.3",
@@ -53,6 +54,8 @@
     "fs-extra": "^11.1.1",
     "glob": "^10.3.3",
     "jotai": "^2.3.1",
+    "lodash-es": "^4.17.21",
+    "rxjs": "^7.8.1",
     "ts-node": "^10.9.1",
     "undici": "^5.23.0",
     "uuid": "^9.0.0",
@@ -61,13 +64,10 @@
     "zx": "^7.2.3"
   },
   "dependencies": {
-    "@toeverything/infra": "workspace:*",
     "async-call-rpc": "^6.3.1",
     "electron-updater": "^6.1.4",
     "link-preview-js": "^3.0.5",
-    "lodash-es": "^4.17.21",
     "nanoid": "^4.0.2",
-    "rxjs": "^7.8.1",
     "yjs": "^13.6.7"
   },
   "build": {

apps/electron/resources/app-update.yml

@@ -1,4 +1,4 @@
 owner: toeverything
 repo: AFFiNE
-provider: github
+provider: custom
 private: false

apps/electron/scripts/common.mjs

@@ -44,6 +44,7 @@ export const config = () => {
         'electron-updater',
         '@toeverything/plugin-infra',
         'yjs',
+        'semver',
       ],
       define: define,
       format: 'cjs',

apps/electron/src/helper/db/migration.ts

@@ -1,7 +1,11 @@
+import { equal } from 'node:assert';
 import { resolve } from 'node:path';
 
-import { migrateToSubdoc } from '@affine/env/blocksuite';
 import { SqliteConnection } from '@affine/native';
+import {
+  migrateToSubdoc,
+  WorkspaceVersion,
+} from '@toeverything/infra/blocksuite';
 import fs from 'fs-extra';
 import { nanoid } from 'nanoid';
 import { applyUpdate, Doc as YDoc, encodeStateAsUpdate } from 'yjs';
@@ -30,6 +34,74 @@ export const migrateToSubdocAndReplaceDatabase = async (path: string) => {
   await db.close();
 };
 
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import { Schema, Workspace } from '@blocksuite/store';
+import { migrateWorkspace } from '@toeverything/infra/blocksuite';
+
+// v1 v2 -> v3
+export const migrateToLatestDatabase = async (path: string) => {
+  const connection = new SqliteConnection(path);
+  await connection.connect();
+  await connection.initVersion();
+  const schema = new Schema();
+  schema.register(AffineSchemas).register(__unstableSchemas);
+  const rootDoc = new YDoc();
+  const downloadBinary = async (doc: YDoc, isRoot: boolean): Promise<void> => {
+    const update = (
+      await connection.getUpdates(isRoot ? undefined : doc.guid)
+    ).map(update => update.data);
+    // Buffer[] -> Uint8Array[]
+    const data = update.map(update => new Uint8Array(update));
+    data.forEach(data => {
+      applyUpdate(doc, data);
+    });
+    // trigger data manually
+    if (isRoot) {
+      doc.getMap('meta');
+      doc.getMap('spaces');
+    } else {
+      doc.getMap('blocks');
+    }
+    await Promise.all(
+      [...doc.subdocs].map(subdoc => {
+        return downloadBinary(subdoc, false);
+      })
+    );
+  };
+  await downloadBinary(rootDoc, true);
+  const result = await migrateWorkspace(WorkspaceVersion.SubDoc, {
+    getSchema: () => schema,
+    getCurrentRootDoc: () => Promise.resolve(rootDoc),
+    createWorkspace: () =>
+      Promise.resolve(
+        new Workspace({
+          id: nanoid(10),
+          schema,
+          blobStorages: [],
+          providerCreators: [],
+        })
+      ),
+  });
+  equal(
+    typeof result,
+    'boolean',
+    'migrateWorkspace should return boolean value'
+  );
+  const uploadBinary = async (doc: YDoc, isRoot: boolean) => {
+    await connection.replaceUpdates(doc.guid, [
+      { docId: isRoot ? undefined : doc.guid, data: encodeStateAsUpdate(doc) },
+    ]);
+    // connection..applyUpdate(encodeStateAsUpdate(doc), 'self', doc.guid)
+    await Promise.all(
+      [...doc.subdocs].map(subdoc => {
+        return uploadBinary(subdoc, false);
+      })
+    );
+  };
+  await uploadBinary(rootDoc, true);
+  await connection.close();
+};
+
 export const copyToTemp = async (path: string) => {
   const tmpDirPath = resolve(await mainRPC.getPath('sessionData'), 'tmp');
   const tmpFilePath = resolve(tmpDirPath, nanoid());

apps/electron/src/helper/dialog/dialog.ts

@@ -12,7 +12,11 @@ import fs from 'fs-extra';
 import { nanoid } from 'nanoid';
 
 import { ensureSQLiteDB } from '../db/ensure-db';
-import { copyToTemp, migrateToSubdocAndReplaceDatabase } from '../db/migration';
+import {
+  copyToTemp,
+  migrateToLatestDatabase,
+  migrateToSubdocAndReplaceDatabase,
+} from '../db/migration';
 import type { WorkspaceSQLiteDB } from '../db/workspace-db-adapter';
 import { logger } from '../logger';
 import { mainRPC } from '../main-rpc';
@@ -197,7 +201,22 @@ export async function loadDBFile(): Promise<LoadDBFileResult> {
       }
     }
 
+    if (validationResult === ValidationResult.MissingVersionColumn) {
+      try {
+        const tmpDBPath = await copyToTemp(originalPath);
+        await migrateToLatestDatabase(tmpDBPath);
+        originalPath = tmpDBPath;
+      } catch (error) {
+        logger.warn(
+          `loadDBFile, migration version column failed: ${originalPath}`,
+          error
+        );
+        return { error: 'DB_FILE_MIGRATION_FAILED' };
+      }
+    }
+
     if (
+      validationResult !== ValidationResult.MissingVersionColumn &&
       validationResult !== ValidationResult.MissingDocIdColumn &&
       validationResult !== ValidationResult.Valid
     ) {

apps/electron/src/main/updater/custom-github-provider.ts

@@ -0,0 +1,250 @@
+// credits: migrated from https://github.com/electron-userland/electron-builder/blob/master/packages/electron-updater/src/providers/GitHubProvider.ts
+import type {
+  CustomPublishOptions,
+  GithubOptions,
+  ReleaseNoteInfo,
+  XElement,
+} from 'builder-util-runtime';
+import { HttpError, newError, parseXml } from 'builder-util-runtime';
+import {
+  type AppUpdater,
+  CancellationToken,
+  type ResolvedUpdateFileInfo,
+  type UpdateInfo,
+} from 'electron-updater';
+import { BaseGitHubProvider } from 'electron-updater/out/providers/GitHubProvider';
+import {
+  parseUpdateInfo,
+  type ProviderRuntimeOptions,
+  resolveFiles,
+} from 'electron-updater/out/providers/Provider';
+import * as semver from 'semver';
+
+interface GithubUpdateInfo extends UpdateInfo {
+  tag: string;
+}
+
+const hrefRegExp = /\/tag\/([^/]+)$/;
+
+export class CustomGitHubProvider extends BaseGitHubProvider<GithubUpdateInfo> {
+  constructor(
+    options: CustomPublishOptions,
+    private updater: AppUpdater,
+    runtimeOptions: ProviderRuntimeOptions
+  ) {
+    super(options as unknown as GithubOptions, 'github.com', runtimeOptions);
+  }
+
+  async getLatestVersion(): Promise<GithubUpdateInfo> {
+    const cancellationToken = new CancellationToken();
+
+    const feedXml = await this.httpRequest(
+      newUrlFromBase(`${this.basePath}.atom`, this.baseUrl),
+      {
+        accept: 'application/xml, application/atom+xml, text/xml, */*',
+      },
+      cancellationToken
+    );
+
+    if (!feedXml) {
+      throw new Error(
+        `Cannot find feed in the remote server (${this.baseUrl.href})`
+      );
+    }
+
+    const feed = parseXml(feedXml);
+    // noinspection TypeScriptValidateJSTypes
+    const latestRelease = feed.element(
+      'entry',
+      false,
+      `No published versions on GitHub`
+    );
+    let tag: string | null = null;
+    try {
+      const currentChannel =
+        this.options.channel ||
+        this.updater?.channel ||
+        (semver.prerelease(this.updater.currentVersion)?.[0] as string) ||
+        null;
+
+      if (currentChannel === null) {
+        throw newError(
+          `Cannot parse channel from version: ${this.updater.currentVersion}`,
+          'ERR_UPDATER_INVALID_VERSION'
+        );
+      }
+
+      for (const element of feed.getElements('entry')) {
+        // noinspection TypeScriptValidateJSTypes
+        const hrefElement = hrefRegExp.exec(
+          element.element('link').attribute('href')
+        );
+
+        // If this is null then something is wrong and skip this release
+        if (hrefElement === null) continue;
+
+        // This Release's Tag
+        const hrefTag = hrefElement[1];
+        // Get Channel from this release's tag
+        // If it is null, we believe it is stable version
+        const hrefChannel =
+          (semver.prerelease(hrefTag)?.[0] as string) || 'stable';
+
+        const isNextPreRelease = hrefChannel === currentChannel;
+        if (isNextPreRelease) {
+          tag = hrefTag;
+          break;
+        }
+      }
+    } catch (e: any) {
+      throw newError(
+        `Cannot parse releases feed: ${
+          e.stack || e.message
+        },\nXML:\n${feedXml}`,
+        'ERR_UPDATER_INVALID_RELEASE_FEED'
+      );
+    }
+
+    if (tag == null) {
+      throw newError(
+        `No published versions on GitHub`,
+        'ERR_UPDATER_NO_PUBLISHED_VERSIONS'
+      );
+    }
+
+    let rawData: string | null = null;
+    let channelFile = '';
+    let channelFileUrl: any = '';
+    const fetchData = async (channelName: string) => {
+      channelFile = getChannelFilename(channelName);
+      channelFileUrl = newUrlFromBase(
+        this.getBaseDownloadPath(String(tag), channelFile),
+        this.baseUrl
+      );
+      const requestOptions = this.createRequestOptions(channelFileUrl);
+      try {
+        return await this.executor.request(requestOptions, cancellationToken);
+      } catch (e: any) {
+        if (e instanceof HttpError && e.statusCode === 404) {
+          throw newError(
+            `Cannot find ${channelFile} in the latest release artifacts (${channelFileUrl}): ${
+              e.stack || e.message
+            }`,
+            'ERR_UPDATER_CHANNEL_FILE_NOT_FOUND'
+          );
+        }
+        throw e;
+      }
+    };
+
+    try {
+      const channel = this.updater.allowPrerelease
+        ? this.getCustomChannelName(
+            String(semver.prerelease(tag)?.[0] || 'latest')
+          )
+        : this.getDefaultChannelName();
+      rawData = await fetchData(channel);
+    } catch (e: any) {
+      if (this.updater.allowPrerelease) {
+        // Allow fallback to `latest.yml`
+        rawData = await fetchData(this.getDefaultChannelName());
+      } else {
+        throw e;
+      }
+    }
+
+    const result = parseUpdateInfo(rawData, channelFile, channelFileUrl);
+    if (result.releaseName == null) {
+      result.releaseName = latestRelease.elementValueOrEmpty('title');
+    }
+
+    if (result.releaseNotes == null) {
+      result.releaseNotes = computeReleaseNotes(
+        this.updater.currentVersion,
+        this.updater.fullChangelog,
+        feed,
+        latestRelease
+      );
+    }
+    return {
+      tag: tag,
+      ...result,
+    };
+  }
+
+  private get basePath(): string {
+    return `/${this.options.owner}/${this.options.repo}/releases`;
+  }
+
+  resolveFiles(updateInfo: GithubUpdateInfo): Array<ResolvedUpdateFileInfo> {
+    // still replace space to - due to backward compatibility
+    return resolveFiles(updateInfo, this.baseUrl, p =>
+      this.getBaseDownloadPath(updateInfo.tag, p.replace(/ /g, '-'))
+    );
+  }
+
+  private getBaseDownloadPath(tag: string, fileName: string): string {
+    return `${this.basePath}/download/${tag}/${fileName}`;
+  }
+}
+
+export interface CustomGitHubOptions {
+  channel: string;
+  repo: string;
+  owner: string;
+  releaseType: 'release' | 'prerelease';
+}
+
+function getNoteValue(parent: XElement): string {
+  const result = parent.elementValueOrEmpty('content');
+  // GitHub reports empty notes as <content>No content.</content>
+  return result === 'No content.' ? '' : result;
+}
+
+export function computeReleaseNotes(
+  currentVersion: semver.SemVer,
+  isFullChangelog: boolean,
+  feed: XElement,
+  latestRelease: any
+): string | Array<ReleaseNoteInfo> | null {
+  if (!isFullChangelog) {
+    return getNoteValue(latestRelease);
+  }
+
+  const releaseNotes: Array<ReleaseNoteInfo> = [];
+  for (const release of feed.getElements('entry')) {
+    // noinspection TypeScriptValidateJSTypes
+    const versionRelease = /\/tag\/v?([^/]+)$/.exec(
+      release.element('link').attribute('href')
+    )?.[1];
+    if (versionRelease && semver.lt(currentVersion, versionRelease)) {
+      releaseNotes.push({
+        version: versionRelease,
+        note: getNoteValue(release),
+      });
+    }
+  }
+  return releaseNotes.sort((a, b) => semver.rcompare(a.version, b.version));
+}
+
+// addRandomQueryToAvoidCaching is false by default because in most cases URL already contains version number,
+// so, it makes sense only for Generic Provider for channel files
+function newUrlFromBase(
+  pathname: string,
+  baseUrl: URL,
+  addRandomQueryToAvoidCaching = false
+): URL {
+  const result = new URL(pathname, baseUrl);
+  // search is not propagated (search is an empty string if not specified)
+  const search = baseUrl.search;
+  if (search != null && search.length !== 0) {
+    result.search = search;
+  } else if (addRandomQueryToAvoidCaching) {
+    result.search = `noCache=${Date.now().toString(32)}`;
+  }
+  return result;
+}
+
+function getChannelFilename(channel: string): string {
+  return `${channel}.yml`;
+}

apps/electron/src/main/updater/electron-updater.ts

@@ -4,6 +4,7 @@ import { z } from 'zod';
 
 import { isMacOS, isWindows } from '../../shared/utils';
 import { logger } from '../logger';
+import { CustomGitHubProvider } from './custom-github-provider';
 import { updaterSubjects } from './event';
 
 export const ReleaseTypeSchema = z.enum([
@@ -36,7 +37,7 @@ export const checkForUpdates = async (force = true) => {
 
 export const registerUpdater = async () => {
   // skip auto update in dev mode & internal
-  if (isDev || buildType === 'internal') {
+  if (buildType === 'internal') {
     return;
   }
 
@@ -51,11 +52,14 @@ export const registerUpdater = async () => {
 
   const feedUrl: Parameters<typeof autoUpdater.setFeedURL>[0] = {
     channel: buildType,
-    provider: 'github',
+    // hack for custom provider
+    provider: 'custom' as 'github',
     // @ts-expect-error - just ignore for now
     repo: buildType !== 'internal' ? 'AFFiNE' : 'AFFiNE-Releases',
     owner: 'toeverything',
     releaseType: buildType === 'stable' ? 'release' : 'prerelease',
+    // @ts-expect-error hack for custom provider
+    updateProvider: CustomGitHubProvider,
   };
 
   logger.debug('auto-updater feed config', feedUrl);

apps/prototype/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/prototype",
   "private": true,
-  "version": "0.8.0",
+  "version": "0.8.2",
   "type": "module",
   "scripts": {
     "dev": "vite --host --port 3003",

apps/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/server",
   "private": true,
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Affine Node.js server",
   "type": "module",
   "bin": {

apps/storybook/package.json

@@ -52,5 +52,5 @@
     "@blocksuite/lit": "*",
     "@blocksuite/store": "*"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/monorepo",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "private": true,
   "author": "toeverything",
   "license": "MPL-2.0",

packages/@types/env/package.json

@@ -7,5 +7,5 @@
     "@affine/env": "workspace:*",
     "@toeverything/infra": "workspace:*"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/cli/package.json

@@ -20,5 +20,5 @@
   "peerDependencies": {
     "ts-node": "*"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/component/package.json

@@ -66,5 +66,5 @@
     "vite": "^4.4.9",
     "yjs": "^13.6.7"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/debug/package.json

@@ -8,5 +8,5 @@
   "devDependencies": {
     "@types/debug": "^4.1.8"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/env/package.json

@@ -27,5 +27,5 @@
   "dependencies": {
     "lit": "^2.8.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/env/src/__tests__/subdoc-migration.spec.ts

@@ -1,83 +0,0 @@
-import { readFileSync } from 'fs';
-import { dirname, resolve } from 'path';
-import { fileURLToPath } from 'url';
-import { describe, expect, test } from 'vitest';
-import type { Array as YArray, Map as YMap } from 'yjs';
-import { applyUpdate, Doc } from 'yjs';
-
-import { migrateToSubdoc } from '../blocksuite/index.js';
-
-const fixturePath = resolve(
-  dirname(fileURLToPath(import.meta.url)),
-  'workspace.ydoc'
-);
-const yDocBuffer = readFileSync(fixturePath);
-const doc = new Doc();
-applyUpdate(doc, new Uint8Array(yDocBuffer));
-const migratedDoc = migrateToSubdoc(doc);
-
-describe('subdoc', () => {
-  test('Migration to subdoc', async () => {
-    const { default: json } = await import('@affine-test/fixtures/output.json');
-    const length = Object.keys(json).length;
-    const binary = new Uint8Array(length);
-    for (let i = 0; i < length; i++) {
-      binary[i] = (json as any)[i];
-    }
-    const doc = new Doc();
-    applyUpdate(doc, binary);
-    {
-      // invoke data
-      doc.getMap('space:hello-world');
-      doc.getMap('space:meta');
-    }
-    const blocks = doc.getMap('space:hello-world').toJSON();
-    const newDoc = migrateToSubdoc(doc);
-    const subDoc = newDoc.getMap('spaces').get('space:hello-world') as Doc;
-    const data = (subDoc.toJSON() as any).blocks;
-    Object.keys(data).forEach(id => {
-      if (id === 'xyWNqindHH') {
-        return;
-      }
-      if (
-        blocks[id]['sys:flavour'] === 'affine:surface' &&
-        !blocks[id]['prop:elements']
-      ) {
-        blocks[id]['prop:elements'] = data[id]['prop:elements'];
-      }
-      expect(data[id]).toEqual(blocks[id]);
-    });
-  });
-
-  test('Test fixture should be set correctly', () => {
-    const meta = doc.getMap('space:meta');
-    const versions = meta.get('versions') as YMap<unknown>;
-    expect(versions.get('affine:code')).toBeTypeOf('number');
-  });
-
-  test('Meta data should be migrated correctly', () => {
-    const originalMeta = doc.getMap('space:meta');
-    const originalVersions = originalMeta.get('versions') as YMap<unknown>;
-
-    const meta = migratedDoc.getMap('meta');
-    const blockVersions = meta.get('blockVersions') as YMap<unknown>;
-
-    expect(meta.get('workspaceVersion')).toBe(1);
-    expect(blockVersions.get('affine:code')).toBe(
-      originalVersions.get('affine:code')
-    );
-    expect((meta.get('pages') as YArray<unknown>).length).toBe(
-      (originalMeta.get('pages') as YArray<unknown>).length
-    );
-
-    expect(blockVersions.get('affine:embed')).toBeUndefined();
-    expect(blockVersions.get('affine:image')).toBe(
-      originalVersions.get('affine:embed')
-    );
-
-    expect(blockVersions.get('affine:frame')).toBeUndefined();
-    expect(blockVersions.get('affine:note')).toBe(
-      originalVersions.get('affine:frame')
-    );
-  });
-});

packages/env/src/blocksuite/index.ts

@@ -12,5 +12,3 @@ export async function initEmptyPage(page: Page, title?: string) {
   const noteBlockId = page.addBlock('affine:note', {}, pageBlockId);
   page.addBlock('affine:paragraph', {}, noteBlockId);
 }
-
-export * from './subdoc-migration.js';

packages/env/src/blocksuite/subdoc-migration.ts

@@ -1,267 +0,0 @@
-import type { Schema } from '@blocksuite/store';
-import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs';
-
-type XYWH = [number, number, number, number];
-
-function deserializeXYWH(xywh: string): XYWH {
-  return JSON.parse(xywh) as XYWH;
-}
-
-function migrateDatabase(data: YMap<unknown>) {
-  data.delete('prop:mode');
-  data.set('prop:views', new YArray());
-  const columns = (data.get('prop:columns') as YArray<unknown>).toJSON() as {
-    id: string;
-    name: string;
-    hide: boolean;
-    type: string;
-    width: number;
-    selection?: unknown[];
-  }[];
-  const views = [
-    {
-      id: 'default',
-      name: 'Table',
-      columns: columns.map(col => ({
-        id: col.id,
-        width: col.width,
-        hide: col.hide,
-      })),
-      filter: { type: 'group', op: 'and', conditions: [] },
-      mode: 'table',
-    },
-  ];
-  const cells = (data.get('prop:cells') as YMap<unknown>).toJSON() as Record<
-    string,
-    Record<
-      string,
-      {
-        id: string;
-        value: unknown;
-      }
-    >
-  >;
-  const convertColumn = (
-    id: string,
-    update: (cell: { id: string; value: unknown }) => void
-  ) => {
-    Object.values(cells).forEach(row => {
-      if (row[id] != null) {
-        update(row[id]);
-      }
-    });
-  };
-  const newColumns = columns.map(v => {
-    let data: Record<string, unknown> = {};
-    if (v.type === 'select' || v.type === 'multi-select') {
-      data = { options: v.selection };
-      if (v.type === 'select') {
-        convertColumn(v.id, cell => {
-          if (Array.isArray(cell.value)) {
-            cell.value = cell.value[0]?.id;
-          }
-        });
-      } else {
-        convertColumn(v.id, cell => {
-          if (Array.isArray(cell.value)) {
-            cell.value = cell.value.map(v => v.id);
-          }
-        });
-      }
-    }
-    if (v.type === 'number') {
-      convertColumn(v.id, cell => {
-        if (typeof cell.value === 'string') {
-          cell.value = Number.parseFloat(cell.value.toString());
-        }
-      });
-    }
-    return {
-      id: v.id,
-      type: v.type,
-      name: v.name,
-      data,
-    };
-  });
-  data.set('prop:columns', newColumns);
-  data.set('prop:views', views);
-  data.set('prop:cells', cells);
-}
-
-function runBlockMigration(
-  flavour: string,
-  data: YMap<unknown>,
-  version: number
-) {
-  if (flavour === 'affine:frame') {
-    data.set('sys:flavour', 'affine:note');
-    return;
-  }
-  if (flavour === 'affine:surface' && version <= 3) {
-    if (data.has('elements')) {
-      const elements = data.get('elements') as YMap<unknown>;
-      migrateSurface(elements);
-      data.set('prop:elements', elements.clone());
-      data.delete('elements');
-    } else {
-      data.set('prop:elements', new YMap());
-    }
-  }
-  if (flavour === 'affine:embed') {
-    data.set('sys:flavour', 'affine:image');
-    data.delete('prop:type');
-  }
-  if (flavour === 'affine:database' && version < 2) {
-    migrateDatabase(data);
-  }
-}
-
-function migrateSurface(data: YMap<unknown>) {
-  for (const [, value] of <IterableIterator<[string, YMap<unknown>]>>(
-    data.entries()
-  )) {
-    if (value.get('type') === 'connector') {
-      migrateSurfaceConnector(value);
-    }
-  }
-}
-
-function migrateSurfaceConnector(data: YMap<any>) {
-  let id = data.get('startElement')?.id;
-  const controllers = data.get('controllers');
-  const length = controllers.length;
-  const xywh = deserializeXYWH(data.get('xywh'));
-  if (id) {
-    data.set('source', { id });
-  } else {
-    data.set('source', {
-      position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]],
-    });
-  }
-
-  id = data.get('endElement')?.id;
-  if (id) {
-    data.set('target', { id });
-  } else {
-    data.set('target', {
-      position: [
-        controllers[length - 1].x + xywh[0],
-        controllers[length - 1].y + xywh[1],
-      ],
-    });
-  }
-
-  const width = data.get('lineWidth') ?? 4;
-  data.set('strokeWidth', width);
-  const color = data.get('color');
-  data.set('stroke', color);
-
-  data.delete('startElement');
-  data.delete('endElement');
-  data.delete('controllers');
-  data.delete('lineWidth');
-  data.delete('color');
-  data.delete('xywh');
-}
-
-function updateBlockVersions(versions: YMap<number>) {
-  const frameVersion = versions.get('affine:frame');
-  if (frameVersion !== undefined) {
-    versions.set('affine:note', frameVersion);
-    versions.delete('affine:frame');
-  }
-  const embedVersion = versions.get('affine:embed');
-  if (embedVersion !== undefined) {
-    versions.set('affine:image', embedVersion);
-    versions.delete('affine:embed');
-  }
-  const databaseVersion = versions.get('affine:database');
-  if (databaseVersion !== undefined && databaseVersion < 2) {
-    versions.set('affine:database', 2);
-  }
-}
-
-function migrateMeta(oldDoc: YDoc, newDoc: YDoc) {
-  const originalMeta = oldDoc.getMap('space:meta');
-  const originalVersions = originalMeta.get('versions') as YMap<number>;
-  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
-  const meta = newDoc.getMap('meta');
-  const pages = new YArray();
-  const blockVersions = originalVersions.clone();
-
-  meta.set('workspaceVersion', 1);
-  meta.set('blockVersions', blockVersions);
-  meta.set('pages', pages);
-  meta.set('name', originalMeta.get('name') as string);
-
-  updateBlockVersions(blockVersions);
-  const mapList = originalPages.map(page => {
-    const map = new YMap();
-    Array.from(page.entries())
-      .filter(([key]) => key !== 'subpageIds')
-      .forEach(([key, value]) => {
-        map.set(key, value);
-      });
-    return map;
-  });
-  pages.push(mapList);
-}
-
-function migrateBlocks(oldDoc: YDoc, newDoc: YDoc) {
-  const spaces = newDoc.getMap('spaces');
-  const originalMeta = oldDoc.getMap('space:meta');
-  const originalVersions = originalMeta.get('versions') as YMap<number>;
-  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
-  originalPages.forEach(page => {
-    const id = page.get('id') as string;
-    const spaceId = id.startsWith('space:') ? id : `space:${id}`;
-    const originalBlocks = oldDoc.getMap(spaceId) as YMap<unknown>;
-    const subdoc = new YDoc();
-    spaces.set(spaceId, subdoc);
-    const blocks = subdoc.getMap('blocks');
-    Array.from(originalBlocks.entries()).forEach(([key, value]) => {
-      const blockData = value.clone();
-      blocks.set(key, blockData);
-      const flavour = blockData.get('sys:flavour') as string;
-      const version = originalVersions.get(flavour);
-      if (version !== undefined) {
-        runBlockMigration(flavour, blockData, version);
-      }
-    });
-  });
-}
-
-export function migrateToSubdoc(doc: YDoc): YDoc {
-  const needMigration = Array.from(doc.getMap('space:meta').keys()).length > 0;
-  if (!needMigration) {
-    return doc;
-  }
-  const output = new YDoc();
-  migrateMeta(doc, output);
-  migrateBlocks(doc, output);
-  return output;
-}
-
-export async function migrateDatabaseBlockTo3(rootDoc: YDoc, schema: Schema) {
-  const spaces = rootDoc.getMap('spaces') as YMap<any>;
-  spaces.forEach(space => {
-    schema.upgradePage(
-      {
-        'affine:note': 1,
-        'affine:bookmark': 1,
-        'affine:database': 2,
-        'affine:divider': 1,
-        'affine:image': 1,
-        'affine:list': 1,
-        'affine:code': 1,
-        'affine:page': 2,
-        'affine:paragraph': 1,
-        'affine:surface': 3,
-      },
-      space
-    );
-  });
-  const meta = rootDoc.getMap('meta') as YMap<unknown>;
-  const versions = meta.get('blockVersions') as YMap<number>;
-  versions.set('affine:database', 3);
-}

packages/env/src/workspace.ts

@@ -10,11 +10,6 @@ import type { PropsWithChildren, ReactNode } from 'react';
 
 import type { Collection } from './filter.js';
 
-export enum WorkspaceVersion {
-  SubDoc = 2,
-  DatabaseV3 = 3,
-}
-
 export enum WorkspaceSubPath {
   ALL = 'all',
   SETTING = 'setting',

packages/graphql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/graphql",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Autogenerated GraphQL client for affine.pro",
   "license": "MPL-2.0",
   "type": "module",

packages/hooks/package.json

@@ -53,5 +53,5 @@
       "optional": true
     }
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/i18n/package.json

@@ -37,5 +37,5 @@
     "ts-node": "^10.9.1",
     "typescript": "^5.1.6"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/infra/package.json

@@ -63,7 +63,8 @@
     "electron": "link:../../apps/electron/node_modules/electron",
     "react": "^18.2.0",
     "vite": "^4.4.9",
-    "vite-plugin-dts": "3.5.2"
+    "vite-plugin-dts": "3.5.2",
+    "yjs": "^13.6.7"
   },
   "peerDependencies": {
     "@affine/templates": "*",
@@ -71,7 +72,8 @@
     "@blocksuite/lit": "*",
     "async-call-rpc": "*",
     "electron": "*",
-    "react": "*"
+    "react": "*",
+    "yjs": "^13"
   },
   "peerDependenciesMeta": {
     "@affine/templates": {
@@ -91,7 +93,10 @@
     },
     "react": {
       "optional": true
+    },
+    "yjs": {
+      "optional": true
     }
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/infra/src/blocksuite/index.ts

@@ -1,5 +1,7 @@
 import type { PageMeta, Workspace } from '@blocksuite/store';
+import { createIndexeddbStorage } from '@blocksuite/store';
 import type { createStore, WritableAtom } from 'jotai/vanilla';
+import { Array as YArray, Doc as YDoc, Map as YMap } from 'yjs';
 
 export async function buildShowcaseWorkspace(
   workspace: Workspace,
@@ -26,7 +28,9 @@ export async function buildShowcaseWorkspace(
     'affine:bookmark': 1,
     'affine:database': 2,
   };
-  workspace.doc.getMap('meta').set('blockVersions', showcaseWorkspaceVersions);
+  workspace.doc
+    .getMap('meta')
+    .set('blockVersions', new YMap(Object.entries(showcaseWorkspaceVersions)));
   const prototypes = {
     tags: {
       options: [
@@ -188,3 +192,380 @@ export async function buildShowcaseWorkspace(
     })
   );
 }
+
+import { applyUpdate, encodeStateAsUpdate } from 'yjs';
+
+const migrationOrigin = 'affine-migration';
+
+import type { Schema } from '@blocksuite/store';
+
+type XYWH = [number, number, number, number];
+
+function deserializeXYWH(xywh: string): XYWH {
+  return JSON.parse(xywh) as XYWH;
+}
+
+function migrateDatabase(data: YMap<unknown>) {
+  data.delete('prop:mode');
+  data.set('prop:views', new YArray());
+  const columns = (data.get('prop:columns') as YArray<unknown>).toJSON() as {
+    id: string;
+    name: string;
+    hide: boolean;
+    type: string;
+    width: number;
+    selection?: unknown[];
+  }[];
+  const views = [
+    {
+      id: 'default',
+      name: 'Table',
+      columns: columns.map(col => ({
+        id: col.id,
+        width: col.width,
+        hide: col.hide,
+      })),
+      filter: { type: 'group', op: 'and', conditions: [] },
+      mode: 'table',
+    },
+  ];
+  const cells = (data.get('prop:cells') as YMap<unknown>).toJSON() as Record<
+    string,
+    Record<
+      string,
+      {
+        id: string;
+        value: unknown;
+      }
+    >
+  >;
+  const convertColumn = (
+    id: string,
+    update: (cell: { id: string; value: unknown }) => void
+  ) => {
+    Object.values(cells).forEach(row => {
+      if (row[id] != null) {
+        update(row[id]);
+      }
+    });
+  };
+  const newColumns = columns.map(v => {
+    let data: Record<string, unknown> = {};
+    if (v.type === 'select' || v.type === 'multi-select') {
+      data = { options: v.selection };
+      if (v.type === 'select') {
+        convertColumn(v.id, cell => {
+          if (Array.isArray(cell.value)) {
+            cell.value = cell.value[0]?.id;
+          }
+        });
+      } else {
+        convertColumn(v.id, cell => {
+          if (Array.isArray(cell.value)) {
+            cell.value = cell.value.map(v => v.id);
+          }
+        });
+      }
+    }
+    if (v.type === 'number') {
+      convertColumn(v.id, cell => {
+        if (typeof cell.value === 'string') {
+          cell.value = Number.parseFloat(cell.value.toString());
+        }
+      });
+    }
+    return {
+      id: v.id,
+      type: v.type,
+      name: v.name,
+      data,
+    };
+  });
+  data.set('prop:columns', newColumns);
+  data.set('prop:views', views);
+  data.set('prop:cells', cells);
+}
+
+function runBlockMigration(
+  flavour: string,
+  data: YMap<unknown>,
+  version: number
+) {
+  if (flavour === 'affine:frame') {
+    data.set('sys:flavour', 'affine:note');
+    return;
+  }
+  if (flavour === 'affine:surface' && version <= 3) {
+    if (data.has('elements')) {
+      const elements = data.get('elements') as YMap<unknown>;
+      migrateSurface(elements);
+      data.set('prop:elements', elements.clone());
+      data.delete('elements');
+    } else {
+      data.set('prop:elements', new YMap());
+    }
+  }
+  if (flavour === 'affine:embed') {
+    data.set('sys:flavour', 'affine:image');
+    data.delete('prop:type');
+  }
+  if (flavour === 'affine:database' && version < 2) {
+    migrateDatabase(data);
+  }
+}
+
+function migrateSurface(data: YMap<unknown>) {
+  for (const [, value] of <IterableIterator<[string, YMap<unknown>]>>(
+    data.entries()
+  )) {
+    if (value.get('type') === 'connector') {
+      migrateSurfaceConnector(value);
+    }
+  }
+}
+
+function migrateSurfaceConnector(data: YMap<any>) {
+  let id = data.get('startElement')?.id;
+  const controllers = data.get('controllers');
+  const length = controllers.length;
+  const xywh = deserializeXYWH(data.get('xywh'));
+  if (id) {
+    data.set('source', { id });
+  } else {
+    data.set('source', {
+      position: [controllers[0].x + xywh[0], controllers[0].y + xywh[1]],
+    });
+  }
+
+  id = data.get('endElement')?.id;
+  if (id) {
+    data.set('target', { id });
+  } else {
+    data.set('target', {
+      position: [
+        controllers[length - 1].x + xywh[0],
+        controllers[length - 1].y + xywh[1],
+      ],
+    });
+  }
+
+  const width = data.get('lineWidth') ?? 4;
+  data.set('strokeWidth', width);
+  const color = data.get('color');
+  data.set('stroke', color);
+
+  data.delete('startElement');
+  data.delete('endElement');
+  data.delete('controllers');
+  data.delete('lineWidth');
+  data.delete('color');
+  data.delete('xywh');
+}
+
+function updateBlockVersions(versions: YMap<number>) {
+  const frameVersion = versions.get('affine:frame');
+  if (frameVersion !== undefined) {
+    versions.set('affine:note', frameVersion);
+    versions.delete('affine:frame');
+  }
+  const embedVersion = versions.get('affine:embed');
+  if (embedVersion !== undefined) {
+    versions.set('affine:image', embedVersion);
+    versions.delete('affine:embed');
+  }
+  const databaseVersion = versions.get('affine:database');
+  if (databaseVersion !== undefined && databaseVersion < 2) {
+    versions.set('affine:database', 2);
+  }
+}
+
+function migrateMeta(oldDoc: YDoc, newDoc: YDoc) {
+  const originalMeta = oldDoc.getMap('space:meta');
+  const originalVersions = originalMeta.get('versions') as YMap<number>;
+  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
+  const meta = newDoc.getMap('meta');
+  const pages = new YArray();
+  const blockVersions = originalVersions.clone();
+
+  meta.set('workspaceVersion', 1);
+  meta.set('blockVersions', blockVersions);
+  meta.set('pages', pages);
+  meta.set('name', originalMeta.get('name') as string);
+
+  updateBlockVersions(blockVersions);
+  const mapList = originalPages.map(page => {
+    const map = new YMap();
+    Array.from(page.entries())
+      .filter(([key]) => key !== 'subpageIds')
+      .forEach(([key, value]) => {
+        map.set(key, value);
+      });
+    return map;
+  });
+  pages.push(mapList);
+}
+
+function migrateBlocks(oldDoc: YDoc, newDoc: YDoc) {
+  const spaces = newDoc.getMap('spaces');
+  const originalMeta = oldDoc.getMap('space:meta');
+  const originalVersions = originalMeta.get('versions') as YMap<number>;
+  const originalPages = originalMeta.get('pages') as YArray<YMap<unknown>>;
+  originalPages.forEach(page => {
+    const id = page.get('id') as string;
+    const spaceId = id.startsWith('space:') ? id : `space:${id}`;
+    const originalBlocks = oldDoc.getMap(spaceId) as YMap<unknown>;
+    const subdoc = new YDoc();
+    spaces.set(spaceId, subdoc);
+    const blocks = subdoc.getMap('blocks');
+    Array.from(originalBlocks.entries()).forEach(([key, value]) => {
+      const blockData = value.clone();
+      blocks.set(key, blockData);
+      const flavour = blockData.get('sys:flavour') as string;
+      const version = originalVersions.get(flavour);
+      if (version !== undefined) {
+        runBlockMigration(flavour, blockData, version);
+      }
+    });
+  });
+}
+
+export function migrateToSubdoc(oldDoc: YDoc): YDoc {
+  const needMigration =
+    Array.from(oldDoc.getMap('space:meta').keys()).length > 0;
+  if (!needMigration) {
+    return oldDoc;
+  }
+  const newDoc = new YDoc();
+  migrateMeta(oldDoc, newDoc);
+  migrateBlocks(oldDoc, newDoc);
+  return newDoc;
+}
+
+export async function migrateDatabaseBlockTo3(rootDoc: YDoc, schema: Schema) {
+  const spaces = rootDoc.getMap('spaces') as YMap<any>;
+  spaces.forEach(space => {
+    schema.upgradePage(
+      {
+        'affine:note': 1,
+        'affine:bookmark': 1,
+        'affine:database': 2,
+        'affine:divider': 1,
+        'affine:image': 1,
+        'affine:list': 1,
+        'affine:code': 1,
+        'affine:page': 2,
+        'affine:paragraph': 1,
+        'affine:surface': 3,
+      },
+      space
+    );
+  });
+  const meta = rootDoc.getMap('meta') as YMap<unknown>;
+  const versions = meta.get('blockVersions') as YMap<number>;
+  versions.set('affine:database', 3);
+}
+
+export type UpgradeOptions = {
+  getCurrentRootDoc: () => Promise<YDoc>;
+  createWorkspace: () => Promise<Workspace>;
+  getSchema: () => Schema;
+};
+
+const upgradeV1ToV2 = async (options: UpgradeOptions) => {
+  const oldDoc = await options.getCurrentRootDoc();
+  const newDoc = migrateToSubdoc(oldDoc);
+  const newWorkspace = await options.createWorkspace();
+  applyUpdate(newWorkspace.doc, encodeStateAsUpdate(newDoc), migrationOrigin);
+  newDoc.getSubdocs().forEach(subdoc => {
+    newWorkspace.doc.getSubdocs().forEach(newDoc => {
+      if (subdoc.guid === newDoc.guid) {
+        applyUpdate(newDoc, encodeStateAsUpdate(subdoc), migrationOrigin);
+      }
+    });
+  });
+  return newWorkspace;
+};
+
+const upgradeV2ToV3 = async (options: UpgradeOptions): Promise<boolean> => {
+  const rootDoc = await options.getCurrentRootDoc();
+  const spaces = rootDoc.getMap('spaces') as YMap<any>;
+  const meta = rootDoc.getMap('meta') as YMap<unknown>;
+  const versions = meta.get('blockVersions') as YMap<number>;
+  if ('affine:database' in versions) {
+    if (versions['affine:database'] === 3) {
+      return false;
+    }
+  } else if (versions.get('affine:database') === 3) {
+    return false;
+  }
+  const schema = options.getSchema();
+  spaces.forEach(space => {
+    schema.upgradePage(
+      {
+        'affine:note': 1,
+        'affine:bookmark': 1,
+        'affine:database': 2,
+        'affine:divider': 1,
+        'affine:image': 1,
+        'affine:list': 1,
+        'affine:code': 1,
+        'affine:page': 2,
+        'affine:paragraph': 1,
+        'affine:surface': 3,
+      },
+      space
+    );
+  });
+  if ('affine:database' in versions) {
+    versions['affine:database'] = 3;
+    meta.set('blockVersions', new YMap(Object.entries(versions)));
+  } else {
+    versions.set('affine:database', 3);
+  }
+  return true;
+};
+
+export enum WorkspaceVersion {
+  // v1 is treated as undefined
+  SubDoc = 2,
+  DatabaseV3 = 3,
+}
+
+/**
+ * If returns false, it means no migration is needed.
+ * If returns true, it means migration is done.
+ * If returns Workspace, it means new workspace is created,
+ *  and the old workspace should be deleted.
+ */
+export async function migrateWorkspace(
+  currentVersion: WorkspaceVersion | undefined,
+  options: UpgradeOptions
+): Promise<Workspace | boolean> {
+  if (currentVersion === undefined) {
+    const workspace = await upgradeV1ToV2(options);
+    await upgradeV2ToV3({
+      ...options,
+      getCurrentRootDoc: () => Promise.resolve(workspace.doc),
+    });
+    return workspace;
+  }
+  if (currentVersion === WorkspaceVersion.SubDoc) {
+    return upgradeV2ToV3(options);
+  } else {
+    return false;
+  }
+}
+
+export async function migrateLocalBlobStorage(from: string, to: string) {
+  const fromStorage = createIndexeddbStorage(from);
+  const toStorage = createIndexeddbStorage(to);
+  const keys = await fromStorage.crud.list();
+  for (const key of keys) {
+    const value = await fromStorage.crud.get(key);
+    if (!value) {
+      console.warn('cannot find blob:', key);
+      continue;
+    }
+    await toStorage.crud.set(key, value);
+  }
+}

packages/infra/vite.config.ts

@@ -34,6 +34,7 @@ export default defineConfig({
         'rxjs',
         'zod',
         'react',
+        'yjs',
         /^jotai/,
         /^@blocksuite/,
         /^@affine\/templates/,

packages/jotai/package.json

@@ -23,5 +23,5 @@
     "@blocksuite/store": "*",
     "lottie-web": "*"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/native/__tests__/db.spec.mts

@@ -0,0 +1,15 @@
+import assert from 'node:assert';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+import { SqliteConnection, ValidationResult } from '../index';
+
+test('db', { concurrency: false }, async t => {
+  await t.test('validate', async () => {
+    const path = fileURLToPath(
+      new URL('./fixtures/test01.affine', import.meta.url)
+    );
+    const result = await SqliteConnection.validate(path);
+    assert.equal(result, ValidationResult.MissingVersionColumn);
+  });
+});

packages/native/index.d.ts

@@ -41,8 +41,9 @@ export interface InsertRow {
 export enum ValidationResult {
   MissingTables = 0,
   MissingDocIdColumn = 1,
-  GeneralError = 2,
-  Valid = 3,
+  MissingVersionColumn = 2,
+  GeneralError = 3,
+  Valid = 4,
 }
 export class Subscription {
   toString(): string;
@@ -75,6 +76,8 @@ export class SqliteConnection {
     docId: string | undefined | null,
     updates: Array<InsertRow>
   ): Promise<void>;
+  initVersion(): Promise<void>;
+  setVersion(version: number): Promise<void>;
   close(): Promise<void>;
   get isClose(): boolean;
   static validate(path: string): Promise<ValidationResult>;

packages/native/package.json

@@ -38,5 +38,5 @@
     "test": "cross-env TS_NODE_TRANSPILE_ONLY=1 TS_NODE_PROJECT=./tsconfig.json node --test --loader ts-node/esm --experimental-specifier-resolution=node ./__tests__/**/*.mts",
     "version": "napi version"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/native/schema/src/lib.rs

@@ -11,4 +11,9 @@ CREATE TABLE IF NOT EXISTS "blobs" (
   key TEXT PRIMARY KEY NOT NULL,
   data BLOB NOT NULL,
   timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
-);"#;
+);
+CREATE TABLE IF NOT EXISTS "version_info" (
+  version NUMBER NOT NULL,
+  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+)
+"#;

packages/native/src/sqlite/mod.rs

@@ -7,6 +7,9 @@ use sqlx::{
   Pool, Row,
 };
 
+// latest version
+const LATEST_VERSION: i32 = 3;
+
 #[napi(object)]
 pub struct BlobRow {
   pub key: String,
@@ -38,6 +41,7 @@ pub struct SqliteConnection {
 pub enum ValidationResult {
   MissingTables,
   MissingDocIdColumn,
+  MissingVersionColumn,
   GeneralError,
   Valid,
 }
@@ -228,6 +232,39 @@ impl SqliteConnection {
     Ok(())
   }
 
+  #[napi]
+  pub async fn init_version(&self) -> napi::Result<()> {
+    // create version_info table
+    sqlx::query!(
+      "CREATE TABLE IF NOT EXISTS version_info (
+        version NUMBER NOT NULL,
+        timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+      )"
+    )
+    .execute(&self.pool)
+    .await
+    .map_err(anyhow::Error::from)?;
+    // `3` is the first version that has version_info table,
+    //  do not modify the version number.
+    sqlx::query!("INSERT INTO version_info (version) VALUES (3)")
+      .execute(&self.pool)
+      .await
+      .map_err(anyhow::Error::from)?;
+    Ok(())
+  }
+
+  #[napi]
+  pub async fn set_version(&self, version: i32) -> napi::Result<()> {
+    if version > LATEST_VERSION {
+      return Err(anyhow::Error::msg("Version is too new").into());
+    }
+    sqlx::query!("UPDATE version_info SET version = ?", version)
+      .execute(&self.pool)
+      .await
+      .map_err(anyhow::Error::from)?;
+    Ok(())
+  }
+
   #[napi]
   pub async fn close(&self) {
     self.pool.close().await;
@@ -261,6 +298,18 @@ impl SqliteConnection {
       Err(_) => return ValidationResult::GeneralError,
     };
 
+    let tables_res = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
+      .fetch_all(&pool)
+      .await;
+
+    let version_exist = match tables_res {
+      Ok(res) => {
+        let names: Vec<String> = res.iter().map(|row| row.get(0)).collect();
+        names.contains(&"version_info".to_string())
+      }
+      Err(_) => return ValidationResult::GeneralError,
+    };
+
     let columns_res = sqlx::query("PRAGMA table_info(updates)")
       .fetch_all(&pool)
       .await;
@@ -277,6 +326,8 @@ impl SqliteConnection {
       ValidationResult::MissingTables
     } else if !doc_id_exist {
       ValidationResult::MissingDocIdColumn
+    } else if !version_exist {
+      ValidationResult::MissingVersionColumn
     } else {
       ValidationResult::Valid
     }

packages/plugin-cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/plugin-cli",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "bin": {
     "af": "./src/af.mjs"
   },

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/sdk",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "type": "module",
   "scripts": {
     "build": "vite build",

packages/storage/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/storage",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "engines": {
     "node": ">= 10.16.0 < 11 || >= 11.8.0"
   },

packages/templates/package.json

@@ -7,5 +7,5 @@
     "./v1/*.json": "./v1/*.json",
     "./preloading.json": "./preloading.json"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/workers/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@affine/workers",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "private": true,
   "scripts": {
     "dev": "wrangler dev"

packages/workspace/package.json

@@ -34,5 +34,5 @@
     "@types/ws": "^8.5.5",
     "ws": "^8.13.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

packages/workspace/src/atom.ts

@@ -1,7 +1,8 @@
 import type { WorkspaceAdapter } from '@affine/env/workspace';
-import { WorkspaceFlavour, WorkspaceVersion } from '@affine/env/workspace';
+import { WorkspaceFlavour } from '@affine/env/workspace';
 import type { BlockHub } from '@blocksuite/blocks';
 import { assertExists } from '@blocksuite/global/utils';
+import { WorkspaceVersion } from '@toeverything/infra/blocksuite';
 import { atom } from 'jotai';
 import { z } from 'zod';
 

packages/workspace/src/migration/index.ts

@@ -1,51 +0,0 @@
-import { migrateToSubdoc } from '@affine/env/blocksuite';
-import type { LocalWorkspace } from '@affine/env/workspace';
-import { WorkspaceFlavour } from '@affine/env/workspace';
-import { getOrCreateWorkspace } from '@affine/workspace/manager';
-import { nanoid, Workspace } from '@blocksuite/store';
-import { createIndexeddbStorage } from '@blocksuite/store';
-const Y = Workspace.Y;
-
-export function upgradeV1ToV2(oldWorkspace: LocalWorkspace): LocalWorkspace {
-  const oldDoc = oldWorkspace.blockSuiteWorkspace.doc;
-  const newDoc = migrateToSubdoc(oldDoc);
-  if (newDoc === oldDoc) {
-    console.warn('do not need update');
-    return oldWorkspace;
-  } else {
-    const id = nanoid();
-    const newBlockSuiteWorkspace = getOrCreateWorkspace(
-      id,
-      WorkspaceFlavour.LOCAL
-    );
-    Y.applyUpdate(newBlockSuiteWorkspace.doc, Y.encodeStateAsUpdate(newDoc));
-    newDoc.getSubdocs().forEach(subdoc => {
-      newBlockSuiteWorkspace.doc.getSubdocs().forEach(newDoc => {
-        if (subdoc.guid === newDoc.guid) {
-          Y.applyUpdate(newDoc, Y.encodeStateAsUpdate(subdoc));
-        }
-      });
-    });
-    console.log('migration result', newBlockSuiteWorkspace.doc.toJSON());
-
-    return {
-      blockSuiteWorkspace: newBlockSuiteWorkspace,
-      flavour: WorkspaceFlavour.LOCAL,
-      id,
-    };
-  }
-}
-
-export async function migrateLocalBlobStorage(from: string, to: string) {
-  const fromStorage = createIndexeddbStorage(from);
-  const toStorage = createIndexeddbStorage(to);
-  const keys = await fromStorage.crud.list();
-  for (const key of keys) {
-    const value = await fromStorage.crud.get(key);
-    if (!value) {
-      console.warn('cannot find blob:', key);
-      continue;
-    }
-    await toStorage.crud.set(key, value);
-  }
-}

packages/y-indexeddb/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@toeverything/y-indexeddb",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "IndexedDB database adapter for Yjs",
   "repository": "toeverything/AFFiNE",
   "author": "toeverything",

packages/y-provider/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/y-provider",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Yjs provider utilities for AFFiNE",
   "main": "./src/index.ts",
   "module": "./src/index.ts",

plugins/bookmark/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/bookmark-plugin",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Bookmark Plugin",
   "affinePlugin": {
     "release": true,

plugins/copilot/package.json

@@ -35,5 +35,5 @@
     "react": "*",
     "react-dom": "*"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

plugins/hello-world/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "private": true,
   "description": "Hello world plugin",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "scripts": {
     "dev": "af dev",
     "build": "af build"

plugins/image-preview/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine/image-preview-plugin",
   "type": "module",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "description": "Image preview plugin",
   "affinePlugin": {
     "release": true,

plugins/outline/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "private": true,
   "description": "Outline plugin",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "scripts": {
     "dev": "af dev",
     "build": "af build"

plugins/vue-hello-world/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "private": true,
   "description": "Vue hello world plugin",
-  "version": "0.8.0",
+  "version": "0.8.2",
   "scripts": {
     "dev": "af dev",
     "build": "af build"

tests/affine-legacy/0.7.0-canary.18/package.json

@@ -19,5 +19,5 @@
     "http-proxy-middleware": "^3.0.0-beta.1",
     "serve": "^14.2.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

tests/affine-legacy/0.8.0-canary.7/package.json

@@ -18,5 +18,5 @@
     "http-proxy-middleware": "^3.0.0-beta.1",
     "serve": "^14.2.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

tests/affine-local/package.json

@@ -9,5 +9,5 @@
     "@affine-test/kit": "workspace:*",
     "@playwright/test": "^1.37.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

tests/affine-plugin/package.json

@@ -9,5 +9,5 @@
     "@affine-test/kit": "workspace:*",
     "@playwright/test": "^1.37.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

tests/affine-prototype/package.json

@@ -9,5 +9,5 @@
     "@affine-test/kit": "workspace:*",
     "@playwright/test": "^1.37.0"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

tests/fixtures/package.json

@@ -3,5 +3,5 @@
   "exports": {
     "./*": "./*"
   },
-  "version": "0.8.0"
+  "version": "0.8.2"
 }

tests/kit/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@affine-test/kit",
   "private": true,
-  "version": "0.8.0",
+  "version": "0.8.2",
   "exports": {
     "./playwright": "./playwright.ts",
     "./utils/*": "./utils/*.ts"

yarn.lock

@@ -11790,6 +11790,7 @@ __metadata:
     react: ^18.2.0
     vite: ^4.4.9
     vite-plugin-dts: 3.5.2
+    yjs: ^13.6.7
     zod: ^3.22.1
   peerDependencies:
     "@affine/templates": "*"
@@ -11798,6 +11799,7 @@ __metadata:
     async-call-rpc: "*"
     electron: "*"
     react: "*"
+    yjs: ^13
   peerDependenciesMeta:
     "@affine/templates":
       optional: true
@@ -11811,6 +11813,8 @@ __metadata:
       optional: true
     react:
       optional: true
+    yjs:
+      optional: true
   languageName: unknown
   linkType: soft