feat(infra): new doc sync engine (#6205)

https://github.com/toeverything/AFFiNE/blob/eyhn/feat/new-sync/packages/common/infra/src/workspace/engine/doc/README.md
2026-02-19 15:26:59 +08:00 · 2024-03-22 16:43:26 +00:00
parent 05c44db5a9
commit 34703a3b7d
85 changed files with 3248 additions and 2286 deletions
--- a/packages/frontend/core/src/atoms/sync-engine-status.ts
+++ b/packages/frontend/core/src/atoms/sync-engine-status.ts
@@ -1,4 +0,0 @@
-import type { SyncEngineStatus } from '@toeverything/infra';
-import { atom } from 'jotai';
-
-export const syncEngineStatusAtom = atom<SyncEngineStatus | null>(null);
--- a/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/export.tsx
+++ b/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/export.tsx
@@ -31,7 +31,7 @@ export const ExportPanel = ({
    setSaving(true);
    try {
      if (isOnline) {
-        await workspace.engine.sync.waitForSynced();
+        await workspace.engine.waitForSynced();
        await workspace.engine.blob.sync();
      }

--- a/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/index.tsx
+++ b/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/index.tsx
@@ -8,6 +8,8 @@ import { useWorkspace } from '@affine/core/hooks/use-workspace';
 import { useWorkspaceInfo } from '@affine/core/hooks/use-workspace-info';
 import { UNTITLED_WORKSPACE_NAME } from '@affine/env/constant';
 import { useAFFiNEI18N } from '@affine/i18n/hooks';
+import { ArrowRightSmallIcon } from '@blocksuite/icons';
+import { useCallback } from 'react';

 import { DeleteLeaveWorkspace } from './delete-leave-workspace';
 import { EnableCloudPanel } from './enable-cloud';
@@ -29,6 +31,17 @@ export const WorkspaceSettingDetail = (props: WorkspaceSettingDetailProps) => {

  const workspaceInfo = useWorkspaceInfo(workspaceMetadata);

+  const handleResetSyncStatus = useCallback(() => {
+    workspace?.engine.doc
+      .resetSyncStatus()
+      .then(() => {
+        window.location.reload();
+      })
+      .catch(err => {
+        console.error(err);
+      });
+  }, [workspace]);
+
  return (
    <>
      <SettingHeader
@@ -64,6 +77,19 @@ export const WorkspaceSettingDetail = (props: WorkspaceSettingDetailProps) => {
      )}
      <SettingWrapper>
        <DeleteLeaveWorkspace {...props} />
+        <SettingRow
+          name={
+            <span style={{ color: 'var(--affine-text-secondary-color)' }}>
+              {t['com.affine.resetSyncStatus.button']()}
+            </span>
+          }
+          desc={t['com.affine.resetSyncStatus.description']()}
+          style={{ cursor: 'pointer' }}
+          onClick={handleResetSyncStatus}
+          data-testid="reset-sync-status"
+        >
+          <ArrowRightSmallIcon />
+        </SettingRow>
      </SettingWrapper>
    </>
  );
--- a/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/profile.tsx
+++ b/packages/frontend/core/src/components/affine/setting-modal/workspace-setting/new-workspace-setting-detail/profile.tsx
@@ -5,13 +5,12 @@ import { Button } from '@affine/component/ui/button';
 import { Upload } from '@affine/core/components/pure/file-upload';
 import { useAsyncCallback } from '@affine/core/hooks/affine-async-hooks';
 import { useWorkspaceBlobObjectUrl } from '@affine/core/hooks/use-workspace-blob';
-import { useWorkspaceStatus } from '@affine/core/hooks/use-workspace-status';
 import { validateAndReduceImage } from '@affine/core/utils/reduce-image';
 import { UNTITLED_WORKSPACE_NAME } from '@affine/env/constant';
 import { useAFFiNEI18N } from '@affine/i18n/hooks';
 import { CameraIcon } from '@blocksuite/icons';
 import type { Workspace } from '@toeverything/infra';
-import { SyncPeerStep } from '@toeverything/infra';
+import { useLiveData } from '@toeverything/infra';
 import { useSetAtom } from 'jotai';
 import {
  type KeyboardEvent,
@@ -32,13 +31,7 @@ export const ProfilePanel = ({ isOwner, workspace }: ProfilePanelProps) => {
  const t = useAFFiNEI18N();
  const pushNotification = useSetAtom(pushNotificationAtom);

-  const workspaceIsLoading =
-    useWorkspaceStatus(
-      workspace,
-      status =>
-        !status.engine.sync.local ||
-        status.engine.sync.local?.step <= SyncPeerStep.LoadingRootDoc
-    ) ?? true;
+  const workspaceIsReady = useLiveData(workspace?.engine.rootDocState)?.ready;

  const [avatarBlob, setAvatarBlob] = useState<string | null>(null);
  const [name, setName] = useState('');
@@ -158,7 +151,7 @@ export const ProfilePanel = ({ isOwner, workspace }: ProfilePanelProps) => {
    [pushNotification, setWorkspaceAvatar]
  );

-  const canAdjustAvatar = !workspaceIsLoading && avatarUrl && isOwner;
+  const canAdjustAvatar = workspaceIsReady && avatarUrl && isOwner;

  return (
    <div className={style.profileWrapper}>
@@ -194,7 +187,7 @@ export const ProfilePanel = ({ isOwner, workspace }: ProfilePanelProps) => {
        <div className={style.label}>{t['Workspace Name']()}</div>
        <FlexWrapper alignItems="center" flexGrow="1">
          <Input
-            disabled={workspaceIsLoading || !isOwner}
+            disabled={!workspaceIsReady || !isOwner}
            value={input}
            style={{ width: 280, height: 32 }}
            data-testid="workspace-name-input"
--- a/packages/frontend/core/src/components/pure/cmdk/main.tsx
+++ b/packages/frontend/core/src/components/pure/cmdk/main.tsx
@@ -1,10 +1,9 @@
 import { Loading } from '@affine/component/ui/loading';
 import { formatDate } from '@affine/core/components/page-list';
-import { useSyncEngineStatus } from '@affine/core/hooks/affine/use-sync-engine-status';
+import { useDocEngineStatus } from '@affine/core/hooks/affine/use-doc-engine-status';
 import { useAsyncCallback } from '@affine/core/hooks/affine-async-hooks';
 import { useAFFiNEI18N } from '@affine/i18n/hooks';
 import type { DocMeta } from '@blocksuite/store';
-import { SyncEngineStep } from '@toeverything/infra';
 import type { CommandCategory } from '@toeverything/infra/command';
 import clsx from 'clsx';
 import { Command } from 'cmdk';
@@ -163,7 +162,7 @@ export const CMDKContainer = ({
  const [value, setValue] = useAtom(cmdkValueAtom);
  const isInEditor = pageMeta !== undefined;
  const [opening, setOpening] = useState(open);
-  const { syncEngineStatus, progress } = useSyncEngineStatus();
+  const { syncing, progress } = useDocEngineStatus();
  const inputRef = useRef<HTMLInputElement>(null);

  // fix list height animation on opening
@@ -205,8 +204,7 @@ export const CMDKContainer = ({
          inEditor: isInEditor,
        })}
      >
-        {!syncEngineStatus ||
-        syncEngineStatus.step === SyncEngineStep.Syncing ? (
+        {syncing ? (
          <Loading
            size={24}
            progress={progress ? Math.max(progress, 0.2) : undefined}
--- a/packages/frontend/core/src/components/pure/workspace-slider-bar/workspace-card/index.tsx
+++ b/packages/frontend/core/src/components/pure/workspace-slider-bar/workspace-card/index.tsx
@@ -3,8 +3,8 @@ import { Avatar } from '@affine/component/ui/avatar';
 import { Loading } from '@affine/component/ui/loading';
 import { Tooltip } from '@affine/component/ui/tooltip';
 import { openSettingModalAtom } from '@affine/core/atoms';
+import { useDocEngineStatus } from '@affine/core/hooks/affine/use-doc-engine-status';
 import { useIsWorkspaceOwner } from '@affine/core/hooks/affine/use-is-workspace-owner';
-import { useSyncEngineStatus } from '@affine/core/hooks/affine/use-sync-engine-status';
 import { useWorkspaceBlobObjectUrl } from '@affine/core/hooks/use-workspace-blob';
 import { useWorkspaceInfo } from '@affine/core/hooks/use-workspace-info';
 import { UNTITLED_WORKSPACE_NAME } from '@affine/env/constant';
@@ -17,7 +17,7 @@ import {
  NoNetworkIcon,
  UnsyncIcon,
 } from '@blocksuite/icons';
-import { SyncEngineStep, Workspace } from '@toeverything/infra';
+import { Workspace } from '@toeverything/infra';
 import { useService } from '@toeverything/infra/di';
 import { useSetAtom } from 'jotai';
 import { debounce } from 'lodash-es';
@@ -94,8 +94,7 @@ const useSyncEngineSyncProgress = () => {
  const t = useAFFiNEI18N();
  const isOnline = useSystemOnline();
  const pushNotification = useSetAtom(pushNotificationAtom);
-  const { syncEngineStatus, setSyncEngineStatus, progress } =
-    useSyncEngineStatus();
+  const { syncing, progress, retrying, errorMessage } = useDocEngineStatus();
  const [isOverCapacity, setIsOverCapacity] = useState(false);

  const currentWorkspace = useService(Workspace);
@@ -111,19 +110,6 @@ const useSyncEngineSyncProgress = () => {

  // debounce sync engine status
  useEffect(() => {
-    setSyncEngineStatus(currentWorkspace.engine.sync.status);
-    const disposable = currentWorkspace.engine.sync.onStatusChange.on(
-      debounce(
-        status => {
-          setSyncEngineStatus(status);
-        },
-        300,
-        {
-          maxWait: 500,
-          trailing: true,
-        }
-      )
-    );
    const disposableOverCapacity =
      currentWorkspace.engine.blob.onStatusChange.on(
        debounce(status => {
@@ -153,17 +139,9 @@ const useSyncEngineSyncProgress = () => {
        })
      );
    return () => {
-      disposable?.dispose();
      disposableOverCapacity?.dispose();
    };
-  }, [
-    currentWorkspace,
-    isOwner,
-    jumpToPricePlan,
-    pushNotification,
-    setSyncEngineStatus,
-    t,
-  ]);
+  }, [currentWorkspace, isOwner, jumpToPricePlan, pushNotification, t]);

  const content = useMemo(() => {
    // TODO: add i18n
@@ -176,21 +154,15 @@ const useSyncEngineSyncProgress = () => {
    if (!isOnline) {
      return 'Disconnected, please check your network connection';
    }
-    if (!syncEngineStatus || syncEngineStatus.step === SyncEngineStep.Syncing) {
+    if (syncing) {
      return (
        `Syncing with AFFiNE Cloud` +
        (progress ? ` (${Math.floor(progress * 100)}%)` : '')
      );
-    } else if (
-      syncEngineStatus &&
-      syncEngineStatus.step < SyncEngineStep.Syncing
-    ) {
-      return (
-        syncEngineStatus.error ||
-        'Disconnected, please check your network connection'
-      );
+    } else if (retrying && errorMessage) {
+      return `${errorMessage}, reconnecting.`;
    }
-    if (syncEngineStatus.retrying) {
+    if (retrying) {
      return 'Sync disconnected due to unexpected issues, reconnecting.';
    }
    if (isOverCapacity) {
@@ -199,29 +171,31 @@ const useSyncEngineSyncProgress = () => {
    return 'Synced with AFFiNE Cloud';
  }, [
    currentWorkspace.flavour,
+    errorMessage,
    isOnline,
    isOverCapacity,
    progress,
-    syncEngineStatus,
+    retrying,
+    syncing,
  ]);

  const CloudWorkspaceSyncStatus = useCallback(() => {
-    if (!syncEngineStatus || syncEngineStatus.step === SyncEngineStep.Syncing) {
+    if (syncing) {
      return SyncingWorkspaceStatus({
        progress: progress ? Math.max(progress, 0.2) : undefined,
      });
-    } else if (syncEngineStatus.retrying || isOverCapacity) {
+    } else if (retrying) {
      return UnSyncWorkspaceStatus();
    } else {
      return CloudWorkspaceStatus();
    }
-  }, [isOverCapacity, progress, syncEngineStatus]);
+  }, [progress, retrying, syncing]);

  return {
    message: content,
    icon:
      currentWorkspace.flavour === WorkspaceFlavour.AFFINE_CLOUD ? (
-        !isOnline || syncEngineStatus?.error ? (
+        !isOnline ? (
          <OfflineStatus />
        ) : (
          <CloudWorkspaceSyncStatus />
--- a/packages/frontend/core/src/hooks/affine/use-doc-engine-status.tsx
+++ b/packages/frontend/core/src/hooks/affine/use-doc-engine-status.tsx
@@ -0,0 +1,20 @@
+import { useLiveData, useService, Workspace } from '@toeverything/infra';
+import { useMemo } from 'react';
+
+export function useDocEngineStatus() {
+  const workspace = useService(Workspace);
+
+  const engineState = useLiveData(workspace.engine.docEngineState);
+
+  const progress =
+    (engineState.total - engineState.syncing) / engineState.total;
+
+  return useMemo(
+    () => ({
+      ...engineState,
+      progress,
+      syncing: engineState.syncing > 0,
+    }),
+    [engineState, progress]
+  );
+}
--- a/packages/frontend/core/src/hooks/affine/use-sync-engine-status.tsx
+++ b/packages/frontend/core/src/hooks/affine/use-sync-engine-status.tsx
@@ -1,35 +0,0 @@
-import { syncEngineStatusAtom } from '@affine/core/atoms/sync-engine-status';
-import { useAtom } from 'jotai';
-import { mean } from 'lodash-es';
-import { useMemo } from 'react';
-
-export function useSyncEngineStatus() {
-  const [syncEngineStatus, setSyncEngineStatus] = useAtom(syncEngineStatusAtom);
-
-  const progress = useMemo(() => {
-    if (!syncEngineStatus?.remotes || syncEngineStatus?.remotes.length === 0) {
-      return null;
-    }
-    return mean(
-      syncEngineStatus.remotes.map(peer => {
-        if (!peer) {
-          return 0;
-        }
-        const totalTask =
-          peer.totalDocs + peer.pendingPullUpdates + peer.pendingPushUpdates;
-        const doneTask = peer.loadedDocs;
-
-        return doneTask / totalTask;
-      })
-    );
-  }, [syncEngineStatus?.remotes]);
-
-  return useMemo(
-    () => ({
-      syncEngineStatus,
-      setSyncEngineStatus,
-      progress,
-    }),
-    [progress, setSyncEngineStatus, syncEngineStatus]
-  );
-}
--- a/packages/frontend/core/src/modules/infra-web/storage/index.ts
+++ b/packages/frontend/core/src/modules/infra-web/storage/index.ts
@@ -4,6 +4,17 @@ import { Observable } from 'rxjs';
 export class LocalStorageMemento implements Memento {
  constructor(private readonly prefix: string) {}

+  keys(): string[] {
+    const keys: string[] = [];
+    for (let i = 0; i < localStorage.length; i++) {
+      const key = localStorage.key(i);
+      if (key && key.startsWith(this.prefix)) {
+        keys.push(key.slice(this.prefix.length));
+      }
+    }
+    return keys;
+  }
+
  get<T>(key: string): T | null {
    const json = localStorage.getItem(this.prefix + key);
    return json ? JSON.parse(json) : null;
@@ -29,6 +40,16 @@ export class LocalStorageMemento implements Memento {
    channel.postMessage(value);
    channel.close();
  }
+
+  del(key: string): void {
+    localStorage.removeItem(this.prefix + key);
+  }
+
+  clear(): void {
+    for (const key of this.keys()) {
+      this.del(key);
+    }
+  }
 }

 export class LocalStorageGlobalCache
--- a/packages/frontend/core/src/pages/share/share-detail-page.tsx
+++ b/packages/frontend/core/src/pages/share/share-detail-page.tsx
@@ -15,12 +15,11 @@ import type { AffineEditorContainer } from '@blocksuite/presets';
 import type { Doc as BlockSuiteDoc } from '@blocksuite/store';
 import type { Doc } from '@toeverything/infra';
 import {
+  DocStorageImpl,
  EmptyBlobStorage,
  LocalBlobStorage,
-  LocalSyncStorage,
  PageManager,
  type PageMode,
-  ReadonlyMappingSyncStorage,
  RemoteBlobStorage,
  ServiceProviderContext,
  useLiveData,
@@ -29,6 +28,7 @@ import {
  WorkspaceManager,
  WorkspaceScope,
 } from '@toeverything/infra';
+import { ReadonlyDocStorage } from '@toeverything/infra';
 import { useCallback, useEffect, useState } from 'react';
 import type { LoaderFunction } from 'react-router-dom';
 import {
@@ -152,8 +152,8 @@ export const Component = () => {
          ])
          .addImpl(RemoteBlobStorage('static'), StaticBlobStorage)
          .addImpl(
-            LocalSyncStorage,
-            ReadonlyMappingSyncStorage({
+            DocStorageImpl,
+            new ReadonlyDocStorage({
              [workspaceId]: new Uint8Array(workspaceArrayBuffer),
              [pageId]: new Uint8Array(pageArrayBuffer),
            })
@@ -161,8 +161,8 @@ export const Component = () => {
      }
    );

-    workspace.engine.sync
-      .waitForSynced()
+    workspace.engine
+      .waitForRootDocReady()
      .then(() => {
        const { page } = workspace.services.get(PageManager).open(pageId);

--- a/packages/frontend/core/src/pages/workspace/detail-page/detail-page.tsx
+++ b/packages/frontend/core/src/pages/workspace/detail-page/detail-page.tsx
@@ -304,7 +304,10 @@ export const DetailPage = ({ pageId }: { pageId: string }): ReactElement => {

  // set sync engine priority target
  useEffect(() => {
-    currentWorkspace.setPriorityRule(id => id.endsWith(pageId));
+    currentWorkspace.setPriorityLoad(pageId, 10);
+    return () => {
+      currentWorkspace.setPriorityLoad(pageId, 5);
+    };
  }, [currentWorkspace, pageId]);

  const jumpOnce = useLiveData(pageRecord?.meta.map(meta => meta.jumpOnce));
--- a/packages/frontend/core/src/pages/workspace/index.tsx
+++ b/packages/frontend/core/src/pages/workspace/index.tsx
@@ -70,7 +70,8 @@ export const Component = (): ReactElement => {
  }, [meta, workspaceManager, workspace, currentWorkspaceService]);

  //  avoid doing operation, before workspace is loaded
-  const isRootDocLoaded = useLiveData(workspace?.engine.sync.isRootDocLoaded);
+  const isRootDocReady =
+    useLiveData(workspace?.engine.rootDocState)?.ready ?? false;

  // if listLoading is false, we can show 404 page, otherwise we should show loading page.
  if (listLoading === false && meta === undefined) {
@@ -81,7 +82,7 @@ export const Component = (): ReactElement => {
    return <WorkspaceFallback key="workspaceLoading" />;
  }

-  if (!isRootDocLoaded) {
+  if (!isRootDocReady) {
    return (
      <ServiceProviderContext.Provider value={workspace.services}>
        <WorkspaceFallback key="workspaceLoading" />
--- a/packages/frontend/core/src/testing.ts
+++ b/packages/frontend/core/src/testing.ts
@@ -34,7 +34,7 @@ export async function configureTestingEnvironment() {
    })
  );

-  await workspace.engine.sync.waitForSynced();
+  await workspace.engine.waitForSynced();

  const { page } = workspace.services.get(PageManager).open('page0');