feat: support get datasource status (#3645)

packages/env/src/blocksuite/index.ts

@@ -2,11 +2,7 @@ import type { Page } from '@blocksuite/store';
 
 export async function initPageWithPreloading(page: Page) {
   const workspace = page.workspace;
-  const {
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-expect-error
-    data,
-  } = await import('@affine/templates/preloading.json');
+  const { data } = await import('@affine/templates/preloading.json');
   await page.waitForLoaded();
   await workspace.importPageSnapshot(data['space:hello-world'], page.id);
 }

packages/env/src/workspace.ts

@@ -1,3 +1,4 @@
+import type { StatusAdapter } from '@affine/y-provider';
 import type { EditorContainer } from '@blocksuite/editor';
 import type { Page } from '@blocksuite/store';
 import type {
@@ -35,7 +36,9 @@ export interface BroadCastChannelProvider extends PassiveDocProvider {
 /**
  * Long polling provider with local indexeddb
  */
-export interface LocalIndexedDBBackgroundProvider extends PassiveDocProvider {
+export interface LocalIndexedDBBackgroundProvider
+  extends StatusAdapter,
+    PassiveDocProvider {
   flavour: 'local-indexeddb-background';
 }
 
@@ -43,7 +46,7 @@ export interface LocalIndexedDBDownloadProvider extends ActiveDocProvider {
   flavour: 'local-indexeddb';
 }
 
-export interface SQLiteProvider extends PassiveDocProvider {
+export interface SQLiteProvider extends PassiveDocProvider, StatusAdapter {
   flavour: 'sqlite';
 }
 

packages/env/tsconfig.json

@@ -4,7 +4,6 @@
   "compilerOptions": {
     "composite": true,
     "noEmit": false,
-    "moduleResolution": "Node16",
     "outDir": "lib"
   },
   "references": [

packages/hooks/package.json

@@ -6,10 +6,11 @@
   },
   "private": true,
   "dependencies": {
-    "@affine/env": "workspace:*",
-    "@toeverything/y-indexeddb": "workspace:*"
+    "foxact": "^0.2.17"
   },
   "devDependencies": {
+    "@affine/env": "workspace:*",
+    "@affine/y-provider": "workspace:*",
     "@blocksuite/block-std": "0.0.0-20230810005427-25adb757-nightly",
     "@blocksuite/blocks": "0.0.0-20230810005427-25adb757-nightly",
     "@blocksuite/editor": "0.0.0-20230810005427-25adb757-nightly",
@@ -18,6 +19,7 @@
     "@blocksuite/store": "0.0.0-20230810005427-25adb757-nightly"
   },
   "peerDependencies": {
+    "@affine/y-provider": "workspace:*",
     "@blocksuite/block-std": "*",
     "@blocksuite/blocks": "*",
     "@blocksuite/editor": "*",
@@ -25,5 +27,31 @@
     "@blocksuite/lit": "*",
     "@blocksuite/store": "*"
   },
+  "peerDependenciesMeta": {
+    "@affine/env": {
+      "optional": true
+    },
+    "@affine/y-provider": {
+      "optional": true
+    },
+    "@blocksuite/block-std": {
+      "optional": true
+    },
+    "@blocksuite/blocks": {
+      "optional": true
+    },
+    "@blocksuite/editor": {
+      "optional": true
+    },
+    "@blocksuite/global": {
+      "optional": true
+    },
+    "@blocksuite/lit": {
+      "optional": true
+    },
+    "@blocksuite/store": {
+      "optional": true
+    }
+  },
   "version": "0.8.0-canary.16"
 }

packages/hooks/src/use-data-source-status.ts

@@ -0,0 +1,15 @@
+import type { Status, StatusAdapter } from '@affine/y-provider';
+import { useCallback, useSyncExternalStore } from 'react';
+
+type UIStatus =
+  | Status
+  | {
+      type: 'unknown';
+    };
+
+export function useDataSourceStatus(datasource: StatusAdapter): UIStatus {
+  return useSyncExternalStore(
+    datasource.subscribeStatusChange,
+    useCallback(() => datasource.status, [datasource])
+  );
+}

packages/workspace/src/providers/index.ts

@@ -26,10 +26,15 @@ const createIndexedDBBackgroundProvider: DocProviderCreator = (
   blockSuiteWorkspace
 ): LocalIndexedDBBackgroundProvider => {
   const indexeddbProvider = create(blockSuiteWorkspace);
+
   let connected = false;
   return {
     flavour: 'local-indexeddb-background',
     passive: true,
+    get status() {
+      return indexeddbProvider.status;
+    },
+    subscribeStatusChange: indexeddbProvider.subscribeStatusChange,
     get connected() {
       return connected;
     },

packages/workspace/src/providers/sqlite-providers.ts

@@ -6,6 +6,7 @@ import {
   createLazyProvider,
   type DatasourceDocAdapter,
 } from '@affine/y-provider';
+import { assertExists } from '@blocksuite/global/utils';
 import type { DocProviderCreator } from '@blocksuite/store';
 import { Workspace as BlockSuiteWorkspace } from '@blocksuite/store';
 import type { Doc } from 'yjs';
@@ -51,6 +52,14 @@ export const createSQLiteProvider: DocProviderCreator = (
   return {
     flavour: 'sqlite',
     passive: true,
+    get status() {
+      assertExists(provider);
+      return provider.status;
+    },
+    subscribeStatusChange(onStatusChange) {
+      assertExists(provider);
+      return provider.subscribeStatusChange(onStatusChange);
+    },
     connect: () => {
       datasource = createDatasource(id);
       provider = createLazyProvider(rootDoc, datasource, { origin: 'sqlite' });

packages/y-indexeddb/src/provider.ts

@@ -3,6 +3,7 @@ import {
   type DatasourceDocAdapter,
   writeOperation,
 } from '@affine/y-provider';
+import { assertExists } from '@blocksuite/global/utils';
 import { openDB } from 'idb';
 import type { Doc } from 'yjs';
 import { diffUpdate, mergeUpdates } from 'yjs';
@@ -77,7 +78,6 @@ const createDatasource = ({
           const merged = mergeUpdates(rows.map(({ update }) => update));
           rows = [{ timestamp: Date.now(), update: merged }];
         }
-
         await writeOperation(
           store.put({
             id: guid,
@@ -112,6 +112,14 @@ export const createIndexedDBProvider = (
   let provider: ReturnType<typeof createLazyProvider> | null = null;
 
   const apis = {
+    get status() {
+      assertExists(provider);
+      return provider.status;
+    },
+    subscribeStatusChange(onStatusChange) {
+      assertExists(provider);
+      return provider.subscribeStatusChange(onStatusChange);
+    },
     connect: () => {
       if (apis.connected) {
         apis.disconnect();
@@ -132,7 +140,7 @@ export const createIndexedDBProvider = (
     get connected() {
       return provider?.connected || false;
     },
-  };
+  } satisfies IndexedDBProvider;
 
   return apis;
 };

packages/y-indexeddb/src/shared.ts

@@ -1,3 +1,4 @@
+import type { StatusAdapter } from '@affine/y-provider';
 import type { DBSchema, IDBPDatabase } from 'idb';
 
 export const dbVersion = 1;
@@ -8,7 +9,7 @@ export function upgradeDB(db: IDBPDatabase<BlockSuiteBinaryDB>) {
   db.createObjectStore('milestone', { keyPath: 'id' });
 }
 
-export interface IndexedDBProvider {
+export interface IndexedDBProvider extends StatusAdapter {
   connect: () => void;
   disconnect: () => void;
   cleanup: () => Promise<void>;

packages/y-provider/package.json

@@ -4,6 +4,10 @@
   "version": "0.8.0-canary.16",
   "description": "Yjs provider utilities for AFFiNE",
   "main": "./src/index.ts",
+  "module": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
   "devDependencies": {
     "@blocksuite/store": "0.0.0-20230810005427-25adb757-nightly"
   },

packages/y-provider/src/lazy-provider.ts

@@ -7,7 +7,8 @@ import {
   encodeStateVectorFromUpdate,
 } from 'yjs';
 
-import type { DatasourceDocAdapter } from './types';
+import type { DatasourceDocAdapter, StatusAdapter } from './types';
+import type { Status } from './types';
 
 function getDoc(doc: Doc, guid: string): Doc | undefined {
   if (doc.guid === guid) {
@@ -33,7 +34,7 @@ export const createLazyProvider = (
   rootDoc: Doc,
   datasource: DatasourceDocAdapter,
   options: LazyProviderOptions = {}
-): Omit<PassiveDocProvider, 'flavour'> => {
+): Omit<PassiveDocProvider, 'flavour'> & StatusAdapter => {
   let connected = false;
   const pendingMap = new Map<string, Uint8Array[]>(); // guid -> pending-updates
   const disposableMap = new Map<string, Set<() => void>>();
@@ -42,11 +43,59 @@ export const createLazyProvider = (
 
   const { origin = 'lazy-provider' } = options;
 
+  // todo: should we use a real state machine here like `xstate`?
+  let currentStatus: Status = {
+    type: 'idle',
+  };
+  let syncingStack = 0;
+  const callbackSet = new Set<() => void>();
+  const changeStatus = (newStatus: Status) => {
+    // simulate a stack, each syncing and synced should be paired
+    if (newStatus.type === 'idle') {
+      if (syncingStack !== 0) {
+        console.error('syncingStatus !== 0, this should not happen');
+      }
+      syncingStack = 0;
+    }
+    if (newStatus.type === 'syncing') {
+      syncingStack++;
+    }
+    if (newStatus.type === 'synced' || newStatus.type === 'error') {
+      syncingStack--;
+    }
+
+    if (syncingStack < 0) {
+      console.error('syncingStatus < 0, this should not happen');
+    }
+
+    if (syncingStack === 0) {
+      currentStatus = newStatus;
+    }
+    if (newStatus.type !== 'synced') {
+      currentStatus = newStatus;
+    }
+    callbackSet.forEach(cb => cb());
+  };
+
   async function syncDoc(doc: Doc) {
     const guid = doc.guid;
 
-    const remoteUpdate = await datasource.queryDocState(guid, {
-      stateVector: encodeStateVector(doc),
+    changeStatus({
+      type: 'syncing',
+    });
+    const remoteUpdate = await datasource
+      .queryDocState(guid, {
+        stateVector: encodeStateVector(doc),
+      })
+      .catch(error => {
+        changeStatus({
+          type: 'error',
+          error,
+        });
+        throw error;
+      });
+    changeStatus({
+      type: 'synced',
     });
 
     pendingMap.set(guid, []);
@@ -59,6 +108,9 @@ export const createLazyProvider = (
       ? encodeStateVectorFromUpdate(remoteUpdate)
       : undefined;
 
+    if (!connected) {
+      return;
+    }
     // perf: optimize me
     // it is possible the doc is only in memory but not yet in the datasource
     // we need to send the whole update to the datasource
@@ -76,7 +128,23 @@ export const createLazyProvider = (
       if (origin === updateOrigin) {
         return;
       }
-      datasource.sendDocUpdate(doc.guid, update).catch(console.error);
+      changeStatus({
+        type: 'syncing',
+      });
+      datasource
+        .sendDocUpdate(doc.guid, update)
+        .then(() => {
+          changeStatus({
+            type: 'synced',
+          });
+        })
+        .catch(error => {
+          changeStatus({
+            type: 'error',
+            error,
+          });
+          console.error(error);
+        });
     };
 
     const subdocsHandler = (event: { loaded: Set<Doc>; removed: Set<Doc> }) => {
@@ -103,6 +171,9 @@ export const createLazyProvider = (
    */
   function setupDatasourceListeners() {
     datasourceUnsub = datasource.onDocUpdate?.((guid, update) => {
+      changeStatus({
+        type: 'syncing',
+      });
       const doc = getDoc(rootDoc, guid);
       if (doc) {
         applyUpdate(doc, update, origin);
@@ -120,6 +191,9 @@ export const createLazyProvider = (
         console.warn('idb: doc not found', guid);
         pendingMap.set(guid, (pendingMap.get(guid) ?? []).concat(update));
       }
+      changeStatus({
+        type: 'synced',
+      });
     });
   }
 
@@ -165,20 +239,44 @@ export const createLazyProvider = (
   function connect() {
     connected = true;
 
+    changeStatus({
+      type: 'syncing',
+    });
     // root doc should be already loaded,
     // but we want to populate the cache for later update events
-    connectDoc(rootDoc).catch(console.error);
+    connectDoc(rootDoc).catch(error => {
+      changeStatus({
+        type: 'error',
+        error,
+      });
+      console.error(error);
+    });
+    changeStatus({
+      type: 'synced',
+    });
     setupDatasourceListeners();
   }
 
   async function disconnect() {
     connected = false;
+    changeStatus({
+      type: 'idle',
+    });
     disposeAll();
     datasourceUnsub?.();
     datasourceUnsub = undefined;
   }
 
   return {
+    get status() {
+      return currentStatus;
+    },
+    subscribeStatusChange(cb: () => void) {
+      callbackSet.add(cb);
+      return () => {
+        callbackSet.delete(cb);
+      };
+    },
     get connected() {
       return connected;
     },

packages/y-provider/src/types.ts

@@ -1,4 +1,24 @@
-export interface DatasourceDocAdapter {
+export type Status =
+  | {
+      type: 'idle';
+    }
+  | {
+      type: 'syncing';
+    }
+  | {
+      type: 'synced';
+    }
+  | {
+      type: 'error';
+      error: Error;
+    };
+
+export interface StatusAdapter {
+  readonly status: Status;
+  subscribeStatusChange(onStatusChange: () => void): () => void;
+}
+
+export interface DatasourceDocAdapter extends Partial<StatusAdapter> {
   // request diff update from other clients
   queryDocState: (
     guid: string,