feat(nbstore): add blob sync (#8996)

packages/common/nbstore/src/sync/blob/index.ts

@@ -0,0 +1,89 @@
+import { difference } from 'lodash-es';
+
+import type { BlobStorage } from '../../storage';
+import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
+
+export class BlobSyncEngine {
+  constructor(
+    readonly local: BlobStorage,
+    readonly remotes: BlobStorage[]
+  ) {}
+
+  private async sync(signal?: AbortSignal) {
+    throwIfAborted(signal);
+
+    for (const remote of this.remotes) {
+      let localList: string[] = [];
+      let remoteList: string[] = [];
+
+      try {
+        localList = (await this.local.list(signal)).map(b => b.key);
+        throwIfAborted(signal);
+        remoteList = (await remote.list(signal)).map(b => b.key);
+        throwIfAborted(signal);
+      } catch (err) {
+        if (err === MANUALLY_STOP) {
+          throw err;
+        }
+        console.error(`error when sync`, err);
+        continue;
+      }
+
+      const needUpload = difference(localList, remoteList);
+      for (const key of needUpload) {
+        try {
+          const data = await this.local.get(key, signal);
+          throwIfAborted(signal);
+          if (data) {
+            await remote.set(data, signal);
+            throwIfAborted(signal);
+          }
+        } catch (err) {
+          if (err === MANUALLY_STOP) {
+            throw err;
+          }
+          console.error(
+            `error when sync ${key} from [${this.local.peer}] to [${remote.peer}]`,
+            err
+          );
+        }
+      }
+
+      const needDownload = difference(remoteList, localList);
+
+      for (const key of needDownload) {
+        try {
+          const data = await remote.get(key, signal);
+          throwIfAborted(signal);
+          if (data) {
+            await this.local.set(data, signal);
+            throwIfAborted(signal);
+          }
+        } catch (err) {
+          if (err === MANUALLY_STOP) {
+            throw err;
+          }
+          console.error(
+            `error when sync ${key} from [${remote.peer}] to [${this.local.peer}]`,
+            err
+          );
+        }
+      }
+    }
+  }
+
+  async run(signal?: AbortSignal) {
+    if (signal?.aborted) {
+      return;
+    }
+
+    try {
+      await this.sync(signal);
+    } catch (error) {
+      if (error === MANUALLY_STOP) {
+        return;
+      }
+      console.error('sync blob error', error);
+    }
+  }
+}

packages/common/nbstore/src/sync/doc/peer.ts

@@ -620,7 +620,19 @@ export class DocSyncPeer {
 
   setPriority(docId: string, priority: number) {
     this.prioritySettings.set(docId, priority);
-    this.status.jobDocQueue.updatePriority(docId, priority);
+    return this.status.jobDocQueue.setPriority(docId, priority);
+  }
+
+  addPriority(id: string, priority: number) {
+    const oldPriority = this.prioritySettings.get(id) ?? 0;
+    this.prioritySettings.set(id, priority);
+    this.status.jobDocQueue.setPriority(id, oldPriority + priority);
+
+    return () => {
+      const currentPriority = this.prioritySettings.get(id) ?? 0;
+      this.prioritySettings.set(id, currentPriority - priority);
+      this.status.jobDocQueue.setPriority(id, currentPriority - priority);
+    };
   }
 
   protected mergeUpdates(updates: Uint8Array[]) {

packages/common/nbstore/src/sync/index.ts

@@ -1,4 +1,5 @@
-import type { DocStorage, SpaceStorage } from '../storage';
+import type { BlobStorage, DocStorage, SpaceStorage } from '../storage';
+import { BlobSyncEngine } from './blob';
 import { DocSyncEngine } from './doc';
 
 export class SyncEngine {
@@ -9,15 +10,30 @@ export class SyncEngine {
 
   async run(signal?: AbortSignal) {
     const doc = this.local.tryGet('doc');
+    const blob = this.local.tryGet('blob');
     const sync = this.local.tryGet('sync');
 
-    if (doc && sync) {
-      const peerDocs = this.peers
-        .map(peer => peer.tryGet('doc'))
-        .filter((v): v is DocStorage => !!v);
+    await Promise.allSettled([
+      (async () => {
+        if (doc && sync) {
+          const peerDocs = this.peers
+            .map(peer => peer.tryGet('doc'))
+            .filter((v): v is DocStorage => !!v);
 
-      const engine = new DocSyncEngine(doc, sync, peerDocs);
-      await engine.run(signal);
-    }
+          const engine = new DocSyncEngine(doc, sync, peerDocs);
+          await engine.run(signal);
+        }
+      })(),
+      (async () => {
+        if (blob) {
+          const peerBlobs = this.peers
+            .map(peer => peer.tryGet('blob'))
+            .filter((v): v is BlobStorage => !!v);
+
+          const engine = new BlobSyncEngine(blob, peerBlobs);
+          await engine.run(signal);
+        }
+      })(),
+    ]);
   }
 }