feat(nbstore): add blob sync (#8996)

2026-08-04 02:49:57 +08:00 · 2024-12-07 08:05:03 +00:00
parent f54f6e88cb
commit 7225f59138
7 changed files with 214 additions and 19 deletions
@@ -0,0 +1,89 @@
+import { difference } from 'lodash-es';
+
+import type { BlobStorage } from '../../storage';
+import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
+
+export class BlobSyncEngine {
+  constructor(
+    readonly local: BlobStorage,
+    readonly remotes: BlobStorage[]
+  ) {}
+
+  private async sync(signal?: AbortSignal) {
+    throwIfAborted(signal);
+
+    for (const remote of this.remotes) {
+      let localList: string[] = [];
+      let remoteList: string[] = [];
+
+      try {
+        localList = (await this.local.list(signal)).map(b => b.key);
+        throwIfAborted(signal);
+        remoteList = (await remote.list(signal)).map(b => b.key);
+        throwIfAborted(signal);
+      } catch (err) {
+        if (err === MANUALLY_STOP) {
+          throw err;
+        }
+        console.error(`error when sync`, err);
+        continue;
+      }
+
+      const needUpload = difference(localList, remoteList);
+      for (const key of needUpload) {
+        try {
+          const data = await this.local.get(key, signal);
+          throwIfAborted(signal);
+          if (data) {
+            await remote.set(data, signal);
+            throwIfAborted(signal);
+          }
+        } catch (err) {
+          if (err === MANUALLY_STOP) {
+            throw err;
+          }
+          console.error(
+            `error when sync ${key} from [${this.local.peer}] to [${remote.peer}]`,
+            err
+          );
+        }
+      }
+
+      const needDownload = difference(remoteList, localList);
+
+      for (const key of needDownload) {
+        try {
+          const data = await remote.get(key, signal);
+          throwIfAborted(signal);
+          if (data) {
+            await this.local.set(data, signal);
+            throwIfAborted(signal);
+          }
+        } catch (err) {
+          if (err === MANUALLY_STOP) {
+            throw err;
+          }
+          console.error(
+            `error when sync ${key} from [${remote.peer}] to [${this.local.peer}]`,
+            err
+          );
+        }
+      }
+    }
+  }
+
+  async run(signal?: AbortSignal) {
+    if (signal?.aborted) {
+      return;
+    }
+
+    try {
+      await this.sync(signal);
+    } catch (error) {
+      if (error === MANUALLY_STOP) {
+        return;
+      }
+      console.error('sync blob error', error);
+    }
+  }
+}