refactor(workspace): blob sync (#5037)

This pr implements a blob engine.
It exposes a single `BlobStorage` to the `blocksuite`, and in it we sync blobs between multiple storages.

The implement still have few issues, but we can merge this pr first and fix them in future.

* BlobEngine currently **do nothing when delete**, because synchronization logic conflicts with deletion logic.
* BlobEngine sync between storages by querying the blob list at regular intervals. This will **cause many queries**, we can avoid this in the future by subscribing to remote changes.

packages/frontend/workspace/src/blob/cloud-blob-storage.ts

@@ -1,79 +0,0 @@
-import {
-  checkBlobSizesQuery,
-  deleteBlobMutation,
-  fetchWithTraceReport,
-  listBlobsQuery,
-  setBlobMutation,
-} from '@affine/graphql';
-import { fetcher } from '@affine/workspace/affine/gql';
-import type { BlobStorage } from '@blocksuite/store';
-
-import { predefinedStaticFiles } from './local-static-storage';
-import { bufferToBlob } from './util';
-
-export const createCloudBlobStorage = (workspaceId: string): BlobStorage => {
-  return {
-    crud: {
-      get: async key => {
-        const suffix = key.startsWith('/')
-          ? key
-          : predefinedStaticFiles.includes(key)
-            ? `/static/${key}`
-            : `/api/workspaces/${workspaceId}/blobs/${key}`;
-
-        return fetchWithTraceReport(
-          runtimeConfig.serverUrlPrefix + suffix
-        ).then(async res => {
-          if (!res.ok) {
-            // status not in the range 200-299
-            return null;
-          }
-          return bufferToBlob(await res.arrayBuffer());
-        });
-      },
-      set: async (key, value) => {
-        const {
-          checkBlobSize: { size },
-        } = await fetcher({
-          query: checkBlobSizesQuery,
-          variables: {
-            workspaceId,
-            size: value.size,
-          },
-        });
-
-        if (size <= 0) {
-          throw new Error('Blob size limit exceeded');
-        }
-
-        const result = await fetcher({
-          query: setBlobMutation,
-          variables: {
-            workspaceId,
-            blob: new File([value], key),
-          },
-        });
-        console.assert(result.setBlob === key, 'Blob hash mismatch');
-        return key;
-      },
-      list: async () => {
-        const result = await fetcher({
-          query: listBlobsQuery,
-          variables: {
-            workspaceId,
-          },
-        });
-        return result.listBlobs;
-      },
-      delete: async (key: string) => {
-        await fetcher({
-          query: deleteBlobMutation,
-          variables: {
-            workspaceId,
-            hash: key,
-          },
-        });
-      },
-    },
-  };
-};

packages/frontend/workspace/src/blob/engine.ts

@@ -0,0 +1,139 @@
+import { DebugLogger } from '@affine/debug';
+import { difference } from 'lodash-es';
+
+const logger = new DebugLogger('affine:blob-engine');
+
+export class BlobEngine {
+  constructor(
+    private local: BlobStorage,
+    private remotes: BlobStorage[]
+  ) {}
+
+  get storages() {
+    return [this.local, ...this.remotes];
+  }
+
+  async sync() {
+    if (this.local.readonly) {
+      return;
+    }
+    logger.debug('start syncing blob...');
+    for (const remote of this.remotes) {
+      let localList;
+      let remoteList;
+      try {
+        localList = await this.local.list();
+        remoteList = await remote.list();
+      } catch (err) {
+        logger.error(`error when sync`, err);
+        continue;
+      }
+
+      if (!remote.readonly) {
+        const needUpload = difference(localList, remoteList);
+        for (const key of needUpload) {
+          try {
+            const data = await this.local.get(key);
+            if (data) {
+              await remote.set(key, data);
+            }
+          } catch (err) {
+            logger.error(
+              `error when sync ${key} from [${this.local.name}] to [${remote.name}]`,
+              err
+            );
+          }
+        }
+      }
+
+      const needDownload = difference(remoteList, localList);
+
+      for (const key of needDownload) {
+        try {
+          const data = await remote.get(key);
+          if (data) {
+            await this.local.set(key, data);
+          }
+        } catch (err) {
+          logger.error(
+            `error when sync ${key} from [${remote.name}] to [${this.local.name}]`,
+            err
+          );
+        }
+      }
+    }
+
+    logger.debug('finish syncing blob');
+  }
+
+  async get(key: string) {
+    logger.debug('get blob', key);
+    for (const storage of this.storages) {
+      const data = await storage.get(key);
+      if (data) {
+        return data;
+      }
+    }
+    return undefined;
+  }
+
+  async set(key: string, value: Blob) {
+    if (this.local.readonly) {
+      throw new Error('local peer is readonly');
+    }
+
+    // await upload to the local peer
+    await this.local.set(key, value);
+
+    // uploads to other peers in the background
+    Promise.allSettled(
+      this.remotes
+        .filter(r => !r.readonly)
+        .map(peer =>
+          peer.set(key, value).catch(err => {
+            logger.error('error when upload to peer', err);
+          })
+        )
+    )
+      .then(result => {
+        if (result.some(({ status }) => status === 'rejected')) {
+          logger.error(
+            `blob ${key} update finish, but some peers failed to update`
+          );
+        } else {
+          logger.debug(`blob ${key} update finish`);
+        }
+      })
+      .catch(() => {
+        // Promise.allSettled never reject
+      });
+  }
+
+  async delete(_key: string) {
+    // not supported
+  }
+
+  async list() {
+    const blobList = new Set<string>();
+
+    for (const peer of this.storages) {
+      const list = await peer.list();
+      if (list) {
+        for (const blob of list) {
+          blobList.add(blob);
+        }
+      }
+    }
+
+    return Array.from(blobList);
+  }
+}
+
+export interface BlobStorage {
+  name: string;
+  readonly: boolean;
+  get: (key: string) => Promise<Blob | undefined>;
+  set: (key: string, value: Blob) => Promise<void>;
+  delete: (key: string) => Promise<void>;
+  list: () => Promise<string[]>;
+}

packages/frontend/workspace/src/blob/index.ts

@@ -0,0 +1,37 @@
+import { BlobEngine } from './engine';
+import {
+  createAffineCloudBlobStorage,
+  createIndexeddbBlobStorage,
+  createSQLiteBlobStorage,
+  createStaticBlobStorage,
+} from './storage';
+
+export * from './engine';
+export * from './storage';
+
+export function createLocalBlobStorage(workspaceId: string) {
+  if (environment.isDesktop) {
+    return createSQLiteBlobStorage(workspaceId);
+  } else {
+    return createIndexeddbBlobStorage(workspaceId);
+  }
+}
+
+export function createLocalBlobEngine(workspaceId: string) {
+  return new BlobEngine(createLocalBlobStorage(workspaceId), [
+    createStaticBlobStorage(),
+  ]);
+}
+
+export function createAffineCloudBlobEngine(workspaceId: string) {
+  return new BlobEngine(createLocalBlobStorage(workspaceId), [
+    createStaticBlobStorage(),
+    createAffineCloudBlobStorage(workspaceId),
+  ]);
+}
+
+export function createAffinePublicBlobEngine(workspaceId: string) {
+  return new BlobEngine(createAffineCloudBlobStorage(workspaceId), [
+    createStaticBlobStorage(),
+  ]);
+}

packages/frontend/workspace/src/blob/sqlite-blob-storage.ts

@@ -1,34 +0,0 @@
-import { assertExists } from '@blocksuite/global/utils';
-import type { BlobStorage } from '@blocksuite/store';
-
-import { bufferToBlob } from './util';
-
-export const createSQLiteStorage = (workspaceId: string): BlobStorage => {
-  const apis = window.apis;
-  assertExists(apis);
-  return {
-    crud: {
-      get: async (key: string) => {
-        const buffer = await apis.db.getBlob(workspaceId, key);
-        if (buffer) {
-          return bufferToBlob(buffer);
-        }
-        return null;
-      },
-      set: async (key: string, value: Blob) => {
-        await apis.db.addBlob(
-          workspaceId,
-          key,
-          new Uint8Array(await value.arrayBuffer())
-        );
-        return key;
-      },
-      delete: async (key: string) => {
-        return apis.db.deleteBlob(workspaceId, key);
-      },
-      list: async () => {
-        return apis.db.getBlobKeys(workspaceId);
-      },
-    },
-  };
-};

packages/frontend/workspace/src/blob/storage/affine-cloud.ts

@@ -0,0 +1,76 @@
+import {
+  checkBlobSizesQuery,
+  deleteBlobMutation,
+  fetchWithTraceReport,
+  listBlobsQuery,
+  setBlobMutation,
+} from '@affine/graphql';
+import { fetcher } from '@affine/workspace/affine/gql';
+
+import type { BlobStorage } from '../engine';
+
+export const createAffineCloudBlobStorage = (
+  workspaceId: string
+): BlobStorage => {
+  return {
+    name: 'affine-cloud',
+    readonly: false,
+    get: async key => {
+      const suffix = key.startsWith('/')
+        ? key
+        : `/api/workspaces/${workspaceId}/blobs/${key}`;
+
+      return fetchWithTraceReport(runtimeConfig.serverUrlPrefix + suffix).then(
+        async res => {
+          if (!res.ok) {
+            // status not in the range 200-299
+            return undefined;
+          }
+          return await res.blob();
+        }
+      );
+    },
+    set: async (key, value) => {
+      const {
+        checkBlobSize: { size },
+      } = await fetcher({
+        query: checkBlobSizesQuery,
+        variables: {
+          workspaceId,
+          size: value.size,
+        },
+      });
+
+      if (size <= 0) {
+        throw new Error('Blob size limit exceeded');
+      }
+
+      const result = await fetcher({
+        query: setBlobMutation,
+        variables: {
+          workspaceId,
+          blob: new File([value], key),
+        },
+      });
+      console.assert(result.setBlob === key, 'Blob hash mismatch');
+    },
+    list: async () => {
+      const result = await fetcher({
+        query: listBlobsQuery,
+        variables: {
+          workspaceId,
+        },
+      });
+      return result.listBlobs;
+    },
+    delete: async (key: string) => {
+      await fetcher({
+        query: deleteBlobMutation,
+        variables: {
+          workspaceId,
+          hash: key,
+        },
+      });
+    },
+  };
+};

packages/frontend/workspace/src/blob/storage/index.ts

@@ -0,0 +1,4 @@
+export * from './affine-cloud';
+export * from './indexeddb';
+export * from './sqlite';
+export * from './static';

packages/frontend/workspace/src/blob/storage/indexeddb.ts

@@ -0,0 +1,32 @@
+import { createStore, del, get, keys, set } from 'idb-keyval';
+
+import type { BlobStorage } from '../engine';
+
+export const createIndexeddbBlobStorage = (
+  workspaceId: string
+): BlobStorage => {
+  const db = createStore(`${workspaceId}_blob`, 'blob');
+  const mimeTypeDb = createStore(`${workspaceId}_blob_mime`, 'blob_mime');
+  return {
+    name: 'indexeddb',
+    readonly: false,
+    get: async (key: string) => {
+      const res = await get<ArrayBuffer>(key, db);
+      if (res) {
+        return new Blob([res], { type: await get(key, mimeTypeDb) });
+      }
+      return undefined;
+    },
+    set: async (key: string, value: Blob) => {
+      await set(key, await value.arrayBuffer(), db);
+      await set(key, value.type, mimeTypeDb);
+    },
+    delete: async (key: string) => {
+      await del(key, db);
+      await del(key, mimeTypeDb);
+    },
+    list: async () => {
+      return keys<string>(db);
+    },
+  };
+};

packages/frontend/workspace/src/blob/storage/sqlite.ts

@@ -0,0 +1,33 @@
+import { assertExists } from '@blocksuite/global/utils';
+
+import type { BlobStorage } from '../engine';
+import { bufferToBlob } from '../util';
+
+export const createSQLiteBlobStorage = (workspaceId: string): BlobStorage => {
+  const apis = window.apis;
+  assertExists(apis);
+  return {
+    name: 'sqlite',
+    readonly: false,
+    get: async (key: string) => {
+      const buffer = await apis.db.getBlob(workspaceId, key);
+      if (buffer) {
+        return bufferToBlob(buffer);
+      }
+      return undefined;
+    },
+    set: async (key: string, value: Blob) => {
+      await apis.db.addBlob(
+        workspaceId,
+        key,
+        new Uint8Array(await value.arrayBuffer())
+      );
+    },
+    delete: async (key: string) => {
+      return apis.db.deleteBlob(workspaceId, key);
+    },
+    list: async () => {
+      return apis.db.getBlobKeys(workspaceId);
+    },
+  };
+};

packages/frontend/workspace/src/blob/storage/static.ts

@@ -1,6 +1,4 @@
-import type { BlobStorage } from '@blocksuite/store';
-
-import { bufferToBlob } from './util';
+import type { BlobStorage } from '../engine';
 
 export const predefinedStaticFiles = [
   '029uztLz2CzJezK7UUhrbGiWUdZ0J7NVs_qR6RDsvb8=',
@@ -38,38 +36,36 @@ export const predefinedStaticFiles = [
   'v2yF7lY2L5rtorTtTmYFsoMb9dBPKs5M1y9cUKxcI1M=',
 ];
 
-export const createStaticStorage = (): BlobStorage => {
+export const createStaticBlobStorage = (): BlobStorage => {
   return {
-    crud: {
-      get: async (key: string) => {
-        const isStaticResource =
-          predefinedStaticFiles.includes(key) || key.startsWith('/static/');
+    name: 'static',
+    readonly: true,
+    get: async (key: string) => {
+      const isStaticResource =
+        predefinedStaticFiles.includes(key) || key.startsWith('/static/');
 
-        if (!isStaticResource) {
-          return null;
-        }
+      if (!isStaticResource) {
+        return undefined;
+      }
 
-        const path = key.startsWith('/static/') ? key : `/static/${key}`;
-        const response = await fetch(path);
+      const path = key.startsWith('/static/') ? key : `/static/${key}`;
+      const response = await fetch(path);
 
-        if (response.ok) {
-          const buffer = await response.arrayBuffer();
-          return bufferToBlob(buffer);
-        }
+      if (response.ok) {
+        return await response.blob();
+      }
 
-        return null;
-      },
-      set: async (key: string) => {
-        // ignore
-        return key;
-      },
-      delete: async () => {
-        // ignore
-      },
-      list: async () => {
-        // ignore
-        return [];
-      },
+      return undefined;
+    },
+    set: async () => {
+      // ignore
+    },
+    delete: async () => {
+      // ignore
+    },
+    list: async () => {
+      // ignore
+      return [];
     },
   };
 };