refactor(workspace): blob sync (#5037)

This pr implements a blob engine.
It exposes a single `BlobStorage` to the `blocksuite`, and in it we sync blobs between multiple storages.

The implement still have few issues, but we can merge this pr first and fix them in future.

* BlobEngine currently **do nothing when delete**, because synchronization logic conflicts with deletion logic.
* BlobEngine sync between storages by querying the blob list at regular intervals. This will **cause many queries**, we can avoid this in the future by subscribing to remote changes.

packages/frontend/workspace/src/blob/storage/affine-cloud.ts

@@ -0,0 +1,76 @@
+import {
+  checkBlobSizesQuery,
+  deleteBlobMutation,
+  fetchWithTraceReport,
+  listBlobsQuery,
+  setBlobMutation,
+} from '@affine/graphql';
+import { fetcher } from '@affine/workspace/affine/gql';
+
+import type { BlobStorage } from '../engine';
+
+export const createAffineCloudBlobStorage = (
+  workspaceId: string
+): BlobStorage => {
+  return {
+    name: 'affine-cloud',
+    readonly: false,
+    get: async key => {
+      const suffix = key.startsWith('/')
+        ? key
+        : `/api/workspaces/${workspaceId}/blobs/${key}`;
+
+      return fetchWithTraceReport(runtimeConfig.serverUrlPrefix + suffix).then(
+        async res => {
+          if (!res.ok) {
+            // status not in the range 200-299
+            return undefined;
+          }
+          return await res.blob();
+        }
+      );
+    },
+    set: async (key, value) => {
+      const {
+        checkBlobSize: { size },
+      } = await fetcher({
+        query: checkBlobSizesQuery,
+        variables: {
+          workspaceId,
+          size: value.size,
+        },
+      });
+
+      if (size <= 0) {
+        throw new Error('Blob size limit exceeded');
+      }
+
+      const result = await fetcher({
+        query: setBlobMutation,
+        variables: {
+          workspaceId,
+          blob: new File([value], key),
+        },
+      });
+      console.assert(result.setBlob === key, 'Blob hash mismatch');
+    },
+    list: async () => {
+      const result = await fetcher({
+        query: listBlobsQuery,
+        variables: {
+          workspaceId,
+        },
+      });
+      return result.listBlobs;
+    },
+    delete: async (key: string) => {
+      await fetcher({
+        query: deleteBlobMutation,
+        variables: {
+          workspaceId,
+          hash: key,
+        },
+      });
+    },
+  };
+};

packages/frontend/workspace/src/blob/storage/index.ts

@@ -0,0 +1,4 @@
+export * from './affine-cloud';
+export * from './indexeddb';
+export * from './sqlite';
+export * from './static';

packages/frontend/workspace/src/blob/storage/indexeddb.ts

@@ -0,0 +1,32 @@
+import { createStore, del, get, keys, set } from 'idb-keyval';
+
+import type { BlobStorage } from '../engine';
+
+export const createIndexeddbBlobStorage = (
+  workspaceId: string
+): BlobStorage => {
+  const db = createStore(`${workspaceId}_blob`, 'blob');
+  const mimeTypeDb = createStore(`${workspaceId}_blob_mime`, 'blob_mime');
+  return {
+    name: 'indexeddb',
+    readonly: false,
+    get: async (key: string) => {
+      const res = await get<ArrayBuffer>(key, db);
+      if (res) {
+        return new Blob([res], { type: await get(key, mimeTypeDb) });
+      }
+      return undefined;
+    },
+    set: async (key: string, value: Blob) => {
+      await set(key, await value.arrayBuffer(), db);
+      await set(key, value.type, mimeTypeDb);
+    },
+    delete: async (key: string) => {
+      await del(key, db);
+      await del(key, mimeTypeDb);
+    },
+    list: async () => {
+      return keys<string>(db);
+    },
+  };
+};

packages/frontend/workspace/src/blob/storage/sqlite.ts

@@ -0,0 +1,33 @@
+import { assertExists } from '@blocksuite/global/utils';
+
+import type { BlobStorage } from '../engine';
+import { bufferToBlob } from '../util';
+
+export const createSQLiteBlobStorage = (workspaceId: string): BlobStorage => {
+  const apis = window.apis;
+  assertExists(apis);
+  return {
+    name: 'sqlite',
+    readonly: false,
+    get: async (key: string) => {
+      const buffer = await apis.db.getBlob(workspaceId, key);
+      if (buffer) {
+        return bufferToBlob(buffer);
+      }
+      return undefined;
+    },
+    set: async (key: string, value: Blob) => {
+      await apis.db.addBlob(
+        workspaceId,
+        key,
+        new Uint8Array(await value.arrayBuffer())
+      );
+    },
+    delete: async (key: string) => {
+      return apis.db.deleteBlob(workspaceId, key);
+    },
+    list: async () => {
+      return apis.db.getBlobKeys(workspaceId);
+    },
+  };
+};

packages/frontend/workspace/src/blob/storage/static.ts

@@ -0,0 +1,71 @@
+import type { BlobStorage } from '../engine';
+
+export const predefinedStaticFiles = [
+  '029uztLz2CzJezK7UUhrbGiWUdZ0J7NVs_qR6RDsvb8=',
+  '047ebf2c9a5c7c9d8521c2ea5e6140ff7732ef9e28a9f944e9bf3ca4',
+  '0hjYqQd8SvwHT2gPds7qFw8W6qIEGVbZvG45uzoYjUU=',
+  '1326bc48553a572c6756d9ee1b30a0dfdda26222fc2d2c872b14e609',
+  '27f983d0765289c19d10ee0b51c00c3c7665236a1a82406370d46e0a',
+  '28516717d63e469cd98729ff46be6595711898bab3dc43302319a987',
+  '4HXJrnBZGaGPFpowNawNog0aMg3dgoVaAnNqEMeUxq0=',
+  '5Cfem_137WmzR35ZeIC76oTkq5SQt-eHlZwJiLy0hgU=',
+  '6aa785ee927547ce9dd9d7b43e01eac948337fe57571443e87bc3a60',
+  '8oj6ym4HlTcshT40Zn6D5DeOgaVCSOOXJvT_EyiqUw8=',
+  '9288be57321c8772d04e05dbb69a22742372b3534442607a2d6a9998',
+  '9vXwWGEX5W9v5pzwpu0eK4pf22DZ_sCloO0zCH1aVQ4=',
+  'Bd5F0WRI0fLh8RK1al9PawPVT3jv7VwBrqiiBEtdV-g=',
+  'CBWoKrhSDndjBJzscQKENRqiXOOZnzIA5qyiCoy4-A0=',
+  'D7g-4LMqOsVWBNOD-_kGgCOvJEoc8rcpYbkfDlF2u5U=',
+  'Vqc8rxFbGyc5L1QeE_Zr10XEcIai_0Xw4Qv6d3ldRPE=',
+  'VuXYyM9JUv1Fv_qjg1v5Go4Zksz0r4NXFeh3Na7JkIc=',
+  'bfXllFddegV9vvxPcSWnOtm-_tuzXm-0OQ59z9Su1zA=',
+  'c820edeeba50006b531883903f5bb0b96bf523c9a6b3ce5868f03db5',
+  'cw9XjQ-pCeSW7LKMzVREGHeCPTXWYbtE-QbZLEY3RrI=',
+  'e93536e1be97e3b5206d43bf0793fdef24e60044d174f0abdefebe08',
+  'f9yKnlNMgKhF-CxOgHBsXkxfViCCkC6KwTv6Uj2Fcjw=',
+  'fb0SNPtMpQlzBQ90_PB7vCu34WpiSUJbNKocFkL2vIo=',
+  'gZLmSgmwumNdgf0eIfOSW44emctrLyFUaZapbk8eZ6s=',
+  'i39ZQ24NlUfWI0MhkbtvHTzGnWMVdr-aC2aOjvHPVg4=',
+  'k07JiWnb-S7qgd9gDQNgqo-LYMe03RX8fR0TXQ-SpG4=',
+  'nSEEkYxrThpZfLoPNOzMp6HWekvutAIYmADElDe1J6I=',
+  'pIqdA3pM1la1gKzxOmAcpLmTh3yXBrL9mGTz_hGj5xE=',
+  'qezoK6du9n3PF4dl4aq5r7LeXz_sV3xOVpFzVVgjNsE=',
+  'rY96Bunn-69CnNe5X_e5CJLwgCJnN6rcbUisecs8kkQ=',
+  'sNVNYDBzUDN2J9OFVJdLJlryBLzRZBLl-4MTNoPF1tA=',
+  'uvpOG9DrldeqIGNaqfwjFdMw_CcfXKfiEjYf7RXdeL0=',
+  'v2yF7lY2L5rtorTtTmYFsoMb9dBPKs5M1y9cUKxcI1M=',
+];
+
+export const createStaticBlobStorage = (): BlobStorage => {
+  return {
+    name: 'static',
+    readonly: true,
+    get: async (key: string) => {
+      const isStaticResource =
+        predefinedStaticFiles.includes(key) || key.startsWith('/static/');
+
+      if (!isStaticResource) {
+        return undefined;
+      }
+
+      const path = key.startsWith('/static/') ? key : `/static/${key}`;
+      const response = await fetch(path);
+
+      if (response.ok) {
+        return await response.blob();
+      }
+
+      return undefined;
+    },
+    set: async () => {
+      // ignore
+    },
+    delete: async () => {
+      // ignore
+    },
+    list: async () => {
+      // ignore
+      return [];
+    },
+  };
+};