feat: init affine blob storage (#2045)

packages/component/package.json

@@ -4,7 +4,7 @@
   "main": "./src/index.ts",
   "scripts": {
     "storybook": "storybook dev -p 6006",
-    "build-storybook": "storybook build",
+    "build-storybook": "NODE_OPTIONS=--max_old_space_size=4096 storybook build",
     "test-storybook": "test-storybook"
   },
   "exports": {
@@ -48,11 +48,11 @@
     "react-is": "^18.2.0"
   },
   "devDependencies": {
-    "@blocksuite/blocks": "0.0.0-20230416194015-c6ae6f0f-nightly",
-    "@blocksuite/editor": "0.0.0-20230416194015-c6ae6f0f-nightly",
-    "@blocksuite/global": "0.0.0-20230416194015-c6ae6f0f-nightly",
+    "@blocksuite/blocks": "0.0.0-20230420070759-dbe39fdf-nightly",
+    "@blocksuite/editor": "0.0.0-20230420070759-dbe39fdf-nightly",
+    "@blocksuite/global": "0.0.0-20230420070759-dbe39fdf-nightly",
     "@blocksuite/icons": "^2.1.10",
-    "@blocksuite/store": "0.0.0-20230416194015-c6ae6f0f-nightly",
+    "@blocksuite/store": "0.0.0-20230420070759-dbe39fdf-nightly",
     "@storybook/addon-actions": "^7.0.5",
     "@storybook/addon-coverage": "^0.0.8",
     "@storybook/addon-essentials": "^7.0.5",

packages/component/src/components/block-suite-editor/index.stories.tsx

@@ -2,7 +2,7 @@
 import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
 import type { EditorContainer } from '@blocksuite/editor';
 import type { Page } from '@blocksuite/store';
-import { Workspace } from '@blocksuite/store';
+import { createMemoryStorage, Workspace } from '@blocksuite/store';
 import { expect } from '@storybook/jest';
 import type { Meta, StoryFn } from '@storybook/react';
 import { useState } from 'react';
@@ -29,8 +29,9 @@ function initPage(page: Page): void {
 
 const blockSuiteWorkspace = new Workspace({
   id: 'test',
-  blobOptionsGetter: () => void 0,
+  blobStorages: [createMemoryStorage],
 });
+
 blockSuiteWorkspace.register(AffineSchemas).register(__unstableSchemas);
 const page = blockSuiteWorkspace.createPage('page0');
 initPage(page);

packages/component/src/components/workspace-list/index.stories.tsx

@@ -18,19 +18,28 @@ export const Default = () => {
       {
         id: '1',
         flavour: WorkspaceFlavour.LOCAL,
-        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace('1'),
+        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace(
+          '1',
+          WorkspaceFlavour.LOCAL
+        ),
         providers: [],
       },
       {
         id: '2',
         flavour: WorkspaceFlavour.LOCAL,
-        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace('2'),
+        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace(
+          '2',
+          WorkspaceFlavour.LOCAL
+        ),
         providers: [],
       },
       {
         id: '3',
         flavour: WorkspaceFlavour.LOCAL,
-        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace('3'),
+        blockSuiteWorkspace: createEmptyBlockSuiteWorkspace(
+          '3',
+          WorkspaceFlavour.LOCAL
+        ),
         providers: [],
       },
     ] satisfies WorkspaceListProps['items'];

packages/component/src/stories/ShareMenu.stories.tsx

@@ -31,7 +31,10 @@ function initPage(page: Page): void {
   page.resetHistory();
 }
 
-const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace('test-workspace');
+const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
+  'test-workspace',
+  WorkspaceFlavour.LOCAL
+);
 
 initPage(blockSuiteWorkspace.createPage('page0'));
 initPage(blockSuiteWorkspace.createPage('page1'));

packages/component/src/stories/WorkspaceAvatar.stories.tsx

@@ -21,7 +21,6 @@ export default {
 
 const basicBlockSuiteWorkspace = new Workspace({
   id: 'blocksuite-local',
-  blobOptionsGetter: (_: string) => undefined,
 });
 
 basicBlockSuiteWorkspace.meta.setName('Hello World');
@@ -46,19 +45,17 @@ Basic.args = {
 
 const avatarBlockSuiteWorkspace = new Workspace({
   id: 'blocksuite-local',
-  blobOptionsGetter: (_: string) => undefined,
 });
 
 avatarBlockSuiteWorkspace.meta.setName('Hello World');
-avatarBlockSuiteWorkspace.blobs.then(async blobs => {
-  if (blobs) {
-    const buffer = await (
-      await fetch(new URL('@affine-test/fixtures/smile.png', import.meta.url))
-    ).arrayBuffer();
-    const id = await blobs.set(new Blob([buffer], { type: 'image/png' }));
+fetch(new URL('@affine-test/fixtures/smile.png', import.meta.url))
+  .then(res => res.arrayBuffer())
+  .then(async buffer => {
+    const id = await avatarBlockSuiteWorkspace.blobs.set(
+      new Blob([buffer], { type: 'image/png' })
+    );
     avatarBlockSuiteWorkspace.meta.setAvatar(id);
-  }
-});
+  });
 
 export const BlobExample: StoryFn<WorkspaceAvatarProps> = props => {
   return (

packages/env/package.json

@@ -4,7 +4,7 @@
   "main": "./src/index.ts",
   "module": "./src/index.ts",
   "devDependencies": {
-    "@blocksuite/global": "0.0.0-20230416194015-c6ae6f0f-nightly",
+    "@blocksuite/global": "0.0.0-20230420070759-dbe39fdf-nightly",
     "next": "=13.2.3",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",

packages/hooks/src/use-block-suite-workspace-avatar-url.ts

@@ -14,8 +14,9 @@ export function useBlockSuiteWorkspaceAvatarUrl(
     fetcher: async avatar => {
       assertExists(blockSuiteWorkspace);
       const blobs = await blockSuiteWorkspace.blobs;
-      if (blobs) {
-        return blobs.get(avatar);
+      const blob = await blobs.get(avatar);
+      if (blob) {
+        return URL.createObjectURL(blob);
       }
       return null;
     },
@@ -27,7 +28,6 @@ export function useBlockSuiteWorkspaceAvatarUrl(
       assertExists(blockSuiteWorkspace);
       const blob = new Blob([file], { type: file.type });
       const blobs = await blockSuiteWorkspace.blobs;
-      assertExists(blobs);
       const blobId = await blobs.set(blob);
       blockSuiteWorkspace.meta.setAvatar(blobId);
       await mutate(blobId);

packages/jotai/package.json

@@ -7,10 +7,10 @@
     "jotai": "^2.0.4"
   },
   "devDependencies": {
-    "@blocksuite/blocks": "0.0.0-20230416194015-c6ae6f0f-nightly",
-    "@blocksuite/editor": "0.0.0-20230416194015-c6ae6f0f-nightly",
-    "@blocksuite/global": "0.0.0-20230416194015-c6ae6f0f-nightly",
-    "@blocksuite/store": "0.0.0-20230416194015-c6ae6f0f-nightly",
+    "@blocksuite/blocks": "0.0.0-20230420070759-dbe39fdf-nightly",
+    "@blocksuite/editor": "0.0.0-20230420070759-dbe39fdf-nightly",
+    "@blocksuite/global": "0.0.0-20230420070759-dbe39fdf-nightly",
+    "@blocksuite/store": "0.0.0-20230420070759-dbe39fdf-nightly",
     "lottie-web": "^5.11.0"
   },
   "peerDependencies": {

packages/workspace/package.json

@@ -3,6 +3,7 @@
   "private": true,
   "exports": {
     "./atom": "./src/atom.ts",
+    "./blob": "./src/blob/index.ts",
     "./utils": "./src/utils.ts",
     "./type": "./src/type.ts",
     "./local/crud": "./src/local/crud.ts",

packages/workspace/src/affine/__tests__/api.spec.ts

@@ -8,6 +8,7 @@ import { readFile } from 'node:fs/promises';
 import { MessageCode } from '@affine/env/constant';
 import { createStatusApis } from '@affine/workspace/affine/api/status';
 import { KeckProvider } from '@affine/workspace/affine/keck';
+import { WorkspaceFlavour } from '@affine/workspace/type';
 import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
 import user1 from '@affine-test/fixtures/built-in-user1.json';
 import user2 from '@affine-test/fixtures/built-in-user2.json';
@@ -119,7 +120,7 @@ async function createWorkspace(
 ): Promise<string> {
   const workspace = createEmptyBlockSuiteWorkspace(
     faker.datatype.uuid(),
-    _ => undefined
+    WorkspaceFlavour.LOCAL
   );
   if (callback) {
     callback(workspace);
@@ -408,9 +409,15 @@ describe('api', () => {
         );
       });
       const binary = await workspaceApis.downloadWorkspace(id, false);
-      const workspace = createEmptyBlockSuiteWorkspace(id, () => undefined);
+      const workspace = createEmptyBlockSuiteWorkspace(
+        id,
+        WorkspaceFlavour.LOCAL
+      );
       Workspace.Y.applyUpdate(workspace.doc, new Uint8Array(binary));
-      const workspace2 = createEmptyBlockSuiteWorkspace(id, () => undefined);
+      const workspace2 = createEmptyBlockSuiteWorkspace(
+        id,
+        WorkspaceFlavour.LOCAL
+      );
       {
         const wsUrl = `ws://127.0.0.1:3000/api/sync/`;
         const provider = new KeckProvider(wsUrl, workspace.id, workspace.doc, {
@@ -459,7 +466,7 @@ describe('api', () => {
         );
         const publicWorkspace = createEmptyBlockSuiteWorkspace(
           id,
-          () => undefined
+          WorkspaceFlavour.LOCAL
         );
         Workspace.Y.applyUpdate(publicWorkspace.doc, new Uint8Array(binary));
         const publicPage = publicWorkspace.getPage(pageId) as Page;

packages/workspace/src/affine/__tests__/sync.spec.ts

@@ -1,3 +1,8 @@
+/**
+ * @vitest-environment happy-dom
+ */
+import 'fake-indexeddb/auto';
+
 import type { Workspace } from '@affine/workspace/affine/api';
 import {
   createWorkspaceApis,
@@ -6,6 +11,7 @@ import {
 import { KeckProvider } from '@affine/workspace/affine/keck';
 import type { LoginResponse } from '@affine/workspace/affine/login';
 import { loginResponseSchema } from '@affine/workspace/affine/login';
+import { WorkspaceFlavour } from '@affine/workspace/type';
 import { createEmptyBlockSuiteWorkspace } from '@affine/workspace/utils';
 import user1 from '@affine-test/fixtures/built-in-user1.json';
 import user2 from '@affine-test/fixtures/built-in-user2.json';
@@ -80,11 +86,17 @@ describe('ydoc sync', () => {
       const binary = await workspaceApis.downloadWorkspace(root.id);
       const workspace1 = createEmptyBlockSuiteWorkspace(
         root.id,
-        (k: string) => ({ api: '/api/workspace', token: user1Token.token }[k])
+        WorkspaceFlavour.AFFINE,
+        {
+          workspaceApis,
+        }
       );
       const workspace2 = createEmptyBlockSuiteWorkspace(
         root.id,
-        (k: string) => ({ api: '/api/workspace', token: user2Token.token }[k])
+        WorkspaceFlavour.AFFINE,
+        {
+          workspaceApis,
+        }
       );
       BlockSuiteWorkspace.Y.applyUpdate(workspace1.doc, new Uint8Array(binary));
       BlockSuiteWorkspace.Y.applyUpdate(workspace2.doc, new Uint8Array(binary));

packages/workspace/src/blob/index.ts

@@ -0,0 +1,106 @@
+import type { BlobStorage } from '@blocksuite/store';
+import { createIndexeddbStorage } from '@blocksuite/store';
+import { openDB } from 'idb';
+import type { DBSchema } from 'idb/build/entry';
+
+import type { createWorkspaceApis } from '../affine/api';
+
+type UploadingBlob = {
+  key: string;
+  arrayBuffer: ArrayBuffer;
+  type: string;
+};
+
+interface AffineBlob extends DBSchema {
+  uploading: {
+    key: string;
+    value: UploadingBlob;
+  };
+  // todo: migrate blob storage from `createIndexeddbStorage`
+}
+
+export const createAffineBlobStorage = (
+  workspaceId: string,
+  workspaceApis: ReturnType<typeof createWorkspaceApis>
+): BlobStorage => {
+  const storage = createIndexeddbStorage(workspaceId);
+  const dbPromise = openDB<AffineBlob>('affine-blob', 1, {
+    upgrade(db) {
+      db.createObjectStore('uploading', { keyPath: 'key' });
+    },
+  });
+  dbPromise.then(async db => {
+    const t = db.transaction('uploading', 'readwrite').objectStore('uploading');
+    await t.getAll().then(blobs =>
+      blobs.map(({ arrayBuffer, type }) =>
+        workspaceApis.uploadBlob(workspaceId, arrayBuffer, type).then(key => {
+          const t = db
+            .transaction('uploading', 'readwrite')
+            .objectStore('uploading');
+          return t.delete(key);
+        })
+      )
+    );
+  });
+  return {
+    crud: {
+      get: async key => {
+        const blob = await storage.crud.get(key);
+        if (!blob) {
+          const buffer = await workspaceApis.getBlob(workspaceId, key);
+          return new Blob([buffer]);
+        } else {
+          return blob;
+        }
+      },
+      set: async (key, value) => {
+        const db = await dbPromise;
+        const t = db
+          .transaction('uploading', 'readwrite')
+          .objectStore('uploading');
+        let uploaded = false;
+        t.put({
+          key,
+          arrayBuffer: await value.arrayBuffer(),
+          type: value.type,
+        }).then(() => {
+          // delete the uploading blob after uploaded
+          if (uploaded) {
+            const t = db
+              .transaction('uploading', 'readwrite')
+              .objectStore('uploading');
+            t.delete(key);
+          }
+        });
+        await Promise.all([
+          storage.crud.set(key, value),
+          workspaceApis
+            .uploadBlob(workspaceId, await value.arrayBuffer(), value.type)
+            .then(async () => {
+              uploaded = true;
+              const t = db
+                .transaction('uploading', 'readwrite')
+                .objectStore('uploading');
+              // delete the uploading blob after uploaded
+              if (await t.get(key)) {
+                await t.delete(key);
+              }
+            }),
+        ]);
+        return key;
+      },
+      delete: async (key: string) => {
+        await Promise.all([
+          storage.crud.delete(key),
+          // we don't support deleting a blob in API?
+          // workspaceApis.deleteBlob(workspaceId, key)
+        ]);
+      },
+      list: async () => {
+        const blobs = await storage.crud.list();
+        // we don't support listing blobs in API?
+        return [...blobs];
+      },
+    },
+  };
+};

packages/workspace/src/local/crud.ts

@@ -43,7 +43,7 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.LOCAL> = {
     }
     const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
       id,
-      (_: string) => undefined
+      WorkspaceFlavour.LOCAL
     );
     const workspace: LocalWorkspace = {
       id,
@@ -62,7 +62,7 @@ export const CRUD: WorkspaceCRUD<WorkspaceFlavour.LOCAL> = {
     const id = nanoid();
     const blockSuiteWorkspace = createEmptyBlockSuiteWorkspace(
       id,
-      (_: string) => undefined
+      WorkspaceFlavour.LOCAL
     );
     BlockSuiteWorkspace.Y.applyUpdateV2(blockSuiteWorkspace.doc, binary);
     const persistence = createIndexedDBProvider(id, blockSuiteWorkspace.doc);

packages/workspace/src/utils.ts

@@ -1,16 +1,47 @@
+import type { createWorkspaceApis } from '@affine/workspace/affine/api';
+import { createAffineBlobStorage } from '@affine/workspace/blob';
 import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
-import type { BlobOptionsGetter, Generator } from '@blocksuite/store';
-import { Workspace } from '@blocksuite/store';
+import type { Generator } from '@blocksuite/store';
+import { createIndexeddbStorage, Workspace } from '@blocksuite/store';
+
+import { WorkspaceFlavour } from './type';
 
 const hashMap = new Map<string, Workspace>();
-export const createEmptyBlockSuiteWorkspace = (
+
+export function createEmptyBlockSuiteWorkspace(
   id: string,
-  blobOptionsGetter?: BlobOptionsGetter,
-  config?: {
+  flavour: WorkspaceFlavour.AFFINE,
+  config: {
+    workspaceApis: ReturnType<typeof createWorkspaceApis>;
     cachePrefix?: string;
     idGenerator?: Generator;
   }
-): Workspace => {
+): Workspace;
+export function createEmptyBlockSuiteWorkspace(
+  id: string,
+  flavour: WorkspaceFlavour.LOCAL,
+  config?: {
+    workspaceApis?: ReturnType<typeof createWorkspaceApis>;
+    cachePrefix?: string;
+    idGenerator?: Generator;
+  }
+): Workspace;
+export function createEmptyBlockSuiteWorkspace(
+  id: string,
+  flavour: WorkspaceFlavour,
+  config?: {
+    workspaceApis?: ReturnType<typeof createWorkspaceApis>;
+    cachePrefix?: string;
+    idGenerator?: Generator;
+  }
+): Workspace {
+  if (
+    flavour === WorkspaceFlavour.AFFINE &&
+    !config?.workspaceApis?.getBlob &&
+    !config?.workspaceApis?.uploadBlob
+  ) {
+    throw new Error('workspaceApis is required for affine flavour');
+  }
   const prefix: string = config?.cachePrefix ?? '';
   const cacheKey = `${prefix}${id}`;
   if (hashMap.has(cacheKey)) {
@@ -20,11 +51,16 @@ export const createEmptyBlockSuiteWorkspace = (
   const workspace = new Workspace({
     id,
     isSSR: typeof window === 'undefined',
-    blobOptionsGetter,
+    blobStorages:
+      flavour === WorkspaceFlavour.AFFINE
+        ? [id => createAffineBlobStorage(id, config!.workspaceApis!)]
+        : typeof window === 'undefined'
+        ? []
+        : [createIndexeddbStorage],
     idGenerator,
   })
     .register(AffineSchemas)
     .register(__unstableSchemas);
   hashMap.set(cacheKey, workspace);
   return workspace;
-};
+}

packages/y-indexeddb/package.json

@@ -26,8 +26,8 @@
     "idb": "^7.1.1"
   },
   "devDependencies": {
-    "@blocksuite/blocks": "0.0.0-20230416194015-c6ae6f0f-nightly",
-    "@blocksuite/store": "0.0.0-20230416194015-c6ae6f0f-nightly",
+    "@blocksuite/blocks": "0.0.0-20230420070759-dbe39fdf-nightly",
+    "@blocksuite/store": "0.0.0-20230420070759-dbe39fdf-nightly",
     "vite": "^4.2.1",
     "vite-plugin-dts": "^2.2.0",
     "y-indexeddb": "^9.0.10"