feat(nbstore): add blob sync storage (#10752)

packages/common/nbstore/src/impls/cloud/blob.ts

@@ -1,11 +1,18 @@
+import { UserFriendlyError } from '@affine/error';
 import {
   deleteBlobMutation,
   listBlobsQuery,
   releaseDeletedBlobsMutation,
   setBlobMutation,
+  workspaceQuotaQuery,
 } from '@affine/graphql';
 
-import { type BlobRecord, BlobStorageBase } from '../../storage';
+import {
+  type BlobRecord,
+  BlobStorageBase,
+  OverCapacityError,
+  OverSizeError,
+} from '../../storage';
 import { HttpConnection } from './http';
 
 interface CloudBlobStorageOptions {
@@ -15,6 +22,7 @@ interface CloudBlobStorageOptions {
 
 export class CloudBlobStorage extends BlobStorageBase {
   static readonly identifier = 'CloudBlobStorage';
+  override readonly isReadonly = false;
 
   constructor(private readonly options: CloudBlobStorageOptions) {
     super();
@@ -22,7 +30,7 @@ export class CloudBlobStorage extends BlobStorageBase {
 
   readonly connection = new HttpConnection(this.options.serverBaseUrl);
 
-  override async get(key: string) {
+  override async get(key: string, signal?: AbortSignal) {
     const res = await this.connection.fetch(
       '/api/workspaces/' + this.options.id + '/blobs/' + key,
       {
@@ -30,6 +38,7 @@ export class CloudBlobStorage extends BlobStorageBase {
         headers: {
           'x-affine-version': BUILD_CONFIG.appVersion,
         },
+        signal,
       }
     );
 
@@ -52,14 +61,32 @@ export class CloudBlobStorage extends BlobStorageBase {
     }
   }
 
-  override async set(blob: BlobRecord) {
-    await this.connection.gql({
-      query: setBlobMutation,
-      variables: {
-        workspaceId: this.options.id,
-        blob: new File([blob.data], blob.key, { type: blob.mime }),
-      },
-    });
+  override async set(blob: BlobRecord, signal?: AbortSignal) {
+    try {
+      const blobSizeLimit = await this.getBlobSizeLimit();
+      if (blob.data.byteLength > blobSizeLimit) {
+        throw new OverSizeError();
+      }
+      await this.connection.gql({
+        query: setBlobMutation,
+        variables: {
+          workspaceId: this.options.id,
+          blob: new File([blob.data], blob.key, { type: blob.mime }),
+        },
+        context: {
+          signal,
+        },
+      });
+    } catch (err) {
+      const userFriendlyError = UserFriendlyError.fromAny(err);
+      if (userFriendlyError.is('BLOB_QUOTA_EXCEEDED')) {
+        throw new OverCapacityError();
+      }
+      if (userFriendlyError.is('CONTENT_TOO_LARGE')) {
+        throw new OverSizeError();
+      }
+      throw err;
+    }
   }
 
   override async delete(key: string, permanently: boolean) {
@@ -87,4 +114,28 @@ export class CloudBlobStorage extends BlobStorageBase {
       createdAt: new Date(blob.createdAt),
     }));
   }
+
+  private blobSizeLimitCache: number | null = null;
+  private blobSizeLimitCacheTime = 0;
+  private async getBlobSizeLimit() {
+    // If cache time is less than 120 seconds, return the cached value directly
+    if (
+      this.blobSizeLimitCache !== null &&
+      Date.now() - this.blobSizeLimitCacheTime < 120 * 1000
+    ) {
+      return this.blobSizeLimitCache;
+    }
+    try {
+      const res = await this.connection.gql({
+        query: workspaceQuotaQuery,
+        variables: { id: this.options.id },
+      });
+
+      this.blobSizeLimitCache = res.workspace.quota.blobLimit;
+      this.blobSizeLimitCacheTime = Date.now();
+      return this.blobSizeLimitCache;
+    } catch (err) {
+      throw UserFriendlyError.fromAny(err);
+    }
+  }
 }

packages/common/nbstore/src/impls/cloud/http.ts

@@ -1,3 +1,4 @@
+import { UserFriendlyError } from '@affine/error';
 import { gqlFetcherFactory } from '@affine/graphql';
 
 import { DummyConnection } from '../../connection';
@@ -29,19 +30,32 @@ export class HttpConnection extends DummyConnection {
         },
       })
       .catch(err => {
-        throw new Error('fetch error: ' + err);
+        throw new UserFriendlyError({
+          status: 504,
+          code: 'NETWORK_ERROR',
+          type: 'NETWORK_ERROR',
+          name: 'NETWORK_ERROR',
+          message: `Network error: ${err.message}`,
+          stacktrace: err.stack,
+        });
       });
     clearTimeout(timeoutId);
     if (!res.ok && res.status !== 404) {
-      let reason: string | any = '';
-      if (res.headers.get('Content-Type')?.includes('application/json')) {
-        try {
-          reason = JSON.stringify(await res.json());
-        } catch {
-          // ignore
-        }
+      if (res.status === 413) {
+        throw new UserFriendlyError({
+          status: 413,
+          code: 'CONTENT_TOO_LARGE',
+          type: 'CONTENT_TOO_LARGE',
+          name: 'CONTENT_TOO_LARGE',
+          message: 'Content too large',
+        });
+      } else if (
+        res.headers.get('Content-Type')?.startsWith('application/json')
+      ) {
+        throw UserFriendlyError.fromAny(await res.json());
+      } else {
+        throw UserFriendlyError.fromAny(await res.text());
       }
-      throw new Error('fetch error status: ' + res.status + ' ' + reason);
     }
     return res;
   };

packages/common/nbstore/src/impls/idb/blob-sync.ts

@@ -0,0 +1,36 @@
+import { share } from '../../connection';
+import { BlobSyncStorageBase } from '../../storage';
+import { IDBConnection, type IDBConnectionOptions } from './db';
+
+export class IndexedDBBlobSyncStorage extends BlobSyncStorageBase {
+  static readonly identifier = 'IndexedDBBlobSyncStorage';
+
+  readonly connection = share(new IDBConnection(this.options));
+
+  constructor(private readonly options: IDBConnectionOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection;
+  }
+
+  async setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void> {
+    const trx = this.db.inner.db.transaction('blobSync', 'readwrite');
+    await trx.store.put({
+      peer,
+      key: blobId,
+      uploadedAt,
+    });
+  }
+
+  async getBlobUploadedAt(peer: string, blobId: string): Promise<Date | null> {
+    const trx = this.db.inner.db.transaction('blobSync', 'readonly');
+    const record = await trx.store.get([peer, blobId]);
+    return record?.uploadedAt ?? null;
+  }
+}

packages/common/nbstore/src/impls/idb/blob.ts

@@ -8,6 +8,7 @@ import { IDBConnection, type IDBConnectionOptions } from './db';
 
 export class IndexedDBBlobStorage extends BlobStorageBase {
   static readonly identifier = 'IndexedDBBlobStorage';
+  override readonly isReadonly = false;
 
   readonly connection = share(new IDBConnection(this.options));
 

packages/common/nbstore/src/impls/idb/index.ts

@@ -1,9 +1,11 @@
 import type { StorageConstructor } from '..';
 import { IndexedDBBlobStorage } from './blob';
+import { IndexedDBBlobSyncStorage } from './blob-sync';
 import { IndexedDBDocStorage } from './doc';
 import { IndexedDBDocSyncStorage } from './doc-sync';
 
 export * from './blob';
+export * from './blob-sync';
 export * from './doc';
 export * from './doc-sync';
 
@@ -11,4 +13,5 @@ export const idbStorages = [
   IndexedDBDocStorage,
   IndexedDBBlobStorage,
   IndexedDBDocSyncStorage,
+  IndexedDBBlobSyncStorage,
 ] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/idb/schema.ts

@@ -36,6 +36,11 @@ Table(PeerClocks)
 | peer | docId |   clock   |  pushed   |
 |------|-------|-----------|-----------|
 | str  |  str  |   Date    |   Date    |
+
+Table(BlobSync)
+| peer | key | uploadedAt |
+|------|-----|------------|
+| str  | str |   Date     |
  */
 export interface DocStorageSchema extends DBSchema {
   snapshots: {
@@ -81,6 +86,17 @@ export interface DocStorageSchema extends DBSchema {
       deletedAt: Date | null;
     };
   };
+  blobSync: {
+    key: [string, string];
+    value: {
+      peer: string;
+      key: string;
+      uploadedAt: Date | null;
+    };
+    indexes: {
+      peer: string;
+    };
+  };
   blobData: {
     key: string;
     value: {
@@ -175,11 +191,19 @@ const init: Migrate = db => {
     autoIncrement: false,
   });
 };
+const initBlobSync: Migrate = db => {
+  const blobSync = db.createObjectStore('blobSync', {
+    keyPath: ['peer', 'key'],
+    autoIncrement: false,
+  });
+
+  blobSync.createIndex('peer', 'peer', { unique: false });
+};
 // END REGION
 
 // 1. all schema changed should be put in migrations
 // 2. order matters
-const migrations: Migrate[] = [init];
+const migrations: Migrate[] = [init, initBlobSync];
 
 export const migrator = {
   version: migrations.length,

packages/common/nbstore/src/impls/idb/v1/blob.ts

@@ -7,6 +7,7 @@ import { BlobIDBConnection, type BlobIDBConnectionOptions } from './db';
  */
 export class IndexedDBV1BlobStorage extends BlobStorageBase {
   static readonly identifier = 'IndexedDBV1BlobStorage';
+  override readonly isReadonly = true;
 
   constructor(private readonly options: BlobIDBConnectionOptions) {
     super();

packages/common/nbstore/src/impls/sqlite/blob-sync.ts

@@ -0,0 +1,32 @@
+import { share } from '../../connection';
+import { BlobSyncStorageBase } from '../../storage';
+import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
+
+export class SqliteBlobSyncStorage extends BlobSyncStorageBase {
+  static readonly identifier = 'SqliteBlobSyncStorage';
+
+  override connection = share(new NativeDBConnection(this.options));
+
+  constructor(private readonly options: SqliteNativeDBOptions) {
+    super();
+  }
+
+  get db() {
+    return this.connection.apis;
+  }
+
+  override async setBlobUploadedAt(
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ): Promise<void> {
+    await this.db.setBlobUploadedAt(peer, blobId, uploadedAt);
+  }
+
+  override async getBlobUploadedAt(
+    peer: string,
+    blobId: string
+  ): Promise<Date | null> {
+    return this.db.getBlobUploadedAt(peer, blobId);
+  }
+}

packages/common/nbstore/src/impls/sqlite/blob.ts

@@ -4,6 +4,7 @@ import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
 
 export class SqliteBlobStorage extends BlobStorageBase {
   static readonly identifier = 'SqliteBlobStorage';
+  override readonly isReadonly = false;
 
   override connection = share(new NativeDBConnection(this.options));
 

packages/common/nbstore/src/impls/sqlite/db.ts

@@ -13,67 +13,75 @@ export interface SqliteNativeDBOptions {
   readonly id: string;
 }
 
-export type NativeDBApis = {
-  connect(id: string): Promise<void>;
-  disconnect(id: string): Promise<void>;
-  pushUpdate(id: string, docId: string, update: Uint8Array): Promise<Date>;
-  getDocSnapshot(id: string, docId: string): Promise<DocRecord | null>;
-  setDocSnapshot(id: string, snapshot: DocRecord): Promise<boolean>;
-  getDocUpdates(id: string, docId: string): Promise<DocRecord[]>;
-  markUpdatesMerged(
+export interface NativeDBApis {
+  connect: (id: string) => Promise<void>;
+  disconnect: (id: string) => Promise<void>;
+  pushUpdate: (id: string, docId: string, update: Uint8Array) => Promise<Date>;
+  getDocSnapshot: (id: string, docId: string) => Promise<DocRecord | null>;
+  setDocSnapshot: (id: string, snapshot: DocRecord) => Promise<boolean>;
+  getDocUpdates: (id: string, docId: string) => Promise<DocRecord[]>;
+  markUpdatesMerged: (
     id: string,
     docId: string,
     updates: Date[]
-  ): Promise<number>;
-  deleteDoc(id: string, docId: string): Promise<void>;
-  getDocClocks(
-    id: string,
-    after?: Date | undefined | null
-  ): Promise<DocClock[]>;
-  getDocClock(id: string, docId: string): Promise<DocClock | null>;
-  getBlob(id: string, key: string): Promise<BlobRecord | null>;
-  setBlob(id: string, blob: BlobRecord): Promise<void>;
-  deleteBlob(id: string, key: string, permanently: boolean): Promise<void>;
-  releaseBlobs(id: string): Promise<void>;
-  listBlobs(id: string): Promise<ListedBlobRecord[]>;
-  getPeerRemoteClocks(id: string, peer: string): Promise<DocClock[]>;
-  getPeerRemoteClock(
+  ) => Promise<number>;
+  deleteDoc: (id: string, docId: string) => Promise<void>;
+  getDocClocks: (id: string, after?: Date | null) => Promise<DocClock[]>;
+  getDocClock: (id: string, docId: string) => Promise<DocClock | null>;
+  getBlob: (id: string, key: string) => Promise<BlobRecord | null>;
+  setBlob: (id: string, blob: BlobRecord) => Promise<void>;
+  deleteBlob: (id: string, key: string, permanently: boolean) => Promise<void>;
+  releaseBlobs: (id: string) => Promise<void>;
+  listBlobs: (id: string) => Promise<ListedBlobRecord[]>;
+  getPeerRemoteClocks: (id: string, peer: string) => Promise<DocClock[]>;
+  getPeerRemoteClock: (
     id: string,
     peer: string,
     docId: string
-  ): Promise<DocClock | null>;
-  setPeerRemoteClock(
+  ) => Promise<DocClock | null>;
+  setPeerRemoteClock: (
     id: string,
     peer: string,
     docId: string,
     clock: Date
-  ): Promise<void>;
-  getPeerPulledRemoteClocks(id: string, peer: string): Promise<DocClock[]>;
-  getPeerPulledRemoteClock(
+  ) => Promise<void>;
+  getPeerPulledRemoteClocks: (id: string, peer: string) => Promise<DocClock[]>;
+  getPeerPulledRemoteClock: (
     id: string,
     peer: string,
     docId: string
-  ): Promise<DocClock | null>;
-  setPeerPulledRemoteClock(
+  ) => Promise<DocClock | null>;
+  setPeerPulledRemoteClock: (
     id: string,
     peer: string,
     docId: string,
     clock: Date
-  ): Promise<void>;
-  getPeerPushedClocks(id: string, peer: string): Promise<DocClock[]>;
-  getPeerPushedClock(
+  ) => Promise<void>;
+  getPeerPushedClocks: (id: string, peer: string) => Promise<DocClock[]>;
+  getPeerPushedClock: (
     id: string,
     peer: string,
     docId: string
-  ): Promise<DocClock | null>;
-  setPeerPushedClock(
+  ) => Promise<DocClock | null>;
+  setPeerPushedClock: (
     id: string,
     peer: string,
     docId: string,
     clock: Date
-  ): Promise<void>;
-  clearClocks(id: string): Promise<void>;
-};
+  ) => Promise<void>;
+  clearClocks: (id: string) => Promise<void>;
+  setBlobUploadedAt: (
+    id: string,
+    peer: string,
+    blobId: string,
+    uploadedAt: Date | null
+  ) => Promise<void>;
+  getBlobUploadedAt: (
+    id: string,
+    peer: string,
+    blobId: string
+  ) => Promise<Date | null>;
+}
 
 type NativeDBApisWrapper = NativeDBApis extends infer APIs
   ? {

packages/common/nbstore/src/impls/sqlite/index.ts

@@ -1,9 +1,11 @@
 import type { StorageConstructor } from '..';
 import { SqliteBlobStorage } from './blob';
+import { SqliteBlobSyncStorage } from './blob-sync';
 import { SqliteDocStorage } from './doc';
 import { SqliteDocSyncStorage } from './doc-sync';
 
 export * from './blob';
+export * from './blob-sync';
 export { bindNativeDBApis, type NativeDBApis } from './db';
 export * from './doc';
 export * from './doc-sync';
@@ -12,4 +14,5 @@ export const sqliteStorages = [
   SqliteDocStorage,
   SqliteBlobStorage,
   SqliteDocSyncStorage,
+  SqliteBlobSyncStorage,
 ] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/sqlite/v1/blob.ts

@@ -9,6 +9,7 @@ import { apis } from './db';
 export class SqliteV1BlobStorage extends BlobStorageBase {
   static identifier = 'SqliteV1BlobStorage';
   override connection = new DummyConnection();
+  override readonly isReadonly = true;
 
   constructor(private readonly options: { type: SpaceType; id: string }) {
     super();