feat: multipart blob sync support (#14138)

## Summary by CodeRabbit * **New Features** * Flexible blob uploads: GRAPHQL, presigned, and multipart flows with per‑part URLs, abort/complete operations, presigned proxy endpoints, and nightly cleanup of expired pending uploads. * **API / Schema** * GraphQL additions: new types, mutations, enum and error to manage upload lifecycle (create, complete, abort, get part URL). * **Database** * New blob status enum and columns (status, upload_id); listing now defaults to completed blobs. * **Localization** * Added user-facing message: "Blob is invalid." * **Tests** * Expanded unit and end‑to‑end coverage for upload flows, proxy behavior, multipart and provider integrations. <sub>✏️ Tip: You can customize this high-level summary in your review settings.</sub>
2026-07-25 14:28:51 +08:00 · 2025-12-23 22:09:21 +08:00
parent a9937e18b6
commit 76524084d1
36 changed files with 2880 additions and 33 deletions
@@ -1,6 +1,11 @@
 import { UserFriendlyError } from '@affine/error';
 import {
+  abortBlobUploadMutation,
+  BlobUploadMethod,
+  completeBlobUploadMutation,
+  createBlobUploadMutation,
  deleteBlobMutation,
+  getBlobUploadPartUrlMutation,
  listBlobsQuery,
  releaseDeletedBlobsMutation,
  setBlobMutation,
@@ -21,6 +26,7 @@ interface CloudBlobStorageOptions {
 }

 const SHOULD_MANUAL_REDIRECT = BUILD_CONFIG.isAndroid || BUILD_CONFIG.isIOS;
+const UPLOAD_REQUEST_TIMEOUT = 0;

 export class CloudBlobStorage extends BlobStorageBase {
  static readonly identifier = 'CloudBlobStorage';
@@ -97,16 +103,69 @@ export class CloudBlobStorage extends BlobStorageBase {
      if (blob.data.byteLength > blobSizeLimit) {
        throw new OverSizeError(this.humanReadableBlobSizeLimitCache);
      }
-      await this.connection.gql({
-        query: setBlobMutation,
+
+      const init = await this.connection.gql({
+        query: createBlobUploadMutation,
        variables: {
          workspaceId: this.options.id,
-          blob: new File([blob.data], blob.key, { type: blob.mime }),
-        },
-        context: {
-          signal,
+          key: blob.key,
+          size: blob.data.byteLength,
+          mime: blob.mime,
        },
+        context: { signal },
      });
+
+      const upload = init.createBlobUpload;
+      if (upload.alreadyUploaded) {
+        return;
+      }
+      if (upload.method === BlobUploadMethod.GRAPHQL) {
+        await this.uploadViaGraphql(blob, signal);
+        return;
+      }
+
+      if (upload.method === BlobUploadMethod.PRESIGNED) {
+        try {
+          await this.uploadViaPresigned(
+            upload.uploadUrl!,
+            upload.headers,
+            blob.data,
+            signal
+          );
+          await this.completeUpload(blob.key, undefined, undefined, signal);
+          return;
+        } catch {
+          await this.uploadViaGraphql(blob, signal);
+          return;
+        }
+      }
+
+      if (upload.method === BlobUploadMethod.MULTIPART) {
+        try {
+          const parts = await this.uploadViaMultipart(
+            blob.key,
+            upload.uploadId!,
+            upload.partSize!,
+            blob.data,
+            upload.uploadedParts,
+            signal
+          );
+          await this.completeUpload(blob.key, upload.uploadId!, parts, signal);
+          return;
+        } catch {
+          if (upload.uploadId) {
+            await this.tryAbortMultipartUpload(
+              blob.key,
+              upload.uploadId,
+              signal
+            );
+          }
+          await this.uploadViaGraphql(blob, signal);
+          return;
+        }
+      }
+
+      await this.uploadViaGraphql(blob, signal);
    } catch (err) {
      const userFriendlyError = UserFriendlyError.fromAny(err);
      if (userFriendlyError.is('STORAGE_QUOTA_EXCEEDED')) {
@@ -151,6 +210,159 @@ export class CloudBlobStorage extends BlobStorageBase {
    }));
  }

+  private async uploadViaGraphql(blob: BlobRecord, signal?: AbortSignal) {
+    await this.connection.gql({
+      query: setBlobMutation,
+      variables: {
+        workspaceId: this.options.id,
+        blob: new File([blob.data], blob.key, { type: blob.mime }),
+      },
+      context: { signal },
+      timeout: UPLOAD_REQUEST_TIMEOUT,
+    });
+  }
+
+  private async uploadViaPresigned(
+    uploadUrl: string,
+    headers: Record<string, string> | null | undefined,
+    data: Uint8Array,
+    signal?: AbortSignal
+  ) {
+    const res = await this.fetchWithTimeout(uploadUrl, {
+      method: 'PUT',
+      headers: headers ?? undefined,
+      body: data,
+      signal,
+      timeout: UPLOAD_REQUEST_TIMEOUT,
+    });
+    if (!res.ok) {
+      throw new Error(`Presigned upload failed with status ${res.status}`);
+    }
+  }
+
+  private async uploadViaMultipart(
+    key: string,
+    uploadId: string,
+    partSize: number,
+    data: Uint8Array,
+    uploadedParts: { partNumber: number; etag: string }[] | null | undefined,
+    signal?: AbortSignal
+  ) {
+    const partsMap = new Map<number, string>();
+    for (const part of uploadedParts ?? []) {
+      partsMap.set(part.partNumber, part.etag);
+    }
+    const total = data.byteLength;
+    const totalParts = Math.ceil(total / partSize);
+
+    for (let partNumber = 1; partNumber <= totalParts; partNumber += 1) {
+      if (partsMap.has(partNumber)) {
+        continue;
+      }
+      const start = (partNumber - 1) * partSize;
+      const end = Math.min(start + partSize, total);
+      const chunk = data.subarray(start, end);
+
+      const part = await this.connection.gql({
+        query: getBlobUploadPartUrlMutation,
+        variables: { workspaceId: this.options.id, key, uploadId, partNumber },
+        context: { signal },
+      });
+
+      const res = await this.fetchWithTimeout(
+        part.getBlobUploadPartUrl.uploadUrl,
+        {
+          method: 'PUT',
+          headers: part.getBlobUploadPartUrl.headers ?? undefined,
+          body: chunk,
+          signal,
+          timeout: UPLOAD_REQUEST_TIMEOUT,
+        }
+      );
+      if (!res.ok) {
+        throw new Error(
+          `Multipart upload failed at part ${partNumber} with status ${res.status}`
+        );
+      }
+
+      const etag = res.headers.get('etag');
+      if (!etag) {
+        throw new Error(`Missing ETag for part ${partNumber}.`);
+      }
+      partsMap.set(partNumber, etag);
+    }
+
+    if (partsMap.size !== totalParts) {
+      throw new Error('Multipart upload has missing parts.');
+    }
+
+    return [...partsMap.entries()]
+      .sort((left, right) => left[0] - right[0])
+      .map(([partNumber, etag]) => ({ partNumber, etag }));
+  }
+
+  private async completeUpload(
+    key: string,
+    uploadId: string | undefined,
+    parts: { partNumber: number; etag: string }[] | undefined,
+    signal?: AbortSignal
+  ) {
+    await this.connection.gql({
+      query: completeBlobUploadMutation,
+      variables: { workspaceId: this.options.id, key, uploadId, parts },
+      context: { signal },
+      timeout: UPLOAD_REQUEST_TIMEOUT,
+    });
+  }
+
+  private async tryAbortMultipartUpload(
+    key: string,
+    uploadId: string,
+    signal?: AbortSignal
+  ) {
+    try {
+      await this.connection.gql({
+        query: abortBlobUploadMutation,
+        variables: { workspaceId: this.options.id, key, uploadId },
+        context: { signal },
+      });
+    } catch {}
+  }
+
+  private async fetchWithTimeout(
+    input: string,
+    init: RequestInit & { timeout?: number }
+  ) {
+    const externalSignal = init.signal;
+    if (externalSignal?.aborted) {
+      throw externalSignal.reason;
+    }
+
+    const abortController = new AbortController();
+    externalSignal?.addEventListener('abort', reason => {
+      abortController.abort(reason);
+    });
+
+    const timeout = init.timeout ?? 15000;
+    const timeoutId =
+      timeout > 0
+        ? setTimeout(() => {
+            abortController.abort(new Error('request timeout'));
+          }, timeout)
+        : undefined;
+
+    try {
+      return await globalThis.fetch(input, {
+        ...init,
+        signal: abortController.signal,
+      });
+    } finally {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+    }
+  }
+
  private humanReadableBlobSizeLimitCache: string | null = null;
  private blobSizeLimitCache: number | null = null;
  private blobSizeLimitCacheTime = 0;
@@ -19,9 +19,12 @@ export class HttpConnection extends DummyConnection {
    });

    const timeout = init?.timeout ?? 15000;
-    const timeoutId = setTimeout(() => {
-      abortController.abort(new Error('request timeout'));
-    }, timeout);
+    const timeoutId =
+      timeout > 0
+        ? setTimeout(() => {
+            abortController.abort(new Error('request timeout'));
+          }, timeout)
+        : undefined;

    const res = await globalThis
      .fetch(new URL(input, this.serverBaseUrl), {
@@ -43,7 +46,9 @@ export class HttpConnection extends DummyConnection {
          stacktrace: err.stack,
        });
      });
-    clearTimeout(timeoutId);
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
    if (!res.ok && res.status !== 404) {
      if (res.status === 413) {
        throw new UserFriendlyError({