From 00576e1e7842fb63095cdc5d7a236321957b550c Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:43:43 +0800 Subject: [PATCH] feat(server): improve blob sync (#15367) ## Summary by CodeRabbit - **New Features** - Standardized `usePresignedURL` configuration for AWS S3 and Cloudflare R2 (including `enabled`, `urlPrefix`, and `signKey`). - Storage upload URL generation now supports both direct provider presigning and server-mediated proxying based on configuration. - **Bug Fixes** - Tightened upload and multipart validation (content type/length checks, header vs query consistency, and stricter expiration handling). - Improved fallback behavior when direct upload URL initialization fails. - **Tests** - Updated R2 storage proxy end-to-end coverage to match the new URL/token behavior. - **Documentation** - Refreshed self-hosted JSON schema guidance for upload URL settings. --- .docker/selfhost/schema.json | 162 +++++++++----- .../__tests__/e2e/storage/r2-proxy.spec.ts | 203 ++++++++++-------- .../src/base/storage/providers/index.ts | 53 +++-- .../backend/server/src/base/storage/utils.ts | 18 ++ .../server/src/core/storage/r2-proxy.ts | 131 +++++------ .../server/src/core/storage/wrappers/blob.ts | 165 +++++++++----- .../src/core/workspaces/resolvers/blob.ts | 105 ++++----- 7 files changed, 490 insertions(+), 347 deletions(-) diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index 8272b06953..b3b4bc7991 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -483,6 +483,24 @@ "type": "string" } } + }, + "usePresignedURL": { + "type": "object", + "description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether browser upload URLs are enabled." + }, + "urlPrefix": { + "type": "string", + "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + }, + "signKey": { + "type": "string", + "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + } + } } } } @@ -549,6 +567,24 @@ } } }, + "usePresignedURL": { + "type": "object", + "description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether browser upload URLs are enabled." + }, + "urlPrefix": { + "type": "string", + "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + }, + "signKey": { + "type": "string", + "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + } + } + }, "accountId": { "type": "string", "description": "The account id for the cloudflare r2 storage provider." @@ -560,24 +596,6 @@ "eu" ], "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." - }, - "usePresignedURL": { - "type": "object", - "description": "The presigned url config for the cloudflare r2 storage provider.", - "properties": { - "enabled": { - "type": "boolean", - "description": "Whether to use presigned url for the cloudflare r2 storage provider." - }, - "urlPrefix": { - "type": "string", - "description": "The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: \"https://storage.example.com\"\nExample rule: is_timed_hmac_valid_v0(\"your_secret\", http.request.uri, 10800, http.request.timestamp.sec, 6)" - }, - "signKey": { - "type": "string", - "description": "The presigned key for the cloudflare r2 storage provider." - } - } } } } @@ -712,6 +730,24 @@ "type": "string" } } + }, + "usePresignedURL": { + "type": "object", + "description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether browser upload URLs are enabled." + }, + "urlPrefix": { + "type": "string", + "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + }, + "signKey": { + "type": "string", + "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + } + } } } } @@ -778,6 +814,24 @@ } } }, + "usePresignedURL": { + "type": "object", + "description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether browser upload URLs are enabled." + }, + "urlPrefix": { + "type": "string", + "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + }, + "signKey": { + "type": "string", + "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + } + } + }, "accountId": { "type": "string", "description": "The account id for the cloudflare r2 storage provider." @@ -789,24 +843,6 @@ "eu" ], "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." - }, - "usePresignedURL": { - "type": "object", - "description": "The presigned url config for the cloudflare r2 storage provider.", - "properties": { - "enabled": { - "type": "boolean", - "description": "Whether to use presigned url for the cloudflare r2 storage provider." - }, - "urlPrefix": { - "type": "string", - "description": "The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: \"https://storage.example.com\"\nExample rule: is_timed_hmac_valid_v0(\"your_secret\", http.request.uri, 10800, http.request.timestamp.sec, 6)" - }, - "signKey": { - "type": "string", - "description": "The presigned key for the cloudflare r2 storage provider." - } - } } } } @@ -1391,6 +1427,24 @@ "type": "string" } } + }, + "usePresignedURL": { + "type": "object", + "description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether browser upload URLs are enabled." + }, + "urlPrefix": { + "type": "string", + "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + }, + "signKey": { + "type": "string", + "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + } + } } } } @@ -1457,6 +1511,24 @@ } } }, + "usePresignedURL": { + "type": "object", + "description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether browser upload URLs are enabled." + }, + "urlPrefix": { + "type": "string", + "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured." + }, + "signKey": { + "type": "string", + "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin." + } + } + }, "accountId": { "type": "string", "description": "The account id for the cloudflare r2 storage provider." @@ -1468,24 +1540,6 @@ "eu" ], "description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets." - }, - "usePresignedURL": { - "type": "object", - "description": "The presigned url config for the cloudflare r2 storage provider.", - "properties": { - "enabled": { - "type": "boolean", - "description": "Whether to use presigned url for the cloudflare r2 storage provider." - }, - "urlPrefix": { - "type": "string", - "description": "The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: \"https://storage.example.com\"\nExample rule: is_timed_hmac_valid_v0(\"your_secret\", http.request.uri, 10800, http.request.timestamp.sec, 6)" - }, - "signKey": { - "type": "string", - "description": "The presigned key for the cloudflare r2 storage provider." - } - } } } } diff --git a/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts b/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts index f705ffe2e1..64d9c3d7be 100644 --- a/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts @@ -1,4 +1,4 @@ -import { createHash, createHmac } from 'node:crypto'; +import { createHash } from 'node:crypto'; import { mock } from 'node:test'; import { @@ -9,9 +9,13 @@ import { type R2StorageConfig, SIGNED_URL_EXPIRED, type StorageProviderConfig, + URLHelper, } from '../../../base'; import { EntitlementService } from '../../../core/entitlement'; -import { MULTIPART_THRESHOLD } from '../../../core/storage/constants'; +import { + MULTIPART_PART_SIZE, + MULTIPART_THRESHOLD, +} from '../../../core/storage/constants'; import { StorageRuntimeProvider } from '../../../core/storage-runtime'; import { SubscriptionPlan, @@ -39,7 +43,6 @@ class MockStorageRuntime { async providerCapabilities() { const storage = app.get(Config).storages.blob.storage; - const usePresignedURL = (storage.config as R2StorageConfig).usePresignedURL; if (storage.provider !== 'cloudflare-r2') { return { put: true, @@ -64,7 +67,7 @@ class MockStorageRuntime { presignPut: true, presignGet: false, multipartDirect: true, - proxyUpload: !!usePresignedURL?.enabled, + proxyUpload: false, assetpack: false, serverMediatedOnly: false, }; @@ -75,31 +78,17 @@ class MockStorageRuntime { key: string, metadata: { contentType?: string; contentLength?: number } = {} ) { - const storage = app.get(Config).storages.blob.storage; - const r2 = storage.config as R2StorageConfig; - if (!r2.usePresignedURL?.enabled) { - return { - url: 'https://test-bucket.r2.example.com/object?X-Amz-Algorithm=AWS4-HMAC-SHA256', - headers: {}, - expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000), - }; - } - const [workspaceId, blobKey] = key.split('/'); - return createProxyUrl( - PROXY_UPLOAD_PATH, - [ - workspaceId, - blobKey, - metadata.contentType ?? 'application/octet-stream', - metadata.contentLength, - ], - { - workspaceId, - key: blobKey, - contentType: metadata.contentType ?? 'application/octet-stream', - contentLength: metadata.contentLength, - } + const url = new URL(`https://test-bucket.r2.example.com/${key}`); + url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256'); + url.searchParams.set( + 'X-Amz-SignedHeaders', + metadata.contentType ? 'content-type;host' : 'host' ); + return { + url: url.toString(), + headers: {}, + expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000), + }; } async createMultipartUpload() { @@ -116,17 +105,15 @@ class MockStorageRuntime { uploadId: string, partNumber: number ) { - const [workspaceId, blobKey] = key.split('/'); - return createProxyUrl( - PROXY_MULTIPART_PATH, - [workspaceId, blobKey, uploadId, partNumber], - { - workspaceId, - key: blobKey, - uploadId, - partNumber, - } - ); + const url = new URL(`https://test-bucket.r2.example.com/${key}`); + url.searchParams.set('uploadId', uploadId); + url.searchParams.set('partNumber', partNumber.toString()); + url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256'); + return { + url: url.toString(), + headers: {}, + expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000), + }; } async listMultipartUploadParts( @@ -390,25 +377,25 @@ e2e.serial('should proxy multipart upload and return etag', async t => { t.is(init.uploadId, 'upload-id'); t.deepEqual(init.uploadedParts, []); - const part = await getBlobUploadPartUrl(workspace.id, key, init.uploadId, 1); + const part = await getBlobUploadPartUrl(workspace.id, key, init.uploadId, 3); const partUrl = new URL(part.uploadUrl, app.url); t.is(partUrl.origin, 'https://cdn.example.com'); t.is(partUrl.pathname, PROXY_MULTIPART_PATH); - const payload = Buffer.from('part-body'); + const payload = Buffer.alloc(1024); const res = await app .PUT(partUrl.pathname + partUrl.search) .set('content-length', payload.length.toString()) .send(payload); t.is(res.status, 200); - t.is(res.get('etag'), 'etag-1'); + t.is(res.get('etag'), 'etag-3'); const calls = getRuntime().partCalls; t.is(calls.length, 1); t.is(calls[0].key, `${workspace.id}/${key}`); t.is(calls[0].uploadId, 'upload-id'); - t.is(calls[0].partNumber, 1); + t.is(calls[0].partNumber, 3); t.is(calls[0].contentLength, payload.length); t.deepEqual(calls[0].body, payload); }); @@ -432,7 +419,7 @@ e2e.serial( init1.uploadId, 1 ); - const payload = Buffer.from('part-body'); + const payload = Buffer.alloc(MULTIPART_PART_SIZE); const partUrl = new URL(part.uploadUrl, app.url); await app .PUT(partUrl.pathname + partUrl.search) @@ -482,7 +469,7 @@ e2e.serial('should reject upload when url is expired', async t => { ); const uploadUrl = new URL(init.uploadUrl, app.url); uploadUrl.searchParams.set( - 'exp', + 'expiresAt', (Math.floor(Date.now() / 1000) - 1).toString() ); @@ -497,16 +484,49 @@ e2e.serial('should reject upload when url is expired', async t => { t.is(getRuntime().putCalls.length, 0); }); +e2e.serial('should use graphql when upload urls are disabled', async t => { + await useR2Storage({ + enabled: false, + urlPrefix: undefined, + signKey: undefined, + }); + const { workspace } = await setupWorkspace(); + const buffer = Buffer.from('plain'); + + const init = await createBlobUpload( + workspace.id, + sha256Base64urlWithPadding(buffer), + buffer.length, + 'text/plain' + ); + + t.is(init.method, 'GRAPHQL'); +}); + +e2e.serial('should use the server origin with only a signing key', async t => { + await useR2Storage({ urlPrefix: undefined }); + const { workspace } = await setupWorkspace(); + const buffer = Buffer.from('server-proxy'); + + const init = await createBlobUpload( + workspace.id, + sha256Base64urlWithPadding(buffer), + buffer.length, + 'text/plain' + ); + + const uploadUrl = new URL(init.uploadUrl, app.url); + t.is(init.method, 'PRESIGNED'); + t.is(uploadUrl.origin, new URL(app.get(URLHelper).baseUrl).origin); + t.is(uploadUrl.pathname, PROXY_UPLOAD_PATH); +}); + e2e.serial( - 'should fall back to direct presign when custom domain is disabled', + 'should use a custom origin for provider presigned urls without a signing key', async t => { - await useR2Storage({ - enabled: false, - urlPrefix: undefined, - signKey: undefined, - }); + await useR2Storage({ signKey: undefined }); const { workspace } = await setupWorkspace(); - const buffer = Buffer.from('plain'); + const buffer = Buffer.from('custom-presign'); const init = await createBlobUpload( workspace.id, @@ -515,9 +535,50 @@ e2e.serial( 'text/plain' ); + const uploadUrl = new URL(init.uploadUrl, app.url); t.is(init.method, 'PRESIGNED'); - t.truthy(init.uploadUrl.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256')); - t.not(new URL(init.uploadUrl, app.url).pathname, PROXY_UPLOAD_PATH); + t.is(uploadUrl.origin, 'https://cdn.example.com'); + t.not(uploadUrl.pathname, PROXY_UPLOAD_PATH); + t.is(uploadUrl.searchParams.get('X-Amz-Algorithm'), 'AWS4-HMAC-SHA256'); + } +); + +e2e.serial( + 'should use provider presigned urls without custom settings', + async t => { + await useR2Storage({ urlPrefix: undefined, signKey: undefined }); + const { workspace } = await setupWorkspace(); + const buffer = Buffer.from('provider-presign'); + + const init = await createBlobUpload( + workspace.id, + sha256Base64urlWithPadding(buffer), + buffer.length, + 'text/plain' + ); + + const uploadUrl = new URL(init.uploadUrl, app.url); + t.is(init.method, 'PRESIGNED'); + t.is(uploadUrl.origin, 'https://test-bucket.r2.example.com'); + t.is(uploadUrl.searchParams.get('X-Amz-Algorithm'), 'AWS4-HMAC-SHA256'); + } +); + +e2e.serial( + 'should fall back to graphql when creating an upload url fails', + async t => { + await useR2Storage({ urlPrefix: 'invalid-url', signKey: undefined }); + const { workspace } = await setupWorkspace(); + const buffer = Buffer.from('fallback'); + + const init = await createBlobUpload( + workspace.id, + sha256Base64urlWithPadding(buffer), + buffer.length, + 'text/plain' + ); + + t.is(init.method, 'GRAPHQL'); } ); @@ -545,40 +606,6 @@ e2e.serial( } ); -function createProxyUrl( - path: string, - canonicalFields: (string | number | undefined)[], - query: Record -) { - const signKey = ( - app.get(Config).storages.blob.storage.config as R2StorageConfig - ).usePresignedURL?.signKey; - if (!signKey) { - throw new Error('missing R2 proxy sign key'); - } - const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED; - const canonical = [ - path, - ...canonicalFields.map(field => - field === undefined ? '' : field.toString() - ), - exp.toString(), - ].join('\n'); - const token = createHmac('sha256', signKey) - .update(canonical) - .digest('base64'); - - const url = new URL(`http://localhost${path}`); - for (const [key, value] of Object.entries(query)) { - if (value !== undefined) { - url.searchParams.set(key, value.toString()); - } - } - url.searchParams.set('exp', exp.toString()); - url.searchParams.set('token', `${exp}-${token}`); - return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) }; -} - function sha256Base64urlWithPadding(buffer: Buffer) { return createHash('sha256') .update(buffer) diff --git a/packages/backend/server/src/base/storage/providers/index.ts b/packages/backend/server/src/base/storage/providers/index.ts index f5451dbd1a..7d07dc1943 100644 --- a/packages/backend/server/src/base/storage/providers/index.ts +++ b/packages/backend/server/src/base/storage/providers/index.ts @@ -27,6 +27,11 @@ export interface S3StorageConfig { expiresInSeconds?: number; signContentTypeForPut?: boolean; }; + usePresignedURL?: { + enabled: boolean; + urlPrefix?: string; + signKey?: string; + }; } export const R2_JURISDICTIONS = ['default', 'eu'] as const; @@ -34,11 +39,6 @@ export const R2_JURISDICTIONS = ['default', 'eu'] as const; export interface R2StorageConfig extends Omit { accountId: string; jurisdiction?: (typeof R2_JURISDICTIONS)[number]; - usePresignedURL?: { - enabled: boolean; - urlPrefix?: string; - signKey?: string; - }; } export type StorageProviderConfig = { bucket: string } & ( @@ -115,6 +115,27 @@ const S3ConfigSchema: JSONSchema = { }, }, }, + usePresignedURL: { + type: 'object', + description: + 'Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.', + properties: { + enabled: { + type: 'boolean', + description: 'Whether browser upload URLs are enabled.', + }, + urlPrefix: { + type: 'string', + description: + 'Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured.', + }, + signKey: { + type: 'string', + description: + 'Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin.', + }, + }, + }, }, }; @@ -189,28 +210,6 @@ export const StorageJSONSchema: JSONSchema = { description: 'Optional jurisdiction for the cloudflare r2 endpoint. Set to "eu" for EU buckets.', }, - usePresignedURL: { - type: 'object' as const, - description: - 'The presigned url config for the cloudflare r2 storage provider.', - properties: { - enabled: { - type: 'boolean' as const, - description: - 'Whether to use presigned url for the cloudflare r2 storage provider.', - }, - urlPrefix: { - type: 'string' as const, - description: - 'The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: "https://storage.example.com"\nExample rule: is_timed_hmac_valid_v0("your_secret", http.request.uri, 10800, http.request.timestamp.sec, 6)', - }, - signKey: { - type: 'string' as const, - description: - 'The presigned key for the cloudflare r2 storage provider.', - }, - }, - }, }, }, }, diff --git a/packages/backend/server/src/base/storage/utils.ts b/packages/backend/server/src/base/storage/utils.ts index 903b9f192d..6e87633c53 100644 --- a/packages/backend/server/src/base/storage/utils.ts +++ b/packages/backend/server/src/base/storage/utils.ts @@ -1,3 +1,4 @@ +import { createHmac } from 'node:crypto'; import { Readable } from 'node:stream'; import type { Response } from 'express'; @@ -68,3 +69,20 @@ export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour export const STORAGE_PROXY_ROOT = '/api/storage'; export const PROXY_UPLOAD_PATH = `${STORAGE_PROXY_ROOT}/upload`; export const PROXY_MULTIPART_PATH = `${STORAGE_PROXY_ROOT}/multipart`; + +export function createStorageUploadToken( + path: string, + fields: (string | number)[], + expiresAt: number, + signKey: string +) { + const canonical = JSON.stringify([ + 'affine-storage-upload', + 1, + 'PUT', + path, + ...fields, + expiresAt, + ]); + return createHmac('sha256', signKey).update(canonical).digest('base64url'); +} diff --git a/packages/backend/server/src/core/storage/r2-proxy.ts b/packages/backend/server/src/core/storage/r2-proxy.ts index c283e6b7e3..c2340b0958 100644 --- a/packages/backend/server/src/core/storage/r2-proxy.ts +++ b/packages/backend/server/src/core/storage/r2-proxy.ts @@ -1,4 +1,4 @@ -import { createHmac, timingSafeEqual } from 'node:crypto'; +import { timingSafeEqual } from 'node:crypto'; import { Controller, Logger, Put, Req, Res } from '@nestjs/common'; import type { Request, Response } from 'express'; @@ -7,9 +7,10 @@ import { BlobInvalid, CallMetric, Config, + createStorageUploadToken, PROXY_MULTIPART_PATH, PROXY_UPLOAD_PATH, - type R2StorageConfig, + type S3StorageConfig, STORAGE_PROXY_ROOT, type StorageProviderConfig, toBuffer, @@ -19,15 +20,9 @@ import { Public } from '../auth/guard'; import { StorageRuntimeProvider } from '../storage-runtime'; import { MULTIPART_PART_SIZE } from './constants'; -type R2BlobStorageConfig = StorageProviderConfig & { - provider: 'cloudflare-r2'; - config: R2StorageConfig; -}; - type QueryValue = Request['query'][string]; -type R2Config = { - storage: R2BlobStorageConfig; +type UploadProxyConfig = { signKey: string; }; @@ -41,25 +36,17 @@ export class R2UploadController { private readonly rt: StorageRuntimeProvider ) {} - private getR2Config(): R2Config { + private getUploadProxyConfig(): UploadProxyConfig { const storage = this.config.storages.blob.storage as StorageProviderConfig; - if (storage.provider !== 'cloudflare-r2') { + if (storage.provider !== 'cloudflare-r2' && storage.provider !== 'aws-s3') { throw new BlobInvalid('Invalid endpoint'); } - const r2Config = storage.config as R2StorageConfig; - const signKey = r2Config.usePresignedURL?.signKey; - if ( - !r2Config.usePresignedURL?.enabled || - !r2Config.usePresignedURL.urlPrefix || - !signKey - ) { + const uploadConfig = (storage.config as S3StorageConfig).usePresignedURL; + const signKey = uploadConfig?.signKey; + if (!uploadConfig?.enabled || !signKey) { throw new BlobInvalid('Invalid endpoint'); } - return { storage: storage as R2BlobStorageConfig, signKey }; - } - - private sign(canonical: string, signKey: string) { - return createHmac('sha256', signKey).update(canonical).digest('base64'); + return { signKey }; } private safeEqual(expected: string, actual: string) { @@ -75,55 +62,32 @@ export class R2UploadController { private verifyToken( path: string, - canonicalFields: (string | number | undefined)[], - exp: number, + canonicalFields: (string | number)[], + expiresAt: number, token: string, signKey: string ) { - const canonical = [ + const expected = createStorageUploadToken( path, - ...canonicalFields.map(field => - field === undefined ? '' : field.toString() - ), - exp.toString(), - ].join('\n'); - const expected = `${exp}-${this.sign(canonical, signKey)}`; + canonicalFields, + expiresAt, + signKey + ); return this.safeEqual(expected, token); } private expectString(value: QueryValue, field: string): string { - if (Array.isArray(value)) { - return String(value[0]); - } if (typeof value === 'string' && value.length > 0) { return value; } throw new BlobInvalid(`Missing ${field}.`); } - private optionalString(value: QueryValue) { - if (Array.isArray(value)) { - return String(value[0]); - } - return typeof value === 'string' && value.length > 0 ? value : undefined; - } - private number(value: QueryValue, field: string): number { const str = this.expectString(value, field); const num = Number(str); - if (!Number.isFinite(num)) { - throw new BlobInvalid(`Invalid ${field}.`); - } - return num; - } - - private optionalNumber(value: QueryValue, field: string): number | undefined { - if (value === undefined) { - return undefined; - } - const num = Number(Array.isArray(value) ? value[0] : value); - if (!Number.isFinite(num)) { + if (!Number.isSafeInteger(num)) { throw new BlobInvalid(`Invalid ${field}.`); } return num; @@ -135,15 +99,15 @@ export class R2UploadController { return undefined; } const num = Number(raw); - if (!Number.isFinite(num) || num < 0) { + if (!Number.isSafeInteger(num) || num < 0) { throw new BlobInvalid('Invalid Content-Length header'); } return num; } - private ensureNotExpired(exp: number) { + private ensureNotExpired(expiresAt: number) { const now = Math.floor(Date.now() / 1000); - if (exp < now) { + if (expiresAt < now) { throw new BlobInvalid('Upload URL expired'); } } @@ -152,25 +116,31 @@ export class R2UploadController { @Put('upload') @CallMetric('controllers', 'r2_proxy_upload') async upload(@Req() req: Request, @Res() res: Response) { - const { signKey } = this.getR2Config(); + const { signKey } = this.getUploadProxyConfig(); const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId'); const key = this.expectString(req.query.key, 'key'); const token = this.expectString(req.query.token, 'token'); - const exp = this.number(req.query.exp, 'exp'); - const contentType = this.optionalString(req.query.contentType); - const contentLengthFromQuery = this.optionalNumber( + const expiresAt = this.number(req.query.expiresAt, 'expiresAt'); + const contentType = this.expectString(req.query.contentType, 'contentType'); + const contentLengthFromQuery = this.number( req.query.contentLength, 'contentLength' ); + if ( + !Number.isInteger(contentLengthFromQuery) || + contentLengthFromQuery < 0 + ) { + throw new BlobInvalid('Invalid content length'); + } - this.ensureNotExpired(exp); + this.ensureNotExpired(expiresAt); if ( !this.verifyToken( PROXY_UPLOAD_PATH, [workspaceId, key, contentType, contentLengthFromQuery], - exp, + expiresAt, token, signKey ) @@ -188,7 +158,6 @@ export class R2UploadController { const contentLengthHeader = this.parseContentLength(req); if ( - contentLengthFromQuery !== undefined && contentLengthHeader !== undefined && contentLengthFromQuery !== contentLengthHeader ) { @@ -196,15 +165,11 @@ export class R2UploadController { } const contentLength = contentLengthHeader ?? contentLengthFromQuery; - if (contentLength === undefined) { - throw new BlobInvalid('Missing Content-Length header'); - } if (record.size && contentLength !== record.size) { throw new BlobInvalid('Content length does not match upload metadata'); } - const mime = contentType ?? record.mime; - if (record.mime && mime && record.mime !== mime) { + if (record.mime && contentType && record.mime !== contentType) { throw new BlobInvalid('Mime type mismatch'); } @@ -213,10 +178,7 @@ export class R2UploadController { 'blob', `${workspaceId}/${key}`, await toBuffer(req), - { - contentType: mime, - contentLength, - } + { contentType, contentLength } ); } catch (error) { this.logger.error('Failed to proxy upload', error as Error); @@ -230,26 +192,36 @@ export class R2UploadController { @Put('multipart') @CallMetric('controllers', 'r2_proxy_multipart') async uploadPart(@Req() req: Request, @Res() res: Response) { - const { signKey } = this.getR2Config(); + const { signKey } = this.getUploadProxyConfig(); const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId'); const key = this.expectString(req.query.key, 'key'); const uploadId = this.expectString(req.query.uploadId, 'uploadId'); const token = this.expectString(req.query.token, 'token'); - const exp = this.number(req.query.exp, 'exp'); + const expiresAt = this.number(req.query.expiresAt, 'expiresAt'); const partNumber = this.number(req.query.partNumber, 'partNumber'); + const contentLengthFromQuery = this.number( + req.query.contentLength, + 'contentLength' + ); if (partNumber < 1) { throw new BlobInvalid('Invalid part number'); } + if ( + !Number.isInteger(contentLengthFromQuery) || + contentLengthFromQuery < 1 + ) { + throw new BlobInvalid('Invalid content length'); + } - this.ensureNotExpired(exp); + this.ensureNotExpired(expiresAt); if ( !this.verifyToken( PROXY_MULTIPART_PATH, - [workspaceId, key, uploadId, partNumber], - exp, + [workspaceId, key, uploadId, partNumber, contentLengthFromQuery], + expiresAt, token, signKey ) @@ -272,6 +244,9 @@ export class R2UploadController { if (contentLength === undefined || contentLength === 0) { throw new BlobInvalid('Missing Content-Length header'); } + if (contentLength !== contentLengthFromQuery) { + throw new BlobInvalid('Content length mismatch'); + } const maxPartNumber = Math.ceil(record.size / MULTIPART_PART_SIZE); if (partNumber > maxPartNumber) { diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index 09387fc74b..089e2cd1a2 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -1,17 +1,17 @@ -import { createHmac } from 'node:crypto'; - import { Injectable, Logger } from '@nestjs/common'; import { + BlobInvalid, type BlobOutputType, Config, + createStorageUploadToken, EventBus, type GetObjectMetadata, OnEvent, PROXY_MULTIPART_PATH, PROXY_UPLOAD_PATH, type PutObjectMetadata, - type R2StorageConfig, + type S3StorageConfig, SIGNED_URL_EXPIRED, type StorageProviderConfig, URLHelper, @@ -19,6 +19,7 @@ import { import { Models } from '../../../models'; import type { StorageProviderCapabilities } from '../../../native'; import { StorageRuntimeProvider } from '../../storage-runtime'; +import { MULTIPART_PART_SIZE } from '../constants'; declare global { interface Events { @@ -50,7 +51,12 @@ type BlobGetResult = { metadata?: GetObjectMetadata; }; -type R2ProxyConfig = { +type UploadURLConfig = { + signKey?: string; + urlPrefix?: string; +}; + +type UploadProxyConfig = { signKey: string; urlPrefix: string; }; @@ -82,7 +88,17 @@ export class WorkspaceBlobStorage { async capabilities(): Promise { const capabilities = await this.rt.providerCapabilities('blob'); - if (!this.r2ProxyConfig()) { + const config = this.uploadURLConfig(); + if (!config) { + return { + ...capabilities, + presignPut: false, + multipartDirect: false, + proxyUpload: false, + serverMediatedOnly: true, + }; + } + if (!config.signKey) { return capabilities; } return { @@ -116,11 +132,22 @@ export class WorkspaceBlobStorage { key: string, metadata?: PutObjectMetadata ) { - const proxy = this.r2ProxyConfig(); - if (proxy) { - return this.createProxyUploadUrl(workspaceId, key, metadata, proxy); + const config = this.uploadURLConfig(); + if (!config) return; + if (config.signKey) { + return this.createProxyUploadUrl(workspaceId, key, metadata, { + signKey: config.signKey, + urlPrefix: config.urlPrefix ?? this.url.baseUrl, + }); } - return this.rt.presignPut('blob', `${workspaceId}/${key}`, metadata); + const presigned = await this.rt.presignPut( + 'blob', + `${workspaceId}/${key}`, + metadata + ); + return config.urlPrefix && presigned + ? this.withURLPrefix(presigned, config.urlPrefix) + : presigned; } async createMultipartUpload( @@ -141,22 +168,36 @@ export class WorkspaceBlobStorage { uploadId: string, partNumber: number ) { - const proxy = this.r2ProxyConfig(); - if (proxy) { + const config = this.uploadURLConfig(); + if (!config) return; + const contentLength = await this.multipartPartContentLength( + workspaceId, + key, + uploadId, + partNumber + ); + if (config.signKey) { return this.createProxyMultipartUrl( workspaceId, key, uploadId, partNumber, - proxy + contentLength, + { + signKey: config.signKey, + urlPrefix: config.urlPrefix ?? this.url.baseUrl, + } ); } - return this.rt.presignUploadPart( + const presigned = await this.rt.presignUploadPart( 'blob', `${workspaceId}/${key}`, uploadId, partNumber ); + return config.urlPrefix && presigned + ? this.withURLPrefix(presigned, config.urlPrefix) + : presigned; } async listMultipartUploadParts( @@ -308,56 +349,38 @@ export class WorkspaceBlobStorage { await this.delete(workspaceId, key, true); } - private r2ProxyConfig() { + private uploadURLConfig(): UploadURLConfig | undefined { const storage = this.config.storages.blob.storage as StorageProviderConfig; - if (storage.provider !== 'cloudflare-r2') { + if (storage.provider !== 'cloudflare-r2' && storage.provider !== 'aws-s3') { return; } - const r2 = storage.config as R2StorageConfig; - const usePresignedURL = r2.usePresignedURL; - if ( - !usePresignedURL?.enabled || - !usePresignedURL.urlPrefix || - !usePresignedURL.signKey - ) { + const usePresignedURL = (storage.config as S3StorageConfig).usePresignedURL; + if (!usePresignedURL?.enabled) { return; } return { - signKey: usePresignedURL.signKey, - urlPrefix: usePresignedURL.urlPrefix, + signKey: usePresignedURL.signKey || undefined, + urlPrefix: usePresignedURL.urlPrefix || undefined, }; } - private signProxy( - path: string, - canonicalFields: (string | number | undefined)[], - exp: number, - signKey: string - ) { - const canonical = [ - path, - ...canonicalFields.map(field => - field === undefined ? '' : field.toString() - ), - exp.toString(), - ].join('\n'); - return `${exp}-${createHmac('sha256', signKey).update(canonical).digest('base64')}`; - } - private createProxyUploadUrl( workspaceId: string, key: string, metadata: PutObjectMetadata | undefined, - proxy: R2ProxyConfig + proxy: UploadProxyConfig ) { const contentType = metadata?.contentType ?? 'application/octet-stream'; const contentLength = metadata?.contentLength; + if (contentLength === undefined) { + throw new BlobInvalid('Missing upload content length'); + } const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000); - const exp = Math.floor(expiresAt.getTime() / 1000); - const token = this.signProxy( + const expiresAtSeconds = Math.floor(expiresAt.getTime() / 1000); + const token = createStorageUploadToken( PROXY_UPLOAD_PATH, [workspaceId, key, contentType, contentLength], - exp, + expiresAtSeconds, proxy.signKey ); return { @@ -366,7 +389,7 @@ export class WorkspaceBlobStorage { key, contentType, contentLength, - exp, + expiresAt: expiresAtSeconds, token, }), headers: {}, @@ -379,14 +402,15 @@ export class WorkspaceBlobStorage { key: string, uploadId: string, partNumber: number, - proxy: R2ProxyConfig + contentLength: number, + proxy: UploadProxyConfig ) { const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000); - const exp = Math.floor(expiresAt.getTime() / 1000); - const token = this.signProxy( + const expiresAtSeconds = Math.floor(expiresAt.getTime() / 1000); + const token = createStorageUploadToken( PROXY_MULTIPART_PATH, - [workspaceId, key, uploadId, partNumber], - exp, + [workspaceId, key, uploadId, partNumber, contentLength], + expiresAtSeconds, proxy.signKey ); return { @@ -395,7 +419,8 @@ export class WorkspaceBlobStorage { key, uploadId, partNumber, - exp, + contentLength, + expiresAt: expiresAtSeconds, token, }), headers: {}, @@ -418,4 +443,42 @@ export class WorkspaceBlobStorage { } return url.toString(); } + + private withURLPrefix( + presigned: T, + urlPrefix: string + ): T { + const url = new URL(presigned.url); + const prefix = new URL(urlPrefix); + if (prefix.pathname !== '/' || prefix.search || prefix.hash) { + throw new BlobInvalid('Upload URL prefix must contain only an origin'); + } + url.protocol = prefix.protocol; + url.host = prefix.host; + return { ...presigned, url: url.toString() }; + } + + private async multipartPartContentLength( + workspaceId: string, + key: string, + uploadId: string, + partNumber: number + ) { + const record = await this.models.blob.get(workspaceId, key); + if (!record || record.status === 'completed') { + throw new BlobInvalid('Multipart upload is not pending'); + } + if (record.uploadId !== uploadId) { + throw new BlobInvalid('Upload id mismatch'); + } + const offset = (partNumber - 1) * MULTIPART_PART_SIZE; + if ( + !Number.isInteger(partNumber) || + partNumber < 1 || + offset >= record.size + ) { + throw new BlobInvalid('Invalid part number'); + } + return Math.min(MULTIPART_PART_SIZE, record.size - offset); + } } diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.ts index 464d150def..bd86b1b07a 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts @@ -248,63 +248,70 @@ export class WorkspaceBlobResolver { } const metadata = { contentType: mime, contentLength: size }; - const capabilities = await this.storage.capabilities(); let init: BlobUploadInit | null = null; let uploadIdForRecord: string | null = null; - // try to resume multipart uploads - if (capabilities.multipartDirect && record && record.uploadId) { - const uploadedParts = await this.storage.listMultipartUploadParts( - workspaceId, - key, - record.uploadId - ); + try { + const capabilities = await this.storage.capabilities(); - if (uploadedParts) { - return { - method: BlobUploadMethod.MULTIPART, - blobKey: key, - uploadId: record.uploadId, - partSize: MULTIPART_PART_SIZE, - uploadedParts, - }; - } - } + // try to resume multipart uploads + if (capabilities.multipartDirect && record && record.uploadId) { + const uploadedParts = await this.storage.listMultipartUploadParts( + workspaceId, + key, + record.uploadId + ); - if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) { - const multipart = await this.storage.createMultipartUpload( - workspaceId, - key, - metadata - ); - if (multipart) { - uploadIdForRecord = multipart.uploadId; - init = { - method: BlobUploadMethod.MULTIPART, - blobKey: key, - uploadId: multipart.uploadId, - partSize: MULTIPART_PART_SIZE, - expiresAt: multipart.expiresAt, - uploadedParts: [], - }; + if (uploadedParts) { + return { + method: BlobUploadMethod.MULTIPART, + blobKey: key, + uploadId: record.uploadId, + partSize: MULTIPART_PART_SIZE, + uploadedParts, + }; + } } - } - if (!init && capabilities.presignPut) { - const presigned = await this.storage.presignPut( - workspaceId, - key, - metadata - ); - if (presigned) { - init = { - method: BlobUploadMethod.PRESIGNED, - blobKey: key, - uploadUrl: presigned.url, - headers: presigned.headers, - expiresAt: presigned.expiresAt, - }; + if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) { + const multipart = await this.storage.createMultipartUpload( + workspaceId, + key, + metadata + ); + if (multipart) { + uploadIdForRecord = multipart.uploadId; + init = { + method: BlobUploadMethod.MULTIPART, + blobKey: key, + uploadId: multipart.uploadId, + partSize: MULTIPART_PART_SIZE, + expiresAt: multipart.expiresAt, + uploadedParts: [], + }; + } } + + if (!init && capabilities.presignPut) { + const presigned = await this.storage.presignPut( + workspaceId, + key, + metadata + ); + if (presigned) { + init = { + method: BlobUploadMethod.PRESIGNED, + blobKey: key, + uploadUrl: presigned.url, + headers: presigned.headers, + expiresAt: presigned.expiresAt, + }; + } + } + } catch (error) { + this.logger.warn('Failed to initialize direct blob upload', error); + init = null; + uploadIdForRecord = null; } if (!init) {