Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-29 08:09:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): improve blob sync (#15367)

Browse Source 
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Standardized `usePresignedURL` configuration for AWS S3 and Cloudflare
R2 (including `enabled`, `urlPrefix`, and `signKey`).
- Storage upload URL generation now supports both direct provider
presigning and server-mediated proxying based on configuration.
- **Bug Fixes**
- Tightened upload and multipart validation (content type/length checks,
header vs query consistency, and stricter expiration handling).
- Improved fallback behavior when direct upload URL initialization
fails.
- **Tests**
- Updated R2 storage proxy end-to-end coverage to match the new
URL/token behavior.
- **Documentation**
  - Refreshed self-hosted JSON schema guidance for upload URL settings.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-28 23:43:43 +08:00
committed by GitHub
parent b6fc0a2192
commit 00576e1e78
7 changed files with 490 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.docker/selfhost/schema.json
+108 -54
View File
@@ -483,6 +483,24 @@
"type": "string"
}
}
},
"usePresignedURL": {
"type": "object",
"description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether browser upload URLs are enabled."
},
"urlPrefix": {
"type": "string",
"description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
},
"signKey": {
"type": "string",
"description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
}
}
}
}
}
@@ -549,6 +567,24 @@
}
}
},
"usePresignedURL": {
"type": "object",
"description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether browser upload URLs are enabled."
},
"urlPrefix": {
"type": "string",
"description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
},
"signKey": {
"type": "string",
"description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
}
}
},
"accountId": {
"type": "string",
"description": "The account id for the cloudflare r2 storage provider."
@@ -560,24 +596,6 @@
"eu"
],
"description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets."
},
"usePresignedURL": {
"type": "object",
"description": "The presigned url config for the cloudflare r2 storage provider.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether to use presigned url for the cloudflare r2 storage provider."
},
"urlPrefix": {
"type": "string",
"description": "The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: \"https://storage.example.com\"\nExample rule: is_timed_hmac_valid_v0(\"your_secret\", http.request.uri, 10800, http.request.timestamp.sec, 6)"
},
"signKey": {
"type": "string",
"description": "The presigned key for the cloudflare r2 storage provider."
}
}
}
}
}
@@ -712,6 +730,24 @@
"type": "string"
}
}
},
"usePresignedURL": {
"type": "object",
"description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether browser upload URLs are enabled."
},
"urlPrefix": {
"type": "string",
"description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
},
"signKey": {
"type": "string",
"description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
}
}
}
}
}
@@ -778,6 +814,24 @@
}
}
},
"usePresignedURL": {
"type": "object",
"description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether browser upload URLs are enabled."
},
"urlPrefix": {
"type": "string",
"description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
},
"signKey": {
"type": "string",
"description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
}
}
},
"accountId": {
"type": "string",
"description": "The account id for the cloudflare r2 storage provider."
@@ -789,24 +843,6 @@
"eu"
],
"description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets."
},
"usePresignedURL": {
"type": "object",
"description": "The presigned url config for the cloudflare r2 storage provider.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether to use presigned url for the cloudflare r2 storage provider."
},
"urlPrefix": {
"type": "string",
"description": "The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: \"https://storage.example.com\"\nExample rule: is_timed_hmac_valid_v0(\"your_secret\", http.request.uri, 10800, http.request.timestamp.sec, 6)"
},
"signKey": {
"type": "string",
"description": "The presigned key for the cloudflare r2 storage provider."
}
}
}
}
}
@@ -1391,6 +1427,24 @@
"type": "string"
}
}
},
"usePresignedURL": {
"type": "object",
"description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether browser upload URLs are enabled."
},
"urlPrefix": {
"type": "string",
"description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
},
"signKey": {
"type": "string",
"description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
}
}
}
}
}
@@ -1457,6 +1511,24 @@
}
}
},
"usePresignedURL": {
"type": "object",
"description": "Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether browser upload URLs are enabled."
},
"urlPrefix": {
"type": "string",
"description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
},
"signKey": {
"type": "string",
"description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
}
}
},
"accountId": {
"type": "string",
"description": "The account id for the cloudflare r2 storage provider."
@@ -1468,24 +1540,6 @@
"eu"
],
"description": "Optional jurisdiction for the cloudflare r2 endpoint. Set to \"eu\" for EU buckets."
},
"usePresignedURL": {
"type": "object",
"description": "The presigned url config for the cloudflare r2 storage provider.",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether to use presigned url for the cloudflare r2 storage provider."
},
"urlPrefix": {
"type": "string",
"description": "The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: \"https://storage.example.com\"\nExample rule: is_timed_hmac_valid_v0(\"your_secret\", http.request.uri, 10800, http.request.timestamp.sec, 6)"
},
"signKey": {
"type": "string",
"description": "The presigned key for the cloudflare r2 storage provider."
}
}
}
}
}
packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts
+115 -88
View File
@@ -1,4 +1,4 @@
import { createHash, createHmac } from 'node:crypto';
import { createHash } from 'node:crypto';
import { mock } from 'node:test';
import {
@@ -9,9 +9,13 @@ import {
type R2StorageConfig,
SIGNED_URL_EXPIRED,
type StorageProviderConfig,
URLHelper,
} from '../../../base';
import { EntitlementService } from '../../../core/entitlement';
import { MULTIPART_THRESHOLD } from '../../../core/storage/constants';
import {
MULTIPART_PART_SIZE,
MULTIPART_THRESHOLD,
} from '../../../core/storage/constants';
import { StorageRuntimeProvider } from '../../../core/storage-runtime';
import {
SubscriptionPlan,
@@ -39,7 +43,6 @@ class MockStorageRuntime {
async providerCapabilities() {
const storage = app.get(Config).storages.blob.storage;
const usePresignedURL = (storage.config as R2StorageConfig).usePresignedURL;
if (storage.provider !== 'cloudflare-r2') {
return {
put: true,
@@ -64,7 +67,7 @@ class MockStorageRuntime {
presignPut: true,
presignGet: false,
multipartDirect: true,
proxyUpload: !!usePresignedURL?.enabled,
proxyUpload: false,
assetpack: false,
serverMediatedOnly: false,
};
@@ -75,31 +78,17 @@ class MockStorageRuntime {
key: string,
metadata: { contentType?: string; contentLength?: number } = {}
) {
const storage = app.get(Config).storages.blob.storage;
const r2 = storage.config as R2StorageConfig;
if (!r2.usePresignedURL?.enabled) {
return {
url: 'https://test-bucket.r2.example.com/object?X-Amz-Algorithm=AWS4-HMAC-SHA256',
headers: {},
expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
};
}
const [workspaceId, blobKey] = key.split('/');
return createProxyUrl(
PROXY_UPLOAD_PATH,
[
workspaceId,
blobKey,
metadata.contentType ?? 'application/octet-stream',
metadata.contentLength,
],
{
workspaceId,
key: blobKey,
contentType: metadata.contentType ?? 'application/octet-stream',
contentLength: metadata.contentLength,
}
const url = new URL(`https://test-bucket.r2.example.com/${key}`);
url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256');
url.searchParams.set(
'X-Amz-SignedHeaders',
metadata.contentType ? 'content-type;host' : 'host'
);
return {
url: url.toString(),
headers: {},
expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
};
}
async createMultipartUpload() {
@@ -116,17 +105,15 @@ class MockStorageRuntime {
uploadId: string,
partNumber: number
) {
const [workspaceId, blobKey] = key.split('/');
return createProxyUrl(
PROXY_MULTIPART_PATH,
[workspaceId, blobKey, uploadId, partNumber],
{
workspaceId,
key: blobKey,
uploadId,
partNumber,
}
);
const url = new URL(`https://test-bucket.r2.example.com/${key}`);
url.searchParams.set('uploadId', uploadId);
url.searchParams.set('partNumber', partNumber.toString());
url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256');
return {
url: url.toString(),
headers: {},
expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
};
}
async listMultipartUploadParts(
@@ -390,25 +377,25 @@ e2e.serial('should proxy multipart upload and return etag', async t => {
t.is(init.uploadId, 'upload-id');
t.deepEqual(init.uploadedParts, []);
const part = await getBlobUploadPartUrl(workspace.id, key, init.uploadId, 1);
const part = await getBlobUploadPartUrl(workspace.id, key, init.uploadId, 3);
const partUrl = new URL(part.uploadUrl, app.url);
t.is(partUrl.origin, 'https://cdn.example.com');
t.is(partUrl.pathname, PROXY_MULTIPART_PATH);
const payload = Buffer.from('part-body');
const payload = Buffer.alloc(1024);
const res = await app
.PUT(partUrl.pathname + partUrl.search)
.set('content-length', payload.length.toString())
.send(payload);
t.is(res.status, 200);
t.is(res.get('etag'), 'etag-1');
t.is(res.get('etag'), 'etag-3');
const calls = getRuntime().partCalls;
t.is(calls.length, 1);
t.is(calls[0].key, `${workspace.id}/${key}`);
t.is(calls[0].uploadId, 'upload-id');
t.is(calls[0].partNumber, 1);
t.is(calls[0].partNumber, 3);
t.is(calls[0].contentLength, payload.length);
t.deepEqual(calls[0].body, payload);
});
@@ -432,7 +419,7 @@ e2e.serial(
init1.uploadId,
1
);
const payload = Buffer.from('part-body');
const payload = Buffer.alloc(MULTIPART_PART_SIZE);
const partUrl = new URL(part.uploadUrl, app.url);
await app
.PUT(partUrl.pathname + partUrl.search)
@@ -482,7 +469,7 @@ e2e.serial('should reject upload when url is expired', async t => {
);
const uploadUrl = new URL(init.uploadUrl, app.url);
uploadUrl.searchParams.set(
'exp',
'expiresAt',
(Math.floor(Date.now() / 1000) - 1).toString()
);
@@ -497,16 +484,49 @@ e2e.serial('should reject upload when url is expired', async t => {
t.is(getRuntime().putCalls.length, 0);
});
e2e.serial('should use graphql when upload urls are disabled', async t => {
await useR2Storage({
enabled: false,
urlPrefix: undefined,
signKey: undefined,
});
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('plain');
const init = await createBlobUpload(
workspace.id,
sha256Base64urlWithPadding(buffer),
buffer.length,
'text/plain'
);
t.is(init.method, 'GRAPHQL');
});
e2e.serial('should use the server origin with only a signing key', async t => {
await useR2Storage({ urlPrefix: undefined });
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('server-proxy');
const init = await createBlobUpload(
workspace.id,
sha256Base64urlWithPadding(buffer),
buffer.length,
'text/plain'
);
const uploadUrl = new URL(init.uploadUrl, app.url);
t.is(init.method, 'PRESIGNED');
t.is(uploadUrl.origin, new URL(app.get(URLHelper).baseUrl).origin);
t.is(uploadUrl.pathname, PROXY_UPLOAD_PATH);
});
e2e.serial(
'should fall back to direct presign when custom domain is disabled',
'should use a custom origin for provider presigned urls without a signing key',
async t => {
await useR2Storage({
enabled: false,
urlPrefix: undefined,
signKey: undefined,
});
await useR2Storage({ signKey: undefined });
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('plain');
const buffer = Buffer.from('custom-presign');
const init = await createBlobUpload(
workspace.id,
@@ -515,9 +535,50 @@ e2e.serial(
'text/plain'
);
const uploadUrl = new URL(init.uploadUrl, app.url);
t.is(init.method, 'PRESIGNED');
t.truthy(init.uploadUrl.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
t.not(new URL(init.uploadUrl, app.url).pathname, PROXY_UPLOAD_PATH);
t.is(uploadUrl.origin, 'https://cdn.example.com');
t.not(uploadUrl.pathname, PROXY_UPLOAD_PATH);
t.is(uploadUrl.searchParams.get('X-Amz-Algorithm'), 'AWS4-HMAC-SHA256');
}
);
e2e.serial(
'should use provider presigned urls without custom settings',
async t => {
await useR2Storage({ urlPrefix: undefined, signKey: undefined });
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('provider-presign');
const init = await createBlobUpload(
workspace.id,
sha256Base64urlWithPadding(buffer),
buffer.length,
'text/plain'
);
const uploadUrl = new URL(init.uploadUrl, app.url);
t.is(init.method, 'PRESIGNED');
t.is(uploadUrl.origin, 'https://test-bucket.r2.example.com');
t.is(uploadUrl.searchParams.get('X-Amz-Algorithm'), 'AWS4-HMAC-SHA256');
}
);
e2e.serial(
'should fall back to graphql when creating an upload url fails',
async t => {
await useR2Storage({ urlPrefix: 'invalid-url', signKey: undefined });
const { workspace } = await setupWorkspace();
const buffer = Buffer.from('fallback');
const init = await createBlobUpload(
workspace.id,
sha256Base64urlWithPadding(buffer),
buffer.length,
'text/plain'
);
t.is(init.method, 'GRAPHQL');
}
);
@@ -545,40 +606,6 @@ e2e.serial(
}
);
function createProxyUrl(
path: string,
canonicalFields: (string | number | undefined)[],
query: Record<string, string | number | undefined>
) {
const signKey = (
app.get(Config).storages.blob.storage.config as R2StorageConfig
).usePresignedURL?.signKey;
if (!signKey) {
throw new Error('missing R2 proxy sign key');
}
const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED;
const canonical = [
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
const token = createHmac('sha256', signKey)
.update(canonical)
.digest('base64');
const url = new URL(`http://localhost${path}`);
for (const [key, value] of Object.entries(query)) {
if (value !== undefined) {
url.searchParams.set(key, value.toString());
}
}
url.searchParams.set('exp', exp.toString());
url.searchParams.set('token', `${exp}-${token}`);
return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) };
}
function sha256Base64urlWithPadding(buffer: Buffer) {
return createHash('sha256')
.update(buffer)
packages/backend/server/src/base/storage/providers/index.ts
+26 -27
View File
@@ -27,6 +27,11 @@ export interface S3StorageConfig {
expiresInSeconds?: number;
signContentTypeForPut?: boolean;
};
usePresignedURL?: {
enabled: boolean;
urlPrefix?: string;
signKey?: string;
};
}
export const R2_JURISDICTIONS = ['default', 'eu'] as const;
@@ -34,11 +39,6 @@ export const R2_JURISDICTIONS = ['default', 'eu'] as const;
export interface R2StorageConfig extends Omit<S3StorageConfig, 'endpoint'> {
accountId: string;
jurisdiction?: (typeof R2_JURISDICTIONS)[number];
usePresignedURL?: {
enabled: boolean;
urlPrefix?: string;
signKey?: string;
};
}
export type StorageProviderConfig = { bucket: string } & (
@@ -115,6 +115,27 @@ const S3ConfigSchema: JSONSchema = {
},
},
},
usePresignedURL: {
type: 'object',
description:
'Controls browser upload URLs. Disabled or unavailable modes fall back to server uploads.',
properties: {
enabled: {
type: 'boolean',
description: 'Whether browser upload URLs are enabled.',
},
urlPrefix: {
type: 'string',
description:
'Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured.',
},
signKey: {
type: 'string',
description:
'Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin.',
},
},
},
},
};
@@ -189,28 +210,6 @@ export const StorageJSONSchema: JSONSchema = {
description:
'Optional jurisdiction for the cloudflare r2 endpoint. Set to "eu" for EU buckets.',
},
usePresignedURL: {
type: 'object' as const,
description:
'The presigned url config for the cloudflare r2 storage provider.',
properties: {
enabled: {
type: 'boolean' as const,
description:
'Whether to use presigned url for the cloudflare r2 storage provider.',
},
urlPrefix: {
type: 'string' as const,
description:
'The custom domain URL prefix for the cloudflare r2 storage provider.\nWhen `enabled=true` and `urlPrefix` + `signKey` are provided, the server will:\n- Redirect GET requests to this custom domain with an HMAC token.\n- Return upload URLs under `/api/storage/*` for uploads.\nPresigned/upload proxy TTL is 1 hour.\nsee https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it.\nExample value: "https://storage.example.com"\nExample rule: is_timed_hmac_valid_v0("your_secret", http.request.uri, 10800, http.request.timestamp.sec, 6)',
},
signKey: {
type: 'string' as const,
description:
'The presigned key for the cloudflare r2 storage provider.',
},
},
},
},
},
},
packages/backend/server/src/base/storage/utils.ts
+18
View File
@@ -1,3 +1,4 @@
import { createHmac } from 'node:crypto';
import { Readable } from 'node:stream';
import type { Response } from 'express';
@@ -68,3 +69,20 @@ export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour
export const STORAGE_PROXY_ROOT = '/api/storage';
export const PROXY_UPLOAD_PATH = `${STORAGE_PROXY_ROOT}/upload`;
export const PROXY_MULTIPART_PATH = `${STORAGE_PROXY_ROOT}/multipart`;
export function createStorageUploadToken(
path: string,
fields: (string | number)[],
expiresAt: number,
signKey: string
) {
const canonical = JSON.stringify([
'affine-storage-upload',
1,
'PUT',
path,
...fields,
expiresAt,
]);
return createHmac('sha256', signKey).update(canonical).digest('base64url');
}
packages/backend/server/src/core/storage/r2-proxy.ts
+53 -78
View File
@@ -1,4 +1,4 @@
import { createHmac, timingSafeEqual } from 'node:crypto';
import { timingSafeEqual } from 'node:crypto';
import { Controller, Logger, Put, Req, Res } from '@nestjs/common';
import type { Request, Response } from 'express';
@@ -7,9 +7,10 @@ import {
BlobInvalid,
CallMetric,
Config,
createStorageUploadToken,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type R2StorageConfig,
type S3StorageConfig,
STORAGE_PROXY_ROOT,
type StorageProviderConfig,
toBuffer,
@@ -19,15 +20,9 @@ import { Public } from '../auth/guard';
import { StorageRuntimeProvider } from '../storage-runtime';
import { MULTIPART_PART_SIZE } from './constants';
type R2BlobStorageConfig = StorageProviderConfig & {
provider: 'cloudflare-r2';
config: R2StorageConfig;
};
type QueryValue = Request['query'][string];
type R2Config = {
storage: R2BlobStorageConfig;
type UploadProxyConfig = {
signKey: string;
};
@@ -41,25 +36,17 @@ export class R2UploadController {
private readonly rt: StorageRuntimeProvider
) {}
private getR2Config(): R2Config {
private getUploadProxyConfig(): UploadProxyConfig {
const storage = this.config.storages.blob.storage as StorageProviderConfig;
if (storage.provider !== 'cloudflare-r2') {
if (storage.provider !== 'cloudflare-r2' && storage.provider !== 'aws-s3') {
throw new BlobInvalid('Invalid endpoint');
}
const r2Config = storage.config as R2StorageConfig;
const signKey = r2Config.usePresignedURL?.signKey;
if (
!r2Config.usePresignedURL?.enabled ||
!r2Config.usePresignedURL.urlPrefix ||
!signKey
) {
const uploadConfig = (storage.config as S3StorageConfig).usePresignedURL;
const signKey = uploadConfig?.signKey;
if (!uploadConfig?.enabled || !signKey) {
throw new BlobInvalid('Invalid endpoint');
}
return { storage: storage as R2BlobStorageConfig, signKey };
}
private sign(canonical: string, signKey: string) {
return createHmac('sha256', signKey).update(canonical).digest('base64');
return { signKey };
}
private safeEqual(expected: string, actual: string) {
@@ -75,55 +62,32 @@ export class R2UploadController {
private verifyToken(
path: string,
canonicalFields: (string | number | undefined)[],
exp: number,
canonicalFields: (string | number)[],
expiresAt: number,
token: string,
signKey: string
) {
const canonical = [
const expected = createStorageUploadToken(
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
const expected = `${exp}-${this.sign(canonical, signKey)}`;
canonicalFields,
expiresAt,
signKey
);
return this.safeEqual(expected, token);
}
private expectString(value: QueryValue, field: string): string {
if (Array.isArray(value)) {
return String(value[0]);
}
if (typeof value === 'string' && value.length > 0) {
return value;
}
throw new BlobInvalid(`Missing ${field}.`);
}
private optionalString(value: QueryValue) {
if (Array.isArray(value)) {
return String(value[0]);
}
return typeof value === 'string' && value.length > 0 ? value : undefined;
}
private number(value: QueryValue, field: string): number {
const str = this.expectString(value, field);
const num = Number(str);
if (!Number.isFinite(num)) {
throw new BlobInvalid(`Invalid ${field}.`);
}
return num;
}
private optionalNumber(value: QueryValue, field: string): number | undefined {
if (value === undefined) {
return undefined;
}
const num = Number(Array.isArray(value) ? value[0] : value);
if (!Number.isFinite(num)) {
if (!Number.isSafeInteger(num)) {
throw new BlobInvalid(`Invalid ${field}.`);
}
return num;
@@ -135,15 +99,15 @@ export class R2UploadController {
return undefined;
}
const num = Number(raw);
if (!Number.isFinite(num) || num < 0) {
if (!Number.isSafeInteger(num) || num < 0) {
throw new BlobInvalid('Invalid Content-Length header');
}
return num;
}
private ensureNotExpired(exp: number) {
private ensureNotExpired(expiresAt: number) {
const now = Math.floor(Date.now() / 1000);
if (exp < now) {
if (expiresAt < now) {
throw new BlobInvalid('Upload URL expired');
}
}
@@ -152,25 +116,31 @@ export class R2UploadController {
@Put('upload')
@CallMetric('controllers', 'r2_proxy_upload')
async upload(@Req() req: Request, @Res() res: Response) {
const { signKey } = this.getR2Config();
const { signKey } = this.getUploadProxyConfig();
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
const key = this.expectString(req.query.key, 'key');
const token = this.expectString(req.query.token, 'token');
const exp = this.number(req.query.exp, 'exp');
const contentType = this.optionalString(req.query.contentType);
const contentLengthFromQuery = this.optionalNumber(
const expiresAt = this.number(req.query.expiresAt, 'expiresAt');
const contentType = this.expectString(req.query.contentType, 'contentType');
const contentLengthFromQuery = this.number(
req.query.contentLength,
'contentLength'
);
if (
!Number.isInteger(contentLengthFromQuery) ||
contentLengthFromQuery < 0
) {
throw new BlobInvalid('Invalid content length');
}
this.ensureNotExpired(exp);
this.ensureNotExpired(expiresAt);
if (
!this.verifyToken(
PROXY_UPLOAD_PATH,
[workspaceId, key, contentType, contentLengthFromQuery],
exp,
expiresAt,
token,
signKey
)
@@ -188,7 +158,6 @@ export class R2UploadController {
const contentLengthHeader = this.parseContentLength(req);
if (
contentLengthFromQuery !== undefined &&
contentLengthHeader !== undefined &&
contentLengthFromQuery !== contentLengthHeader
) {
@@ -196,15 +165,11 @@ export class R2UploadController {
}
const contentLength = contentLengthHeader ?? contentLengthFromQuery;
if (contentLength === undefined) {
throw new BlobInvalid('Missing Content-Length header');
}
if (record.size && contentLength !== record.size) {
throw new BlobInvalid('Content length does not match upload metadata');
}
const mime = contentType ?? record.mime;
if (record.mime && mime && record.mime !== mime) {
if (record.mime && contentType && record.mime !== contentType) {
throw new BlobInvalid('Mime type mismatch');
}
@@ -213,10 +178,7 @@ export class R2UploadController {
'blob',
`${workspaceId}/${key}`,
await toBuffer(req),
{
contentType: mime,
contentLength,
}
{ contentType, contentLength }
);
} catch (error) {
this.logger.error('Failed to proxy upload', error as Error);
@@ -230,26 +192,36 @@ export class R2UploadController {
@Put('multipart')
@CallMetric('controllers', 'r2_proxy_multipart')
async uploadPart(@Req() req: Request, @Res() res: Response) {
const { signKey } = this.getR2Config();
const { signKey } = this.getUploadProxyConfig();
const workspaceId = this.expectString(req.query.workspaceId, 'workspaceId');
const key = this.expectString(req.query.key, 'key');
const uploadId = this.expectString(req.query.uploadId, 'uploadId');
const token = this.expectString(req.query.token, 'token');
const exp = this.number(req.query.exp, 'exp');
const expiresAt = this.number(req.query.expiresAt, 'expiresAt');
const partNumber = this.number(req.query.partNumber, 'partNumber');
const contentLengthFromQuery = this.number(
req.query.contentLength,
'contentLength'
);
if (partNumber < 1) {
throw new BlobInvalid('Invalid part number');
}
if (
!Number.isInteger(contentLengthFromQuery) ||
contentLengthFromQuery < 1
) {
throw new BlobInvalid('Invalid content length');
}
this.ensureNotExpired(exp);
this.ensureNotExpired(expiresAt);
if (
!this.verifyToken(
PROXY_MULTIPART_PATH,
[workspaceId, key, uploadId, partNumber],
exp,
[workspaceId, key, uploadId, partNumber, contentLengthFromQuery],
expiresAt,
token,
signKey
)
@@ -272,6 +244,9 @@ export class R2UploadController {
if (contentLength === undefined || contentLength === 0) {
throw new BlobInvalid('Missing Content-Length header');
}
if (contentLength !== contentLengthFromQuery) {
throw new BlobInvalid('Content length mismatch');
}
const maxPartNumber = Math.ceil(record.size / MULTIPART_PART_SIZE);
if (partNumber > maxPartNumber) {
packages/backend/server/src/core/storage/wrappers/blob.ts
+114 -51
View File
@@ -1,17 +1,17 @@
import { createHmac } from 'node:crypto';
import { Injectable, Logger } from '@nestjs/common';
import {
BlobInvalid,
type BlobOutputType,
Config,
createStorageUploadToken,
EventBus,
type GetObjectMetadata,
OnEvent,
PROXY_MULTIPART_PATH,
PROXY_UPLOAD_PATH,
type PutObjectMetadata,
type R2StorageConfig,
type S3StorageConfig,
SIGNED_URL_EXPIRED,
type StorageProviderConfig,
URLHelper,
@@ -19,6 +19,7 @@ import {
import { Models } from '../../../models';
import type { StorageProviderCapabilities } from '../../../native';
import { StorageRuntimeProvider } from '../../storage-runtime';
import { MULTIPART_PART_SIZE } from '../constants';
declare global {
interface Events {
@@ -50,7 +51,12 @@ type BlobGetResult = {
metadata?: GetObjectMetadata;
};
type R2ProxyConfig = {
type UploadURLConfig = {
signKey?: string;
urlPrefix?: string;
};
type UploadProxyConfig = {
signKey: string;
urlPrefix: string;
};
@@ -82,7 +88,17 @@ export class WorkspaceBlobStorage {
async capabilities(): Promise<StorageProviderCapabilities> {
const capabilities = await this.rt.providerCapabilities('blob');
if (!this.r2ProxyConfig()) {
const config = this.uploadURLConfig();
if (!config) {
return {
...capabilities,
presignPut: false,
multipartDirect: false,
proxyUpload: false,
serverMediatedOnly: true,
};
}
if (!config.signKey) {
return capabilities;
}
return {
@@ -116,11 +132,22 @@ export class WorkspaceBlobStorage {
key: string,
metadata?: PutObjectMetadata
) {
const proxy = this.r2ProxyConfig();
if (proxy) {
return this.createProxyUploadUrl(workspaceId, key, metadata, proxy);
const config = this.uploadURLConfig();
if (!config) return;
if (config.signKey) {
return this.createProxyUploadUrl(workspaceId, key, metadata, {
signKey: config.signKey,
urlPrefix: config.urlPrefix ?? this.url.baseUrl,
});
}
return this.rt.presignPut('blob', `${workspaceId}/${key}`, metadata);
const presigned = await this.rt.presignPut(
'blob',
`${workspaceId}/${key}`,
metadata
);
return config.urlPrefix && presigned
? this.withURLPrefix(presigned, config.urlPrefix)
: presigned;
}
async createMultipartUpload(
@@ -141,22 +168,36 @@ export class WorkspaceBlobStorage {
uploadId: string,
partNumber: number
) {
const proxy = this.r2ProxyConfig();
if (proxy) {
const config = this.uploadURLConfig();
if (!config) return;
const contentLength = await this.multipartPartContentLength(
workspaceId,
key,
uploadId,
partNumber
);
if (config.signKey) {
return this.createProxyMultipartUrl(
workspaceId,
key,
uploadId,
partNumber,
proxy
contentLength,
{
signKey: config.signKey,
urlPrefix: config.urlPrefix ?? this.url.baseUrl,
}
);
}
return this.rt.presignUploadPart(
const presigned = await this.rt.presignUploadPart(
'blob',
`${workspaceId}/${key}`,
uploadId,
partNumber
);
return config.urlPrefix && presigned
? this.withURLPrefix(presigned, config.urlPrefix)
: presigned;
}
async listMultipartUploadParts(
@@ -308,56 +349,38 @@ export class WorkspaceBlobStorage {
await this.delete(workspaceId, key, true);
}
private r2ProxyConfig() {
private uploadURLConfig(): UploadURLConfig | undefined {
const storage = this.config.storages.blob.storage as StorageProviderConfig;
if (storage.provider !== 'cloudflare-r2') {
if (storage.provider !== 'cloudflare-r2' && storage.provider !== 'aws-s3') {
return;
}
const r2 = storage.config as R2StorageConfig;
const usePresignedURL = r2.usePresignedURL;
if (
!usePresignedURL?.enabled ||
!usePresignedURL.urlPrefix ||
!usePresignedURL.signKey
) {
const usePresignedURL = (storage.config as S3StorageConfig).usePresignedURL;
if (!usePresignedURL?.enabled) {
return;
}
return {
signKey: usePresignedURL.signKey,
urlPrefix: usePresignedURL.urlPrefix,
signKey: usePresignedURL.signKey || undefined,
urlPrefix: usePresignedURL.urlPrefix || undefined,
};
}
private signProxy(
path: string,
canonicalFields: (string | number | undefined)[],
exp: number,
signKey: string
) {
const canonical = [
path,
...canonicalFields.map(field =>
field === undefined ? '' : field.toString()
),
exp.toString(),
].join('\n');
return `${exp}-${createHmac('sha256', signKey).update(canonical).digest('base64')}`;
}
private createProxyUploadUrl(
workspaceId: string,
key: string,
metadata: PutObjectMetadata | undefined,
proxy: R2ProxyConfig
proxy: UploadProxyConfig
) {
const contentType = metadata?.contentType ?? 'application/octet-stream';
const contentLength = metadata?.contentLength;
if (contentLength === undefined) {
throw new BlobInvalid('Missing upload content length');
}
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
const exp = Math.floor(expiresAt.getTime() / 1000);
const token = this.signProxy(
const expiresAtSeconds = Math.floor(expiresAt.getTime() / 1000);
const token = createStorageUploadToken(
PROXY_UPLOAD_PATH,
[workspaceId, key, contentType, contentLength],
exp,
expiresAtSeconds,
proxy.signKey
);
return {
@@ -366,7 +389,7 @@ export class WorkspaceBlobStorage {
key,
contentType,
contentLength,
exp,
expiresAt: expiresAtSeconds,
token,
}),
headers: {},
@@ -379,14 +402,15 @@ export class WorkspaceBlobStorage {
key: string,
uploadId: string,
partNumber: number,
proxy: R2ProxyConfig
contentLength: number,
proxy: UploadProxyConfig
) {
const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
const exp = Math.floor(expiresAt.getTime() / 1000);
const token = this.signProxy(
const expiresAtSeconds = Math.floor(expiresAt.getTime() / 1000);
const token = createStorageUploadToken(
PROXY_MULTIPART_PATH,
[workspaceId, key, uploadId, partNumber],
exp,
[workspaceId, key, uploadId, partNumber, contentLength],
expiresAtSeconds,
proxy.signKey
);
return {
@@ -395,7 +419,8 @@ export class WorkspaceBlobStorage {
key,
uploadId,
partNumber,
exp,
contentLength,
expiresAt: expiresAtSeconds,
token,
}),
headers: {},
@@ -418,4 +443,42 @@ export class WorkspaceBlobStorage {
}
return url.toString();
}
private withURLPrefix<T extends { url: string }>(
presigned: T,
urlPrefix: string
): T {
const url = new URL(presigned.url);
const prefix = new URL(urlPrefix);
if (prefix.pathname !== '/' || prefix.search || prefix.hash) {
throw new BlobInvalid('Upload URL prefix must contain only an origin');
}
url.protocol = prefix.protocol;
url.host = prefix.host;
return { ...presigned, url: url.toString() };
}
private async multipartPartContentLength(
workspaceId: string,
key: string,
uploadId: string,
partNumber: number
) {
const record = await this.models.blob.get(workspaceId, key);
if (!record || record.status === 'completed') {
throw new BlobInvalid('Multipart upload is not pending');
}
if (record.uploadId !== uploadId) {
throw new BlobInvalid('Upload id mismatch');
}
const offset = (partNumber - 1) * MULTIPART_PART_SIZE;
if (
!Number.isInteger(partNumber) ||
partNumber < 1 ||
offset >= record.size
) {
throw new BlobInvalid('Invalid part number');
}
return Math.min(MULTIPART_PART_SIZE, record.size - offset);
}
}
packages/backend/server/src/core/workspaces/resolvers/blob.ts
+56 -49
View File
@@ -248,63 +248,70 @@ export class WorkspaceBlobResolver {
}
const metadata = { contentType: mime, contentLength: size };
const capabilities = await this.storage.capabilities();
let init: BlobUploadInit | null = null;
let uploadIdForRecord: string | null = null;
// try to resume multipart uploads
if (capabilities.multipartDirect && record && record.uploadId) {
const uploadedParts = await this.storage.listMultipartUploadParts(
workspaceId,
key,
record.uploadId
);
try {
const capabilities = await this.storage.capabilities();
if (uploadedParts) {
return {
method: BlobUploadMethod.MULTIPART,
blobKey: key,
uploadId: record.uploadId,
partSize: MULTIPART_PART_SIZE,
uploadedParts,
};
}
}
// try to resume multipart uploads
if (capabilities.multipartDirect && record && record.uploadId) {
const uploadedParts = await this.storage.listMultipartUploadParts(
workspaceId,
key,
record.uploadId
);
if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) {
const multipart = await this.storage.createMultipartUpload(
workspaceId,
key,
metadata
);
if (multipart) {
uploadIdForRecord = multipart.uploadId;
init = {
method: BlobUploadMethod.MULTIPART,
blobKey: key,
uploadId: multipart.uploadId,
partSize: MULTIPART_PART_SIZE,
expiresAt: multipart.expiresAt,
uploadedParts: [],
};
if (uploadedParts) {
return {
method: BlobUploadMethod.MULTIPART,
blobKey: key,
uploadId: record.uploadId,
partSize: MULTIPART_PART_SIZE,
uploadedParts,
};
}
}
}
if (!init && capabilities.presignPut) {
const presigned = await this.storage.presignPut(
workspaceId,
key,
metadata
);
if (presigned) {
init = {
method: BlobUploadMethod.PRESIGNED,
blobKey: key,
uploadUrl: presigned.url,
headers: presigned.headers,
expiresAt: presigned.expiresAt,
};
if (capabilities.multipartDirect && size >= MULTIPART_THRESHOLD) {
const multipart = await this.storage.createMultipartUpload(
workspaceId,
key,
metadata
);
if (multipart) {
uploadIdForRecord = multipart.uploadId;
init = {
method: BlobUploadMethod.MULTIPART,
blobKey: key,
uploadId: multipart.uploadId,
partSize: MULTIPART_PART_SIZE,
expiresAt: multipart.expiresAt,
uploadedParts: [],
};
}
}
if (!init && capabilities.presignPut) {
const presigned = await this.storage.presignPut(
workspaceId,
key,
metadata
);
if (presigned) {
init = {
method: BlobUploadMethod.PRESIGNED,
blobKey: key,
uploadUrl: presigned.url,
headers: presigned.headers,
expiresAt: presigned.expiresAt,
};
}
}
} catch (error) {
this.logger.warn('Failed to initialize direct blob upload', error);
init = null;
uploadIdForRecord = null;
}
if (!init) {