From f2e20728785ce870ad5db37bba6a69b8af2b47fb Mon Sep 17 00:00:00 2001 From: darkskygit Date: Tue, 1 Apr 2025 15:14:06 +0000 Subject: [PATCH] feat(server): s3 presigned url (#11364) --- .docker/selfhost/schema.json | 49 ++++++++++++--- packages/backend/server/package.json | 3 +- .../src/base/storage/providers/index.ts | 10 +++ .../src/base/storage/providers/provider.ts | 9 ++- .../server/src/base/storage/providers/r2.ts | 63 ++++++++++++++++++- .../server/src/base/storage/providers/s3.ts | 43 ++++++++++--- .../src/base/storage/providers/utils.ts | 2 + .../server/src/core/storage/wrappers/blob.ts | 4 +- .../server/src/core/workspaces/controller.ts | 11 +++- .../server/src/plugins/copilot/controller.ts | 12 +++- .../server/src/plugins/copilot/storage.ts | 9 ++- packages/frontend/admin/src/config.json | 17 +++-- yarn.lock | 33 +++++++++- 13 files changed, 233 insertions(+), 32 deletions(-) diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index 130a0479fb..05e9d91e24 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -9,27 +9,27 @@ "properties": { "db": { "type": "number", - "description": "The database index of redis server to be used(Must be less than 10).\n@default 0\n@environment `REDIS_DATABASE`", + "description": "The database index of redis server to be used(Must be less than 10).\n@default 0\n@environment `REDIS_SERVER_DATABASE`", "default": 0 }, "host": { "type": "string", - "description": "The host of the redis server.\n@default \"localhost\"\n@environment `REDIS_HOST`", + "description": "The host of the redis server.\n@default \"localhost\"\n@environment `REDIS_SERVER_HOST`", "default": "localhost" }, "port": { "type": "number", - "description": "The port of the redis server.\n@default 6379\n@environment `REDIS_PORT`", + "description": "The port of the redis server.\n@default 6379\n@environment `REDIS_SERVER_PORT`", "default": 6379 }, "username": { "type": "string", - "description": "The username of the redis server.\n@default \"\"\n@environment `REDIS_USERNAME`", + "description": "The username of the redis server.\n@default \"\"\n@environment `REDIS_SERVER_USERNAME`", "default": "" }, "password": { "type": "string", - "description": "The password of the redis server.\n@default \"\"\n@environment `REDIS_PASSWORD`", + "description": "The password of the redis server.\n@default \"\"\n@environment `REDIS_SERVER_PASSWORD`", "default": "" }, "ioredis": { @@ -427,6 +427,14 @@ "accountId": { "type": "string", "description": "The account id for the cloudflare r2 storage provider." + }, + "signDomain": { + "type": "string", + "description": "The presigned domain for the cloudflare r2 storage provider." + }, + "signKey": { + "type": "string", + "description": "The presigned key for the cloudflare r2 storage provider." } } } @@ -530,6 +538,14 @@ "accountId": { "type": "string", "description": "The account id for the cloudflare r2 storage provider." + }, + "signDomain": { + "type": "string", + "description": "The presigned domain for the cloudflare r2 storage provider." + }, + "signKey": { + "type": "string", + "description": "The presigned key for the cloudflare r2 storage provider." } } } @@ -557,8 +573,8 @@ }, "externalUrl": { "type": "string", - "description": "Base url of AFFiNE server, used for generating external urls.\nDefault to be `[server.protocol]://[server.host][:server.port]` if not specified.\n \n@default \"http://localhost:3010\"\n@environment `AFFINE_SERVER_EXTERNAL_URL`", - "default": "http://localhost:3010" + "description": "Base url of AFFiNE server, used for generating external urls.\nDefault to be `[server.protocol]://[server.host][:server.port]` if not specified.\n \n@default \"\"\n@environment `AFFINE_SERVER_EXTERNAL_URL`", + "default": "" }, "https": { "type": "boolean", @@ -593,6 +609,17 @@ } } }, + "docService": { + "type": "object", + "description": "Configuration for docService module", + "properties": { + "endpoint": { + "type": "string", + "description": "The endpoint of the doc service.\n@default \"\"\n@environment `DOC_SERVICE_ENDPOINT`", + "default": "" + } + } + }, "client": { "type": "object", "description": "Configuration for client module", @@ -765,6 +792,14 @@ "accountId": { "type": "string", "description": "The account id for the cloudflare r2 storage provider." + }, + "signDomain": { + "type": "string", + "description": "The presigned domain for the cloudflare r2 storage provider." + }, + "signKey": { + "type": "string", + "description": "The presigned key for the cloudflare r2 storage provider." } } } diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index bfd8e65f8e..c0dbd4f0ca 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -27,7 +27,8 @@ "dependencies": { "@ai-sdk/google": "^1.1.19", "@apollo/server": "^4.11.3", - "@aws-sdk/client-s3": "^3.709.0", + "@aws-sdk/client-s3": "^3.779.0", + "@aws-sdk/s3-request-presigner": "^3.779.0", "@fal-ai/serverless-client": "^0.15.0", "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.20.0", "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.4.1", diff --git a/packages/backend/server/src/base/storage/providers/index.ts b/packages/backend/server/src/base/storage/providers/index.ts index 01be8e9fd6..9521f83833 100644 --- a/packages/backend/server/src/base/storage/providers/index.ts +++ b/packages/backend/server/src/base/storage/providers/index.ts @@ -105,6 +105,16 @@ export const StorageJSONSchema: JSONSchema = { description: 'The account id for the cloudflare r2 storage provider.', }, + signDomain: { + type: 'string' as const, + description: + 'The presigned domain for the cloudflare r2 storage provider.', + }, + signKey: { + type: 'string' as const, + description: + 'The presigned key for the cloudflare r2 storage provider.', + }, }, }, }, diff --git a/packages/backend/server/src/base/storage/providers/provider.ts b/packages/backend/server/src/base/storage/providers/provider.ts index 06b8ac4bfb..2c765e20d0 100644 --- a/packages/backend/server/src/base/storage/providers/provider.ts +++ b/packages/backend/server/src/base/storage/providers/provider.ts @@ -33,8 +33,13 @@ export interface StorageProvider { ): Promise; head(key: string): Promise; get( - key: string - ): Promise<{ body?: BlobOutputType; metadata?: GetObjectMetadata }>; + key: string, + signedUrl?: boolean + ): Promise<{ + redirectUrl?: string; + body?: BlobOutputType; + metadata?: GetObjectMetadata; + }>; list(prefix?: string): Promise; delete(key: string): Promise; } diff --git a/packages/backend/server/src/base/storage/providers/r2.ts b/packages/backend/server/src/base/storage/providers/r2.ts index e7fe8f100d..be8fc337ba 100644 --- a/packages/backend/server/src/base/storage/providers/r2.ts +++ b/packages/backend/server/src/base/storage/providers/r2.ts @@ -1,15 +1,28 @@ import assert from 'node:assert'; +import { Readable } from 'node:stream'; import { Logger } from '@nestjs/common'; +import { GetObjectMetadata } from './provider'; import { S3StorageConfig, S3StorageProvider } from './s3'; export interface R2StorageConfig extends S3StorageConfig { accountId: string; + // r2 public domain with verification + // see https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it + // example rule: is_timed_hmac_valid_v0("your_secret", http.request.uri, 10800, http.request.timestamp.sec, 6) + signDomain?: string; + signKey?: string; } export class R2StorageProvider extends S3StorageProvider { - constructor(config: R2StorageConfig, bucket: string) { + private readonly encoder = new TextEncoder(); + private readonly key: Uint8Array; + + constructor( + private readonly config: R2StorageConfig, + bucket: string + ) { assert(config.accountId, 'accountId is required for R2 storage provider'); super( { @@ -23,5 +36,53 @@ export class R2StorageProvider extends S3StorageProvider { bucket ); this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`); + this.key = this.encoder.encode(config.signKey); + } + + private async signUrl(url: URL): Promise { + const timestamp = Math.floor(Date.now() / 1000); + const key = await crypto.subtle.importKey( + 'raw', + this.key, + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign', 'verify'] + ); + const mac = await crypto.subtle.sign( + 'HMAC', + key, + this.encoder.encode(`${url.pathname}${timestamp}`) + ); + + const base64Mac = Buffer.from(mac).toString('base64'); + url.searchParams.set('sign', `${timestamp}-${base64Mac}`); + return url.toString(); + } + + override async get( + key: string, + signedUrl?: boolean + ): Promise<{ + body?: Readable; + metadata?: GetObjectMetadata; + redirectUrl?: string; + }> { + const { signDomain } = this.config; + if (signedUrl && signDomain) { + const metadata = await this.head(key); + const url = await this.signUrl(new URL(`/${key}`, signDomain)); + if (metadata) { + return { + redirectUrl: url.toString(), + metadata, + }; + } + + // object not found + return {}; + } + + // fallback to s3 presigned url if signDomain is not configured + return super.get(key, signDomain ? false : signedUrl); } } diff --git a/packages/backend/server/src/base/storage/providers/s3.ts b/packages/backend/server/src/base/storage/providers/s3.ts index 7db115dcc0..b0576ec93f 100644 --- a/packages/backend/server/src/base/storage/providers/s3.ts +++ b/packages/backend/server/src/base/storage/providers/s3.ts @@ -11,6 +11,7 @@ import { S3Client, S3ClientConfig, } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { Logger } from '@nestjs/common'; import { @@ -20,7 +21,7 @@ import { PutObjectMetadata, StorageProvider, } from './provider'; -import { autoMetadata, toBuffer } from './utils'; +import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils'; export type S3StorageConfig = S3ClientConfig; @@ -106,17 +107,43 @@ export class S3StorageProvider implements StorageProvider { } } - async get(key: string): Promise<{ + async get( + key: string, + signedUrl?: boolean + ): Promise<{ body?: Readable; metadata?: GetObjectMetadata; + redirectUrl?: string; }> { try { - const obj = await this.client.send( - new GetObjectCommand({ - Bucket: this.bucket, - Key: key, - }) - ); + const command = new GetObjectCommand({ + Bucket: this.bucket, + Key: key, + }); + + if (signedUrl) { + const metadata = await this.head(key); + if (metadata) { + const url = await getSignedUrl( + this.client, + new GetObjectCommand({ + Bucket: this.bucket, + Key: key, + }), + { expiresIn: SIGNED_URL_EXPIRED } + ); + + return { + redirectUrl: url, + metadata, + }; + } + + // object not found + return {}; + } + + const obj = await this.client.send(command); if (!obj.Body) { this.logger.verbose(`Object \`${key}\` not found`); diff --git a/packages/backend/server/src/base/storage/providers/utils.ts b/packages/backend/server/src/base/storage/providers/utils.ts index ede448ba8f..4f926e8201 100644 --- a/packages/backend/server/src/base/storage/providers/utils.ts +++ b/packages/backend/server/src/base/storage/providers/utils.ts @@ -42,3 +42,5 @@ export function autoMetadata( return metadata; } + +export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index bb0e73c228..8ddefad9c1 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -67,8 +67,8 @@ export class WorkspaceBlobStorage { }); } - async get(workspaceId: string, key: string) { - return this.provider.get(`${workspaceId}/${key}`); + async get(workspaceId: string, key: string, signedUrl?: boolean) { + return this.provider.get(`${workspaceId}/${key}`, signedUrl); } async list(workspaceId: string, syncBlobMeta = true) { diff --git a/packages/backend/server/src/core/workspaces/controller.ts b/packages/backend/server/src/core/workspaces/controller.ts index 1d9367e265..0daea1d09a 100644 --- a/packages/backend/server/src/core/workspaces/controller.ts +++ b/packages/backend/server/src/core/workspaces/controller.ts @@ -43,7 +43,16 @@ export class WorkspacesController { .user(user?.id ?? 'anonymous') .workspace(workspaceId) .assert('Workspace.Read'); - const { body, metadata } = await this.storage.get(workspaceId, name); + const { body, metadata, redirectUrl } = await this.storage.get( + workspaceId, + name, + true + ); + + if (redirectUrl) { + // redirect to signed url + return res.redirect(redirectUrl); + } if (!body) { throw new BlobNotFound({ diff --git a/packages/backend/server/src/plugins/copilot/controller.ts b/packages/backend/server/src/plugins/copilot/controller.ts index 6417b10212..31dfff5ca7 100644 --- a/packages/backend/server/src/plugins/copilot/controller.ts +++ b/packages/backend/server/src/plugins/copilot/controller.ts @@ -602,7 +602,17 @@ export class CopilotController implements BeforeApplicationShutdown { @Param('workspaceId') workspaceId: string, @Param('key') key: string ) { - const { body, metadata } = await this.storage.get(userId, workspaceId, key); + const { body, metadata, redirectUrl } = await this.storage.get( + userId, + workspaceId, + key, + true + ); + + if (redirectUrl) { + // redirect to signed url + return res.redirect(redirectUrl); + } if (!body) { throw new BlobNotFound({ diff --git a/packages/backend/server/src/plugins/copilot/storage.ts b/packages/backend/server/src/plugins/copilot/storage.ts index 90813809e9..b40647486c 100644 --- a/packages/backend/server/src/plugins/copilot/storage.ts +++ b/packages/backend/server/src/plugins/copilot/storage.ts @@ -56,8 +56,13 @@ export class CopilotStorage { } @CallMetric('ai', 'blob_get') - async get(userId: string, workspaceId: string, key: string) { - return this.provider.get(`${userId}/${workspaceId}/${key}`); + async get( + userId: string, + workspaceId: string, + key: string, + signedUrl?: boolean + ) { + return this.provider.get(`${userId}/${workspaceId}/${key}`, signedUrl); } @CallMetric('ai', 'blob_delete') diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index 3b732be37d..47a0de6cb7 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -3,27 +3,27 @@ "db": { "type": "Number", "desc": "The database index of redis server to be used(Must be less than 10).", - "env": "REDIS_DATABASE" + "env": "REDIS_SERVER_DATABASE" }, "host": { "type": "String", "desc": "The host of the redis server.", - "env": "REDIS_HOST" + "env": "REDIS_SERVER_HOST" }, "port": { "type": "Number", "desc": "The port of the redis server.", - "env": "REDIS_PORT" + "env": "REDIS_SERVER_PORT" }, "username": { "type": "String", "desc": "The username of the redis server.", - "env": "REDIS_USERNAME" + "env": "REDIS_SERVER_USERNAME" }, "password": { "type": "String", "desc": "The password of the redis server.", - "env": "REDIS_PASSWORD" + "env": "REDIS_SERVER_PASSWORD" }, "ioredis": { "type": "Object", @@ -239,6 +239,13 @@ "desc": "Only allow users with early access features to access the app" } }, + "docService": { + "endpoint": { + "type": "String", + "desc": "The endpoint of the doc service.", + "env": "DOC_SERVICE_ENDPOINT" + } + }, "client": { "versionControl.enabled": { "type": "Boolean", diff --git a/yarn.lock b/yarn.lock index 8253946570..3d83a14f89 100644 --- a/yarn.lock +++ b/yarn.lock @@ -868,7 +868,8 @@ __metadata: "@affine/server-native": "workspace:*" "@ai-sdk/google": "npm:^1.1.19" "@apollo/server": "npm:^4.11.3" - "@aws-sdk/client-s3": "npm:^3.709.0" + "@aws-sdk/client-s3": "npm:^3.779.0" + "@aws-sdk/s3-request-presigner": "npm:^3.779.0" "@faker-js/faker": "npm:^9.6.0" "@fal-ai/serverless-client": "npm:^0.15.0" "@google-cloud/opentelemetry-cloud-monitoring-exporter": "npm:^0.20.0" @@ -1475,7 +1476,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-s3@npm:^3.709.0": +"@aws-sdk/client-s3@npm:^3.709.0, @aws-sdk/client-s3@npm:^3.779.0": version: 3.779.0 resolution: "@aws-sdk/client-s3@npm:3.779.0" dependencies: @@ -1922,6 +1923,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/s3-request-presigner@npm:^3.779.0": + version: 3.779.0 + resolution: "@aws-sdk/s3-request-presigner@npm:3.779.0" + dependencies: + "@aws-sdk/signature-v4-multi-region": "npm:3.775.0" + "@aws-sdk/types": "npm:3.775.0" + "@aws-sdk/util-format-url": "npm:3.775.0" + "@smithy/middleware-endpoint": "npm:^4.1.0" + "@smithy/protocol-http": "npm:^5.1.0" + "@smithy/smithy-client": "npm:^4.2.0" + "@smithy/types": "npm:^4.2.0" + tslib: "npm:^2.6.2" + checksum: 10/7a0b4d4b42fe0fbd603675a53ca831ce3ccf989c0d54be2e77aa4fea49c3c9873526fdf4f58103c856fa44d2b9fd29bec0f821f54bc5f2e37c5eaa340aa3b5fa + languageName: node + linkType: hard + "@aws-sdk/signature-v4-multi-region@npm:3.775.0": version: 3.775.0 resolution: "@aws-sdk/signature-v4-multi-region@npm:3.775.0" @@ -1981,6 +1998,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/util-format-url@npm:3.775.0": + version: 3.775.0 + resolution: "@aws-sdk/util-format-url@npm:3.775.0" + dependencies: + "@aws-sdk/types": "npm:3.775.0" + "@smithy/querystring-builder": "npm:^4.0.2" + "@smithy/types": "npm:^4.2.0" + tslib: "npm:^2.6.2" + checksum: 10/67bd40d7b9683af754dcbfa06cb08e482be541b453cb86f42465263742d6fd2882f3531ba5bf08a2f159f5eb0960f754f62be20e000722b49c2b4dcee9d36e3f + languageName: node + linkType: hard + "@aws-sdk/util-locate-window@npm:^3.0.0": version: 3.723.0 resolution: "@aws-sdk/util-locate-window@npm:3.723.0"