feat(server): s3 presigned url (#11364)

packages/backend/server/src/base/storage/providers/index.ts

@@ -105,6 +105,16 @@ export const StorageJSONSchema: JSONSchema = {
               description:
                 'The account id for the cloudflare r2 storage provider.',
             },
+            signDomain: {
+              type: 'string' as const,
+              description:
+                'The presigned domain for the cloudflare r2 storage provider.',
+            },
+            signKey: {
+              type: 'string' as const,
+              description:
+                'The presigned key for the cloudflare r2 storage provider.',
+            },
           },
         },
       },

packages/backend/server/src/base/storage/providers/provider.ts

@@ -33,8 +33,13 @@ export interface StorageProvider {
   ): Promise<void>;
   head(key: string): Promise<GetObjectMetadata | undefined>;
   get(
-    key: string
-  ): Promise<{ body?: BlobOutputType; metadata?: GetObjectMetadata }>;
+    key: string,
+    signedUrl?: boolean
+  ): Promise<{
+    redirectUrl?: string;
+    body?: BlobOutputType;
+    metadata?: GetObjectMetadata;
+  }>;
   list(prefix?: string): Promise<ListObjectsMetadata[]>;
   delete(key: string): Promise<void>;
 }

packages/backend/server/src/base/storage/providers/r2.ts

@@ -1,15 +1,28 @@
 import assert from 'node:assert';
+import { Readable } from 'node:stream';
 
 import { Logger } from '@nestjs/common';
 
+import { GetObjectMetadata } from './provider';
 import { S3StorageConfig, S3StorageProvider } from './s3';
 
 export interface R2StorageConfig extends S3StorageConfig {
   accountId: string;
+  // r2 public domain with verification
+  // see https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it
+  // example rule: is_timed_hmac_valid_v0("your_secret", http.request.uri, 10800, http.request.timestamp.sec, 6)
+  signDomain?: string;
+  signKey?: string;
 }
 
 export class R2StorageProvider extends S3StorageProvider {
-  constructor(config: R2StorageConfig, bucket: string) {
+  private readonly encoder = new TextEncoder();
+  private readonly key: Uint8Array;
+
+  constructor(
+    private readonly config: R2StorageConfig,
+    bucket: string
+  ) {
     assert(config.accountId, 'accountId is required for R2 storage provider');
     super(
       {
@@ -23,5 +36,53 @@ export class R2StorageProvider extends S3StorageProvider {
       bucket
     );
     this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
+    this.key = this.encoder.encode(config.signKey);
+  }
+
+  private async signUrl(url: URL): Promise<string> {
+    const timestamp = Math.floor(Date.now() / 1000);
+    const key = await crypto.subtle.importKey(
+      'raw',
+      this.key,
+      { name: 'HMAC', hash: 'SHA-256' },
+      false,
+      ['sign', 'verify']
+    );
+    const mac = await crypto.subtle.sign(
+      'HMAC',
+      key,
+      this.encoder.encode(`${url.pathname}${timestamp}`)
+    );
+
+    const base64Mac = Buffer.from(mac).toString('base64');
+    url.searchParams.set('sign', `${timestamp}-${base64Mac}`);
+    return url.toString();
+  }
+
+  override async get(
+    key: string,
+    signedUrl?: boolean
+  ): Promise<{
+    body?: Readable;
+    metadata?: GetObjectMetadata;
+    redirectUrl?: string;
+  }> {
+    const { signDomain } = this.config;
+    if (signedUrl && signDomain) {
+      const metadata = await this.head(key);
+      const url = await this.signUrl(new URL(`/${key}`, signDomain));
+      if (metadata) {
+        return {
+          redirectUrl: url.toString(),
+          metadata,
+        };
+      }
+
+      // object not found
+      return {};
+    }
+
+    // fallback to s3 presigned url if signDomain is not configured
+    return super.get(key, signDomain ? false : signedUrl);
   }
 }

packages/backend/server/src/base/storage/providers/s3.ts

@@ -11,6 +11,7 @@ import {
   S3Client,
   S3ClientConfig,
 } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Logger } from '@nestjs/common';
 
 import {
@@ -20,7 +21,7 @@ import {
   PutObjectMetadata,
   StorageProvider,
 } from './provider';
-import { autoMetadata, toBuffer } from './utils';
+import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils';
 
 export type S3StorageConfig = S3ClientConfig;
 
@@ -106,17 +107,43 @@ export class S3StorageProvider implements StorageProvider {
     }
   }
 
-  async get(key: string): Promise<{
+  async get(
+    key: string,
+    signedUrl?: boolean
+  ): Promise<{
     body?: Readable;
     metadata?: GetObjectMetadata;
+    redirectUrl?: string;
   }> {
     try {
-      const obj = await this.client.send(
-        new GetObjectCommand({
-          Bucket: this.bucket,
-          Key: key,
-        })
-      );
+      const command = new GetObjectCommand({
+        Bucket: this.bucket,
+        Key: key,
+      });
+
+      if (signedUrl) {
+        const metadata = await this.head(key);
+        if (metadata) {
+          const url = await getSignedUrl(
+            this.client,
+            new GetObjectCommand({
+              Bucket: this.bucket,
+              Key: key,
+            }),
+            { expiresIn: SIGNED_URL_EXPIRED }
+          );
+
+          return {
+            redirectUrl: url,
+            metadata,
+          };
+        }
+
+        // object not found
+        return {};
+      }
+
+      const obj = await this.client.send(command);
 
       if (!obj.Body) {
         this.logger.verbose(`Object \`${key}\` not found`);

packages/backend/server/src/base/storage/providers/utils.ts

@@ -42,3 +42,5 @@ export function autoMetadata(
 
   return metadata;
 }
+
+export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour

packages/backend/server/src/core/storage/wrappers/blob.ts

@@ -67,8 +67,8 @@ export class WorkspaceBlobStorage {
     });
   }
 
-  async get(workspaceId: string, key: string) {
-    return this.provider.get(`${workspaceId}/${key}`);
+  async get(workspaceId: string, key: string, signedUrl?: boolean) {
+    return this.provider.get(`${workspaceId}/${key}`, signedUrl);
   }
 
   async list(workspaceId: string, syncBlobMeta = true) {

packages/backend/server/src/core/workspaces/controller.ts

@@ -43,7 +43,16 @@ export class WorkspacesController {
       .user(user?.id ?? 'anonymous')
       .workspace(workspaceId)
       .assert('Workspace.Read');
-    const { body, metadata } = await this.storage.get(workspaceId, name);
+    const { body, metadata, redirectUrl } = await this.storage.get(
+      workspaceId,
+      name,
+      true
+    );
+
+    if (redirectUrl) {
+      // redirect to signed url
+      return res.redirect(redirectUrl);
+    }
 
     if (!body) {
       throw new BlobNotFound({

packages/backend/server/src/plugins/copilot/controller.ts

@@ -602,7 +602,17 @@ export class CopilotController implements BeforeApplicationShutdown {
     @Param('workspaceId') workspaceId: string,
     @Param('key') key: string
   ) {
-    const { body, metadata } = await this.storage.get(userId, workspaceId, key);
+    const { body, metadata, redirectUrl } = await this.storage.get(
+      userId,
+      workspaceId,
+      key,
+      true
+    );
+
+    if (redirectUrl) {
+      // redirect to signed url
+      return res.redirect(redirectUrl);
+    }
 
     if (!body) {
       throw new BlobNotFound({

packages/backend/server/src/plugins/copilot/storage.ts

@@ -56,8 +56,13 @@ export class CopilotStorage {
   }
 
   @CallMetric('ai', 'blob_get')
-  async get(userId: string, workspaceId: string, key: string) {
-    return this.provider.get(`${userId}/${workspaceId}/${key}`);
+  async get(
+    userId: string,
+    workspaceId: string,
+    key: string,
+    signedUrl?: boolean
+  ) {
+    return this.provider.get(`${userId}/${workspaceId}/${key}`, signedUrl);
   }
 
   @CallMetric('ai', 'blob_delete')