feat(server): s3 presigned url (#11364)

.docker/selfhost/schema.json

@@ -9,27 +9,27 @@
       "properties": {
         "db": {
           "type": "number",
-          "description": "The database index of redis server to be used(Must be less than 10).\n@default 0\n@environment `REDIS_DATABASE`",
+          "description": "The database index of redis server to be used(Must be less than 10).\n@default 0\n@environment `REDIS_SERVER_DATABASE`",
           "default": 0
         },
         "host": {
           "type": "string",
-          "description": "The host of the redis server.\n@default \"localhost\"\n@environment `REDIS_HOST`",
+          "description": "The host of the redis server.\n@default \"localhost\"\n@environment `REDIS_SERVER_HOST`",
           "default": "localhost"
         },
         "port": {
           "type": "number",
-          "description": "The port of the redis server.\n@default 6379\n@environment `REDIS_PORT`",
+          "description": "The port of the redis server.\n@default 6379\n@environment `REDIS_SERVER_PORT`",
           "default": 6379
         },
         "username": {
           "type": "string",
-          "description": "The username of the redis server.\n@default \"\"\n@environment `REDIS_USERNAME`",
+          "description": "The username of the redis server.\n@default \"\"\n@environment `REDIS_SERVER_USERNAME`",
           "default": ""
         },
         "password": {
           "type": "string",
-          "description": "The password of the redis server.\n@default \"\"\n@environment `REDIS_PASSWORD`",
+          "description": "The password of the redis server.\n@default \"\"\n@environment `REDIS_SERVER_PASSWORD`",
           "default": ""
         },
         "ioredis": {
@@ -427,6 +427,14 @@
                     "accountId": {
                       "type": "string",
                       "description": "The account id for the cloudflare r2 storage provider."
+                    },
+                    "signDomain": {
+                      "type": "string",
+                      "description": "The presigned domain for the cloudflare r2 storage provider."
+                    },
+                    "signKey": {
+                      "type": "string",
+                      "description": "The presigned key for the cloudflare r2 storage provider."
                     }
                   }
                 }
@@ -530,6 +538,14 @@
                     "accountId": {
                       "type": "string",
                       "description": "The account id for the cloudflare r2 storage provider."
+                    },
+                    "signDomain": {
+                      "type": "string",
+                      "description": "The presigned domain for the cloudflare r2 storage provider."
+                    },
+                    "signKey": {
+                      "type": "string",
+                      "description": "The presigned key for the cloudflare r2 storage provider."
                     }
                   }
                 }
@@ -557,8 +573,8 @@
         },
         "externalUrl": {
           "type": "string",
-          "description": "Base url of AFFiNE server, used for generating external urls.\nDefault to be `[server.protocol]://[server.host][:server.port]` if not specified.\n    \n@default \"http://localhost:3010\"\n@environment `AFFINE_SERVER_EXTERNAL_URL`",
+          "description": "Base url of AFFiNE server, used for generating external urls.\nDefault to be `[server.protocol]://[server.host][:server.port]` if not specified.\n    \n@default \"\"\n@environment `AFFINE_SERVER_EXTERNAL_URL`",
-          "default": "http://localhost:3010"
+          "default": ""
         },
         "https": {
           "type": "boolean",
@@ -593,6 +609,17 @@
         }
       }
     },
+    "docService": {
+      "type": "object",
+      "description": "Configuration for docService module",
+      "properties": {
+        "endpoint": {
+          "type": "string",
+          "description": "The endpoint of the doc service.\n@default \"\"\n@environment `DOC_SERVICE_ENDPOINT`",
+          "default": ""
+        }
+      }
+    },
     "client": {
       "type": "object",
       "description": "Configuration for client module",
@@ -765,6 +792,14 @@
                     "accountId": {
                       "type": "string",
                       "description": "The account id for the cloudflare r2 storage provider."
+                    },
+                    "signDomain": {
+                      "type": "string",
+                      "description": "The presigned domain for the cloudflare r2 storage provider."
+                    },
+                    "signKey": {
+                      "type": "string",
+                      "description": "The presigned key for the cloudflare r2 storage provider."
                     }
                   }
                 }

packages/backend/server/package.json

@@ -27,7 +27,8 @@
   "dependencies": {
     "@ai-sdk/google": "^1.1.19",
     "@apollo/server": "^4.11.3",
-    "@aws-sdk/client-s3": "^3.709.0",
+    "@aws-sdk/client-s3": "^3.779.0",
+    "@aws-sdk/s3-request-presigner": "^3.779.0",
     "@fal-ai/serverless-client": "^0.15.0",
     "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.20.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.4.1",

packages/backend/server/src/base/storage/providers/index.ts

@@ -105,6 +105,16 @@ export const StorageJSONSchema: JSONSchema = {
               description:
                 'The account id for the cloudflare r2 storage provider.',
             },
+            signDomain: {
+              type: 'string' as const,
+              description:
+                'The presigned domain for the cloudflare r2 storage provider.',
+            },
+            signKey: {
+              type: 'string' as const,
+              description:
+                'The presigned key for the cloudflare r2 storage provider.',
+            },
           },
         },
       },

packages/backend/server/src/base/storage/providers/provider.ts

@@ -33,8 +33,13 @@ export interface StorageProvider {
   ): Promise<void>;
   head(key: string): Promise<GetObjectMetadata | undefined>;
   get(
-    key: string
+    key: string,
-  ): Promise<{ body?: BlobOutputType; metadata?: GetObjectMetadata }>;
+    signedUrl?: boolean
+  ): Promise<{
+    redirectUrl?: string;
+    body?: BlobOutputType;
+    metadata?: GetObjectMetadata;
+  }>;
   list(prefix?: string): Promise<ListObjectsMetadata[]>;
   delete(key: string): Promise<void>;
 }

packages/backend/server/src/base/storage/providers/r2.ts

@@ -1,15 +1,28 @@
 import assert from 'node:assert';
+import { Readable } from 'node:stream';
 
 import { Logger } from '@nestjs/common';
 
+import { GetObjectMetadata } from './provider';
 import { S3StorageConfig, S3StorageProvider } from './s3';
 
 export interface R2StorageConfig extends S3StorageConfig {
   accountId: string;
+  // r2 public domain with verification
+  // see https://developers.cloudflare.com/waf/custom-rules/use-cases/configure-token-authentication/ to configure it
+  // example rule: is_timed_hmac_valid_v0("your_secret", http.request.uri, 10800, http.request.timestamp.sec, 6)
+  signDomain?: string;
+  signKey?: string;
 }
 
 export class R2StorageProvider extends S3StorageProvider {
-  constructor(config: R2StorageConfig, bucket: string) {
+  private readonly encoder = new TextEncoder();
+  private readonly key: Uint8Array;
+
+  constructor(
+    private readonly config: R2StorageConfig,
+    bucket: string
+  ) {
     assert(config.accountId, 'accountId is required for R2 storage provider');
     super(
       {
@@ -23,5 +36,53 @@ export class R2StorageProvider extends S3StorageProvider {
       bucket
     );
     this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
+    this.key = this.encoder.encode(config.signKey);
+  }
+
+  private async signUrl(url: URL): Promise<string> {
+    const timestamp = Math.floor(Date.now() / 1000);
+    const key = await crypto.subtle.importKey(
+      'raw',
+      this.key,
+      { name: 'HMAC', hash: 'SHA-256' },
+      false,
+      ['sign', 'verify']
+    );
+    const mac = await crypto.subtle.sign(
+      'HMAC',
+      key,
+      this.encoder.encode(`${url.pathname}${timestamp}`)
+    );
+
+    const base64Mac = Buffer.from(mac).toString('base64');
+    url.searchParams.set('sign', `${timestamp}-${base64Mac}`);
+    return url.toString();
+  }
+
+  override async get(
+    key: string,
+    signedUrl?: boolean
+  ): Promise<{
+    body?: Readable;
+    metadata?: GetObjectMetadata;
+    redirectUrl?: string;
+  }> {
+    const { signDomain } = this.config;
+    if (signedUrl && signDomain) {
+      const metadata = await this.head(key);
+      const url = await this.signUrl(new URL(`/${key}`, signDomain));
+      if (metadata) {
+        return {
+          redirectUrl: url.toString(),
+          metadata,
+        };
+      }
+
+      // object not found
+      return {};
+    }
+
+    // fallback to s3 presigned url if signDomain is not configured
+    return super.get(key, signDomain ? false : signedUrl);
   }
 }

packages/backend/server/src/base/storage/providers/s3.ts

@@ -11,6 +11,7 @@ import {
   S3Client,
   S3ClientConfig,
 } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Logger } from '@nestjs/common';
 
 import {
@@ -20,7 +21,7 @@ import {
   PutObjectMetadata,
   StorageProvider,
 } from './provider';
-import { autoMetadata, toBuffer } from './utils';
+import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils';
 
 export type S3StorageConfig = S3ClientConfig;
 
@@ -106,17 +107,43 @@ export class S3StorageProvider implements StorageProvider {
     }
   }
 
-  async get(key: string): Promise<{
+  async get(
+    key: string,
+    signedUrl?: boolean
+  ): Promise<{
     body?: Readable;
     metadata?: GetObjectMetadata;
+    redirectUrl?: string;
   }> {
     try {
-      const obj = await this.client.send(
+      const command = new GetObjectCommand({
-        new GetObjectCommand({
+        Bucket: this.bucket,
-          Bucket: this.bucket,
+        Key: key,
-          Key: key,
+      });
-        })
+
-      );
+      if (signedUrl) {
+        const metadata = await this.head(key);
+        if (metadata) {
+          const url = await getSignedUrl(
+            this.client,
+            new GetObjectCommand({
+              Bucket: this.bucket,
+              Key: key,
+            }),
+            { expiresIn: SIGNED_URL_EXPIRED }
+          );
+
+          return {
+            redirectUrl: url,
+            metadata,
+          };
+        }
+
+        // object not found
+        return {};
+      }
+
+      const obj = await this.client.send(command);
 
       if (!obj.Body) {
         this.logger.verbose(`Object \`${key}\` not found`);

packages/backend/server/src/base/storage/providers/utils.ts

@@ -42,3 +42,5 @@ export function autoMetadata(
 
   return metadata;
 }
+
+export const SIGNED_URL_EXPIRED = 60 * 60; // 1 hour

packages/backend/server/src/core/storage/wrappers/blob.ts

@@ -67,8 +67,8 @@ export class WorkspaceBlobStorage {
     });
   }
 
-  async get(workspaceId: string, key: string) {
+  async get(workspaceId: string, key: string, signedUrl?: boolean) {
-    return this.provider.get(`${workspaceId}/${key}`);
+    return this.provider.get(`${workspaceId}/${key}`, signedUrl);
   }
 
   async list(workspaceId: string, syncBlobMeta = true) {

packages/backend/server/src/core/workspaces/controller.ts

@@ -43,7 +43,16 @@ export class WorkspacesController {
       .user(user?.id ?? 'anonymous')
       .workspace(workspaceId)
       .assert('Workspace.Read');
-    const { body, metadata } = await this.storage.get(workspaceId, name);
+    const { body, metadata, redirectUrl } = await this.storage.get(
+      workspaceId,
+      name,
+      true
+    );
+
+    if (redirectUrl) {
+      // redirect to signed url
+      return res.redirect(redirectUrl);
+    }
 
     if (!body) {
       throw new BlobNotFound({

packages/backend/server/src/plugins/copilot/controller.ts

@@ -602,7 +602,17 @@ export class CopilotController implements BeforeApplicationShutdown {
     @Param('workspaceId') workspaceId: string,
     @Param('key') key: string
   ) {
-    const { body, metadata } = await this.storage.get(userId, workspaceId, key);
+    const { body, metadata, redirectUrl } = await this.storage.get(
+      userId,
+      workspaceId,
+      key,
+      true
+    );
+
+    if (redirectUrl) {
+      // redirect to signed url
+      return res.redirect(redirectUrl);
+    }
 
     if (!body) {
       throw new BlobNotFound({

packages/backend/server/src/plugins/copilot/storage.ts

@@ -56,8 +56,13 @@ export class CopilotStorage {
   }
 
   @CallMetric('ai', 'blob_get')
-  async get(userId: string, workspaceId: string, key: string) {
+  async get(
-    return this.provider.get(`${userId}/${workspaceId}/${key}`);
+    userId: string,
+    workspaceId: string,
+    key: string,
+    signedUrl?: boolean
+  ) {
+    return this.provider.get(`${userId}/${workspaceId}/${key}`, signedUrl);
   }
 
   @CallMetric('ai', 'blob_delete')

packages/frontend/admin/src/config.json

@@ -3,27 +3,27 @@
     "db": {
       "type": "Number",
       "desc": "The database index of redis server to be used(Must be less than 10).",
-      "env": "REDIS_DATABASE"
+      "env": "REDIS_SERVER_DATABASE"
     },
     "host": {
       "type": "String",
       "desc": "The host of the redis server.",
-      "env": "REDIS_HOST"
+      "env": "REDIS_SERVER_HOST"
     },
     "port": {
       "type": "Number",
       "desc": "The port of the redis server.",
-      "env": "REDIS_PORT"
+      "env": "REDIS_SERVER_PORT"
     },
     "username": {
       "type": "String",
       "desc": "The username of the redis server.",
-      "env": "REDIS_USERNAME"
+      "env": "REDIS_SERVER_USERNAME"
     },
     "password": {
       "type": "String",
       "desc": "The password of the redis server.",
-      "env": "REDIS_PASSWORD"
+      "env": "REDIS_SERVER_PASSWORD"
     },
     "ioredis": {
       "type": "Object",
@@ -239,6 +239,13 @@
       "desc": "Only allow users with early access features to access the app"
     }
   },
+  "docService": {
+    "endpoint": {
+      "type": "String",
+      "desc": "The endpoint of the doc service.",
+      "env": "DOC_SERVICE_ENDPOINT"
+    }
+  },
   "client": {
     "versionControl.enabled": {
       "type": "Boolean",

yarn.lock

@@ -868,7 +868,8 @@ __metadata:
     "@affine/server-native": "workspace:*"
     "@ai-sdk/google": "npm:^1.1.19"
     "@apollo/server": "npm:^4.11.3"
-    "@aws-sdk/client-s3": "npm:^3.709.0"
+    "@aws-sdk/client-s3": "npm:^3.779.0"
+    "@aws-sdk/s3-request-presigner": "npm:^3.779.0"
     "@faker-js/faker": "npm:^9.6.0"
     "@fal-ai/serverless-client": "npm:^0.15.0"
     "@google-cloud/opentelemetry-cloud-monitoring-exporter": "npm:^0.20.0"
@@ -1475,7 +1476,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@aws-sdk/client-s3@npm:^3.709.0":
+"@aws-sdk/client-s3@npm:^3.709.0, @aws-sdk/client-s3@npm:^3.779.0":
   version: 3.779.0
   resolution: "@aws-sdk/client-s3@npm:3.779.0"
   dependencies:
@@ -1922,6 +1923,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@aws-sdk/s3-request-presigner@npm:^3.779.0":
+  version: 3.779.0
+  resolution: "@aws-sdk/s3-request-presigner@npm:3.779.0"
+  dependencies:
+    "@aws-sdk/signature-v4-multi-region": "npm:3.775.0"
+    "@aws-sdk/types": "npm:3.775.0"
+    "@aws-sdk/util-format-url": "npm:3.775.0"
+    "@smithy/middleware-endpoint": "npm:^4.1.0"
+    "@smithy/protocol-http": "npm:^5.1.0"
+    "@smithy/smithy-client": "npm:^4.2.0"
+    "@smithy/types": "npm:^4.2.0"
+    tslib: "npm:^2.6.2"
+  checksum: 10/7a0b4d4b42fe0fbd603675a53ca831ce3ccf989c0d54be2e77aa4fea49c3c9873526fdf4f58103c856fa44d2b9fd29bec0f821f54bc5f2e37c5eaa340aa3b5fa
+  languageName: node
+  linkType: hard
+
 "@aws-sdk/signature-v4-multi-region@npm:3.775.0":
   version: 3.775.0
   resolution: "@aws-sdk/signature-v4-multi-region@npm:3.775.0"
@@ -1981,6 +1998,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@aws-sdk/util-format-url@npm:3.775.0":
+  version: 3.775.0
+  resolution: "@aws-sdk/util-format-url@npm:3.775.0"
+  dependencies:
+    "@aws-sdk/types": "npm:3.775.0"
+    "@smithy/querystring-builder": "npm:^4.0.2"
+    "@smithy/types": "npm:^4.2.0"
+    tslib: "npm:^2.6.2"
+  checksum: 10/67bd40d7b9683af754dcbfa06cb08e482be541b453cb86f42465263742d6fd2882f3531ba5bf08a2f159f5eb0960f754f62be20e000722b49c2b4dcee9d36e3f
+  languageName: node
+  linkType: hard
+
 "@aws-sdk/util-locate-window@npm:^3.0.0":
   version: 3.723.0
   resolution: "@aws-sdk/util-locate-window@npm:3.723.0"