feat(server): lightweight s3 client (#14348)

#### PR Dependency Tree


* **PR #14348** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a dedicated S3-compatible client package and expanded
S3-compatible storage config (endpoint, region, forcePathStyle,
requestTimeoutMs, minPartSize, presign options, sessionToken).
* Document sync now broadcasts batched/compressed doc updates for more
efficient real-time syncing.

* **Tests**
* New unit and benchmark tests for base64 utilities and S3 multipart
listing; updated storage-related tests to match new formats.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

tools/cli/package.json

@@ -16,7 +16,7 @@
   },
   "dependencies": {
     "@affine-tools/utils": "workspace:*",
-    "@aws-sdk/client-s3": "^3.948.0",
+    "@affine/s3-compat": "workspace:*",
     "@napi-rs/simple-git": "^0.1.22",
     "@perfsee/webpack": "^1.13.0",
     "@sentry/webpack-plugin": "^3.0.0",

tools/cli/src/webpack/s3-plugin.ts

@@ -1,8 +1,7 @@
 import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 
-import type { PutObjectCommandInput } from '@aws-sdk/client-s3';
-import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { createS3CompatClient } from '@affine/s3-compat';
 import { lookup } from 'mime-types';
 import type { Compiler, WebpackPluginInstance } from 'webpack';
 
@@ -11,16 +10,18 @@ export const R2_BUCKET =
   (process.env.BUILD_TYPE === 'canary' ? 'assets-dev' : 'assets-prod');
 
 export class WebpackS3Plugin implements WebpackPluginInstance {
-  private readonly s3 = new S3Client({
-    region: 'auto',
-    endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
-    credentials: {
+  private readonly s3 = createS3CompatClient(
+    {
+      region: 'auto',
+      bucket: R2_BUCKET,
+      forcePathStyle: true,
+      endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
+    },
+    {
       accessKeyId: process.env.R2_ACCESS_KEY_ID!,
       secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
-    },
-    requestChecksumCalculation: 'WHEN_REQUIRED',
-    responseChecksumValidation: 'WHEN_REQUIRED',
-  });
+    }
+  );
 
   apply(compiler: Compiler) {
     compiler.hooks.assetEmitted.tapPromise(
@@ -31,16 +32,11 @@ export class WebpackS3Plugin implements WebpackPluginInstance {
         }
         const assetPath = join(outputPath, asset);
         const assetSource = await readFile(assetPath);
-        const putObjectCommandOptions: PutObjectCommandInput = {
-          Body: assetSource,
-          Bucket: R2_BUCKET,
-          Key: asset,
-        };
-        const contentType = lookup(asset);
-        if (contentType) {
-          putObjectCommandOptions.ContentType = contentType;
-        }
-        await this.s3.send(new PutObjectCommand(putObjectCommandOptions));
+        const contentType = lookup(asset) || undefined;
+        await this.s3.putObject(asset, assetSource, {
+          contentType,
+          contentLength: assetSource.byteLength,
+        });
       }
     );
   }

tools/cli/tsconfig.json

@@ -6,5 +6,8 @@
     "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo"
   },
   "include": ["./src"],
-  "references": [{ "path": "../utils" }]
+  "references": [
+    { "path": "../utils" },
+    { "path": "../../packages/common/s3-compat" }
+  ]
 }

tools/utils/src/workspace.gen.ts

@@ -1166,6 +1166,7 @@ export const PackageList = [
     location: 'packages/backend/server',
     name: '@affine/server',
     workspaceDependencies: [
+      'packages/common/s3-compat',
       'packages/backend/native',
       'tools/cli',
       'tools/utils',
@@ -1222,6 +1223,11 @@ export const PackageList = [
     name: '@affine/reader',
     workspaceDependencies: ['blocksuite/affine/all'],
   },
+  {
+    location: 'packages/common/s3-compat',
+    name: '@affine/s3-compat',
+    workspaceDependencies: [],
+  },
   {
     location: 'packages/frontend/admin',
     name: '@affine/admin',
@@ -1462,7 +1468,7 @@ export const PackageList = [
   {
     location: 'tools/cli',
     name: '@affine-tools/cli',
-    workspaceDependencies: ['tools/utils'],
+    workspaceDependencies: ['tools/utils', 'packages/common/s3-compat'],
   },
   {
     location: 'tools/commitlint',
@@ -1580,6 +1586,7 @@ export type PackageName =
   | '@toeverything/infra'
   | '@affine/nbstore'
   | '@affine/reader'
+  | '@affine/s3-compat'
   | '@affine/admin'
   | '@affine/android'
   | '@affine/electron'