fix: test & lint

packages/backend/server/src/__tests__/e2e/create-app.ts

@@ -13,6 +13,7 @@ import {
   AFFiNELogger,
   CacheInterceptor,
   CloudThrottlerGuard,
+  ConfigFactory,
   EventBus,
   GlobalExceptionFilter,
   JobQueue,
@@ -250,6 +251,31 @@ export async function createApp(
   }
 
   const module = await builder.compile();
+  module.get(ConfigFactory).override({
+    storages: {
+      avatar: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'avatars',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+      blob: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'blobs',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+    },
+    copilot: {
+      storage: {
+        provider: 'assetpack',
+        bucket: 'copilot',
+        config: { path: '/tmp/affine-test-storage' },
+      },
+    },
+  });
 
   module.useCustomApplicationConstructor(TestingApp);
 

packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts

@@ -362,6 +362,7 @@ e2e.serial('should proxy single upload with valid signature', async t => {
   t.is(init.method, 'PRESIGNED');
   t.truthy(init.uploadUrl);
   const uploadUrl = new URL(init.uploadUrl, app.url);
+  t.is(uploadUrl.origin, 'https://cdn.example.com');
   t.is(uploadUrl.pathname, PROXY_UPLOAD_PATH);
 
   const res = await app
@@ -391,6 +392,7 @@ e2e.serial('should proxy multipart upload and return etag', async t => {
 
   const part = await getBlobUploadPartUrl(workspace.id, key, init.uploadId, 1);
   const partUrl = new URL(part.uploadUrl, app.url);
+  t.is(partUrl.origin, 'https://cdn.example.com');
   t.is(partUrl.pathname, PROXY_MULTIPART_PATH);
 
   const payload = Buffer.from('part-body');

packages/backend/server/src/__tests__/utils/testing-module.ts

@@ -9,7 +9,7 @@ import {
 import { PrismaClient } from '@prisma/client';
 
 import { buildAppModule, FunctionalityModules } from '../../app.module';
-import { AFFiNELogger, JobQueue } from '../../base';
+import { AFFiNELogger, ConfigFactory, JobQueue } from '../../base';
 import { GqlModule } from '../../base/graphql';
 import { ServerConfigModule } from '../../core';
 import { AuthGuard, AuthModule } from '../../core/auth';
@@ -99,6 +99,31 @@ export async function createTestingModule(
   }
 
   const module = await builder.compile();
+  module.get(ConfigFactory).override({
+    storages: {
+      avatar: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'avatars',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+      blob: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'blobs',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+    },
+    copilot: {
+      storage: {
+        provider: 'assetpack',
+        bucket: 'copilot',
+        config: { path: '/tmp/affine-test-storage' },
+      },
+    },
+  });
 
   const testingModule = module as TestingModule;
 

packages/backend/server/src/__tests__/workspace/blobs.e2e.ts

@@ -35,6 +35,22 @@ const RESTRICTED_QUOTA = {
 
 let app: TestingApp;
 let model: WorkspaceFeatureModel;
+type CompleteResult =
+  | {
+      ok: true;
+      contentType: string;
+      contentLength: number;
+      lastModifiedMs: number;
+    }
+  | {
+      ok: false;
+      reason:
+        | 'not_found'
+        | 'size_mismatch'
+        | 'mime_mismatch'
+        | 'checksum_mismatch'
+        | 'size_too_large';
+    };
 const objects = new Map<
   string,
   {
@@ -46,6 +62,7 @@ const objects = new Map<
     };
   }
 >();
+const completeResults = new Map<string, CompleteResult>();
 const storageRuntime = {
   providerCapabilities: async () => ({
     put: true,
@@ -100,25 +117,12 @@ const storageRuntime = {
   presignUploadPart: async () => undefined,
   listMultipartUploadParts: async () => undefined,
   completeMultipartUpload: async () => undefined,
-  completeWorkspaceBlobUpload: async (
-    workspaceId: string,
-    key: string,
-    expected: { size: number; mime: string }
-  ) => {
+  completeWorkspaceBlobUpload: async (workspaceId: string, key: string) => {
     const objectKey = `${workspaceId}/${key}`;
+    const configured = completeResults.get(objectKey);
+    if (configured) return configured;
     const object = objects.get(objectKey);
-    if (!object) {
-      return { ok: false, reason: 'not_found' };
-    }
-    if (object.metadata.contentLength !== expected.size) {
-      return { ok: false, reason: 'size_mismatch' };
-    }
-    if (object.metadata.contentType !== expected.mime) {
-      return { ok: false, reason: 'mime_mismatch' };
-    }
-    if (sha256Base64urlWithPadding(object.body) !== key) {
-      return { ok: false, reason: 'checksum_mismatch' };
-    }
+    if (!object) return { ok: false, reason: 'not_found' };
     await app.get(BlobModel).upsert({
       workspaceId,
       key,
@@ -159,6 +163,7 @@ test.before(async () => {
 test.beforeEach(async () => {
   await app.initTestingDB();
   objects.clear();
+  completeResults.clear();
 });
 
 test.after.always(async () => {
@@ -333,6 +338,10 @@ test('should reject complete when blob key mismatched', async t => {
     contentType: mime,
     contentLength: buffer.length,
   });
+  completeResults.set(`${workspace.id}/${wrongKey}`, {
+    ok: false,
+    reason: 'checksum_mismatch',
+  });
 
   await t.throwsAsync(() => completeBlobUpload(app, workspace.id, wrongKey), {
     message: 'Blob key mismatch',

packages/backend/server/src/base/config/register.ts

@@ -21,6 +21,7 @@ export type JSONSchema = { description?: string } & (
   | {
       type: 'object';
       properties?: Record<string, JSONSchema>;
+      required?: string[];
     }
 );
 

packages/backend/server/src/base/storage/providers/index.ts

@@ -232,8 +232,10 @@ export const StorageJSONSchema: JSONSchema = {
               type: 'string',
             },
           },
+          required: ['path'],
         },
       },
+      required: ['provider', 'bucket', 'config'],
     },
   ],
 };

packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts

@@ -4,8 +4,36 @@ import Sinon from 'sinon';
 import { StorageRuntimeProvider } from '../provider';
 
 function createProvider() {
-  const provider = new StorageRuntimeProvider();
+  const provider = new StorageRuntimeProvider({
+    db: {
+      datasourceUrl: 'postgresql://localhost:5432/affine',
+    },
+    storages: {
+      blob: {
+        storage: {
+          provider: 'fs',
+          bucket: 'blobs',
+          config: { path: '~/.affine/storage' },
+        },
+      },
+      avatar: {
+        storage: {
+          provider: 'fs',
+          bucket: 'avatars',
+          config: { path: '~/.affine/storage' },
+        },
+      },
+    },
+    copilot: {
+      storage: {
+        provider: 'fs',
+        bucket: 'copilot',
+        config: { path: '~/.affine/storage' },
+      },
+    },
+  } as any);
   const runtime = {
+    configure: Sinon.stub(),
     start: Sinon.stub().resolves(),
     stop: Sinon.stub().resolves(),
     runMigrations: Sinon.stub().resolves(),
@@ -26,6 +54,29 @@ test('storage-runtime provider restarts on storage config changes', async t => {
   await provider.onConfigChanged({ updates: { storages: {} } });
 
   t.is(runtime.stop.callCount, 1);
+  t.is(runtime.configure.callCount, 2);
+  t.is(runtime.start.callCount, 2);
+  t.is(runtime.runMigrations.callCount, 2);
+});
+
+test('storage-runtime provider restarts on copilot storage config changes', async t => {
+  const { provider, runtime } = createProvider();
+
+  await provider.start();
+  await provider.onConfigChanged({
+    updates: {
+      copilot: {
+        storage: {
+          provider: 'fs',
+          bucket: 'new-copilot',
+          config: { path: '~/.affine/storage' },
+        },
+      },
+    },
+  });
+
+  t.is(runtime.stop.callCount, 1);
+  t.is(runtime.configure.callCount, 2);
   t.is(runtime.start.callCount, 2);
   t.is(runtime.runMigrations.callCount, 2);
 });

packages/backend/server/src/core/storage-runtime/provider.ts

@@ -14,7 +14,7 @@ import type {
   PresignedUpload,
   PutObjectMetadata,
 } from '../../base';
-import { OnEvent } from '../../base';
+import { Config, OnEvent } from '../../base';
 import { wrapCallMetric } from '../../base/metrics';
 import {
   type RuntimeObjectGetResult,
@@ -36,6 +36,8 @@ export class StorageRuntimeProvider
   private readonly runtime: RuntimeInstance = new StorageRuntime();
   private migrationsStarted = false;
 
+  constructor(private readonly config: Config) {}
+
   async onApplicationBootstrap() {
     await this.start();
   }
@@ -45,6 +47,7 @@ export class StorageRuntimeProvider
   }
 
   async start() {
+    this.configureRuntime();
     await this.runtime.start();
     await this.runMigrationsOnce();
     const health = await this.runtime.health();
@@ -65,7 +68,11 @@ export class StorageRuntimeProvider
 
   @OnEvent('config.changed')
   async onConfigChanged({ updates }: Events['config.changed']) {
-    if (!('storages' in updates) && !('db' in updates)) {
+    if (
+      !('storages' in updates) &&
+      !('db' in updates) &&
+      !updates.copilot?.storage
+    ) {
       return;
     }
     await this.restart();
@@ -293,6 +300,23 @@ export class StorageRuntimeProvider
     this.migrationsStarted = false;
     await this.start();
   }
+
+  private configureRuntime() {
+    this.runtime.configure(
+      JSON.stringify({
+        db: {
+          datasourceUrl: this.config.db.datasourceUrl,
+        },
+        storages: {
+          'blob.storage': this.config.storages.blob.storage,
+          'avatar.storage': this.config.storages.avatar.storage,
+        },
+        copilot: {
+          storage: this.config.copilot.storage,
+        },
+      })
+    );
+  }
 }
 
 function toRuntimeMetadata(metadata?: PutObjectMetadata) {

packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts

@@ -32,6 +32,7 @@ test.beforeEach(t => {
     health: Sinon.stub().resolves({
       databaseConnected: true,
       providerConfigured: true,
+      provider: 'fs',
     }),
     backfillMissingBlobMetadata: Sinon.stub(),
     rebuildWorkspaceDocBlobRefs: Sinon.stub(),
@@ -103,7 +104,8 @@ for (const scenario of objectStorageRequiredCases) {
   test(`${scenario.name} skips when object storage is not configured`, async t => {
     t.context.runtime.health.resolves({
       databaseConnected: true,
-      providerConfigured: false,
+      providerConfigured: true,
+      provider: undefined,
     });
 
     await scenario.run(t.context);

packages/backend/server/src/core/storage/blob-job.ts

@@ -411,7 +411,7 @@ export class StorageBlobJob {
 
   private async hasObjectStorage(operation: string) {
     const health = await this.rt.health();
-    if (health.providerConfigured) {
+    if (health.provider) {
       return true;
     }
 

packages/backend/server/src/core/storage/wrappers/blob.ts

@@ -50,6 +50,11 @@ type BlobGetResult = {
   metadata?: GetObjectMetadata;
 };
 
+type R2ProxyConfig = {
+  signKey: string;
+  urlPrefix: string;
+};
+
 @Injectable()
 export class WorkspaceBlobStorage {
   private readonly logger = new Logger(WorkspaceBlobStorage.name);
@@ -317,7 +322,10 @@ export class WorkspaceBlobStorage {
     ) {
       return;
     }
-    return { signKey: usePresignedURL.signKey };
+    return {
+      signKey: usePresignedURL.signKey,
+      urlPrefix: usePresignedURL.urlPrefix,
+    };
   }
 
   private signProxy(
@@ -340,7 +348,7 @@ export class WorkspaceBlobStorage {
     workspaceId: string,
     key: string,
     metadata: PutObjectMetadata | undefined,
-    proxy: { signKey: string }
+    proxy: R2ProxyConfig
   ) {
     const contentType = metadata?.contentType ?? 'application/octet-stream';
     const contentLength = metadata?.contentLength;
@@ -353,7 +361,7 @@ export class WorkspaceBlobStorage {
       proxy.signKey
     );
     return {
-      url: this.url.link(PROXY_UPLOAD_PATH, {
+      url: this.linkProxyUrl(proxy.urlPrefix, PROXY_UPLOAD_PATH, {
         workspaceId,
         key,
         contentType,
@@ -371,7 +379,7 @@ export class WorkspaceBlobStorage {
     key: string,
     uploadId: string,
     partNumber: number,
-    proxy: { signKey: string }
+    proxy: R2ProxyConfig
   ) {
     const expiresAt = new Date(Date.now() + SIGNED_URL_EXPIRED * 1000);
     const exp = Math.floor(expiresAt.getTime() / 1000);
@@ -382,7 +390,7 @@ export class WorkspaceBlobStorage {
       proxy.signKey
     );
     return {
-      url: this.url.link(PROXY_MULTIPART_PATH, {
+      url: this.linkProxyUrl(proxy.urlPrefix, PROXY_MULTIPART_PATH, {
         workspaceId,
         key,
         uploadId,
@@ -394,4 +402,20 @@ export class WorkspaceBlobStorage {
       expiresAt,
     };
   }
+
+  private linkProxyUrl(
+    urlPrefix: string,
+    path: string,
+    query: Record<string, string | number | undefined>
+  ) {
+    const url = new URL(
+      `${urlPrefix.replace(/\/+$/, '')}${path.startsWith('/') ? path : `/${path}`}`
+    );
+    for (const [key, value] of Object.entries(query)) {
+      if (value !== undefined) {
+        url.searchParams.set(key, value.toString());
+      }
+    }
+    return url.toString();
+  }
 }

packages/backend/server/src/core/user/controller.ts

@@ -16,9 +16,10 @@ export class UserAvatarController {
 
   @Get('/:id')
   async getAvatar(@Res() res: Response, @Param('id') id: string) {
-    if (this.storage.config.storage.provider !== 'fs') {
+    const provider = this.storage.config.storage.provider;
+    if (!['assetpack', 'fs'].includes(provider)) {
       throw new ActionForbidden(
-        'Only available when avatar storage provider set to fs.'
+        'Only available when avatar storage provider is fs or assetpack.'
       );
     }
 

packages/backend/server/src/core/workspaces/resolvers/blob.ts

@@ -398,6 +398,9 @@ export class WorkspaceBlobResolver {
       if (result.reason === 'mime_mismatch') {
         throw new BlobInvalid('Blob mime mismatch');
       }
+      if (result.reason === 'size_too_large') {
+        throw new BlobInvalid('Blob size too large');
+      }
       throw new BlobInvalid('Blob key mismatch');
     }
 

packages/backend/server/src/plugins/copilot/storage.ts

@@ -39,7 +39,10 @@ export class CopilotStorage {
   ) {
     const name = `${userId}/${workspaceId}/${key}`;
     const buffer = await toBuffer(blob);
-    await this.rt.putObject('copilot', name, buffer);
+    await this.rt.putObject('copilot', name, buffer, {
+      contentType: mimeType,
+      contentLength: buffer.length,
+    });
     if (!env.prod) {
       // return image base64url for dev environment
       return `data:${mimeType};base64,${buffer.toString('base64')}`;