feat(server): impl storage runtime (#15181)

#### PR Dependency Tree


* **PR #15181** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added an additional storage backend option: asset-pack based storage
(provider for avatar, blob, and copilot).
* Introduced a dedicated storage runtime with provider capability
reporting and expanded object operations (put/head/get/list/delete),
including presigned and multipart flows where supported.
* Cloudflare R2 `jurisdiction` now uses an explicit default when
omitted.
* **Bug Fixes**
  * Broadened avatar access to allow both fs and asset-pack providers.
* Improved workspace blob upload completion validation and handling when
stored objects are missing or mismatched.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/__tests__/copilot/copilot-provider.spec.ts

@@ -1,13 +1,16 @@
 import { randomUUID } from 'node:crypto';
 
+import { Global, Module } from '@nestjs/common';
 import type { Prisma } from '@prisma/client';
 import type { ExecutionContext, TestFn } from 'ava';
 import ava from 'ava';
 import Sinon from 'sinon';
 import { z } from 'zod';
 
+import { AppModuleBuilder, FunctionalityModules } from '../../app.module';
+import { JobModule, JobQueue } from '../../base';
 import { ServerFeature, ServerService } from '../../core';
-import { AuthService } from '../../core/auth';
+import { AuthModule, AuthService } from '../../core/auth';
 import { QuotaModule } from '../../core/quota';
 import { Models } from '../../models';
 import { llmImageDispatchPlan } from '../../native';
@@ -25,6 +28,7 @@ import { ChatSession, ChatSessionService } from '../../plugins/copilot/session';
 import { TranscriptPayloadSchema } from '../../plugins/copilot/transcript/schema';
 import { CopilotTranscriptionService } from '../../plugins/copilot/transcript/service';
 import { TestingPromptService } from '../mocks/prompt-service.mock';
+import { MockJobQueue } from '../mocks/queue.mock';
 import { createTestingModule, TestingModule } from '../utils';
 import { TestAssets } from '../utils/copilot';
 import {
@@ -48,6 +52,13 @@ type Tester = {
 
 const test = ava as TestFn<Tester>;
 
+@Global()
+@Module({
+  providers: [{ provide: JobQueue, useClass: MockJobQueue }],
+  exports: [JobQueue],
+})
+class MockJobModule {}
+
 let isCopilotConfigured = false;
 const runIfCopilotConfigured = test.macro(
   async (
@@ -64,8 +75,20 @@ const runIfCopilotConfigured = test.macro(
 );
 
 test.serial.before(async t => {
+  const appModule = new AppModuleBuilder()
+    .use(
+      ...FunctionalityModules.filter(module => {
+        const moduleType = 'module' in module ? module.module : module;
+        return moduleType !== JobModule;
+      }),
+      MockJobModule,
+      AuthModule,
+      QuotaModule,
+      CopilotModule
+    )
+    .compile();
   const module = await createTestingModule({
-    imports: [QuotaModule, CopilotModule],
+    imports: [appModule],
     tapModule: builder => {
       builder.overrideProvider(PromptService).useClass(TestingPromptService);
     },

packages/backend/server/src/__tests__/copilot/copilot.spec.ts

@@ -156,8 +156,6 @@ test.before(async t => {
       CopilotModule,
     ],
     tapModule: builder => {
-      // use real JobQueue for testing
-      builder.overrideProvider(JobQueue).useClass(JobQueue);
       builder.overrideProvider(RequestMutex).useValue({
         acquire: async () => ({
           async [Symbol.asyncDispose]() {},
@@ -811,7 +809,9 @@ test('should schedule title generation as a background job', async t => {
   const chatSession = await session.get(sessionId);
   t.truthy(chatSession);
 
-  const addJob = Sinon.stub(jobs, 'add').resolves();
+  const addJob = jobs.add as Sinon.SinonStub;
+  addJob.resetHistory();
+  addJob.resolves();
 
   chatSession!.pushTurn(
     buildTurn(sessionId, {
@@ -1835,6 +1835,14 @@ test('should be able to manage workspace embedding', async t => {
       fileId: file.fileId,
       fileName: file.fileName,
     });
+    await jobs.embedPendingFile({
+      userId,
+      workspaceId: ws.id,
+      contextId: undefined,
+      blobId,
+      fileId: file.fileId,
+      fileName: file.fileName,
+    });
 
     let ret = 0;
     while (!ret) {
@@ -2059,7 +2067,9 @@ test('should handle copilot cron jobs correctly', async t => {
     copilotSession,
     'toBeGenerateTitle'
   ).resolves(mockSessions);
-  const jobAddStub = Sinon.stub(cronJobs['jobs'], 'add').resolves();
+  const jobAddStub = cronJobs['jobs'].add as Sinon.SinonStub;
+  jobAddStub.resetHistory();
+  jobAddStub.resolves();
 
   // daily cleanup job scheduling
   {
@@ -2107,7 +2117,7 @@ test('should handle copilot cron jobs correctly', async t => {
 
   cleanupStub.restore();
   toBeGenerateStub.restore();
-  jobAddStub.restore();
+  jobAddStub.resetHistory();
 });
 
 test('model selection policy should resolve requested optional models consistently', async t => {

packages/backend/server/src/__tests__/create-module.ts

@@ -7,8 +7,13 @@ import {
 import { PrismaClient } from '@prisma/client';
 
 import { FunctionalityModules } from '../app.module';
-import { AFFiNELogger, EventBus, JobQueue } from '../base';
-import { createFactory, MockEventBus, MockJobQueue } from './mocks';
+import { AFFiNELogger, EventBus, JobModule, JobQueue } from '../base';
+import {
+  createFactory,
+  MockEventBus,
+  MockJobModule,
+  MockJobQueue,
+} from './mocks';
 import { TEST_LOG_LEVEL } from './utils';
 
 interface TestingModuleMetadata extends ModuleMetadata {
@@ -26,10 +31,17 @@ export async function createModule(
   metadata: TestingModuleMetadata = {}
 ): Promise<TestingModule> {
   const { tapModule, ...meta } = metadata;
+  const functionalityModules = [
+    ...FunctionalityModules.filter(module => {
+      const moduleType = 'module' in module ? module.module : module;
+      return moduleType !== JobModule;
+    }),
+    MockJobModule,
+  ];
 
   const builder = Test.createTestingModule({
     ...meta,
-    imports: [...FunctionalityModules, ...(meta.imports ?? [])],
+    imports: [...functionalityModules, ...(meta.imports ?? [])],
   });
 
   builder

packages/backend/server/src/__tests__/e2e/create-app.ts

@@ -13,6 +13,7 @@ import {
   AFFiNELogger,
   CacheInterceptor,
   CloudThrottlerGuard,
+  ConfigFactory,
   EventBus,
   GlobalExceptionFilter,
   JobQueue,
@@ -250,6 +251,31 @@ export async function createApp(
   }
 
   const module = await builder.compile();
+  module.get(ConfigFactory).override({
+    storages: {
+      avatar: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'avatars',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+      blob: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'blobs',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+    },
+    copilot: {
+      storage: {
+        provider: 'assetpack',
+        bucket: 'copilot',
+        config: { path: '/tmp/affine-test-storage' },
+      },
+    },
+  });
 
   module.useCustomApplicationConstructor(TestingApp);
 

packages/backend/server/src/__tests__/e2e/storage/r2-proxy.spec.ts

@@ -1,4 +1,4 @@
-import { createHash } from 'node:crypto';
+import { createHash, createHmac } from 'node:crypto';
 import { mock } from 'node:test';
 
 import {
@@ -6,22 +6,13 @@ import {
   ConfigFactory,
   PROXY_MULTIPART_PATH,
   PROXY_UPLOAD_PATH,
-  StorageProviderConfig,
-  StorageProviderFactory,
-  toBuffer,
+  type R2StorageConfig,
+  SIGNED_URL_EXPIRED,
+  type StorageProviderConfig,
 } from '../../../base';
-import {
-  R2StorageConfig,
-  R2StorageProvider,
-} from '../../../base/storage/providers/r2';
-import { SIGNED_URL_EXPIRED } from '../../../base/storage/providers/utils';
 import { EntitlementService } from '../../../core/entitlement';
-import {
-  CommentAttachmentStorage,
-  WorkspaceBlobStorage,
-} from '../../../core/storage';
 import { MULTIPART_THRESHOLD } from '../../../core/storage/constants';
-import { R2UploadController } from '../../../core/storage/r2-proxy';
+import { StorageRuntimeProvider } from '../../../core/storage-runtime';
 import {
   SubscriptionPlan,
   SubscriptionRecurring,
@@ -29,7 +20,7 @@ import {
 } from '../../../plugins/payment/types';
 import { app, e2e, Mockers } from '../test';
 
-class MockR2Provider extends R2StorageProvider {
+class MockStorageRuntime {
   createMultipartCalls = 0;
   putCalls: {
     key: string;
@@ -46,45 +37,72 @@ class MockR2Provider extends R2StorageProvider {
     contentLength?: number;
   }[] = [];
 
-  constructor(config: R2StorageConfig, bucket: string) {
-    super(config, bucket);
+  async providerCapabilities() {
+    const storage = app.get(Config).storages.blob.storage;
+    const usePresignedURL = (storage.config as R2StorageConfig).usePresignedURL;
+    if (storage.provider !== 'cloudflare-r2') {
+      return {
+        put: true,
+        get: true,
+        head: true,
+        list: true,
+        delete: true,
+        presignPut: false,
+        presignGet: false,
+        multipartDirect: false,
+        proxyUpload: false,
+        assetpack: false,
+        serverMediatedOnly: true,
+      };
+    }
+    return {
+      put: true,
+      get: true,
+      head: true,
+      list: true,
+      delete: true,
+      presignPut: true,
+      presignGet: false,
+      multipartDirect: true,
+      proxyUpload: !!usePresignedURL?.enabled,
+      assetpack: false,
+      serverMediatedOnly: false,
+    };
   }
 
-  destroy() {}
-
-  override async proxyPutObject(
+  async presignPut(
+    _scope: string,
     key: string,
-    body: any,
-    options: { contentType?: string; contentLength?: number } = {}
+    metadata: { contentType?: string; contentLength?: number } = {}
   ) {
-    this.putCalls.push({
-      key,
-      body: await toBuffer(body),
-      contentType: options.contentType,
-      contentLength: options.contentLength,
-    });
+    const storage = app.get(Config).storages.blob.storage;
+    const r2 = storage.config as R2StorageConfig;
+    if (!r2.usePresignedURL?.enabled) {
+      return {
+        url: 'https://test-bucket.r2.example.com/object?X-Amz-Algorithm=AWS4-HMAC-SHA256',
+        headers: {},
+        expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
+      };
+    }
+    const [workspaceId, blobKey] = key.split('/');
+    return createProxyUrl(
+      PROXY_UPLOAD_PATH,
+      [
+        workspaceId,
+        blobKey,
+        metadata.contentType ?? 'application/octet-stream',
+        metadata.contentLength,
+      ],
+      {
+        workspaceId,
+        key: blobKey,
+        contentType: metadata.contentType ?? 'application/octet-stream',
+        contentLength: metadata.contentLength,
+      }
+    );
   }
 
-  override async proxyUploadPart(
-    key: string,
-    uploadId: string,
-    partNumber: number,
-    body: any,
-    options: { contentLength?: number } = {}
-  ) {
-    const etag = `etag-${partNumber}`;
-    this.partCalls.push({
-      key,
-      uploadId,
-      partNumber,
-      etag,
-      body: await toBuffer(body),
-      contentLength: options.contentLength,
-    });
-    return etag;
-  }
-
-  override async createMultipartUpload() {
+  async createMultipartUpload() {
     this.createMultipartCalls += 1;
     return {
       uploadId: 'upload-id',
@@ -92,7 +110,30 @@ class MockR2Provider extends R2StorageProvider {
     };
   }
 
-  override async listMultipartUploadParts(key: string, uploadId: string) {
+  async presignUploadPart(
+    _scope: string,
+    key: string,
+    uploadId: string,
+    partNumber: number
+  ) {
+    const [workspaceId, blobKey] = key.split('/');
+    return createProxyUrl(
+      PROXY_MULTIPART_PATH,
+      [workspaceId, blobKey, uploadId, partNumber],
+      {
+        workspaceId,
+        key: blobKey,
+        uploadId,
+        partNumber,
+      }
+    );
+  }
+
+  async listMultipartUploadParts(
+    _scope: string,
+    key: string,
+    uploadId: string
+  ) {
     const latest = new Map<number, string>();
     for (const part of this.partCalls) {
       if (part.key !== key || part.uploadId !== uploadId) {
@@ -104,6 +145,45 @@ class MockR2Provider extends R2StorageProvider {
       .sort((left, right) => left[0] - right[0])
       .map(([partNumber, etag]) => ({ partNumber, etag }));
   }
+
+  async putObject(
+    _scope: string,
+    key: string,
+    body: Buffer,
+    options: { contentType?: string; contentLength?: number } = {}
+  ) {
+    this.putCalls.push({
+      key,
+      body,
+      contentType: options.contentType,
+      contentLength: options.contentLength,
+    });
+    return {
+      contentType: options.contentType ?? 'application/octet-stream',
+      contentLength: options.contentLength ?? body.length,
+      lastModified: new Date(),
+    };
+  }
+
+  async proxyUploadPart(
+    _scope: string,
+    key: string,
+    uploadId: string,
+    partNumber: number,
+    body: Buffer,
+    contentLength?: number
+  ) {
+    const etag = `etag-${partNumber}`;
+    this.partCalls.push({
+      key,
+      uploadId,
+      partNumber,
+      etag,
+      body,
+      contentLength,
+    });
+    return etag;
+  }
 }
 
 const baseR2Storage: StorageProviderConfig = {
@@ -125,55 +205,40 @@ const baseR2Storage: StorageProviderConfig = {
 };
 
 let defaultBlobStorage: StorageProviderConfig;
-let provider: MockR2Provider | null = null;
-let factoryCreateUnmocked: StorageProviderFactory['create'];
+let runtime: MockStorageRuntime;
 
 e2e.before(() => {
   defaultBlobStorage = structuredClone(app.get(Config).storages.blob.storage);
-  const factory = app.get(StorageProviderFactory);
-  factoryCreateUnmocked = factory.create.bind(factory);
 });
 
 e2e.beforeEach(async () => {
-  provider?.destroy();
-  provider = null;
-
-  const factory = app.get(StorageProviderFactory);
-  mock.method(factory, 'create', (config: StorageProviderConfig) => {
-    if (config.provider === 'cloudflare-r2') {
-      if (!provider) {
-        provider = new MockR2Provider(
-          config.config as R2StorageConfig,
-          config.bucket
-        );
-      }
-      return provider;
-    }
-    return factoryCreateUnmocked(config);
-  });
+  runtime = new MockStorageRuntime();
+  const rt = app.get(StorageRuntimeProvider);
+  for (const method of [
+    'providerCapabilities',
+    'presignPut',
+    'createMultipartUpload',
+    'presignUploadPart',
+    'listMultipartUploadParts',
+    'putObject',
+    'proxyUploadPart',
+  ] as const) {
+    mock.method(rt, method, (...args: any[]) =>
+      (runtime[method] as any)(...args)
+    );
+  }
 
   await useR2Storage();
 });
 
 e2e.afterEach.always(async () => {
   await setBlobStorage(defaultBlobStorage);
-  provider?.destroy();
-  provider = null;
   mock.reset();
 });
 
 async function setBlobStorage(storage: StorageProviderConfig) {
-  provider?.destroy();
-  provider = null;
   const configFactory = app.get(ConfigFactory);
   configFactory.override({ storages: { blob: { storage } } });
-  const blobStorage = app.get(WorkspaceBlobStorage);
-  await blobStorage.onConfigInit();
-  const commentAttachmentStorage = app.get(CommentAttachmentStorage);
-  await commentAttachmentStorage.onConfigInit();
-  const controller = app.get(R2UploadController);
-  // reset cached provider in controller
-  (controller as any).provider = null;
 }
 
 async function useR2Storage(
@@ -193,11 +258,8 @@ async function useR2Storage(
   return storage;
 }
 
-function getProvider(): MockR2Provider {
-  if (!provider) {
-    throw new Error('R2 provider is not initialized');
-  }
-  return provider;
+function getRuntime(): MockStorageRuntime {
+  return runtime;
 }
 
 async function createBlobUpload(
@@ -285,7 +347,7 @@ async function gql<QueryData = any>(
   return res.body.data;
 }
 
-e2e('should proxy single upload with valid signature', async t => {
+e2e.serial('should proxy single upload with valid signature', async t => {
   const { workspace } = await setupWorkspace();
   const buffer = Buffer.from('r2-proxy');
   const key = sha256Base64urlWithPadding(buffer);
@@ -300,6 +362,7 @@ e2e('should proxy single upload with valid signature', async t => {
   t.is(init.method, 'PRESIGNED');
   t.truthy(init.uploadUrl);
   const uploadUrl = new URL(init.uploadUrl, app.url);
+  t.is(uploadUrl.origin, 'https://cdn.example.com');
   t.is(uploadUrl.pathname, PROXY_UPLOAD_PATH);
 
   const res = await app
@@ -309,7 +372,7 @@ e2e('should proxy single upload with valid signature', async t => {
     .send(buffer);
 
   t.is(res.status, 200);
-  const calls = getProvider().putCalls;
+  const calls = getRuntime().putCalls;
   t.is(calls.length, 1);
   t.is(calls[0].key, `${workspace.id}/${key}`);
   t.is(calls[0].contentType, 'text/plain');
@@ -317,7 +380,7 @@ e2e('should proxy single upload with valid signature', async t => {
   t.deepEqual(calls[0].body, buffer);
 });
 
-e2e('should proxy multipart upload and return etag', async t => {
+e2e.serial('should proxy multipart upload and return etag', async t => {
   const { workspace } = await setupWorkspace();
   const key = 'multipart-object';
   const totalSize = MULTIPART_THRESHOLD + 1024;
@@ -329,6 +392,7 @@ e2e('should proxy multipart upload and return etag', async t => {
 
   const part = await getBlobUploadPartUrl(workspace.id, key, init.uploadId, 1);
   const partUrl = new URL(part.uploadUrl, app.url);
+  t.is(partUrl.origin, 'https://cdn.example.com');
   t.is(partUrl.pathname, PROXY_MULTIPART_PATH);
 
   const payload = Buffer.from('part-body');
@@ -340,7 +404,7 @@ e2e('should proxy multipart upload and return etag', async t => {
   t.is(res.status, 200);
   t.is(res.get('etag'), 'etag-1');
 
-  const calls = getProvider().partCalls;
+  const calls = getRuntime().partCalls;
   t.is(calls.length, 1);
   t.is(calls[0].key, `${workspace.id}/${key}`);
   t.is(calls[0].uploadId, 'upload-id');
@@ -349,34 +413,42 @@ e2e('should proxy multipart upload and return etag', async t => {
   t.deepEqual(calls[0].body, payload);
 });
 
-e2e('should resume multipart upload and return uploaded parts', async t => {
-  const { workspace } = await setupWorkspace();
-  const key = 'multipart-resume';
-  const totalSize = MULTIPART_THRESHOLD + 1024;
+e2e.serial(
+  'should resume multipart upload and return uploaded parts',
+  async t => {
+    const { workspace } = await setupWorkspace();
+    const key = 'multipart-resume';
+    const totalSize = MULTIPART_THRESHOLD + 1024;
 
-  const init1 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
-  t.is(init1.method, 'MULTIPART');
-  t.is(init1.uploadId, 'upload-id');
-  t.deepEqual(init1.uploadedParts, []);
-  t.is(getProvider().createMultipartCalls, 1);
+    const init1 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
+    t.is(init1.method, 'MULTIPART');
+    t.is(init1.uploadId, 'upload-id');
+    t.deepEqual(init1.uploadedParts, []);
+    t.is(getRuntime().createMultipartCalls, 1);
 
-  const part = await getBlobUploadPartUrl(workspace.id, key, init1.uploadId, 1);
-  const payload = Buffer.from('part-body');
-  const partUrl = new URL(part.uploadUrl, app.url);
-  await app
-    .PUT(partUrl.pathname + partUrl.search)
-    .set('content-length', payload.length.toString())
-    .send(payload)
-    .expect(200);
+    const part = await getBlobUploadPartUrl(
+      workspace.id,
+      key,
+      init1.uploadId,
+      1
+    );
+    const payload = Buffer.from('part-body');
+    const partUrl = new URL(part.uploadUrl, app.url);
+    await app
+      .PUT(partUrl.pathname + partUrl.search)
+      .set('content-length', payload.length.toString())
+      .send(payload)
+      .expect(200);
 
-  const init2 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
-  t.is(init2.method, 'MULTIPART');
-  t.is(init2.uploadId, 'upload-id');
-  t.deepEqual(init2.uploadedParts, [{ partNumber: 1, etag: 'etag-1' }]);
-  t.is(getProvider().createMultipartCalls, 1);
-});
+    const init2 = await createBlobUpload(workspace.id, key, totalSize, 'bin');
+    t.is(init2.method, 'MULTIPART');
+    t.is(init2.uploadId, 'upload-id');
+    t.deepEqual(init2.uploadedParts, [{ partNumber: 1, etag: 'etag-1' }]);
+    t.is(getRuntime().createMultipartCalls, 1);
+  }
+);
 
-e2e('should reject upload when token is invalid', async t => {
+e2e.serial('should reject upload when token is invalid', async t => {
   const { workspace } = await setupWorkspace();
   const buffer = Buffer.from('payload');
   const init = await createBlobUpload(
@@ -396,10 +468,10 @@ e2e('should reject upload when token is invalid', async t => {
 
   t.is(res.status, 400);
   t.is(res.body.message, 'Invalid upload token');
-  t.is(getProvider().putCalls.length, 0);
+  t.is(getRuntime().putCalls.length, 0);
 });
 
-e2e('should reject upload when url is expired', async t => {
+e2e.serial('should reject upload when url is expired', async t => {
   const { workspace } = await setupWorkspace();
   const buffer = Buffer.from('expired');
   const init = await createBlobUpload(
@@ -422,10 +494,10 @@ e2e('should reject upload when url is expired', async t => {
 
   t.is(res.status, 400);
   t.is(res.body.message, 'Upload URL expired');
-  t.is(getProvider().putCalls.length, 0);
+  t.is(getRuntime().putCalls.length, 0);
 });
 
-e2e(
+e2e.serial(
   'should fall back to direct presign when custom domain is disabled',
   async t => {
     await useR2Storage({
@@ -449,7 +521,7 @@ e2e(
   }
 );
 
-e2e(
+e2e.serial(
   'should still fallback to graphql when provider does not support presign',
   async t => {
     await setBlobStorage({
@@ -473,6 +545,40 @@ e2e(
   }
 );
 
+function createProxyUrl(
+  path: string,
+  canonicalFields: (string | number | undefined)[],
+  query: Record<string, string | number | undefined>
+) {
+  const signKey = (
+    app.get(Config).storages.blob.storage.config as R2StorageConfig
+  ).usePresignedURL?.signKey;
+  if (!signKey) {
+    throw new Error('missing R2 proxy sign key');
+  }
+  const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED;
+  const canonical = [
+    path,
+    ...canonicalFields.map(field =>
+      field === undefined ? '' : field.toString()
+    ),
+    exp.toString(),
+  ].join('\n');
+  const token = createHmac('sha256', signKey)
+    .update(canonical)
+    .digest('base64');
+
+  const url = new URL(`http://localhost${path}`);
+  for (const [key, value] of Object.entries(query)) {
+    if (value !== undefined) {
+      url.searchParams.set(key, value.toString());
+    }
+  }
+  url.searchParams.set('exp', exp.toString());
+  url.searchParams.set('token', `${exp}-${token}`);
+  return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) };
+}
+
 function sha256Base64urlWithPadding(buffer: Buffer) {
   return createHash('sha256')
     .update(buffer)

packages/backend/server/src/__tests__/e2e/workspace/controller.spec.ts

@@ -1,4 +1,5 @@
 import { randomUUID } from 'node:crypto';
+import { Readable } from 'node:stream';
 import { mock } from 'node:test';
 
 import {
@@ -7,6 +8,8 @@ import {
   type StorageProviderConfig,
 } from '../../../base';
 import { CommentAttachmentStorage } from '../../../core/storage';
+import { StorageRuntimeProvider } from '../../../core/storage-runtime';
+import { getMime } from '../../../native';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
@@ -26,14 +29,68 @@ e2e.afterEach.always(() => {
   mock.reset();
 });
 
+const objects = new Map<
+  string,
+  {
+    body: Buffer;
+    metadata?: {
+      contentLength?: number;
+      contentType?: string;
+      checksumCRC32?: string;
+      lastModified?: Date;
+    };
+  }
+>();
+
+e2e.beforeEach(() => {
+  objects.clear();
+  const rt = app.get(StorageRuntimeProvider);
+  mock.method(
+    rt,
+    'putObject',
+    async (
+      _scope: string,
+      key: string,
+      body: Buffer,
+      metadata?: {
+        contentLength?: number;
+        contentType?: string;
+        checksumCRC32?: string;
+      }
+    ) => {
+      const object = {
+        body,
+        metadata: {
+          ...metadata,
+          contentType: metadata?.contentType ?? getMime(body),
+          contentLength: metadata?.contentLength ?? body.length,
+          lastModified: new Date(),
+        },
+      };
+      objects.set(key, object);
+      return object.metadata;
+    }
+  );
+  mock.method(rt, 'getObject', async (_scope: string, key: string) => {
+    const object = objects.get(key);
+    if (!object) {
+      return {};
+    }
+    return {
+      body: Readable.from(object.body),
+      metadata: object.metadata,
+    };
+  });
+  mock.method(rt, 'presignGet', async () => undefined);
+});
+
 async function useCommentAttachmentBlobStorage(storage: StorageProviderConfig) {
   app.get(ConfigFactory).override({ storages: { blob: { storage } } });
-  await app.get(CommentAttachmentStorage).onConfigInit();
 }
 
 // #region comment attachment
 
-e2e(
+e2e.serial(
   'should get comment attachment not found when key is not exists',
   async t => {
     const { owner, workspace } = await createWorkspace();
@@ -50,7 +107,7 @@ e2e(
   }
 );
 
-e2e(
+e2e.serial(
   'should get comment attachment no permission when user is not member',
   async t => {
     const { workspace } = await createWorkspace();
@@ -117,7 +174,7 @@ e2e.serial('should get comment attachment body', async t => {
   }
 });
 
-e2e('should get comment attachment redirect url', async t => {
+e2e.serial('should get comment attachment redirect url', async t => {
   const { owner, workspace } = await createWorkspace();
   await app.login(owner);
 

packages/backend/server/src/__tests__/mocks/index.ts

@@ -12,7 +12,7 @@ import { MockDocSnapshot } from './doc-snapshot.mock';
 import { MockDocUser } from './doc-user.mock';
 import { MockEventBus } from './eventbus.mock';
 import { MockMailer } from './mailer.mock';
-import { MockJobQueue } from './queue.mock';
+import { MockJobModule, MockJobQueue } from './queue.mock';
 import { MockTeamWorkspace } from './team-workspace.mock';
 import { MockUser } from './user.mock';
 import { MockUserSettings } from './user-settings.mock';
@@ -35,6 +35,7 @@ export {
   installMockCopilotRuntime,
   MockCopilotProvider,
   MockEventBus,
+  MockJobModule,
   MockJobQueue,
   MockMailer,
 };

packages/backend/server/src/__tests__/mocks/queue.mock.ts

@@ -1,12 +1,16 @@
+import { Global, Module } from '@nestjs/common';
 import { interval, map, take, takeUntil } from 'rxjs';
 import Sinon from 'sinon';
 
 import { JobQueue } from '../../base';
 
 export class MockJobQueue {
-  add = Sinon.createStubInstance(JobQueue).add.resolves();
-  remove = Sinon.createStubInstance(JobQueue).remove.resolves();
-  removeWhere = Sinon.createStubInstance(JobQueue).removeWhere.resolves([]);
+  private readonly sandbox = Sinon.createSandbox();
+
+  add = this.sandbox.stub().resolves();
+  get = this.sandbox.stub().resolves();
+  remove = this.sandbox.stub().resolves();
+  removeWhere = this.sandbox.stub().resolves([]);
 
   last<Job extends JobName>(name: Job): { name: Job; payload: Jobs[Job] } {
     const addJobName = this.add.lastCall?.args[0];
@@ -57,3 +61,10 @@ export class MockJobQueue {
     return this.add.getCalls().filter(call => call.args[0] === name).length;
   }
 }
+
+@Global()
+@Module({
+  providers: [{ provide: JobQueue, useClass: MockJobQueue }],
+  exports: [JobQueue],
+})
+export class MockJobModule {}

packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts

@@ -4,9 +4,9 @@ import ava, { TestFn } from 'ava';
 import Sinon from 'sinon';
 
 import { OneDay } from '../../base';
-import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { StorageModule, WorkspaceBlobStorage } from '../../core/storage';
 import { BlobUploadCleanupJob } from '../../core/storage/job';
+import { StorageRuntimeProvider } from '../../core/storage-runtime';
 import { MockUser, MockWorkspace } from '../mocks';
 import { createTestingModule, TestingModule } from '../utils';
 
@@ -28,7 +28,7 @@ test.before(async t => {
     imports: [ScheduleModule.forRoot(), StorageModule],
     tapModule: builder => {
       builder
-        .overrideProvider(BackendRuntimeProvider)
+        .overrideProvider(StorageRuntimeProvider)
         .useValue(t.context.runtime);
     },
   });

packages/backend/server/src/__tests__/utils/testing-module.ts

@@ -9,7 +9,7 @@ import {
 import { PrismaClient } from '@prisma/client';
 
 import { buildAppModule, FunctionalityModules } from '../../app.module';
-import { AFFiNELogger, JobQueue } from '../../base';
+import { AFFiNELogger, ConfigFactory, JobModule, JobQueue } from '../../base';
 import { GqlModule } from '../../base/graphql';
 import { ServerConfigModule } from '../../core';
 import { AuthGuard, AuthModule } from '../../core/auth';
@@ -18,7 +18,7 @@ import { ModelsModule } from '../../models';
 // for jsdoc inference
 // oxlint-disable-next-line no-unused-vars
 import type { createModule } from '../create-module';
-import { createFactory, MockJobQueue } from '../mocks';
+import { createFactory, MockJobModule, MockJobQueue } from '../mocks';
 import { MockMailer } from '../mocks/mailer.mock';
 import { initTestingDB, TEST_LOG_LEVEL } from './utils';
 
@@ -48,6 +48,16 @@ function dedupeModules(modules: NonNullable<ModuleMetadata['imports']>) {
   return Array.from(map.values());
 }
 
+function testingFunctionalityModules() {
+  return [
+    ...FunctionalityModules.filter(module => {
+      const moduleType = 'module' in module ? module.module : module;
+      return moduleType !== JobModule;
+    }),
+    MockJobModule,
+  ];
+}
+
 @Resolver(() => String)
 class MockResolver {
   @Query(() => String)
@@ -70,7 +80,7 @@ export async function createTestingModule(
     imports[0].module?.name === 'AppModule'
       ? imports
       : dedupeModules([
-          ...FunctionalityModules,
+          ...testingFunctionalityModules(),
           ModelsModule,
           AuthModule,
           GqlModule,
@@ -99,6 +109,31 @@ export async function createTestingModule(
   }
 
   const module = await builder.compile();
+  module.get(ConfigFactory).override({
+    storages: {
+      avatar: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'avatars',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+      blob: {
+        storage: {
+          provider: 'assetpack',
+          bucket: 'blobs',
+          config: { path: '/tmp/affine-test-storage' },
+        },
+      },
+    },
+    copilot: {
+      storage: {
+        provider: 'assetpack',
+        bucket: 'copilot',
+        config: { path: '/tmp/affine-test-storage' },
+      },
+    },
+  });
 
   const testingModule = module as TestingModule;
 

packages/backend/server/src/__tests__/workspace/blobs.e2e.ts

@@ -1,12 +1,15 @@
 import { createHash } from 'node:crypto';
+import { Readable } from 'node:stream';
 
 import test from 'ava';
 import Sinon from 'sinon';
 
-import { Config, ConfigFactory, StorageProviderFactory } from '../../base';
+import { ConfigFactory } from '../../base';
 import { QuotaStateService } from '../../core/quota/state';
 import { WorkspaceBlobStorage } from '../../core/storage/wrappers/blob';
+import { StorageRuntimeProvider } from '../../core/storage-runtime';
 import { BlobModel, WorkspaceFeatureModel } from '../../models';
+import { getMime } from '../../native';
 import {
   collectAllBlobSizes,
   completeBlobUpload,
@@ -32,9 +35,117 @@ const RESTRICTED_QUOTA = {
 
 let app: TestingApp;
 let model: WorkspaceFeatureModel;
+type CompleteResult =
+  | {
+      ok: true;
+      contentType: string;
+      contentLength: number;
+      lastModifiedMs: number;
+    }
+  | {
+      ok: false;
+      reason:
+        | 'not_found'
+        | 'size_mismatch'
+        | 'mime_mismatch'
+        | 'checksum_mismatch'
+        | 'size_too_large';
+    };
+const objects = new Map<
+  string,
+  {
+    body: Buffer;
+    metadata: {
+      contentType: string;
+      contentLength: number;
+      lastModified: Date;
+    };
+  }
+>();
+const completeResults = new Map<string, CompleteResult>();
+const storageRuntime = {
+  providerCapabilities: async () => ({
+    put: true,
+    get: true,
+    head: true,
+    list: true,
+    delete: true,
+    presignPut: false,
+    presignGet: false,
+    multipartDirect: false,
+    proxyUpload: false,
+    assetpack: false,
+    serverMediatedOnly: true,
+  }),
+  putObject: async (
+    _scope: string,
+    key: string,
+    body: Buffer,
+    metadata?: { contentType?: string; contentLength?: number }
+  ) => {
+    const object = {
+      body,
+      metadata: {
+        contentType: metadata?.contentType ?? getMime(body),
+        contentLength: metadata?.contentLength ?? body.length,
+        lastModified: new Date(),
+      },
+    };
+    objects.set(key, object);
+    return object.metadata;
+  },
+  headObject: async (_scope: string, key: string) => {
+    return objects.get(key)?.metadata;
+  },
+  getObject: async (_scope: string, key: string) => {
+    const object = objects.get(key);
+    return object
+      ? { body: Readable.from(object.body), metadata: object.metadata }
+      : {};
+  },
+  listObjects: async (_scope: string, prefix?: string) => {
+    return Array.from(objects.entries())
+      .filter(([key]) => !prefix || key.startsWith(prefix))
+      .map(([key, object]) => ({ key, ...object.metadata }));
+  },
+  deleteObject: async (_scope: string, key: string) => {
+    objects.delete(key);
+  },
+  presignPut: async () => undefined,
+  presignGet: async () => undefined,
+  createMultipartUpload: async () => undefined,
+  presignUploadPart: async () => undefined,
+  listMultipartUploadParts: async () => undefined,
+  completeMultipartUpload: async () => undefined,
+  completeWorkspaceBlobUpload: async (workspaceId: string, key: string) => {
+    const objectKey = `${workspaceId}/${key}`;
+    const configured = completeResults.get(objectKey);
+    if (configured) return configured;
+    const object = objects.get(objectKey);
+    if (!object) return { ok: false, reason: 'not_found' };
+    await app.get(BlobModel).upsert({
+      workspaceId,
+      key,
+      mime: object.metadata.contentType,
+      size: object.metadata.contentLength,
+      status: 'completed',
+      uploadId: null,
+    });
+    return {
+      ok: true,
+      contentType: object.metadata.contentType,
+      contentLength: object.metadata.contentLength,
+      lastModifiedMs: object.metadata.lastModified.getTime(),
+    };
+  },
+};
 
 test.before(async () => {
-  app = await createTestingApp();
+  app = await createTestingApp({
+    tapModule: builder => {
+      builder.overrideProvider(StorageRuntimeProvider).useValue(storageRuntime);
+    },
+  });
   model = app.get(WorkspaceFeatureModel);
   app.get(ConfigFactory).override({
     storages: {
@@ -47,11 +158,12 @@ test.before(async () => {
       },
     },
   });
-  await app.get(WorkspaceBlobStorage).onConfigInit();
 });
 
 test.beforeEach(async () => {
   await app.initTestingDB();
+  objects.clear();
+  completeResults.clear();
 });
 
 test.after.always(async () => {
@@ -124,22 +236,19 @@ test('should keep partial blob metadata listing on DB path without storage scan'
 
   const workspace = await createWorkspace(app);
   const storage = app.get(WorkspaceBlobStorage);
-  const rawProvider = (storage as any).provider;
-  const listSpy = Sinon.spy(rawProvider, 'list');
+  const rt = app.get(StorageRuntimeProvider);
+  const listSpy = Sinon.spy(rt, 'listObjects');
   t.teardown(() => listSpy.restore());
 
   const buffer1 = Buffer.from('with metadata');
   const buffer2 = Buffer.from('without metadata');
   const key1 = sha256Base64urlWithPadding(buffer1);
   const key2 = sha256Base64urlWithPadding(buffer2);
-  const config = app.get(Config);
-  const factory = app.get(StorageProviderFactory);
-  const provider = factory.create(config.storages.blob.storage);
-  await provider.put(`${workspace.id}/${key1}`, buffer1, {
+  await rt.putObject('blob', `${workspace.id}/${key1}`, buffer1, {
     contentType: 'text/plain',
     contentLength: buffer1.length,
   });
-  await provider.put(`${workspace.id}/${key2}`, buffer2, {
+  await rt.putObject('blob', `${workspace.id}/${key2}`, buffer2, {
     contentType: 'text/plain',
     contentLength: buffer2.length,
   });
@@ -194,11 +303,9 @@ test('should complete pending blob upload', async t => {
 
   await createBlobUpload(app, workspace.id, key, buffer.length, mime);
 
-  const config = app.get(Config);
-  const factory = app.get(StorageProviderFactory);
-  const provider = factory.create(config.storages.blob.storage);
+  const rt = app.get(StorageRuntimeProvider);
 
-  await provider.put(`${workspace.id}/${key}`, buffer, {
+  await rt.putObject('blob', `${workspace.id}/${key}`, buffer, {
     contentType: mime,
     contentLength: buffer.length,
   });
@@ -225,14 +332,16 @@ test('should reject complete when blob key mismatched', async t => {
   const wrongKey = sha256Base64urlWithPadding(Buffer.from('other'));
   await createBlobUpload(app, workspace.id, wrongKey, buffer.length, mime);
 
-  const config = app.get(Config);
-  const factory = app.get(StorageProviderFactory);
-  const provider = factory.create(config.storages.blob.storage);
+  const rt = app.get(StorageRuntimeProvider);
 
-  await provider.put(`${workspace.id}/${wrongKey}`, buffer, {
+  await rt.putObject('blob', `${workspace.id}/${wrongKey}`, buffer, {
     contentType: mime,
     contentLength: buffer.length,
   });
+  completeResults.set(`${workspace.id}/${wrongKey}`, {
+    ok: false,
+    reason: 'checksum_mismatch',
+  });
 
   await t.throwsAsync(() => completeBlobUpload(app, workspace.id, wrongKey), {
     message: 'Blob key mismatch',
@@ -265,9 +374,8 @@ test('should auto delete blobs when workspace is deleted', async t => {
   const blobs = await listBlobs(app, workspace.id);
   t.is(blobs.length, 2);
 
-  const storage = app.get(WorkspaceBlobStorage);
-  const rawProvider = (storage as any).provider;
-  const listSpy = Sinon.spy(rawProvider, 'list');
+  const rt = app.get(StorageRuntimeProvider);
+  const listSpy = Sinon.spy(rt, 'listObjects');
   t.teardown(() => listSpy.restore());
 
   await deleteWorkspace(app, workspace.id);

packages/backend/server/src/__tests__/workspace/controller.spec.ts

@@ -212,8 +212,7 @@ test('should be able to get permission granted workspace', async t => {
 test('should return 404 if blob not found', async t => {
   const { app, storage } = t.context;
 
-  // @ts-expect-error mock
-  storage.get.resolves({ body: null });
+  storage.get.resolves({ body: undefined });
   const res = await app.GET('/api/workspaces/public/blobs/test');
 
   t.is(res.status, HttpStatus.NOT_FOUND);