feat(server): impl storage runtime (#15181)

#### PR Dependency Tree


* **PR #15181** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added an additional storage backend option: asset-pack based storage
(provider for avatar, blob, and copilot).
* Introduced a dedicated storage runtime with provider capability
reporting and expanded object operations (put/head/get/list/delete),
including presigned and multipart flows where supported.
* Cloudflare R2 `jurisdiction` now uses an explicit default when
omitted.
* **Bug Fixes**
  * Broadened avatar access to allow both fs and asset-pack providers.
* Improved workspace blob upload completion validation and handling when
stored objects are missing or mismatched.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/base/config/register.ts

@@ -21,6 +21,7 @@ export type JSONSchema = { description?: string } & (
   | {
       type: 'object';
       properties?: Record<string, JSONSchema>;
+      required?: string[];
     }
 );
 

packages/backend/server/src/base/error/errors.gen.ts

@@ -1059,6 +1059,16 @@ export class UnsupportedClientVersion extends UserFriendlyError {
     super('action_forbidden', 'unsupported_client_version', message, args);
   }
 }
+@ObjectType()
+class UnsupportedServerVersionDataType {
+  @Field() requiredVersion!: string
+}
+
+export class UnsupportedServerVersion extends UserFriendlyError {
+  constructor(args: UnsupportedServerVersionDataType, message?: string | ((args: UnsupportedServerVersionDataType) => string)) {
+    super('action_forbidden', 'unsupported_server_version', message, args);
+  }
+}
 
 export class NotificationNotFound extends UserFriendlyError {
   constructor(message?: string) {
@@ -1288,6 +1298,7 @@ export enum ErrorNames {
   INVALID_LICENSE_UPDATE_PARAMS,
   LICENSE_EXPIRED,
   UNSUPPORTED_CLIENT_VERSION,
+  UNSUPPORTED_SERVER_VERSION,
   NOTIFICATION_NOT_FOUND,
   MENTION_USER_DOC_ACCESS_DENIED,
   MENTION_USER_ONESELF_DENIED,
@@ -1308,5 +1319,5 @@ registerEnumType(ErrorNames, {
 export const ErrorDataUnionType = createUnionType({
   name: 'ErrorDataUnion',
   types: () =>
-    [GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
+    [GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, UnsupportedServerVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
 });

packages/backend/server/src/base/index.ts

@@ -30,11 +30,6 @@ export { Lock, Locker, Mutex, RequestMutex } from './mutex';
 export * from './nestjs';
 export { type PrismaTransaction } from './prisma';
 export * from './storage';
-export {
-  autoMetadata,
-  type StorageProvider,
-  type StorageProviderConfig,
-  StorageProviderFactory,
-} from './storage';
+export { type StorageProviderConfig } from './storage';
 export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';
 export * from './utils';

packages/backend/server/src/base/redis/instances.ts

@@ -8,6 +8,13 @@ import { Redis as IORedis, RedisOptions } from 'ioredis';
 
 import { Config } from '../config';
 
+function redisOptions(options: RedisOptions) {
+  return {
+    ...(env.testing ? { lazyConnect: true } : {}),
+    ...options,
+  };
+}
+
 class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
   private readonly logger = new Logger(this.constructor.name);
 
@@ -47,41 +54,47 @@ class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
 @Injectable()
 export class CacheRedis extends Redis {
   constructor(config: Config) {
-    super({ ...config.redis, ...config.redis.ioredis });
+    super(redisOptions({ ...config.redis, ...config.redis.ioredis }));
   }
 }
 
 @Injectable()
 export class SessionRedis extends Redis {
   constructor(config: Config) {
-    super({
-      ...config.redis,
-      ...config.redis.ioredis,
-      db: (config.redis.db ?? 0) + 2,
-    });
+    super(
+      redisOptions({
+        ...config.redis,
+        ...config.redis.ioredis,
+        db: (config.redis.db ?? 0) + 2,
+      })
+    );
   }
 }
 
 @Injectable()
 export class SocketIoRedis extends Redis {
   constructor(config: Config) {
-    super({
-      ...config.redis,
-      ...config.redis.ioredis,
-      db: (config.redis.db ?? 0) + 3,
-    });
+    super(
+      redisOptions({
+        ...config.redis,
+        ...config.redis.ioredis,
+        db: (config.redis.db ?? 0) + 3,
+      })
+    );
   }
 }
 
 @Injectable()
 export class QueueRedis extends Redis {
   constructor(config: Config) {
-    super({
-      ...config.redis,
-      ...config.redis.ioredis,
-      db: (config.redis.db ?? 0) + 4,
-      // required explicitly set to `null` by bullmq
-      maxRetriesPerRequest: null,
-    });
+    super(
+      redisOptions({
+        ...config.redis,
+        ...config.redis.ioredis,
+        db: (config.redis.db ?? 0) + 4,
+        // required explicitly set to `null` by bullmq
+        maxRetriesPerRequest: null,
+      })
+    );
   }
 }

packages/backend/server/src/base/storage/__tests__/fs.spec.ts

@@ -1,130 +0,0 @@
-import { promises as fs } from 'node:fs';
-import { join } from 'node:path';
-
-import test from 'ava';
-import { getStreamAsBuffer } from 'get-stream';
-
-import { ListObjectsMetadata } from '../providers';
-import { FsStorageProvider } from '../providers/fs';
-
-const config = {
-  path: join(process.cwd(), 'node_modules', '.cache/affine-test-storage'),
-};
-
-function createProvider() {
-  return new FsStorageProvider(
-    config,
-    'test' + Math.random().toString(16).substring(2, 8)
-  );
-}
-
-function keys(list: ListObjectsMetadata[]) {
-  return list.map(i => i.key);
-}
-
-async function randomPut(
-  provider: FsStorageProvider,
-  prefix = ''
-): Promise<string> {
-  const key = prefix + 'test-key-' + Math.random().toString(16).substring(2, 8);
-  const body = Buffer.from(key);
-  await provider.put(key, body);
-  return key;
-}
-
-test.after.always(() => {
-  fs.rm(config.path, { recursive: true }).catch(console.error);
-});
-
-test('put & get', async t => {
-  const provider = createProvider();
-  const key = 'testKey';
-  const body = Buffer.from('testBody');
-  await provider.put(key, body);
-
-  const result = await provider.get(key);
-
-  t.deepEqual(await getStreamAsBuffer(result.body!), body);
-  t.is(result.metadata?.contentLength, body.length);
-});
-
-test('list - one level', async t => {
-  const provider = createProvider();
-  const list = await Promise.all(
-    Array.from({ length: 100 }).map(() => randomPut(provider))
-  );
-  list.sort();
-  // random order, use set
-  const result = await provider.list();
-  t.deepEqual(keys(result), list);
-
-  const result2 = await provider.list('test-key');
-  t.deepEqual(keys(result2), list);
-
-  const result3 = await provider.list('testKey');
-  t.is(result3.length, 0);
-});
-
-test('list recursively', async t => {
-  const provider = createProvider();
-
-  await Promise.all([
-    Promise.all(Array.from({ length: 10 }).map(() => randomPut(provider))),
-    Promise.all(
-      Array.from({ length: 10 }).map(() => randomPut(provider, 'a/'))
-    ),
-    Promise.all(
-      Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/'))
-    ),
-    Promise.all(
-      Array.from({ length: 10 }).map(() => randomPut(provider, 'a/b/t/'))
-    ),
-  ]);
-
-  const r1 = await provider.list();
-  t.is(r1.length, 40);
-
-  // contains all `a/xxx` and `a/b/xxx` and `a/b/c/xxx`
-  const r2 = await provider.list('a');
-  t.is(r2.length, 30);
-
-  // contains only `a/b/xxx`
-  const r3 = await provider.list('a/b');
-  const r4 = await provider.list('a/b/');
-  t.is(r3.length, 20);
-  t.deepEqual(r3, r4);
-
-  // prefix is not ended with '/', it's open to all files and sub dirs
-  // contains all `a/b/t/xxx` and `a/b/t{xxxx}`
-  const r5 = await provider.list('a/b/t');
-
-  t.is(r5.length, 20);
-});
-
-test('delete', async t => {
-  const provider = createProvider();
-  const key = 'testKey';
-  const body = Buffer.from('testBody');
-  await provider.put(key, body);
-
-  await provider.delete(key);
-
-  await t.throwsAsync(() => fs.access(join(config.path, provider.bucket, key)));
-});
-
-test('rejects unsafe object keys', async t => {
-  const provider = createProvider();
-
-  await t.throwsAsync(() => provider.put('../escape', Buffer.from('nope')));
-  await t.throwsAsync(() => provider.get('nested/../escape'));
-  await t.throwsAsync(() => provider.head('./escape'));
-  t.throws(() => provider.delete('nested//escape'));
-});
-
-test('rejects unsafe list prefixes', async t => {
-  const provider = createProvider();
-
-  await t.throwsAsync(() => provider.list('../escape'));
-  await t.throwsAsync(() => provider.list('nested/../../escape'));
-  await t.throwsAsync(() => provider.list('/absolute'));
-});

packages/backend/server/src/base/storage/__tests__/r2.spec.ts

@@ -1,82 +0,0 @@
-import test from 'ava';
-
-import { R2StorageProvider } from '../providers/r2';
-
-function endpointOf(provider: R2StorageProvider) {
-  return provider.endpointUrl;
-}
-
-test('R2 provider should use account endpoint by default', t => {
-  const provider = new R2StorageProvider(
-    {
-      accountId: 'test-account',
-      region: 'auto',
-      credentials: {
-        accessKeyId: 'test',
-        secretAccessKey: 'test',
-      },
-    },
-    'test-bucket'
-  );
-
-  t.is(
-    endpointOf(provider),
-    'https://test-account.r2.cloudflarestorage.com/test-bucket'
-  );
-});
-
-test('R2 provider should append jurisdiction suffix for EU buckets', t => {
-  const provider = new R2StorageProvider(
-    {
-      accountId: 'test-account',
-      jurisdiction: 'eu',
-      region: 'auto',
-      credentials: {
-        accessKeyId: 'test',
-        secretAccessKey: 'test',
-      },
-    },
-    'test-bucket'
-  );
-
-  t.is(
-    endpointOf(provider),
-    'https://test-account.eu.r2.cloudflarestorage.com/test-bucket'
-  );
-});
-
-test('R2 provider should throw when accountId is missing', t => {
-  t.throws(
-    () =>
-      new R2StorageProvider(
-        {
-          region: 'auto',
-          credentials: {
-            accessKeyId: 'test',
-            secretAccessKey: 'test',
-          },
-        } as any,
-        'test-bucket'
-      )
-  );
-});
-
-test('R2 provider should use default endpoint when jurisdiction is explicitly undefined', t => {
-  const provider = new R2StorageProvider(
-    {
-      accountId: 'test-account',
-      jurisdiction: undefined,
-      region: 'auto',
-      credentials: {
-        accessKeyId: 'test',
-        secretAccessKey: 'test',
-      },
-    },
-    'test-bucket'
-  );
-
-  t.is(
-    endpointOf(provider),
-    'https://test-account.r2.cloudflarestorage.com/test-bucket'
-  );
-});

packages/backend/server/src/base/storage/__tests__/s3-list-parts.spec.ts

@@ -1,49 +0,0 @@
-import { parseListPartsXml } from '@affine/s3-compat';
-import test from 'ava';
-
-test('parseListPartsXml handles array parts and pagination', t => {
-  const xml = `<?xml version="1.0" encoding="UTF-8"?>
-<ListPartsResult>
-  <Bucket>test</Bucket>
-  <Key>key</Key>
-  <UploadId>upload-id</UploadId>
-  <PartNumberMarker>0</PartNumberMarker>
-  <NextPartNumberMarker>3</NextPartNumberMarker>
-  <IsTruncated>true</IsTruncated>
-  <Part>
-    <PartNumber>1</PartNumber>
-    <ETag>"etag-1"</ETag>
-  </Part>
-  <Part>
-    <PartNumber>2</PartNumber>
-    <ETag>etag-2</ETag>
-  </Part>
-</ListPartsResult>`;
-
-  const result = parseListPartsXml(xml);
-  t.deepEqual(result.parts, [
-    { partNumber: 1, etag: 'etag-1' },
-    { partNumber: 2, etag: 'etag-2' },
-  ]);
-  t.true(result.isTruncated);
-  t.is(result.nextPartNumberMarker, '3');
-});
-
-test('parseListPartsXml handles single part', t => {
-  const xml = `<?xml version="1.0" encoding="UTF-8"?>
-<ListPartsResult>
-  <Bucket>test</Bucket>
-  <Key>key</Key>
-  <UploadId>upload-id</UploadId>
-  <IsTruncated>false</IsTruncated>
-  <Part>
-    <PartNumber>5</PartNumber>
-    <ETag>"etag-5"</ETag>
-  </Part>
-</ListPartsResult>`;
-
-  const result = parseListPartsXml(xml);
-  t.deepEqual(result.parts, [{ partNumber: 5, etag: 'etag-5' }]);
-  t.false(result.isTruncated);
-  t.is(result.nextPartNumberMarker, undefined);
-});

packages/backend/server/src/base/storage/__tests__/s3.spec.ts

@@ -1,90 +0,0 @@
-import test from 'ava';
-
-import { S3StorageProvider } from '../providers/s3';
-import { SIGNED_URL_EXPIRED } from '../providers/utils';
-
-const config = {
-  region: 'us-east-1',
-  endpoint: 'https://s3.us-east-1.amazonaws.com',
-  credentials: {
-    accessKeyId: 'test',
-    secretAccessKey: 'test',
-  },
-};
-
-function createProvider() {
-  return new S3StorageProvider(config, 'test-bucket');
-}
-
-test('presignPut should return url and headers', async t => {
-  const provider = createProvider();
-  const result = await provider.presignPut('key', {
-    contentType: 'text/plain',
-  });
-
-  t.truthy(result);
-  t.true(result!.url.length > 0);
-  t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
-  t.true(result!.url.includes('X-Amz-SignedHeaders='));
-  t.true(result!.url.includes('content-type'));
-  t.deepEqual(result!.headers, { 'Content-Type': 'text/plain' });
-  const now = Date.now();
-  t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000);
-  t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000);
-});
-
-test('presignUploadPart should return url', async t => {
-  const provider = createProvider();
-  const result = await provider.presignUploadPart('key', 'upload-1', 3);
-
-  t.truthy(result);
-  t.true(result!.url.length > 0);
-  t.true(result!.url.includes('X-Amz-Algorithm=AWS4-HMAC-SHA256'));
-});
-
-test('createMultipartUpload should return uploadId', async t => {
-  const provider = createProvider();
-  let receivedKey: string | undefined;
-  let receivedMeta: any;
-  (provider as any).client = {
-    createMultipartUpload: async (key: string, meta: any) => {
-      receivedKey = key;
-      receivedMeta = meta;
-      return { uploadId: 'upload-1' };
-    },
-  };
-
-  const now = Date.now();
-  const result = await provider.createMultipartUpload('key', {
-    contentType: 'text/plain',
-  });
-
-  t.is(result?.uploadId, 'upload-1');
-  t.true(result!.expiresAt.getTime() >= now + SIGNED_URL_EXPIRED * 1000 - 2000);
-  t.true(result!.expiresAt.getTime() <= now + SIGNED_URL_EXPIRED * 1000 + 2000);
-  t.is(receivedKey, 'key');
-  t.is(receivedMeta.contentType, 'text/plain');
-});
-
-test('completeMultipartUpload should order parts', async t => {
-  const provider = createProvider();
-  let receivedParts: any;
-  (provider as any).client = {
-    completeMultipartUpload: async (
-      _key: string,
-      _uploadId: string,
-      parts: any
-    ) => {
-      receivedParts = parts;
-    },
-  };
-
-  await provider.completeMultipartUpload('key', 'upload-1', [
-    { partNumber: 2, etag: 'b' },
-    { partNumber: 1, etag: 'a' },
-  ]);
-  t.deepEqual(receivedParts, [
-    { partNumber: 1, etag: 'a' },
-    { partNumber: 2, etag: 'b' },
-  ]);
-});

packages/backend/server/src/base/storage/factory.ts

@@ -1,20 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import {
-  StorageProvider,
-  StorageProviderConfig,
-  StorageProviders,
-} from './providers';
-
-@Injectable()
-export class StorageProviderFactory {
-  create(config: StorageProviderConfig): StorageProvider {
-    const Provider = StorageProviders[config.provider];
-
-    if (!Provider) {
-      throw new Error(`Unknown storage provider type: ${config.provider}`);
-    }
-
-    return new Provider(config.config, config.bucket);
-  }
-}

packages/backend/server/src/base/storage/index.ts

@@ -1,12 +1,3 @@
-import { Global, Module } from '@nestjs/common';
-
-import { StorageProviderFactory } from './factory';
-
-@Global()
-@Module({
-  providers: [StorageProviderFactory],
-  exports: [StorageProviderFactory],
-})
-export class StorageProviderModule {}
-export { StorageProviderFactory } from './factory';
 export * from './providers';
+export type * from './types';
+export * from './utils';

packages/backend/server/src/base/storage/providers/fs.ts

@@ -1,318 +0,0 @@
-import {
-  accessSync,
-  constants,
-  createReadStream,
-  Dirent,
-  mkdirSync,
-  readdirSync,
-  readFileSync,
-  rmSync,
-  statSync,
-  writeFileSync,
-} from 'node:fs';
-import { homedir } from 'node:os';
-import { join, parse } from 'node:path';
-import { Readable } from 'node:stream';
-
-import { Logger } from '@nestjs/common';
-
-import {
-  BlobInputType,
-  GetObjectMetadata,
-  ListObjectsMetadata,
-  PutObjectMetadata,
-  StorageProvider,
-} from './provider';
-import { autoMetadata, toBuffer } from './utils';
-
-function normalizeStorageKey(key: string): string {
-  const normalized = key.replaceAll('\\', '/');
-  const segments = normalized.split('/');
-
-  if (
-    !normalized ||
-    normalized.startsWith('/') ||
-    segments.some(segment => !segment || segment === '.' || segment === '..')
-  ) {
-    throw new Error(`Invalid storage key: ${key}`);
-  }
-
-  return segments.join('/');
-}
-
-function normalizeStoragePrefix(prefix: string): string {
-  const normalized = prefix.replaceAll('\\', '/');
-  if (!normalized) {
-    return normalized;
-  }
-  if (normalized.startsWith('/')) {
-    throw new Error(`Invalid storage prefix: ${prefix}`);
-  }
-
-  const segments = normalized.split('/');
-  const lastSegment = segments.pop();
-
-  if (
-    lastSegment === undefined ||
-    segments.some(segment => !segment || segment === '.' || segment === '..') ||
-    lastSegment === '.' ||
-    lastSegment === '..'
-  ) {
-    throw new Error(`Invalid storage prefix: ${prefix}`);
-  }
-
-  if (lastSegment === '') {
-    return `${segments.join('/')}/`;
-  }
-
-  return [...segments, lastSegment].join('/');
-}
-
-export interface FsStorageConfig {
-  path: string;
-}
-
-export class FsStorageProvider implements StorageProvider {
-  private readonly path: string;
-  private readonly logger: Logger;
-
-  readonly type = 'fs';
-
-  constructor(
-    config: FsStorageConfig,
-    public readonly bucket: string
-  ) {
-    this.path = config.path.startsWith('~/')
-      ? join(homedir(), config.path.slice(2), bucket)
-      : join(config.path, bucket);
-    this.ensureAvailability();
-
-    this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`);
-  }
-
-  async put(
-    key: string,
-    body: BlobInputType,
-    metadata: PutObjectMetadata = {}
-  ): Promise<void> {
-    key = normalizeStorageKey(key);
-    const blob = await toBuffer(body);
-
-    // write object
-    this.writeObject(key, blob);
-    // write metadata
-    await this.writeMetadata(key, blob, metadata);
-    this.logger.verbose(`Object \`${key}\` put`);
-  }
-
-  async head(key: string) {
-    key = normalizeStorageKey(key);
-    const metadata = this.readMetadata(key);
-    if (!metadata) {
-      this.logger.verbose(`Object \`${key}\` not found`);
-      return undefined;
-    }
-    return metadata;
-  }
-
-  async get(key: string): Promise<{
-    body?: Readable;
-    metadata?: GetObjectMetadata;
-  }> {
-    key = normalizeStorageKey(key);
-
-    try {
-      const metadata = this.readMetadata(key);
-      const stream = this.readObject(this.join(key));
-      this.logger.verbose(`Read object \`${key}\``);
-      return {
-        body: stream,
-        metadata,
-      };
-    } catch (e) {
-      this.logger.error(`Failed to read object \`${key}\``, e);
-      return {};
-    }
-  }
-
-  async list(prefix?: string): Promise<ListObjectsMetadata[]> {
-    // prefix cases:
-    // - `undefined`: list all objects
-    // - `a/b`: list objects under dir `a` with prefix `b`, `b` might be a dir under `a` as well.
-    // - `a/b/` list objects under dir `a/b`
-
-    // read dir recursively and filter out '.metadata.json' files
-    let dir = this.path;
-    if (prefix) {
-      prefix = normalizeStoragePrefix(prefix);
-      const parts = prefix.split(/[/\\]/);
-      // for prefix `a/b/c`, move `a/b` to dir and `c` to key prefix
-      if (parts.length > 1) {
-        dir = join(dir, ...parts.slice(0, -1));
-        prefix = parts[parts.length - 1];
-      }
-    }
-
-    const results: ListObjectsMetadata[] = [];
-    async function getFiles(dir: string, prefix?: string): Promise<void> {
-      try {
-        const entries: Dirent[] = readdirSync(dir, { withFileTypes: true });
-
-        for (const entry of entries) {
-          const res = join(dir, entry.name);
-
-          if (entry.isDirectory()) {
-            if (!prefix || entry.name.startsWith(prefix)) {
-              await getFiles(res);
-            }
-          } else if (
-            (!prefix || entry.name.startsWith(prefix)) &&
-            !entry.name.endsWith('.metadata.json')
-          ) {
-            const stat = statSync(res);
-            results.push({
-              key: res,
-              lastModified: stat.mtime,
-              contentLength: stat.size,
-            });
-          }
-        }
-      } catch {
-        // failed to read dir, stop recursion
-      }
-    }
-
-    await getFiles(dir, prefix);
-
-    // trim path with `this.path` prefix
-    results.forEach(r => (r.key = r.key.slice(this.path.length + 1)));
-
-    return results;
-  }
-
-  delete(key: string): Promise<void> {
-    key = normalizeStorageKey(key);
-
-    try {
-      rmSync(this.join(key), { force: true });
-      rmSync(this.join(`${key}.metadata.json`), { force: true });
-    } catch (e) {
-      throw new Error(`Failed to delete object \`${key}\``, {
-        cause: e,
-      });
-    }
-
-    this.logger.verbose(`Object \`${key}\` deleted`);
-
-    return Promise.resolve();
-  }
-
-  ensureAvailability() {
-    // check stats
-    const stats = statSync(this.path, {
-      throwIfNoEntry: false,
-    });
-
-    // not existing, create it
-    if (!stats) {
-      try {
-        mkdirSync(this.path, { recursive: true });
-      } catch (e) {
-        throw new Error(
-          `Failed to create target directory for fs storage provider: ${this.path}`,
-          {
-            cause: e,
-          }
-        );
-      }
-    } else if (stats.isDirectory()) {
-      // the target directory has already existed, check if it is readable & writable
-      try {
-        accessSync(this.path, constants.W_OK | constants.R_OK);
-      } catch (e) {
-        throw new Error(
-          `The target directory for fs storage provider has already existed, but it is not readable & writable: ${this.path}`,
-          {
-            cause: e,
-          }
-        );
-      }
-    } else if (stats.isFile()) {
-      throw new Error(
-        `The target directory for fs storage provider is a file: ${this.path}`
-      );
-    }
-  }
-
-  private join(...paths: string[]) {
-    return join(this.path, ...paths);
-  }
-
-  private readObject(file: string): Readable | undefined {
-    const state = statSync(file, { throwIfNoEntry: false });
-
-    if (state?.isFile()) {
-      return createReadStream(file);
-    }
-
-    return undefined;
-  }
-
-  private writeObject(key: string, blob: Buffer) {
-    const path = this.join(key);
-    mkdirSync(parse(path).dir, { recursive: true });
-    writeFileSync(path, blob);
-  }
-
-  private async writeMetadata(
-    key: string,
-    blob: Buffer,
-    raw: PutObjectMetadata
-  ) {
-    try {
-      const metadata = autoMetadata(blob, raw);
-
-      if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) {
-        throw new Error(
-          'The checksum of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
-        );
-      }
-
-      if (raw.contentLength && metadata.contentLength !== raw.contentLength) {
-        throw new Error(
-          'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'
-        );
-      }
-
-      writeFileSync(
-        this.join(`${key}.metadata.json`),
-        JSON.stringify({
-          ...metadata,
-          lastModified: Date.now(),
-        })
-      );
-    } catch (e) {
-      this.logger.warn(`Failed to write metadata of object \`${key}\``, e);
-    }
-  }
-
-  private readMetadata(key: string): GetObjectMetadata | undefined {
-    try {
-      const raw = JSON.parse(
-        readFileSync(this.join(`${key}.metadata.json`), {
-          encoding: 'utf-8',
-        })
-      );
-
-      return {
-        ...raw,
-        lastModified: new Date(raw.lastModified),
-        expires: raw.expires ? new Date(raw.expires) : undefined,
-      };
-    } catch (e) {
-      this.logger.warn(`Failed to read metadata of object \`${key}\``, e);
-
-      return;
-    }
-  }
-}

packages/backend/server/src/base/storage/providers/index.ts

@@ -1,20 +1,45 @@
-import { Type } from '@nestjs/common';
-
 import { JSONSchema } from '../../config';
-import { FsStorageConfig, FsStorageProvider } from './fs';
-import { StorageProvider } from './provider';
-import { R2_JURISDICTIONS, R2StorageConfig, R2StorageProvider } from './r2';
-import { S3StorageConfig, S3StorageProvider } from './s3';
 
-export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2';
-export const StorageProviders: Record<
-  StorageProviderName,
-  Type<StorageProvider>
-> = {
-  fs: FsStorageProvider,
-  'aws-s3': S3StorageProvider,
-  'cloudflare-r2': R2StorageProvider,
-};
+export type StorageProviderName =
+  | 'fs'
+  | 'aws-s3'
+  | 'cloudflare-r2'
+  | 'assetpack';
+
+export interface FsStorageConfig {
+  path: string;
+}
+
+export type AssetpackStorageConfig = FsStorageConfig;
+
+export interface S3StorageConfig {
+  endpoint?: string;
+  region: string;
+  credentials?: {
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    sessionToken?: string;
+  };
+  forcePathStyle?: boolean;
+  requestTimeoutMs?: number;
+  minPartSize?: number;
+  presign?: {
+    expiresInSeconds?: number;
+    signContentTypeForPut?: boolean;
+  };
+}
+
+export const R2_JURISDICTIONS = ['default', 'eu'] as const;
+
+export interface R2StorageConfig extends Omit<S3StorageConfig, 'endpoint'> {
+  accountId: string;
+  jurisdiction?: (typeof R2_JURISDICTIONS)[number];
+  usePresignedURL?: {
+    enabled: boolean;
+    urlPrefix?: string;
+    signKey?: string;
+  };
+}
 
 export type StorageProviderConfig = { bucket: string } & (
   | {
@@ -29,6 +54,10 @@ export type StorageProviderConfig = { bucket: string } & (
       provider: 'cloudflare-r2';
       config: R2StorageConfig;
     }
+  | {
+      provider: 'assetpack';
+      config: AssetpackStorageConfig;
+    }
 );
 
 const S3ConfigSchema: JSONSchema = {
@@ -186,16 +215,37 @@ export const StorageJSONSchema: JSONSchema = {
         },
       },
     },
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['assetpack'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: {
+          type: 'object',
+          properties: {
+            path: {
+              type: 'string',
+            },
+          },
+          required: ['path'],
+        },
+      },
+      required: ['provider', 'bucket', 'config'],
+    },
   ],
 };
 
-export type * from './provider';
+export type * from '../types';
 export {
   applyAttachHeaders,
-  autoMetadata,
   PROXY_MULTIPART_PATH,
   PROXY_UPLOAD_PATH,
   sniffMime,
   STORAGE_PROXY_ROOT,
   toBuffer,
-} from './utils';
+} from '../utils';

packages/backend/server/src/base/storage/providers/provider.ts

@@ -1,84 +0,0 @@
-import type { Readable } from 'node:stream';
-
-export interface GetObjectMetadata {
-  /**
-   * @default 'application/octet-stream'
-   */
-  contentType: string;
-  contentLength: number;
-  lastModified: Date;
-  checksumCRC32?: string;
-}
-
-export interface PutObjectMetadata {
-  contentType?: string;
-  contentLength?: number;
-  checksumCRC32?: string;
-}
-
-export interface ListObjectsMetadata {
-  key: string;
-  lastModified: Date;
-  contentLength: number;
-}
-
-export type BlobInputType = Buffer | Readable | string;
-export type BlobOutputType = Readable;
-
-export interface PresignedUpload {
-  url: string;
-  headers?: Record<string, string>;
-  expiresAt: Date;
-}
-
-export interface MultipartUploadInit {
-  uploadId: string;
-  expiresAt: Date;
-}
-
-export interface MultipartUploadPart {
-  partNumber: number;
-  etag: string;
-}
-
-export interface StorageProvider {
-  put(
-    key: string,
-    body: BlobInputType,
-    metadata?: PutObjectMetadata
-  ): Promise<void>;
-  presignPut?(
-    key: string,
-    metadata?: PutObjectMetadata
-  ): Promise<PresignedUpload | undefined>;
-  createMultipartUpload?(
-    key: string,
-    metadata?: PutObjectMetadata
-  ): Promise<MultipartUploadInit | undefined>;
-  presignUploadPart?(
-    key: string,
-    uploadId: string,
-    partNumber: number
-  ): Promise<PresignedUpload | undefined>;
-  listMultipartUploadParts?(
-    key: string,
-    uploadId: string
-  ): Promise<MultipartUploadPart[] | undefined>;
-  completeMultipartUpload?(
-    key: string,
-    uploadId: string,
-    parts: MultipartUploadPart[]
-  ): Promise<void>;
-  abortMultipartUpload?(key: string, uploadId: string): Promise<void>;
-  head(key: string): Promise<GetObjectMetadata | undefined>;
-  get(
-    key: string,
-    signedUrl?: boolean
-  ): Promise<{
-    redirectUrl?: string;
-    body?: BlobOutputType;
-    metadata?: GetObjectMetadata;
-  }>;
-  list(prefix?: string): Promise<ListObjectsMetadata[]>;
-  delete(key: string): Promise<void>;
-}

packages/backend/server/src/base/storage/providers/r2.ts

@@ -1,251 +0,0 @@
-import assert from 'node:assert';
-import { Readable } from 'node:stream';
-
-import { Logger } from '@nestjs/common';
-
-import {
-  GetObjectMetadata,
-  PresignedUpload,
-  PutObjectMetadata,
-} from './provider';
-import { S3StorageConfig, S3StorageProvider } from './s3';
-import {
-  PROXY_MULTIPART_PATH,
-  PROXY_UPLOAD_PATH,
-  SIGNED_URL_EXPIRED,
-} from './utils';
-
-export const R2_JURISDICTIONS = ['eu'] as const;
-type R2Jurisdiction = (typeof R2_JURISDICTIONS)[number];
-
-export interface R2StorageConfig extends Omit<
-  S3StorageConfig,
-  'endpoint' | 'forcePathStyle'
-> {
-  accountId: string;
-  jurisdiction?: R2Jurisdiction;
-  usePresignedURL?: {
-    enabled: boolean;
-    urlPrefix?: string;
-    signKey?: string;
-  };
-}
-
-export class R2StorageProvider extends S3StorageProvider {
-  private readonly encoder = new TextEncoder();
-  private readonly key: Uint8Array;
-
-  constructor(
-    private readonly config: R2StorageConfig,
-    bucket: string
-  ) {
-    assert(config.accountId, 'accountId is required for R2 storage provider');
-    const account = config.jurisdiction
-      ? `${config.accountId}.${config.jurisdiction}`
-      : config.accountId;
-    const endpoint = `https://${account}.r2.cloudflarestorage.com`;
-
-    super(
-      {
-        ...config,
-        forcePathStyle: true,
-        endpoint,
-      },
-      bucket
-    );
-    this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
-    this.key = this.encoder.encode(config.usePresignedURL?.signKey ?? '');
-  }
-
-  private get shouldUseProxyUpload() {
-    const { usePresignedURL } = this.config;
-    return (
-      !!usePresignedURL?.enabled &&
-      !!usePresignedURL.signKey &&
-      this.key.length > 0
-    );
-  }
-
-  private parseWorkspaceKey(fullKey: string) {
-    const [workspaceId, ...rest] = fullKey.split('/');
-    if (!workspaceId || rest.length !== 1) {
-      return null;
-    }
-    return { workspaceId, key: rest.join('/') };
-  }
-
-  private async signPayload(payload: string) {
-    const key = await crypto.subtle.importKey(
-      'raw',
-      this.key,
-      { name: 'HMAC', hash: 'SHA-256' },
-      false,
-      ['sign', 'verify']
-    );
-    const mac = await crypto.subtle.sign(
-      'HMAC',
-      key,
-      this.encoder.encode(payload)
-    );
-
-    return Buffer.from(mac).toString('base64');
-  }
-
-  private async signUrl(url: URL): Promise<string> {
-    const timestamp = Math.floor(Date.now() / 1000);
-    const base64Mac = await this.signPayload(`${url.pathname}${timestamp}`);
-    url.searchParams.set('sign', `${timestamp}-${base64Mac}`);
-    return url.toString();
-  }
-
-  private async createProxyUrl(
-    path: string,
-    canonicalFields: (string | number | undefined)[],
-    query: Record<string, string | number | undefined>
-  ) {
-    const exp = Math.floor(Date.now() / 1000) + SIGNED_URL_EXPIRED;
-    const canonical = [
-      path,
-      ...canonicalFields.map(field =>
-        field === undefined ? '' : field.toString()
-      ),
-      exp.toString(),
-    ].join('\n');
-    const token = await this.signPayload(canonical);
-
-    const url = new URL(`http://localhost${path}`);
-    for (const [key, value] of Object.entries(query)) {
-      if (value === undefined) continue;
-      url.searchParams.set(key, value.toString());
-    }
-    url.searchParams.set('exp', exp.toString());
-    url.searchParams.set('token', `${exp}-${token}`);
-
-    return { url: url.pathname + url.search, expiresAt: new Date(exp * 1000) };
-  }
-
-  override async presignPut(
-    key: string,
-    metadata: PutObjectMetadata = {}
-  ): Promise<PresignedUpload | undefined> {
-    if (!this.shouldUseProxyUpload) {
-      return super.presignPut(key, metadata);
-    }
-
-    const parsed = this.parseWorkspaceKey(key);
-    if (!parsed) {
-      return super.presignPut(key, metadata);
-    }
-
-    const contentType = metadata.contentType ?? 'application/octet-stream';
-    const { url, expiresAt } = await this.createProxyUrl(
-      PROXY_UPLOAD_PATH,
-      [parsed.workspaceId, parsed.key, contentType, metadata.contentLength],
-      {
-        workspaceId: parsed.workspaceId,
-        key: parsed.key,
-        contentType,
-        contentLength: metadata.contentLength,
-      }
-    );
-
-    return {
-      url,
-      headers: { 'Content-Type': contentType },
-      expiresAt,
-    };
-  }
-
-  override async presignUploadPart(
-    key: string,
-    uploadId: string,
-    partNumber: number
-  ): Promise<PresignedUpload | undefined> {
-    if (!this.shouldUseProxyUpload) {
-      return super.presignUploadPart(key, uploadId, partNumber);
-    }
-
-    const parsed = this.parseWorkspaceKey(key);
-    if (!parsed) {
-      return super.presignUploadPart(key, uploadId, partNumber);
-    }
-
-    return this.createProxyUrl(
-      PROXY_MULTIPART_PATH,
-      [parsed.workspaceId, parsed.key, uploadId, partNumber],
-      {
-        workspaceId: parsed.workspaceId,
-        key: parsed.key,
-        uploadId,
-        partNumber,
-      }
-    );
-  }
-
-  async proxyPutObject(
-    key: string,
-    body: Readable | Buffer | Uint8Array | string,
-    options: { contentType?: string; contentLength?: number } = {}
-  ) {
-    return this.client.putObject(key, this.normalizeBody(body), {
-      contentType: options.contentType,
-      contentLength: options.contentLength,
-    });
-  }
-
-  async proxyUploadPart(
-    key: string,
-    uploadId: string,
-    partNumber: number,
-    body: Readable | Buffer | Uint8Array | string,
-    options: { contentLength?: number } = {}
-  ) {
-    const result = await this.client.uploadPart(
-      key,
-      uploadId,
-      partNumber,
-      this.normalizeBody(body),
-      { contentLength: options.contentLength }
-    );
-
-    return result.etag;
-  }
-
-  private normalizeBody(body: Readable | Buffer | Uint8Array | string) {
-    // s3mini does not accept Node.js Readable directly.
-    // Convert it to Web ReadableStream for compatibility.
-    if (body instanceof Readable) {
-      return Readable.toWeb(body);
-    } else if (typeof body === 'string') {
-      return this.encoder.encode(body);
-    }
-    return body;
-  }
-
-  override async get(
-    key: string,
-    signedUrl?: boolean
-  ): Promise<{
-    body?: Readable;
-    metadata?: GetObjectMetadata;
-    redirectUrl?: string;
-  }> {
-    const { usePresignedURL: { enabled, urlPrefix } = {} } = this.config;
-    if (signedUrl && enabled && urlPrefix) {
-      const metadata = await this.head(key);
-      const url = await this.signUrl(new URL(`/${key}`, urlPrefix));
-      if (metadata) {
-        return {
-          redirectUrl: url.toString(),
-          metadata,
-        };
-      }
-
-      // object not found
-      return {};
-    }
-
-    // fallback to s3 get
-    return super.get(key, signedUrl);
-  }
-}

packages/backend/server/src/base/storage/providers/s3.ts

@@ -1,363 +0,0 @@
-/* oxlint-disable @typescript-eslint/no-non-null-assertion */
-import { Readable } from 'node:stream';
-
-import type {
-  S3CompatClient,
-  S3CompatConfig,
-  S3CompatCredentials,
-} from '@affine/s3-compat';
-import { createS3CompatClient } from '@affine/s3-compat';
-import { Logger } from '@nestjs/common';
-
-import {
-  BlobInputType,
-  GetObjectMetadata,
-  ListObjectsMetadata,
-  MultipartUploadInit,
-  MultipartUploadPart,
-  PresignedUpload,
-  PutObjectMetadata,
-  StorageProvider,
-} from './provider';
-import { autoMetadata, SIGNED_URL_EXPIRED, toBuffer } from './utils';
-
-export interface S3StorageConfig {
-  endpoint?: string;
-  region: string;
-  credentials: S3CompatCredentials;
-  forcePathStyle?: boolean;
-  requestTimeoutMs?: number;
-  minPartSize?: number;
-  presign?: {
-    expiresInSeconds?: number;
-    signContentTypeForPut?: boolean;
-  };
-  usePresignedURL?: {
-    enabled: boolean;
-  };
-}
-
-function resolveEndpoint(config: S3StorageConfig) {
-  if (config.endpoint) {
-    return config.endpoint;
-  }
-  if (config.region === 'us-east-1') {
-    return 'https://s3.amazonaws.com';
-  }
-  return `https://s3.${config.region}.amazonaws.com`;
-}
-
-function joinPath(basePath: string, suffix: string) {
-  const trimmedBase = basePath.endsWith('/') ? basePath.slice(0, -1) : basePath;
-  const trimmedSuffix = suffix.startsWith('/') ? suffix.slice(1) : suffix;
-  if (!trimmedBase) {
-    return `/${trimmedSuffix}`;
-  }
-  if (!trimmedSuffix) {
-    return trimmedBase;
-  }
-  return `${trimmedBase}/${trimmedSuffix}`;
-}
-
-function composeEndpointUrl(config: S3CompatConfig) {
-  const url = new URL(config.endpoint);
-  if (config.forcePathStyle) {
-    const firstSegment = url.pathname.split('/').find(Boolean);
-    if (firstSegment !== config.bucket) {
-      url.pathname = joinPath(url.pathname, config.bucket);
-    }
-    return url.toString();
-  }
-
-  const firstSegment = url.pathname.split('/').find(Boolean);
-  const hostHasBucket = url.hostname.startsWith(`${config.bucket}.`);
-  const pathHasBucket = firstSegment === config.bucket;
-  if (!hostHasBucket && !pathHasBucket) {
-    url.hostname = `${config.bucket}.${url.hostname}`;
-  }
-  return url.toString();
-}
-
-export class S3StorageProvider implements StorageProvider {
-  protected logger: Logger;
-  protected client: S3CompatClient;
-  private readonly usePresignedURL: boolean;
-  private readonly endpoint: string;
-
-  get endpointUrl() {
-    return this.endpoint;
-  }
-
-  constructor(
-    config: S3StorageConfig,
-    public readonly bucket: string
-  ) {
-    const { usePresignedURL, presign, credentials, ...clientConfig } = config;
-
-    const compatConfig: S3CompatConfig = {
-      ...clientConfig,
-      endpoint: resolveEndpoint(config),
-      bucket,
-      requestTimeoutMs: clientConfig.requestTimeoutMs ?? 60_000,
-      presign: {
-        expiresInSeconds: presign?.expiresInSeconds ?? SIGNED_URL_EXPIRED,
-        signContentTypeForPut: presign?.signContentTypeForPut ?? true,
-      },
-    };
-
-    this.endpoint = composeEndpointUrl(compatConfig);
-    this.client = createS3CompatClient(compatConfig, credentials);
-    this.usePresignedURL = usePresignedURL?.enabled ?? false;
-    this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
-  }
-
-  async put(
-    key: string,
-    body: BlobInputType,
-    metadata: PutObjectMetadata = {}
-  ): Promise<void> {
-    const blob = await toBuffer(body);
-
-    metadata = autoMetadata(blob, metadata);
-
-    try {
-      await this.client.putObject(key, blob, {
-        contentType: metadata.contentType,
-        contentLength: metadata.contentLength,
-      });
-
-      this.logger.verbose(`Object \`${key}\` put`);
-    } catch (e) {
-      this.logger.error(
-        `Failed to put object (${JSON.stringify({
-          key,
-          bucket: this.bucket,
-          metadata,
-        })})`
-      );
-      throw e;
-    }
-  }
-
-  async presignPut(
-    key: string,
-    metadata: PutObjectMetadata = {}
-  ): Promise<PresignedUpload | undefined> {
-    try {
-      const contentType = metadata.contentType ?? 'application/octet-stream';
-      const result = await this.client.presignPutObject(key, { contentType });
-
-      return {
-        url: result.url,
-        headers: result.headers,
-        expiresAt: result.expiresAt,
-      };
-    } catch (e) {
-      this.logger.error(
-        `Failed to presign put object (${JSON.stringify({
-          key,
-          bucket: this.bucket,
-          metadata,
-        })}`
-      );
-      throw e;
-    }
-  }
-
-  async createMultipartUpload(
-    key: string,
-    metadata: PutObjectMetadata = {}
-  ): Promise<MultipartUploadInit | undefined> {
-    try {
-      const contentType = metadata.contentType ?? 'application/octet-stream';
-      const response = await this.client.createMultipartUpload(key, {
-        contentType,
-      });
-
-      if (!response.uploadId) {
-        return;
-      }
-
-      return {
-        uploadId: response.uploadId,
-        expiresAt: new Date(Date.now() + SIGNED_URL_EXPIRED * 1000),
-      };
-    } catch (e) {
-      this.logger.error(
-        `Failed to create multipart upload (${JSON.stringify({
-          key,
-          bucket: this.bucket,
-          metadata,
-        })}`
-      );
-      throw e;
-    }
-  }
-
-  async presignUploadPart(
-    key: string,
-    uploadId: string,
-    partNumber: number
-  ): Promise<PresignedUpload | undefined> {
-    try {
-      const result = await this.client.presignUploadPart(
-        key,
-        uploadId,
-        partNumber
-      );
-
-      return {
-        url: result.url,
-        expiresAt: result.expiresAt,
-      };
-    } catch (e) {
-      this.logger.error(
-        `Failed to presign upload part (${JSON.stringify({ key, bucket: this.bucket, uploadId, partNumber })}`
-      );
-      throw e;
-    }
-  }
-
-  async listMultipartUploadParts(
-    key: string,
-    uploadId: string
-  ): Promise<MultipartUploadPart[] | undefined> {
-    try {
-      return await this.client.listParts(key, uploadId);
-    } catch (e) {
-      this.logger.error(`Failed to list multipart upload parts for \`${key}\``);
-      throw e;
-    }
-  }
-
-  async completeMultipartUpload(
-    key: string,
-    uploadId: string,
-    parts: MultipartUploadPart[]
-  ): Promise<void> {
-    try {
-      const orderedParts = [...parts].sort(
-        (left, right) => left.partNumber - right.partNumber
-      );
-
-      await this.client.completeMultipartUpload(key, uploadId, orderedParts);
-    } catch (e) {
-      this.logger.error(`Failed to complete multipart upload for \`${key}\``);
-      throw e;
-    }
-  }
-
-  async abortMultipartUpload(key: string, uploadId: string): Promise<void> {
-    try {
-      await this.client.abortMultipartUpload(key, uploadId);
-    } catch (e) {
-      this.logger.error(`Failed to abort multipart upload for \`${key}\``);
-      throw e;
-    }
-  }
-
-  async head(key: string) {
-    try {
-      const obj = await this.client.headObject(key);
-      if (!obj) {
-        this.logger.verbose(`Object \`${key}\` not found`);
-        return undefined;
-      }
-
-      return {
-        contentType: obj.contentType ?? 'application/octet-stream',
-        contentLength: obj.contentLength ?? 0,
-        lastModified: obj.lastModified ?? new Date(0),
-        checksumCRC32: obj.checksumCRC32,
-      };
-    } catch (e) {
-      this.logger.error(`Failed to head object \`${key}\``);
-      throw e;
-    }
-  }
-
-  async get(
-    key: string,
-    signedUrl?: boolean
-  ): Promise<{
-    body?: Readable;
-    metadata?: GetObjectMetadata;
-    redirectUrl?: string;
-  }> {
-    try {
-      if (this.usePresignedURL && signedUrl) {
-        const metadata = await this.head(key);
-        if (metadata) {
-          const result = await this.client.presignGetObject(key);
-
-          return {
-            redirectUrl: result.url,
-            metadata,
-          };
-        }
-
-        // object not found
-        return {};
-      }
-
-      const obj = await this.client.getObjectResponse(key);
-      if (!obj || !obj.body) {
-        this.logger.verbose(`Object \`${key}\` not found`);
-        return {};
-      }
-
-      const contentType = obj.headers.get('content-type') ?? undefined;
-      const contentLengthHeader = obj.headers.get('content-length');
-      const contentLength = contentLengthHeader
-        ? Number(contentLengthHeader)
-        : undefined;
-      const lastModifiedHeader = obj.headers.get('last-modified');
-      const lastModified = lastModifiedHeader
-        ? new Date(lastModifiedHeader)
-        : undefined;
-
-      this.logger.verbose(`Read object \`${key}\``);
-      return {
-        body: Readable.fromWeb(obj.body),
-        metadata: {
-          contentType: contentType ?? 'application/octet-stream',
-          contentLength: contentLength ?? 0,
-          lastModified: lastModified ?? new Date(0),
-          checksumCRC32: obj.headers.get('x-amz-checksum-crc32') ?? undefined,
-        },
-      };
-    } catch (e) {
-      this.logger.error(`Failed to read object \`${key}\``);
-      throw e;
-    }
-  }
-
-  async list(prefix?: string): Promise<ListObjectsMetadata[]> {
-    try {
-      const result = await this.client.listObjectsV2(prefix);
-
-      this.logger.verbose(
-        `List ${result.length} objects with prefix \`${prefix}\``
-      );
-      return result;
-    } catch (e) {
-      this.logger.error(`Failed to list objects with prefix \`${prefix}\``);
-      throw e;
-    }
-  }
-
-  async delete(key: string): Promise<void> {
-    try {
-      await this.client.deleteObject(key);
-
-      this.logger.verbose(`Deleted object \`${key}\``);
-    } catch (e) {
-      this.logger.error(`Failed to delete object \`${key}\``, {
-        bucket: this.bucket,
-        key,
-        cause: e,
-      });
-      throw e;
-    }
-  }
-}

packages/backend/server/src/base/storage/types.ts

@@ -0,0 +1,32 @@
+import type { Readable } from 'node:stream';
+
+export interface GetObjectMetadata {
+  /**
+   * @default 'application/octet-stream'
+   */
+  contentType: string;
+  contentLength: number;
+  lastModified: Date;
+  checksumCRC32?: string;
+}
+
+export interface PutObjectMetadata {
+  contentType?: string;
+  contentLength?: number;
+  checksumCRC32?: string;
+}
+
+export interface ListObjectsMetadata {
+  key: string;
+  lastModified: Date;
+  contentLength: number;
+}
+
+export type BlobInputType = Buffer | Readable | string;
+export type BlobOutputType = Readable;
+
+export interface PresignedUpload {
+  url: string;
+  headers?: Record<string, string>;
+  expiresAt: Date;
+}

packages/backend/server/src/base/storage/utils.ts

@@ -1,11 +1,10 @@
 import { Readable } from 'node:stream';
 
-import { crc32 } from '@node-rs/crc32';
 import type { Response } from 'express';
 import { getStreamAsBuffer } from 'get-stream';
 
-import { getMime } from '../../../native';
-import { BlobInputType, PutObjectMetadata } from './provider';
+import { getMime } from '../../native';
+import type { BlobInputType } from './types';
 
 export async function toBuffer(input: BlobInputType): Promise<Buffer> {
   return input instanceof Readable
@@ -15,35 +14,6 @@ export async function toBuffer(input: BlobInputType): Promise<Buffer> {
       : Buffer.from(input as string);
 }
 
-export function autoMetadata(
-  blob: Buffer,
-  raw: PutObjectMetadata = {}
-): PutObjectMetadata {
-  const metadata = {
-    ...raw,
-  };
-
-  if (!metadata.contentLength) {
-    metadata.contentLength = blob.byteLength;
-  }
-
-  try {
-    // checksum
-    if (!metadata.checksumCRC32) {
-      metadata.checksumCRC32 = crc32(blob).toString(16);
-    }
-
-    // mime type
-    if (!metadata.contentType) {
-      metadata.contentType = getMime(blob);
-    }
-  } catch {
-    // noop
-  }
-
-  return metadata;
-}
-
 const DANGEROUS_INLINE_MIME_PREFIXES = [
   'text/html',
   'application/xhtml+xml',