refactor(server): config system (#11081)

packages/backend/server/src/base/storage/config.ts

@@ -1,75 +0,0 @@
-import { homedir } from 'node:os';
-import { join } from 'node:path';
-
-import { defineStartupConfig, ModuleConfig } from '../config';
-
-export interface FsStorageConfig {
-  path: string;
-}
-
-export interface StorageProvidersConfig {
-  fs?: FsStorageConfig;
-}
-
-declare module '../config' {
-  interface AppConfig {
-    storageProviders: ModuleConfig<StorageProvidersConfig>;
-  }
-}
-
-defineStartupConfig('storageProviders', {
-  fs: {
-    path: join(homedir(), '.affine/storage'),
-  },
-});
-
-export type StorageProviderType = keyof StorageProvidersConfig;
-
-export type StorageConfig<Ext = unknown> = {
-  provider: StorageProviderType;
-  bucket: string;
-} & Ext;
-
-export interface StoragesConfig {
-  avatar: StorageConfig<{ publicLinkFactory: (key: string) => string }>;
-  blob: StorageConfig;
-  copilot: StorageConfig;
-}
-
-export interface AFFiNEStorageConfig {
-  /**
-   * All providers for object storage
-   *
-   * Support different providers for different usage at the same time.
-   */
-  providers: StorageProvidersConfig;
-  storages: StoragesConfig;
-}
-
-export type StorageProviders = AFFiNEStorageConfig['providers'];
-export type Storages = keyof AFFiNEStorageConfig['storages'];
-
-export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig {
-  return {
-    providers: {
-      fs: {
-        path: join(homedir(), '.affine/storage'),
-      },
-    },
-    storages: {
-      avatar: {
-        provider: 'fs',
-        bucket: 'avatars',
-        publicLinkFactory: key => `/api/avatars/${key}`,
-      },
-      blob: {
-        provider: 'fs',
-        bucket: 'blobs',
-      },
-      copilot: {
-        provider: 'fs',
-        bucket: 'copilot',
-      },
-    },
-  };
-}

packages/backend/server/src/base/storage/factory.ts

@@ -0,0 +1,20 @@
+import { Injectable } from '@nestjs/common';
+
+import {
+  StorageProvider,
+  StorageProviderConfig,
+  StorageProviders,
+} from './providers';
+
+@Injectable()
+export class StorageProviderFactory {
+  create(config: StorageProviderConfig): StorageProvider {
+    const Provider = StorageProviders[config.provider];
+
+    if (!Provider) {
+      throw new Error(`Unknown storage provider type: ${config.provider}`);
+    }
+
+    return new Provider(config.config, config.bucket);
+  }
+}

packages/backend/server/src/base/storage/index.ts

@@ -1,17 +1,6 @@
-import './config';
-
 import { Global, Module } from '@nestjs/common';
 
-import { registerStorageProvider, StorageProviderFactory } from './providers';
-import { FsStorageProvider } from './providers/fs';
-
-registerStorageProvider('fs', (config, bucket) => {
-  if (!config.storageProviders.fs) {
-    throw new Error('Missing fs storage provider configuration');
-  }
-
-  return new FsStorageProvider(config.storageProviders.fs, bucket);
-});
+import { StorageProviderFactory } from './factory';
 
 @Global()
 @Module({
@@ -19,16 +8,5 @@ registerStorageProvider('fs', (config, bucket) => {
   exports: [StorageProviderFactory],
 })
 export class StorageProviderModule {}
-
-export * from '../../native';
-export type { StorageProviderType } from './config';
-export type {
-  BlobInputType,
-  BlobOutputType,
-  GetObjectMetadata,
-  ListObjectsMetadata,
-  PutObjectMetadata,
-  StorageProvider,
-} from './providers';
-export { registerStorageProvider, StorageProviderFactory } from './providers';
-export { autoMetadata, toBuffer } from './providers/utils';
+export { StorageProviderFactory } from './factory';
+export * from './providers';

packages/backend/server/src/base/storage/providers/fs.ts

@@ -10,12 +10,12 @@ import {
   statSync,
   writeFileSync,
 } from 'node:fs';
-import { join, parse, resolve } from 'node:path';
+import { homedir } from 'node:os';
+import { join, parse } from 'node:path';
 import { Readable } from 'node:stream';
 
 import { Logger } from '@nestjs/common';
 
-import { FsStorageConfig } from '../config';
 import {
   BlobInputType,
   GetObjectMetadata,
@@ -30,6 +30,10 @@ function escapeKey(key: string): string {
   return key.replace(/\.?\.[/\\]/g, '%');
 }
 
+export interface FsStorageConfig {
+  path: string;
+}
+
 export class FsStorageProvider implements StorageProvider {
   private readonly path: string;
   private readonly logger: Logger;
@@ -40,7 +44,9 @@ export class FsStorageProvider implements StorageProvider {
     config: FsStorageConfig,
     public readonly bucket: string
   ) {
-    this.path = resolve(config.path, bucket);
+    this.path = config.path.startsWith('~/')
+      ? join(homedir(), config.path.slice(2), bucket)
+      : join(config.path, bucket);
     this.ensureAvailability();
 
     this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`);

packages/backend/server/src/base/storage/providers/index.ts

@@ -1,34 +1,116 @@
-import { Injectable } from '@nestjs/common';
+import { Type } from '@nestjs/common';
 
-import { Config } from '../../config';
-import { StorageConfig, StorageProviderType } from '../config';
-import type { StorageProvider } from './provider';
+import { JSONSchema } from '../../config';
+import { FsStorageConfig, FsStorageProvider } from './fs';
+import { StorageProvider } from './provider';
+import { R2StorageConfig, R2StorageProvider } from './r2';
+import { S3StorageConfig, S3StorageProvider } from './s3';
 
-const availableProviders = new Map<
-  StorageProviderType,
-  (config: Config, bucket: string) => StorageProvider
->();
+export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2';
+export const StorageProviders: Record<
+  StorageProviderName,
+  Type<StorageProvider>
+> = {
+  fs: FsStorageProvider,
+  'aws-s3': S3StorageProvider,
+  'cloudflare-r2': R2StorageProvider,
+};
 
-export function registerStorageProvider(
-  type: StorageProviderType,
-  providerFactory: (config: Config, bucket: string) => StorageProvider
-) {
-  availableProviders.set(type, providerFactory);
-}
-
-@Injectable()
-export class StorageProviderFactory {
-  constructor(private readonly config: Config) {}
-
-  create(storage: StorageConfig): StorageProvider {
-    const providerFactory = availableProviders.get(storage.provider);
-
-    if (!providerFactory) {
-      throw new Error(`Unknown storage provider type: ${storage.provider}`);
+export type StorageProviderConfig = { bucket: string } & (
+  | {
+      provider: 'fs';
+      config: FsStorageConfig;
     }
+  | {
+      provider: 'aws-s3';
+      config: S3StorageConfig;
+    }
+  | {
+      provider: 'cloudflare-r2';
+      config: R2StorageConfig;
+    }
+);
 
-    return providerFactory(this.config, storage.bucket);
-  }
-}
+const S3ConfigSchema: JSONSchema = {
+  type: 'object',
+  description:
+    'The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html',
+  properties: {
+    credentials: {
+      type: 'object',
+      description: 'The credentials for the s3 compatible storage provider.',
+      properties: {
+        accessKeyId: {
+          type: 'string',
+        },
+        secretAccessKey: {
+          type: 'string',
+        },
+      },
+    },
+  },
+};
+
+export const StorageJSONSchema: JSONSchema = {
+  oneOf: [
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['fs'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: {
+          type: 'object',
+          properties: {
+            path: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['aws-s3'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: S3ConfigSchema,
+      },
+    },
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['cloudflare-r2'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: {
+          ...S3ConfigSchema,
+          properties: {
+            ...S3ConfigSchema.properties,
+            accountId: {
+              type: 'string' as const,
+              description:
+                'The account id for the cloudflare r2 storage provider.',
+            },
+          },
+        },
+      },
+    },
+  ],
+};
 
 export type * from './provider';
+export { autoMetadata, toBuffer } from './utils';

packages/backend/server/src/base/storage/providers/provider.ts

@@ -1,7 +1,5 @@
 import type { Readable } from 'node:stream';
 
-import { StorageProviderType } from '../config';
-
 export interface GetObjectMetadata {
   /**
    * @default 'application/octet-stream'
@@ -28,7 +26,6 @@ export type BlobInputType = Buffer | Readable | string;
 export type BlobOutputType = Readable;
 
 export interface StorageProvider {
-  readonly type: StorageProviderType;
   put(
     key: string,
     body: BlobInputType,

packages/backend/server/src/base/storage/providers/r2.ts

@@ -0,0 +1,27 @@
+import assert from 'node:assert';
+
+import { Logger } from '@nestjs/common';
+
+import { S3StorageConfig, S3StorageProvider } from './s3';
+
+export interface R2StorageConfig extends S3StorageConfig {
+  accountId: string;
+}
+
+export class R2StorageProvider extends S3StorageProvider {
+  constructor(config: R2StorageConfig, bucket: string) {
+    assert(config.accountId, 'accountId is required for R2 storage provider');
+    super(
+      {
+        ...config,
+        forcePathStyle: true,
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        // see https://github.com/aws/aws-sdk-js-v3/issues/6810
+        requestChecksumCalculation: 'WHEN_REQUIRED',
+        responseChecksumValidation: 'WHEN_REQUIRED',
+      },
+      bucket
+    );
+    this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
+  }
+}

packages/backend/server/src/base/storage/providers/s3.ts

@@ -0,0 +1,207 @@
+/* oxlint-disable @typescript-eslint/no-non-null-assertion */
+import { Readable } from 'node:stream';
+
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  NoSuchKey,
+  PutObjectCommand,
+  S3Client,
+  S3ClientConfig,
+} from '@aws-sdk/client-s3';
+import { Logger } from '@nestjs/common';
+
+import {
+  BlobInputType,
+  GetObjectMetadata,
+  ListObjectsMetadata,
+  PutObjectMetadata,
+  StorageProvider,
+} from './provider';
+import { autoMetadata, toBuffer } from './utils';
+
+export type S3StorageConfig = S3ClientConfig;
+
+export class S3StorageProvider implements StorageProvider {
+  protected logger: Logger;
+  protected client: S3Client;
+
+  constructor(
+    config: S3StorageConfig,
+    public readonly bucket: string
+  ) {
+    this.client = new S3Client({
+      region: 'auto',
+      // s3 client uses keep-alive by default to accelerate requests, and max requests queue is 50.
+      // If some of them are long holding or dead without response, the whole queue will block.
+      // By default no timeout is set for requests or connections, so we set them here.
+      requestHandler: { requestTimeout: 60_000, connectionTimeout: 10_000 },
+      ...config,
+    });
+    this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
+  }
+
+  async put(
+    key: string,
+    body: BlobInputType,
+    metadata: PutObjectMetadata = {}
+  ): Promise<void> {
+    const blob = await toBuffer(body);
+
+    metadata = autoMetadata(blob, metadata);
+
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+          Body: blob,
+
+          // metadata
+          ContentType: metadata.contentType,
+          ContentLength: metadata.contentLength,
+          // TODO(@forehalo): Cloudflare doesn't support CRC32, use md5 instead later.
+          // ChecksumCRC32: metadata.checksumCRC32,
+        })
+      );
+
+      this.logger.verbose(`Object \`${key}\` put`);
+    } catch (e) {
+      this.logger.error(
+        `Failed to put object (${JSON.stringify({
+          key,
+          bucket: this.bucket,
+          metadata,
+        })})`
+      );
+      throw e;
+    }
+  }
+
+  async head(key: string) {
+    try {
+      const obj = await this.client.send(
+        new HeadObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      return {
+        contentType: obj.ContentType!,
+        contentLength: obj.ContentLength!,
+        lastModified: obj.LastModified!,
+        checksumCRC32: obj.ChecksumCRC32,
+      };
+    } catch (e) {
+      // 404
+      if (e instanceof NoSuchKey) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return undefined;
+      }
+      this.logger.error(`Failed to head object \`${key}\``);
+      throw e;
+    }
+  }
+
+  async get(key: string): Promise<{
+    body?: Readable;
+    metadata?: GetObjectMetadata;
+  }> {
+    try {
+      const obj = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      if (!obj.Body) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return {};
+      }
+
+      this.logger.verbose(`Read object \`${key}\``);
+      return {
+        // @ts-expect-errors ignore browser response type `Blob`
+        body: obj.Body,
+        metadata: {
+          // always set when putting object
+          contentType: obj.ContentType!,
+          contentLength: obj.ContentLength!,
+          lastModified: obj.LastModified!,
+          checksumCRC32: obj.ChecksumCRC32,
+        },
+      };
+    } catch (e) {
+      // 404
+      if (e instanceof NoSuchKey) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return {};
+      }
+      this.logger.error(`Failed to read object \`${key}\``);
+      throw e;
+    }
+  }
+
+  async list(prefix?: string): Promise<ListObjectsMetadata[]> {
+    // continuationToken should be `string | undefined`,
+    // but TypeScript will fail on type infer in the code below.
+    // Seems to be a bug in TypeScript
+    let continuationToken: any = undefined;
+    let hasMore = true;
+    let result: ListObjectsMetadata[] = [];
+
+    try {
+      while (hasMore) {
+        const listResult = await this.client.send(
+          new ListObjectsV2Command({
+            Bucket: this.bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+          })
+        );
+
+        if (listResult.Contents?.length) {
+          result = result.concat(
+            listResult.Contents.map(r => ({
+              key: r.Key!,
+              lastModified: r.LastModified!,
+              contentLength: r.Size!,
+            }))
+          );
+        }
+
+        // has more items not listed
+        hasMore = !!listResult.IsTruncated;
+        continuationToken = listResult.NextContinuationToken;
+      }
+
+      this.logger.verbose(
+        `List ${result.length} objects with prefix \`${prefix}\``
+      );
+      return result;
+    } catch (e) {
+      this.logger.error(`Failed to list objects with prefix \`${prefix}\``);
+      throw e;
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    try {
+      await this.client.send(
+        new DeleteObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      this.logger.verbose(`Deleted object \`${key}\``);
+    } catch (e) {
+      this.logger.error(`Failed to delete object \`${key}\``);
+      throw e;
+    }
+  }
+}