refactor(server): separate s3 & r2 storage to plugin (#5805)

packages/backend/server/src/plugins/config.ts

@@ -1,12 +1,15 @@
 import { GCloudConfig } from './gcloud/config';
 import { PaymentConfig } from './payment';
 import { RedisOptions } from './redis';
+import { R2StorageConfig, S3StorageConfig } from './storage';
 
 declare module '../fundamentals/config' {
   interface PluginsConfig {
     readonly payment: PaymentConfig;
     readonly redis: RedisOptions;
     readonly gcloud: GCloudConfig;
+    readonly 'cloudflare-r2': R2StorageConfig;
+    readonly 'aws-s3': S3StorageConfig;
   }
 
   export type AvailablePlugins = keyof PluginsConfig;

packages/backend/server/src/plugins/index.ts

@@ -2,9 +2,12 @@ import type { AvailablePlugins } from '../fundamentals/config';
 import { GCloudModule } from './gcloud';
 import { PaymentModule } from './payment';
 import { RedisModule } from './redis';
+import { AwsS3Module, CloudflareR2Module } from './storage';
 
 export const pluginsMap = new Map<AvailablePlugins, AFFiNEModule>([
   ['payment', PaymentModule],
   ['redis', RedisModule],
   ['gcloud', GCloudModule],
+  ['cloudflare-r2', CloudflareR2Module],
+  ['aws-s3', AwsS3Module],
 ]);

packages/backend/server/src/plugins/storage/index.ts

@@ -0,0 +1,40 @@
+import { OptionalModule } from '../../fundamentals';
+import { registerStorageProvider } from '../../fundamentals/storage';
+import { R2StorageProvider } from './providers/r2';
+import { S3StorageProvider } from './providers/s3';
+
+registerStorageProvider('cloudflare-r2', (config, bucket) => {
+  if (!config.plugins['cloudflare-r2']) {
+    throw new Error('Missing cloudflare-r2 storage provider configuration');
+  }
+
+  return new R2StorageProvider(config.plugins['cloudflare-r2'], bucket);
+});
+registerStorageProvider('aws-s3', (config, bucket) => {
+  if (!config.plugins['aws-s3']) {
+    throw new Error('Missing aws-s3 storage provider configuration');
+  }
+
+  return new S3StorageProvider(config.plugins['aws-s3'], bucket);
+});
+
+@OptionalModule({
+  requires: [
+    'plugins.cloudflare-r2.accountId',
+    'plugins.cloudflare-r2.credentials.accessKeyId',
+    'plugins.cloudflare-r2.credentials.secretAccessKey',
+  ],
+  if: config => config.flavor.graphql,
+})
+export class CloudflareR2Module {}
+
+@OptionalModule({
+  requires: [
+    'plugins.aws-s3.credentials.accessKeyId',
+    'plugins.aws-s3.credentials.secretAccessKey',
+  ],
+  if: config => config.flavor.graphql,
+})
+export class AwsS3Module {}
+
+export type { R2StorageConfig, S3StorageConfig } from './types';

packages/backend/server/src/plugins/storage/providers/r2.ts

@@ -0,0 +1,20 @@
+import { Logger } from '@nestjs/common';
+
+import type { R2StorageConfig } from '../types';
+import { S3StorageProvider } from './s3';
+
+export class R2StorageProvider extends S3StorageProvider {
+  override readonly type = 'cloudflare-r2' as any /* cast 'r2' to 's3' */;
+
+  constructor(config: R2StorageConfig, bucket: string) {
+    super(
+      {
+        ...config,
+        forcePathStyle: true,
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+      },
+      bucket
+    );
+    this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
+  }
+}

packages/backend/server/src/plugins/storage/providers/s3.ts

@@ -0,0 +1,171 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+import { Readable } from 'node:stream';
+
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  NoSuchKey,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { Logger } from '@nestjs/common';
+
+import {
+  autoMetadata,
+  BlobInputType,
+  GetObjectMetadata,
+  ListObjectsMetadata,
+  PutObjectMetadata,
+  StorageProvider,
+  toBuffer,
+} from '../../../fundamentals/storage';
+import type { S3StorageConfig } from '../types';
+
+export class S3StorageProvider implements StorageProvider {
+  protected logger: Logger;
+  protected client: S3Client;
+
+  readonly type = 'aws-s3';
+
+  constructor(
+    config: S3StorageConfig,
+    public readonly bucket: string
+  ) {
+    this.client = new S3Client({ region: 'auto', ...config });
+    this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
+  }
+
+  async put(
+    key: string,
+    body: BlobInputType,
+    metadata: PutObjectMetadata = {}
+  ): Promise<void> {
+    const blob = await toBuffer(body);
+
+    metadata = await autoMetadata(blob, metadata);
+
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+          Body: blob,
+
+          // metadata
+          ContentType: metadata.contentType,
+          ContentLength: metadata.contentLength,
+          // TODO: Cloudflare doesn't support CRC32, use md5 instead later.
+          // ChecksumCRC32: metadata.checksumCRC32,
+        })
+      );
+
+      this.logger.verbose(`Object \`${key}\` put`);
+    } catch (e) {
+      throw new Error(`Failed to put object \`${key}\``, {
+        cause: e,
+      });
+    }
+  }
+
+  async get(key: string): Promise<{
+    body?: Readable;
+    metadata?: GetObjectMetadata;
+  }> {
+    try {
+      const obj = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      if (!obj.Body) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return {};
+      }
+
+      this.logger.verbose(`Read object \`${key}\``);
+      return {
+        // @ts-expect-errors ignore browser response type `Blob`
+        body: obj.Body,
+        metadata: {
+          // always set when putting object
+          contentType: obj.ContentType!,
+          contentLength: obj.ContentLength!,
+          lastModified: obj.LastModified!,
+          checksumCRC32: obj.ChecksumCRC32,
+        },
+      };
+    } catch (e) {
+      // 404
+      if (e instanceof NoSuchKey) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return {};
+      } else {
+        throw new Error(`Failed to read object \`${key}\``, {
+          cause: e,
+        });
+      }
+    }
+  }
+
+  async list(prefix?: string): Promise<ListObjectsMetadata[]> {
+    // continuationToken should be `string | undefined`,
+    // but TypeScript will fail on type infer in the code below.
+    // Seems to be a bug in TypeScript
+    let continuationToken: any = undefined;
+    let hasMore = true;
+    let result: ListObjectsMetadata[] = [];
+
+    try {
+      while (hasMore) {
+        const listResult = await this.client.send(
+          new ListObjectsV2Command({
+            Bucket: this.bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+          })
+        );
+
+        if (listResult.Contents?.length) {
+          result = result.concat(
+            listResult.Contents.map(r => ({
+              key: r.Key!,
+              lastModified: r.LastModified!,
+              size: r.Size!,
+            }))
+          );
+        }
+
+        // has more items not listed
+        hasMore = !!listResult.IsTruncated;
+        continuationToken = listResult.NextContinuationToken;
+      }
+
+      this.logger.verbose(
+        `List ${result.length} objects with prefix \`${prefix}\``
+      );
+      return result;
+    } catch (e) {
+      throw new Error(`Failed to list objects with prefix \`${prefix}\``, {
+        cause: e,
+      });
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    try {
+      await this.client.send(
+        new DeleteObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+    } catch (e) {
+      throw new Error(`Failed to delete object \`${key}\``, {
+        cause: e,
+      });
+    }
+  }
+}

packages/backend/server/src/plugins/storage/types.ts

@@ -0,0 +1,16 @@
+import { S3ClientConfigType } from '@aws-sdk/client-s3';
+
+type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__';
+export type R2StorageConfig = S3ClientConfigType & {
+  accountId: string;
+};
+
+export type S3StorageConfig = S3ClientConfigType;
+
+declare module '../../fundamentals/config/storage' {
+  interface StorageProvidersConfig {
+    // the type here is only existing for extends [StorageProviderType] with better type inference and checking.
+    'cloudflare-r2'?: WARNING;
+    'aws-s3'?: WARNING;
+  }
+}