refactor(server): use new storage providers (#5433)

packages/backend/server/src/modules/storage/fs.ts

@@ -1,30 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import { createWriteStream } from 'node:fs';
-import { mkdir } from 'node:fs/promises';
-import { join } from 'node:path';
-import { pipeline } from 'node:stream/promises';
-
-import { Injectable } from '@nestjs/common';
-
-import { Config } from '../../config';
-import { FileUpload } from '../../types';
-
-@Injectable()
-export class FSService {
-  constructor(private readonly config: Config) {}
-
-  async writeFile(key: string, file: FileUpload) {
-    const dest = this.config.objectStorage.fs.path;
-    const fileName = `${key}-${randomUUID()}`;
-    const prefix = this.config.node.dev
-      ? `${this.config.https ? 'https' : 'http'}://${this.config.host}:${
-          this.config.port
-        }`
-      : '';
-    await mkdir(dest, { recursive: true });
-    const destFile = join(dest, fileName);
-    await pipeline(file.createReadStream(), createWriteStream(destFile));
-
-    return `${prefix}/assets/${fileName}`;
-  }
-}

packages/backend/server/src/modules/storage/index.ts

@@ -1,11 +1,11 @@
 import { Module } from '@nestjs/common';
 
-import { FSService } from './fs';
-import { S3 } from './s3';
-import { StorageService } from './storage.service';
+import { AvatarStorage, WorkspaceBlobStorage } from './wrappers';
 
 @Module({
-  providers: [S3, StorageService, FSService],
-  exports: [StorageService],
+  providers: [WorkspaceBlobStorage, AvatarStorage],
+  exports: [WorkspaceBlobStorage, AvatarStorage],
 })
 export class StorageModule {}
+
+export { AvatarStorage, WorkspaceBlobStorage };

packages/backend/server/src/modules/storage/providers/fs.ts

@@ -34,6 +34,8 @@ export class FsStorageProvider implements StorageProvider {
   private readonly path: string;
   private readonly logger: Logger;
 
+  readonly type = 'fs';
+
   constructor(
     config: FsStorageConfig,
     public readonly bucket: string

packages/backend/server/src/modules/storage/providers/provider.ts

@@ -1,5 +1,7 @@
 import type { Readable } from 'node:stream';
 
+import { StorageProviderType } from '../../../config';
+
 export interface GetObjectMetadata {
   /**
    * @default 'application/octet-stream'
@@ -26,6 +28,7 @@ export type BlobInputType = Buffer | Readable | string;
 export type BlobOutputType = Readable;
 
 export interface StorageProvider {
+  readonly type: StorageProviderType;
   put(
     key: string,
     body: BlobInputType,

packages/backend/server/src/modules/storage/providers/r2.ts

@@ -2,6 +2,8 @@ import { R2StorageConfig } from '../../../config/storage';
 import { S3StorageProvider } from './s3';
 
 export class R2StorageProvider extends S3StorageProvider {
+  override readonly type = 'r2' as any /* cast 'r2' to 's3' */;
+
   constructor(config: R2StorageConfig, bucket: string) {
     super(
       {

packages/backend/server/src/modules/storage/providers/s3.ts

@@ -21,8 +21,11 @@ import {
 import { autoMetadata, toBuffer } from './utils';
 
 export class S3StorageProvider implements StorageProvider {
-  logger: Logger;
-  client: S3Client;
+  private readonly logger: Logger;
+  protected client: S3Client;
+
+  readonly type = 's3';
+
   constructor(
     config: S3StorageConfig,
     public readonly bucket: string

packages/backend/server/src/modules/storage/s3.ts

@@ -1,22 +0,0 @@
-import { S3Client } from '@aws-sdk/client-s3';
-import { FactoryProvider } from '@nestjs/common';
-
-import { Config } from '../../config';
-
-export const S3_SERVICE = Symbol('S3_SERVICE');
-
-export const S3: FactoryProvider<S3Client> = {
-  provide: S3_SERVICE,
-  useFactory: (config: Config) => {
-    const s3 = new S3Client({
-      region: 'auto',
-      endpoint: `https://${config.objectStorage.r2.accountId}.r2.cloudflarestorage.com`,
-      credentials: {
-        accessKeyId: config.objectStorage.r2.accessKeyId,
-        secretAccessKey: config.objectStorage.r2.secretAccessKey,
-      },
-    });
-    return s3;
-  },
-  inject: [Config],
-};

packages/backend/server/src/modules/storage/storage.service.ts

@@ -1,43 +0,0 @@
-import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
-import { Inject, Injectable } from '@nestjs/common';
-import { crc32 } from '@node-rs/crc32';
-import { fileTypeFromBuffer } from 'file-type';
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore - no types
-import { getStreamAsBuffer } from 'get-stream';
-
-import { Config } from '../../config';
-import { FileUpload } from '../../types';
-import { FSService } from './fs';
-import { S3_SERVICE } from './s3';
-
-@Injectable()
-export class StorageService {
-  constructor(
-    @Inject(S3_SERVICE) private readonly s3: S3Client,
-    private readonly fs: FSService,
-    private readonly config: Config
-  ) {}
-
-  async uploadFile(key: string, file: FileUpload) {
-    if (this.config.objectStorage.r2.enabled) {
-      const readableFile = file.createReadStream();
-      const fileBuffer = await getStreamAsBuffer(readableFile);
-      const mime = (await fileTypeFromBuffer(fileBuffer))?.mime;
-      const crc32Value = crc32(fileBuffer);
-      const keyWithCrc32 = `${crc32Value}-${key}`;
-      await this.s3.send(
-        new PutObjectCommand({
-          Body: fileBuffer,
-          Bucket: this.config.objectStorage.r2.bucket,
-          Key: keyWithCrc32,
-          ContentLength: fileBuffer.length,
-          ContentType: mime,
-        })
-      );
-      return `https://avatar.affineassets.com/${keyWithCrc32}`;
-    } else {
-      return this.fs.writeFile(key, file);
-    }
-  }
-}

packages/backend/server/src/modules/storage/wrappers/avatar.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common';
 
-import { Config } from '../../../config';
+import { AFFiNEStorageConfig, Config } from '../../../config';
+import { type EventPayload, OnEvent } from '../../../event';
 import {
   BlobInputType,
   createStorageProvider,
@@ -11,20 +12,36 @@ import {
 @Injectable()
 export class AvatarStorage {
   public readonly provider: StorageProvider;
+  private readonly storageConfig: AFFiNEStorageConfig['storages']['avatar'];
 
-  constructor({ storage }: Config) {
-    this.provider = createStorageProvider(storage, 'avatar');
+  constructor(private readonly config: Config) {
+    this.provider = createStorageProvider(this.config.storage, 'avatar');
+    this.storageConfig = this.config.storage.storages.avatar;
   }
 
-  put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) {
-    return this.provider.put(key, blob, metadata);
+  async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) {
+    await this.provider.put(key, blob, metadata);
+    let link = this.storageConfig.publicLinkFactory(key);
+
+    if (link.startsWith('/')) {
+      link = this.config.baseUrl + link;
+    }
+
+    return link;
   }
 
   get(key: string) {
     return this.provider.get(key);
   }
 
-  async delete(key: string) {
+  delete(key: string) {
     return this.provider.delete(key);
   }
+
+  @OnEvent('user.deleted')
+  async onUserDeleted(user: EventPayload<'user.deleted'>) {
+    if (user.avatarUrl) {
+      await this.delete(user.avatarUrl);
+    }
+  }
 }

packages/backend/server/src/modules/storage/wrappers/blob.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common';
 
 import { Config } from '../../../config';
+import { EventEmitter, type EventPayload, OnEvent } from '../../../event';
 import {
   BlobInputType,
   createStorageProvider,
@@ -10,7 +11,10 @@ import {
 @Injectable()
 export class WorkspaceBlobStorage {
   public readonly provider: StorageProvider;
-  constructor({ storage }: Config) {
+  constructor(
+    private readonly event: EventEmitter,
+    { storage }: Config
+  ) {
     this.provider = createStorageProvider(storage, 'blob');
   }
 
@@ -42,4 +46,25 @@ export class WorkspaceBlobStorage {
     // how could we ignore the ones get soft-deleted?
     return blobs.reduce((acc, item) => acc + item.size, 0);
   }
+
+  @OnEvent('workspace.deleted')
+  async onWorkspaceDeleted(workspaceId: EventPayload<'workspace.deleted'>) {
+    const blobs = await this.list(workspaceId);
+
+    // to reduce cpu time holding
+    blobs.forEach(blob => {
+      this.event.emit('workspace.blob.deleted', {
+        workspaceId: workspaceId,
+        name: blob.key,
+      });
+    });
+  }
+
+  @OnEvent('workspace.blob.deleted')
+  async onDeleteWorkspaceBlob({
+    workspaceId,
+    name,
+  }: EventPayload<'workspace.blob.deleted'>) {
+    await this.delete(workspaceId, name);
+  }
 }