feat(server): blob data migration (#5461)

packages/backend/server/src/data/app.ts

@@ -1,18 +1,32 @@
 import { Logger, Module } from '@nestjs/common';
 import { CommandFactory } from 'nest-commander';
 
-import { PrismaModule } from '../prisma';
+import { AppModule as BusinessAppModule } from '../app';
+import { ConfigModule } from '../config';
 import { CreateCommand, NameQuestion } from './commands/create';
 import { RevertCommand, RunCommand } from './commands/run';
 
 @Module({
-  imports: [PrismaModule],
+  imports: [
+    ConfigModule.forRoot({
+      doc: {
+        manager: {
+          enableUpdateAutoMerging: false,
+        },
+      },
+    }),
+    BusinessAppModule,
+  ],
   providers: [NameQuestion, CreateCommand, RunCommand, RevertCommand],
 })
 class AppModule {}
 
 async function bootstrap() {
-  await CommandFactory.run(AppModule, new Logger());
+  await CommandFactory.run(AppModule, new Logger()).catch(e => {
+    console.error(e);
+    process.exit(1);
+  });
+  process.exit(0);
 }
 
 await bootstrap();

packages/backend/server/src/data/commands/run.ts

@@ -3,6 +3,7 @@ import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { Logger } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
 import { Command, CommandRunner } from 'nest-commander';
 
 import { PrismaService } from '../../prisma';
@@ -10,8 +11,8 @@ import { PrismaService } from '../../prisma';
 interface Migration {
   file: string;
   name: string;
-  up: (db: PrismaService) => Promise<void>;
-  down: (db: PrismaService) => Promise<void>;
+  up: (db: PrismaService, injector: ModuleRef) => Promise<void>;
+  down: (db: PrismaService, injector: ModuleRef) => Promise<void>;
 }
 
 export async function collectMigrations(): Promise<Migration[]> {
@@ -46,7 +47,10 @@ export async function collectMigrations(): Promise<Migration[]> {
 })
 export class RunCommand extends CommandRunner {
   logger = new Logger(RunCommand.name);
-  constructor(private readonly db: PrismaService) {
+  constructor(
+    private readonly db: PrismaService,
+    private readonly injector: ModuleRef
+  ) {
     super();
   }
 
@@ -103,14 +107,14 @@ export class RunCommand extends CommandRunner {
     });
 
     try {
-      await migration.up(this.db);
+      await migration.up(this.db, this.injector);
     } catch (e) {
       await this.db.dataMigration.delete({
         where: {
           id: record.id,
         },
       });
-      await migration.down(this.db);
+      await migration.down(this.db, this.injector);
       this.logger.error('Failed to run data migration', e);
       process.exit(1);
     }
@@ -134,7 +138,10 @@ export class RunCommand extends CommandRunner {
 export class RevertCommand extends CommandRunner {
   logger = new Logger(RevertCommand.name);
 
-  constructor(private readonly db: PrismaService) {
+  constructor(
+    private readonly db: PrismaService,
+    private readonly injector: ModuleRef
+  ) {
     super();
   }
 
@@ -168,7 +175,7 @@ export class RevertCommand extends CommandRunner {
 
     try {
       this.logger.log(`Reverting ${name}...`);
-      await migration.down(this.db);
+      await migration.down(this.db, this.injector);
       this.logger.log('Done reverting');
     } catch (e) {
       this.logger.error(`Failed to revert data migration ${name}`, e);

packages/backend/server/src/data/migrations/1703828796699-workspace-blobs.ts

@@ -0,0 +1,38 @@
+import { ModuleRef } from '@nestjs/core';
+
+import { WorkspaceBlobStorage } from '../../modules/storage';
+import { PrismaService } from '../../prisma';
+
+export class WorkspaceBlobs1703828796699 {
+  // do the migration
+  static async up(db: PrismaService, injector: ModuleRef) {
+    const blobStorage = injector.get(WorkspaceBlobStorage, { strict: false });
+    let hasMore = true;
+    let turn = 0;
+    const eachTurnCount = 50;
+
+    while (hasMore) {
+      const blobs = await db.blob.findMany({
+        skip: turn * eachTurnCount,
+        take: eachTurnCount,
+        orderBy: {
+          createdAt: 'asc',
+        },
+      });
+
+      hasMore = blobs.length === eachTurnCount;
+      turn += 1;
+
+      await Promise.all(
+        blobs.map(async ({ workspaceId, hash, blob }) =>
+          blobStorage.put(workspaceId, hash, blob)
+        )
+      );
+    }
+  }
+
+  // revert the migration
+  static async down(_db: PrismaService) {
+    // old data kept, no need to downgrade the migration
+  }
+}

packages/backend/server/src/modules/storage/wrappers/blob.ts

@@ -37,8 +37,11 @@ export class WorkspaceBlobStorage {
     return blobs;
   }
 
-  async delete(workspaceId: string, key: string) {
-    return this.provider.delete(`${workspaceId}/${key}`);
+  /**
+   * we won't really delete the blobs until the doc blobs manager is implemented sounded
+   */
+  async delete(_workspaceId: string, _key: string) {
+    // return this.provider.delete(`${workspaceId}/${key}`);
   }
 
   async totalSize(workspaceId: string) {