feat(server): blob data migration (#5461)

.github/actions/deploy/deploy.mjs

@@ -13,7 +13,6 @@ const {
   R2_ACCOUNT_ID,
   R2_ACCESS_KEY_ID,
   R2_SECRET_ACCESS_KEY,
-  R2_BUCKET,
   ENABLE_CAPTCHA,
   CAPTCHA_TURNSTILE_SECRET,
   OAUTH_EMAIL_SENDER,
@@ -96,7 +95,6 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`,
     `--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`,
     `--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`,
-    `--set-string graphql.app.objectStorage.r2.bucket="${R2_BUCKET}"`,
     `--set-string graphql.app.oauth.email.sender="${OAUTH_EMAIL_SENDER}"`,
     `--set-string graphql.app.oauth.email.login="${OAUTH_EMAIL_LOGIN}"`,
     `--set-string graphql.app.oauth.email.password="${OAUTH_EMAIL_PASSWORD}"`,

.github/helm/affine/charts/graphql/templates/deployment.yaml

@@ -136,11 +136,6 @@ spec:
               secretKeyRef:
                 name: "{{ .Values.app.objectStorage.r2.secretName }}"
                 key: secretAccessKey
-          - name: R2_OBJECT_STORAGE_BUCKET
-            valueFrom:
-              secretKeyRef:
-                name: "{{ .Values.app.objectStorage.r2.secretName }}"
-                key: bucket
           {{ end }}
           {{ if .Values.app.captcha.enabled }}
           - name: CAPTCHA_TURNSTILE_SECRET

.github/helm/affine/charts/graphql/templates/migration.yaml

@@ -35,6 +35,23 @@ spec:
           - name: DATABASE_URL
             value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
           {{ end }}
+          {{ if .Values.app.objectStorage.r2.enabled }}
+          - name: R2_OBJECT_STORAGE_ACCOUNT_ID
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                key: accountId
+          - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                key: accessKeyId
+          - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.objectStorage.r2.secretName }}"
+                key: secretAccessKey
+          {{ end }}
         resources:
           requests:
             cpu: '100m'

.github/helm/affine/charts/graphql/templates/r2-secret.yaml

@@ -8,5 +8,4 @@ data:
   accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }}
   accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }}
   secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }}
-  bucket: {{ .Values.app.objectStorage.r2.bucket | b64enc }}
 {{- end }}

.github/helm/affine/charts/graphql/values.yaml

@@ -34,7 +34,6 @@ app:
       accountId: ''
       accessKeyId: ''
       secretAccessKey: ''
-      bucket: ''
   oauth:
     email:
       secretName: 'oauth-email'

.github/workflows/build-test.yml

@@ -315,6 +315,12 @@ jobs:
           electron-install: false
           full-cache: true
 
+      - name: Download storage.node
+        uses: actions/download-artifact@v4
+        with:
+          name: storage.node
+          path: ./packages/backend/server
+
       - name: Initialize database
         run: |
           psql -h localhost -U postgres -c "CREATE DATABASE affine;"
@@ -337,12 +343,6 @@ jobs:
         env:
           DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
 
-      - name: Download storage.node
-        uses: actions/download-artifact@v4
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
       - name: Run server tests
         run: yarn workspace @affine/server test:coverage
         env:
@@ -407,6 +407,18 @@ jobs:
           playwright-install: true
           hard-link-nm: false
 
+      - name: Download storage.node
+        uses: actions/download-artifact@v4
+        with:
+          name: storage.node
+          path: ./packages/backend/server
+
+      - name: Download affine.linux-x64-gnu.node
+        uses: actions/download-artifact@v4
+        with:
+          name: affine.linux-x64-gnu.node
+          path: ./packages/frontend/native
+
       - name: Initialize database
         run: |
           psql -h localhost -U postgres -c "CREATE DATABASE affine;"
@@ -426,17 +438,6 @@ jobs:
         run: |
           yarn workspace @affine/server data-migration run
           yarn workspace @affine/server exec node --loader ts-node/esm/transpile-only ./scripts/init-db.ts
-      - name: Download storage.node
-        uses: actions/download-artifact@v4
-        with:
-          name: storage.node
-          path: ./packages/backend/server
-
-      - name: Download affine.linux-x64-gnu.node
-        uses: actions/download-artifact@v4
-        with:
-          name: affine.linux-x64-gnu.node
-          path: ./packages/frontend/native
 
       - name: ${{ matrix.tests.name }}
         run: |

.github/workflows/deploy.yml

@@ -240,7 +240,6 @@ jobs:
           R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          R2_BUCKET: ${{ secrets.R2_BUCKET }}
           ENABLE_CAPTCHA: true
           CAPTCHA_TURNSTILE_SECRET: ${{ secrets.CAPTCHA_TURNSTILE_SECRET }}
           OAUTH_EMAIL_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}

packages/backend/server/src/data/app.ts

@@ -1,18 +1,32 @@
 import { Logger, Module } from '@nestjs/common';
 import { CommandFactory } from 'nest-commander';
 
-import { PrismaModule } from '../prisma';
+import { AppModule as BusinessAppModule } from '../app';
+import { ConfigModule } from '../config';
 import { CreateCommand, NameQuestion } from './commands/create';
 import { RevertCommand, RunCommand } from './commands/run';
 
 @Module({
-  imports: [PrismaModule],
+  imports: [
+    ConfigModule.forRoot({
+      doc: {
+        manager: {
+          enableUpdateAutoMerging: false,
+        },
+      },
+    }),
+    BusinessAppModule,
+  ],
   providers: [NameQuestion, CreateCommand, RunCommand, RevertCommand],
 })
 class AppModule {}
 
 async function bootstrap() {
-  await CommandFactory.run(AppModule, new Logger());
+  await CommandFactory.run(AppModule, new Logger()).catch(e => {
+    console.error(e);
+    process.exit(1);
+  });
+  process.exit(0);
 }
 
 await bootstrap();

packages/backend/server/src/data/commands/run.ts

@@ -3,6 +3,7 @@ import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { Logger } from '@nestjs/common';
+import { ModuleRef } from '@nestjs/core';
 import { Command, CommandRunner } from 'nest-commander';
 
 import { PrismaService } from '../../prisma';
@@ -10,8 +11,8 @@ import { PrismaService } from '../../prisma';
 interface Migration {
   file: string;
   name: string;
-  up: (db: PrismaService) => Promise<void>;
+  up: (db: PrismaService, injector: ModuleRef) => Promise<void>;
-  down: (db: PrismaService) => Promise<void>;
+  down: (db: PrismaService, injector: ModuleRef) => Promise<void>;
 }
 
 export async function collectMigrations(): Promise<Migration[]> {
@@ -46,7 +47,10 @@ export async function collectMigrations(): Promise<Migration[]> {
 })
 export class RunCommand extends CommandRunner {
   logger = new Logger(RunCommand.name);
-  constructor(private readonly db: PrismaService) {
+  constructor(
+    private readonly db: PrismaService,
+    private readonly injector: ModuleRef
+  ) {
     super();
   }
 
@@ -103,14 +107,14 @@ export class RunCommand extends CommandRunner {
     });
 
     try {
-      await migration.up(this.db);
+      await migration.up(this.db, this.injector);
     } catch (e) {
       await this.db.dataMigration.delete({
         where: {
           id: record.id,
         },
       });
-      await migration.down(this.db);
+      await migration.down(this.db, this.injector);
       this.logger.error('Failed to run data migration', e);
       process.exit(1);
     }
@@ -134,7 +138,10 @@ export class RunCommand extends CommandRunner {
 export class RevertCommand extends CommandRunner {
   logger = new Logger(RevertCommand.name);
 
-  constructor(private readonly db: PrismaService) {
+  constructor(
+    private readonly db: PrismaService,
+    private readonly injector: ModuleRef
+  ) {
     super();
   }
 
@@ -168,7 +175,7 @@ export class RevertCommand extends CommandRunner {
 
     try {
       this.logger.log(`Reverting ${name}...`);
-      await migration.down(this.db);
+      await migration.down(this.db, this.injector);
       this.logger.log('Done reverting');
     } catch (e) {
       this.logger.error(`Failed to revert data migration ${name}`, e);

packages/backend/server/src/data/migrations/1703828796699-workspace-blobs.ts

@@ -0,0 +1,38 @@
+import { ModuleRef } from '@nestjs/core';
+
+import { WorkspaceBlobStorage } from '../../modules/storage';
+import { PrismaService } from '../../prisma';
+
+export class WorkspaceBlobs1703828796699 {
+  // do the migration
+  static async up(db: PrismaService, injector: ModuleRef) {
+    const blobStorage = injector.get(WorkspaceBlobStorage, { strict: false });
+    let hasMore = true;
+    let turn = 0;
+    const eachTurnCount = 50;
+
+    while (hasMore) {
+      const blobs = await db.blob.findMany({
+        skip: turn * eachTurnCount,
+        take: eachTurnCount,
+        orderBy: {
+          createdAt: 'asc',
+        },
+      });
+
+      hasMore = blobs.length === eachTurnCount;
+      turn += 1;
+
+      await Promise.all(
+        blobs.map(async ({ workspaceId, hash, blob }) =>
+          blobStorage.put(workspaceId, hash, blob)
+        )
+      );
+    }
+  }
+
+  // revert the migration
+  static async down(_db: PrismaService) {
+    // old data kept, no need to downgrade the migration
+  }
+}

packages/backend/server/src/modules/storage/wrappers/blob.ts

@@ -37,8 +37,11 @@ export class WorkspaceBlobStorage {
     return blobs;
   }
 
-  async delete(workspaceId: string, key: string) {
+  /**
-    return this.provider.delete(`${workspaceId}/${key}`);
+   * we won't really delete the blobs until the doc blobs manager is implemented sounded
+   */
+  async delete(_workspaceId: string, _key: string) {
+    // return this.provider.delete(`${workspaceId}/${key}`);
   }
 
   async totalSize(workspaceId: string) {