diff --git a/packages/backend/server/src/affine.config.ts b/packages/backend/server/src/affine.config.ts index ae879a4132..43e9a83fac 100644 --- a/packages/backend/server/src/affine.config.ts +++ b/packages/backend/server/src/affine.config.ts @@ -19,5 +19,7 @@ if (node.prod && env.R2_OBJECT_STORAGE_ACCOUNT_ID) { `https://avatar.affineassets.com/${key}`; AFFiNE.storage.storages.blob.provider = 'r2'; - AFFiNE.storage.storages.blob.bucket = 'workspace-blobs'; + AFFiNE.storage.storages.blob.bucket = `workspace-blobs-${ + AFFiNE.affine.canary ? 'canary' : 'prod' + }`; } diff --git a/packages/backend/server/src/modules/storage/providers/provider.ts b/packages/backend/server/src/modules/storage/providers/provider.ts index 95740d5a62..6e83189184 100644 --- a/packages/backend/server/src/modules/storage/providers/provider.ts +++ b/packages/backend/server/src/modules/storage/providers/provider.ts @@ -9,7 +9,7 @@ export interface GetObjectMetadata { contentType: string; contentLength: number; lastModified: Date; - checksumCRC32: string; + checksumCRC32?: string; } export interface PutObjectMetadata { diff --git a/packages/backend/server/src/modules/storage/providers/s3.ts b/packages/backend/server/src/modules/storage/providers/s3.ts index 241d7272fc..30c1f9ca8e 100644 --- a/packages/backend/server/src/modules/storage/providers/s3.ts +++ b/packages/backend/server/src/modules/storage/providers/s3.ts @@ -30,7 +30,7 @@ export class S3StorageProvider implements StorageProvider { config: S3StorageConfig, public readonly bucket: string ) { - this.client = new S3Client(config); + this.client = new S3Client({ region: 'auto', ...config }); this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`); } @@ -53,7 +53,8 @@ export class S3StorageProvider implements StorageProvider { // metadata ContentType: metadata.contentType, ContentLength: metadata.contentLength, - ChecksumCRC32: metadata.checksumCRC32, + // TODO: Cloudflare doesn't support CRC32, use md5 instead later. + // ChecksumCRC32: metadata.checksumCRC32, }) ); @@ -90,8 +91,8 @@ export class S3StorageProvider implements StorageProvider { // always set when putting object contentType: obj.ContentType!, contentLength: obj.ContentLength!, - checksumCRC32: obj.ChecksumCRC32!, lastModified: obj.LastModified!, + checksumCRC32: obj.ChecksumCRC32, }, }; } catch (e) { diff --git a/packages/backend/server/src/modules/storage/wrappers/blob.ts b/packages/backend/server/src/modules/storage/wrappers/blob.ts index 04028a9478..0eca2d4375 100644 --- a/packages/backend/server/src/modules/storage/wrappers/blob.ts +++ b/packages/backend/server/src/modules/storage/wrappers/blob.ts @@ -1,29 +1,70 @@ -import { Injectable } from '@nestjs/common'; +import { Readable } from 'node:stream'; + +import type { Storage } from '@affine/storage'; +import { Injectable, OnModuleInit } from '@nestjs/common'; import { Config } from '../../../config'; import { EventEmitter, type EventPayload, OnEvent } from '../../../event'; +import { OctoBaseStorageModule } from '../../../storage'; import { BlobInputType, createStorageProvider, StorageProvider, } from '../providers'; +import { toBuffer } from '../providers/utils'; @Injectable() -export class WorkspaceBlobStorage { +export class WorkspaceBlobStorage implements OnModuleInit { public readonly provider: StorageProvider; + + /** + * @deprecated for backwards compatibility, need to be removed in next stable release + */ + private octobase: Storage | null = null; + constructor( private readonly event: EventEmitter, - { storage }: Config + private readonly config: Config ) { - this.provider = createStorageProvider(storage, 'blob'); + this.provider = createStorageProvider(this.config.storage, 'blob'); } - put(workspaceId: string, key: string, blob: BlobInputType) { - return this.provider.put(`${workspaceId}/${key}`, blob); + async onModuleInit() { + if (!this.config.node.test) { + this.octobase = await OctoBaseStorageModule.Storage.connect( + this.config.db.url + ); + } } - get(workspaceId: string, key: string) { - return this.provider.get(`${workspaceId}/${key}`); + async put(workspaceId: string, key: string, blob: BlobInputType) { + const buf = await toBuffer(blob); + await this.provider.put(`${workspaceId}/${key}`, buf); + if (this.octobase) { + await this.octobase.uploadBlob(workspaceId, buf); + } + } + + async get(workspaceId: string, key: string) { + const result = await this.provider.get(`${workspaceId}/${key}`); + if (!result.body && this.octobase) { + const blob = await this.octobase.getBlob(workspaceId, key); + + if (!blob) { + return result; + } + + return { + body: Readable.from(blob.data), + metadata: { + contentType: blob.contentType, + contentLength: blob.size, + lastModified: new Date(blob.lastModified), + }, + }; + } + + return result; } async list(workspaceId: string) { diff --git a/packages/backend/server/src/modules/workspaces/controller.ts b/packages/backend/server/src/modules/workspaces/controller.ts index 02fbaea87f..e38ed68071 100644 --- a/packages/backend/server/src/modules/workspaces/controller.ts +++ b/packages/backend/server/src/modules/workspaces/controller.ts @@ -53,7 +53,6 @@ export class WorkspacesController { res.setHeader('content-type', metadata.contentType); res.setHeader('last-modified', metadata.lastModified.toISOString()); res.setHeader('content-length', metadata.contentLength); - res.setHeader('x-checksum-crc32', metadata.checksumCRC32); } else { this.logger.warn(`Blob ${workspaceId}/${name} has no metadata`); } diff --git a/packages/backend/server/src/storage/index.ts b/packages/backend/server/src/storage/index.ts index 0b8a8e5058..ad121b5ad8 100644 --- a/packages/backend/server/src/storage/index.ts +++ b/packages/backend/server/src/storage/index.ts @@ -1,8 +1,7 @@ -// NODE: -// This file has been deprecated after blob storage moved to cloudflare r2 storage. -// It only exists for backward compatibility. import { createRequire } from 'node:module'; +export const StorageProvide = Symbol('Storage'); + let storageModule: typeof import('@affine/storage'); try { storageModule = await import('@affine/storage'); @@ -14,6 +13,8 @@ try { : require('../../storage.node'); } +export { storageModule as OctoBaseStorageModule }; + export const mergeUpdatesInApplyWay = storageModule.mergeUpdatesInApplyWay; export const verifyChallengeResponse = async (