From abcca8b09edb646b15054a543d645647e265479a Mon Sep 17 00:00:00 2001 From: liuyi Date: Tue, 2 Jan 2024 07:01:25 +0000 Subject: [PATCH] refactor(server): object storages (#5405) --- packages/backend/server/src/affine.config.ts | 21 +++++++ packages/backend/server/src/affine.ts | 3 + packages/backend/server/src/config/def.ts | 8 +++ packages/backend/server/src/config/default.ts | 7 +-- packages/backend/server/src/config/index.ts | 1 + .../server/src/config/storage/index.ts | 58 +++++++++++++++++++ packages/backend/server/src/prelude.ts | 6 +- 7 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 packages/backend/server/src/affine.config.ts create mode 100644 packages/backend/server/src/affine.ts create mode 100644 packages/backend/server/src/config/storage/index.ts diff --git a/packages/backend/server/src/affine.config.ts b/packages/backend/server/src/affine.config.ts new file mode 100644 index 0000000000..394d8ed0ee --- /dev/null +++ b/packages/backend/server/src/affine.config.ts @@ -0,0 +1,21 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +// Custom configurations + +const env = process.env; +const node = AFFiNE.node; + +// TODO: may be separate config overring in `affine.[env].config`? +if (node.prod && env.R2_OBJECT_STORAGE_ACCOUNT_ID) { + AFFiNE.storage.providers.r2 = { + accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID, + credentials: { + accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!, + secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!, + }, + }; + AFFiNE.storage.storages.avatar.provider = 'r2'; + AFFiNE.storage.storages.avatar.bucket = 'account-avatar'; + + AFFiNE.storage.storages.blob.provider = 'r2'; + AFFiNE.storage.storages.blob.bucket = 'workspace-blobs'; +} diff --git a/packages/backend/server/src/affine.ts b/packages/backend/server/src/affine.ts new file mode 100644 index 0000000000..ceb37536da --- /dev/null +++ b/packages/backend/server/src/affine.ts @@ -0,0 +1,3 @@ +import { getDefaultAFFiNEConfig } from './config/default'; + +globalThis.AFFiNE = getDefaultAFFiNEConfig(); diff --git a/packages/backend/server/src/config/def.ts b/packages/backend/server/src/config/def.ts index 145e1b3b5d..e09d6db80c 100644 --- a/packages/backend/server/src/config/def.ts +++ b/packages/backend/server/src/config/def.ts @@ -1,6 +1,7 @@ import type { ApolloDriverConfig } from '@nestjs/apollo'; import type { LeafPaths } from '../utils/types'; +import type { AFFiNEStorageConfig } from './storage'; declare global { // eslint-disable-next-line @typescript-eslint/no-namespace @@ -165,11 +166,18 @@ export interface AFFiNEConfig { featureFlags: { earlyAccessPreview: boolean; }; + + /** + * Configuration for Object Storage, which defines how blobs and avatar assets are stored. + */ + storage: AFFiNEStorageConfig; + /** * object storage Config * * all artifacts and logs will be stored on instance disk, * and can not shared between instances if not configured + * @deprecated use `storage` instead */ objectStorage: { /** diff --git a/packages/backend/server/src/config/default.ts b/packages/backend/server/src/config/default.ts index aa2f0ba76f..f0aa7e2926 100644 --- a/packages/backend/server/src/config/default.ts +++ b/packages/backend/server/src/config/default.ts @@ -9,6 +9,7 @@ import parse from 'parse-duration'; import pkg from '../../package.json' assert { type: 'json' }; import type { AFFiNEConfig, ServerFlavor } from './def'; import { applyEnvToConfig } from './env'; +import { getDefaultAFFiNEStorageConfig } from './storage'; export const SERVER_FLAVOR = (process.env.SERVER_FLAVOR ?? 'allinone') as ServerFlavor; @@ -59,11 +60,6 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { AFFINE_SERVER_SUB_PATH: 'path', AFFINE_ENV: 'affineEnv', DATABASE_URL: 'db.url', - ENABLE_R2_OBJECT_STORAGE: ['objectStorage.r2.enabled', 'boolean'], - R2_OBJECT_STORAGE_ACCOUNT_ID: 'objectStorage.r2.accountId', - R2_OBJECT_STORAGE_ACCESS_KEY_ID: 'objectStorage.r2.accessKeyId', - R2_OBJECT_STORAGE_SECRET_ACCESS_KEY: 'objectStorage.r2.secretAccessKey', - R2_OBJECT_STORAGE_BUCKET: 'objectStorage.r2.bucket', ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'], CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'], OAUTH_GOOGLE_ENABLED: ['auth.oauthProviders.google.enabled', 'boolean'], @@ -180,6 +176,7 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => { password: '', }, }, + storage: getDefaultAFFiNEStorageConfig(), objectStorage: { r2: { enabled: false, diff --git a/packages/backend/server/src/config/index.ts b/packages/backend/server/src/config/index.ts index cdd76a91bb..722dee3e93 100644 --- a/packages/backend/server/src/config/index.ts +++ b/packages/backend/server/src/config/index.ts @@ -74,3 +74,4 @@ export class ConfigModule { export type { AFFiNEConfig } from './def'; export { SERVER_FLAVOR } from './default'; +export * from './storage'; diff --git a/packages/backend/server/src/config/storage/index.ts b/packages/backend/server/src/config/storage/index.ts new file mode 100644 index 0000000000..6d3e9c5ab2 --- /dev/null +++ b/packages/backend/server/src/config/storage/index.ts @@ -0,0 +1,58 @@ +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +import { S3ClientConfigType } from '@aws-sdk/client-s3'; + +export type StorageProviderType = 'fs' | 'r2' | 's3'; +export interface FsStorageConfig { + path: string; +} +export type R2StorageConfig = S3ClientConfigType & { + accountId: string; +}; +export type S3StorageConfig = S3ClientConfigType; + +export type StorageTargetConfig = { + provider: StorageProviderType; + bucket: string; +}; + +export interface AFFiNEStorageConfig { + /** + * All providers for object storage + * + * Support different providers for different usage at the same time. + */ + providers: { + fs?: FsStorageConfig; + s3?: S3StorageConfig; + r2?: R2StorageConfig; + }; + storages: { + avatar: StorageTargetConfig; + blob: StorageTargetConfig; + }; +} + +export type StorageProviders = AFFiNEStorageConfig['providers']; +export type Storages = keyof AFFiNEStorageConfig['storages']; + +export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig { + return { + providers: { + fs: { + path: join(homedir(), '.affine/storage'), + }, + }, + storages: { + avatar: { + provider: 'fs', + bucket: 'avatars', + }, + blob: { + provider: 'fs', + bucket: 'blobs', + }, + }, + }; +} diff --git a/packages/backend/server/src/prelude.ts b/packages/backend/server/src/prelude.ts index 0c0fae35fc..bac73ad323 100644 --- a/packages/backend/server/src/prelude.ts +++ b/packages/backend/server/src/prelude.ts @@ -1,9 +1,7 @@ import 'reflect-metadata'; import 'dotenv/config'; - -import { getDefaultAFFiNEConfig } from './config/default'; - -globalThis.AFFiNE = getDefaultAFFiNEConfig(); +import './affine'; +import './affine.config'; if (process.env.NODE_ENV === 'development') { console.log('AFFiNE Config:', globalThis.AFFiNE);