refactor(server): config system (#11081)

packages/backend/server/src/base/config/__tests__/config.spec.ts

@@ -0,0 +1,90 @@
+import test from 'ava';
+
+import { createModule } from '../../../__tests__/create-module';
+import { ConfigFactory, ConfigModule } from '..';
+import { Config } from '../config';
+
+const module = await createModule();
+test.after.always(async () => {
+  await module.close();
+});
+
+test('should create config', t => {
+  const config = module.get(Config);
+
+  t.is(typeof config.auth.passwordRequirements.max, 'number');
+  t.is(typeof config.job.queue, 'object');
+});
+
+test('should override config', async t => {
+  await using module = await createModule({
+    imports: [
+      ConfigModule.override({
+        auth: {
+          passwordRequirements: {
+            max: 100,
+            min: 6,
+          },
+        },
+        job: {
+          queues: {
+            notification: {
+              concurrency: 1000,
+            },
+          },
+        },
+      }),
+    ],
+  });
+
+  const config = module.get(Config);
+  const configFactory = module.get(ConfigFactory);
+
+  t.deepEqual(config.auth.passwordRequirements, {
+    max: 100,
+    min: 6,
+  });
+
+  configFactory.override({
+    auth: {
+      passwordRequirements: {
+        max: 10,
+      },
+    },
+  });
+
+  t.deepEqual(config.auth.passwordRequirements, {
+    max: 10,
+    min: 6,
+  });
+});
+
+test('should validate config', t => {
+  const config = module.get(ConfigFactory);
+
+  t.notThrows(() =>
+    config.validate([
+      {
+        module: 'auth',
+        key: 'passwordRequirements',
+        value: { max: 10, min: 6 },
+      },
+    ])
+  );
+
+  t.throws(
+    () =>
+      config.validate([
+        {
+          module: 'auth',
+          key: 'passwordRequirements',
+          value: { max: 10, min: 10 },
+        },
+      ]),
+    {
+      message: `Invalid config for module [auth] with key [passwordRequirements]
+Value: {"max":10,"min":10}
+Error: Minimum length of password must be less than maximum length`,
+    }
+  );
+});

packages/backend/server/src/base/config/config.ts

@@ -0,0 +1,3 @@
+import { ApplyType } from '../utils';
+
+export class Config extends ApplyType<AppConfig>() {}

packages/backend/server/src/base/config/def.ts

@@ -1,55 +0,0 @@
-import type { LeafPaths } from '../utils/types';
-import { AppStartupConfig } from './types';
-
-export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean';
-export type ServerFlavor =
-  | 'allinone'
-  | 'graphql'
-  | 'sync'
-  | 'renderer'
-  | 'doc'
-  | 'script';
-
-export type AFFINE_ENV = 'dev' | 'beta' | 'production';
-export type NODE_ENV = 'development' | 'test' | 'production';
-
-export enum DeploymentType {
-  Affine = 'affine',
-  Selfhosted = 'selfhosted',
-}
-
-export type ConfigPaths = LeafPaths<AppStartupConfig, '', '......'>;
-
-export interface PreDefinedAFFiNEConfig {
-  ENV_MAP: Record<string, ConfigPaths | [ConfigPaths, EnvConfigType?]>;
-  serverId: string;
-  serverName: string;
-  readonly projectRoot: string;
-  readonly AFFINE_ENV: AFFINE_ENV;
-  readonly NODE_ENV: NODE_ENV;
-  readonly version: string;
-  readonly type: DeploymentType;
-  readonly isSelfhosted: boolean;
-  readonly flavor: { type: string } & { [key in ServerFlavor]: boolean };
-  readonly affine: { canary: boolean; beta: boolean; stable: boolean };
-  readonly node: {
-    prod: boolean;
-    dev: boolean;
-    test: boolean;
-  };
-  readonly deploy: boolean;
-}
-
-export interface AppPluginsConfig {}
-
-export type AFFiNEConfig = PreDefinedAFFiNEConfig &
-  AppStartupConfig &
-  AppPluginsConfig;
-
-declare global {
-  // oxlint-disable-next-line @typescript-eslint/no-namespace
-  namespace globalThis {
-    // oxlint-disable-next-line no-var
-    var AFFiNE: AFFiNEConfig;
-  }
-}

packages/backend/server/src/base/config/default.ts

@@ -1,142 +0,0 @@
-import { resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import pkg from '../../../package.json' with { type: 'json' };
-import {
-  AFFINE_ENV,
-  AFFiNEConfig,
-  DeploymentType,
-  NODE_ENV,
-  PreDefinedAFFiNEConfig,
-  ServerFlavor,
-} from './def';
-import { readEnv } from './env';
-import { defaultStartupConfig } from './register';
-
-function expectFlavor(flavor: ServerFlavor, expected: ServerFlavor) {
-  return flavor === expected || flavor === 'allinone';
-}
-
-function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
-  const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
-    'development',
-    'test',
-    'production',
-  ]);
-  const AFFINE_ENV = readEnv<AFFINE_ENV>('AFFINE_ENV', 'production', [
-    'dev',
-    'beta',
-    'production',
-  ]);
-  const flavor = readEnv<ServerFlavor>('SERVER_FLAVOR', 'allinone', [
-    'allinone',
-    'graphql',
-    'sync',
-    'renderer',
-    'doc',
-    'script',
-  ]);
-  const deploymentType = readEnv<DeploymentType>(
-    'DEPLOYMENT_TYPE',
-    NODE_ENV === 'development'
-      ? DeploymentType.Affine
-      : DeploymentType.Selfhosted,
-    Object.values(DeploymentType)
-  );
-  const isSelfhosted = deploymentType === DeploymentType.Selfhosted;
-  const affine = {
-    canary: AFFINE_ENV === 'dev',
-    beta: AFFINE_ENV === 'beta',
-    stable: AFFINE_ENV === 'production',
-  };
-  const node = {
-    prod: NODE_ENV === 'production',
-    dev: NODE_ENV === 'development',
-    test: NODE_ENV === 'test',
-  };
-
-  return {
-    ENV_MAP: {},
-    NODE_ENV,
-    AFFINE_ENV,
-    serverId: 'some-randome-uuid',
-    serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud',
-    version: pkg.version,
-    type: deploymentType,
-    isSelfhosted,
-    flavor: {
-      type: flavor,
-      allinone: flavor === 'allinone',
-      graphql: expectFlavor(flavor, 'graphql'),
-      sync: expectFlavor(flavor, 'sync'),
-      renderer: expectFlavor(flavor, 'renderer'),
-      doc: expectFlavor(flavor, 'doc'),
-      script: expectFlavor(flavor, 'script'),
-    },
-    affine,
-    node,
-    deploy: !node.dev && !node.test,
-    projectRoot: resolve(fileURLToPath(import.meta.url), '../../../../'),
-  };
-}
-
-export function getAFFiNEConfigModifier(): AFFiNEConfig {
-  const predefined = getPredefinedAFFiNEConfig() as AFFiNEConfig;
-
-  return chainableProxy(predefined);
-}
-
-function merge(a: any, b: any) {
-  if (typeof b !== 'object' || b instanceof Map || b instanceof Set) {
-    return b;
-  }
-
-  if (Array.isArray(b)) {
-    if (Array.isArray(a)) {
-      return a.concat(b);
-    }
-    return b;
-  }
-
-  const result = { ...a };
-  Object.keys(b).forEach(key => {
-    result[key] = merge(result[key], b[key]);
-  });
-
-  return result;
-}
-
-export function mergeConfigOverride(override: any) {
-  return merge(defaultStartupConfig, override);
-}
-
-function chainableProxy(obj: any) {
-  const keys: Set<string> = new Set(Object.keys(obj));
-  return new Proxy(obj, {
-    get(target, prop) {
-      if (!(prop in target)) {
-        keys.add(prop as string);
-        target[prop] = chainableProxy({});
-      }
-      return target[prop];
-    },
-    set(target, prop, value) {
-      keys.add(prop as string);
-      if (
-        typeof value === 'object' &&
-        !(
-          value instanceof Map ||
-          value instanceof Set ||
-          value instanceof Array
-        )
-      ) {
-        value = chainableProxy(value);
-      }
-      target[prop] = value;
-      return true;
-    },
-    ownKeys() {
-      return Array.from(keys);
-    },
-  });
-}

packages/backend/server/src/base/config/env.ts

@@ -1,11 +1,9 @@
-import { set } from 'lodash-es';
-
-import type { AFFiNEConfig, EnvConfigType } from './def';
+export type EnvConfigType = 'string' | 'integer' | 'float' | 'boolean';
 
 /**
  * parse number value from environment variables
  */
-function int(value: string) {
+function integer(value: string) {
   const n = parseInt(value);
   return Number.isNaN(n) ? undefined : n;
 }
@@ -20,7 +18,7 @@ function boolean(value: string) {
 }
 
 const envParsers: Record<EnvConfigType, (value: string) => unknown> = {
-  int,
+  integer,
   float,
   boolean,
   string: value => value,
@@ -33,38 +31,3 @@ export function parseEnvValue(value: string | undefined, type: EnvConfigType) {
 
   return envParsers[type](value);
 }
-
-export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
-  for (const env in rawConfig.ENV_MAP) {
-    const config = rawConfig.ENV_MAP[env];
-    const [path, value] =
-      typeof config === 'string'
-        ? [config, parseEnvValue(process.env[env], 'string')]
-        : [config[0], parseEnvValue(process.env[env], config[1] ?? 'string')];
-
-    if (value !== undefined) {
-      set(rawConfig, path, value);
-    }
-  }
-}
-
-export function readEnv<T>(
-  env: string,
-  defaultValue: T,
-  availableValues?: T[]
-) {
-  const value = process.env[env];
-  if (value === undefined) {
-    return defaultValue;
-  }
-
-  if (availableValues && !availableValues.includes(value as any)) {
-    throw new Error(
-      `Invalid value '${value}' for environment variable ${env}, expected one of [${availableValues.join(
-        ', '
-      )}]`
-    );
-  }
-
-  return value as T;
-}

packages/backend/server/src/base/config/factory.ts

@@ -0,0 +1,60 @@
+import { Inject, Injectable, Optional } from '@nestjs/common';
+import { merge } from 'lodash-es';
+
+import { InvalidAppConfig } from '../error';
+import { APP_CONFIG_DESCRIPTORS, getDefaultConfig } from './register';
+
+export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN');
+
+@Injectable()
+export class ConfigFactory {
+  readonly #config: DeepReadonly<AppConfig>;
+
+  constructor(
+    @Inject(OVERRIDE_CONFIG_TOKEN)
+    @Optional()
+    private readonly overrides: DeepPartial<AppConfig> = {}
+  ) {
+    this.#config = this.loadDefault();
+  }
+
+  get config() {
+    return this.#config;
+  }
+
+  override(updates: DeepPartial<AppConfig>) {
+    merge(this.#config, updates);
+  }
+
+  validate(updates: Array<{ module: string; key: string; value: any }>) {
+    const errors: string[] = [];
+
+    updates.forEach(update => {
+      const descriptor = APP_CONFIG_DESCRIPTORS[update.module]?.[update.key];
+      if (!descriptor) {
+        errors.push(
+          `Invalid config for module [${update.module}] with unknown key [${update.key}]`
+        );
+        return;
+      }
+
+      const { success, error } = descriptor.validate(update.value);
+      if (!success) {
+        error.issues.forEach(issue => {
+          errors.push(`Invalid config for module [${update.module}] with key [${update.key}]
+Value: ${JSON.stringify(update.value)}
+Error: ${issue.message}`);
+        });
+      }
+    });
+
+    if (errors.length > 0) {
+      throw new InvalidAppConfig(errors.join('\n'));
+    }
+  }
+
+  private loadDefault(): DeepReadonly<AppConfig> {
+    const config = getDefaultConfig();
+    return merge(config, this.overrides);
+  }
+}

packages/backend/server/src/base/config/index.ts

@@ -1,37 +1,29 @@
-import { DynamicModule, FactoryProvider } from '@nestjs/common';
-import { merge } from 'lodash-es';
+import { DynamicModule, Global, Module, Provider } from '@nestjs/common';
 
-import { AFFiNEConfig } from './def';
-import { Config } from './provider';
-
-export * from './def';
-export * from './default';
-export { applyEnvToConfig, parseEnvValue } from './env';
-export * from './provider';
-export { defineRuntimeConfig, defineStartupConfig } from './register';
-export type { AppConfig, ConfigItem, ModuleConfig } from './types';
-
-function createConfigProvider(
-  override?: DeepPartial<Config>
-): FactoryProvider<Config> {
-  return {
-    provide: Config,
-    useFactory: () => {
-      return Object.freeze(merge({}, globalThis.AFFiNE, override));
-    },
-    inject: [],
-  };
-}
+import { Config } from './config';
+import { ConfigFactory, OVERRIDE_CONFIG_TOKEN } from './factory';
+import { ConfigProvider } from './provider';
 
+@Global()
+@Module({
+  providers: [ConfigProvider, ConfigFactory],
+  exports: [ConfigProvider, ConfigFactory],
+})
 export class ConfigModule {
-  static forRoot = (override?: DeepPartial<AFFiNEConfig>): DynamicModule => {
-    const provider = createConfigProvider(override);
+  static override(overrides: DeepPartial<AppConfigSchema> = {}): DynamicModule {
+    const provider: Provider = {
+      provide: OVERRIDE_CONFIG_TOKEN,
+      useValue: overrides,
+    };
 
     return {
       global: true,
-      module: ConfigModule,
+      module: class ConfigOverrideModule {},
       providers: [provider],
       exports: [provider],
     };
-  };
+  }
 }
+
+export { Config, ConfigFactory };
+export { defineModuleConfig, type JSONSchema } from './register';

packages/backend/server/src/base/config/provider.ts

@@ -1,16 +1,12 @@
-import { ApplyType } from '../utils/types';
-import { AFFiNEConfig } from './def';
+import { FactoryProvider } from '@nestjs/common';
 
-/**
- * @example
- *
- * import { Config } from '@affine/server'
- *
- * class TestConfig {
- *   constructor(private readonly config: Config) {}
- *   test() {
- *     return this.config.env
- *   }
- * }
- */
-export class Config extends ApplyType<AFFiNEConfig>() {}
+import { Config } from './config';
+import { ConfigFactory } from './factory';
+
+export const ConfigProvider: FactoryProvider = {
+  provide: Config,
+  useFactory: (factory: ConfigFactory) => {
+    return factory.config;
+  },
+  inject: [ConfigFactory],
+};

packages/backend/server/src/base/config/register.ts

@@ -1,66 +1,239 @@
-import { Prisma, RuntimeConfigType } from '@prisma/client';
-import { get, merge, set } from 'lodash-es';
+import { once, set } from 'lodash-es';
+import { z } from 'zod';
 
-import {
-  AppModulesConfigDef,
-  AppStartupConfig,
-  ModuleRuntimeConfigDescriptions,
-  ModuleStartupConfigDescriptions,
-} from './types';
+import { type EnvConfigType, parseEnvValue } from './env';
+import { AppConfigByPath } from './types';
 
-export const defaultStartupConfig: AppStartupConfig = {} as any;
-export const defaultRuntimeConfig: Record<
-  string,
-  Prisma.RuntimeConfigCreateInput
-> = {} as any;
+export type JSONSchema = { description?: string } & (
+  | { type?: undefined; oneOf?: JSONSchema[] }
+  | {
+      type: 'string' | 'number' | 'boolean';
+      enum?: string[];
+    }
+  | {
+      type: 'array';
+      items?: JSONSchema;
+    }
+  | {
+      type: 'object';
+      properties?: Record<string, JSONSchema>;
+    }
+);
 
-export function runtimeConfigType(val: any): RuntimeConfigType {
-  if (Array.isArray(val)) {
-    return RuntimeConfigType.Array;
-  }
+type ConfigType = EnvConfigType | 'array' | 'object' | 'any';
+export type ConfigDescriptor<T> = {
+  desc: string;
+  type: ConfigType;
+  validate: (value: T) => z.SafeParseReturnType<T, T>;
+  schema: JSONSchema;
+  default: T;
+  env?: [string, EnvConfigType];
+  link?: string;
+};
 
-  switch (typeof val) {
-    case 'string':
-      return RuntimeConfigType.String;
-    case 'number':
-      return RuntimeConfigType.Number;
-    case 'boolean':
-      return RuntimeConfigType.Boolean;
+type ConfigDefineDescriptor<T> = {
+  desc: string;
+  default: T;
+  validate?: (value: T) => boolean;
+  shape?: z.ZodType<T>;
+  env?: string | [string, EnvConfigType];
+  link?: string;
+  schema?: JSONSchema;
+};
+
+function typeFromShape(shape: z.ZodType<any>): ConfigType {
+  switch (shape.constructor) {
+    case z.ZodString:
+      return 'string';
+    case z.ZodNumber:
+      return 'float';
+    case z.ZodBoolean:
+      return 'boolean';
+    case z.ZodArray:
+      return 'array';
+    case z.ZodObject:
+      return 'object';
     default:
-      return RuntimeConfigType.Object;
+      return 'any';
   }
 }
 
-function registerRuntimeConfig<T extends keyof AppModulesConfigDef>(
-  module: T,
-  configs: ModuleRuntimeConfigDescriptions<T>
-) {
-  Object.entries(configs).forEach(([key, value]) => {
-    defaultRuntimeConfig[`${module}/${key}`] = {
-      id: `${module}/${key}`,
+function shapeFromType(type: ConfigType): z.ZodType<any> {
+  switch (type) {
+    case 'string':
+      return z.string();
+    case 'float':
+      return z.number();
+    case 'boolean':
+      return z.boolean();
+    case 'integer':
+      return z.number().int();
+    case 'array':
+      return z.array(z.any());
+    case 'object':
+      return z.object({});
+    default:
+      return z.any();
+  }
+}
+
+function typeFromSchema(schema: JSONSchema): ConfigType {
+  if ('type' in schema) {
+    switch (schema.type) {
+      case 'string':
+        return 'string';
+      case 'number':
+        return 'float';
+      case 'boolean':
+        return 'boolean';
+      case 'array':
+        return 'array';
+      case 'object':
+        return 'object';
+    }
+  }
+
+  return 'any';
+}
+
+function schemaFromType(type: ConfigType): JSONSchema['type'] {
+  switch (type) {
+    case 'any':
+      return undefined;
+    case 'float':
+    case 'integer':
+      return 'number';
+    default:
+      return type;
+  }
+}
+
+function typeFromDefault<T>(defaultValue: T): ConfigType {
+  if (Array.isArray(defaultValue)) {
+    return 'array';
+  }
+
+  switch (typeof defaultValue) {
+    case 'string':
+      return 'string';
+    case 'number':
+      return 'float';
+    case 'boolean':
+      return 'boolean';
+    case 'object':
+      return 'object';
+    default:
+      return 'any';
+  }
+}
+
+function standardizeDescriptor<T>(
+  desc: ConfigDefineDescriptor<T>
+): ConfigDescriptor<T> {
+  const env = desc.env
+    ? Array.isArray(desc.env)
+      ? desc.env
+      : ([desc.env, 'string'] as [string, EnvConfigType])
+    : undefined;
+
+  let type: ConfigType = 'any';
+
+  if (desc.default !== undefined && desc.default !== null) {
+    type = typeFromDefault(desc.default);
+  } else if (env) {
+    type = env[1];
+  } else if (desc.shape) {
+    type = typeFromShape(desc.shape);
+  } else if (desc.schema) {
+    type = typeFromSchema(desc.schema);
+  }
+
+  const shape = desc.shape ?? shapeFromType(type);
+
+  return {
+    desc: desc.desc,
+    default: desc.default,
+    type,
+    validate: (value: T) => {
+      return shape.safeParse(value);
+    },
+    env,
+    link: desc.link,
+    schema: {
+      type: schemaFromType(type),
+      description: desc.desc,
+      ...desc.schema,
+    },
+  };
+}
+
+type ModuleConfigDescriptors<T> = {
+  [K in keyof T]: ConfigDefineDescriptor<T[K]>;
+};
+
+export const APP_CONFIG_DESCRIPTORS: Record<
+  string,
+  Record<string, ConfigDescriptor<any>>
+> = {};
+
+export const getDescriptors = once(() => {
+  return Object.entries(APP_CONFIG_DESCRIPTORS).map(
+    ([module, descriptors]) => ({
       module,
-      key,
-      description: value.desc,
-      value: value.default,
-      type: runtimeConfigType(value.default),
-    };
-  });
-}
-
-export function defineStartupConfig<T extends keyof AppModulesConfigDef>(
-  module: T,
-  configs: ModuleStartupConfigDescriptions<AppModulesConfigDef[T]>
-) {
-  set(
-    defaultStartupConfig,
-    module,
-    merge(get(defaultStartupConfig, module, {}), configs)
+      descriptors: Object.entries(descriptors).map(([key, descriptor]) => ({
+        key,
+        descriptor,
+      })),
+    })
   );
+});
+
+export function defineModuleConfig<T extends keyof AppConfigSchema>(
+  module: T,
+  defs: ModuleConfigDescriptors<AppConfigByPath<T>>
+) {
+  const descriptors: Record<string, ConfigDescriptor<any>> = {};
+  Object.entries(defs).forEach(([key, desc]) => {
+    descriptors[key] = standardizeDescriptor(
+      desc as ConfigDefineDescriptor<any>
+    );
+  });
+
+  APP_CONFIG_DESCRIPTORS[module] = {
+    ...APP_CONFIG_DESCRIPTORS[module],
+    ...descriptors,
+  };
 }
 
-export function defineRuntimeConfig<T extends keyof AppModulesConfigDef>(
-  module: T,
-  configs: ModuleRuntimeConfigDescriptions<T>
-) {
-  registerRuntimeConfig(module, configs);
+export function getDefaultConfig(): AppConfigSchema {
+  const config: Record<string, any> = {};
+  const envs = process.env;
+
+  for (const [module, defs] of Object.entries(APP_CONFIG_DESCRIPTORS)) {
+    const modulizedConfig = {};
+
+    for (const [key, desc] of Object.entries(defs)) {
+      let defaultValue = desc.default;
+
+      if (desc.env) {
+        const [env, parser] = desc.env;
+        const envValue = envs[env];
+        if (envValue) {
+          defaultValue = parseEnvValue(envValue, parser);
+        }
+      }
+
+      const { success, error } = desc.validate(defaultValue);
+
+      if (!success) {
+        throw error;
+      }
+
+      set(modulizedConfig, key, defaultValue);
+    }
+
+    config[module] = modulizedConfig;
+  }
+
+  return config as AppConfigSchema;
 }

packages/backend/server/src/base/config/types.ts

@@ -1,127 +1,20 @@
-import { Join, PathType } from '../utils/types';
+import { LeafPaths, PathType } from '../utils';
 
-export type ConfigItem<T> = T & { __type: 'ConfigItem' };
-
-type ConfigDef = Record<string, any> | never;
-
-export interface ModuleConfig<
-  Startup extends ConfigDef = never,
-  Runtime extends ConfigDef = never,
-> {
-  startup: Startup;
-  runtime: Runtime;
+declare global {
+  type ConfigItem<T> = Leaf<T>;
+  interface AppConfigSchema {}
+  type AppConfig = DeeplyEraseLeaf<AppConfigSchema>;
 }
 
-export type RuntimeConfigDescription<T> = {
-  desc: string;
-  default: T;
-};
-
-type ConfigItemLeaves<T, P extends string = ''> =
-  T extends Record<string, any>
+export type AppConfigByPath<Module extends keyof AppConfigSchema> =
+  AppConfigSchema[Module] extends infer Config
     ? {
-        [K in keyof T]: K extends string
-          ? T[K] extends { __type: 'ConfigItem' }
-            ? K
-            : T[K] extends PrimitiveType
-              ? K
-              : Join<K, ConfigItemLeaves<T[K], P>>
-          : never;
-      }[keyof T]
-    : never;
-
-type StartupConfigDescriptions<T extends ConfigDef> = {
-  [K in keyof T]: T[K] extends Record<string, any>
-    ? T[K] extends ConfigItem<infer V>
-      ? V
-      : T[K]
-    : T[K];
-};
-
-type ModuleConfigLeaves<T, P extends string = ''> =
-  T extends Record<string, any>
-    ? {
-        [K in keyof T]: K extends string
-          ? T[K] extends ModuleConfig<any, any>
-            ? K
-            : Join<K, ModuleConfigLeaves<T[K], P>>
-          : never;
-      }[keyof T]
-    : never;
-
-type FlattenModuleConfigs<T extends Record<string, any>> = {
-  // @ts-expect-error allow
-  [K in ModuleConfigLeaves<T>]: PathType<T, K>;
-};
-
-type _AppStartupConfig<T extends Record<string, any>> = {
-  [K in keyof T]: T[K] extends ModuleConfig<infer S, any>
-    ? S
-    : _AppStartupConfig<T[K]>;
-};
-
-// for extending
-export interface AppConfig {}
-export type AppModulesConfigDef = FlattenModuleConfigs<AppConfig>;
-export type AppConfigModules = keyof AppModulesConfigDef;
-export type AppStartupConfig = _AppStartupConfig<AppConfig>;
-
-// app runtime config keyed by module names
-export type AppRuntimeConfigByModules = {
-  [Module in keyof AppModulesConfigDef]: AppModulesConfigDef[Module] extends ModuleConfig<
-    any,
-    infer Runtime
-  >
-    ? Runtime extends never
-      ? never
-      : {
-          // @ts-expect-error allow
-          [K in ConfigItemLeaves<Runtime>]: PathType<
-            Runtime,
-            K
-          > extends infer Config
-            ? Config extends ConfigItem<infer V>
+        [Path in LeafPaths<Config>]: Path extends string
+          ? PathType<Config, Path> extends infer Item
+            ? Item extends Leaf<infer V>
               ? V
-              : Config
-            : never;
-        }
+              : Item
+            : never
+          : never;
+      }
     : never;
-};
-
-// names of modules that have runtime config
-export type AppRuntimeConfigModules = {
-  [Module in keyof AppRuntimeConfigByModules]: AppRuntimeConfigByModules[Module] extends never
-    ? never
-    : Module;
-}[keyof AppRuntimeConfigByModules];
-
-// runtime config keyed by module names flattened into config names
-// { auth: { allowSignup: boolean } } => { 'auth/allowSignup': boolean }
-export type FlattenedAppRuntimeConfig = UnionToIntersection<
-  {
-    [Module in keyof AppRuntimeConfigByModules]: AppModulesConfigDef[Module] extends never
-      ? never
-      : {
-          [K in keyof AppRuntimeConfigByModules[Module] as K extends string
-            ? `${Module}/${K}`
-            : never]: AppRuntimeConfigByModules[Module][K];
-        };
-  }[keyof AppRuntimeConfigByModules]
->;
-
-export type ModuleStartupConfigDescriptions<T extends ModuleConfig<any, any>> =
-  T extends ModuleConfig<infer S, any>
-    ? S extends never
-      ? undefined
-      : StartupConfigDescriptions<S>
-    : never;
-
-export type ModuleRuntimeConfigDescriptions<
-  Module extends keyof AppRuntimeConfigByModules,
-> = AppModulesConfigDef[Module] extends never
-  ? never
-  : {
-      [K in keyof AppRuntimeConfigByModules[Module]]: RuntimeConfigDescription<
-        AppRuntimeConfigByModules[Module][K]
-      >;
-    };

packages/backend/server/src/base/error/def.ts

@@ -815,4 +815,10 @@ export const USER_FRIENDLY_ERRORS = {
     type: 'action_forbidden',
     message: 'You can not mention yourself.',
   },
+
+  // app config
+  invalid_app_config: {
+    type: 'invalid_input',
+    message: 'Invalid app config.',
+  },
 } satisfies Record<string, UserFriendlyErrorOptions>;

packages/backend/server/src/base/error/errors.gen.ts

@@ -917,6 +917,12 @@ export class MentionUserOneselfDenied extends UserFriendlyError {
     super('action_forbidden', 'mention_user_oneself_denied', message);
   }
 }
+
+export class InvalidAppConfig extends UserFriendlyError {
+  constructor(message?: string) {
+    super('invalid_input', 'invalid_app_config', message);
+  }
+}
 export enum ErrorNames {
   INTERNAL_SERVER_ERROR,
   NETWORK_ERROR,
@@ -1034,7 +1040,8 @@ export enum ErrorNames {
   UNSUPPORTED_CLIENT_VERSION,
   NOTIFICATION_NOT_FOUND,
   MENTION_USER_DOC_ACCESS_DENIED,
-  MENTION_USER_ONESELF_DENIED
+  MENTION_USER_ONESELF_DENIED,
+  INVALID_APP_CONFIG
 }
 registerEnumType(ErrorNames, {
   name: 'ErrorNames'

packages/backend/server/src/base/error/index.ts

@@ -5,7 +5,6 @@ import { fileURLToPath } from 'node:url';
 import { Logger, Module, OnModuleInit } from '@nestjs/common';
 import { Args, Query, Resolver } from '@nestjs/graphql';
 
-import { Config } from '../config/provider';
 import { generateUserFriendlyErrors } from './def';
 import { ActionForbidden, ErrorDataUnionType, ErrorNames } from './errors.gen';
 
@@ -23,9 +22,8 @@ class ErrorResolver {
 })
 export class ErrorModule implements OnModuleInit {
   logger = new Logger('ErrorModule');
-  constructor(private readonly config: Config) {}
   onModuleInit() {
-    if (!this.config.node.dev) {
+    if (!env.dev) {
       return;
     }
     this.logger.log('Generating UserFriendlyError classes');

packages/backend/server/src/base/event/def.ts

@@ -1,6 +1,6 @@
 import { OnOptions } from 'eventemitter2';
 
-import { PushMetadata, sliceMetadata } from '../nestjs';
+import { PushMetadata, sliceMetadata } from '../nestjs/decorator';
 
 declare global {
   /**

packages/backend/server/src/base/event/index.ts

@@ -6,7 +6,10 @@ import { EventHandlerScanner } from './scanner';
 
 const EmitProvider = {
   provide: EventEmitter2,
-  useFactory: () => new EventEmitter2(),
+  useFactory: () =>
+    new EventEmitter2({
+      maxListeners: 100,
+    }),
 };
 
 @Global()

packages/backend/server/src/base/event/scanner.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@nestjs/common';
 import { once } from 'lodash-es';
 
-import { ModuleScanner } from '../nestjs';
+import { ModuleScanner } from '../nestjs/scanner';
 import {
   type EventName,
   type EventOptions,

packages/backend/server/src/base/graphql/config.ts

@@ -1,17 +1,27 @@
 import { ApolloDriverConfig } from '@nestjs/apollo';
 
-import { defineStartupConfig, ModuleConfig } from '../../base/config';
+import { defineModuleConfig } from '../config';
 
-declare module '../../base/config' {
-  interface AppConfig {
-    graphql: ModuleConfig<ApolloDriverConfig>;
+declare global {
+  interface AppConfigSchema {
+    graphql: {
+      apolloDriverConfig: ConfigItem<ApolloDriverConfig>;
+    };
   }
 }
 
-defineStartupConfig('graphql', {
-  buildSchemaOptions: {
-    numberScalarMode: 'integer',
+defineModuleConfig('graphql', {
+  apolloDriverConfig: {
+    desc: 'The config for underlying nestjs GraphQL and apollo driver engine.',
+    default: {
+      buildSchemaOptions: {
+        numberScalarMode: 'integer',
+      },
+      useGlobalPrefix: true,
+      playground: true,
+      introspection: true,
+      sortSchema: true,
+    },
+    link: 'https://docs.nestjs.com/graphql/quick-start',
   },
-  introspection: true,
-  playground: true,
 });

packages/backend/server/src/base/graphql/index.ts

@@ -1,13 +1,12 @@
 import './config';
 
 import { join } from 'node:path';
-import { fileURLToPath } from 'node:url';
 
 import type { ApolloDriverConfig } from '@nestjs/apollo';
 import { ApolloDriver } from '@nestjs/apollo';
 import { Global, Module } from '@nestjs/common';
 import { GraphQLModule } from '@nestjs/graphql';
-import { Request, Response } from 'express';
+import type { Request, Response } from 'express';
 
 import { Config } from '../config';
 import { mapAnyError } from '../nestjs/exception';
@@ -26,18 +25,17 @@ export type GraphqlContext = {
       driver: ApolloDriver,
       useFactory: (config: Config) => {
         return {
-          ...config.graphql,
-          path: `${config.server.path}/graphql`,
+          ...config.graphql.apolloDriverConfig,
+          autoSchemaFile: join(
+            env.projectRoot,
+            env.testing
+              ? './node_modules/.cache/schema.gql'
+              : './src/schema.gql'
+          ),
+          path: '/graphql',
           csrfPrevention: {
             requestHeaders: ['content-type'],
           },
-          autoSchemaFile: join(
-            fileURLToPath(import.meta.url),
-            config.node.dev
-              ? '../../../schema.gql'
-              : '../../../../node_modules/.cache/schema.gql'
-          ),
-          sortSchema: true,
           context: ({
             req,
             res,
@@ -55,7 +53,7 @@ export type GraphqlContext = {
 
             // @ts-expect-error allow assign
             formattedError.extensions = ufe.toJSON();
-            if (config.affine.canary) {
+            if (env.namespaces.canary) {
               formattedError.extensions.stacktrace = ufe.stacktrace;
             }
             return formattedError;

packages/backend/server/src/base/graphql/register.ts

@@ -1,4 +1,3 @@
-import { Type } from '@nestjs/common';
 import { Field, FieldOptions, ObjectType } from '@nestjs/graphql';
 
 import { ApplyType } from '../utils/types';
@@ -7,7 +6,7 @@ export function registerObjectType<T>(
   fields: Record<
     string,
     {
-      type: () => Type<any>;
+      type: () => any;
       options?: FieldOptions;
     }
   >,

packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts

@@ -1,5 +1,3 @@
-import { createPrivateKey, createPublicKey } from 'node:crypto';
-
 import ava, { TestFn } from 'ava';
 import Sinon from 'sinon';
 
@@ -9,42 +7,19 @@ const test = ava as TestFn<{
   crypto: CryptoHelper;
 }>;
 
-const key = `-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49
-AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI
-3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg==
------END EC PRIVATE KEY-----`;
-const privateKey = createPrivateKey({
-  key,
-  format: 'pem',
-  type: 'sec1',
-})
-  .export({
-    type: 'pkcs8',
-    format: 'pem',
-  })
-  .toString('utf8');
-
-const publicKey = createPublicKey({
-  key,
-  format: 'pem',
-  type: 'spki',
-})
-  .export({
-    format: 'pem',
-    type: 'spki',
-  })
-  .toString('utf8');
+const privateKey = `-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgS3IAkshQuSmFWGpe
+rGTg2vwaC3LdcvBQlYHHMBYJZMyhRANCAAQXdT/TAh4neNEpd4UqpDIEqWv0XvFo
+BRJxGsC5I/fetqObdx1+KEjcm8zFU2xLaUTw9IZCu8OslloOjQv4ur0a
+-----END PRIVATE KEY-----`;
 
 test.beforeEach(async t => {
   t.context.crypto = new CryptoHelper({
     crypto: {
-      secret: {
-        publicKey,
-        privateKey,
-      },
+      privateKey,
     },
   } as any);
+  t.context.crypto.onConfigInit();
 });
 
 test('should be able to sign and verify', t => {

packages/backend/server/src/base/helpers/config.ts

@@ -1,53 +1,18 @@
-import { createPrivateKey, createPublicKey } from 'node:crypto';
+import { defineModuleConfig } from '../config';
 
-import { defineStartupConfig, ModuleConfig } from '../config';
-
-declare module '../config' {
-  interface AppConfig {
-    crypto: ModuleConfig<{
-      secret: {
-        publicKey: string;
-        privateKey: string;
-      };
-    }>;
+declare global {
+  interface AppConfigSchema {
+    crypto: {
+      privateKey: string;
+    };
   }
 }
 
-// Don't use this in production
-const examplePrivateKey = `-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49
-AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI
-3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg==
------END EC PRIVATE KEY-----`;
-
-defineStartupConfig('crypto', {
-  secret: (function () {
-    const AFFINE_PRIVATE_KEY =
-      process.env.AFFINE_PRIVATE_KEY ?? examplePrivateKey;
-    const privateKey = createPrivateKey({
-      key: Buffer.from(AFFINE_PRIVATE_KEY),
-      format: 'pem',
-      type: 'sec1',
-    })
-      .export({
-        format: 'pem',
-        type: 'pkcs8',
-      })
-      .toString('utf8');
-    const publicKey = createPublicKey({
-      key: Buffer.from(AFFINE_PRIVATE_KEY),
-      format: 'pem',
-      type: 'spki',
-    })
-      .export({
-        format: 'pem',
-        type: 'spki',
-      })
-      .toString('utf8');
-
-    return {
-      publicKey,
-      privateKey,
-    };
-  })(),
+defineModuleConfig('crypto', {
+  privateKey: {
+    desc: 'The private key for used by the crypto module to create signed tokens or encrypt data.',
+    env: 'AFFINE_PRIVATE_KEY',
+    default: '',
+    schema: { type: 'string' },
+  },
 });

packages/backend/server/src/base/helpers/crypto.ts

@@ -2,8 +2,11 @@ import {
   createCipheriv,
   createDecipheriv,
   createHash,
+  createPrivateKey,
+  createPublicKey,
   createSign,
   createVerify,
+  generateKeyPairSync,
   randomBytes,
   randomInt,
   timingSafeEqual,
@@ -16,13 +19,48 @@ import {
 } from '@node-rs/argon2';
 
 import { Config } from '../config';
+import { OnEvent } from '../event';
 
 const NONCE_LENGTH = 12;
 const AUTH_TAG_LENGTH = 12;
 
+function generatePrivateKey(): string {
+  const { privateKey } = generateKeyPairSync('ec', {
+    namedCurve: 'prime256v1',
+  });
+
+  const key = privateKey.export({
+    type: 'sec1',
+    format: 'pem',
+  });
+
+  return key.toString('utf8');
+}
+
+function readPrivateKey(privateKey: string) {
+  return createPrivateKey({
+    key: Buffer.from(privateKey),
+    format: 'pem',
+    type: 'sec1',
+  })
+    .export({
+      format: 'pem',
+      type: 'pkcs8',
+    })
+    .toString('utf8');
+}
+
+function readPublicKey(privateKey: string) {
+  return createPublicKey({
+    key: Buffer.from(privateKey),
+  })
+    .export({ format: 'pem', type: 'spki' })
+    .toString('utf8');
+}
+
 @Injectable()
 export class CryptoHelper {
-  keyPair: {
+  keyPair!: {
     publicKey: Buffer;
     privateKey: Buffer;
     sha256: {
@@ -31,13 +69,31 @@ export class CryptoHelper {
     };
   };
 
-  constructor(config: Config) {
+  constructor(private readonly config: Config) {}
+
+  @OnEvent('config.init')
+  onConfigInit() {
+    this.setup();
+  }
+
+  @OnEvent('config.changed')
+  onConfigChanged(event: Events['config.changed']) {
+    if (event.updates.crypto?.privateKey) {
+      this.setup();
+    }
+  }
+
+  private setup() {
+    const key = this.config.crypto.privateKey || generatePrivateKey();
+    const privateKey = readPrivateKey(key);
+    const publicKey = readPublicKey(key);
+
     this.keyPair = {
-      publicKey: Buffer.from(config.crypto.secret.publicKey, 'utf8'),
-      privateKey: Buffer.from(config.crypto.secret.privateKey, 'utf8'),
+      publicKey: Buffer.from(publicKey),
+      privateKey: Buffer.from(privateKey),
       sha256: {
-        publicKey: this.sha256(config.crypto.secret.publicKey),
-        privateKey: this.sha256(config.crypto.secret.privateKey),
+        publicKey: this.sha256(publicKey),
+        privateKey: this.sha256(privateKey),
       },
     };
   }

packages/backend/server/src/base/helpers/url.ts

@@ -4,16 +4,23 @@ import { Injectable } from '@nestjs/common';
 import type { Response } from 'express';
 
 import { Config } from '../config';
+import { OnEvent } from '../event';
 
 @Injectable()
 export class URLHelper {
-  private readonly redirectAllowHosts: string[];
+  redirectAllowHosts!: string[];
 
-  readonly origin: string;
-  readonly baseUrl: string;
-  readonly home: string;
+  origin!: string;
+  baseUrl!: string;
+  home!: string;
 
   constructor(private readonly config: Config) {
+    this.init();
+  }
+
+  @OnEvent('config.changed')
+  @OnEvent('config.init')
+  init() {
     if (this.config.server.externalUrl) {
       if (!this.verify(this.config.server.externalUrl)) {
         throw new Error(

packages/backend/server/src/base/index.ts

@@ -6,17 +6,14 @@ export {
   SessionCache,
 } from './cache';
 export {
-  type AFFiNEConfig,
-  applyEnvToConfig,
   Config,
-  type ConfigPaths,
-  DeploymentType,
-  getAFFiNEConfigModifier,
+  ConfigFactory,
+  defineModuleConfig,
+  type JSONSchema,
 } from './config';
 export * from './error';
 export { EventBus, OnEvent } from './event';
 export {
-  type GraphqlContext,
   paginate,
   Paginated,
   PaginationInput,
@@ -30,8 +27,12 @@ export { CallMetric, metrics } from './metrics';
 export { Lock, Locker, Mutex, RequestMutex } from './mutex';
 export * from './nestjs';
 export { type PrismaTransaction } from './prisma';
-export { Runtime } from './runtime';
 export * from './storage';
-export { type StorageProvider, StorageProviderFactory } from './storage';
+export {
+  autoMetadata,
+  type StorageProvider,
+  type StorageProviderConfig,
+  StorageProviderFactory,
+} from './storage';
 export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler';
 export * from './utils';

packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts

@@ -52,13 +52,15 @@ class JobHandlers {
 test.before(async () => {
   module = await createTestingModule({
     imports: [
-      ConfigModule.forRoot({
+      ConfigModule.override({
         job: {
           worker: {
-            // NOTE(@forehalo):
-            //   bullmq will hold the connection to check stalled jobs,
-            //   which will keep the test process alive to timeout.
-            stalledInterval: 100,
+            defaultWorkerOptions: {
+              // NOTE(@forehalo):
+              //   bullmq will hold the connection to check stalled jobs,
+              //   which will keep the test process alive to timeout.
+              stalledInterval: 100,
+            },
           },
           queue: {
             defaultJobOptions: { delay: 1000 },
@@ -82,7 +84,7 @@ test.afterEach(async () => {
   // @ts-expect-error private api
   const inner = queue.getQueue('nightly');
   await inner.obliterate({ force: true });
-  inner.resume();
+  await inner.resume();
 });
 
 test.after.always(async () => {
@@ -132,7 +134,7 @@ test('should remove job from queue', async t => {
 // #region executor
 test('should start workers', async t => {
   // @ts-expect-error private api
-  const worker = executor.workers['nightly'];
+  const worker = executor.workers.get('nightly')!;
 
   t.truthy(worker);
   t.true(worker.isRunning());

packages/backend/server/src/base/job/queue/config.ts

@@ -1,61 +1,86 @@
 import { QueueOptions, WorkerOptions } from 'bullmq';
 
-import {
-  defineRuntimeConfig,
-  defineStartupConfig,
-  ModuleConfig,
-} from '../../config';
+import { defineModuleConfig, JSONSchema } from '../../config';
 import { Queue } from './def';
 
-declare module '../../config' {
-  interface AppConfig {
-    job: ModuleConfig<
-      {
-        queue: Omit<QueueOptions, 'connection'>;
-        worker: Omit<WorkerOptions, 'connection'>;
-      },
-      {
-        queues: {
-          [key in Queue]: {
-            concurrency: number;
-          };
-        };
-      }
-    >;
+declare global {
+  interface AppConfigSchema {
+    job: {
+      queue: ConfigItem<Omit<QueueOptions, 'connection' | 'telemetry'>>;
+      worker: ConfigItem<{
+        defaultWorkerOptions: Omit<WorkerOptions, 'connection' | 'telemetry'>;
+      }>;
+      queues: {
+        [key in Queue]: ConfigItem<{
+          concurrency: number;
+        }>;
+      };
+    };
   }
 }
 
-defineStartupConfig('job', {
+const schema: JSONSchema = {
+  type: 'object',
+  properties: {
+    concurrency: { type: 'number' },
+  },
+};
+
+defineModuleConfig('job', {
   queue: {
-    prefix: AFFiNE.node.test ? 'affine_job_test' : 'affine_job',
-    defaultJobOptions: {
-      attempts: 5,
-      // should remove job after it's completed, because we will add a new job with the same job id
-      removeOnComplete: true,
-      removeOnFail: {
-        age: 24 * 3600 /* 1 day */,
-        count: 500,
+    desc: 'The config for job queues',
+    default: {
+      prefix: env.testing ? 'affine_job_test' : 'affine_job',
+      defaultJobOptions: {
+        attempts: 5,
+        // should remove job after it's completed, because we will add a new job with the same job id
+        removeOnComplete: true,
+        removeOnFail: {
+          age: 24 * 3600 /* 1 day */,
+          count: 500,
+        },
       },
     },
+    link: 'https://api.docs.bullmq.io/interfaces/v5.QueueOptions.html',
   },
-  worker: {},
-});
 
-defineRuntimeConfig('job', {
-  'queues.nightly.concurrency': {
-    default: 1,
-    desc: 'Concurrency of worker consuming of nightly checking job queue',
+  worker: {
+    desc: 'The config for job workers',
+    default: {
+      defaultWorkerOptions: {},
+    },
+    link: 'https://api.docs.bullmq.io/interfaces/v5.WorkerOptions.html',
   },
-  'queues.notification.concurrency': {
-    default: 10,
-    desc: 'Concurrency of worker consuming of notification job queue',
+
+  'queues.copilot': {
+    desc: 'The config for copilot job queue',
+    default: {
+      concurrency: 1,
+    },
+    schema,
   },
-  'queues.doc.concurrency': {
-    default: 1,
-    desc: 'Concurrency of worker consuming of doc job queue',
+
+  'queues.doc': {
+    desc: 'The config for doc job queue',
+    default: {
+      concurrency: 1,
+    },
+    schema,
   },
-  'queues.copilot.concurrency': {
-    default: 1,
-    desc: 'Concurrency of worker consuming of copilot job queue',
+
+  'queues.notification': {
+    desc: 'The config for notification job queue',
+    default: {
+      concurrency: 10,
+    },
+    schema,
+  },
+
+  'queues.nightly': {
+    desc: 'The config for nightly job queue',
+    default: {
+      concurrency: 1,
+    },
+    schema,
   },
 });

packages/backend/server/src/base/job/queue/def.ts

@@ -49,7 +49,7 @@ export const OnJob = (job: JobName) => {
   if (!QUEUES.includes(ns as Queue)) {
     throw new Error(
       `Invalid job queue: ${ns}, must be one of [${QUEUES.join(', ')}].
-If you want to introduce new job queue, please modify the Queue enum first in ${join(AFFiNE.projectRoot, 'src/base/job/queue/def.ts')}`
+If you want to introduce new job queue, please modify the Queue enum first in ${join(env.projectRoot, 'src/base/job/queue/def.ts')}`
     );
   }
 

packages/backend/server/src/base/job/queue/executor.ts

@@ -1,49 +1,51 @@
-import {
-  Injectable,
-  Logger,
-  OnApplicationBootstrap,
-  OnApplicationShutdown,
-} from '@nestjs/common';
+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
 import { Worker } from 'bullmq';
-import { difference } from 'lodash-es';
+import { difference, merge } from 'lodash-es';
 import { CLS_ID, ClsServiceManager } from 'nestjs-cls';
 
 import { Config } from '../../config';
+import { OnEvent } from '../../event';
 import { metrics, wrapCallMetric } from '../../metrics';
 import { QueueRedis } from '../../redis';
-import { Runtime } from '../../runtime';
 import { genRequestId } from '../../utils';
 import { JOB_SIGNAL, namespace, Queue, QUEUES } from './def';
 import { JobHandlerScanner } from './scanner';
 
 @Injectable()
-export class JobExecutor
-  implements OnApplicationBootstrap, OnApplicationShutdown
-{
+export class JobExecutor implements OnModuleDestroy {
   private readonly logger = new Logger('job');
-  private readonly workers: Record<string, Worker> = {};
+  private readonly workers: Map<Queue, Worker> = new Map();
 
   constructor(
     private readonly config: Config,
     private readonly redis: QueueRedis,
-    private readonly scanner: JobHandlerScanner,
-    private readonly runtime: Runtime
+    private readonly scanner: JobHandlerScanner
   ) {}
 
-  async onApplicationBootstrap() {
-    const queues = this.config.flavor.graphql
-      ? difference(QUEUES, [Queue.DOC])
-      : [];
+  @OnEvent('config.init')
+  async onConfigInit() {
+    const queues = env.flavors.graphql ? difference(QUEUES, [Queue.DOC]) : [];
 
     // NOTE(@forehalo): only enable doc queue in doc service
-    if (this.config.flavor.doc) {
+    if (env.flavors.doc) {
       queues.push(Queue.DOC);
     }
 
     await this.startWorkers(queues);
   }
 
-  async onApplicationShutdown() {
+  @OnEvent('config.changed')
+  async onConfigChanged({ updates }: Events['config.changed']) {
+    if (updates.job?.queues) {
+      Object.entries(updates.job.queues).forEach(([queue, options]) => {
+        if (options.concurrency) {
+          this.setConcurrency(queue as Queue, options.concurrency);
+        }
+      });
+    }
+  }
+
+  async onModuleDestroy() {
     await this.stopWorkers();
   }
 
@@ -98,38 +100,35 @@ export class JobExecutor
     }
   }
 
-  private async startWorkers(queues: Queue[]) {
-    const configs =
-      (await this.runtime.fetchAll(
-        queues.reduce(
-          (ret, queue) => {
-            ret[`job/queues.${queue}.concurrency`] = true;
-            return ret;
-          },
-          {} as {
-            [key in `job/queues.${Queue}.concurrency`]: true;
-          }
-        )
-        // TODO(@forehalo): fix the override by [payment/service.spec.ts]
-      )) ?? {};
+  setConcurrency(queue: Queue, concurrency: number) {
+    const worker = this.workers.get(queue);
+    if (!worker) {
+      throw new Error(`Worker for [${queue}] not found.`);
+    }
 
+    worker.concurrency = concurrency;
+  }
+
+  private async startWorkers(queues: Queue[]) {
     for (const queue of queues) {
-      const concurrency =
-        (configs[`job/queues.${queue}.concurrency`] as number) ??
-        this.config.job.worker.concurrency ??
-        1;
+      const queueOptions = this.config.job.queues[queue];
+      const concurrency = queueOptions.concurrency ?? 1;
 
       const worker = new Worker(
         queue,
         async job => {
           await this.run(job.name as JobName, job.data);
         },
-        {
-          ...this.config.job.queue,
-          ...this.config.job.worker,
-          connection: this.redis,
-          concurrency,
-        }
+        merge(
+          {},
+          this.config.job.queue,
+          this.config.job.worker.defaultWorkerOptions,
+          queueOptions,
+          {
+            concurrency,
+            connection: this.redis,
+          }
+        )
       );
 
       worker.on('error', error => {
@@ -140,13 +139,13 @@ export class JobExecutor
         `Queue Worker [${queue}] started; concurrency=${concurrency};`
       );
 
-      this.workers[queue] = worker;
+      this.workers.set(queue, worker);
     }
   }
 
   private async stopWorkers() {
     await Promise.all(
-      Object.values(this.workers).map(async worker => {
+      Array.from(this.workers.values()).map(async worker => {
         await worker.close(true);
       })
     );

packages/backend/server/src/base/metrics/config.ts

@@ -1,33 +1,16 @@
-import { defineStartupConfig, ModuleConfig } from '../config';
+import { defineModuleConfig } from '../config';
 
-declare module '../config' {
-  interface AppConfig {
-    metrics: ModuleConfig<{
-      /**
-       * Enable metric and tracing collection
-       */
+declare global {
+  interface AppConfigSchema {
+    metrics: {
       enabled: boolean;
-      /**
-       * Enable telemetry
-       */
-      telemetry: {
-        enabled: boolean;
-        token: string;
-      };
-      customerIo: {
-        token: string;
-      };
-    }>;
+    };
   }
 }
 
-defineStartupConfig('metrics', {
-  enabled: false,
-  telemetry: {
-    enabled: false,
-    token: '',
-  },
-  customerIo: {
-    token: '',
+defineModuleConfig('metrics', {
+  enabled: {
+    desc: 'Enable metric and tracing collection',
+    default: false,
   },
 });

packages/backend/server/src/base/metrics/index.ts

@@ -1,54 +1,14 @@
 import './config';
 
-import {
-  Global,
-  Module,
-  OnModuleDestroy,
-  OnModuleInit,
-  Provider,
-} from '@nestjs/common';
-import { ModuleRef } from '@nestjs/core';
-import { NodeSDK } from '@opentelemetry/sdk-node';
+import { Global, Module } from '@nestjs/common';
 
-import { Config } from '../config';
-import {
-  LocalOpentelemetryFactory,
-  OpentelemetryFactory,
-  registerCustomMetrics,
-} from './opentelemetry';
-
-const factorProvider: Provider = {
-  provide: OpentelemetryFactory,
-  useFactory: (config: Config) => {
-    return config.metrics.enabled ? new LocalOpentelemetryFactory() : null;
-  },
-  inject: [Config],
-};
+import { OpentelemetryFactory } from './opentelemetry';
 
 @Global()
 @Module({
-  providers: [factorProvider],
-  exports: [factorProvider],
+  providers: [OpentelemetryFactory],
 })
-export class MetricsModule implements OnModuleInit, OnModuleDestroy {
-  private sdk: NodeSDK | null = null;
-  constructor(private readonly ref: ModuleRef) {}
-
-  onModuleInit() {
-    const factor = this.ref.get(OpentelemetryFactory, { strict: false });
-    if (factor) {
-      this.sdk = factor.create();
-      this.sdk.start();
-      registerCustomMetrics();
-    }
-  }
-
-  async onModuleDestroy() {
-    if (this.sdk) {
-      await this.sdk.shutdown();
-    }
-  }
-}
+export class MetricsModule {}
 
 export * from './metrics';
 export * from './utils';

packages/backend/server/src/base/metrics/metrics.ts

@@ -2,11 +2,28 @@ import {
   Gauge,
   Histogram,
   Meter,
+  MeterProvider,
   MetricOptions,
+  metrics as otelMetrics,
   UpDownCounter,
 } from '@opentelemetry/api';
+import { HostMetrics } from '@opentelemetry/host-metrics';
 
-import { getMeter } from './opentelemetry';
+function getMeterProvider() {
+  return otelMetrics.getMeterProvider();
+}
+
+export function registerCustomMetrics() {
+  const hostMetricsMonitoring = new HostMetrics({
+    name: 'instance-host-metrics',
+    meterProvider: getMeterProvider() as MeterProvider,
+  });
+  hostMetricsMonitoring.start();
+}
+
+export function getMeter(name = 'business') {
+  return getMeterProvider().getMeter(name);
+}
 
 type MetricType = 'counter' | 'gauge' | 'histogram';
 type Metric<T extends MetricType> = T extends 'counter'
@@ -122,5 +139,3 @@ export const metrics = new Proxy<Record<KnownMetricScopes, ScopedMetrics>>(
     },
   }
 );
-
-export function stopMetrics() {}

packages/backend/server/src/base/metrics/opentelemetry.ts

@@ -1,5 +1,4 @@
-import { OnModuleDestroy } from '@nestjs/common';
-import { metrics } from '@opentelemetry/api';
+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
 import {
   CompositePropagator,
   W3CBaggagePropagator,
@@ -7,7 +6,6 @@ import {
 } from '@opentelemetry/core';
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { ZipkinExporter } from '@opentelemetry/exporter-zipkin';
-import { HostMetrics } from '@opentelemetry/host-metrics';
 import { Instrumentation } from '@opentelemetry/instrumentation';
 import { GraphQLInstrumentation } from '@opentelemetry/instrumentation-graphql';
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
@@ -15,7 +13,6 @@ import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
 import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
 import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io';
 import { Resource } from '@opentelemetry/resources';
-import type { MeterProvider } from '@opentelemetry/sdk-metrics';
 import { MetricProducer, MetricReader } from '@opentelemetry/sdk-metrics';
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import {
@@ -30,11 +27,14 @@ import {
 } from '@opentelemetry/semantic-conventions/incubating';
 import prismaInstrument from '@prisma/instrumentation';
 
+import { Config } from '../config';
+import { OnEvent } from '../event/def';
+import { registerCustomMetrics } from './metrics';
 import { PrismaMetricProducer } from './prisma';
 
 const { PrismaInstrumentation } = prismaInstrument;
 
-export abstract class OpentelemetryFactory {
+export abstract class BaseOpentelemetryFactory {
   abstract getMetricReader(): MetricReader;
   abstract getSpanExporter(): SpanExporter;
 
@@ -55,9 +55,9 @@ export abstract class OpentelemetryFactory {
 
   getResource() {
     return new Resource({
-      [ATTR_K8S_NAMESPACE_NAME]: AFFiNE.AFFINE_ENV,
-      [ATTR_SERVICE_NAME]: AFFiNE.flavor.type,
-      [ATTR_SERVICE_VERSION]: AFFiNE.version,
+      [ATTR_K8S_NAMESPACE_NAME]: env.NAMESPACE,
+      [ATTR_SERVICE_NAME]: env.FLAVOR,
+      [ATTR_SERVICE_VERSION]: env.version,
     });
   }
 
@@ -81,39 +81,58 @@ export abstract class OpentelemetryFactory {
   }
 }
 
-export class LocalOpentelemetryFactory
-  extends OpentelemetryFactory
+@Injectable()
+export class OpentelemetryFactory
+  extends BaseOpentelemetryFactory
   implements OnModuleDestroy
 {
-  private readonly metricsExporter = new PrometheusExporter({
-    metricProducers: this.getMetricsProducers(),
-  });
+  private readonly logger = new Logger(OpentelemetryFactory.name);
+  #sdk: NodeSDK | null = null;
+
+  constructor(private readonly config: Config) {
+    super();
+  }
+
+  @OnEvent('config.init')
+  async init(event: Events['config.init']) {
+    if (event.config.metrics.enabled) {
+      await this.setup();
+      registerCustomMetrics();
+    }
+  }
+
+  @OnEvent('config.changed')
+  async onConfigChanged(event: Events['config.changed']) {
+    if ('metrics' in event.updates) {
+      await this.setup();
+    }
+  }
 
   async onModuleDestroy() {
-    await this.metricsExporter.shutdown();
+    await this.#sdk?.shutdown();
   }
 
   override getMetricReader(): MetricReader {
-    return this.metricsExporter;
+    return new PrometheusExporter({
+      metricProducers: this.getMetricsProducers(),
+    });
   }
 
   override getSpanExporter(): SpanExporter {
     return new ZipkinExporter();
   }
-}
 
-function getMeterProvider() {
-  return metrics.getMeterProvider();
-}
-
-export function registerCustomMetrics() {
-  const hostMetricsMonitoring = new HostMetrics({
-    name: 'instance-host-metrics',
-    meterProvider: getMeterProvider() as MeterProvider,
-  });
-  hostMetricsMonitoring.start();
-}
-
-export function getMeter(name = 'business') {
-  return getMeterProvider().getMeter(name);
+  private async setup() {
+    if (this.config.metrics.enabled) {
+      if (!this.#sdk) {
+        this.#sdk = this.create();
+      }
+      this.#sdk.start();
+      this.logger.log('OpenTelemetry SDK started');
+    } else {
+      await this.#sdk?.shutdown();
+      this.#sdk = null;
+      this.logger.log('OpenTelemetry SDK stopped');
+    }
+  }
 }

packages/backend/server/src/base/metrics/prisma.ts

@@ -10,7 +10,7 @@ import {
   ScopeMetrics,
 } from '@opentelemetry/sdk-metrics';
 
-import { PrismaService } from '../prisma';
+import { PrismaFactory } from '../prisma/factory';
 
 function transformPrismaKey(key: string) {
   // replace first '_' to '/' as a scope prefix
@@ -30,11 +30,11 @@ export class PrismaMetricProducer implements MetricProducer {
       errors: [],
     };
 
-    if (!PrismaService.INSTANCE) {
+    if (!PrismaFactory.INSTANCE) {
       return result;
     }
 
-    const prisma = PrismaService.INSTANCE;
+    const prisma = PrismaFactory.INSTANCE;
 
     const endTime = hrTime();
 

packages/backend/server/src/base/nestjs/config.ts

@@ -1,39 +0,0 @@
-import { defineStartupConfig, ModuleConfig } from '../../base/config';
-
-export interface ServerStartupConfigurations {
-  /**
-   * Base url of AFFiNE server, used for generating external urls.
-   * default to be `[AFFiNE.protocol]://[AFFiNE.host][:AFFiNE.port]/[AFFiNE.path]` if not specified
-   */
-  externalUrl: string;
-  /**
-   * Whether the server is hosted on a ssl enabled domain
-   */
-  https: boolean;
-  /**
-   * where the server get deployed(FQDN).
-   */
-  host: string;
-  /**
-   * which port the server will listen on
-   */
-  port: number;
-  /**
-   * subpath where the server get deployed if there is.
-   */
-  path: string;
-}
-
-declare module '../../base/config' {
-  interface AppConfig {
-    server: ModuleConfig<ServerStartupConfigurations>;
-  }
-}
-
-defineStartupConfig('server', {
-  externalUrl: '',
-  https: false,
-  host: 'localhost',
-  port: 3010,
-  path: '',
-});

packages/backend/server/src/base/nestjs/index.ts

@@ -1,5 +1,3 @@
-import './config';
 export * from './decorator';
 export * from './exception';
-export * from './optional-module';
 export * from './scanner';

packages/backend/server/src/base/nestjs/optional-module.ts

@@ -1,72 +0,0 @@
-import {
-  DynamicModule,
-  Module,
-  ModuleMetadata,
-  Provider,
-  Type,
-} from '@nestjs/common';
-import { omit } from 'lodash-es';
-
-import type { AFFiNEConfig, ConfigPaths } from '../config';
-
-export interface OptionalModuleMetadata extends ModuleMetadata {
-  /**
-   * Only install module if given config paths are defined in AFFiNE config.
-   */
-  requires?: ConfigPaths[];
-
-  /**
-   * Only install module if the predication returns true.
-   */
-  if?: (config: AFFiNEConfig) => boolean;
-
-  /**
-   * Defines which feature will be enabled if the module installed.
-   */
-  contributesTo?: import('../../core/config').ServerFeature; // avoid circlar dependency
-
-  /**
-   * Defines which providers provided by other modules will be overridden if the module installed.
-   */
-  overrides?: Provider[];
-}
-
-const additionalOptions = [
-  'contributesTo',
-  'requires',
-  'if',
-  'overrides',
-] as const satisfies Array<keyof OptionalModuleMetadata>;
-
-type OptionalDynamicModule = DynamicModule & OptionalModuleMetadata;
-
-export function OptionalModule(metadata: OptionalModuleMetadata) {
-  return (target: Type) => {
-    additionalOptions.forEach(option => {
-      if (Object.hasOwn(metadata, option)) {
-        Reflect.defineMetadata(option, metadata[option], target);
-      }
-    });
-
-    if (metadata.overrides) {
-      metadata.providers = (metadata.providers ?? []).concat(
-        metadata.overrides
-      );
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      metadata.exports = (metadata.exports ?? []).concat(metadata.overrides);
-    }
-
-    const nestMetadata = omit(metadata, additionalOptions);
-    Module(nestMetadata)(target);
-  };
-}
-
-export function getOptionalModuleMetadata<
-  T extends keyof OptionalModuleMetadata,
->(target: Type | OptionalDynamicModule, key: T): OptionalModuleMetadata[T] {
-  if ('module' in target) {
-    return target[key];
-  } else {
-    return Reflect.getMetadata(key, target);
-  }
-}

packages/backend/server/src/base/prisma/config.ts

@@ -1,17 +1,27 @@
 import type { Prisma } from '@prisma/client';
+import { z } from 'zod';
 
-import { defineStartupConfig, ModuleConfig } from '../config';
+import { defineModuleConfig } from '../config';
 
-interface PrismaStartupConfiguration extends Prisma.PrismaClientOptions {
-  datasourceUrl: string;
-}
-
-declare module '../config' {
-  interface AppConfig {
-    prisma: ModuleConfig<PrismaStartupConfiguration>;
+declare global {
+  interface AppConfigSchema {
+    db: {
+      datasourceUrl: string;
+      prisma: ConfigItem<Prisma.PrismaClientOptions>;
+    };
   }
 }
 
-defineStartupConfig('prisma', {
-  datasourceUrl: '',
+defineModuleConfig('db', {
+  datasourceUrl: {
+    desc: 'The datasource url for the prisma client.',
+    default: 'postgresql://localhost:5432/affine',
+    env: 'DATABASE_URL',
+    shape: z.string().url(),
+  },
+  prisma: {
+    desc: 'The config for the prisma client.',
+    default: {},
+    link: 'https://www.prisma.io/docs/reference/api-reference/prisma-client-reference',
+  },
 });

packages/backend/server/src/base/prisma/factory.ts

@@ -0,0 +1,25 @@
+import type { OnModuleDestroy } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
+import { PrismaClient } from '@prisma/client';
+
+import { Config } from '../config';
+
+@Injectable()
+export class PrismaFactory implements OnModuleDestroy {
+  static INSTANCE: PrismaClient | null = null;
+  readonly #instance: PrismaClient;
+
+  constructor(config: Config) {
+    this.#instance = new PrismaClient(config.db.prisma);
+    PrismaFactory.INSTANCE = this.#instance;
+  }
+
+  get() {
+    return this.#instance;
+  }
+
+  async onModuleDestroy() {
+    await PrismaFactory.INSTANCE?.$disconnect();
+    PrismaFactory.INSTANCE = null;
+  }
+}

packages/backend/server/src/base/prisma/index.ts

@@ -3,29 +3,24 @@ import './config';
 import { Global, Module, Provider } from '@nestjs/common';
 import { PrismaClient } from '@prisma/client';
 
-import { Config } from '../config';
-import { PrismaService } from './service';
+import { PrismaFactory } from './factory';
 
 // only `PrismaClient` can be injected
 const clientProvider: Provider = {
   provide: PrismaClient,
-  useFactory: (config: Config) => {
-    if (PrismaService.INSTANCE) {
-      return PrismaService.INSTANCE;
-    }
-
-    return new PrismaService(config.prisma);
+  useFactory: (factory: PrismaFactory) => {
+    return factory.get();
   },
-  inject: [Config],
+  inject: [PrismaFactory],
 };
 
 @Global()
 @Module({
-  providers: [clientProvider],
+  providers: [PrismaFactory, clientProvider],
   exports: [clientProvider],
 })
 export class PrismaModule {}
-export { PrismaService } from './service';
+export { PrismaFactory };
 
 export type PrismaTransaction = Parameters<
   Parameters<PrismaClient['$transaction']>[0]

packages/backend/server/src/base/prisma/service.ts

@@ -1,27 +0,0 @@
-import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
-import { Injectable } from '@nestjs/common';
-import { Prisma, PrismaClient } from '@prisma/client';
-
-@Injectable()
-export class PrismaService
-  extends PrismaClient
-  implements OnModuleInit, OnApplicationShutdown
-{
-  static INSTANCE: PrismaService | null = null;
-
-  constructor(opts: Prisma.PrismaClientOptions) {
-    super(opts);
-    PrismaService.INSTANCE = this;
-  }
-
-  async onModuleInit() {
-    await this.$connect();
-  }
-
-  async onApplicationShutdown(): Promise<void> {
-    if (!AFFiNE.node.test) {
-      await this.$disconnect();
-      PrismaService.INSTANCE = null;
-    }
-  }
-}

packages/backend/server/src/base/redis/config.ts

@@ -1,11 +1,54 @@
 import { RedisOptions } from 'ioredis';
+import { z } from 'zod';
 
-import { defineStartupConfig, ModuleConfig } from '../../base/config';
+import { defineModuleConfig } from '../config';
 
-declare module '../config' {
-  interface AppConfig {
-    redis: ModuleConfig<RedisOptions>;
+declare global {
+  interface AppConfigSchema {
+    redis: {
+      host: string;
+      port: number;
+      db: number;
+      username: string;
+      password: string;
+      ioredis: ConfigItem<
+        Omit<RedisOptions, 'host' | 'port' | 'db' | 'username' | 'password'>
+      >;
+    };
   }
 }
 
-defineStartupConfig('redis', {});
+defineModuleConfig('redis', {
+  db: {
+    desc: 'The database index of redis server to be used(Must be less than 10).',
+    default: 0,
+    env: ['REDIS_DATABASE', 'integer'],
+    validate: val => val >= 0 && val < 10,
+  },
+  host: {
+    desc: 'The host of the redis server.',
+    default: 'localhost',
+    env: ['REDIS_HOST', 'string'],
+  },
+  port: {
+    desc: 'The port of the redis server.',
+    default: 6379,
+    env: ['REDIS_PORT', 'integer'],
+    shape: z.number().positive(),
+  },
+  username: {
+    desc: 'The username of the redis server.',
+    default: '',
+    env: ['REDIS_USERNAME', 'string'],
+  },
+  password: {
+    desc: 'The password of the redis server.',
+    default: '',
+    env: ['REDIS_PASSWORD', 'string'],
+  },
+  ioredis: {
+    desc: 'The config for the ioredis client.',
+    default: {},
+    link: 'https://github.com/luin/ioredis',
+  },
+});

packages/backend/server/src/base/redis/instances.ts

@@ -6,13 +6,10 @@ import {
 } from '@nestjs/common';
 import { Redis as IORedis, RedisOptions } from 'ioredis';
 
-import { Config } from '../../base/config';
+import { Config } from '../config';
 
 class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
   private readonly logger = new Logger(this.constructor.name);
-  constructor(opts: RedisOptions) {
-    super(opts);
-  }
 
   errorHandler = (err: Error) => {
     this.logger.error(err);
@@ -46,21 +43,29 @@ class Redis extends IORedis implements OnModuleInit, OnModuleDestroy {
 @Injectable()
 export class CacheRedis extends Redis {
   constructor(config: Config) {
-    super(config.redis);
+    super({ ...config.redis, ...config.redis.ioredis });
   }
 }
 
 @Injectable()
 export class SessionRedis extends Redis {
   constructor(config: Config) {
-    super({ ...config.redis, db: (config.redis.db ?? 0) + 2 });
+    super({
+      ...config.redis,
+      ...config.redis.ioredis,
+      db: (config.redis.db ?? 0) + 2,
+    });
   }
 }
 
 @Injectable()
 export class SocketIoRedis extends Redis {
   constructor(config: Config) {
-    super({ ...config.redis, db: (config.redis.db ?? 0) + 3 });
+    super({
+      ...config.redis,
+      ...config.redis.ioredis,
+      db: (config.redis.db ?? 0) + 3,
+    });
   }
 }
 
@@ -69,6 +74,7 @@ export class QueueRedis extends Redis {
   constructor(config: Config) {
     super({
       ...config.redis,
+      ...config.redis.ioredis,
       db: (config.redis.db ?? 0) + 4,
       // required explicitly set to `null` by bullmq
       maxRetriesPerRequest: null,

packages/backend/server/src/base/runtime/event.ts

@@ -1,7 +0,0 @@
-import { FlattenedAppRuntimeConfig } from '../config/types';
-
-declare global {
-  interface Events {
-    'runtime.changed__NOT_IMPLEMENTED__': Partial<FlattenedAppRuntimeConfig>;
-  }
-}

packages/backend/server/src/base/runtime/index.ts

@@ -1,11 +0,0 @@
-import { Global, Module } from '@nestjs/common';
-
-import { Runtime } from './service';
-
-@Global()
-@Module({
-  providers: [Runtime],
-  exports: [Runtime],
-})
-export class RuntimeModule {}
-export { Runtime };

packages/backend/server/src/base/runtime/service.ts

@@ -1,258 +0,0 @@
-import {
-  forwardRef,
-  Inject,
-  Injectable,
-  Logger,
-  OnModuleInit,
-} from '@nestjs/common';
-import { PrismaClient } from '@prisma/client';
-import { difference, keyBy } from 'lodash-es';
-
-import { Cache } from '../cache';
-import { defaultRuntimeConfig, runtimeConfigType } from '../config/register';
-import {
-  AppRuntimeConfigModules,
-  FlattenedAppRuntimeConfig,
-} from '../config/types';
-import { InvalidRuntimeConfigType, RuntimeConfigNotFound } from '../error';
-import { defer } from '../utils/promise';
-
-function validateConfigType<K extends keyof FlattenedAppRuntimeConfig>(
-  key: K,
-  value: any
-) {
-  const config = defaultRuntimeConfig[key];
-
-  if (!config) {
-    throw new RuntimeConfigNotFound({ key });
-  }
-
-  const want = config.type;
-  const get = runtimeConfigType(value);
-  if (get !== want) {
-    throw new InvalidRuntimeConfigType({
-      key,
-      want,
-      get,
-    });
-  }
-}
-
-/**
- * runtime.fetch(k) // v1
- * runtime.fetchAll(k1, k2, k3) // [v1, v2, v3]
- * runtime.set(k, v)
- * runtime.update(k, (v) => {
- *   v.xxx = 'yyy';
- *   return v
- * })
- */
-@Injectable()
-export class Runtime implements OnModuleInit {
-  private readonly logger = new Logger('App:RuntimeConfig');
-
-  constructor(
-    private readonly db: PrismaClient,
-    // circular deps: runtime => cache => redis(maybe) => config => runtime
-    @Inject(forwardRef(() => Cache)) private readonly cache: Cache
-  ) {}
-
-  async onModuleInit() {
-    await this.upgradeDB();
-  }
-
-  async fetch<K extends keyof FlattenedAppRuntimeConfig>(
-    k: K
-  ): Promise<FlattenedAppRuntimeConfig[K]> {
-    const cached = await this.loadCache<K>(k);
-
-    if (cached !== undefined) {
-      return cached;
-    }
-
-    const dbValue = await this.loadDb<K>(k);
-
-    if (dbValue === undefined) {
-      throw new RuntimeConfigNotFound({ key: k });
-    }
-
-    await this.setCache(k, dbValue);
-
-    return dbValue;
-  }
-
-  async fetchAll<
-    Selector extends { [Key in keyof FlattenedAppRuntimeConfig]?: true },
-  >(
-    selector: Selector
-  ): Promise<{
-    // @ts-expect-error allow
-    [Key in keyof Selector]: FlattenedAppRuntimeConfig[Key];
-  }> {
-    const keys = Object.keys(selector);
-
-    if (keys.length === 0) {
-      return {} as any;
-    }
-
-    const records = await this.db.runtimeConfig.findMany({
-      select: {
-        id: true,
-        value: true,
-      },
-      where: {
-        id: {
-          in: keys,
-        },
-        deletedAt: null,
-      },
-    });
-
-    const keyed = keyBy(records, 'id');
-    return keys.reduce((ret, key) => {
-      ret[key] = keyed[key]?.value ?? defaultRuntimeConfig[key].value;
-      return ret;
-    }, {} as any);
-  }
-
-  async list(module?: AppRuntimeConfigModules) {
-    return await this.db.runtimeConfig.findMany({
-      where: module ? { module, deletedAt: null } : { deletedAt: null },
-    });
-  }
-
-  async set<
-    K extends keyof FlattenedAppRuntimeConfig,
-    V = FlattenedAppRuntimeConfig[K],
-  >(key: K, value: V) {
-    validateConfigType(key, value);
-    const config = await this.db.runtimeConfig.upsert({
-      where: {
-        id: key,
-        deletedAt: null,
-      },
-      create: {
-        ...defaultRuntimeConfig[key],
-        value: value as any,
-      },
-      update: {
-        value: value as any,
-        deletedAt: null,
-      },
-    });
-
-    await this.setCache(key, config.value as FlattenedAppRuntimeConfig[K]);
-    return config;
-  }
-
-  async update<
-    K extends keyof FlattenedAppRuntimeConfig,
-    V = FlattenedAppRuntimeConfig[K],
-  >(k: K, modifier: (v: V) => V | Promise<V>) {
-    const data = await this.fetch<K>(k);
-
-    const updated = await modifier(data as V);
-
-    await this.set(k, updated);
-
-    return updated;
-  }
-
-  async loadDb<K extends keyof FlattenedAppRuntimeConfig>(
-    k: K
-  ): Promise<FlattenedAppRuntimeConfig[K] | undefined> {
-    const v = await this.db.runtimeConfig.findFirst({
-      where: {
-        id: k,
-        deletedAt: null,
-      },
-    });
-
-    if (v) {
-      return v.value as FlattenedAppRuntimeConfig[K];
-    } else {
-      const record = await this.db.runtimeConfig.create({
-        data: defaultRuntimeConfig[k],
-      });
-
-      return record.value as any;
-    }
-  }
-
-  async loadCache<K extends keyof FlattenedAppRuntimeConfig>(
-    k: K
-  ): Promise<FlattenedAppRuntimeConfig[K] | undefined> {
-    return this.cache.get<FlattenedAppRuntimeConfig[K]>(`SERVER_RUNTIME:${k}`);
-  }
-
-  async setCache<K extends keyof FlattenedAppRuntimeConfig>(
-    k: K,
-    v: FlattenedAppRuntimeConfig[K]
-  ): Promise<boolean> {
-    return this.cache.set<FlattenedAppRuntimeConfig[K]>(
-      `SERVER_RUNTIME:${k}`,
-      v,
-      { ttl: 60 * 1000 }
-    );
-  }
-
-  /**
-   * Upgrade the DB with latest runtime configs
-   */
-  private async upgradeDB() {
-    const existingConfig = await this.db.runtimeConfig.findMany({
-      select: {
-        id: true,
-      },
-      where: {
-        deletedAt: null,
-      },
-    });
-
-    const defined = Object.keys(defaultRuntimeConfig);
-    const existing = existingConfig.map(c => c.id);
-    const newConfigs = difference(defined, existing);
-    const deleteConfigs = difference(existing, defined);
-
-    if (!newConfigs.length && !deleteConfigs.length) {
-      return;
-    }
-
-    this.logger.log(`Found runtime config changes, upgrading...`);
-    const acquired = await this.cache.setnx('runtime:upgrade', 1, {
-      ttl: 10 * 60 * 1000,
-    });
-    await using _ = defer(async () => {
-      await this.cache.delete('runtime:upgrade');
-    });
-
-    if (acquired) {
-      for (const key of newConfigs) {
-        await this.db.runtimeConfig.upsert({
-          create: defaultRuntimeConfig[key],
-          // old deleted setting should be restored
-          update: {
-            ...defaultRuntimeConfig[key],
-            deletedAt: null,
-          },
-          where: {
-            id: key,
-          },
-        });
-      }
-
-      await this.db.runtimeConfig.updateMany({
-        where: {
-          id: {
-            in: deleteConfigs,
-          },
-        },
-        data: {
-          deletedAt: new Date(),
-        },
-      });
-    }
-
-    this.logger.log('Upgrade completed');
-  }
-}

packages/backend/server/src/base/storage/config.ts

@@ -1,75 +0,0 @@
-import { homedir } from 'node:os';
-import { join } from 'node:path';
-
-import { defineStartupConfig, ModuleConfig } from '../config';
-
-export interface FsStorageConfig {
-  path: string;
-}
-
-export interface StorageProvidersConfig {
-  fs?: FsStorageConfig;
-}
-
-declare module '../config' {
-  interface AppConfig {
-    storageProviders: ModuleConfig<StorageProvidersConfig>;
-  }
-}
-
-defineStartupConfig('storageProviders', {
-  fs: {
-    path: join(homedir(), '.affine/storage'),
-  },
-});
-
-export type StorageProviderType = keyof StorageProvidersConfig;
-
-export type StorageConfig<Ext = unknown> = {
-  provider: StorageProviderType;
-  bucket: string;
-} & Ext;
-
-export interface StoragesConfig {
-  avatar: StorageConfig<{ publicLinkFactory: (key: string) => string }>;
-  blob: StorageConfig;
-  copilot: StorageConfig;
-}
-
-export interface AFFiNEStorageConfig {
-  /**
-   * All providers for object storage
-   *
-   * Support different providers for different usage at the same time.
-   */
-  providers: StorageProvidersConfig;
-  storages: StoragesConfig;
-}
-
-export type StorageProviders = AFFiNEStorageConfig['providers'];
-export type Storages = keyof AFFiNEStorageConfig['storages'];
-
-export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig {
-  return {
-    providers: {
-      fs: {
-        path: join(homedir(), '.affine/storage'),
-      },
-    },
-    storages: {
-      avatar: {
-        provider: 'fs',
-        bucket: 'avatars',
-        publicLinkFactory: key => `/api/avatars/${key}`,
-      },
-      blob: {
-        provider: 'fs',
-        bucket: 'blobs',
-      },
-      copilot: {
-        provider: 'fs',
-        bucket: 'copilot',
-      },
-    },
-  };
-}

packages/backend/server/src/base/storage/factory.ts

@@ -0,0 +1,20 @@
+import { Injectable } from '@nestjs/common';
+
+import {
+  StorageProvider,
+  StorageProviderConfig,
+  StorageProviders,
+} from './providers';
+
+@Injectable()
+export class StorageProviderFactory {
+  create(config: StorageProviderConfig): StorageProvider {
+    const Provider = StorageProviders[config.provider];
+
+    if (!Provider) {
+      throw new Error(`Unknown storage provider type: ${config.provider}`);
+    }
+
+    return new Provider(config.config, config.bucket);
+  }
+}

packages/backend/server/src/base/storage/index.ts

@@ -1,17 +1,6 @@
-import './config';
-
 import { Global, Module } from '@nestjs/common';
 
-import { registerStorageProvider, StorageProviderFactory } from './providers';
-import { FsStorageProvider } from './providers/fs';
-
-registerStorageProvider('fs', (config, bucket) => {
-  if (!config.storageProviders.fs) {
-    throw new Error('Missing fs storage provider configuration');
-  }
-
-  return new FsStorageProvider(config.storageProviders.fs, bucket);
-});
+import { StorageProviderFactory } from './factory';
 
 @Global()
 @Module({
@@ -19,16 +8,5 @@ registerStorageProvider('fs', (config, bucket) => {
   exports: [StorageProviderFactory],
 })
 export class StorageProviderModule {}
-
-export * from '../../native';
-export type { StorageProviderType } from './config';
-export type {
-  BlobInputType,
-  BlobOutputType,
-  GetObjectMetadata,
-  ListObjectsMetadata,
-  PutObjectMetadata,
-  StorageProvider,
-} from './providers';
-export { registerStorageProvider, StorageProviderFactory } from './providers';
-export { autoMetadata, toBuffer } from './providers/utils';
+export { StorageProviderFactory } from './factory';
+export * from './providers';

packages/backend/server/src/base/storage/providers/fs.ts

@@ -10,12 +10,12 @@ import {
   statSync,
   writeFileSync,
 } from 'node:fs';
-import { join, parse, resolve } from 'node:path';
+import { homedir } from 'node:os';
+import { join, parse } from 'node:path';
 import { Readable } from 'node:stream';
 
 import { Logger } from '@nestjs/common';
 
-import { FsStorageConfig } from '../config';
 import {
   BlobInputType,
   GetObjectMetadata,
@@ -30,6 +30,10 @@ function escapeKey(key: string): string {
   return key.replace(/\.?\.[/\\]/g, '%');
 }
 
+export interface FsStorageConfig {
+  path: string;
+}
+
 export class FsStorageProvider implements StorageProvider {
   private readonly path: string;
   private readonly logger: Logger;
@@ -40,7 +44,9 @@ export class FsStorageProvider implements StorageProvider {
     config: FsStorageConfig,
     public readonly bucket: string
   ) {
-    this.path = resolve(config.path, bucket);
+    this.path = config.path.startsWith('~/')
+      ? join(homedir(), config.path.slice(2), bucket)
+      : join(config.path, bucket);
     this.ensureAvailability();
 
     this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`);

packages/backend/server/src/base/storage/providers/index.ts

@@ -1,34 +1,116 @@
-import { Injectable } from '@nestjs/common';
+import { Type } from '@nestjs/common';
 
-import { Config } from '../../config';
-import { StorageConfig, StorageProviderType } from '../config';
-import type { StorageProvider } from './provider';
+import { JSONSchema } from '../../config';
+import { FsStorageConfig, FsStorageProvider } from './fs';
+import { StorageProvider } from './provider';
+import { R2StorageConfig, R2StorageProvider } from './r2';
+import { S3StorageConfig, S3StorageProvider } from './s3';
 
-const availableProviders = new Map<
-  StorageProviderType,
-  (config: Config, bucket: string) => StorageProvider
->();
+export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2';
+export const StorageProviders: Record<
+  StorageProviderName,
+  Type<StorageProvider>
+> = {
+  fs: FsStorageProvider,
+  'aws-s3': S3StorageProvider,
+  'cloudflare-r2': R2StorageProvider,
+};
 
-export function registerStorageProvider(
-  type: StorageProviderType,
-  providerFactory: (config: Config, bucket: string) => StorageProvider
-) {
-  availableProviders.set(type, providerFactory);
-}
-
-@Injectable()
-export class StorageProviderFactory {
-  constructor(private readonly config: Config) {}
-
-  create(storage: StorageConfig): StorageProvider {
-    const providerFactory = availableProviders.get(storage.provider);
-
-    if (!providerFactory) {
-      throw new Error(`Unknown storage provider type: ${storage.provider}`);
+export type StorageProviderConfig = { bucket: string } & (
+  | {
+      provider: 'fs';
+      config: FsStorageConfig;
     }
+  | {
+      provider: 'aws-s3';
+      config: S3StorageConfig;
+    }
+  | {
+      provider: 'cloudflare-r2';
+      config: R2StorageConfig;
+    }
+);
 
-    return providerFactory(this.config, storage.bucket);
-  }
-}
+const S3ConfigSchema: JSONSchema = {
+  type: 'object',
+  description:
+    'The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html',
+  properties: {
+    credentials: {
+      type: 'object',
+      description: 'The credentials for the s3 compatible storage provider.',
+      properties: {
+        accessKeyId: {
+          type: 'string',
+        },
+        secretAccessKey: {
+          type: 'string',
+        },
+      },
+    },
+  },
+};
+
+export const StorageJSONSchema: JSONSchema = {
+  oneOf: [
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['fs'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: {
+          type: 'object',
+          properties: {
+            path: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['aws-s3'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: S3ConfigSchema,
+      },
+    },
+    {
+      type: 'object',
+      properties: {
+        provider: {
+          type: 'string',
+          enum: ['cloudflare-r2'],
+        },
+        bucket: {
+          type: 'string',
+        },
+        config: {
+          ...S3ConfigSchema,
+          properties: {
+            ...S3ConfigSchema.properties,
+            accountId: {
+              type: 'string' as const,
+              description:
+                'The account id for the cloudflare r2 storage provider.',
+            },
+          },
+        },
+      },
+    },
+  ],
+};
 
 export type * from './provider';
+export { autoMetadata, toBuffer } from './utils';

packages/backend/server/src/base/storage/providers/provider.ts

@@ -1,7 +1,5 @@
 import type { Readable } from 'node:stream';
 
-import { StorageProviderType } from '../config';
-
 export interface GetObjectMetadata {
   /**
    * @default 'application/octet-stream'
@@ -28,7 +26,6 @@ export type BlobInputType = Buffer | Readable | string;
 export type BlobOutputType = Readable;
 
 export interface StorageProvider {
-  readonly type: StorageProviderType;
   put(
     key: string,
     body: BlobInputType,

packages/backend/server/src/base/storage/providers/r2.ts

@@ -0,0 +1,27 @@
+import assert from 'node:assert';
+
+import { Logger } from '@nestjs/common';
+
+import { S3StorageConfig, S3StorageProvider } from './s3';
+
+export interface R2StorageConfig extends S3StorageConfig {
+  accountId: string;
+}
+
+export class R2StorageProvider extends S3StorageProvider {
+  constructor(config: R2StorageConfig, bucket: string) {
+    assert(config.accountId, 'accountId is required for R2 storage provider');
+    super(
+      {
+        ...config,
+        forcePathStyle: true,
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        // see https://github.com/aws/aws-sdk-js-v3/issues/6810
+        requestChecksumCalculation: 'WHEN_REQUIRED',
+        responseChecksumValidation: 'WHEN_REQUIRED',
+      },
+      bucket
+    );
+    this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
+  }
+}

packages/backend/server/src/base/storage/providers/s3.ts

@@ -0,0 +1,207 @@
+/* oxlint-disable @typescript-eslint/no-non-null-assertion */
+import { Readable } from 'node:stream';
+
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  NoSuchKey,
+  PutObjectCommand,
+  S3Client,
+  S3ClientConfig,
+} from '@aws-sdk/client-s3';
+import { Logger } from '@nestjs/common';
+
+import {
+  BlobInputType,
+  GetObjectMetadata,
+  ListObjectsMetadata,
+  PutObjectMetadata,
+  StorageProvider,
+} from './provider';
+import { autoMetadata, toBuffer } from './utils';
+
+export type S3StorageConfig = S3ClientConfig;
+
+export class S3StorageProvider implements StorageProvider {
+  protected logger: Logger;
+  protected client: S3Client;
+
+  constructor(
+    config: S3StorageConfig,
+    public readonly bucket: string
+  ) {
+    this.client = new S3Client({
+      region: 'auto',
+      // s3 client uses keep-alive by default to accelerate requests, and max requests queue is 50.
+      // If some of them are long holding or dead without response, the whole queue will block.
+      // By default no timeout is set for requests or connections, so we set them here.
+      requestHandler: { requestTimeout: 60_000, connectionTimeout: 10_000 },
+      ...config,
+    });
+    this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
+  }
+
+  async put(
+    key: string,
+    body: BlobInputType,
+    metadata: PutObjectMetadata = {}
+  ): Promise<void> {
+    const blob = await toBuffer(body);
+
+    metadata = autoMetadata(blob, metadata);
+
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+          Body: blob,
+
+          // metadata
+          ContentType: metadata.contentType,
+          ContentLength: metadata.contentLength,
+          // TODO(@forehalo): Cloudflare doesn't support CRC32, use md5 instead later.
+          // ChecksumCRC32: metadata.checksumCRC32,
+        })
+      );
+
+      this.logger.verbose(`Object \`${key}\` put`);
+    } catch (e) {
+      this.logger.error(
+        `Failed to put object (${JSON.stringify({
+          key,
+          bucket: this.bucket,
+          metadata,
+        })})`
+      );
+      throw e;
+    }
+  }
+
+  async head(key: string) {
+    try {
+      const obj = await this.client.send(
+        new HeadObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      return {
+        contentType: obj.ContentType!,
+        contentLength: obj.ContentLength!,
+        lastModified: obj.LastModified!,
+        checksumCRC32: obj.ChecksumCRC32,
+      };
+    } catch (e) {
+      // 404
+      if (e instanceof NoSuchKey) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return undefined;
+      }
+      this.logger.error(`Failed to head object \`${key}\``);
+      throw e;
+    }
+  }
+
+  async get(key: string): Promise<{
+    body?: Readable;
+    metadata?: GetObjectMetadata;
+  }> {
+    try {
+      const obj = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      if (!obj.Body) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return {};
+      }
+
+      this.logger.verbose(`Read object \`${key}\``);
+      return {
+        // @ts-expect-errors ignore browser response type `Blob`
+        body: obj.Body,
+        metadata: {
+          // always set when putting object
+          contentType: obj.ContentType!,
+          contentLength: obj.ContentLength!,
+          lastModified: obj.LastModified!,
+          checksumCRC32: obj.ChecksumCRC32,
+        },
+      };
+    } catch (e) {
+      // 404
+      if (e instanceof NoSuchKey) {
+        this.logger.verbose(`Object \`${key}\` not found`);
+        return {};
+      }
+      this.logger.error(`Failed to read object \`${key}\``);
+      throw e;
+    }
+  }
+
+  async list(prefix?: string): Promise<ListObjectsMetadata[]> {
+    // continuationToken should be `string | undefined`,
+    // but TypeScript will fail on type infer in the code below.
+    // Seems to be a bug in TypeScript
+    let continuationToken: any = undefined;
+    let hasMore = true;
+    let result: ListObjectsMetadata[] = [];
+
+    try {
+      while (hasMore) {
+        const listResult = await this.client.send(
+          new ListObjectsV2Command({
+            Bucket: this.bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+          })
+        );
+
+        if (listResult.Contents?.length) {
+          result = result.concat(
+            listResult.Contents.map(r => ({
+              key: r.Key!,
+              lastModified: r.LastModified!,
+              contentLength: r.Size!,
+            }))
+          );
+        }
+
+        // has more items not listed
+        hasMore = !!listResult.IsTruncated;
+        continuationToken = listResult.NextContinuationToken;
+      }
+
+      this.logger.verbose(
+        `List ${result.length} objects with prefix \`${prefix}\``
+      );
+      return result;
+    } catch (e) {
+      this.logger.error(`Failed to list objects with prefix \`${prefix}\``);
+      throw e;
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    try {
+      await this.client.send(
+        new DeleteObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        })
+      );
+
+      this.logger.verbose(`Deleted object \`${key}\``);
+    } catch (e) {
+      this.logger.error(`Failed to delete object \`${key}\``);
+      throw e;
+    }
+  }
+}

packages/backend/server/src/base/throttler/config.ts

@@ -1,27 +1,38 @@
-import { defineStartupConfig, ModuleConfig } from '../config';
+import { defineModuleConfig } from '../config';
 
 export type ThrottlerType = 'default' | 'strict';
 
-type ThrottlerStartupConfigurations = {
-  [key in ThrottlerType]: {
-    ttl: number;
-    limit: number;
-  };
-};
-
-declare module '../config' {
-  interface AppConfig {
-    throttler: ModuleConfig<ThrottlerStartupConfigurations>;
+declare global {
+  interface AppConfigSchema {
+    throttle: {
+      enabled: boolean;
+      throttlers: {
+        [key in ThrottlerType]: ConfigItem<{
+          ttl: number;
+          limit: number;
+        }>;
+      };
+    };
   }
 }
 
-defineStartupConfig('throttler', {
-  default: {
-    ttl: 60,
-    limit: 120,
+defineModuleConfig('throttle', {
+  enabled: {
+    desc: 'Whether the throttler is enabled.',
+    default: true,
   },
-  strict: {
-    ttl: 60,
-    limit: 20,
+  'throttlers.default': {
+    desc: 'The config for the default throttler.',
+    default: {
+      ttl: 60,
+      limit: 120,
+    },
+  },
+  'throttlers.strict': {
+    desc: 'The config for the strict throttler.',
+    default: {
+      ttl: 60,
+      limit: 20,
+    },
   },
 });

packages/backend/server/src/base/throttler/index.ts

@@ -24,14 +24,19 @@ export class ThrottlerStorage extends ThrottlerStorageService {}
 
 @Injectable()
 class CustomOptionsFactory implements ThrottlerOptionsFactory {
-  constructor(private readonly storage: ThrottlerStorage) {}
+  constructor(
+    private readonly config: Config,
+    private readonly storage: ThrottlerStorage
+  ) {}
 
   createThrottlerOptions() {
     const options: ThrottlerModuleOptions = {
-      throttlers: Object.entries(AFFiNE.throttler).map(([name, config]) => ({
-        name,
-        ...config,
-      })),
+      throttlers: Object.entries(this.config.throttle.throttlers).map(
+        ([name, config]) => ({
+          name,
+          ...config,
+        })
+      ),
       storage: this.storage,
     };
 
@@ -84,6 +89,7 @@ export class CloudThrottlerGuard extends ThrottlerGuard {
       ttl,
       blockDuration,
     } = request;
+
     let limit = request.limit;
 
     // give it 'default' if no throttler is specified,
@@ -110,13 +116,9 @@ export class CloudThrottlerGuard extends ThrottlerGuard {
 
     let tracker = await this.getTracker(req);
 
-    if (this.config.node.dev) {
-      limit = Number.MAX_SAFE_INTEGER;
-    } else {
-      // custom limit or ttl APIs will be treated standalone
-      if (limit !== throttlerOptions.limit || ttl !== throttlerOptions.ttl) {
-        tracker += ';custom';
-      }
+    // custom limit or ttl APIs will be treated standalone
+    if (limit !== throttlerOptions.limit || ttl !== throttlerOptions.ttl) {
+      tracker += ';custom';
     }
 
     const key = this.generateKey(
@@ -151,6 +153,10 @@ export class CloudThrottlerGuard extends ThrottlerGuard {
   }
 
   override async canActivate(context: ExecutionContext): Promise<boolean> {
+    if (!this.config.throttle.enabled) {
+      return true;
+    }
+
     const { req } = this.getRequestResponse(context);
 
     const throttler = this.getSpecifiedThrottler(context);

packages/backend/server/src/base/utils/request.ts

@@ -94,7 +94,7 @@ export function parseCookies(
 export type RequestType = GqlContextType | 'event' | 'job';
 
 export function genRequestId(type: RequestType) {
-  return `${AFFiNE.flavor.type}:${type}:${randomUUID()}`;
+  return `${env.DEPLOYMENT_TYPE}:${type}:${randomUUID()}`;
 }
 
 export function getOrGenRequestId(type: RequestType) {

packages/backend/server/src/base/utils/types.ts

@@ -2,9 +2,7 @@ import { Readable } from 'node:stream';
 
 export function ApplyType<T>(): ConstructorOf<T> {
   // @ts-expect-error used to fake the type of config
-  return class Inner implements T {
-    constructor() {}
-  };
+  return class Inner implements T {};
 }
 
 export type PathType<T, Path extends string> =
@@ -30,7 +28,7 @@ export type Join<Prefix, Suffixes> = Prefix extends string | number
 
 export type LeafPaths<
   T,
-  Path extends string = '',
+  Prefix extends string = '',
   MaxDepth extends string = '.....',
   Depth extends string = '',
 > = Depth extends MaxDepth
@@ -40,7 +38,9 @@ export type LeafPaths<
         [K in keyof T]-?: K extends string | number
           ? T[K] extends PrimitiveType
             ? K
-            : Join<K, LeafPaths<T[K], Path, MaxDepth, `${Depth}.`>>
+            : T[K] extends { __leaf: true }
+              ? K
+              : Join<K, LeafPaths<T[K], Prefix, MaxDepth, `${Depth}.`>>
           : never;
       }[keyof T]
     : never;

packages/backend/server/src/base/websocket/adapter.ts

@@ -1,7 +1,7 @@
 import { INestApplication } from '@nestjs/common';
 import { IoAdapter } from '@nestjs/platform-socket.io';
 import { createAdapter } from '@socket.io/redis-adapter';
-import { Server } from 'socket.io';
+import { Server, Socket } from 'socket.io';
 
 import { Config } from '../config';
 import { AuthenticationRequired } from '../error';
@@ -14,7 +14,9 @@ export class SocketIoAdapter extends IoAdapter {
   }
 
   override createIOServer(port: number, options?: any): Server {
-    const config = this.app.get(WEBSOCKET_OPTIONS) as Config['websocket'];
+    const config = this.app.get(WEBSOCKET_OPTIONS) as Config['websocket'] & {
+      canActivate: (socket: Socket) => Promise<boolean>;
+    };
     const server: Server = super.createIOServer(port, {
       ...config,
       ...options,
@@ -22,7 +24,6 @@ export class SocketIoAdapter extends IoAdapter {
 
     if (config.canActivate) {
       server.use((socket, next) => {
-        // @ts-expect-error checked
         config
           .canActivate(socket)
           .then(pass => {

packages/backend/server/src/base/websocket/config.ts

@@ -1,20 +1,34 @@
 import { GatewayMetadata } from '@nestjs/websockets';
-import { Socket } from 'socket.io';
+import { z } from 'zod';
 
-import { defineStartupConfig, ModuleConfig } from '../config';
+import { defineModuleConfig } from '../config';
 
-declare module '../config' {
-  interface AppConfig {
-    websocket: ModuleConfig<
-      GatewayMetadata & {
-        canActivate?: (socket: Socket) => Promise<boolean>;
-      }
-    >;
+declare global {
+  interface AppConfigSchema {
+    websocket: {
+      transports: ConfigItem<GatewayMetadata['transports']>;
+      maxHttpBufferSize: number;
+    };
   }
 }
 
-defineStartupConfig('websocket', {
-  transports: ['websocket', 'polling'],
-  // see: https://socket.io/docs/v4/server-options/#maxhttpbuffersize
-  maxHttpBufferSize: 1e8, // 100 MB
+defineModuleConfig('websocket', {
+  transports: {
+    desc: 'The enabled transports for accepting websocket traffics.',
+    default: ['websocket', 'polling'],
+    shape: z.array(z.enum(['websocket', 'polling'])),
+    schema: {
+      type: 'array',
+      items: {
+        type: 'string',
+        enum: ['websocket', 'polling'],
+      },
+    },
+    link: 'https://docs.nestjs.com/websockets/gateways#transports',
+  },
+  maxHttpBufferSize: {
+    desc: 'How many bytes or characters a message can be, before closing the session (to avoid DoS).',
+    default: 1e8, // 100 MB
+    shape: z.number().int().positive(),
+  },
 });