refactor(server): config system (#11081)

This commit is contained in:
forehalo
2025-03-27 12:32:28 +00:00
parent 7091111f85
commit 0ea38680fa
274 changed files with 7583 additions and 5841 deletions
@@ -0,0 +1,90 @@
import test from 'ava';
import { createModule } from '../../../__tests__/create-module';
import { ConfigFactory, ConfigModule } from '..';
import { Config } from '../config';
const module = await createModule();
test.after.always(async () => {
await module.close();
});
test('should create config', t => {
const config = module.get(Config);
t.is(typeof config.auth.passwordRequirements.max, 'number');
t.is(typeof config.job.queue, 'object');
});
test('should override config', async t => {
await using module = await createModule({
imports: [
ConfigModule.override({
auth: {
passwordRequirements: {
max: 100,
min: 6,
},
},
job: {
queues: {
notification: {
concurrency: 1000,
},
},
},
}),
],
});
const config = module.get(Config);
const configFactory = module.get(ConfigFactory);
t.deepEqual(config.auth.passwordRequirements, {
max: 100,
min: 6,
});
configFactory.override({
auth: {
passwordRequirements: {
max: 10,
},
},
});
t.deepEqual(config.auth.passwordRequirements, {
max: 10,
min: 6,
});
});
test('should validate config', t => {
const config = module.get(ConfigFactory);
t.notThrows(() =>
config.validate([
{
module: 'auth',
key: 'passwordRequirements',
value: { max: 10, min: 6 },
},
])
);
t.throws(
() =>
config.validate([
{
module: 'auth',
key: 'passwordRequirements',
value: { max: 10, min: 10 },
},
]),
{
message: `Invalid config for module [auth] with key [passwordRequirements]
Value: {"max":10,"min":10}
Error: Minimum length of password must be less than maximum length`,
}
);
});
@@ -0,0 +1,3 @@
import { ApplyType } from '../utils';
export class Config extends ApplyType<AppConfig>() {}
@@ -1,55 +0,0 @@
import type { LeafPaths } from '../utils/types';
import { AppStartupConfig } from './types';
export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean';
export type ServerFlavor =
| 'allinone'
| 'graphql'
| 'sync'
| 'renderer'
| 'doc'
| 'script';
export type AFFINE_ENV = 'dev' | 'beta' | 'production';
export type NODE_ENV = 'development' | 'test' | 'production';
export enum DeploymentType {
Affine = 'affine',
Selfhosted = 'selfhosted',
}
export type ConfigPaths = LeafPaths<AppStartupConfig, '', '......'>;
export interface PreDefinedAFFiNEConfig {
ENV_MAP: Record<string, ConfigPaths | [ConfigPaths, EnvConfigType?]>;
serverId: string;
serverName: string;
readonly projectRoot: string;
readonly AFFINE_ENV: AFFINE_ENV;
readonly NODE_ENV: NODE_ENV;
readonly version: string;
readonly type: DeploymentType;
readonly isSelfhosted: boolean;
readonly flavor: { type: string } & { [key in ServerFlavor]: boolean };
readonly affine: { canary: boolean; beta: boolean; stable: boolean };
readonly node: {
prod: boolean;
dev: boolean;
test: boolean;
};
readonly deploy: boolean;
}
export interface AppPluginsConfig {}
export type AFFiNEConfig = PreDefinedAFFiNEConfig &
AppStartupConfig &
AppPluginsConfig;
declare global {
// oxlint-disable-next-line @typescript-eslint/no-namespace
namespace globalThis {
// oxlint-disable-next-line no-var
var AFFiNE: AFFiNEConfig;
}
}
@@ -1,142 +0,0 @@
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import pkg from '../../../package.json' with { type: 'json' };
import {
AFFINE_ENV,
AFFiNEConfig,
DeploymentType,
NODE_ENV,
PreDefinedAFFiNEConfig,
ServerFlavor,
} from './def';
import { readEnv } from './env';
import { defaultStartupConfig } from './register';
function expectFlavor(flavor: ServerFlavor, expected: ServerFlavor) {
return flavor === expected || flavor === 'allinone';
}
function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
'development',
'test',
'production',
]);
const AFFINE_ENV = readEnv<AFFINE_ENV>('AFFINE_ENV', 'production', [
'dev',
'beta',
'production',
]);
const flavor = readEnv<ServerFlavor>('SERVER_FLAVOR', 'allinone', [
'allinone',
'graphql',
'sync',
'renderer',
'doc',
'script',
]);
const deploymentType = readEnv<DeploymentType>(
'DEPLOYMENT_TYPE',
NODE_ENV === 'development'
? DeploymentType.Affine
: DeploymentType.Selfhosted,
Object.values(DeploymentType)
);
const isSelfhosted = deploymentType === DeploymentType.Selfhosted;
const affine = {
canary: AFFINE_ENV === 'dev',
beta: AFFINE_ENV === 'beta',
stable: AFFINE_ENV === 'production',
};
const node = {
prod: NODE_ENV === 'production',
dev: NODE_ENV === 'development',
test: NODE_ENV === 'test',
};
return {
ENV_MAP: {},
NODE_ENV,
AFFINE_ENV,
serverId: 'some-randome-uuid',
serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud',
version: pkg.version,
type: deploymentType,
isSelfhosted,
flavor: {
type: flavor,
allinone: flavor === 'allinone',
graphql: expectFlavor(flavor, 'graphql'),
sync: expectFlavor(flavor, 'sync'),
renderer: expectFlavor(flavor, 'renderer'),
doc: expectFlavor(flavor, 'doc'),
script: expectFlavor(flavor, 'script'),
},
affine,
node,
deploy: !node.dev && !node.test,
projectRoot: resolve(fileURLToPath(import.meta.url), '../../../../'),
};
}
export function getAFFiNEConfigModifier(): AFFiNEConfig {
const predefined = getPredefinedAFFiNEConfig() as AFFiNEConfig;
return chainableProxy(predefined);
}
function merge(a: any, b: any) {
if (typeof b !== 'object' || b instanceof Map || b instanceof Set) {
return b;
}
if (Array.isArray(b)) {
if (Array.isArray(a)) {
return a.concat(b);
}
return b;
}
const result = { ...a };
Object.keys(b).forEach(key => {
result[key] = merge(result[key], b[key]);
});
return result;
}
export function mergeConfigOverride(override: any) {
return merge(defaultStartupConfig, override);
}
function chainableProxy(obj: any) {
const keys: Set<string> = new Set(Object.keys(obj));
return new Proxy(obj, {
get(target, prop) {
if (!(prop in target)) {
keys.add(prop as string);
target[prop] = chainableProxy({});
}
return target[prop];
},
set(target, prop, value) {
keys.add(prop as string);
if (
typeof value === 'object' &&
!(
value instanceof Map ||
value instanceof Set ||
value instanceof Array
)
) {
value = chainableProxy(value);
}
target[prop] = value;
return true;
},
ownKeys() {
return Array.from(keys);
},
});
}
+3 -40
View File
@@ -1,11 +1,9 @@
import { set } from 'lodash-es';
import type { AFFiNEConfig, EnvConfigType } from './def';
export type EnvConfigType = 'string' | 'integer' | 'float' | 'boolean';
/**
* parse number value from environment variables
*/
function int(value: string) {
function integer(value: string) {
const n = parseInt(value);
return Number.isNaN(n) ? undefined : n;
}
@@ -20,7 +18,7 @@ function boolean(value: string) {
}
const envParsers: Record<EnvConfigType, (value: string) => unknown> = {
int,
integer,
float,
boolean,
string: value => value,
@@ -33,38 +31,3 @@ export function parseEnvValue(value: string | undefined, type: EnvConfigType) {
return envParsers[type](value);
}
export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
for (const env in rawConfig.ENV_MAP) {
const config = rawConfig.ENV_MAP[env];
const [path, value] =
typeof config === 'string'
? [config, parseEnvValue(process.env[env], 'string')]
: [config[0], parseEnvValue(process.env[env], config[1] ?? 'string')];
if (value !== undefined) {
set(rawConfig, path, value);
}
}
}
export function readEnv<T>(
env: string,
defaultValue: T,
availableValues?: T[]
) {
const value = process.env[env];
if (value === undefined) {
return defaultValue;
}
if (availableValues && !availableValues.includes(value as any)) {
throw new Error(
`Invalid value '${value}' for environment variable ${env}, expected one of [${availableValues.join(
', '
)}]`
);
}
return value as T;
}
@@ -0,0 +1,60 @@
import { Inject, Injectable, Optional } from '@nestjs/common';
import { merge } from 'lodash-es';
import { InvalidAppConfig } from '../error';
import { APP_CONFIG_DESCRIPTORS, getDefaultConfig } from './register';
export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN');
@Injectable()
export class ConfigFactory {
readonly #config: DeepReadonly<AppConfig>;
constructor(
@Inject(OVERRIDE_CONFIG_TOKEN)
@Optional()
private readonly overrides: DeepPartial<AppConfig> = {}
) {
this.#config = this.loadDefault();
}
get config() {
return this.#config;
}
override(updates: DeepPartial<AppConfig>) {
merge(this.#config, updates);
}
validate(updates: Array<{ module: string; key: string; value: any }>) {
const errors: string[] = [];
updates.forEach(update => {
const descriptor = APP_CONFIG_DESCRIPTORS[update.module]?.[update.key];
if (!descriptor) {
errors.push(
`Invalid config for module [${update.module}] with unknown key [${update.key}]`
);
return;
}
const { success, error } = descriptor.validate(update.value);
if (!success) {
error.issues.forEach(issue => {
errors.push(`Invalid config for module [${update.module}] with key [${update.key}]
Value: ${JSON.stringify(update.value)}
Error: ${issue.message}`);
});
}
});
if (errors.length > 0) {
throw new InvalidAppConfig(errors.join('\n'));
}
}
private loadDefault(): DeepReadonly<AppConfig> {
const config = getDefaultConfig();
return merge(config, this.overrides);
}
}
@@ -1,37 +1,29 @@
import { DynamicModule, FactoryProvider } from '@nestjs/common';
import { merge } from 'lodash-es';
import { DynamicModule, Global, Module, Provider } from '@nestjs/common';
import { AFFiNEConfig } from './def';
import { Config } from './provider';
export * from './def';
export * from './default';
export { applyEnvToConfig, parseEnvValue } from './env';
export * from './provider';
export { defineRuntimeConfig, defineStartupConfig } from './register';
export type { AppConfig, ConfigItem, ModuleConfig } from './types';
function createConfigProvider(
override?: DeepPartial<Config>
): FactoryProvider<Config> {
return {
provide: Config,
useFactory: () => {
return Object.freeze(merge({}, globalThis.AFFiNE, override));
},
inject: [],
};
}
import { Config } from './config';
import { ConfigFactory, OVERRIDE_CONFIG_TOKEN } from './factory';
import { ConfigProvider } from './provider';
@Global()
@Module({
providers: [ConfigProvider, ConfigFactory],
exports: [ConfigProvider, ConfigFactory],
})
export class ConfigModule {
static forRoot = (override?: DeepPartial<AFFiNEConfig>): DynamicModule => {
const provider = createConfigProvider(override);
static override(overrides: DeepPartial<AppConfigSchema> = {}): DynamicModule {
const provider: Provider = {
provide: OVERRIDE_CONFIG_TOKEN,
useValue: overrides,
};
return {
global: true,
module: ConfigModule,
module: class ConfigOverrideModule {},
providers: [provider],
exports: [provider],
};
};
}
}
export { Config, ConfigFactory };
export { defineModuleConfig, type JSONSchema } from './register';
@@ -1,16 +1,12 @@
import { ApplyType } from '../utils/types';
import { AFFiNEConfig } from './def';
import { FactoryProvider } from '@nestjs/common';
/**
* @example
*
* import { Config } from '@affine/server'
*
* class TestConfig {
* constructor(private readonly config: Config) {}
* test() {
* return this.config.env
* }
* }
*/
export class Config extends ApplyType<AFFiNEConfig>() {}
import { Config } from './config';
import { ConfigFactory } from './factory';
export const ConfigProvider: FactoryProvider = {
provide: Config,
useFactory: (factory: ConfigFactory) => {
return factory.config;
},
inject: [ConfigFactory],
};
@@ -1,66 +1,239 @@
import { Prisma, RuntimeConfigType } from '@prisma/client';
import { get, merge, set } from 'lodash-es';
import { once, set } from 'lodash-es';
import { z } from 'zod';
import {
AppModulesConfigDef,
AppStartupConfig,
ModuleRuntimeConfigDescriptions,
ModuleStartupConfigDescriptions,
} from './types';
import { type EnvConfigType, parseEnvValue } from './env';
import { AppConfigByPath } from './types';
export const defaultStartupConfig: AppStartupConfig = {} as any;
export const defaultRuntimeConfig: Record<
string,
Prisma.RuntimeConfigCreateInput
> = {} as any;
export type JSONSchema = { description?: string } & (
| { type?: undefined; oneOf?: JSONSchema[] }
| {
type: 'string' | 'number' | 'boolean';
enum?: string[];
}
| {
type: 'array';
items?: JSONSchema;
}
| {
type: 'object';
properties?: Record<string, JSONSchema>;
}
);
export function runtimeConfigType(val: any): RuntimeConfigType {
if (Array.isArray(val)) {
return RuntimeConfigType.Array;
}
type ConfigType = EnvConfigType | 'array' | 'object' | 'any';
export type ConfigDescriptor<T> = {
desc: string;
type: ConfigType;
validate: (value: T) => z.SafeParseReturnType<T, T>;
schema: JSONSchema;
default: T;
env?: [string, EnvConfigType];
link?: string;
};
switch (typeof val) {
case 'string':
return RuntimeConfigType.String;
case 'number':
return RuntimeConfigType.Number;
case 'boolean':
return RuntimeConfigType.Boolean;
type ConfigDefineDescriptor<T> = {
desc: string;
default: T;
validate?: (value: T) => boolean;
shape?: z.ZodType<T>;
env?: string | [string, EnvConfigType];
link?: string;
schema?: JSONSchema;
};
function typeFromShape(shape: z.ZodType<any>): ConfigType {
switch (shape.constructor) {
case z.ZodString:
return 'string';
case z.ZodNumber:
return 'float';
case z.ZodBoolean:
return 'boolean';
case z.ZodArray:
return 'array';
case z.ZodObject:
return 'object';
default:
return RuntimeConfigType.Object;
return 'any';
}
}
function registerRuntimeConfig<T extends keyof AppModulesConfigDef>(
module: T,
configs: ModuleRuntimeConfigDescriptions<T>
) {
Object.entries(configs).forEach(([key, value]) => {
defaultRuntimeConfig[`${module}/${key}`] = {
id: `${module}/${key}`,
function shapeFromType(type: ConfigType): z.ZodType<any> {
switch (type) {
case 'string':
return z.string();
case 'float':
return z.number();
case 'boolean':
return z.boolean();
case 'integer':
return z.number().int();
case 'array':
return z.array(z.any());
case 'object':
return z.object({});
default:
return z.any();
}
}
function typeFromSchema(schema: JSONSchema): ConfigType {
if ('type' in schema) {
switch (schema.type) {
case 'string':
return 'string';
case 'number':
return 'float';
case 'boolean':
return 'boolean';
case 'array':
return 'array';
case 'object':
return 'object';
}
}
return 'any';
}
function schemaFromType(type: ConfigType): JSONSchema['type'] {
switch (type) {
case 'any':
return undefined;
case 'float':
case 'integer':
return 'number';
default:
return type;
}
}
function typeFromDefault<T>(defaultValue: T): ConfigType {
if (Array.isArray(defaultValue)) {
return 'array';
}
switch (typeof defaultValue) {
case 'string':
return 'string';
case 'number':
return 'float';
case 'boolean':
return 'boolean';
case 'object':
return 'object';
default:
return 'any';
}
}
function standardizeDescriptor<T>(
desc: ConfigDefineDescriptor<T>
): ConfigDescriptor<T> {
const env = desc.env
? Array.isArray(desc.env)
? desc.env
: ([desc.env, 'string'] as [string, EnvConfigType])
: undefined;
let type: ConfigType = 'any';
if (desc.default !== undefined && desc.default !== null) {
type = typeFromDefault(desc.default);
} else if (env) {
type = env[1];
} else if (desc.shape) {
type = typeFromShape(desc.shape);
} else if (desc.schema) {
type = typeFromSchema(desc.schema);
}
const shape = desc.shape ?? shapeFromType(type);
return {
desc: desc.desc,
default: desc.default,
type,
validate: (value: T) => {
return shape.safeParse(value);
},
env,
link: desc.link,
schema: {
type: schemaFromType(type),
description: desc.desc,
...desc.schema,
},
};
}
type ModuleConfigDescriptors<T> = {
[K in keyof T]: ConfigDefineDescriptor<T[K]>;
};
export const APP_CONFIG_DESCRIPTORS: Record<
string,
Record<string, ConfigDescriptor<any>>
> = {};
export const getDescriptors = once(() => {
return Object.entries(APP_CONFIG_DESCRIPTORS).map(
([module, descriptors]) => ({
module,
key,
description: value.desc,
value: value.default,
type: runtimeConfigType(value.default),
};
});
}
export function defineStartupConfig<T extends keyof AppModulesConfigDef>(
module: T,
configs: ModuleStartupConfigDescriptions<AppModulesConfigDef[T]>
) {
set(
defaultStartupConfig,
module,
merge(get(defaultStartupConfig, module, {}), configs)
descriptors: Object.entries(descriptors).map(([key, descriptor]) => ({
key,
descriptor,
})),
})
);
});
export function defineModuleConfig<T extends keyof AppConfigSchema>(
module: T,
defs: ModuleConfigDescriptors<AppConfigByPath<T>>
) {
const descriptors: Record<string, ConfigDescriptor<any>> = {};
Object.entries(defs).forEach(([key, desc]) => {
descriptors[key] = standardizeDescriptor(
desc as ConfigDefineDescriptor<any>
);
});
APP_CONFIG_DESCRIPTORS[module] = {
...APP_CONFIG_DESCRIPTORS[module],
...descriptors,
};
}
export function defineRuntimeConfig<T extends keyof AppModulesConfigDef>(
module: T,
configs: ModuleRuntimeConfigDescriptions<T>
) {
registerRuntimeConfig(module, configs);
export function getDefaultConfig(): AppConfigSchema {
const config: Record<string, any> = {};
const envs = process.env;
for (const [module, defs] of Object.entries(APP_CONFIG_DESCRIPTORS)) {
const modulizedConfig = {};
for (const [key, desc] of Object.entries(defs)) {
let defaultValue = desc.default;
if (desc.env) {
const [env, parser] = desc.env;
const envValue = envs[env];
if (envValue) {
defaultValue = parseEnvValue(envValue, parser);
}
}
const { success, error } = desc.validate(defaultValue);
if (!success) {
throw error;
}
set(modulizedConfig, key, defaultValue);
}
config[module] = modulizedConfig;
}
return config as AppConfigSchema;
}
+14 -121
View File
@@ -1,127 +1,20 @@
import { Join, PathType } from '../utils/types';
import { LeafPaths, PathType } from '../utils';
export type ConfigItem<T> = T & { __type: 'ConfigItem' };
type ConfigDef = Record<string, any> | never;
export interface ModuleConfig<
Startup extends ConfigDef = never,
Runtime extends ConfigDef = never,
> {
startup: Startup;
runtime: Runtime;
declare global {
type ConfigItem<T> = Leaf<T>;
interface AppConfigSchema {}
type AppConfig = DeeplyEraseLeaf<AppConfigSchema>;
}
export type RuntimeConfigDescription<T> = {
desc: string;
default: T;
};
type ConfigItemLeaves<T, P extends string = ''> =
T extends Record<string, any>
export type AppConfigByPath<Module extends keyof AppConfigSchema> =
AppConfigSchema[Module] extends infer Config
? {
[K in keyof T]: K extends string
? T[K] extends { __type: 'ConfigItem' }
? K
: T[K] extends PrimitiveType
? K
: Join<K, ConfigItemLeaves<T[K], P>>
: never;
}[keyof T]
: never;
type StartupConfigDescriptions<T extends ConfigDef> = {
[K in keyof T]: T[K] extends Record<string, any>
? T[K] extends ConfigItem<infer V>
? V
: T[K]
: T[K];
};
type ModuleConfigLeaves<T, P extends string = ''> =
T extends Record<string, any>
? {
[K in keyof T]: K extends string
? T[K] extends ModuleConfig<any, any>
? K
: Join<K, ModuleConfigLeaves<T[K], P>>
: never;
}[keyof T]
: never;
type FlattenModuleConfigs<T extends Record<string, any>> = {
// @ts-expect-error allow
[K in ModuleConfigLeaves<T>]: PathType<T, K>;
};
type _AppStartupConfig<T extends Record<string, any>> = {
[K in keyof T]: T[K] extends ModuleConfig<infer S, any>
? S
: _AppStartupConfig<T[K]>;
};
// for extending
export interface AppConfig {}
export type AppModulesConfigDef = FlattenModuleConfigs<AppConfig>;
export type AppConfigModules = keyof AppModulesConfigDef;
export type AppStartupConfig = _AppStartupConfig<AppConfig>;
// app runtime config keyed by module names
export type AppRuntimeConfigByModules = {
[Module in keyof AppModulesConfigDef]: AppModulesConfigDef[Module] extends ModuleConfig<
any,
infer Runtime
>
? Runtime extends never
? never
: {
// @ts-expect-error allow
[K in ConfigItemLeaves<Runtime>]: PathType<
Runtime,
K
> extends infer Config
? Config extends ConfigItem<infer V>
[Path in LeafPaths<Config>]: Path extends string
? PathType<Config, Path> extends infer Item
? Item extends Leaf<infer V>
? V
: Config
: never;
}
: Item
: never
: never;
}
: never;
};
// names of modules that have runtime config
export type AppRuntimeConfigModules = {
[Module in keyof AppRuntimeConfigByModules]: AppRuntimeConfigByModules[Module] extends never
? never
: Module;
}[keyof AppRuntimeConfigByModules];
// runtime config keyed by module names flattened into config names
// { auth: { allowSignup: boolean } } => { 'auth/allowSignup': boolean }
export type FlattenedAppRuntimeConfig = UnionToIntersection<
{
[Module in keyof AppRuntimeConfigByModules]: AppModulesConfigDef[Module] extends never
? never
: {
[K in keyof AppRuntimeConfigByModules[Module] as K extends string
? `${Module}/${K}`
: never]: AppRuntimeConfigByModules[Module][K];
};
}[keyof AppRuntimeConfigByModules]
>;
export type ModuleStartupConfigDescriptions<T extends ModuleConfig<any, any>> =
T extends ModuleConfig<infer S, any>
? S extends never
? undefined
: StartupConfigDescriptions<S>
: never;
export type ModuleRuntimeConfigDescriptions<
Module extends keyof AppRuntimeConfigByModules,
> = AppModulesConfigDef[Module] extends never
? never
: {
[K in keyof AppRuntimeConfigByModules[Module]]: RuntimeConfigDescription<
AppRuntimeConfigByModules[Module][K]
>;
};