chore: rename fundamentals to base (#9119)

This commit is contained in:
forehalo
2024-12-13 06:27:12 +00:00
parent 8c24f2b906
commit 4c23991047
185 changed files with 183 additions and 193 deletions
@@ -0,0 +1,49 @@
import type { LeafPaths } from '../utils/types';
import { AppStartupConfig } from './types';
export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean';
export type ServerFlavor = 'allinone' | 'graphql' | 'sync' | 'renderer';
export type AFFINE_ENV = 'dev' | 'beta' | 'production';
export type NODE_ENV = 'development' | 'test' | 'production' | 'script';
export enum DeploymentType {
Affine = 'affine',
Selfhosted = 'selfhosted',
}
export type ConfigPaths = LeafPaths<AppStartupConfig, '', '......'>;
export interface PreDefinedAFFiNEConfig {
ENV_MAP: Record<string, ConfigPaths | [ConfigPaths, EnvConfigType?]>;
serverId: string;
serverName: string;
readonly projectRoot: string;
readonly AFFINE_ENV: AFFINE_ENV;
readonly NODE_ENV: NODE_ENV;
readonly version: string;
readonly type: DeploymentType;
readonly isSelfhosted: boolean;
readonly flavor: { type: string } & { [key in ServerFlavor]: boolean };
readonly affine: { canary: boolean; beta: boolean; stable: boolean };
readonly node: {
prod: boolean;
dev: boolean;
test: boolean;
script: boolean;
};
readonly deploy: boolean;
}
export interface AppPluginsConfig {}
export type AFFiNEConfig = PreDefinedAFFiNEConfig &
AppStartupConfig &
AppPluginsConfig;
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace globalThis {
// eslint-disable-next-line no-var
var AFFiNE: AFFiNEConfig;
}
}
@@ -0,0 +1,136 @@
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import pkg from '../../../package.json' with { type: 'json' };
import {
AFFINE_ENV,
AFFiNEConfig,
DeploymentType,
NODE_ENV,
PreDefinedAFFiNEConfig,
ServerFlavor,
} from './def';
import { readEnv } from './env';
import { defaultStartupConfig } from './register';
function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig {
const NODE_ENV = readEnv<NODE_ENV>('NODE_ENV', 'production', [
'development',
'test',
'production',
'script',
]);
const AFFINE_ENV = readEnv<AFFINE_ENV>('AFFINE_ENV', 'dev', [
'dev',
'beta',
'production',
]);
const flavor = readEnv<ServerFlavor>('SERVER_FLAVOR', 'allinone', [
'allinone',
'graphql',
'sync',
'renderer',
]);
const deploymentType = readEnv<DeploymentType>(
'DEPLOYMENT_TYPE',
NODE_ENV === 'development'
? DeploymentType.Affine
: DeploymentType.Selfhosted,
Object.values(DeploymentType)
);
const isSelfhosted = deploymentType === DeploymentType.Selfhosted;
const affine = {
canary: AFFINE_ENV === 'dev',
beta: AFFINE_ENV === 'beta',
stable: AFFINE_ENV === 'production',
};
const node = {
prod: NODE_ENV === 'production',
dev: NODE_ENV === 'development',
test: NODE_ENV === 'test',
script: NODE_ENV === 'script',
};
return {
ENV_MAP: {},
NODE_ENV,
AFFINE_ENV,
serverId: 'some-randome-uuid',
serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud',
version: pkg.version,
type: deploymentType,
isSelfhosted,
flavor: {
type: flavor,
allinone: flavor === 'allinone',
graphql: flavor === 'graphql' || flavor === 'allinone',
sync: flavor === 'sync' || flavor === 'allinone',
renderer: flavor === 'renderer' || flavor === 'allinone',
},
affine,
node,
deploy: !node.dev && !node.test,
projectRoot: resolve(fileURLToPath(import.meta.url), '../../../../'),
};
}
export function getAFFiNEConfigModifier(): AFFiNEConfig {
const predefined = getPredefinedAFFiNEConfig() as AFFiNEConfig;
return chainableProxy(predefined);
}
function merge(a: any, b: any) {
if (typeof b !== 'object' || b instanceof Map || b instanceof Set) {
return b;
}
if (Array.isArray(b)) {
if (Array.isArray(a)) {
return a.concat(b);
}
return b;
}
const result = { ...a };
Object.keys(b).forEach(key => {
result[key] = merge(result[key], b[key]);
});
return result;
}
export function mergeConfigOverride(override: any) {
return merge(defaultStartupConfig, override);
}
function chainableProxy(obj: any) {
const keys: Set<string> = new Set(Object.keys(obj));
return new Proxy(obj, {
get(target, prop) {
if (!(prop in target)) {
keys.add(prop as string);
target[prop] = chainableProxy({});
}
return target[prop];
},
set(target, prop, value) {
keys.add(prop as string);
if (
typeof value === 'object' &&
!(
value instanceof Map ||
value instanceof Set ||
value instanceof Array
)
) {
value = chainableProxy(value);
}
target[prop] = value;
return true;
},
ownKeys() {
return Array.from(keys);
},
});
}
@@ -0,0 +1,70 @@
import { set } from 'lodash-es';
import type { AFFiNEConfig, EnvConfigType } from './def';
/**
* parse number value from environment variables
*/
function int(value: string) {
const n = parseInt(value);
return Number.isNaN(n) ? undefined : n;
}
function float(value: string) {
const n = parseFloat(value);
return Number.isNaN(n) ? undefined : n;
}
function boolean(value: string) {
return value === '1' || value.toLowerCase() === 'true';
}
const envParsers: Record<EnvConfigType, (value: string) => unknown> = {
int,
float,
boolean,
string: value => value,
};
export function parseEnvValue(value: string | undefined, type: EnvConfigType) {
if (value === undefined) {
return;
}
return envParsers[type](value);
}
export function applyEnvToConfig(rawConfig: AFFiNEConfig) {
for (const env in rawConfig.ENV_MAP) {
const config = rawConfig.ENV_MAP[env];
const [path, value] =
typeof config === 'string'
? [config, parseEnvValue(process.env[env], 'string')]
: [config[0], parseEnvValue(process.env[env], config[1] ?? 'string')];
if (value !== undefined) {
set(rawConfig, path, value);
}
}
}
export function readEnv<T>(
env: string,
defaultValue: T,
availableValues?: T[]
) {
const value = process.env[env];
if (value === undefined) {
return defaultValue;
}
if (availableValues && !availableValues.includes(value as any)) {
throw new Error(
`Invalid value '${value}' for environment variable ${env}, expected one of [${availableValues.join(
', '
)}]`
);
}
return value as T;
}
@@ -0,0 +1,40 @@
import { DynamicModule, FactoryProvider } from '@nestjs/common';
import { merge } from 'lodash-es';
import { AFFiNEConfig } from './def';
import { Config } from './provider';
import { Runtime } from './runtime/service';
export * from './def';
export * from './default';
export { applyEnvToConfig, parseEnvValue } from './env';
export * from './provider';
export { defineRuntimeConfig, defineStartupConfig } from './register';
export type { AppConfig, ConfigItem, ModuleConfig } from './types';
function createConfigProvider(
override?: DeepPartial<Config>
): FactoryProvider<Config> {
return {
provide: Config,
useFactory: (runtime: Runtime) => {
return Object.freeze(merge({}, globalThis.AFFiNE, override, { runtime }));
},
inject: [Runtime],
};
}
export class ConfigModule {
static forRoot = (override?: DeepPartial<AFFiNEConfig>): DynamicModule => {
const provider = createConfigProvider(override);
return {
global: true,
module: ConfigModule,
providers: [provider, Runtime],
exports: [provider],
};
};
}
export { Runtime };
@@ -0,0 +1,19 @@
import { ApplyType } from '../utils/types';
import { AFFiNEConfig } from './def';
import type { Runtime } from './runtime/service';
/**
* @example
*
* import { Config } from '@affine/server'
*
* class TestConfig {
* constructor(private readonly config: Config) {}
* test() {
* return this.config.env
* }
* }
*/
export class Config extends ApplyType<AFFiNEConfig>() {
runtime!: Runtime;
}
@@ -0,0 +1,66 @@
import { Prisma, RuntimeConfigType } from '@prisma/client';
import { get, merge, set } from 'lodash-es';
import {
AppModulesConfigDef,
AppStartupConfig,
ModuleRuntimeConfigDescriptions,
ModuleStartupConfigDescriptions,
} from './types';
export const defaultStartupConfig: AppStartupConfig = {} as any;
export const defaultRuntimeConfig: Record<
string,
Prisma.RuntimeConfigCreateInput
> = {} as any;
export function runtimeConfigType(val: any): RuntimeConfigType {
if (Array.isArray(val)) {
return RuntimeConfigType.Array;
}
switch (typeof val) {
case 'string':
return RuntimeConfigType.String;
case 'number':
return RuntimeConfigType.Number;
case 'boolean':
return RuntimeConfigType.Boolean;
default:
return RuntimeConfigType.Object;
}
}
function registerRuntimeConfig<T extends keyof AppModulesConfigDef>(
module: T,
configs: ModuleRuntimeConfigDescriptions<T>
) {
Object.entries(configs).forEach(([key, value]) => {
defaultRuntimeConfig[`${module}/${key}`] = {
id: `${module}/${key}`,
module,
key,
description: value.desc,
value: value.default,
type: runtimeConfigType(value.default),
};
});
}
export function defineStartupConfig<T extends keyof AppModulesConfigDef>(
module: T,
configs: ModuleStartupConfigDescriptions<AppModulesConfigDef[T]>
) {
set(
defaultStartupConfig,
module,
merge(get(defaultStartupConfig, module, {}), configs)
);
}
export function defineRuntimeConfig<T extends keyof AppModulesConfigDef>(
module: T,
configs: ModuleRuntimeConfigDescriptions<T>
) {
registerRuntimeConfig(module, configs);
}
@@ -0,0 +1,22 @@
import { OnEvent } from '../../event';
import { Payload } from '../../event/def';
import { FlattenedAppRuntimeConfig } from '../types';
declare module '../../event/def' {
interface EventDefinitions {
runtimeConfig: {
[K in keyof FlattenedAppRuntimeConfig]: {
changed: Payload<FlattenedAppRuntimeConfig[K]>;
};
};
}
}
/**
* not implemented yet
*/
export const OnRuntimeConfigChange_DO_NOT_USE = (
nameWithModule: keyof FlattenedAppRuntimeConfig
) => {
return OnEvent(`runtimeConfig.${nameWithModule}.changed`);
};
@@ -0,0 +1,250 @@
import {
forwardRef,
Inject,
Injectable,
Logger,
OnModuleInit,
} from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { difference, keyBy } from 'lodash-es';
import { Cache } from '../../cache';
import { InvalidRuntimeConfigType, RuntimeConfigNotFound } from '../../error';
import { defer } from '../../utils/promise';
import { defaultRuntimeConfig, runtimeConfigType } from '../register';
import { AppRuntimeConfigModules, FlattenedAppRuntimeConfig } from '../types';
function validateConfigType<K extends keyof FlattenedAppRuntimeConfig>(
key: K,
value: any
) {
const config = defaultRuntimeConfig[key];
if (!config) {
throw new RuntimeConfigNotFound({ key });
}
const want = config.type;
const get = runtimeConfigType(value);
if (get !== want) {
throw new InvalidRuntimeConfigType({
key,
want,
get,
});
}
}
/**
* runtime.fetch(k) // v1
* runtime.fetchAll(k1, k2, k3) // [v1, v2, v3]
* runtime.set(k, v)
* runtime.update(k, (v) => {
* v.xxx = 'yyy';
* return v
* })
*/
@Injectable()
export class Runtime implements OnModuleInit {
private readonly logger = new Logger('App:RuntimeConfig');
constructor(
private readonly db: PrismaClient,
// circular deps: runtime => cache => redis(maybe) => config => runtime
@Inject(forwardRef(() => Cache)) private readonly cache: Cache
) {}
async onModuleInit() {
await this.upgradeDB();
}
async fetch<K extends keyof FlattenedAppRuntimeConfig>(
k: K
): Promise<FlattenedAppRuntimeConfig[K]> {
const cached = await this.loadCache<K>(k);
if (cached !== undefined) {
return cached;
}
const dbValue = await this.loadDb<K>(k);
if (dbValue === undefined) {
throw new RuntimeConfigNotFound({ key: k });
}
await this.setCache(k, dbValue);
return dbValue;
}
async fetchAll<
Selector extends { [Key in keyof FlattenedAppRuntimeConfig]?: true },
>(
selector: Selector
): Promise<{
// @ts-expect-error allow
[Key in keyof Selector]: FlattenedAppRuntimeConfig[Key];
}> {
const keys = Object.keys(selector);
if (keys.length === 0) {
return {} as any;
}
const records = await this.db.runtimeConfig.findMany({
select: {
id: true,
value: true,
},
where: {
id: {
in: keys,
},
deletedAt: null,
},
});
const keyed = keyBy(records, 'id');
return keys.reduce((ret, key) => {
ret[key] = keyed[key]?.value ?? defaultRuntimeConfig[key].value;
return ret;
}, {} as any);
}
async list(module?: AppRuntimeConfigModules) {
return await this.db.runtimeConfig.findMany({
where: module ? { module, deletedAt: null } : { deletedAt: null },
});
}
async set<
K extends keyof FlattenedAppRuntimeConfig,
V = FlattenedAppRuntimeConfig[K],
>(key: K, value: V) {
validateConfigType(key, value);
const config = await this.db.runtimeConfig.update({
where: {
id: key,
deletedAt: null,
},
data: {
value: value as any,
},
});
await this.setCache(key, config.value as FlattenedAppRuntimeConfig[K]);
return config;
}
async update<
K extends keyof FlattenedAppRuntimeConfig,
V = FlattenedAppRuntimeConfig[K],
>(k: K, modifier: (v: V) => V | Promise<V>) {
const data = await this.fetch<K>(k);
const updated = await modifier(data as V);
await this.set(k, updated);
return updated;
}
async loadDb<K extends keyof FlattenedAppRuntimeConfig>(
k: K
): Promise<FlattenedAppRuntimeConfig[K] | undefined> {
const v = await this.db.runtimeConfig.findFirst({
where: {
id: k,
deletedAt: null,
},
});
if (v) {
return v.value as FlattenedAppRuntimeConfig[K];
} else {
const record = await this.db.runtimeConfig.create({
data: defaultRuntimeConfig[k],
});
return record.value as any;
}
}
async loadCache<K extends keyof FlattenedAppRuntimeConfig>(
k: K
): Promise<FlattenedAppRuntimeConfig[K] | undefined> {
return this.cache.get<FlattenedAppRuntimeConfig[K]>(`SERVER_RUNTIME:${k}`);
}
async setCache<K extends keyof FlattenedAppRuntimeConfig>(
k: K,
v: FlattenedAppRuntimeConfig[K]
): Promise<boolean> {
return this.cache.set<FlattenedAppRuntimeConfig[K]>(
`SERVER_RUNTIME:${k}`,
v,
{ ttl: 60 * 1000 }
);
}
/**
* Upgrade the DB with latest runtime configs
*/
private async upgradeDB() {
const existingConfig = await this.db.runtimeConfig.findMany({
select: {
id: true,
},
where: {
deletedAt: null,
},
});
const defined = Object.keys(defaultRuntimeConfig);
const existing = existingConfig.map(c => c.id);
const newConfigs = difference(defined, existing);
const deleteConfigs = difference(existing, defined);
if (!newConfigs.length && !deleteConfigs.length) {
return;
}
this.logger.log(`Found runtime config changes, upgrading...`);
const acquired = await this.cache.setnx('runtime:upgrade', 1, {
ttl: 10 * 60 * 1000,
});
await using _ = defer(async () => {
await this.cache.delete('runtime:upgrade');
});
if (acquired) {
for (const key of newConfigs) {
await this.db.runtimeConfig.upsert({
create: defaultRuntimeConfig[key],
// old deleted setting should be restored
update: {
...defaultRuntimeConfig[key],
deletedAt: null,
},
where: {
id: key,
},
});
}
await this.db.runtimeConfig.updateMany({
where: {
id: {
in: deleteConfigs,
},
},
data: {
deletedAt: new Date(),
},
});
}
this.logger.log('Upgrade completed');
}
}
@@ -0,0 +1,127 @@
import { Join, PathType } from '../utils/types';
export type ConfigItem<T> = T & { __type: 'ConfigItem' };
type ConfigDef = Record<string, any> | never;
export interface ModuleConfig<
Startup extends ConfigDef = never,
Runtime extends ConfigDef = never,
> {
startup: Startup;
runtime: Runtime;
}
export type RuntimeConfigDescription<T> = {
desc: string;
default: T;
};
type ConfigItemLeaves<T, P extends string = ''> =
T extends Record<string, any>
? {
[K in keyof T]: K extends string
? T[K] extends { __type: 'ConfigItem' }
? K
: T[K] extends PrimitiveType
? K
: Join<K, ConfigItemLeaves<T[K], P>>
: never;
}[keyof T]
: never;
type StartupConfigDescriptions<T extends ConfigDef> = {
[K in keyof T]: T[K] extends Record<string, any>
? T[K] extends ConfigItem<infer V>
? V
: T[K]
: T[K];
};
type ModuleConfigLeaves<T, P extends string = ''> =
T extends Record<string, any>
? {
[K in keyof T]: K extends string
? T[K] extends ModuleConfig<any, any>
? K
: Join<K, ModuleConfigLeaves<T[K], P>>
: never;
}[keyof T]
: never;
type FlattenModuleConfigs<T extends Record<string, any>> = {
// @ts-expect-error allow
[K in ModuleConfigLeaves<T>]: PathType<T, K>;
};
type _AppStartupConfig<T extends Record<string, any>> = {
[K in keyof T]: T[K] extends ModuleConfig<infer S, any>
? S
: _AppStartupConfig<T[K]>;
};
// for extending
export interface AppConfig {}
export type AppModulesConfigDef = FlattenModuleConfigs<AppConfig>;
export type AppConfigModules = keyof AppModulesConfigDef;
export type AppStartupConfig = _AppStartupConfig<AppConfig>;
// app runtime config keyed by module names
export type AppRuntimeConfigByModules = {
[Module in keyof AppModulesConfigDef]: AppModulesConfigDef[Module] extends ModuleConfig<
any,
infer Runtime
>
? Runtime extends never
? never
: {
// @ts-expect-error allow
[K in ConfigItemLeaves<Runtime>]: PathType<
Runtime,
K
> extends infer Config
? Config extends ConfigItem<infer V>
? V
: Config
: never;
}
: never;
};
// names of modules that have runtime config
export type AppRuntimeConfigModules = {
[Module in keyof AppRuntimeConfigByModules]: AppRuntimeConfigByModules[Module] extends never
? never
: Module;
}[keyof AppRuntimeConfigByModules];
// runtime config keyed by module names flattened into config names
// { auth: { allowSignup: boolean } } => { 'auth/allowSignup': boolean }
export type FlattenedAppRuntimeConfig = UnionToIntersection<
{
[Module in keyof AppRuntimeConfigByModules]: AppModulesConfigDef[Module] extends never
? never
: {
[K in keyof AppRuntimeConfigByModules[Module] as K extends string
? `${Module}/${K}`
: never]: AppRuntimeConfigByModules[Module][K];
};
}[keyof AppRuntimeConfigByModules]
>;
export type ModuleStartupConfigDescriptions<T extends ModuleConfig<any, any>> =
T extends ModuleConfig<infer S, any>
? S extends never
? undefined
: StartupConfigDescriptions<S>
: never;
export type ModuleRuntimeConfigDescriptions<
Module extends keyof AppRuntimeConfigByModules,
> = AppModulesConfigDef[Module] extends never
? never
: {
[K in keyof AppRuntimeConfigByModules[Module]]: RuntimeConfigDescription<
AppRuntimeConfigByModules[Module][K]
>;
};