chore(server): customize config merge logic (#11400)

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -64,16 +64,16 @@ test.serial.before(async t => {
         copilot: {
           providers: {
             openai: {
-              apiKey: process.env.COPILOT_OPENAI_API_KEY,
+              apiKey: process.env.COPILOT_OPENAI_API_KEY || '',
             },
             fal: {
-              apiKey: process.env.COPILOT_FAL_API_KEY,
+              apiKey: process.env.COPILOT_FAL_API_KEY || '',
             },
             perplexity: {
-              apiKey: process.env.COPILOT_PERPLEXITY_API_KEY,
+              apiKey: process.env.COPILOT_PERPLEXITY_API_KEY || '',
             },
             gemini: {
-              apiKey: process.env.COPILOT_GOOGLE_API_KEY,
+              apiKey: process.env.COPILOT_GOOGLE_API_KEY || '',
             },
           },
         },

packages/backend/server/src/base/config/__tests__/config.spec.ts

@@ -3,6 +3,7 @@ import test from 'ava';
 import { createModule } from '../../../__tests__/create-module';
 import { ConfigFactory, ConfigModule } from '..';
 import { Config } from '../config';
+import { override } from '../register';
 
 const module = await createModule();
 test.after.always(async () => {
@@ -49,13 +50,14 @@ test('should override config', async t => {
     auth: {
       passwordRequirements: {
         max: 10,
+        min: 1,
       },
     },
   });
 
   t.deepEqual(config.auth.passwordRequirements, {
     max: 10,
-    min: 6,
+    min: 1,
   });
 });
 
@@ -88,3 +90,87 @@ Error: Minimum length of password must be less than maximum length`,
     }
   );
 });
+
+test('should override correctly', t => {
+  const config = {
+    auth: {
+      // object config
+      passwordRequirements: {
+        max: 10,
+        min: 6,
+      },
+      allowSignup: false,
+      // keyed config
+      // 'session.ttl', 'session.ttr'
+      session: {
+        ttl: 2000,
+        ttr: 1000,
+      },
+    },
+    storages: {
+      avatar: {
+        // keyed config
+        // "avatar.publicPath: String"
+        publicPath: '/',
+        // object config
+        // "avatar.storage => Object { }"
+        storage: {
+          provider: 'fs',
+          config: {
+            path: '/path/to/avatar',
+          },
+        },
+      },
+    },
+  } as AppConfig;
+
+  override(config, {
+    auth: {
+      passwordRequirements: {
+        max: 20,
+      },
+      allowSignup: true,
+      session: {
+        ttl: 3000,
+      },
+    },
+    storages: {
+      avatar: {
+        storage: {
+          provider: 'aws-s3',
+          config: {
+            credentials: {
+              accessKeyId: '1',
+              accessKeySecret: '1',
+            },
+          },
+        },
+      },
+    },
+  });
+
+  // simple value override
+  t.deepEqual(config.auth.allowSignup, true);
+
+  // right covered left
+  t.deepEqual(config.auth.passwordRequirements, {
+    max: 20,
+  });
+
+  // right merged to left
+  t.deepEqual(config.auth.session, {
+    ttl: 3000,
+    ttr: 1000,
+  });
+
+  // right covered left
+  t.deepEqual(config.storages.avatar.storage, {
+    provider: 'aws-s3',
+    config: {
+      credentials: {
+        accessKeyId: '1',
+        accessKeySecret: '1',
+      },
+    },
+  });
+});

packages/backend/server/src/base/config/factory.ts

@@ -1,14 +1,13 @@
 import { Inject, Injectable, Optional } from '@nestjs/common';
-import { merge } from 'lodash-es';
 
 import { InvalidAppConfig } from '../error';
-import { APP_CONFIG_DESCRIPTORS, getDefaultConfig } from './register';
+import { APP_CONFIG_DESCRIPTORS, getDefaultConfig, override } from './register';
 
 export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN');
 
 @Injectable()
 export class ConfigFactory {
-  readonly #config: DeepReadonly<AppConfig>;
+  readonly #config: AppConfig;
 
   constructor(
     @Inject(OVERRIDE_CONFIG_TOKEN)
@@ -22,8 +21,12 @@ export class ConfigFactory {
     return this.#config;
   }
 
+  clone() {
+    return structuredClone(this.#config);
+  }
+
   override(updates: DeepPartial<AppConfig>) {
-    merge(this.#config, updates);
+    override(this.#config, updates);
   }
 
   validate(updates: Array<{ module: string; key: string; value: any }>) {
@@ -53,8 +56,9 @@ Error: ${issue.message}`);
     }
   }
 
-  private loadDefault(): DeepReadonly<AppConfig> {
+  private loadDefault() {
     const config = getDefaultConfig();
-    return merge(config, this.overrides);
+    override(config, this.overrides);
+    return config;
   }
 }

packages/backend/server/src/base/config/register.ts

@@ -1,7 +1,8 @@
 import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
 import { join } from 'node:path';
 
-import { merge, once, set } from 'lodash-es';
+import { mergeWith, once, set } from 'lodash-es';
 import { z } from 'zod';
 
 import { type EnvConfigType, parseEnvValue } from './env';
@@ -208,14 +209,15 @@ export function defineModuleConfig<T extends keyof AppConfigSchema>(
   };
 }
 
-function readConfigJSONOverrides() {
-  if (existsSync(join(env.projectRoot, 'config.json'))) {
+const CONFIG_JSON_PATHS = [
+  join(env.projectRoot, 'config.json'),
+  `${homedir()}/.affine/config/config.json`,
+];
+function readConfigJSONOverrides(path: string) {
+  const overrides: DeepPartial<AppConfig> = {};
+  if (existsSync(path)) {
     try {
-      const config = JSON.parse(
-        readFileSync(join(env.projectRoot, 'config.json'), 'utf-8')
-      ) as AppConfig;
-
-      const overrides = {};
+      const config = JSON.parse(readFileSync(path, 'utf-8')) as AppConfig;
 
       Object.entries(config).forEach(([key, value]) => {
         if (key === '$schema') {
@@ -226,18 +228,41 @@ function readConfigJSONOverrides() {
           set(overrides, `${key}.${k}`, v);
         });
       });
-
-      return overrides;
     } catch (e) {
       console.error('Invalid json config file', e);
     }
   }
 
-  return {};
+  return overrides;
 }
 
-export function getDefaultConfig(): AppConfigSchema {
-  const config: Record<string, any> = {};
+export function override(config: AppConfig, update: DeepPartial<AppConfig>) {
+  Object.keys(update).forEach(module => {
+    const moduleDescriptors = APP_CONFIG_DESCRIPTORS[module];
+    const configKeys = new Set(Object.keys(moduleDescriptors));
+
+    const moduleConfig = config[module as keyof AppConfig];
+    const moduleOverrides = update[module as keyof AppConfig];
+
+    const merge = (left: any, right: any, path: string = '') => {
+      // if we found the key in the config keys
+      // we should use the override object instead of merge it with left
+      if (configKeys.has(path)) {
+        return right;
+      }
+
+      // go deeper
+      return mergeWith(left, right, (left, right, key) => {
+        return merge(left, right, path === '' ? key : `${path}.${key}`);
+      });
+    };
+
+    config[module as keyof AppConfig] = merge(moduleConfig, moduleOverrides);
+  });
+}
+
+export function getDefaultConfig(): AppConfig {
+  const config = {} as AppConfig;
   const envs = process.env;
 
   for (const [module, defs] of Object.entries(APP_CONFIG_DESCRIPTORS)) {
@@ -271,12 +296,14 @@ Error: ${issue.message}`;
       set(modulizedConfig, key, defaultValue);
     }
 
+    // @ts-expect-error all keys are known
     config[module] = modulizedConfig;
   }
 
-  const fileOverrides = readConfigJSONOverrides();
-
-  merge(config, fileOverrides);
+  CONFIG_JSON_PATHS.forEach(path => {
+    const overrides = readConfigJSONOverrides(path);
+    override(config, overrides);
+  });
 
   return config as AppConfigSchema;
 }

packages/backend/server/src/base/event/eventbus.ts

@@ -155,7 +155,7 @@ export class EventBus
 
     add.call(this.emitter, event, handler as any, opts);
 
-    this.logger.verbose(`Event handler registered ${signature}`);
+    this.logger.log(`Event handler registered ${signature}`);
 
     return () => {
       this.emitter.off(event, handler as any);

packages/backend/server/src/base/job/queue/scanner.ts

@@ -1,4 +1,4 @@
-import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
 
 import { ModuleScanner } from '../../nestjs';
 import { getJobHandlerMetadata, JOB_SIGNAL } from './def';
@@ -11,6 +11,7 @@ interface JobHandler {
 @Injectable()
 export class JobHandlerScanner implements OnModuleInit {
   private readonly handlers: Record<string, JobHandler> = {};
+  private readonly logger = new Logger(JobHandlerScanner.name);
 
   constructor(private readonly scanner: ModuleScanner) {}
 
@@ -70,6 +71,8 @@ export class JobHandlerScanner implements OnModuleInit {
               return instance[method].bind(instance)(payload);
             },
           };
+
+          this.logger.log(`Job handler registered [${jobName}] (${signature})`);
         });
       });
     });

packages/backend/server/src/core/config/__tests__/service.spec.ts

@@ -23,7 +23,7 @@ test.after.always(async () => {
 });
 
 test('should update config', async t => {
-  const oldValue = service.config.server.externalUrl;
+  const oldValue = service.getConfig().server.externalUrl;
   const newValue = faker.internet.url();
   await service.updateConfig(user.id, [
     {
@@ -33,8 +33,8 @@ test('should update config', async t => {
     },
   ]);
 
-  t.not(service.config.server.externalUrl, oldValue);
-  t.is(service.config.server.externalUrl, newValue);
+  t.not(service.getConfig().server.externalUrl, oldValue);
+  t.is(service.getConfig().server.externalUrl, newValue);
 });
 
 test('should validate config before update', async t => {
@@ -53,7 +53,7 @@ Error: Invalid url`,
     }
   );
 
-  t.not(service.config.server.externalUrl, 'invalid-url');
+  t.not(service.getConfig().server.externalUrl, 'invalid-url');
 
   await t.throwsAsync(
     service.updateConfig(user.id, [
@@ -68,8 +68,11 @@ Error: Invalid url`,
     }
   );
 
-  // @ts-expect-error allow
-  t.is(service.config.auth['unknown-key'], undefined);
+  t.is(
+    // @ts-expect-error testing
+    service.getConfig().auth['unknown-key'],
+    undefined
+  );
 });
 
 test('should emit config.init event', async t => {
@@ -77,12 +80,12 @@ test('should emit config.init event', async t => {
   const event = module.event.last('config.init');
   t.is(event.name, 'config.init');
   t.deepEqual(event.payload, {
-    config: service.config,
+    config: service.getConfig(),
   });
 });
 
 test('should revalidate config', async t => {
-  const outdatedValue = service.config.server.externalUrl;
+  const outdatedValue = service.getConfig().server.externalUrl;
   const newValue = faker.internet.url();
 
   await models.appConfig.save(user.id, [
@@ -94,8 +97,8 @@ test('should revalidate config', async t => {
 
   await service.revalidateConfig();
 
-  t.not(service.config.server.externalUrl, outdatedValue);
-  t.is(service.config.server.externalUrl, newValue);
+  t.not(service.getConfig().server.externalUrl, outdatedValue);
+  t.is(service.getConfig().server.externalUrl, newValue);
 });
 
 test('should emit config changed event', async t => {

packages/backend/server/src/core/config/resolver.ts

@@ -191,7 +191,7 @@ export class AppConfigResolver {
     description: 'get the whole app configuration',
   })
   appConfig() {
-    return this.service.config;
+    return this.service.getConfig();
   }
 
   @Mutation(() => GraphQLJSONObject, {

packages/backend/server/src/core/config/service.ts

@@ -56,8 +56,8 @@ export class ServerService implements OnApplicationBootstrap {
     this.#features.delete(feature);
   }
 
-  get config() {
-    return this.configFactory.config;
+  getConfig() {
+    return this.configFactory.clone();
   }
 
   async updateConfig(