From e22f66f1cc62c611a716bad8286c525da9f84559 Mon Sep 17 00:00:00 2001 From: liuyi Date: Fri, 28 Mar 2025 14:12:56 +0800 Subject: [PATCH] fix(server): auto import config.json (#11255) --- .github/actions/copilot-test/action.yml | 19 ---------- .github/actions/server-test-env/action.yml | 8 ++++ .github/workflows/build-test.yml | 29 +++++++-------- .github/workflows/copilot-test.yml | 9 +++-- packages/backend/server/.gitignore | 1 + packages/backend/server/config.example.json | 3 ++ .../server/src/base/config/register.ts | 37 ++++++++++++++++++- .../backend/server/src/core/config/service.ts | 6 +-- 8 files changed, 70 insertions(+), 42 deletions(-) create mode 100644 packages/backend/server/config.example.json diff --git a/.github/actions/copilot-test/action.yml b/.github/actions/copilot-test/action.yml index 3bf5b1af6d..4a0b081b27 100644 --- a/.github/actions/copilot-test/action.yml +++ b/.github/actions/copilot-test/action.yml @@ -5,35 +5,16 @@ inputs: description: 'Script to run' default: 'yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only' required: false - openai-key: - description: 'OpenAI secret key' - required: true - google-key: - description: 'Google secret key' - required: true - fal-key: - description: 'Fal secret key' - required: true - perplexity-key: - description: 'Perplexity secret key' - required: true runs: using: 'composite' steps: - - name: Prepare Server Test Environment - uses: ./.github/actions/server-test-env - - name: Server Copilot E2E Test shell: bash run: ${{ inputs.script }} env: COPILOT: true DEV_SERVER_URL: http://localhost:8080 - COPILOT_OPENAI_API_KEY: ${{ inputs.openai-key }} - COPILOT_GOOGLE_API_KEY: ${{ inputs.google-key }} - COPILOT_FAL_API_KEY: ${{ inputs.fal-key }} - COPILOT_PERPLEXITY_API_KEY: ${{ inputs.perplexity-key }} - name: Upload test results if: ${{ failure() }} diff --git a/.github/actions/server-test-env/action.yml b/.github/actions/server-test-env/action.yml index 260e5209d6..9755ea0e33 100644 --- a/.github/actions/server-test-env/action.yml +++ b/.github/actions/server-test-env/action.yml @@ -21,3 +21,11 @@ runs: yarn affine @affine/server prisma generate yarn affine @affine/server prisma migrate deploy yarn affine @affine/server data-migration run + - name: Import config + shell: bash + run: | + printf '{"copilot":{"enabled":true,"providers.fal":{"apiKey":"%s"},"providers.gemini":{"apiKey":"%s"},"providers.openai":{"apiKey":"%s"},"providers.perplexity":{"apiKey":"%s"}}}' \ + "$COPILOT_FAL_API_KEY" \ + "$COPILOT_GOOGLE_API_KEY" \ + "$COPILOT_OPENAI_API_KEY" \ + "$COPILOT_PERPLEXITY_API_KEY" > ./packages/backend/server/config.json diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b8099813dc..300a013185 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -555,8 +555,6 @@ jobs: run: yarn affine @affine/server test:coverage --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' - COPILOT_OPENAI_API_KEY: 'use_fake_openai_api_key' - COPILOT_GOOGLE_API_KEY: 'use_fake_google_api_key' CI_NODE_INDEX: ${{ matrix.node_index }} CI_NODE_TOTAL: ${{ matrix.total_nodes }} @@ -617,8 +615,6 @@ jobs: - name: Run server tests run: yarn affine @affine/server e2e:coverage --forbid-only - env: - COPILOT_OPENAI_API_KEY: 'use_fake_openai_api_key' - name: Upload server test coverage results uses: codecov/codecov-action@v5 @@ -721,6 +717,11 @@ jobs: - name: Prepare Server Test Environment if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }} + env: + COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} + COPILOT_GOOGLE_API_KEY: ${{ secrets.COPILOT_GOOGLE_API_KEY }} + COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }} + COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} uses: ./.github/actions/server-test-env - name: Run server tests @@ -728,10 +729,6 @@ jobs: run: yarn affine @affine/server test:copilot:coverage --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' - COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} - COPILOT_GOOGLE_API_KEY: ${{ secrets.COPILOT_GOOGLE_API_KEY }} - COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }} - COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} - name: Upload server test coverage results if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }} @@ -813,15 +810,20 @@ jobs: name: server-native.node path: ./packages/backend/server + - name: Prepare Server Test Environment + if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.e2efilter.outputs.changed == 'true' }} + env: + COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} + COPILOT_GOOGLE_API_KEY: ${{ secrets.COPILOT_GOOGLE_API_KEY }} + COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }} + COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} + uses: ./.github/actions/server-test-env + - name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.e2efilter.outputs.changed == 'true' }} uses: ./.github/actions/copilot-test with: script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }} - google-key: ${{ secrets.COPILOT_GOOGLE_API_KEY }} - fal-key: ${{ secrets.COPILOT_FAL_API_KEY }} - perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} cloud-e2e-test: name: ${{ matrix.tests.name }} @@ -917,9 +919,6 @@ jobs: ${{ matrix.tests.script }} env: DEV_SERVER_URL: http://localhost:8080 - COPILOT_OPENAI_API_KEY: 1 - COPILOT_FAL_API_KEY: 1 - COPILOT_PERPLEXITY_API_KEY: 1 - name: Upload test results if: always() diff --git a/.github/workflows/copilot-test.yml b/.github/workflows/copilot-test.yml index 305376a717..bae394e36b 100644 --- a/.github/workflows/copilot-test.yml +++ b/.github/workflows/copilot-test.yml @@ -76,16 +76,17 @@ jobs: path: ./packages/backend/server - name: Prepare Server Test Environment + env: + COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} + COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }} + COPILOT_GOOGLE_API_KEY: ${{ secrets.COPILOT_GOOGLE_API_KEY }} + COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} uses: ./.github/actions/server-test-env - name: Run server tests run: yarn affine @affine/server test:copilot:coverage --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' - COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} - COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }} - COPILOT_GOOGLE_API_KEY: ${{ secrets.COPILOT_GOOGLE_API_KEY }} - COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} - name: Upload server test coverage results uses: codecov/codecov-action@v5 diff --git a/packages/backend/server/.gitignore b/packages/backend/server/.gitignore index 5d609db0b7..7c3fa798aa 100644 --- a/packages/backend/server/.gitignore +++ b/packages/backend/server/.gitignore @@ -1,2 +1,3 @@ +config.json .env static/ diff --git a/packages/backend/server/config.example.json b/packages/backend/server/config.example.json new file mode 100644 index 0000000000..7b613b95e7 --- /dev/null +++ b/packages/backend/server/config.example.json @@ -0,0 +1,3 @@ +{ + "$schema": "../../../.docker/selfhost/schema.json" +} diff --git a/packages/backend/server/src/base/config/register.ts b/packages/backend/server/src/base/config/register.ts index 11880b4ea7..2d183c588d 100644 --- a/packages/backend/server/src/base/config/register.ts +++ b/packages/backend/server/src/base/config/register.ts @@ -1,4 +1,7 @@ -import { once, set } from 'lodash-es'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { merge, once, set } from 'lodash-es'; import { z } from 'zod'; import { type EnvConfigType, parseEnvValue } from './env'; @@ -205,6 +208,34 @@ export function defineModuleConfig( }; } +function readConfigJSONOverrides() { + if (existsSync(join(env.projectRoot, 'config.json'))) { + try { + const config = JSON.parse( + readFileSync(join(env.projectRoot, 'config.json'), 'utf-8') + ) as AppConfig; + + const overrides = {}; + + Object.entries(config).forEach(([key, value]) => { + if (key === '$schema') { + return; + } + + Object.entries(value).forEach(([k, v]) => { + set(overrides, `${key}.${k}`, v); + }); + }); + + return overrides; + } catch (e) { + console.error('Invalid json config file', e); + } + } + + return {}; +} + export function getDefaultConfig(): AppConfigSchema { const config: Record = {}; const envs = process.env; @@ -235,5 +266,9 @@ export function getDefaultConfig(): AppConfigSchema { config[module] = modulizedConfig; } + const fileOverrides = readConfigJSONOverrides(); + + merge(config, fileOverrides); + return config as AppConfigSchema; } diff --git a/packages/backend/server/src/core/config/service.ts b/packages/backend/server/src/core/config/service.ts index 1ea543ac3a..cddf4a59a7 100644 --- a/packages/backend/server/src/core/config/service.ts +++ b/packages/backend/server/src/core/config/service.ts @@ -90,9 +90,9 @@ export class ServerService implements OnApplicationBootstrap { } @OnEvent('config.changed.broadcast') - onConfigChangedBroadcast(updates: DeepPartial) { - this.configFactory.override(updates); - this.event.emit('config.changed', { updates }); + onConfigChangedBroadcast(event: Events['config.changed.broadcast']) { + this.configFactory.override(event.updates); + this.event.emit('config.changed', event); } async revalidateConfig() {