From 6a13d69dea84fb3f9b99ae01cb41a60517b399b5 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Thu, 15 May 2025 06:16:06 +0000 Subject: [PATCH] chore(server): separate elasticsearch to run independently (#12299) close CLOUD-217 ## Summary by CodeRabbit - **New Features** - Introduced a dedicated test job for Elasticsearch, running only Elasticsearch-specific tests during CI. - **Chores** - Enhanced server test workflows with explicit setup steps and automated coverage uploads. - Improved test suite structure for Elasticsearch provider to enable conditional and asynchronous test execution based on environment variables. --- .github/workflows/build-test.yml | 74 ++++++++++++++++++- .../__tests__/providers/elasticsearch.spec.ts | 61 +++++++++------ 2 files changed, 110 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 3d94455613..4694c338ec 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -586,6 +586,76 @@ jobs: image: manticoresearch/manticore:9.2.14 ports: - 9308:9308 + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + electron-install: false + full-cache: true + + - name: Download server-native.node + uses: actions/download-artifact@v4 + with: + name: server-native.node + path: ./packages/backend/native + + - name: Prepare Server Test Environment + uses: ./.github/actions/server-test-env + + - name: Run server tests + run: yarn affine @affine/server test:coverage --forbid-only + env: + CARGO_TARGET_DIR: '${{ github.workspace }}/target' + CI_NODE_INDEX: ${{ matrix.node_index }} + CI_NODE_TOTAL: ${{ matrix.total_nodes }} + + - name: Upload server test coverage results + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./packages/backend/server/.coverage/lcov.info + flags: server-test + name: affine + fail_ci_if_error: false + + server-test-elasticsearch: + name: Server Test with Elasticsearch + runs-on: ubuntu-latest + needs: + - optimize_ci + - build-server-native + if: needs.optimize_ci.outputs.skip == 'false' + strategy: + fail-fast: false + env: + NODE_ENV: test + DATABASE_URL: postgresql://affine:affine@localhost:5432/affine + REDIS_SERVER_HOST: localhost + AFFINE_INDEXER_SEARCH_PROVIDER: elasticsearch + AFFINE_INDEXER_SEARCH_ENDPOINT: http://localhost:9200 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_PASSWORD: affine + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + redis: + image: redis + ports: + - 6379:6379 + mailer: + image: mailhog/mailhog + ports: + - 1025:1025 + - 8025:8025 steps: # https://github.com/elastic/elastic-github-actions/blob/master/elasticsearch/README.md - name: Configure sysctl limits for Elasticsearch @@ -618,8 +688,8 @@ jobs: - name: Prepare Server Test Environment uses: ./.github/actions/server-test-env - - name: Run server tests - run: yarn affine @affine/server test:coverage --forbid-only + - name: Run server tests with elasticsearch only + run: yarn affine @affine/server test:coverage "**/*/*elasticsearch.spec.ts" --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' CI_NODE_INDEX: ${{ matrix.node_index }} diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts index 941f4e8243..e4968e6be3 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts @@ -2,38 +2,52 @@ import { randomUUID } from 'node:crypto'; import { readFile } from 'node:fs/promises'; import path from 'node:path'; -import test from 'ava'; +import _test from 'ava'; import { omit, pick } from 'lodash-es'; -import { createModule } from '../../../../__tests__/create-module'; +import { + createModule, + TestingModule, +} from '../../../../__tests__/create-module'; import { Mockers } from '../../../../__tests__/mocks'; import { ConfigModule } from '../../../../base/config'; +import { User, Workspace } from '../../../../models'; import { IndexerModule } from '../../'; import { SearchProviderType } from '../../config'; import { AggregateQueryDSL, ElasticsearchProvider } from '../../providers'; import { blockMapping, docMapping, SearchTable } from '../../tables'; -const module = await createModule({ - imports: [ - IndexerModule, - ConfigModule.override({ - indexer: { - provider: { - type: SearchProviderType.Elasticsearch, - endpoint: 'http://localhost:9200', - username: 'elastic', - password: 'affine', - }, - }, - }), - ], - providers: [ElasticsearchProvider], -}); -const searchProvider = module.get(ElasticsearchProvider); -const user = await module.create(Mockers.User); -const workspace = await module.create(Mockers.Workspace); +const test = + process.env.AFFINE_INDEXER_SEARCH_PROVIDER === 'elasticsearch' + ? _test + : _test.skip; + +let module: TestingModule; +let searchProvider: ElasticsearchProvider; +let user: User; +let workspace: Workspace; + +_test.before(async () => { + module = await createModule({ + imports: [ + IndexerModule, + ConfigModule.override({ + indexer: { + provider: { + type: SearchProviderType.Elasticsearch, + endpoint: 'http://localhost:9200', + username: 'elastic', + password: 'affine', + }, + }, + }), + ], + providers: [ElasticsearchProvider], + }); + searchProvider = module.get(ElasticsearchProvider); + user = await module.create(Mockers.User); + workspace = await module.create(Mockers.Workspace); -test.before(async () => { await searchProvider.createTable( SearchTable.block, JSON.stringify(blockMapping) @@ -159,7 +173,7 @@ test.before(async () => { }); }); -test.after.always(async () => { +_test.after.always(async () => { await searchProvider.deleteByQuery( SearchTable.block, { @@ -182,6 +196,7 @@ test.after.always(async () => { refresh: true, } ); + await module.close(); });