refactor(server): indexer & worker & sync perf (#15504)

.github/workflows/build-test.yml

@@ -21,7 +21,6 @@ env:
   COVERAGE: true
   MACOSX_DEPLOYMENT_TARGET: '11.6'
   DEPLOYMENT_TYPE: affine
-  AFFINE_INDEXER_ENABLED: true
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -890,10 +889,6 @@ jobs:
         ports:
           - 1025:1025
           - 8025:8025
-      indexer:
-        image: manticoresearch/manticore:10.1.0
-        ports:
-          - 9308:9308
     steps:
       - uses: actions/checkout@v7
 
@@ -938,8 +933,8 @@ jobs:
       NODE_ENV: test
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       REDIS_SERVER_HOST: localhost
-      AFFINE_INDEXER_SEARCH_PROVIDER: elasticsearch
-      AFFINE_INDEXER_SEARCH_ENDPOINT: http://localhost:9200
+      SERVER_CONFIG: >-
+        {"indexer":{"enabled":true,"provider":{"type":"elasticsearch","endpoint":"http://localhost:9200"}}}
     services:
       postgres:
         image: pgvector/pgvector:pg16
@@ -992,10 +987,12 @@ jobs:
           path: ./packages/backend/native
 
       - name: Prepare Server Test Environment
+        with:
+          indexer-provider: elasticsearch
         uses: ./.github/actions/server-test-env
 
       - name: Run server tests with elasticsearch only
-        run: yarn affine @affine/server test:coverage "**/*/*elasticsearch.spec.ts" --forbid-only
+        run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
 
@@ -1008,6 +1005,93 @@ jobs:
           name: affine
           fail_ci_if_error: false
 
+  server-test-manticore:
+    name: Server Test with Manticore
+    runs-on: ubuntu-latest
+    needs:
+      - build-server-native
+    env:
+      NODE_ENV: test
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      REDIS_SERVER_HOST: localhost
+      SEARCH_MS_URL: http://localhost:9308
+      SERVER_CONFIG: >-
+        {"indexer":{"enabled":true,"provider":{"type":"manticoresearch","endpoint":"http://localhost:9308"}}}
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+      mailer:
+        image: mailhog/mailhog
+        ports:
+          - 1025:1025
+          - 8025:8025
+      indexer:
+        image: manticoresearch/manticore:10.1.0
+        options: >-
+          --health-cmd "wget -q -O /dev/null http://localhost:9308/"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 12
+        ports:
+          - 9308:9308
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine/monorepo @affine/server
+          electron-install: false
+          full-cache: true
+
+      - name: Setup Rust
+        uses: ./.github/actions/build-rust
+        with:
+          target: x86_64-unknown-linux-gnu
+          package: '@affine/server-native'
+          no-build: 'true'
+
+      - name: Download server-native.node
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/native
+
+      - name: Prepare Server Test Environment
+        uses: ./.github/actions/server-test-env
+        with:
+          indexer-provider: manticoresearch
+
+      - name: Run server tests with Manticore
+        run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only
+
+      - name: Run Manticore provider contract
+        run: >-
+          cargo test --manifest-path packages/backend/native/Cargo.toml
+          remote_providers_apply_search_and_delete_the_same_contract --lib
+
+      - name: Upload server test coverage results
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./packages/backend/server/.coverage/lcov.info
+          flags: server-test
+          name: affine
+          fail_ci_if_error: false
+
   server-e2e-test:
     # the new version of server e2e test should be super fast, so sharding testing is not needed
     name: Server E2E Test
@@ -1034,10 +1118,6 @@ jobs:
         image: redis
         ports:
           - 6379:6379
-      indexer:
-        image: manticoresearch/manticore:10.1.0
-        ports:
-          - 9308:9308
     steps:
       - uses: actions/checkout@v7
 
@@ -1165,10 +1245,6 @@ jobs:
         ports:
           - 1025:1025
           - 8025:8025
-      indexer:
-        image: manticoresearch/manticore:10.1.0
-        ports:
-          - 9308:9308
     steps:
       - uses: actions/checkout@v7
 
@@ -1238,10 +1314,6 @@ jobs:
         image: redis
         ports:
           - 6379:6379
-      indexer:
-        image: manticoresearch/manticore:10.1.0
-        ports:
-          - 9308:9308
     steps:
       - uses: actions/checkout@v7
 
@@ -1321,10 +1393,6 @@ jobs:
         ports:
           - 1025:1025
           - 8025:8025
-      indexer:
-        image: manticoresearch/manticore:10.1.0
-        ports:
-          - 9308:9308
     steps:
       - uses: actions/checkout@v7
 
@@ -1528,6 +1596,8 @@ jobs:
       - build-electron-renderer
       - native-unit-test
       - server-test
+      - server-test-elasticsearch
+      - server-test-manticore
       - server-e2e-test
       - rust-test
       - rust-test-filter