feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.github/actions/server-test-env/action.yml

@@ -33,6 +33,9 @@ runs:
       shell: bash
       run: |
         case '${{ inputs.indexer-provider }}' in
+          disabled)
+            default_indexer='{"enabled":false,"provider":{"type":"embedded","endpoint":""}}'
+            ;;
           embedded)
             default_indexer='{"enabled":true,"provider":{"type":"embedded","endpoint":""}}'
             ;;

.github/workflows/build-test.yml

@@ -933,8 +933,6 @@ jobs:
       NODE_ENV: test
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       REDIS_SERVER_HOST: localhost
-      SERVER_CONFIG: >-
-        {"indexer":{"enabled":true,"provider":{"type":"elasticsearch","endpoint":"http://localhost:9200"}}}
     services:
       postgres:
         image: pgvector/pgvector:pg16
@@ -1005,8 +1003,74 @@ jobs:
           name: affine
           fail_ci_if_error: false
 
-  server-test-manticore:
-    name: Server Test with Manticore
+  server-test-embedded:
+    name: Server Test with Embedded Search
+    runs-on: ubuntu-latest
+    needs:
+      - build-server-native
+    env:
+      NODE_ENV: test
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      REDIS_SERVER_HOST: localhost
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine/monorepo @affine/server
+          electron-install: false
+          full-cache: true
+
+      - name: Download server-native.node
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/native
+
+      - name: Prepare Server Test Environment
+        with:
+          indexer-provider: embedded
+        uses: ./.github/actions/server-test-env
+
+      - name: Run embedded Search provider E2E
+        run: >-
+          yarn affine @affine/server e2e:coverage
+          src/__tests__/e2e/indexer/aggregate.spec.ts
+          src/__tests__/e2e/indexer/search-docs.spec.ts
+          src/__tests__/e2e/indexer/search.spec.ts
+          --concurrency 1
+          --forbid-only
+        env:
+          CARGO_TARGET_DIR: '${{ github.workspace }}/target'
+
+      - name: Upload server test coverage results
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./packages/backend/server/.coverage/lcov.info
+          flags: server-test
+          name: affine
+          fail_ci_if_error: false
+
+  server-test-manticoresearch:
+    name: Server Test with Manticore Search
     runs-on: ubuntu-latest
     needs:
       - build-server-native
@@ -1014,9 +1078,6 @@ jobs:
       NODE_ENV: test
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       REDIS_SERVER_HOST: localhost
-      SEARCH_MS_URL: http://localhost:9308
-      SERVER_CONFIG: >-
-        {"indexer":{"enabled":true,"provider":{"type":"manticoresearch","endpoint":"http://localhost:9308"}}}
     services:
       postgres:
         image: pgvector/pgvector:pg16
@@ -1038,15 +1099,18 @@ jobs:
         ports:
           - 1025:1025
           - 8025:8025
-      indexer:
-        image: manticoresearch/manticore:10.1.0
-        options: >-
-          --health-cmd "wget -q -O /dev/null http://localhost:9308/"
-          --health-interval 5s
-          --health-timeout 3s
-          --health-retries 12
+      manticore:
+        image: manticoresearch/manticore:29.0.2
+        env:
+          'searchd.listen': 'http:9308'
+          'searchd.network_timeout': '30'
         ports:
           - 9308:9308
+        options: >-
+          --health-cmd "wget -qO- --post-data='SHOW TABLES' 'http://127.0.0.1:9308/sql?mode=raw' >/dev/null"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 30
     steps:
       - uses: actions/checkout@v7
 
@@ -1057,13 +1121,6 @@ jobs:
           electron-install: false
           full-cache: true
 
-      - name: Setup Rust
-        uses: ./.github/actions/build-rust
-        with:
-          target: x86_64-unknown-linux-gnu
-          package: '@affine/server-native'
-          no-build: 'true'
-
       - name: Download server-native.node
         uses: actions/download-artifact@v4
         with:
@@ -1071,17 +1128,14 @@ jobs:
           path: ./packages/backend/native
 
       - name: Prepare Server Test Environment
-        uses: ./.github/actions/server-test-env
         with:
           indexer-provider: manticoresearch
+        uses: ./.github/actions/server-test-env
 
-      - name: Run server tests with Manticore
-        run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/search-docs.spec.ts --forbid-only
-
-      - name: Run Manticore provider contract
-        run: >-
-          cargo test --manifest-path packages/backend/native/Cargo.toml
-          remote_providers_apply_search_and_delete_the_same_contract --lib
+      - name: Run Manticore Search provider E2E
+        run: yarn affine @affine/server e2e:coverage src/__tests__/e2e/indexer/manticore-provider.spec.ts --forbid-only
+        env:
+          CARGO_TARGET_DIR: '${{ github.workspace }}/target'
 
       - name: Upload server test coverage results
         uses: codecov/codecov-action@v5
@@ -1135,6 +1189,8 @@ jobs:
           path: ./packages/backend/native
 
       - name: Prepare Server Test Environment
+        with:
+          indexer-provider: disabled
         uses: ./.github/actions/server-test-env
 
       - name: Run server tests
@@ -1597,7 +1653,8 @@ jobs:
       - native-unit-test
       - server-test
       - server-test-elasticsearch
-      - server-test-manticore
+      - server-test-embedded
+      - server-test-manticoresearch
       - server-e2e-test
       - rust-test
       - rust-test-filter