feat(server): add cloud indexer with Elasticsearch and Manticoresearch providers (#11835)

close CLOUD-137

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination.
  - Added pluggable search providers: Elasticsearch and Manticoresearch.
  - New GraphQL queries, schema types, and resolver support for search and aggregation.
  - Enhanced configuration options for search providers in self-hosted and cloud deployments.
  - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch.
  - Integrated indexer service into deployment and CI workflows.

- **Bug Fixes**
  - Improved error handling with new user-friendly error messages for search provider and indexer issues.

- **Documentation**
  - Updated configuration examples and environment variable references for indexer and search providers.

- **Tests**
  - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases.
  - Included snapshot tests and test fixtures for search providers.

- **Chores**
  - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.github/actions/deploy/deploy.mjs

@@ -16,6 +16,10 @@ const {
   REDIS_SERVER_HOST,
   REDIS_SERVER_PASSWORD,
   STATIC_IP_NAME,
+  AFFINE_INDEXER_SEARCH_PROVIDER,
+  AFFINE_INDEXER_SEARCH_ENDPOINT,
+  AFFINE_INDEXER_SEARCH_USERNAME,
+  AFFINE_INDEXER_SEARCH_PASSWORD,
 } = process.env;
 
 const buildType = BUILD_TYPE || 'canary';
@@ -81,6 +85,12 @@ const createHelmCommand = ({ isDryRun }) => {
           `--set-string global.redis.password="${REDIS_SERVER_PASSWORD}"`,
         ]
       : [];
+  const indexerOptions = [
+    `--set-string global.indexer.provider="${AFFINE_INDEXER_SEARCH_PROVIDER}"`,
+    `--set-string global.indexer.endpoint="${AFFINE_INDEXER_SEARCH_ENDPOINT}"`,
+    `--set-string global.indexer.username="${AFFINE_INDEXER_SEARCH_USERNAME}"`,
+    `--set-string global.indexer.password="${AFFINE_INDEXER_SEARCH_PASSWORD}"`,
+  ];
   const serviceAnnotations = [
     `--set-json   web.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
     `--set-json   graphql.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
@@ -130,6 +140,7 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string global.ingress.host="${host}"`,
     `--set-string global.version="${APP_VERSION}"`,
     ...redisAndPostgres,
+    ...indexerOptions,
     `--set        web.replicaCount=${replica.web}`,
     `--set-string web.image.tag="${imageTag}"`,
     `--set        graphql.replicaCount=${replica.graphql}`,

.github/helm/affine/charts/doc/templates/deployment.yaml

@@ -69,6 +69,17 @@ spec:
                 key: redis-password
           - name: REDIS_SERVER_DATABASE
             value: "{{ .Values.global.redis.database }}"
+          - name: AFFINE_INDEXER_SEARCH_PROVIDER
+            value: "{{ .Values.global.indexer.provider }}"
+          - name: AFFINE_INDEXER_SEARCH_ENDPOINT
+            value: "{{ .Values.global.indexer.endpoint }}"
+          - name: AFFINE_INDEXER_SEARCH_USERNAME
+            value: "{{ .Values.global.indexer.username }}"
+          - name: AFFINE_INDEXER_SEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: indexer
+                key: indexer-password
           - name: AFFINE_SERVER_PORT
             value: "{{ .Values.global.docService.port }}"
           - name: AFFINE_SERVER_SUB_PATH

.github/helm/affine/charts/graphql/templates/deployment.yaml

@@ -67,6 +67,17 @@ spec:
                 key: redis-password
           - name: REDIS_SERVER_DATABASE
             value: "{{ .Values.global.redis.database }}"
+          - name: AFFINE_INDEXER_SEARCH_PROVIDER
+            value: "{{ .Values.global.indexer.provider }}"
+          - name: AFFINE_INDEXER_SEARCH_ENDPOINT
+            value: "{{ .Values.global.indexer.endpoint }}"
+          - name: AFFINE_INDEXER_SEARCH_USERNAME
+            value: "{{ .Values.global.indexer.username }}"
+          - name: AFFINE_INDEXER_SEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: indexer
+                key: indexer-password
           - name: AFFINE_SERVER_PORT
             value: "{{ .Values.service.port }}"
           - name: AFFINE_SERVER_SUB_PATH

.github/helm/affine/charts/graphql/templates/migration.yaml

@@ -44,6 +44,17 @@ spec:
               secretKeyRef:
                 name: redis
                 key: redis-password
+          - name: AFFINE_INDEXER_SEARCH_PROVIDER
+            value: "{{ .Values.global.indexer.provider }}"
+          - name: AFFINE_INDEXER_SEARCH_ENDPOINT
+            value: "{{ .Values.global.indexer.endpoint }}"
+          - name: AFFINE_INDEXER_SEARCH_USERNAME
+            value: "{{ .Values.global.indexer.username }}"
+          - name: AFFINE_INDEXER_SEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: indexer
+                key: indexer-password
         resources:
           requests:
             cpu: '100m'

.github/helm/affine/charts/renderer/templates/deployment.yaml

@@ -69,6 +69,17 @@ spec:
                 key: redis-password
           - name: REDIS_SERVER_DATABASE
             value: "{{ .Values.global.redis.database }}"
+          - name: AFFINE_INDEXER_SEARCH_PROVIDER
+            value: "{{ .Values.global.indexer.provider }}"
+          - name: AFFINE_INDEXER_SEARCH_ENDPOINT
+            value: "{{ .Values.global.indexer.endpoint }}"
+          - name: AFFINE_INDEXER_SEARCH_USERNAME
+            value: "{{ .Values.global.indexer.username }}"
+          - name: AFFINE_INDEXER_SEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: indexer
+                key: indexer-password
           - name: AFFINE_SERVER_PORT
             value: "{{ .Values.service.port }}"
           - name: AFFINE_SERVER_SUB_PATH

.github/helm/affine/charts/sync/templates/deployment.yaml

@@ -69,6 +69,17 @@ spec:
                 key: redis-password
           - name: REDIS_SERVER_DATABASE
             value: "{{ .Values.global.redis.database }}"
+          - name: AFFINE_INDEXER_SEARCH_PROVIDER
+            value: "{{ .Values.global.indexer.provider }}"
+          - name: AFFINE_INDEXER_SEARCH_ENDPOINT
+            value: "{{ .Values.global.indexer.endpoint }}"
+          - name: AFFINE_INDEXER_SEARCH_USERNAME
+            value: "{{ .Values.global.indexer.username }}"
+          - name: AFFINE_INDEXER_SEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: indexer
+                key: indexer-password
           - name: AFFINE_SERVER_PORT
             value: "{{ .Values.service.port }}"
           - name: AFFINE_SERVER_HOST

.github/helm/affine/templates/indexer-secret.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.global.indexer.password -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: indexer
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "-2"
+    "helm.sh/hook-delete-policy": before-hook-creation
+type: Opaque
+data:
+  indexer-password: {{ .Values.global.indexer.password | b64enc }}
+{{- end }}

.github/helm/affine/values.yaml

@@ -21,6 +21,11 @@ global:
     username: ''
     password: ''
     database: 0
+  indexer:
+    provider: ''
+    endpoint: ''
+    username: ''
+    password: ''
   docService:
     name: 'affine-doc'
     port: 3020

.github/workflows/build-test.yml

@@ -577,7 +577,25 @@ jobs:
         ports:
           - 1025:1025
           - 8025:8025
+      manticoresearch:
+        image: manticoresearch/manticore:9.2.14
+        ports:
+          - 9308:9308
     steps:
+      # https://github.com/elastic/elastic-github-actions/blob/master/elasticsearch/README.md
+      - name: Configure sysctl limits for Elasticsearch
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+
+      - name: Runs Elasticsearch
+        uses: elastic/elastic-github-actions/elasticsearch@master
+        with:
+          stack-version: 9.0.1
+          security-enabled: false
+
       - uses: actions/checkout@v4
 
       - name: Setup Node.js
@@ -639,6 +657,10 @@ jobs:
         image: redis
         ports:
           - 6379:6379
+      indexer:
+        image: manticoresearch/manticore:9.2.14
+        ports:
+          - 9308:9308
     steps:
       - uses: actions/checkout@v4
 
@@ -1076,6 +1098,10 @@ jobs:
         ports:
           - 1025:1025
           - 8025:8025
+      indexer:
+        image: manticoresearch/manticore:9.2.14
+        ports:
+          - 9308:9308
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/deploy.yml

@@ -103,6 +103,10 @@ jobs:
           CLOUD_SQL_IAM_ACCOUNT: ${{ secrets.CLOUD_SQL_IAM_ACCOUNT }}
           APP_IAM_ACCOUNT: ${{ secrets.APP_IAM_ACCOUNT }}
           STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }}
+          AFFINE_INDEXER_SEARCH_PROVIDER: ${{ secrets.AFFINE_INDEXER_SEARCH_PROVIDER }}
+          AFFINE_INDEXER_SEARCH_ENDPOINT: ${{ secrets.AFFINE_INDEXER_SEARCH_ENDPOINT }}
+          AFFINE_INDEXER_SEARCH_USERNAME: ${{ secrets.AFFINE_INDEXER_SEARCH_USERNAME }}
+          AFFINE_INDEXER_SEARCH_PASSWORD: ${{ secrets.AFFINE_INDEXER_SEARCH_PASSWORD }}
 
   deploy-done:
     needs: