feat(server): add cloud indexer with Elasticsearch and Manticoresearch providers (#11835)

close CLOUD-137

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination.
  - Added pluggable search providers: Elasticsearch and Manticoresearch.
  - New GraphQL queries, schema types, and resolver support for search and aggregation.
  - Enhanced configuration options for search providers in self-hosted and cloud deployments.
  - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch.
  - Integrated indexer service into deployment and CI workflows.

- **Bug Fixes**
  - Improved error handling with new user-friendly error messages for search provider and indexer issues.

- **Documentation**
  - Updated configuration examples and environment variable references for indexer and search providers.

- **Tests**
  - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases.
  - Included snapshot tests and test fixtures for search providers.

- **Chores**
  - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.docker/selfhost/.env.example

@@ -20,4 +20,9 @@ CONFIG_LOCATION=~/.affine/self-host/config
 # database credentials
 DB_USERNAME=affine
 DB_PASSWORD=
-DB_DATABASE=affine
+DB_DATABASE=affine
+
+# indexer search provider manticoresearch version
+MANTICORE_VERSION=9.2.14
+# position of the manticoresearch data to persist 
+MANTICORE_DATA_LOCATION=~/.affine/self-host/manticore

.docker/selfhost/compose.yml

@@ -10,6 +10,8 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
+      indexer:
+        condition: service_healthy
       affine_migration:
         condition: service_completed_successfully
     volumes:
@@ -41,6 +43,8 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+      indexer:
+        condition: service_healthy
 
   redis:
     image: redis
@@ -72,3 +76,24 @@ services:
       timeout: 5s
       retries: 5
     restart: unless-stopped
+
+  indexer:
+    image: manticoresearch/manticore:${MANTICORE_VERSION:-9.2.14}
+    container_name: affine_indexer
+    volumes:
+      - ${MANTICORE_DATA_LOCATION}:/var/lib/manticore
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        ['CMD', 'wget', '-O-', 'http://127.0.0.1:9308']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped

.docker/selfhost/schema.json

@@ -794,6 +794,37 @@
         }
       }
     },
+    "indexer": {
+      "type": "object",
+      "description": "Configuration for indexer module",
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "description": "Enable indexer plugin\n@default true",
+          "default": true
+        },
+        "provider.type": {
+          "type": "string",
+          "description": "Indexer search service provider name\n@default \"manticoresearch\"\n@environment `AFFINE_INDEXER_SEARCH_PROVIDER`",
+          "default": "manticoresearch"
+        },
+        "provider.endpoint": {
+          "type": "string",
+          "description": "Indexer search service endpoint\n@default \"http://localhost:9308\"\n@environment `AFFINE_INDEXER_SEARCH_ENDPOINT`",
+          "default": "http://localhost:9308"
+        },
+        "provider.username": {
+          "type": "string",
+          "description": "Indexer search service auth username, if not set, basic auth will be disabled. Optional for elasticsearch\n@default \"\"\n@environment `AFFINE_INDEXER_SEARCH_USERNAME`\n@link https://www.elastic.co/guide/en/elasticsearch/reference/current/http-clients.html",
+          "default": ""
+        },
+        "provider.password": {
+          "type": "string",
+          "description": "Indexer search service auth password, if not set, basic auth will be disabled. Optional for elasticsearch\n@default \"\"\n@environment `AFFINE_INDEXER_SEARCH_PASSWORD`",
+          "default": ""
+        }
+      }
+    },
     "oauth": {
       "type": "object",
       "description": "Configuration for oauth module",