feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/e2e/remote-provider/compose.yml

@@ -0,0 +1,73 @@
+services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: affine_rfc6_remote_e2e
+      POSTGRES_USER: ds
+      POSTGRES_PASSWORD: ds
+    ports:
+      - '55432:5432'
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U ds -d affine_rfc6_remote_e2e']
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 5s
+
+  opensearch:
+    image: opensearchproject/opensearch:2.19.1
+    environment:
+      discovery.type: single-node
+      DISABLE_INSTALL_DEMO_CONFIG: 'true'
+      DISABLE_SECURITY_PLUGIN: 'true'
+      bootstrap.memory_lock: 'true'
+      OPENSEARCH_JAVA_OPTS: '-Xms512m -Xmx512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    ports:
+      - '9200:9200'
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -fsS http://localhost:9200/_cluster/health >/dev/null',
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+      start_period: 10s
+
+  toxiproxy:
+    image: ghcr.io/shopify/toxiproxy:2.12.0
+    ports:
+      - '8474:8474'
+      - '8666:8666'
+
+  proxy-init:
+    image: curlimages/curl:8.12.1
+    depends_on:
+      opensearch:
+        condition: service_healthy
+      toxiproxy:
+        condition: service_started
+    entrypoint: ['/bin/sh', '-ec']
+    command:
+      - >-
+        until curl -fsS http://toxiproxy:8474/version >/dev/null; do sleep 1; done;
+        curl -fsS http://toxiproxy:8474/proxies/opensearch >/dev/null 2>&1 ||
+        curl -fsS -X POST http://toxiproxy:8474/proxies
+        -H 'content-type: application/json'
+        -d '{"name":"opensearch","listen":"0.0.0.0:8666","upstream":"opensearch:9200"}';
+        tail -f /dev/null
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -fsS http://toxiproxy:8474/proxies/opensearch >/dev/null',
+        ]
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 2s