feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/indexer/service.ts

@@ -4,16 +4,20 @@ import {
   InternalServerError,
   InvalidIndexerInput,
   OnEvent,
-  SearchProviderNotFound,
+  SearchIndexFailed,
+  SearchIndexNotReady,
+  SearchPermissionSyncing,
+  SearchProviderUnavailable,
   SpaceAccessDenied,
-  WorkspacePermissionNotFound,
 } from '../../base';
 import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { ServerFeature, ServerService } from '../../core/config';
 import { Models } from '../../models';
 import {
   type AggregateResult,
+  buildBasicSearchDocsInput,
   buildSearchDocsInput,
+  collectBasicSearchDocs,
   collectSearchDocs,
   formatSearchNodes,
   type SearchNode,
@@ -50,12 +54,17 @@ export class IndexerService implements OnApplicationBootstrap {
   }
 
   private async syncFeature() {
+    if (!this.server.getConfig().indexer.enabled) {
+      this.server.disableFeature(ServerFeature.Indexer);
+      return;
+    }
     const status = (await this.runtime.searchStatus()) as { ready: boolean };
     if (status.ready) this.server.enableFeature(ServerFeature.Indexer);
     else this.server.disableFeature(ServerFeature.Indexer);
   }
 
   async search(actorUserId: string, workspaceId: string, input: SearchInput) {
+    await this.syncFeature();
     const result = this.unwrap<SearchResult>(
       await this.runtime.searchAuthorized(actorUserId, workspaceId, input),
       workspaceId
@@ -68,10 +77,15 @@ export class IndexerService implements OnApplicationBootstrap {
     workspaceId: string,
     input: AggregateInput
   ) {
+    await this.syncFeature();
     const result = this.unwrap<AggregateResult>(
       await this.runtime.aggregateAuthorized(actorUserId, workspaceId, input),
       workspaceId
     );
+    return this.formatAggregate(result);
+  }
+
+  private formatAggregate(result: AggregateResult) {
     return {
       ...result,
       buckets: result.buckets.map(bucket => ({
@@ -84,38 +98,42 @@ export class IndexerService implements OnApplicationBootstrap {
     };
   }
 
-  async indexDoc(workspaceId: string, docId: string) {
-    await this.runtime.indexSearchDocument(workspaceId, docId);
-  }
-
-  async deleteDoc(workspaceId: string, docId: string) {
-    await this.runtime.deleteSearchDocument(workspaceId, docId);
-  }
-
-  async reconcileWorkspace(workspaceId: string) {
-    await this.runtime.reconcileSearchWorkspace(workspaceId);
-  }
-
-  async deleteWorkspace(workspaceId: string) {
-    await this.runtime.deleteSearchWorkspace(workspaceId);
-  }
-
   async searchDocsByKeyword(
     actorUserId: string,
     workspaceId: string,
     keyword: string,
     options?: { limit?: number; docIds?: string[] }
   ): Promise<SearchDoc[]> {
+    await this.syncFeature();
+    if (options?.limit !== undefined && options.limit <= 0) {
+      throw new InvalidIndexerInput({
+        reason: 'searchDocs limit must be positive',
+      });
+    }
     if (options?.docIds?.length === 0) return [];
-    const result = await this.aggregate(
+    const aggregateOutput = await this.runtime.aggregateAuthorized(
       actorUserId,
       workspaceId,
       buildSearchDocsInput(workspaceId, keyword, options)
     );
-    const { docs, missingTitles, userIds } = collectSearchDocs(
-      result,
-      workspaceId
-    );
+    const collected =
+      !aggregateOutput.ok && aggregateOutput.errorCode === 'unsupported_query'
+        ? collectBasicSearchDocs(
+            await this.search(
+              actorUserId,
+              workspaceId,
+              buildBasicSearchDocsInput(workspaceId, keyword, options)
+            ),
+            workspaceId,
+            options?.limit ?? 20
+          )
+        : collectSearchDocs(
+            this.formatAggregate(
+              this.unwrap<AggregateResult>(aggregateOutput, workspaceId)
+            ),
+            workspaceId
+          );
+    const { docs, missingTitles, userIds } = collected;
     if (missingTitles.length > 0) {
       const metas = await this.models.doc.findMetas(missingTitles, {
         select: { title: true },
@@ -146,9 +164,15 @@ export class IndexerService implements OnApplicationBootstrap {
       case 'unsupported_query':
         throw new InvalidIndexerInput({ reason: output.errorCode });
       case 'provider_unavailable':
-        throw new SearchProviderNotFound();
-      case 'permission_unavailable':
-        throw new WorkspacePermissionNotFound({ spaceId: workspaceId });
+        throw new SearchProviderUnavailable();
+      case 'index_not_ready':
+        throw new SearchIndexNotReady({ spaceId: workspaceId });
+      case 'permission_syncing':
+        throw new SearchPermissionSyncing();
+      case 'index_failed':
+        throw new SearchIndexFailed({
+          diagnosticId: 'search_workspace_reconcile_failed',
+        });
       default:
         throw new InternalServerError();
     }