feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/native/index.d.ts

@@ -56,10 +56,6 @@ export declare class BackendRuntime {
   getWorkspaceInviteLinkById(inviteId: string): Promise<RuntimeWorkspaceInviteLinkRecord | null>
   revokeWorkspaceInviteLink(workspaceId: string): Promise<boolean>
   cleanupExpiredRuntimeStates(limit: number): Promise<number>
-  refreshWorkspaceAdminStatsDirty(batchLimit: number, owner: string, leaseTtlMs: number): Promise<RuntimeWorkspaceStatsRefreshResult>
-  recalibrateWorkspaceAdminStats(lastSid: number, batchLimit: number, owner: string, leaseTtlMs: number): Promise<RuntimeWorkspaceStatsRecalibrationResult>
-  writeWorkspaceAdminStatsDailySnapshot(owner: string, leaseTtlMs: number): Promise<RuntimeWorkspaceStatsSnapshotResult>
-  recalibrateWorkspaceAdminStatsDaily(batchLimit: number, owner: string, leaseTtlMs: number, lockRetryTimes: number, lockRetryDelayMs: number): Promise<RuntimeWorkspaceStatsDailyRecalibrationResult>
   constructor(privateKey?: string | undefined | null, configPaths?: Array<string> | undefined | null)
   start(): Promise<void>
   stop(): Promise<void>
@@ -68,10 +64,7 @@ export declare class BackendRuntime {
   runMigrations(): Promise<void>
   searchAuthorized(actorUserId: string, workspaceId: string, request: RuntimeSearchRequest): Promise<SearchOperationOutput>
   aggregateAuthorized(actorUserId: string, workspaceId: string, request: RuntimeAggregateRequest): Promise<SearchOperationOutput>
-  indexSearchDocument(workspaceId: string, docId: string): Promise<void>
-  deleteSearchDocument(workspaceId: string, docId: string): Promise<void>
-  reconcileSearchWorkspace(workspaceId: string): Promise<void>
-  deleteSearchWorkspace(workspaceId: string): Promise<void>
+  reconcileSearchProjection(limit?: number | undefined | null): Promise<number>
   filterReadableDocs(actorUserId: string, workspaceId: string, docIds: Array<string>): Promise<Array<string>>
   searchStatus(): Promise<any>
   embeddingHealth(): Promise<EmbeddingHealth>
@@ -113,7 +106,6 @@ export declare class StorageRuntime {
   rebuildWorkspaceDocBlobRefs(workspaceId: string, limit: number): Promise<RuntimeDocBlobRefsResult>
   reconcileWorkspaceDocuments(workspaceId: string): Promise<RuntimeDocumentCleanupReconcileResult>
   executeDocumentCleanupCandidates(workspaceId: string | undefined | null, gracePeriodDays: number, limit: number): Promise<RuntimeDocumentCleanupExecuteResult>
-  ackDocumentCleanupEffect(workspaceId: string, docId: string, cleanupVersion: string, effect: string): Promise<RuntimeDocumentCleanupAckResult>
   constructor()
   start(): Promise<void>
   configure(configJson: string): void
@@ -1261,16 +1253,11 @@ export interface RuntimeDocHistoryInput {
   historyMaxAgeMs: number
 }
 
-export interface RuntimeDocumentCleanupAckResult {
-  completed: boolean
-}
-
 export interface RuntimeDocumentCleanupEffect {
   workspaceId: string
   docId: string
   cleanupVersion: string
   commentObjectsDone: boolean
-  searchDone: boolean
 }
 
 export interface RuntimeDocumentCleanupExecuteResult {
@@ -1551,30 +1538,6 @@ export interface RuntimeWorkspaceInviteQuotaUsage {
   targetDomains: Array<RuntimeQuotaTargetDomainInput>
 }
 
-export interface RuntimeWorkspaceStatsDailyRecalibrationResult {
-  processed: number
-  lastSid: number
-  snapshotted: number
-  skipped: boolean
-}
-
-export interface RuntimeWorkspaceStatsRecalibrationResult {
-  processed: number
-  lastSid: number
-  skipped: boolean
-}
-
-export interface RuntimeWorkspaceStatsRefreshResult {
-  processed: number
-  backlog: number
-  skipped: boolean
-}
-
-export interface RuntimeWorkspaceStatsSnapshotResult {
-  snapshotted: number
-  skipped: boolean
-}
-
 export declare function safeFetch(request: SafeFetchRequest): Promise<SafeFetchResponse>
 
 export type SafeFetchMethod =  'get'|