feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/base/job/queue/config.ts

@@ -71,14 +71,6 @@ defineModuleConfig('job', {
     schema,
   },
 
-  'queues.indexer': {
-    desc: 'The config for indexer job queue',
-    default: {
-      concurrency: 1,
-    },
-    schema,
-  },
-
   'queues.notification': {
     desc: 'The config for notification job queue',
     default: {

packages/backend/server/src/base/job/queue/def.ts

@@ -27,7 +27,6 @@ export enum Queue {
   NOTIFICATION = 'notification',
   DOC = 'doc',
   COPILOT = 'copilot',
-  INDEXER = 'indexer',
   CALENDAR = 'calendar',
   BACKENDRUNTIME = 'backendRuntime',
   INVITE_ABUSE = 'inviteAbuse',

packages/backend/server/src/base/job/queue/owner.ts

@@ -1,11 +1,7 @@
 import { ServerRole } from '../../../env';
 import { Queue, QUEUES } from './def';
 
-export const WORKER_QUEUES = [
-  Queue.DOC,
-  Queue.INDEXER,
-  Queue.BACKENDRUNTIME,
-] as const;
+export const WORKER_QUEUES = [Queue.DOC, Queue.BACKENDRUNTIME] as const;
 
 export function queuesForRole(role: ServerRole | undefined): Queue[] {
   switch (role) {