feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/frontend/admin/src/modules/settings/config.ts

@@ -170,14 +170,13 @@ export const KNOWN_CONFIG_GROUPS = [
       {
         key: 'provider.type',
         type: 'Enum',
-        options: ['embedded', 'manticoresearch', 'elasticsearch'],
-        desc: 'Search provider. Embedded keeps external credentials for later reuse.',
+        options: ['embedded', 'elasticsearch', 'manticoresearch'],
+        desc: 'Search provider. Embedded and Elasticsearch provide full search semantics; Manticore Search provides basic search semantics.',
       },
       'provider.endpoint',
       'provider.apiKey',
       'provider.username',
       'provider.password',
-      'autoIndex.batchSize',
     ],
   } as ConfigGroup<'indexer'>,
 ];

packages/frontend/admin/src/modules/settings/index.spec.tsx

@@ -38,11 +38,8 @@ vi.mock('./config-input-row', () => ({
       <button type="button" onClick={() => onChange?.(field, 'embedded')}>
         set-embedded-{field}
       </button>
-      <button
-        type="button"
-        onClick={() => onChange?.(field, 'manticoresearch')}
-      >
-        set-manticoresearch-{field}
+      <button type="button" onClick={() => onChange?.(field, 'elasticsearch')}>
+        set-elasticsearch-{field}
       </button>
       <button
         type="button"
@@ -107,7 +104,7 @@ vi.mock('./config', () => ({
         {
           key: 'provider.type',
           type: 'Enum',
-          options: ['embedded', 'manticoresearch', 'elasticsearch'],
+          options: ['embedded', 'elasticsearch', 'manticoresearch'],
         },
         'provider.endpoint',
       ],
@@ -197,7 +194,7 @@ describe('SettingsPage', () => {
     expect(authItem?.dataset.state).toBe('open');
   });
 
-  test('encodes embedded without replacing external provider settings', () => {
+  test('enables the selected provider without replacing external settings', () => {
     const update = vi.fn();
     useAppConfigMock.mockReturnValue({
       ...useAppConfigMock(),
@@ -212,28 +209,27 @@ describe('SettingsPage', () => {
     );
 
     fireEvent.click(screen.getAllByRole('button', { name: /Indexer/i })[0]);
-    expect(screen.getByText('indexer/provider.type:embedded')).toBeTruthy();
-    expect(screen.queryByTestId('field-indexer/provider.endpoint')).toBeNull();
+    expect(
+      screen.getByText('indexer/provider.type:elasticsearch')
+    ).toBeTruthy();
+    expect(screen.getByTestId('field-indexer/provider.endpoint')).toBeTruthy();
     fireEvent.click(
       screen.getByRole('button', {
         name: 'set-embedded-indexer/provider.type',
       })
     );
-    expect(update).toHaveBeenCalledWith('indexer/enabled', false);
-    expect(update).not.toHaveBeenCalledWith(
-      'indexer/provider.type',
-      'embedded'
-    );
+    expect(update).toHaveBeenCalledWith('indexer/enabled', true);
+    expect(update).toHaveBeenCalledWith('indexer/provider.type', 'embedded');
 
     fireEvent.click(
       screen.getByRole('button', {
-        name: 'set-manticoresearch-indexer/provider.type',
+        name: 'set-elasticsearch-indexer/provider.type',
       })
     );
     expect(update).toHaveBeenCalledWith('indexer/enabled', true);
     expect(update).toHaveBeenCalledWith(
       'indexer/provider.type',
-      'manticoresearch'
+      'elasticsearch'
     );
   });
 

packages/frontend/admin/src/modules/settings/index.tsx

@@ -175,9 +175,8 @@ const AdminPanel = ({
                     {fields.map(field => {
                       const fieldKey =
                         typeof field === 'string' ? field : String(field.key);
-                      const effectiveIndexerProvider = sourceConfig?.enabled
-                        ? sourceConfig?.provider?.type
-                        : 'embedded';
+                      const effectiveIndexerProvider =
+                        sourceConfig?.provider?.type ?? 'embedded';
                       if (
                         module === 'indexer' &&
                         effectiveIndexerProvider === 'embedded' &&
@@ -219,12 +218,8 @@ const AdminPanel = ({
                             module === 'indexer' &&
                             field.key === 'provider.type'
                               ? (_path, value) => {
-                                  if (value === 'embedded') {
-                                    onUpdate('indexer/enabled', false);
-                                  } else {
-                                    onUpdate('indexer/enabled', true);
-                                    onUpdate('indexer/provider.type', value);
-                                  }
+                                  onUpdate('indexer/enabled', true);
+                                  onUpdate('indexer/provider.type', value);
                                 }
                               : onUpdate,
                         };