feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/frontend/core/src/modules/blob-management/entity/unused-blobs.spec.ts

@@ -0,0 +1,78 @@
+/** @vitest-environment happy-dom */
+
+import { Framework } from '@toeverything/infra';
+import { describe, expect, it, vi } from 'vitest';
+
+import { DocsSearchService } from '../../docs-search';
+import { WorkspaceService } from '../../workspace';
+import { WorkspaceFlavoursService } from '../../workspace/services/flavours';
+import { UnusedBlobs } from './unused-blobs';
+
+describe('UnusedBlobs', () => {
+  it('reads every used blob page from the local index', async () => {
+    const aggregate = vi
+      .fn()
+      .mockResolvedValueOnce({
+        pagination: { hasMore: true },
+        buckets: [{ key: 'used-1' }],
+      })
+      .mockResolvedValueOnce({
+        pagination: { hasMore: false },
+        buckets: [{ key: 'used-2' }],
+      });
+    const flavoursService = {
+      flavours$: {
+        value: [
+          {
+            flavour: 'local',
+            listBlobs: vi
+              .fn()
+              .mockResolvedValue([
+                { key: 'used-1' },
+                { key: 'used-2' },
+                { key: 'unused' },
+              ]),
+          },
+        ],
+      },
+    };
+    const workspaceService = {
+      workspace: {
+        id: 'workspace',
+        flavour: 'local',
+        avatar$: { value: null },
+        engine: { doc: { waitForSynced: vi.fn() } },
+      },
+    };
+    const docsSearchService = {
+      indexer: { aggregate, waitForCompleted: vi.fn() },
+    };
+    const framework = new Framework();
+    framework
+      .service(
+        WorkspaceFlavoursService,
+        flavoursService as unknown as WorkspaceFlavoursService
+      )
+      .service(
+        WorkspaceService,
+        workspaceService as unknown as WorkspaceService
+      )
+      .service(
+        DocsSearchService,
+        docsSearchService as unknown as DocsSearchService
+      )
+      .entity(UnusedBlobs, [
+        WorkspaceFlavoursService,
+        WorkspaceService,
+        DocsSearchService,
+      ]);
+    const entity = framework.provider().createEntity(UnusedBlobs);
+
+    await expect(entity.getUnusedBlobs()).resolves.toEqual([{ key: 'unused' }]);
+    expect(aggregate).toHaveBeenCalledTimes(2);
+    expect(aggregate.mock.calls.map(call => call[3])).toEqual([
+      { pagination: { limit: 1000, skip: 0 }, prefer: 'local' },
+      { pagination: { limit: 1000, skip: 1000 }, prefer: 'local' },
+    ]);
+  });
+});

packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts

@@ -97,27 +97,30 @@ export class UnusedBlobs extends Entity {
   }
 
   private async getUsedBlobs(): Promise<string[]> {
-    const result = await this.docsSearchService.indexer.aggregate(
-      'block',
-      {
-        type: 'boolean',
-        occur: 'must',
-        queries: [
-          {
-            type: 'exists',
-            field: 'blob',
-          },
-        ],
-      },
-      'blob',
-      {
-        pagination: {
-          limit: Number.MAX_SAFE_INTEGER,
+    const limit = 1000;
+    const usedBlobs: string[] = [];
+    for (let skip = 0; ; skip += limit) {
+      const result = await this.docsSearchService.indexer.aggregate(
+        'block',
+        {
+          type: 'boolean',
+          occur: 'must',
+          queries: [
+            {
+              type: 'exists',
+              field: 'blob',
+            },
+          ],
         },
+        'blob',
+        { pagination: { limit, skip }, prefer: 'local' }
+      );
+      usedBlobs.push(...result.buckets.map(bucket => bucket.key));
+      if (!result.pagination.hasMore) return usedBlobs;
+      if (result.buckets.length === 0) {
+        throw new Error('Local blob index pagination did not advance');
       }
-    );
-
-    return result.buckets.map(bucket => bucket.key);
+    }
   }
 
   async hydrateBlob(

packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts

@@ -1,3 +1,4 @@
+import { UserFriendlyError } from '@affine/error';
 import {
   catchErrorInto,
   effect,
@@ -7,7 +8,6 @@ import {
   LiveData,
   onComplete,
   onStart,
-  smartRetry,
 } from '@toeverything/infra';
 import { tap } from 'rxjs';
 
@@ -47,42 +47,57 @@ export class DocBacklinks extends Entity {
     exhaustMapWithTrailing(() =>
       fromPromise(async () => {
         const searchFromCloud =
-          this.featureFlagService.flags.enable_battery_save_mode &&
+          this.featureFlagService.flags.enable_battery_save_mode.value &&
           this.workspaceService.workspace.flavour !== 'local';
-        const { buckets } = await this.docsSearchService.indexer.aggregate(
-          'block',
-          {
-            type: 'boolean',
-            occur: 'must',
-            queries: [
-              {
-                type: 'match',
-                field: 'refDocId',
-                match: this.docService.doc.id,
-              },
-            ],
-          },
-          'docId',
-          {
-            hits: {
-              fields: [
-                'docId',
-                'blockId',
-                'parentBlockId',
-                'parentFlavour',
-                'additional',
-                'markdownPreview',
+        const aggregate = (prefer: 'local' | 'remote') =>
+          this.docsSearchService.indexer.aggregate(
+            'block',
+            {
+              type: 'boolean',
+              occur: 'must',
+              queries: [
+                {
+                  type: 'match',
+                  field: 'refDocId',
+                  match: this.docService.doc.id,
+                },
               ],
-              pagination: {
-                limit: BUILD_CONFIG.isElectron ? 100 : 5, // the max number of backlinks to show for each doc
+            },
+            'docId',
+            {
+              hits: {
+                fields: [
+                  'docId',
+                  'blockId',
+                  'parentBlockId',
+                  'parentFlavour',
+                  'additional',
+                  'markdownPreview',
+                ],
+                pagination: {
+                  limit: BUILD_CONFIG.isElectron ? 100 : 5,
+                },
               },
-            },
-            pagination: {
-              limit: 100,
-            },
-            prefer: searchFromCloud ? 'remote' : 'local',
-          }
-        );
+              pagination: {
+                limit: 100,
+              },
+              prefer,
+            }
+          );
+        const { buckets } = searchFromCloud
+          ? await aggregate('remote').catch(error => {
+              const cause = UserFriendlyError.fromAny(error);
+              if (
+                cause.is('SEARCH_PROVIDER_UNAVAILABLE') ||
+                cause.is('NETWORK_ERROR') ||
+                (cause.is('INVALID_INDEXER_INPUT') &&
+                  cause.data?.reason === 'unsupported_query')
+              ) {
+                return aggregate('local');
+              }
+              throw error;
+            })
+          : await aggregate('local');
         return buckets.flatMap(bucket => {
           const title =
             this.docsService.list.doc$(bucket.key).value?.title$.value ?? '';
@@ -140,7 +155,6 @@ export class DocBacklinks extends Entity {
           });
         });
       }).pipe(
-        smartRetry(),
         tap(backlinks => {
           this.backlinks$.value = backlinks;
         }),