feat(server): improve indexer perf (#15512)

#### PR Dependency Tree


* **PR #15512** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Search now supports generation-based indexing with embedded and remote
providers.
* Added automatic search reconciliation and improved handling of
document, workspace, and permission changes.
* Added clearer search status errors for unavailable, syncing, unready,
or failed indexes.
* Added Manticore Search end-to-end support and provider-specific search
behavior.

* **Improvements**
* Search and aggregate pagination now report returned results and
continuation status more accurately.
* Improved permission filtering to prevent inaccessible documents from
appearing in results.
  * Admin provider selection now consistently enables indexing.

* **Documentation**
* Clarified search pagination, aggregation counts, provider
configuration, and end-to-end setup.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/.env.example

@@ -11,5 +11,3 @@
 # MAILER_USER="noreply@toeverything.info"
 # MAILER_PASSWORD="affine"
 # MAILER_SECURE=false
-
-# AFFINE_INDEXER_ENABLED=true

packages/backend/server/e2e/manticore-provider/README.md

@@ -0,0 +1,45 @@
+# Manticore Search provider E2E
+
+This fixture validates the shared immutable projection contract against a real
+Manticore Search RT table. Aggregate queries remain unsupported and are validated
+through the typed fallback boundary instead.
+
+Start the external dependency:
+
+```bash
+docker compose -f packages/backend/server/e2e/manticore-provider/compose.yml up -d --wait
+```
+
+Use a disposable PostgreSQL database and run the provider-gated E2E:
+
+```bash
+DATABASE_URL=postgresql://ds:ds@localhost:55433/affine_rfc6_manticore_e2e \
+yarn workspace @affine/server prisma migrate deploy
+```
+
+Configure `packages/backend/server/config.json` with:
+
+```json
+{
+  "indexer": {
+    "enabled": true,
+    "provider": {
+      "type": "manticoresearch",
+      "endpoint": "http://127.0.0.1:9308"
+    }
+  }
+}
+```
+
+Then run:
+
+```bash
+DATABASE_URL=postgresql://ds:ds@localhost:55433/affine_rfc6_manticore_e2e \
+yarn af server e2e src/__tests__/e2e/indexer/manticore-provider.spec.ts
+```
+
+Stop only this disposable dependency when finished:
+
+```bash
+docker compose -f packages/backend/server/e2e/manticore-provider/compose.yml down -v
+```

packages/backend/server/e2e/manticore-provider/compose.yml

@@ -0,0 +1,33 @@
+services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: affine_rfc6_manticore_e2e
+      POSTGRES_USER: ds
+      POSTGRES_PASSWORD: ds
+    ports:
+      - '55433:5432'
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U ds -d affine_rfc6_manticore_e2e']
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 5s
+
+  manticore:
+    image: manticoresearch/manticore:29.0.2
+    environment:
+      searchd.listen: 'http:9308'
+      searchd.network_timeout: '30'
+    ports:
+      - '9308:9308'
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          "wget -qO- --post-data='SHOW TABLES' 'http://127.0.0.1:9308/sql?mode=raw' >/dev/null",
+        ]
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 5s

packages/backend/server/e2e/remote-provider/README.md

@@ -0,0 +1,39 @@
+# Remote provider E2E
+
+These tests exercise the Elasticsearch-compatible provider over HTTP against a
+local OpenSearch instance. Toxiproxy is included so the lease test can inject
+provider latency without changing production code.
+
+```bash
+docker compose -f packages/backend/server/e2e/remote-provider/compose.yml up -d --wait
+
+DATABASE_URL=postgresql://ds:ds@localhost:55432/affine_rfc6_remote_e2e \
+yarn workspace @affine/server prisma migrate deploy
+```
+
+Configure `packages/backend/server/config.json` with:
+
+```json
+{
+  "indexer": {
+    "enabled": true,
+    "provider": {
+      "type": "elasticsearch",
+      "endpoint": "http://127.0.0.1:8666"
+    }
+  }
+}
+```
+
+Then run:
+
+```bash
+DATABASE_URL=postgresql://ds:ds@localhost:55432/affine_rfc6_remote_e2e \
+yarn af server e2e src/__tests__/e2e/indexer/remote-provider.spec.ts
+
+docker compose -f packages/backend/server/e2e/remote-provider/compose.yml down -v
+```
+
+The suite uses a disposable PostgreSQL database and creates generation-specific
+indices. It directly deletes or mutates provider rows only as a fault-injection
+fixture; production code still repairs them through the normal reconcile path.

packages/backend/server/e2e/remote-provider/compose.yml

@@ -0,0 +1,73 @@
+services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: affine_rfc6_remote_e2e
+      POSTGRES_USER: ds
+      POSTGRES_PASSWORD: ds
+    ports:
+      - '55432:5432'
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U ds -d affine_rfc6_remote_e2e']
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 5s
+
+  opensearch:
+    image: opensearchproject/opensearch:2.19.1
+    environment:
+      discovery.type: single-node
+      DISABLE_INSTALL_DEMO_CONFIG: 'true'
+      DISABLE_SECURITY_PLUGIN: 'true'
+      bootstrap.memory_lock: 'true'
+      OPENSEARCH_JAVA_OPTS: '-Xms512m -Xmx512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    ports:
+      - '9200:9200'
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -fsS http://localhost:9200/_cluster/health >/dev/null',
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+      start_period: 10s
+
+  toxiproxy:
+    image: ghcr.io/shopify/toxiproxy:2.12.0
+    ports:
+      - '8474:8474'
+      - '8666:8666'
+
+  proxy-init:
+    image: curlimages/curl:8.12.1
+    depends_on:
+      opensearch:
+        condition: service_healthy
+      toxiproxy:
+        condition: service_started
+    entrypoint: ['/bin/sh', '-ec']
+    command:
+      - >-
+        until curl -fsS http://toxiproxy:8474/version >/dev/null; do sleep 1; done;
+        curl -fsS http://toxiproxy:8474/proxies/opensearch >/dev/null 2>&1 ||
+        curl -fsS -X POST http://toxiproxy:8474/proxies
+        -H 'content-type: application/json'
+        -d '{"name":"opensearch","listen":"0.0.0.0:8666","upstream":"opensearch:9200"}';
+        tail -f /dev/null
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -fsS http://toxiproxy:8474/proxies/opensearch >/dev/null',
+        ]
+      interval: 2s
+      timeout: 2s
+      retries: 30
+      start_period: 2s

packages/backend/server/src/__tests__/copilot/runtime-boundaries.spec.ts

@@ -10,7 +10,7 @@ import {
   type Config,
   type EventBus,
   type JobQueue,
-  SearchProviderNotFound,
+  SearchProviderUnavailable,
 } from '../../base';
 import { ServerFeature, type ServerService } from '../../core';
 import type { DocReader } from '../../core/doc';
@@ -450,7 +450,7 @@ test('document tools enforce the user-selected hard scope', async t => {
     readableAc,
     {
       searchDocsByKeyword: async () => {
-        throw new SearchProviderNotFound();
+        throw new SearchProviderUnavailable();
       },
     } as unknown as IndexerService,
     vectorSearch,

packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts

@@ -1,7 +1,6 @@
 import { getCurrentUserQuery } from '@affine/graphql';
 
 import { JobExecutor } from '../../../base/job/queue/executor';
-import { JobHandlerScanner } from '../../../base/job/queue/scanner';
 import { DatabaseDocReader, DocReader } from '../../../core/doc';
 import { RealtimeGateway } from '../../../core/realtime/gateway';
 import { createApp } from '../create-app';
@@ -43,7 +42,6 @@ e2e('should init worker service', async t => {
   await withFlavor('worker', async app => {
     const res = await app.GET('/info').expect(200);
     t.is(res.body.flavor, 'worker');
-    t.truthy(app.get(JobHandlerScanner).getHandler('indexer.indexDoc'));
     t.throws(() => app.get(RealtimeGateway));
 
     await t.throwsAsync(app.gql({ query: getCurrentUserQuery }));
@@ -55,7 +53,6 @@ e2e('should init allinone service with worker handlers', async t => {
   await withFlavor('allinone', async app => {
     const res = await app.GET('/info').expect(200);
     t.is(res.body.flavor, 'allinone');
-    t.truthy(app.get(JobHandlerScanner).getHandler('indexer.indexDoc'));
   });
 });
 

packages/backend/server/src/__tests__/e2e/config/resolver.spec.ts

@@ -1,16 +1,27 @@
 import { serverConfigQuery, ServerFeature } from '@affine/graphql';
 
+import { Config } from '../../../base';
+import { BackendRuntimeProvider } from '../../../core/backend-runtime';
+import { IndexerService } from '../../../plugins/indexer';
 import { app, e2e } from '../test';
 
-e2e('should indexer feature enabled by default', async t => {
-  const { serverConfig } = await app.gql({ query: serverConfigQuery });
+e2e(
+  'should expose the indexer feature when its projection is ready',
+  async t => {
+    const enabled = app.get(Config).indexer.enabled;
+    if (enabled) {
+      await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
+      await app.get(IndexerService).onApplicationBootstrap();
+    }
+    const { serverConfig } = await app.gql({ query: serverConfigQuery });
 
-  t.is(
-    serverConfig.features.includes(ServerFeature.Indexer),
-    true,
-    JSON.stringify(serverConfig, null, 2)
-  );
-});
+    t.is(
+      serverConfig.features.includes(ServerFeature.Indexer),
+      enabled,
+      JSON.stringify(serverConfig, null, 2)
+    );
+  }
+);
 
 e2e('should comment feature enabled by default', async t => {
   const { serverConfig } = await app.gql({ query: serverConfigQuery });

packages/backend/server/src/__tests__/e2e/create-app.ts

@@ -362,7 +362,9 @@ export async function createApp(
     await app.init();
     await app.get(BackendRuntimeProvider, { strict: false }).runMigrations();
     await app.get(StorageRuntimeProvider, { strict: false }).runMigrations();
-    await app.get(IndexerService, { strict: false }).onApplicationBootstrap();
+    if (globalThis.env.isApi || globalThis.env.isFrontend) {
+      await app.get(IndexerService, { strict: false }).onApplicationBootstrap();
+    }
   } catch (error) {
     await app.close();
     throw error;

packages/backend/server/src/__tests__/e2e/indexer/aggregate.spec.ts

@@ -4,14 +4,20 @@ import {
   SearchTable,
 } from '@affine/graphql';
 
+import { Config } from '../../../base';
+import { BackendRuntimeProvider } from '../../../core/backend-runtime';
 import { createDocWithMarkdown } from '../../../native';
-import { IndexerService } from '../../../plugins/indexer/service';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
-e2e('should aggregate by docId', async t => {
+const indexerE2e = app.get(Config).indexer.enabled ? e2e : e2e.skip;
+
+indexerE2e('should aggregate by docId', async t => {
   const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, { owner });
+  const workspace = await app.create(Mockers.Workspace, {
+    owner,
+    snapshot: true,
+  });
   for (const [docId, markdown] of [
     ['doc-0', 'hello world\n\nhello again'],
     ['doc-1', 'hello world'],
@@ -23,7 +29,7 @@ e2e('should aggregate by docId', async t => {
       user: owner,
       blob: createDocWithMarkdown(docId, markdown, docId),
     });
-    await app.get(IndexerService).indexDoc(workspace.id, docId);
+    await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
   }
 
   const result = await app.gql({

packages/backend/server/src/__tests__/e2e/indexer/manticore-provider.spec.ts

@@ -0,0 +1,186 @@
+import {
+  indexerSearchDocsQuery,
+  indexerSearchQuery,
+  SearchQueryType,
+  SearchTable,
+} from '@affine/graphql';
+
+import { Config } from '../../../base';
+import { BackendRuntimeProvider } from '../../../core/backend-runtime';
+import { DocRole } from '../../../models';
+import { createDocWithMarkdown } from '../../../native';
+import { Mockers } from '../../mocks';
+import { app, e2e } from '../test';
+
+const manticoreSearchEnabled =
+  app.get(Config).indexer.enabled &&
+  app.get(Config).indexer.provider.type === 'manticoresearch';
+const manticoreSearchE2e = manticoreSearchEnabled ? e2e : e2e.skip;
+
+async function indexDocument(
+  workspaceId: string,
+  user: { id: string },
+  docId: string,
+  markdown: string,
+  defaultRole = DocRole.Manager
+) {
+  await app.create(Mockers.DocMeta, { workspaceId, docId, defaultRole });
+  await app.create(Mockers.DocSnapshot, {
+    workspaceId,
+    docId,
+    user,
+    blob: createDocWithMarkdown(docId, markdown, docId),
+  });
+  await reconcileSearch();
+}
+
+async function reconcileSearch() {
+  const runtime = app.get(BackendRuntimeProvider);
+  for (let attempt = 0; attempt < 50; attempt++) {
+    await runtime.reconcileSearchProjection(1000);
+    if ((await runtime.searchStatus()).ready) return;
+  }
+  throw new Error('search projection did not become ready');
+}
+
+async function searchPage(
+  workspaceId: string,
+  match: string,
+  pagination: { limit: number; cursor?: string }
+) {
+  const result = await app.gql({
+    query: indexerSearchQuery,
+    variables: {
+      id: workspaceId,
+      input: {
+        table: SearchTable.block,
+        query: {
+          type: SearchQueryType.match,
+          field: 'content',
+          match,
+        },
+        options: {
+          fields: ['docId', 'blockId'],
+          pagination,
+        },
+      },
+    },
+  });
+  return result.workspace.search;
+}
+
+async function searchCount(workspaceId: string, match: string) {
+  return (await searchPage(workspaceId, match, { limit: 20 })).pagination.count;
+}
+
+async function searchDocsCount(workspaceId: string, keyword: string) {
+  const result = await app.gql({
+    query: indexerSearchDocsQuery,
+    variables: {
+      id: workspaceId,
+      input: { keyword, limit: 20 },
+    },
+  });
+  return result.workspace.searchDocs.length;
+}
+
+manticoreSearchE2e(
+  'indexes and searches through the Manticore Search provider',
+  async t => {
+    const owner = await app.signup();
+    const workspace = await app.create(Mockers.Workspace, {
+      owner,
+      snapshot: true,
+    });
+    const docId = `manticore-basic-${Date.now()}`;
+    const marker = 'manticorebasicmarker';
+    await indexDocument(
+      workspace.id,
+      owner,
+      docId,
+      `${marker} first block\n\nsecond block`
+    );
+    await indexDocument(
+      workspace.id,
+      owner,
+      `${docId}-second`,
+      `${marker} from a second document`
+    );
+
+    t.is(await searchCount(workspace.id, marker), 2);
+    const basicDocs = await app.gql({
+      query: indexerSearchDocsQuery,
+      variables: {
+        id: workspace.id,
+        input: { keyword: marker, limit: 2 },
+      },
+    });
+    t.is(basicDocs.workspace.searchDocs.length, 2);
+    await t.throwsAsync(
+      app.gql({
+        query: indexerSearchDocsQuery,
+        variables: {
+          id: workspace.id,
+          input: { keyword: marker, limit: 0 },
+        },
+      })
+    );
+
+    const firstPage = await searchPage(workspace.id, marker, { limit: 1 });
+    const nextCursor = firstPage.pagination.nextCursor;
+    t.truthy(nextCursor);
+    const secondPage = await searchPage(workspace.id, marker, {
+      limit: 1,
+      cursor: nextCursor ?? undefined,
+    });
+    t.is(secondPage.nodes.length, 1);
+    t.not(
+      firstPage.nodes[0]?.fields.docId[0],
+      secondPage.nodes[0]?.fields.docId[0]
+    );
+  }
+);
+
+manticoreSearchE2e(
+  'enforces ACL changes through the Manticore Search provider',
+  async t => {
+    const owner = await app.signup();
+    const workspace = await app.create(Mockers.Workspace, {
+      owner,
+      snapshot: true,
+    });
+    await app.create(Mockers.TeamWorkspace, { id: workspace.id });
+    const marker = `manticore-acl-${Date.now()}`;
+    await indexDocument(
+      workspace.id,
+      owner,
+      `${marker}-doc`,
+      marker,
+      DocRole.None
+    );
+
+    const member = await app.signup();
+    await app.create(Mockers.WorkspaceUser, {
+      workspaceId: workspace.id,
+      userId: member.id,
+    });
+    await reconcileSearch();
+    t.is(await searchCount(workspace.id, marker), 0);
+    t.is(await searchDocsCount(workspace.id, marker), 0);
+
+    await app.create(Mockers.DocUser, {
+      workspaceId: workspace.id,
+      docId: `${marker}-doc`,
+      userId: member.id,
+      type: DocRole.Reader,
+    });
+    await reconcileSearch();
+    t.true((await searchCount(workspace.id, marker)) > 0);
+    t.true((await searchDocsCount(workspace.id, marker)) > 0);
+
+    await app.models.docUser.delete(workspace.id, `${marker}-doc`, member.id);
+    await reconcileSearch();
+    t.is(await searchCount(workspace.id, marker), 0);
+    t.is(await searchDocsCount(workspace.id, marker), 0);
+  }
+);

packages/backend/server/src/__tests__/e2e/indexer/remote-provider.spec.ts

@@ -0,0 +1,83 @@
+import {
+  indexerAggregateQuery,
+  SearchQueryType,
+  SearchTable,
+} from '@affine/graphql';
+
+import { Config } from '../../../base';
+import { BackendRuntimeProvider } from '../../../core/backend-runtime';
+import { createDocWithMarkdown } from '../../../native';
+import { Mockers } from '../../mocks';
+import { app, e2e } from '../test';
+
+const indexer = app.get(Config).indexer;
+const remoteE2e =
+  indexer.enabled && indexer.provider.type === 'elasticsearch' ? e2e : e2e.skip;
+
+async function indexDocument(
+  workspaceId: string,
+  user: { id: string },
+  docId: string,
+  markdown: string
+) {
+  await app.create(Mockers.DocMeta, { workspaceId, docId });
+  await app.create(Mockers.DocSnapshot, {
+    workspaceId,
+    docId,
+    user,
+    blob: createDocWithMarkdown(docId, markdown, docId),
+  });
+  const runtime = app.get(BackendRuntimeProvider);
+  for (let attempt = 0; attempt < 50; attempt++) {
+    await runtime.reconcileSearchProjection(1000);
+    if ((await runtime.searchStatus()).ready) return;
+  }
+  throw new Error('search projection did not become ready');
+}
+
+remoteE2e(
+  'exposes remote aggregation through the GraphQL contract',
+  async t => {
+    const owner = await app.signup();
+    const workspace = await app.create(Mockers.Workspace, {
+      owner,
+      snapshot: true,
+    });
+    const suffix = Date.now();
+    for (const index of [0, 1, 2]) {
+      await indexDocument(
+        workspace.id,
+        owner,
+        `remote-aggregate-${suffix}-${index}`,
+        `remote aggregate marker ${index}`
+      );
+    }
+
+    const result = await app.gql({
+      query: indexerAggregateQuery,
+      variables: {
+        id: workspace.id,
+        input: {
+          table: SearchTable.block,
+          query: {
+            type: SearchQueryType.match,
+            field: 'content',
+            match: 'remote aggregate marker',
+          },
+          field: 'docId',
+          options: {
+            pagination: { limit: 2, skip: 0 },
+            hits: {
+              pagination: { limit: 1, skip: 0 },
+              fields: ['docId', 'blockId'],
+            },
+          },
+        },
+      },
+    });
+
+    t.is(result.workspace.aggregate.buckets.length, 2);
+    t.is(result.workspace.aggregate.pagination.count, 2);
+    t.true(result.workspace.aggregate.pagination.hasMore);
+  }
+);

packages/backend/server/src/__tests__/e2e/indexer/search-docs.spec.ts

@@ -1,15 +1,19 @@
 import { indexerSearchDocsQuery } from '@affine/graphql';
 
-import { ConfigFactory } from '../../../base';
+import { Config } from '../../../base';
+import { BackendRuntimeProvider } from '../../../core/backend-runtime';
 import { createDocWithMarkdown } from '../../../native';
-import { SearchProviderType } from '../../../plugins/indexer/config';
-import { IndexerService } from '../../../plugins/indexer/service';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
-e2e('should search docs by keyword', async t => {
+const indexerE2e = app.get(Config).indexer.enabled ? e2e : e2e.skip;
+
+indexerE2e('should search docs by keyword', async t => {
   const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, { owner });
+  const workspace = await app.create(Mockers.Workspace, {
+    owner,
+    snapshot: true,
+  });
   for (const docId of ['doc-0', 'doc-1', 'doc-2']) {
     await app.create(Mockers.DocMeta, { workspaceId: workspace.id, docId });
     await app.create(Mockers.DocSnapshot, {
@@ -18,33 +22,26 @@ e2e('should search docs by keyword', async t => {
       user: owner,
       blob: createDocWithMarkdown(docId, `${docId} hello`, docId),
     });
-    await app.get(IndexerService).indexDoc(workspace.id, docId);
+    await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
   }
 
   const search = app.gql({
     query: indexerSearchDocsQuery,
     variables: { id: workspace.id, input: { keyword: 'hello', limit: 2 } },
   });
-  if (
-    app.get(ConfigFactory).config.indexer.provider.type ===
-    SearchProviderType.Manticoresearch
-  ) {
-    await t.throwsAsync(search, {
-      message: /Invalid indexer input: unsupported_query/,
-    });
-    return;
-  }
-
   const result = await search;
   t.is(result.workspace.searchDocs.length, 2);
   t.true(result.workspace.searchDocs.every(doc => doc.highlight.length > 0));
 });
 
-e2e(
+indexerE2e(
   'should search docs by keyword failed when workspace is no permission',
   async t => {
     const owner = await app.signup();
-    const workspace = await app.create(Mockers.Workspace, { owner });
+    const workspace = await app.create(Mockers.Workspace, {
+      owner,
+      snapshot: true,
+    });
     await app.signup();
     await t.throwsAsync(
       app.gql({

packages/backend/server/src/__tests__/e2e/indexer/search.spec.ts

@@ -4,12 +4,15 @@ import {
   SearchTable,
 } from '@affine/graphql';
 
+import { Config } from '../../../base';
+import { BackendRuntimeProvider } from '../../../core/backend-runtime';
 import { DocRole } from '../../../models';
 import { createDocWithMarkdown } from '../../../native';
-import { IndexerService } from '../../../plugins/indexer/service';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
+const indexerE2e = app.get(Config).indexer.enabled ? e2e : e2e.skip;
+
 async function indexDoc(
   workspaceId: string,
   user: { id: string },
@@ -24,12 +27,15 @@ async function indexDoc(
     user,
     blob: createDocWithMarkdown(docId, markdown, docId),
   });
-  await app.get(IndexerService).indexDoc(workspaceId, docId);
+  await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
 }
 
-e2e('should search with query', async t => {
+indexerE2e('should search with query', async t => {
   const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, { owner });
+  const workspace = await app.create(Mockers.Workspace, {
+    owner,
+    snapshot: true,
+  });
   await indexDoc(
     workspace.id,
     owner,
@@ -114,100 +120,116 @@ e2e('should search with query', async t => {
   );
 });
 
-e2e('should filter no read permission docs on team workspace', async t => {
-  const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, { owner });
-  await app.create(Mockers.TeamWorkspace, { id: workspace.id });
-  await indexDoc(
-    workspace.id,
-    owner,
-    'private-doc',
-    'team secret searchable',
-    DocRole.None
-  );
+indexerE2e(
+  'should filter no read permission docs on team workspace',
+  async t => {
+    const owner = await app.signup();
+    const workspace = await app.create(Mockers.Workspace, {
+      owner,
+      snapshot: true,
+    });
+    await app.create(Mockers.TeamWorkspace, { id: workspace.id });
+    await indexDoc(
+      workspace.id,
+      owner,
+      'private-doc',
+      'team secret searchable',
+      DocRole.None
+    );
 
-  const member = await app.signup();
-  await app.create(Mockers.WorkspaceUser, {
-    workspaceId: workspace.id,
-    userId: member.id,
-  });
-  await app.get(IndexerService).reconcileWorkspace(workspace.id);
-  const denied = await app.gql({
-    query: indexerSearchQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        table: SearchTable.block,
-        query: {
-          type: SearchQueryType.match,
-          field: 'content',
-          match: 'secret',
+    const member = await app.signup();
+    await app.create(Mockers.WorkspaceUser, {
+      workspaceId: workspace.id,
+      userId: member.id,
+    });
+    await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
+    const denied = await app.gql({
+      query: indexerSearchQuery,
+      variables: {
+        id: workspace.id,
+        input: {
+          table: SearchTable.block,
+          query: {
+            type: SearchQueryType.match,
+            field: 'content',
+            match: 'secret',
+          },
+          options: { fields: ['docId'], pagination: { limit: 10 } },
         },
-        options: { fields: ['docId'], pagination: { limit: 10 } },
       },
-    },
-  });
-  t.is(denied.workspace.search.pagination.count, 0);
+    });
+    t.is(denied.workspace.search.pagination.count, 0);
 
-  await app.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'private-doc',
-    userId: member.id,
-    type: DocRole.Reader,
-  });
-  await app.get(IndexerService).reconcileWorkspace(workspace.id);
-  const allowed = await app.gql({
-    query: indexerSearchQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        table: SearchTable.block,
-        query: {
-          type: SearchQueryType.match,
-          field: 'content',
-          match: 'secret',
+    await app.create(Mockers.DocUser, {
+      workspaceId: workspace.id,
+      docId: 'private-doc',
+      userId: member.id,
+      type: DocRole.Reader,
+    });
+    await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
+    const allowed = await app.gql({
+      query: indexerSearchQuery,
+      variables: {
+        id: workspace.id,
+        input: {
+          table: SearchTable.block,
+          query: {
+            type: SearchQueryType.match,
+            field: 'content',
+            match: 'secret',
+          },
+          options: { fields: ['docId'], pagination: { limit: 10 } },
         },
-        options: { fields: ['docId'], pagination: { limit: 10 } },
       },
-    },
-  });
-  t.true(allowed.workspace.search.pagination.count > 0);
+    });
+    t.true(allowed.workspace.search.pagination.count > 0);
 
-  await app.models.docUser.delete(workspace.id, 'private-doc', member.id);
-  await app.get(IndexerService).reconcileWorkspace(workspace.id);
-  const revoked = await app.gql({
-    query: indexerSearchQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        table: SearchTable.block,
-        query: {
-          type: SearchQueryType.match,
-          field: 'content',
-          match: 'secret',
+    await app.models.docUser.delete(workspace.id, 'private-doc', member.id);
+    await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
+    const revoked = await app.gql({
+      query: indexerSearchQuery,
+      variables: {
+        id: workspace.id,
+        input: {
+          table: SearchTable.block,
+          query: {
+            type: SearchQueryType.match,
+            field: 'content',
+            match: 'secret',
+          },
+          options: { fields: ['docId'], pagination: { limit: 10 } },
         },
-        options: { fields: ['docId'], pagination: { limit: 10 } },
       },
-    },
-  });
-  t.is(revoked.workspace.search.pagination.count, 0);
-});
+    });
+    t.is(revoked.workspace.search.pagination.count, 0);
+  }
+);
 
-e2e('should return empty results when search not match any docs', async t => {
-  const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, { owner });
-  await app.get(IndexerService).reconcileWorkspace(workspace.id);
-  const result = await app.gql({
-    query: indexerSearchQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        table: SearchTable.doc,
-        query: { type: SearchQueryType.match, field: 'title', match: 'absent' },
-        options: { fields: ['docId'], pagination: { limit: 10 } },
+indexerE2e(
+  'should return empty results when search not match any docs',
+  async t => {
+    const owner = await app.signup();
+    const workspace = await app.create(Mockers.Workspace, {
+      owner,
+      snapshot: true,
+    });
+    await app.get(BackendRuntimeProvider).reconcileSearchProjection(1000);
+    const result = await app.gql({
+      query: indexerSearchQuery,
+      variables: {
+        id: workspace.id,
+        input: {
+          table: SearchTable.doc,
+          query: {
+            type: SearchQueryType.match,
+            field: 'title',
+            match: 'absent',
+          },
+          options: { fields: ['docId'], pagination: { limit: 10 } },
+        },
       },
-    },
-  });
-  t.is(result.workspace.search.pagination.count, 0);
-  t.deepEqual(result.workspace.search.nodes, []);
-});
+    });
+    t.is(result.workspace.search.pagination.count, 0);
+    t.deepEqual(result.workspace.search.nodes, []);
+  }
+);

packages/backend/server/src/__tests__/utils/utils.ts

@@ -6,14 +6,18 @@ export const TEST_LOG_LEVEL: LogLevel =
   (process.env.TEST_LOG_LEVEL as LogLevel) ?? 'fatal';
 
 async function flushDB(client: PrismaClient) {
-  const result: { tablename: string }[] =
-    await client.$queryRaw`SELECT tablename
+  const result: { schemaname: string; tablename: string }[] =
+    await client.$queryRaw`SELECT schemaname, tablename
                            FROM pg_catalog.pg_tables
                            WHERE schemaname != 'pg_catalog'
                              AND schemaname != 'information_schema'`;
   const query = `TRUNCATE TABLE ${result
-    .map(({ tablename }) => tablename)
-    .filter(name => !name.includes('migrations'))
+    .filter(({ tablename }) => !tablename.includes('migrations'))
+    .map(({ schemaname, tablename }) =>
+      [schemaname, tablename]
+        .map(identifier => `"${identifier.replaceAll('"', '""')}"`)
+        .join('.')
+    )
     .join(', ')}`;
 
   for (let attempt = 0; attempt < 3; attempt++) {

packages/backend/server/src/app.module.ts

@@ -61,7 +61,7 @@ import { CalendarModule } from './plugins/calendar';
 import { CaptchaModule } from './plugins/captcha';
 import { CopilotModule } from './plugins/copilot';
 import { GCloudModule } from './plugins/gcloud';
-import { IndexerModule, IndexerWorkerModule } from './plugins/indexer';
+import { IndexerModule } from './plugins/indexer';
 import { LicenseModule } from './plugins/license';
 import { OAuthModule } from './plugins/oauth';
 import { PaymentModule } from './plugins/payment';
@@ -169,9 +169,8 @@ export function buildAppModule(env: Env) {
     .use(...FunctionalityModules)
     .useIf(() => !workerOnly, RealtimeGatewayModule)
 
-    // online roles publish indexer events; only the worker registers consumers
+    // Search API and worker runtime are separate from the queue worker application.
     .useIf(() => env.isApi || env.isFrontend, IndexerModule)
-    .useIf(() => env.isWorker, IndexerWorkerModule)
 
     // the worker owns doc consumers and schedulers
     .useIf(() => env.isWorker, DocJobsModule)

packages/backend/server/src/base/error/def.ts

@@ -14,6 +14,7 @@ export type UserFriendlyErrorBaseType =
   | 'no_permission'
   | 'quota_exceeded'
   | 'authentication_required'
+  | 'service_unavailable'
   | 'internal_server_error';
 
 type ErrorArgType = 'string' | 'number' | 'boolean';
@@ -36,6 +37,7 @@ const BaseTypeToHttpStatusMap: Record<UserFriendlyErrorBaseType, HttpStatus> = {
   no_permission: HttpStatus.FORBIDDEN,
   quota_exceeded: HttpStatus.PAYMENT_REQUIRED,
   authentication_required: HttpStatus.UNAUTHORIZED,
+  service_unavailable: HttpStatus.SERVICE_UNAVAILABLE,
   internal_server_error: HttpStatus.INTERNAL_SERVER_ERROR,
 };
 
@@ -969,6 +971,25 @@ export const USER_FRIENDLY_ERRORS = {
   },
 
   // indexer errors
+  search_index_not_ready: {
+    type: 'service_unavailable',
+    args: { spaceId: 'string' },
+    message: ({ spaceId }) =>
+      `Search index for Space ${spaceId} is not ready yet.`,
+  },
+  search_permission_syncing: {
+    type: 'service_unavailable',
+    message: 'Search permissions are still syncing. Please try again shortly.',
+  },
+  search_provider_unavailable: {
+    type: 'service_unavailable',
+    message: 'Search provider is temporarily unavailable.',
+  },
+  search_index_failed: {
+    type: 'service_unavailable',
+    args: { diagnosticId: 'string' },
+    message: 'Search index is temporarily unavailable.',
+  },
   search_provider_not_found: {
     type: 'resource_not_found',
     message: 'Search provider not found.',

packages/backend/server/src/base/error/errors.gen.ts

@@ -1123,6 +1123,38 @@ export class InvalidAppConfigInput extends UserFriendlyError {
     super('invalid_input', 'invalid_app_config_input', message, args);
   }
 }
+@ObjectType()
+class SearchIndexNotReadyDataType {
+  @Field() spaceId!: string
+}
+
+export class SearchIndexNotReady extends UserFriendlyError {
+  constructor(args: SearchIndexNotReadyDataType, message?: string | ((args: SearchIndexNotReadyDataType) => string)) {
+    super('service_unavailable', 'search_index_not_ready', message, args);
+  }
+}
+
+export class SearchPermissionSyncing extends UserFriendlyError {
+  constructor(message?: string) {
+    super('service_unavailable', 'search_permission_syncing', message);
+  }
+}
+
+export class SearchProviderUnavailable extends UserFriendlyError {
+  constructor(message?: string) {
+    super('service_unavailable', 'search_provider_unavailable', message);
+  }
+}
+@ObjectType()
+class SearchIndexFailedDataType {
+  @Field() diagnosticId!: string
+}
+
+export class SearchIndexFailed extends UserFriendlyError {
+  constructor(args: SearchIndexFailedDataType, message?: string | ((args: SearchIndexFailedDataType) => string)) {
+    super('service_unavailable', 'search_index_failed', message, args);
+  }
+}
 
 export class SearchProviderNotFound extends UserFriendlyError {
   constructor(message?: string) {
@@ -1320,6 +1352,10 @@ export enum ErrorNames {
   MENTION_USER_ONESELF_DENIED,
   INVALID_APP_CONFIG,
   INVALID_APP_CONFIG_INPUT,
+  SEARCH_INDEX_NOT_READY,
+  SEARCH_PERMISSION_SYNCING,
+  SEARCH_PROVIDER_UNAVAILABLE,
+  SEARCH_INDEX_FAILED,
   SEARCH_PROVIDER_NOT_FOUND,
   INVALID_SEARCH_PROVIDER_REQUEST,
   INVALID_INDEXER_INPUT,
@@ -1335,5 +1371,5 @@ registerEnumType(ErrorNames, {
 export const ErrorDataUnionType = createUnionType({
   name: 'ErrorDataUnion',
   types: () =>
-    [GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotFailedToAddWorkspaceArtifactDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, UnsupportedServerVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
+    [GraphqlBadRequestDataType, HttpRequestErrorDataType, SsrfBlockedErrorDataType, ResponseTooLargeErrorDataType, ImageFormatNotSupportedDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidOauthResponseDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CalendarProviderRequestErrorDataType, NoCopilotProviderAvailableDataType, CopilotFailedToGenerateEmbeddingDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderNotSupportedDataType, CopilotProviderSideErrorDataType, CopilotFailedToAddWorkspaceArtifactDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, UnsupportedServerVersionDataType, MentionUserDocAccessDeniedDataType, InvalidAppConfigDataType, InvalidAppConfigInputDataType, SearchIndexNotReadyDataType, SearchIndexFailedDataType, InvalidSearchProviderRequestDataType, InvalidIndexerInputDataType] as const,
 });

packages/backend/server/src/base/job/queue/config.ts

@@ -71,14 +71,6 @@ defineModuleConfig('job', {
     schema,
   },
 
-  'queues.indexer': {
-    desc: 'The config for indexer job queue',
-    default: {
-      concurrency: 1,
-    },
-    schema,
-  },
-
   'queues.notification': {
     desc: 'The config for notification job queue',
     default: {

packages/backend/server/src/base/job/queue/def.ts

@@ -27,7 +27,6 @@ export enum Queue {
   NOTIFICATION = 'notification',
   DOC = 'doc',
   COPILOT = 'copilot',
-  INDEXER = 'indexer',
   CALENDAR = 'calendar',
   BACKENDRUNTIME = 'backendRuntime',
   INVITE_ABUSE = 'inviteAbuse',

packages/backend/server/src/base/job/queue/owner.ts

@@ -1,11 +1,7 @@
 import { ServerRole } from '../../../env';
 import { Queue, QUEUES } from './def';
 
-export const WORKER_QUEUES = [
-  Queue.DOC,
-  Queue.INDEXER,
-  Queue.BACKENDRUNTIME,
-] as const;
+export const WORKER_QUEUES = [Queue.DOC, Queue.BACKENDRUNTIME] as const;
 
 export function queuesForRole(role: ServerRole | undefined): Queue[] {
   switch (role) {

packages/backend/server/src/base/metrics/metrics.ts

@@ -63,6 +63,7 @@ export type KnownMetricScopes =
   | 'storage'
   | 'process'
   | 'permission'
+  | 'search'
   | 'workspace';
 
 const metricCreators: MetricCreators = {

packages/backend/server/src/core/backend-runtime/index.ts

@@ -5,6 +5,7 @@ import {
   BackendRuntimeEmbeddingProducer,
   BackendRuntimeEmbeddingService,
   BackendRuntimeHousekeepingJob,
+  BackendRuntimeSearchJob,
 } from './job';
 import {
   BACKEND_RUNTIME_CONFIG_PATHS,
@@ -33,7 +34,11 @@ export class BackendRuntimeProducerModule {}
 
 @Module({
   imports: [BackendRuntimeModule],
-  providers: [BackendRuntimeEmbeddingJob, BackendRuntimeHousekeepingJob],
+  providers: [
+    BackendRuntimeEmbeddingJob,
+    BackendRuntimeHousekeepingJob,
+    BackendRuntimeSearchJob,
+  ],
 })
 export class BackendRuntimeWorkerModule {}
 
@@ -42,6 +47,7 @@ export {
   BackendRuntimeEmbeddingProducer,
   BackendRuntimeEmbeddingService,
   BackendRuntimeHousekeepingJob,
+  BackendRuntimeSearchJob,
 } from './job';
 export {
   BACKEND_RUNTIME_CONFIG_PATHS,

packages/backend/server/src/core/backend-runtime/job.ts

@@ -2,11 +2,13 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
 import {
+  ConfigFactory,
   CopilotSelectedSourcesFailed,
   CopilotSelectedSourcesLimitExceeded,
   CopilotSelectedSourcesProcessing,
   CopilotSelectedSourcesUnavailable,
   JobQueue,
+  metrics,
   OnEvent,
   OnJob,
 } from '../../base';
@@ -30,6 +32,9 @@ declare global {
     'backendRuntime.reconcileDocumentEmbeddings': {
       workspaceId: string;
     };
+    'backendRuntime.reconcileSearchProjection': {
+      limit?: number;
+    };
   }
 }
 
@@ -257,3 +262,98 @@ export class BackendRuntimeHousekeepingJob {
     return total;
   }
 }
+
+@Injectable()
+export class BackendRuntimeSearchJob {
+  constructor(
+    private readonly rt: BackendRuntimeProvider,
+    private readonly queue: JobQueue,
+    private readonly config: ConfigFactory
+  ) {}
+
+  @Cron(CronExpression.EVERY_30_SECONDS)
+  async scheduleReconciliation() {
+    if (!this.config.config.indexer.enabled) return;
+    await this.queue.add(
+      'backendRuntime.reconcileSearchProjection',
+      { limit: 100 },
+      { jobId: 'backend-runtime-search-reconciliation', removeOnFail: true }
+    );
+  }
+
+  @OnJob('backendRuntime.reconcileSearchProjection')
+  async reconcileProjection({
+    limit = 100,
+  }: Jobs['backendRuntime.reconcileSearchProjection']) {
+    if (!this.config.config.indexer.enabled) return 0;
+    const startedAt = performance.now();
+    try {
+      const reconciled = await this.rt.reconcileSearchProjection(limit);
+      const status = (await this.rt.searchStatus()) as {
+        ready?: boolean;
+        state?: string;
+        metrics?: {
+          scanCursor?: number;
+          scanHighWater?: number;
+          pendingPublications?: number;
+          gcBacklog?: number;
+          providerRequests?: number;
+          providerLatencyMicrosAvg?: number;
+          generationGcFailures?: number;
+          filterDrops?: {
+            missingPublished?: number;
+            projectionMismatch?: number;
+            canonicalPermission?: number;
+          };
+        };
+      };
+      metrics.search.counter('reconcile_runs').add(1);
+      metrics.search.gauge('reconciled_workspaces').record(reconciled);
+      metrics.search.gauge('generation_ready').record(status.ready ? 1 : 0, {
+        state: status.state ?? 'unknown',
+      });
+      metrics.search
+        .histogram('reconcile_latency_ms')
+        .record(performance.now() - startedAt);
+      const projection = status.metrics;
+      if (projection) {
+        metrics.search.gauge('scan_cursor').record(projection.scanCursor ?? 0);
+        metrics.search
+          .gauge('scan_high_water')
+          .record(projection.scanHighWater ?? 0);
+        metrics.search
+          .gauge('pending_publications')
+          .record(projection.pendingPublications ?? 0);
+        metrics.search.gauge('gc_backlog').record(projection.gcBacklog ?? 0);
+        metrics.search
+          .gauge('provider_requests')
+          .record(projection.providerRequests ?? 0);
+        metrics.search
+          .gauge('provider_latency_micros_avg')
+          .record(projection.providerLatencyMicrosAvg ?? 0);
+        metrics.search
+          .gauge('generation_gc_failures')
+          .record(projection.generationGcFailures ?? 0);
+        metrics.search
+          .gauge('filter_drops')
+          .record(projection.filterDrops?.missingPublished ?? 0, {
+            reason: 'missing_published',
+          });
+        metrics.search
+          .gauge('filter_drops')
+          .record(projection.filterDrops?.projectionMismatch ?? 0, {
+            reason: 'projection_mismatch',
+          });
+        metrics.search
+          .gauge('filter_drops')
+          .record(projection.filterDrops?.canonicalPermission ?? 0, {
+            reason: 'canonical_permission',
+          });
+      }
+      return reconciled;
+    } catch (error) {
+      metrics.search.counter('reconcile_failures').add(1);
+      throw error;
+    }
+  }
+}

packages/backend/server/src/core/backend-runtime/provider.ts

@@ -385,27 +385,9 @@ export class BackendRuntimeProvider
     );
   }
 
-  async indexSearchDocument(workspaceId: string, docId: string) {
-    await this.measured('indexSearchDocument', runtime =>
-      runtime.indexSearchDocument(workspaceId, docId)
-    );
-  }
-
-  async deleteSearchDocument(workspaceId: string, docId: string) {
-    await this.measured('deleteSearchDocument', runtime =>
-      runtime.deleteSearchDocument(workspaceId, docId)
-    );
-  }
-
-  async reconcileSearchWorkspace(workspaceId: string) {
-    await this.measured('reconcileSearchWorkspace', runtime =>
-      runtime.reconcileSearchWorkspace(workspaceId)
-    );
-  }
-
-  async deleteSearchWorkspace(workspaceId: string) {
-    await this.measured('deleteSearchWorkspace', runtime =>
-      runtime.deleteSearchWorkspace(workspaceId)
+  async reconcileSearchProjection(limit = 100) {
+    return await this.measured('reconcileSearchProjection', runtime =>
+      runtime.reconcileSearchProjection(limit)
     );
   }
 

packages/backend/server/src/core/config/__tests__/service.spec.ts

@@ -7,6 +7,7 @@ import { createModule } from '../../../__tests__/create-module';
 import { Mockers } from '../../../__tests__/mocks';
 import { InvalidAppConfigInput } from '../../../base';
 import { Models } from '../../../models';
+import { SearchProviderType } from '../../../plugins/indexer/config';
 import { ServerService } from '../service';
 
 const module = await createModule({
@@ -40,6 +41,19 @@ test('should update config', async t => {
   t.is(service.getConfig().server.externalUrl, newValue);
 });
 
+test('should enable the selected indexer provider', async t => {
+  await service.updateConfig(user.id, [
+    {
+      module: 'indexer',
+      key: 'provider.type',
+      value: SearchProviderType.Embedded,
+    },
+  ]);
+
+  t.true(service.getConfig().indexer.enabled);
+  t.is(service.getConfig().indexer.provider.type, SearchProviderType.Embedded);
+});
+
 test('should validate config before update', async t => {
   await t.throwsAsync(
     service.updateConfig(user.id, [

packages/backend/server/src/core/config/service.ts

@@ -76,15 +76,7 @@ export class ServerService implements OnApplicationBootstrap {
     const providerType = updates.find(
       update => update.module === 'indexer' && update.key === 'provider.type'
     );
-    if (providerType?.value === 'embedded') {
-      updates = updates.filter(update => update !== providerType);
-      updates = [
-        ...updates.filter(
-          update => !(update.module === 'indexer' && update.key === 'enabled')
-        ),
-        { module: 'indexer', key: 'enabled', value: false },
-      ];
-    } else if (providerType) {
+    if (providerType) {
       updates = [
         ...updates.filter(
           update => !(update.module === 'indexer' && update.key === 'enabled')

packages/backend/server/src/core/permission/__tests__/service.spec.ts

@@ -1,6 +1,13 @@
 import test from 'ava';
 
-import { InternalServerError } from '../../../base';
+import {
+  InternalServerError,
+  SearchIndexFailed,
+  SearchIndexNotReady,
+  SearchPermissionSyncing,
+  SearchProviderUnavailable,
+  SpaceAccessDenied,
+} from '../../../base';
 import { DocRole } from '../../../models';
 import { docLegacyBoundary } from '../context';
 import { PermissionContextLoader } from '../context-loader';
@@ -23,9 +30,11 @@ function createLoader() {
     docPolicies: 0,
     docGrants: 0,
   };
+  const queries: string[] = [];
   const db = {
     $queryRaw: async (strings: TemplateStringsArray) => {
       const sql = strings.join('');
+      queries.push(sql);
       if (sql.includes('FROM workspace_members')) {
         calls.members += 1;
         return [{ role: 'owner', state: 'active' }];
@@ -82,6 +91,7 @@ function createLoader() {
   };
   return {
     calls,
+    queries,
     loader: new PermissionContextLoader(db as never, createCls() as never),
   };
 }
@@ -196,7 +206,7 @@ test('PermissionService supports anonymous preview without doc read', async t =>
 });
 
 test('PermissionContextLoader reads only terminal permission tables', async t => {
-  const { loader } = createLoader();
+  const { loader, queries } = createLoader();
   const input = await loader.load({
     userId: 'u1',
     workspaceId: 'w1',
@@ -212,6 +222,9 @@ test('PermissionContextLoader reads only terminal permission tables', async t =>
   t.is(input.docs?.[0]?.explicitUserRole, 'manager');
   t.is(input.docs?.[0]?.memberDefaultRole, 'manager');
   t.is(input.docs?.[1]?.publicRole, 'external');
+  t.false(
+    queries.some(query => /workspace_permission_|search_runtime_/i.test(query))
+  );
 });
 
 test('PermissionContextLoader treats missing quota state as unknown and stale', async t => {
@@ -261,3 +274,11 @@ test('PermissionService maps native validation errors to internal errors', t =>
 
   t.true(error instanceof InternalServerError);
 });
+
+test('search and permission boundaries expose stable HTTP contracts', t => {
+  t.is(new SearchIndexNotReady({ spaceId: 'w1' }).status, 503);
+  t.is(new SearchPermissionSyncing().status, 503);
+  t.is(new SearchProviderUnavailable().status, 503);
+  t.is(new SearchIndexFailed({ diagnosticId: 'w1' }).status, 503);
+  t.is(new SpaceAccessDenied({ spaceId: 'w1' }).status, 403);
+});

packages/backend/server/src/core/storage-runtime/provider.ts

@@ -287,17 +287,6 @@ export class StorageRuntimeProvider
     );
   }
 
-  async ackDocumentCleanupEffect(
-    workspaceId: string,
-    docId: string,
-    cleanupVersion: string,
-    effect: 'search' | 'copilot'
-  ) {
-    return await this.measured('ackDocumentCleanupEffect', rt =>
-      rt.ackDocumentCleanupEffect(workspaceId, docId, cleanupVersion, effect)
-    );
-  }
-
   async planUnreferencedWorkspaceBlobs(
     workspaceId: string,
     gracePeriodDays: number,

packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts

@@ -13,7 +13,6 @@ interface Context {
     planUnreferencedWorkspaceBlobs: Sinon.SinonStub;
     executeBlobCleanupCandidates: Sinon.SinonStub;
     executeDocumentCleanupCandidates: Sinon.SinonStub;
-    ackDocumentCleanupEffect: Sinon.SinonStub;
   };
   event: {
     emitAsync: Sinon.SinonStub;
@@ -51,7 +50,6 @@ test.beforeEach(t => {
     planUnreferencedWorkspaceBlobs: Sinon.stub(),
     executeBlobCleanupCandidates: Sinon.stub(),
     executeDocumentCleanupCandidates: Sinon.stub(),
-    ackDocumentCleanupEffect: Sinon.stub(),
   };
   t.context.event = {
     emitAsync: Sinon.stub().resolves(undefined),
@@ -316,7 +314,7 @@ test('document projection worker drains metadata incrementally after a document
   );
 });
 
-test('document cleanup dispatches stable search effects', async t => {
+test('document cleanup emits blob updates after comment object cleanup', async t => {
   t.context.runtime.executeDocumentCleanupCandidates.resolves({
     scannedCandidates: 1,
     serializationRetries: 0,
@@ -331,22 +329,13 @@ test('document cleanup dispatches stable search effects', async t => {
         docId: 'doc-1',
         cleanupVersion: 'version-1',
         commentObjectsDone: true,
-        searchDone: false,
       },
     ],
   });
 
   await t.context.job.executeDocumentCleanupCandidates({});
 
-  t.true(
-    t.context.queue.add.calledWith(
-      'indexer.reconcileDocumentCleanup',
-      Sinon.match({ docId: 'doc-1' }),
-      {
-        jobId: 'document-cleanup:search:workspace-1:doc-1:version-1',
-      }
-    )
-  );
+  t.false(t.context.queue.add.called);
   t.true(
     t.context.event.emitAsync.calledWith('workspace.blobs.updated', {
       workspaceId: 'workspace-1',
@@ -354,22 +343,6 @@ test('document cleanup dispatches stable search effects', async t => {
   );
 });
 
-test('document cleanup effect ack delegates to storage runtime', async t => {
-  await t.context.job.ackDocumentCleanupEffect({
-    workspaceId: 'workspace-1',
-    docId: 'doc-1',
-    cleanupVersion: 'version-1',
-    effect: 'search',
-  });
-
-  t.deepEqual(t.context.runtime.ackDocumentCleanupEffect.firstCall.args, [
-    'workspace-1',
-    'doc-1',
-    'version-1',
-    'search',
-  ]);
-});
-
 test('blob cleanup execution sweep drains marked runs and continues by page', async t => {
   t.context.db.$queryRaw
     .onFirstCall()

packages/backend/server/src/core/storage/blob-job.ts

@@ -33,12 +33,6 @@ declare global {
       gracePeriodDays?: number;
       limit?: number;
     };
-    'backendRuntime.ackDocumentCleanupEffect': {
-      workspaceId: string;
-      docId: string;
-      cleanupVersion: string;
-      effect: 'search' | 'copilot';
-    };
     'backendRuntime.planUnreferencedWorkspaceBlobs': {
       workspaceId: string;
       gracePeriodDays?: number;
@@ -321,11 +315,6 @@ export class StorageBlobJob {
       .counter('document_cleanup_execute_failure_total')
       .add(result.failed);
     for (const effect of result.effects) {
-      if (!effect.searchDone) {
-        await this.queue.add('indexer.reconcileDocumentCleanup', effect, {
-          jobId: `document-cleanup:search:${effect.workspaceId}:${effect.docId}:${effect.cleanupVersion}`,
-        });
-      }
       if (effect.commentObjectsDone) {
         await this.event.emitAsync('workspace.blobs.updated', {
           workspaceId: effect.workspaceId,
@@ -336,21 +325,6 @@ export class StorageBlobJob {
     return result;
   }
 
-  @OnJob('backendRuntime.ackDocumentCleanupEffect')
-  async ackDocumentCleanupEffect({
-    workspaceId,
-    docId,
-    cleanupVersion,
-    effect,
-  }: Jobs['backendRuntime.ackDocumentCleanupEffect']) {
-    await this.rt.ackDocumentCleanupEffect(
-      workspaceId,
-      docId,
-      cleanupVersion,
-      effect
-    );
-  }
-
   @OnJob('backendRuntime.planUnreferencedWorkspaceBlobs')
   async planUnreferencedWorkspaceBlobs({
     workspaceId,

packages/backend/server/src/native.ts

@@ -71,7 +71,6 @@ import serverNativeModule, {
   type RuntimeVerificationTokenRecord,
   type RuntimeWorkspaceArtifact,
   type RuntimeWorkspaceInviteLinkRecord,
-  type RuntimeWorkspaceStatsDailyRecalibrationResult,
   type SafeFetchRequest,
   type SafeFetchResponse,
   type StorageProviderCapabilities,
@@ -173,7 +172,6 @@ export type {
   RuntimeVerificationTokenRecord,
   RuntimeWorkspaceArtifact,
   RuntimeWorkspaceInviteLinkRecord,
-  RuntimeWorkspaceStatsDailyRecalibrationResult,
   SafeFetchRequest,
   SafeFetchResponse,
   StorageProviderCapabilities,

packages/backend/server/src/plugins/copilot/retrieval/document.ts

@@ -1,6 +1,6 @@
 import { Inject, Injectable } from '@nestjs/common';
 
-import { SearchProviderNotFound } from '../../../base';
+import { SearchProviderUnavailable } from '../../../base';
 import { PermissionAccess } from '../../../core/permission';
 import type { DocVisibility } from '../../../core/utils/blocksuite';
 import { type DocChunkSimilarity, Models } from '../../../models';
@@ -101,7 +101,7 @@ export class DocumentRetrievalService {
           docIds,
         })
         .catch(error => {
-          if (error instanceof SearchProviderNotFound) return null;
+          if (error instanceof SearchProviderUnavailable) return null;
           throw error;
         }),
       this.context.canEmbedding

packages/backend/server/src/plugins/indexer/__tests__/event.spec.ts

@@ -1,77 +0,0 @@
-import test from 'ava';
-
-import { createModule } from '../../../__tests__/create-module';
-import { JobQueue } from '../../../base';
-import { ConfigModule } from '../../../base/config';
-import { IndexerEvent } from '../event';
-import { IndexerModule } from '../index';
-import { IndexerScheduler } from '../scheduler';
-
-const module = await createModule({
-  imports: [
-    IndexerModule,
-    ConfigModule.override({
-      indexer: {
-        enabled: true,
-      },
-    }),
-  ],
-});
-const indexerEvent = module.get(IndexerEvent);
-const indexerScheduler = new IndexerScheduler(module.get(JobQueue));
-
-test.after.always(async () => {
-  await module.close();
-});
-
-test('should index workspace when root snapshot is updated', async t => {
-  // @ts-expect-error ignore missing fields
-  await indexerEvent.indexWorkspace({
-    workspaceId: 'test-workspace',
-    docId: 'test-workspace',
-  });
-
-  const { payload } = await module.queue.waitFor('indexer.indexWorkspace');
-  t.is(payload.workspaceId, 'test-workspace');
-});
-
-test('should not index workspace when non-root snapshot is updated', async t => {
-  const count = module.queue.count('indexer.indexWorkspace');
-
-  // @ts-expect-error ignore missing fields
-  await indexerEvent.indexWorkspace({
-    workspaceId: 'test-workspace',
-    docId: 'child-doc',
-  });
-
-  t.is(module.queue.count('indexer.indexWorkspace'), count);
-});
-
-test('should reindex documents after document access changes', async t => {
-  await indexerEvent.reindexDocOnGrantChange({
-    workspaceId: 'test-workspace',
-    docId: 'test-doc',
-  });
-  const { payload } = await module.queue.waitFor('indexer.indexDoc');
-  t.deepEqual(payload, {
-    workspaceId: 'test-workspace',
-    docId: 'test-doc',
-  });
-});
-
-test('should delete workspace', async t => {
-  // @ts-expect-error ignore missing fields
-  await indexerEvent.deleteUserWorkspaces({
-    ownedWorkspaces: ['test-workspace'],
-  });
-
-  const { payload } = await module.queue.waitFor('indexer.deleteWorkspace');
-  t.is(payload.workspaceId, 'test-workspace');
-});
-
-test('should schedule auto index workspaces', async t => {
-  await indexerScheduler.autoIndexWorkspaces();
-
-  const { payload } = await module.queue.waitFor('indexer.autoIndexWorkspaces');
-  t.is(payload.lastIndexedWorkspaceSid, undefined);
-});

packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts

@@ -1,215 +0,0 @@
-import { randomUUID } from 'node:crypto';
-
-import test from 'ava';
-import Sinon from 'sinon';
-
-import { createModule } from '../../../__tests__/create-module';
-import { Mockers } from '../../../__tests__/mocks';
-import { JOB_SIGNAL } from '../../../base';
-import { ConfigModule } from '../../../base/config';
-import { ServerConfigModule } from '../../../core/config';
-import { DocReader } from '../../../core/doc';
-import { Models } from '../../../models';
-import { addDocToRootDoc } from '../../../native';
-import { IndexerModule, IndexerService, IndexerWorkerModule } from '../index';
-import { IndexerJob } from '../job';
-
-const module = await createModule({
-  imports: [
-    IndexerModule,
-    IndexerWorkerModule,
-    ServerConfigModule,
-    ConfigModule.override({
-      indexer: {
-        enabled: true,
-      },
-    }),
-  ],
-  providers: [IndexerService],
-});
-const indexerService = module.get(IndexerService);
-const indexerJob = module.get(IndexerJob);
-const models = module.get(Models);
-const docReader = module.get(DocReader);
-
-const user = await module.create(Mockers.User);
-const workspace = await module.create(Mockers.Workspace, {
-  snapshot: true,
-  owner: user,
-});
-
-test.after.always(async () => {
-  await module.close();
-});
-
-test.afterEach.always(() => {
-  Sinon.restore();
-});
-
-test('should handle indexer.indexDoc job', async t => {
-  const spy = Sinon.spy(indexerService, 'indexDoc');
-  await indexerJob.indexDoc({
-    workspaceId: workspace.id,
-    docId: randomUUID(),
-  });
-  t.is(spy.callCount, 1);
-});
-
-test('should handle indexer.deleteDoc job', async t => {
-  const spy = Sinon.spy(indexerService, 'deleteDoc');
-  await indexerJob.deleteDoc({
-    workspaceId: workspace.id,
-    docId: randomUUID(),
-  });
-  t.is(spy.callCount, 1);
-});
-
-test('should handle indexer.indexWorkspace job', async t => {
-  const spy = Sinon.stub(indexerService, 'reconcileWorkspace').resolves();
-
-  await indexerJob.indexWorkspace({
-    workspaceId: workspace.id,
-  });
-
-  t.true(spy.calledOnceWith(workspace.id));
-
-  // workspace should be indexed
-  const ws = await models.workspace.get(workspace.id);
-  t.is(ws!.indexed, true);
-});
-
-test('document cleanup reconcile deletes missing search state before ack', async t => {
-  const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
-  const indexSpy = Sinon.spy(indexerService, 'indexDoc');
-  const cleanupWorkspace = await module.create(Mockers.Workspace, {
-    owner: user,
-  });
-  await module.create(Mockers.DocSnapshot, {
-    workspaceId: cleanupWorkspace.id,
-    docId: cleanupWorkspace.id,
-    user,
-    blob: addDocToRootDoc(Buffer.from([0, 0]), 'live-doc', 'Live'),
-  });
-
-  await indexerJob.reconcileDocumentCleanup({
-    workspaceId: cleanupWorkspace.id,
-    docId: 'missing-doc',
-    cleanupVersion: 'version-1',
-  });
-
-  t.true(deleteSpy.calledOnceWith(cleanupWorkspace.id, 'missing-doc'));
-  t.false(indexSpy.called);
-  const { payload } = await module.queue.waitFor(
-    'backendRuntime.ackDocumentCleanupEffect'
-  );
-  t.deepEqual(payload, {
-    workspaceId: cleanupWorkspace.id,
-    docId: 'missing-doc',
-    cleanupVersion: 'version-1',
-    effect: 'search',
-  });
-});
-
-test('document cleanup reconcile reindexes restored doc before ack', async t => {
-  const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
-  const indexSpy = Sinon.spy(indexerService, 'indexDoc');
-  const cleanupWorkspace = await module.create(Mockers.Workspace, {
-    owner: user,
-  });
-  await module.create(Mockers.DocSnapshot, {
-    workspaceId: cleanupWorkspace.id,
-    docId: cleanupWorkspace.id,
-    user,
-    blob: addDocToRootDoc(Buffer.from([0, 0]), 'restored-doc', 'Restored'),
-  });
-  await module.create(Mockers.DocSnapshot, {
-    workspaceId: cleanupWorkspace.id,
-    docId: 'restored-doc',
-    user,
-  });
-  const getDocSpy = Sinon.spy(docReader, 'getDoc');
-
-  await indexerJob.reconcileDocumentCleanup({
-    workspaceId: cleanupWorkspace.id,
-    docId: 'restored-doc',
-    cleanupVersion: 'version-2',
-  });
-
-  t.true(indexSpy.calledOnceWith(cleanupWorkspace.id, 'restored-doc'));
-  t.false(deleteSpy.called);
-  t.true(getDocSpy.calledWith(cleanupWorkspace.id, cleanupWorkspace.id));
-  t.true(getDocSpy.calledWith(cleanupWorkspace.id, 'restored-doc'));
-  const { payload } = await module.queue.waitFor(
-    'backendRuntime.ackDocumentCleanupEffect'
-  );
-  t.deepEqual(payload, {
-    workspaceId: cleanupWorkspace.id,
-    docId: 'restored-doc',
-    cleanupVersion: 'version-2',
-    effect: 'search',
-  });
-});
-
-test('should handle indexer.deleteWorkspace job', async t => {
-  const spy = Sinon.spy(indexerService, 'deleteWorkspace');
-
-  await indexerJob.deleteWorkspace({
-    workspaceId: workspace.id,
-  });
-
-  t.is(spy.callCount, 1);
-});
-
-test('should handle indexer.autoIndexWorkspaces job', async t => {
-  const workspace = await module.create(Mockers.Workspace, {
-    snapshot: true,
-  });
-
-  const result = await indexerJob.autoIndexWorkspaces({
-    lastIndexedWorkspaceSid: workspace.sid - 1,
-  });
-  t.is(result, JOB_SIGNAL.Repeat);
-
-  const { payload } = await module.queue.waitFor('indexer.indexWorkspace');
-  t.is(payload.workspaceId, workspace.id);
-
-  // no new auto index job
-  const count = module.queue.count('indexer.autoIndexWorkspaces');
-
-  await indexerJob.autoIndexWorkspaces({
-    lastIndexedWorkspaceSid: workspace.sid,
-  });
-
-  t.is(module.queue.count('indexer.autoIndexWorkspaces'), count);
-});
-
-test('should not index workspace if it is not updated in 180 days', async t => {
-  const workspace = await module.create(Mockers.Workspace);
-  await module.create(Mockers.DocSnapshot, {
-    user,
-    workspaceId: workspace.id,
-    docId: workspace.id,
-    updatedAt: new Date(Date.now() - 180 * 24 * 60 * 60 * 1000 - 1),
-  });
-
-  const count = module.queue.count('indexer.indexWorkspace');
-
-  await indexerJob.autoIndexWorkspaces({
-    lastIndexedWorkspaceSid: workspace.sid - 1,
-  });
-
-  t.is(module.queue.count('indexer.indexWorkspace'), count);
-});
-
-test('should not index workspace if snapshot not exists', async t => {
-  // not create snapshot
-  const workspace = await module.create(Mockers.Workspace);
-
-  const count = module.queue.count('indexer.indexWorkspace');
-
-  await indexerJob.autoIndexWorkspaces({
-    lastIndexedWorkspaceSid: workspace.sid - 1,
-  });
-
-  t.is(module.queue.count('indexer.indexWorkspace'), count);
-});

packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts

@@ -4,13 +4,18 @@ import Sinon from 'sinon';
 import {
   InternalServerError,
   InvalidIndexerInput,
-  SearchProviderNotFound,
+  SearchIndexFailed,
+  SearchIndexNotReady,
+  SearchPermissionSyncing,
+  SearchProviderUnavailable,
   SpaceAccessDenied,
-  WorkspacePermissionNotFound,
 } from '../../../base';
+import { ConfigFactory } from '../../../base/config';
 import { BackendRuntimeProvider } from '../../../core/backend-runtime';
-import { ServerService } from '../../../core/config';
+import { BackendRuntimeSearchJob } from '../../../core/backend-runtime/job';
+import { ServerFeature, ServerService } from '../../../core/config';
 import { Models } from '../../../models';
+import { IndexerResolver } from '../resolver';
 import { IndexerService } from '../service';
 import { SearchQueryType, SearchTable } from '../types';
 
@@ -18,13 +23,47 @@ test.afterEach.always(() => {
   Sinon.restore();
 });
 
+function enabledServer() {
+  return {
+    getConfig: Sinon.stub().returns({ indexer: { enabled: true } }),
+    enableFeature: Sinon.stub(),
+    disableFeature: Sinon.stub(),
+  };
+}
+
 test('reflects native search readiness in the Node feature flag', async t => {
   const runtime = {
     searchStatus: Sinon.stub(),
+    searchAuthorized: Sinon.stub().resolves({
+      ok: true,
+      value: { total: 0, nodes: [] },
+    }),
   };
   runtime.searchStatus.onFirstCall().resolves({ ready: true });
   runtime.searchStatus.onSecondCall().resolves({ ready: false });
+  runtime.searchStatus.onThirdCall().resolves({ ready: true });
+  const server = enabledServer();
+  const service = new IndexerService(
+    runtime as unknown as BackendRuntimeProvider,
+    {} as Models,
+    server as unknown as ServerService
+  );
+
+  await service.onApplicationBootstrap();
+  await service.onConfigChanged({ updates: { indexer: {} } } as never);
+  await service.search('actor', 'workspace', {} as never);
+
+  t.is(server.enableFeature.callCount, 2);
+  t.true(server.disableFeature.calledOnce);
+  t.is(runtime.searchStatus.callCount, 3);
+});
+
+test('does not query native search when the indexer is disabled', async t => {
+  const runtime = {
+    searchStatus: Sinon.stub(),
+  };
   const server = {
+    getConfig: Sinon.stub().returns({ indexer: { enabled: false } }),
     enableFeature: Sinon.stub(),
     disableFeature: Sinon.stub(),
   };
@@ -35,21 +74,50 @@ test('reflects native search readiness in the Node feature flag', async t => {
   );
 
   await service.onApplicationBootstrap();
-  await service.onConfigChanged({ updates: { indexer: {} } } as never);
 
-  t.true(server.enableFeature.calledOnce);
-  t.true(server.disableFeature.calledOnce);
-  t.is(runtime.searchStatus.callCount, 2);
+  t.false(runtime.searchStatus.called);
+  t.true(server.disableFeature.calledOnceWith(ServerFeature.Indexer));
+});
+
+test('does not schedule or run native search reconciliation when disabled', async t => {
+  const runtime = {
+    reconcileSearchProjection: Sinon.stub(),
+    searchStatus: Sinon.stub(),
+  };
+  const queue = { add: Sinon.stub() };
+  const config = {
+    config: { indexer: { enabled: false } },
+  } as unknown as ConfigFactory;
+  const job = new BackendRuntimeSearchJob(
+    runtime as unknown as BackendRuntimeProvider,
+    queue as never,
+    config
+  );
+
+  await job.scheduleReconciliation();
+  t.is(queue.add.callCount, 0);
+  t.is(await job.reconcileProjection({ limit: 100 }), 0);
+  t.false(runtime.reconcileSearchProjection.called);
+  t.false(runtime.searchStatus.called);
+
+  config.config.indexer.enabled = true;
+  await job.scheduleReconciliation();
+  t.deepEqual(queue.add.firstCall.args[2], {
+    jobId: 'backend-runtime-search-reconciliation',
+    removeOnFail: true,
+  });
 });
 
 test('maps native search results and typed errors at the Node boundary', async t => {
   const runtime = {
+    searchStatus: Sinon.stub().resolves({ ready: true }),
     searchAuthorized: Sinon.stub(),
+    aggregateAuthorized: Sinon.stub(),
   };
   const service = new IndexerService(
     runtime as unknown as BackendRuntimeProvider,
     {} as Models,
-    {} as ServerService
+    enabledServer() as unknown as ServerService
   );
   const input = {
     table: SearchTable.block,
@@ -59,7 +127,7 @@ test('maps native search results and typed errors at the Node boundary', async t
   runtime.searchAuthorized.resolves({
     ok: true,
     value: {
-      total: 1,
+      total: 99,
       nodes: [
         {
           id: 'node',
@@ -85,12 +153,41 @@ test('maps native search results and typed errors at the Node boundary', async t
     markdownPreview: ['<b>hello</b>'],
   });
 
+  const resolver = new IndexerResolver(service, {
+    user: Sinon.stub().returns({
+      workspace: Sinon.stub().returns({ assert: Sinon.stub().resolves() }),
+    }),
+  } as never);
+  const searchResult = await resolver.search(
+    { id: 'actor' } as never,
+    { id: 'workspace' } as never,
+    input
+  );
+  t.is(searchResult.pagination.count, 1);
+
+  runtime.aggregateAuthorized.resolves({
+    ok: true,
+    value: {
+      total: 99,
+      hasMore: true,
+      buckets: [{ key: 'doc', count: 1, hits: { nodes: [] } }],
+    },
+  });
+  const aggregateResult = await resolver.aggregate(
+    { id: 'actor' } as never,
+    { id: 'workspace' } as never,
+    {} as never
+  );
+  t.is(aggregateResult.pagination.count, 1);
+
   for (const [errorCode, expected] of [
     ['workspace_denied', SpaceAccessDenied],
     ['invalid_request', InvalidIndexerInput],
     ['unsupported_query', InvalidIndexerInput],
-    ['provider_unavailable', SearchProviderNotFound],
-    ['permission_unavailable', WorkspacePermissionNotFound],
+    ['provider_unavailable', SearchProviderUnavailable],
+    ['index_not_ready', SearchIndexNotReady],
+    ['permission_syncing', SearchPermissionSyncing],
+    ['index_failed', SearchIndexFailed],
     ['unexpected', InternalServerError],
   ] as const) {
     runtime.searchAuthorized.resolves({ ok: false, errorCode });
@@ -102,7 +199,29 @@ test('maps native search results and typed errors at the Node boundary', async t
 });
 
 test('searchDocs keeps filtering and enrichment in Node', async t => {
+  const blockNode = {
+    id: 'block',
+    score: 1,
+    fields: {
+      workspace_id: ['workspace'],
+      doc_id: ['doc'],
+      block_id: ['block'],
+      unit_id: ['unit'],
+      projection_version: [1],
+      source_hash: ['hash'],
+      visibility: ['visible'],
+      source_block_id: ['source-block'],
+      flavour: ['affine:paragraph'],
+      content: ['body'],
+      created_at: [2_000],
+      updated_at: [3_000],
+      created_by_user_id: ['creator'],
+      updated_by_user_id: ['updater'],
+    },
+    highlights: { content: ['<b>body</b>'] },
+  };
   const runtime = {
+    searchStatus: Sinon.stub().resolves({ ready: true }),
     aggregateAuthorized: Sinon.stub().resolves({
       ok: true,
       value: {
@@ -113,34 +232,19 @@ test('searchDocs keeps filtering and enrichment in Node', async t => {
             key: 'doc',
             count: 1,
             hits: {
-              nodes: [
-                {
-                  id: 'block',
-                  score: 1,
-                  fields: {
-                    workspace_id: ['workspace'],
-                    doc_id: ['doc'],
-                    block_id: ['block'],
-                    unit_id: ['unit'],
-                    projection_version: [1],
-                    source_hash: ['hash'],
-                    visibility: ['visible'],
-                    source_block_id: ['source-block'],
-                    flavour: ['affine:paragraph'],
-                    content: ['body'],
-                    created_at: [2_000],
-                    updated_at: [3_000],
-                    created_by_user_id: ['creator'],
-                    updated_by_user_id: ['updater'],
-                  },
-                  highlights: { content: ['<b>body</b>'] },
-                },
-              ],
+              nodes: [blockNode],
             },
           },
         ],
       },
     }),
+    searchAuthorized: Sinon.stub().resolves({
+      ok: true,
+      value: {
+        total: 2,
+        nodes: [blockNode, { ...blockNode, id: 'duplicate-block' }],
+      },
+    }),
   };
   const creator = { id: 'creator', name: 'Creator' };
   const updater = { id: 'updater', name: 'Updater' };
@@ -162,9 +266,16 @@ test('searchDocs keeps filtering and enrichment in Node', async t => {
   const service = new IndexerService(
     runtime as unknown as BackendRuntimeProvider,
     models as unknown as Models,
-    {} as ServerService
+    enabledServer() as unknown as ServerService
   );
 
+  for (const limit of [0, -1]) {
+    const error = await t.throwsAsync(
+      service.searchDocsByKeyword('actor', 'workspace', 'body', { limit })
+    );
+    t.true(error instanceof InvalidIndexerInput);
+  }
+  t.false(runtime.aggregateAuthorized.called);
   t.deepEqual(
     await service.searchDocsByKeyword('actor', 'workspace', 'body', {
       docIds: [],
@@ -194,4 +305,22 @@ test('searchDocs keeps filtering and enrichment in Node', async t => {
       }
     )
   );
+
+  runtime.aggregateAuthorized.resolves({
+    ok: false,
+    errorCode: 'unsupported_query',
+  });
+  const basicDocs = await service.searchDocsByKeyword(
+    'actor',
+    'workspace',
+    'body',
+    { limit: 5, docIds: ['doc'] }
+  );
+  t.is(basicDocs.length, 1);
+  t.is(basicDocs[0].docId, 'doc');
+  const basicRequest = runtime.searchAuthorized.firstCall.args[2];
+  t.true(basicRequest.options.fields.includes('docId'));
+  t.is(basicRequest.options.pagination?.limit, 20);
+  t.true(JSON.stringify(basicRequest.query).includes('doc'));
+  t.true(JSON.stringify(basicRequest.query).includes('workspace'));
 });

packages/backend/server/src/plugins/indexer/config.ts

@@ -4,8 +4,8 @@ import { defineModuleConfig } from '../../base';
 
 export enum SearchProviderType {
   Embedded = 'embedded',
-  Manticoresearch = 'manticoresearch',
   Elasticsearch = 'elasticsearch',
+  ManticoreSearch = 'manticoresearch',
 }
 
 const SearchProviderTypeSchema = z.nativeEnum(SearchProviderType);
@@ -21,9 +21,6 @@ declare global {
         username: string;
         password: string;
       };
-      autoIndex: {
-        batchSize: number;
-      };
     };
   }
 }
@@ -32,18 +29,15 @@ defineModuleConfig('indexer', {
   enabled: {
     desc: 'Enable indexer plugin',
     default: false,
-    env: ['AFFINE_INDEXER_ENABLED', 'boolean'],
   },
   'provider.type': {
     desc: 'Indexer search provider. Self-hosted uses the embedded provider by default; remote providers require an endpoint.',
     default: SearchProviderType.Embedded,
     shape: SearchProviderTypeSchema,
-    env: ['AFFINE_INDEXER_SEARCH_PROVIDER', 'string'],
   },
   'provider.endpoint': {
     desc: 'Remote indexer endpoint. Not used by the embedded provider.',
     default: '',
-    env: ['AFFINE_INDEXER_SEARCH_ENDPOINT', 'string'],
     validate: val => {
       // allow to be nullable and empty string
       if (!val) {
@@ -54,25 +48,17 @@ defineModuleConfig('indexer', {
     },
   },
   'provider.apiKey': {
-    desc: 'Indexer search service api key. Optional for elasticsearch',
+    desc: 'Indexer search service api key. Optional for remote providers',
     link: 'https://www.elastic.co/guide/server/current/api-key.html',
     default: '',
-    env: ['AFFINE_INDEXER_SEARCH_API_KEY', 'string'],
   },
   'provider.username': {
-    desc: 'Indexer search service auth username, if not set, basic auth will be disabled. Optional for elasticsearch',
+    desc: 'Indexer search service auth username, if not set, basic auth will be disabled. Optional for remote providers',
     link: 'https://www.elastic.co/guide/en/elasticsearch/reference/current/http-clients.html',
     default: '',
-    env: ['AFFINE_INDEXER_SEARCH_USERNAME', 'string'],
   },
   'provider.password': {
-    desc: 'Indexer search service auth password, if not set, basic auth will be disabled. Optional for elasticsearch',
+    desc: 'Indexer search service auth password, if not set, basic auth will be disabled. Optional for remote providers',
     default: '',
-    env: ['AFFINE_INDEXER_SEARCH_PASSWORD', 'string'],
-  },
-  'autoIndex.batchSize': {
-    desc: 'Number of workspaces automatically indexed per batch',
-    default: 10,
-    shape: z.number().int().positive().max(1000),
   },
 });

packages/backend/server/src/plugins/indexer/event.ts

@@ -1,84 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import { JobQueue, OnEvent } from '../../base';
-
-@Injectable()
-export class IndexerEvent {
-  constructor(private readonly queue: JobQueue) {}
-
-  @OnEvent('doc.grants.changed')
-  async reindexDocOnGrantChange({
-    workspaceId,
-    docId,
-  }: Events['doc.grants.changed']) {
-    await this.indexDoc({ workspaceId, docId });
-  }
-
-  @OnEvent('doc.owner.changed')
-  async reindexDocOnOwnerChange({
-    workspaceId,
-    docId,
-  }: Events['doc.owner.changed']) {
-    await this.indexDoc({ workspaceId, docId });
-  }
-
-  @OnEvent('doc.default_role.changed')
-  async reindexDocOnDefaultRoleChange({
-    workspaceId,
-    docId,
-  }: Events['doc.default_role.changed']) {
-    await this.indexDoc({ workspaceId, docId });
-  }
-
-  @OnEvent('doc.public_state.changed')
-  async reindexDocOnPublicStateChange({
-    workspaceId,
-    docId,
-  }: Events['doc.public_state.changed']) {
-    await this.indexDoc({ workspaceId, docId });
-  }
-
-  @OnEvent('doc.updated')
-  async indexDoc({ workspaceId, docId }: Events['doc.updated']) {
-    await this.queue.add(
-      'indexer.indexDoc',
-      {
-        workspaceId,
-        docId,
-      },
-      {
-        jobId: `indexDoc/${workspaceId}/${docId}`,
-        priority: 100,
-      }
-    );
-  }
-
-  @OnEvent('doc.snapshot.updated')
-  async indexWorkspace({ workspaceId, docId }: Events['doc.snapshot.updated']) {
-    if (workspaceId !== docId) {
-      return;
-    }
-
-    await this.queue.add(
-      'indexer.indexWorkspace',
-      { workspaceId },
-      { jobId: `indexWorkspace/${workspaceId}`, priority: 100 }
-    );
-  }
-
-  @OnEvent('user.deleted')
-  async deleteUserWorkspaces(payload: Events['user.deleted']) {
-    for (const workspace of payload.ownedWorkspaces) {
-      await this.queue.add(
-        'indexer.deleteWorkspace',
-        {
-          workspaceId: workspace,
-        },
-        {
-          jobId: `deleteWorkspace/${workspace}`,
-          priority: 0,
-        }
-      );
-    }
-  }
-}

packages/backend/server/src/plugins/indexer/index.ts

@@ -5,10 +5,7 @@ import { Module } from '@nestjs/common';
 import { ServerConfigModule } from '../../core/config';
 import { DocStorageModule } from '../../core/doc';
 import { PermissionModule } from '../../core/permission';
-import { IndexerEvent } from './event';
-import { IndexerJob } from './job';
 import { IndexerResolver } from './resolver';
-import { IndexerScheduler } from './scheduler';
 import { IndexerService } from './service';
 
 const INDEXER_SHARED_IMPORTS = [
@@ -24,21 +21,9 @@ const INDEXER_SHARED_IMPORTS = [
 })
 export class IndexerServiceModule {}
 
-@Module({
-  imports: [IndexerServiceModule],
-  providers: [IndexerEvent],
-})
-export class IndexerProducerModule {}
-
 @Module({
   imports: [IndexerServiceModule, DocStorageModule, PermissionModule],
-  providers: [IndexerJob, IndexerScheduler],
-})
-export class IndexerWorkerModule {}
-
-@Module({
-  imports: [IndexerServiceModule, DocStorageModule, PermissionModule],
-  providers: [IndexerResolver, IndexerEvent],
+  providers: [IndexerResolver],
   exports: [IndexerServiceModule],
 })
 export class IndexerModule {}

packages/backend/server/src/plugins/indexer/job.ts

@@ -1,192 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-
-import { Config, JOB_SIGNAL, JobQueue, OnJob } from '../../base';
-import { DocReader } from '../../core/doc';
-import { readAllDocIdsFromWorkspaceSnapshot } from '../../core/utils/blocksuite';
-import { Models } from '../../models';
-import { IndexerService } from './service';
-
-declare global {
-  interface Jobs {
-    'indexer.indexDoc': {
-      workspaceId: string;
-      docId: string;
-    };
-    'indexer.deleteDoc': {
-      workspaceId: string;
-      docId: string;
-    };
-    'indexer.indexWorkspace': {
-      workspaceId: string;
-    };
-    'indexer.deleteWorkspace': {
-      workspaceId: string;
-    };
-    'indexer.autoIndexWorkspaces': {
-      lastIndexedWorkspaceSid?: number;
-    };
-    'indexer.reconcileDocumentCleanup': {
-      workspaceId: string;
-      docId: string;
-      cleanupVersion: string;
-    };
-  }
-}
-
-@Injectable()
-export class IndexerJob {
-  private readonly logger = new Logger(IndexerJob.name);
-
-  constructor(
-    private readonly models: Models,
-    private readonly service: IndexerService,
-    private readonly queue: JobQueue,
-    private readonly config: Config,
-    private readonly doc: DocReader
-  ) {}
-
-  @OnJob('indexer.indexDoc')
-  async indexDoc({ workspaceId, docId }: Jobs['indexer.indexDoc']) {
-    // delete the 'indexer.deleteDoc' job from the queue
-    await this.queue.remove(
-      `deleteDoc/${workspaceId}/${docId}`,
-      'indexer.deleteDoc'
-    );
-    await this.service.indexDoc(workspaceId, docId);
-    await this.enqueueBlobRefProjection(workspaceId, docId);
-  }
-
-  @OnJob('indexer.deleteDoc')
-  async deleteDoc({ workspaceId, docId }: Jobs['indexer.deleteDoc']) {
-    // delete the 'indexer.updateDoc' job from the queue
-    await this.queue.remove(
-      `indexDoc/${workspaceId}/${docId}`,
-      'indexer.indexDoc'
-    );
-    await this.service.deleteDoc(workspaceId, docId);
-  }
-
-  @OnJob('indexer.reconcileDocumentCleanup')
-  async reconcileDocumentCleanup({
-    workspaceId,
-    docId,
-    cleanupVersion,
-  }: Jobs['indexer.reconcileDocumentCleanup']) {
-    const root = await this.doc.getDoc(workspaceId, workspaceId);
-    if (!root) {
-      throw new Error(`workspace root ${workspaceId} not found`);
-    }
-    const live = readAllDocIdsFromWorkspaceSnapshot(root.bin, true).includes(
-      docId
-    );
-    if (live) {
-      if (!(await this.doc.getDoc(workspaceId, docId))) {
-        throw new Error(`restored document ${workspaceId}/${docId} not found`);
-      }
-      await this.service.indexDoc(workspaceId, docId);
-      await this.enqueueBlobRefProjection(workspaceId, docId);
-    } else {
-      await this.service.deleteDoc(workspaceId, docId);
-    }
-    await this.queue.add('backendRuntime.ackDocumentCleanupEffect', {
-      workspaceId,
-      docId,
-      cleanupVersion,
-      effect: 'search',
-    });
-  }
-
-  @OnJob('indexer.indexWorkspace')
-  async indexWorkspace({ workspaceId }: Jobs['indexer.indexWorkspace']) {
-    await this.queue.remove(workspaceId, 'indexer.deleteWorkspace');
-    const workspace = await this.models.workspace.get(workspaceId);
-    if (!workspace) {
-      this.logger.warn(`workspace ${workspaceId} not found`);
-      return;
-    }
-
-    await this.service.reconcileWorkspace(workspaceId);
-    if (!workspace.indexed) {
-      await this.models.workspace.update(workspaceId, {
-        indexed: true,
-      });
-    }
-    this.logger.log(`reconciled workspace ${workspaceId}`);
-  }
-
-  @OnJob('indexer.deleteWorkspace')
-  async deleteWorkspace({ workspaceId }: Jobs['indexer.deleteWorkspace']) {
-    await this.queue.remove(
-      `indexWorkspace/${workspaceId}`,
-      'indexer.indexWorkspace'
-    );
-    await this.service.deleteWorkspace(workspaceId);
-  }
-
-  @OnJob('indexer.autoIndexWorkspaces')
-  async autoIndexWorkspaces(payload: Jobs['indexer.autoIndexWorkspaces']) {
-    const startSid = payload.lastIndexedWorkspaceSid ?? 0;
-    const workspaces = await this.models.workspace.list(
-      { sid: { gt: startSid } },
-      { id: true, indexed: true, sid: true },
-      this.config.indexer.autoIndex.batchSize
-    );
-
-    if (workspaces.length === 0) {
-      // Keep the current sid value when repeating
-      return JOB_SIGNAL.Repeat;
-    }
-    let addedCount = 0;
-    for (const workspace of workspaces) {
-      const snapshotMeta = await this.models.doc.getSnapshot(
-        workspace.id,
-        workspace.id,
-        {
-          select: {
-            updatedAt: true,
-          },
-        }
-      );
-      // ignore 180 days not updated workspaces
-      if (
-        !snapshotMeta?.updatedAt ||
-        Date.now() - snapshotMeta.updatedAt.getTime() >
-          180 * 24 * 60 * 60 * 1000
-      ) {
-        continue;
-      }
-      await this.queue.add(
-        'indexer.indexWorkspace',
-        { workspaceId: workspace.id },
-        { jobId: `indexWorkspace/${workspace.id}` }
-      );
-      addedCount++;
-    }
-    const nextSid = workspaces[workspaces.length - 1].sid;
-    this.logger.log(
-      `Auto added ${addedCount} workspaces to queue, lastIndexedWorkspaceSid: ${startSid} -> ${nextSid}`
-    );
-
-    // update the lastIndexedWorkspaceSid in the payload and repeat the job after 30 seconds
-    payload.lastIndexedWorkspaceSid = nextSid;
-    return JOB_SIGNAL.Repeat;
-  }
-
-  private async enqueueBlobRefProjection(workspaceId: string, docId: string) {
-    const snapshot = await this.models.doc.getSnapshot(workspaceId, docId, {
-      select: { updatedAt: true },
-    });
-    if (!snapshot) {
-      return;
-    }
-    const sourceRevision = snapshot.updatedAt.getTime();
-    await this.queue.add(
-      'backendRuntime.projectWorkspaceDocBlobRefs',
-      { workspaceId, docId, sourceRevision },
-      {
-        jobId: `doc:blob-ref-projection:${workspaceId}:${docId}:${sourceRevision}`,
-        priority: 100,
-      }
-    );
-  }
-}

packages/backend/server/src/plugins/indexer/resolver.ts

@@ -35,8 +35,8 @@ export class IndexerResolver {
     return {
       nodes: result.nodes,
       pagination: {
-        count: result.total,
-        hasMore: result.nodes.length > 0,
+        count: result.nodes.length,
+        hasMore: Boolean(result.nextCursor),
         nextCursor: result.nextCursor,
       },
     };
@@ -56,7 +56,7 @@ export class IndexerResolver {
     return {
       buckets: result.buckets,
       pagination: {
-        count: result.total,
+        count: result.buckets.length,
         hasMore: result.hasMore,
       },
     };

packages/backend/server/src/plugins/indexer/result.ts

@@ -3,6 +3,8 @@ import { camelCase, mapKeys } from 'lodash-es';
 import {
   AggregateInput,
   SearchDoc,
+  SearchInput,
+  SearchQuery,
   SearchQueryOccur,
   SearchQueryType,
   SearchTable,
@@ -147,6 +149,71 @@ export function buildSearchDocsInput(
   };
 }
 
+export function buildBasicSearchDocsInput(
+  workspaceId: string,
+  keyword: string,
+  options?: { limit?: number; docIds?: string[] }
+): SearchInput {
+  const aggregate = buildSearchDocsInput(workspaceId, keyword, options);
+  const limit = options?.limit ?? 20;
+  const queries: SearchQuery[] = [
+    {
+      type: SearchQueryType.match as const,
+      field: 'workspaceId',
+      match: workspaceId,
+    },
+    {
+      type: SearchQueryType.match as const,
+      field: 'content',
+      match: keyword,
+    },
+  ];
+  if (options?.docIds) {
+    queries.push({
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.should,
+      queries: options.docIds.map(docId => ({
+        type: SearchQueryType.match as const,
+        field: 'docId',
+        match: docId,
+      })),
+    });
+  }
+  return {
+    table: aggregate.table,
+    query: {
+      type: SearchQueryType.boolean,
+      occur: SearchQueryOccur.must,
+      queries,
+    },
+    options: {
+      fields: ['docId', ...aggregate.options.hits.fields],
+      highlights: aggregate.options.hits.highlights,
+      pagination: { limit: Math.min(Math.max(limit, 1) * 4, 10_000) },
+    },
+  };
+}
+
+export function collectBasicSearchDocs(
+  result: { nodes: SearchNodeWithMeta[] },
+  workspaceId: string,
+  limit: number
+) {
+  const seen = new Set<string>();
+  const buckets: AggregateResult['buckets'] = [];
+  for (const node of result.nodes) {
+    const docId = node._source.docId;
+    if (seen.has(docId)) continue;
+    seen.add(docId);
+    buckets.push({ key: docId, count: 1, hits: { nodes: [node] } });
+    if (buckets.length === limit) break;
+  }
+  return collectSearchDocs(
+    { total: buckets.length, hasMore: false, buckets },
+    workspaceId
+  );
+}
+
 export function collectSearchDocs(
   result: AggregateResult,
   workspaceId: string

packages/backend/server/src/plugins/indexer/scheduler.ts

@@ -1,22 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { Cron, CronExpression } from '@nestjs/schedule';
-
-import { JobQueue } from '../../base';
-
-@Injectable()
-export class IndexerScheduler {
-  constructor(private readonly queue: JobQueue) {}
-
-  @Cron(CronExpression.EVERY_30_SECONDS)
-  async autoIndexWorkspaces() {
-    await this.queue.add(
-      'indexer.autoIndexWorkspaces',
-      {},
-      {
-        // make sure only one job is running at a time
-        delay: 30 * 1000,
-        jobId: 'autoIndexWorkspaces',
-      }
-    );
-  }
-}

packages/backend/server/src/plugins/indexer/service.ts

@@ -4,16 +4,20 @@ import {
   InternalServerError,
   InvalidIndexerInput,
   OnEvent,
-  SearchProviderNotFound,
+  SearchIndexFailed,
+  SearchIndexNotReady,
+  SearchPermissionSyncing,
+  SearchProviderUnavailable,
   SpaceAccessDenied,
-  WorkspacePermissionNotFound,
 } from '../../base';
 import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { ServerFeature, ServerService } from '../../core/config';
 import { Models } from '../../models';
 import {
   type AggregateResult,
+  buildBasicSearchDocsInput,
   buildSearchDocsInput,
+  collectBasicSearchDocs,
   collectSearchDocs,
   formatSearchNodes,
   type SearchNode,
@@ -50,12 +54,17 @@ export class IndexerService implements OnApplicationBootstrap {
   }
 
   private async syncFeature() {
+    if (!this.server.getConfig().indexer.enabled) {
+      this.server.disableFeature(ServerFeature.Indexer);
+      return;
+    }
     const status = (await this.runtime.searchStatus()) as { ready: boolean };
     if (status.ready) this.server.enableFeature(ServerFeature.Indexer);
     else this.server.disableFeature(ServerFeature.Indexer);
   }
 
   async search(actorUserId: string, workspaceId: string, input: SearchInput) {
+    await this.syncFeature();
     const result = this.unwrap<SearchResult>(
       await this.runtime.searchAuthorized(actorUserId, workspaceId, input),
       workspaceId
@@ -68,10 +77,15 @@ export class IndexerService implements OnApplicationBootstrap {
     workspaceId: string,
     input: AggregateInput
   ) {
+    await this.syncFeature();
     const result = this.unwrap<AggregateResult>(
       await this.runtime.aggregateAuthorized(actorUserId, workspaceId, input),
       workspaceId
     );
+    return this.formatAggregate(result);
+  }
+
+  private formatAggregate(result: AggregateResult) {
     return {
       ...result,
       buckets: result.buckets.map(bucket => ({
@@ -84,38 +98,42 @@ export class IndexerService implements OnApplicationBootstrap {
     };
   }
 
-  async indexDoc(workspaceId: string, docId: string) {
-    await this.runtime.indexSearchDocument(workspaceId, docId);
-  }
-
-  async deleteDoc(workspaceId: string, docId: string) {
-    await this.runtime.deleteSearchDocument(workspaceId, docId);
-  }
-
-  async reconcileWorkspace(workspaceId: string) {
-    await this.runtime.reconcileSearchWorkspace(workspaceId);
-  }
-
-  async deleteWorkspace(workspaceId: string) {
-    await this.runtime.deleteSearchWorkspace(workspaceId);
-  }
-
   async searchDocsByKeyword(
     actorUserId: string,
     workspaceId: string,
     keyword: string,
     options?: { limit?: number; docIds?: string[] }
   ): Promise<SearchDoc[]> {
+    await this.syncFeature();
+    if (options?.limit !== undefined && options.limit <= 0) {
+      throw new InvalidIndexerInput({
+        reason: 'searchDocs limit must be positive',
+      });
+    }
     if (options?.docIds?.length === 0) return [];
-    const result = await this.aggregate(
+    const aggregateOutput = await this.runtime.aggregateAuthorized(
       actorUserId,
       workspaceId,
       buildSearchDocsInput(workspaceId, keyword, options)
     );
-    const { docs, missingTitles, userIds } = collectSearchDocs(
-      result,
-      workspaceId
-    );
+    const collected =
+      !aggregateOutput.ok && aggregateOutput.errorCode === 'unsupported_query'
+        ? collectBasicSearchDocs(
+            await this.search(
+              actorUserId,
+              workspaceId,
+              buildBasicSearchDocsInput(workspaceId, keyword, options)
+            ),
+            workspaceId,
+            options?.limit ?? 20
+          )
+        : collectSearchDocs(
+            this.formatAggregate(
+              this.unwrap<AggregateResult>(aggregateOutput, workspaceId)
+            ),
+            workspaceId
+          );
+    const { docs, missingTitles, userIds } = collected;
     if (missingTitles.length > 0) {
       const metas = await this.models.doc.findMetas(missingTitles, {
         select: { title: true },
@@ -146,9 +164,15 @@ export class IndexerService implements OnApplicationBootstrap {
       case 'unsupported_query':
         throw new InvalidIndexerInput({ reason: output.errorCode });
       case 'provider_unavailable':
-        throw new SearchProviderNotFound();
-      case 'permission_unavailable':
-        throw new WorkspacePermissionNotFound({ spaceId: workspaceId });
+        throw new SearchProviderUnavailable();
+      case 'index_not_ready':
+        throw new SearchIndexNotReady({ spaceId: workspaceId });
+      case 'permission_syncing':
+        throw new SearchPermissionSyncing();
+      case 'index_failed':
+        throw new SearchIndexFailed({
+          diagnosticId: 'search_workspace_reconcile_failed',
+        });
       default:
         throw new InternalServerError();
     }

packages/backend/server/src/plugins/indexer/types.ts

@@ -296,13 +296,23 @@ export class SearchNodeObjectType {
 
 @ObjectType()
 export class SearchResultPagination {
-  @Field(() => Int)
+  @Field(() => Int, {
+    description:
+      'Number of results returned in this response, not a global total',
+  })
   count!: number;
 
-  @Field(() => Boolean)
+  @Field(() => Boolean, {
+    description:
+      'Whether the provider has more candidates; remaining visible results are not guaranteed',
+  })
   hasMore!: boolean;
 
-  @Field(() => String, { nullable: true })
+  @Field(() => String, {
+    nullable: true,
+    description:
+      'Opaque provider candidate cursor; it does not guarantee complete visible-result pagination',
+  })
   nextCursor?: string;
 }
 
@@ -326,7 +336,9 @@ export class AggregateBucketObjectType {
   @Field(() => String)
   key!: string;
 
-  @Field(() => Int)
+  @Field(() => Int, {
+    description: 'Number of returned sample hits in this bucket',
+  })
   count!: number;
 
   @Field(() => AggregateBucketHitsObjectType, {

packages/backend/server/src/schema.gql

@@ -203,6 +203,7 @@ type AggregateBucketHitsObjectType {
 }
 
 type AggregateBucketObjectType {
+  """Number of returned sample hits in this bucket"""
   count: Int!
 
   """The hits object"""
@@ -925,7 +926,7 @@ type EditorType {
   name: String!
 }
 
-union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CalendarProviderRequestErrorDataType | CopilotDocNotFoundDataType | CopilotFailedToAddWorkspaceArtifactDataType | CopilotFailedToGenerateEmbeddingDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderNotSupportedDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | ImageFormatNotSupportedDataType | InvalidAppConfigDataType | InvalidAppConfigInputDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidIndexerInputDataType | InvalidLicenseToActivateDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidOauthResponseDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | InvalidSearchProviderRequestDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NoCopilotProviderAvailableDataType | NoMoreSeatDataType | NotInSpaceDataType | QueryTooLongDataType | ResponseTooLargeErrorDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SsrfBlockedErrorDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedServerVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType
+union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CalendarProviderRequestErrorDataType | CopilotDocNotFoundDataType | CopilotFailedToAddWorkspaceArtifactDataType | CopilotFailedToGenerateEmbeddingDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderNotSupportedDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | ImageFormatNotSupportedDataType | InvalidAppConfigDataType | InvalidAppConfigInputDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidIndexerInputDataType | InvalidLicenseToActivateDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidOauthResponseDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | InvalidSearchProviderRequestDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NoCopilotProviderAvailableDataType | NoMoreSeatDataType | NotInSpaceDataType | QueryTooLongDataType | ResponseTooLargeErrorDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SearchIndexFailedDataType | SearchIndexNotReadyDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SsrfBlockedErrorDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedServerVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType
 
 enum ErrorNames {
   ACCESS_DENIED
@@ -1050,7 +1051,11 @@ enum ErrorNames {
   RUNTIME_CONFIG_NOT_FOUND
   SAME_EMAIL_PROVIDED
   SAME_SUBSCRIPTION_RECURRING
+  SEARCH_INDEX_FAILED
+  SEARCH_INDEX_NOT_READY
+  SEARCH_PERMISSION_SYNCING
   SEARCH_PROVIDER_NOT_FOUND
+  SEARCH_PROVIDER_UNAVAILABLE
   SIGN_UP_FORBIDDEN
   SPACE_ACCESS_DENIED
   SPACE_NOT_FOUND
@@ -2141,6 +2146,14 @@ input SearchHighlight {
   field: String!
 }
 
+type SearchIndexFailedDataType {
+  diagnosticId: String!
+}
+
+type SearchIndexNotReadyDataType {
+  spaceId: String!
+}
+
 input SearchInput {
   options: SearchOptions!
   query: SearchQuery!
@@ -2199,8 +2212,17 @@ type SearchResultObjectType {
 }
 
 type SearchResultPagination {
+  """Number of results returned in this response, not a global total"""
   count: Int!
+
+  """
+  Whether the provider has more candidates; remaining visible results are not guaranteed
+  """
   hasMore: Boolean!
+
+  """
+  Opaque provider candidate cursor; it does not guarantee complete visible-result pagination
+  """
   nextCursor: String
 }