refactor(server): indexer & worker & sync perf (#15504)

packages/backend/server/migrations/20260820120000_cleanup_legacy_copilot_runtime/migration.sql

@@ -0,0 +1,46 @@
+-- This migration is intentionally fail-closed. The data migration with the
+-- same release must have admitted every live legacy context blob through the
+-- artifact runtime before these product-owned tables are removed.
+DO $$
+BEGIN
+  IF to_regclass('public.ai_contexts') IS NOT NULL AND EXISTS (
+    SELECT 1
+    FROM ai_contexts context
+    JOIN ai_sessions_metadata session ON session.id = context.session_id
+    JOIN blobs blob
+      ON blob.workspace_id = session.workspace_id
+     AND blob.deleted_at IS NULL
+     AND blob.status = 'completed'
+    WHERE jsonb_path_exists(
+      context.config::jsonb,
+      '$.** ? (@ == $blobKey)',
+      jsonb_build_object('blobKey', to_jsonb(blob.key::text))
+      )
+      AND NOT EXISTS (
+        SELECT 1
+        FROM workspace_artifacts artifact
+        WHERE artifact.workspace_id = session.workspace_id
+          AND artifact.status = 'ready'
+          AND artifact.storage_scope = 'blob'
+          AND artifact.storage_key = concat(session.workspace_id, '/', blob.key)
+      )
+  ) THEN
+    RAISE EXCEPTION
+      'legacy context blob artifact admission is incomplete; run the data migration before cleanup';
+  END IF;
+END $$;
+
+DELETE FROM app_configs WHERE id = 'copilot.providers.defaults';
+
+ALTER TABLE ai_workspace_byok_configs
+  ALTER COLUMN definition DROP DEFAULT,
+  DROP COLUMN IF EXISTS endpoint,
+  DROP COLUMN IF EXISTS disabled_reason,
+  DROP COLUMN IF EXISTS last_validated_at,
+  DROP COLUMN IF EXISTS last_validation_error;
+
+DELETE FROM ai_workspace_byok_configs WHERE definition = '{}'::jsonb;
+
+DROP TABLE IF EXISTS ai_context_embeddings;
+DROP TABLE IF EXISTS ai_workspace_embeddings;
+DROP TABLE IF EXISTS ai_contexts;

packages/backend/server/package.json

@@ -19,7 +19,7 @@
     "seed": "r ./src/seed/index.ts",
     "genconfig": "r ./scripts/genconfig.ts",
     "cli": "cross-env SERVER_FLAVOR=script node ./dist/main.js",
-    "predeploy": "yarn prisma migrate deploy && yarn cli run",
+    "predeploy": "yarn cli admit-legacy-context-blobs && yarn prisma migrate deploy && yarn cli run",
     "postinstall": "prisma generate"
   },
   "dependencies": {

packages/backend/server/src/__tests__/app/selfhost.e2e.ts

@@ -16,6 +16,7 @@ const test = ava as TestFn<{
   app: TestingApp;
   db: PrismaClient;
 }>;
+let originalDeploymentType: typeof env.DEPLOYMENT_TYPE;
 
 const mobileUAString =
   'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Mobile Safari/537.36';
@@ -47,6 +48,7 @@ export class TestResolver {
 }
 
 test.before('init selfhost server', async t => {
+  originalDeploymentType = globalThis.env.DEPLOYMENT_TYPE;
   // @ts-expect-error override
   globalThis.env.DEPLOYMENT_TYPE = 'selfhosted';
   const app = await createTestingApp({
@@ -69,7 +71,12 @@ test.beforeEach(async t => {
 });
 
 test.after.always(async t => {
-  await t.context.app.close();
+  try {
+    await t.context.app.close();
+  } finally {
+    // @ts-expect-error restore mutable test env singleton
+    globalThis.env.DEPLOYMENT_TYPE = originalDeploymentType;
+  }
 });
 
 test('do not allow visit index.html directly', async t => {

packages/backend/server/src/__tests__/copilot/runtime-boundaries.spec.ts

@@ -10,6 +10,7 @@ import {
   type Config,
   type EventBus,
   type JobQueue,
+  SearchProviderNotFound,
 } from '../../base';
 import { ServerFeature, type ServerService } from '../../core';
 import type { DocReader } from '../../core/doc';
@@ -417,7 +418,6 @@ test('document tools enforce the user-selected hard scope', async t => {
     ) => candidates,
   };
   const hybrid = new DocumentRetrievalService(
-    { indexer: { enabled: true } } as Config,
     readableAc,
     lexicalIndexer,
     vectorSearch,
@@ -432,7 +432,6 @@ test('document tools enforce the user-selected hard scope', async t => {
   t.true(hybridResult.hits[0].score > 1 / 61);
 
   const lexicalOnly = new DocumentRetrievalService(
-    { indexer: { enabled: true } } as Config,
     readableAc,
     lexicalIndexer,
     { ...vectorSearch, canEmbedding: false },
@@ -448,9 +447,12 @@ test('document tools enforce the user-selected hard scope', async t => {
   t.is(lexicalResult.degradedReason, 'VECTOR_UNAVAILABLE');
 
   const vectorOnly = new DocumentRetrievalService(
-    { indexer: { enabled: false } } as Config,
     readableAc,
-    lexicalIndexer,
+    {
+      searchDocsByKeyword: async () => {
+        throw new SearchProviderNotFound();
+      },
+    } as unknown as IndexerService,
     vectorSearch,
     documentModels
   );

packages/backend/server/src/__tests__/create-module.ts

@@ -7,7 +7,18 @@ import {
 import { PrismaClient } from '@prisma/client';
 
 import { FunctionalityModules } from '../app.module';
-import { AFFiNELogger, EventBus, JobModule, JobQueue } from '../base';
+import {
+  AFFiNELogger,
+  ConfigFactory,
+  EventBus,
+  JobModule,
+  JobQueue,
+} from '../base';
+import {
+  BACKEND_RUNTIME_CONFIG_PATHS,
+  BackendRuntimeProvider,
+} from '../core/backend-runtime';
+import { StorageRuntimeProvider } from '../core/storage-runtime';
 import {
   createFactory,
   MockEventBus,
@@ -15,6 +26,7 @@ import {
   MockJobQueue,
 } from './mocks';
 import { TEST_LOG_LEVEL } from './utils';
+import { createTestRuntimeConfig } from './utils/runtime-config';
 
 interface TestingModuleMetadata extends ModuleMetadata {
   tapModule?(m: TestingModuleBuilder): void;
@@ -30,6 +42,11 @@ export interface TestingModule extends NestjsTestingModule {
 export async function createModule(
   metadata: TestingModuleMetadata = {}
 ): Promise<TestingModule> {
+  const config = new ConfigFactory().config;
+  const runtimeConfig = await createTestRuntimeConfig(
+    config.db.datasourceUrl,
+    config.indexer
+  );
   const { tapModule, ...meta } = metadata;
   const functionalityModules = [
     ...FunctionalityModules.filter(module => {
@@ -48,14 +65,22 @@ export async function createModule(
     .overrideProvider(JobQueue)
     .useValue(new MockJobQueue())
     .overrideProvider(EventBus)
-    .useValue(new MockEventBus());
+    .useValue(new MockEventBus())
+    .overrideProvider(BACKEND_RUNTIME_CONFIG_PATHS)
+    .useValue([runtimeConfig.configPath]);
 
   // when custom override happens
   if (tapModule) {
     tapModule(builder);
   }
 
-  const module = (await builder.compile()) as TestingModule;
+  let module: TestingModule;
+  try {
+    module = (await builder.compile()) as TestingModule;
+  } catch (error) {
+    await runtimeConfig.cleanup();
+    throw error;
+  }
 
   const logger = new AFFiNELogger();
   // we got a lot smoking tests try to break nestjs
@@ -63,7 +88,33 @@ export async function createModule(
   logger.setLogLevels([TEST_LOG_LEVEL]);
   module.useLogger(logger);
 
-  await module.init();
+  const close = module.close.bind(module);
+  let closePromise: Promise<void> | undefined;
+  module.close = () => {
+    return (closePromise ??= (async () => {
+      try {
+        await close();
+      } finally {
+        await runtimeConfig.cleanup();
+      }
+    })());
+  };
+
+  try {
+    await module.init();
+  } catch (error) {
+    await module.close();
+    throw error;
+  }
+  const backendRuntime = module.get(BackendRuntimeProvider);
+  if (backendRuntime instanceof BackendRuntimeProvider) {
+    await backendRuntime.runMigrations();
+    await backendRuntime.onConfigChanged({ updates: { indexer: {} } });
+  }
+  const storageRuntime = module.get(StorageRuntimeProvider);
+  if (storageRuntime instanceof StorageRuntimeProvider) {
+    await storageRuntime.runMigrations();
+  }
   module[Symbol.asyncDispose] = async () => {
     await module.close();
   };

packages/backend/server/src/__tests__/doc/cron.spec.ts

@@ -4,7 +4,7 @@ import ava, { TestFn } from 'ava';
 import Sinon from 'sinon';
 
 import { BackendRuntimeProvider } from '../../core/backend-runtime';
-import { DocStorageModule } from '../../core/doc';
+import { DocStorageModule, DocStorageWorkerModule } from '../../core/doc';
 import { DocStorageCronJob } from '../../core/doc/job';
 import { createTestingModule, type TestingModule } from '../utils';
 
@@ -23,7 +23,11 @@ test.before(async t => {
     cleanupExpiredSnapshotHistories: Sinon.stub(),
   };
   t.context.module = await createTestingModule({
-    imports: [ScheduleModule.forRoot(), DocStorageModule],
+    imports: [
+      ScheduleModule.forRoot(),
+      DocStorageModule,
+      DocStorageWorkerModule,
+    ],
     tapModule: builder => {
       builder
         .overrideProvider(BackendRuntimeProvider)

packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts

@@ -1,66 +1,93 @@
 import { getCurrentUserQuery } from '@affine/graphql';
 
 import { JobExecutor } from '../../../base/job/queue/executor';
+import { JobHandlerScanner } from '../../../base/job/queue/scanner';
 import { DatabaseDocReader, DocReader } from '../../../core/doc';
 import { createApp } from '../create-app';
 import { e2e } from '../test';
 
-type TestFlavor = 'doc' | 'graphql' | 'sync' | 'renderer' | 'front';
+type TestFlavor =
+  | 'allinone'
+  | 'worker'
+  | 'graphql'
+  | 'sync'
+  | 'renderer'
+  | 'front';
 
-const createFlavorApp = async (flavor: TestFlavor) => {
+const withFlavor = async <T>(
+  flavor: TestFlavor,
+  run: (app: Awaited<ReturnType<typeof createApp>>) => Promise<T>
+) => {
+  const mutableEnv = globalThis.env as unknown as { FLAVOR: string };
+  const previousFlavor = mutableEnv.FLAVOR;
   // @ts-expect-error override
   globalThis.env.FLAVOR = flavor;
-  return await createApp({
-    tapModule(module) {
-      module.overrideProvider(JobExecutor).useValue({
-        onConfigInit: async () => {},
-        onConfigChanged: async () => {},
-        onModuleDestroy: async () => {},
-      });
-    },
-  });
+  try {
+    await using app = await createApp({
+      tapModule(module) {
+        module.overrideProvider(JobExecutor).useValue({
+          onConfigInit: async () => {},
+          onConfigChanged: async () => {},
+          onModuleDestroy: async () => {},
+        });
+      },
+    });
+    return await run(app);
+  } finally {
+    mutableEnv.FLAVOR = previousFlavor;
+  }
 };
 
-e2e('should init doc service', async t => {
-  await using app = await createFlavorApp('doc');
+e2e('should init worker service', async t => {
+  await withFlavor('worker', async app => {
+    const res = await app.GET('/info').expect(200);
+    t.is(res.body.flavor, 'worker');
+    t.truthy(app.get(JobHandlerScanner).getHandler('indexer.indexDoc'));
 
-  const res = await app.GET('/info').expect(200);
-  t.is(res.body.flavor, 'doc');
+    await t.throwsAsync(app.gql({ query: getCurrentUserQuery }));
+    await app.PUT('/api/storage/upload').expect(404);
+  });
+});
 
-  await t.throwsAsync(app.gql({ query: getCurrentUserQuery }));
+e2e('should init allinone service with worker handlers', async t => {
+  await withFlavor('allinone', async app => {
+    const res = await app.GET('/info').expect(200);
+    t.is(res.body.flavor, 'allinone');
+    t.truthy(app.get(JobHandlerScanner).getHandler('indexer.indexDoc'));
+  });
 });
 
 e2e('should init graphql service', async t => {
-  await using app = await createFlavorApp('graphql');
+  await withFlavor('graphql', async app => {
+    const res = await app.GET('/info').expect(200);
 
-  const res = await app.GET('/info').expect(200);
+    t.is(res.body.flavor, 'graphql');
 
-  t.is(res.body.flavor, 'graphql');
-
-  const user = await app.gql({ query: getCurrentUserQuery });
-  t.is(user.currentUser, null);
+    const user = await app.gql({ query: getCurrentUserQuery });
+    t.is(user.currentUser, null);
+  });
 });
 
 e2e('should init sync service', async t => {
-  await using app = await createFlavorApp('sync');
-
-  const res = await app.GET('/info').expect(200);
-  t.is(res.body.flavor, 'sync');
+  await withFlavor('sync', async app => {
+    const res = await app.GET('/info').expect(200);
+    t.is(res.body.flavor, 'sync');
+  });
 });
 
 e2e('should init renderer service', async t => {
-  await using app = await createFlavorApp('renderer');
-
-  const res = await app.GET('/info').expect(200);
-  t.is(res.body.flavor, 'renderer');
+  await withFlavor('renderer', async app => {
+    const res = await app.GET('/info').expect(200);
+    t.is(res.body.flavor, 'renderer');
+  });
 });
 
 e2e('should init front service', async t => {
-  await using app = await createFlavorApp('front');
+  await withFlavor('front', async app => {
+    const res = await app.GET('/info').expect(200);
+    t.is(res.body.flavor, 'front');
 
-  const res = await app.GET('/info').expect(200);
-  t.is(res.body.flavor, 'front');
-
-  const docReader = app.get(DocReader);
-  t.true(docReader instanceof DatabaseDocReader);
+    const docReader = app.get(DocReader);
+    t.true(docReader instanceof DatabaseDocReader);
+  });
 });

packages/backend/server/src/__tests__/e2e/create-app.ts

@@ -1,7 +1,7 @@
 import assert from 'node:assert';
 
 import { gqlFetcherFactory } from '@affine/graphql';
-import { INestApplication, ModuleMetadata } from '@nestjs/common';
+import { INestApplication, ModuleMetadata, Type } from '@nestjs/common';
 import { NestApplication } from '@nestjs/core';
 import {
   Test,
@@ -26,9 +26,15 @@ import {
 import { ThrottlerStorage } from '../../base/throttler';
 import { SocketIoAdapter } from '../../base/websocket';
 import { AuthGuard, AuthService } from '../../core/auth';
-import { BACKEND_RUNTIME_CONFIG_PATHS } from '../../core/backend-runtime';
+import {
+  BACKEND_RUNTIME_CONFIG_PATHS,
+  BackendRuntimeProvider,
+} from '../../core/backend-runtime';
 import { Mailer } from '../../core/mail';
+import { StorageRuntimeProvider } from '../../core/storage-runtime';
+import { ServerRole } from '../../env';
 import { Models } from '../../models';
+import { IndexerService } from '../../plugins/indexer/service';
 import {
   createFactory,
   MockedUser,
@@ -52,8 +58,16 @@ export class TestingApp extends NestApplication {
   private csrfCookie: string | null = null;
   private readonly userCookies: Set<string> = new Set();
 
+  private getOptional<T>(token: Type<T>) {
+    try {
+      return this.get(token, { strict: false });
+    } catch {
+      return undefined;
+    }
+  }
+
   create = createFactory(this.get(PrismaClient, { strict: false }));
-  mails = this.get(Mailer, { strict: false }) as MockMailer;
+  mails = this.getOptional(Mailer) as unknown as MockMailer;
   queue = this.get(JobQueue, { strict: false }) as MockJobQueue;
   eventBus = this.get(EventBus, { strict: false });
   models = this.get(Models, { strict: false });
@@ -241,8 +255,10 @@ export class TestingApp extends NestApplication {
 export async function createApp(
   metadata: TestingAppMetadata = {}
 ): Promise<TestingApp> {
+  const config = new ConfigFactory().config;
   const runtimeConfig = await createTestRuntimeConfig(
-    new ConfigFactory().config.db.datasourceUrl
+    config.db.datasourceUrl,
+    config.indexer
   );
   const { buildAppModule } = await import('../../app.module');
   const { tapModule, tapApp } = metadata;
@@ -326,7 +342,11 @@ export async function createApp(
     })
   );
 
-  app.useGlobalGuards(app.get(AuthGuard), app.get(CloudThrottlerGuard));
+  if (globalThis.env.role === ServerRole.Worker) {
+    app.useGlobalGuards(app.get(CloudThrottlerGuard));
+  } else {
+    app.useGlobalGuards(app.get(AuthGuard), app.get(CloudThrottlerGuard));
+  }
   app.useGlobalInterceptors(app.get(CacheInterceptor));
   app.useGlobalFilters(new GlobalExceptionFilter(app.getHttpAdapter()));
 
@@ -340,6 +360,9 @@ export async function createApp(
 
   try {
     await app.init();
+    await app.get(BackendRuntimeProvider, { strict: false }).runMigrations();
+    await app.get(StorageRuntimeProvider, { strict: false }).runMigrations();
+    await app.get(IndexerService, { strict: false }).onApplicationBootstrap();
   } catch (error) {
     await app.close();
     throw error;

packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md

@@ -1,98 +0,0 @@
-# Snapshot report for `src/__tests__/e2e/doc-service/controller.spec.ts`
-
-The actual snapshot is saved in `controller.spec.ts.snap`.
-
-Generated by [AVA](https://avajs.dev).
-
-## should get doc markdown success
-
-> Snapshot 1
-
-    {
-      knownUnsupportedBlocks: [
-        'RX4CG2zsBk:affine:note',
-        'S1mkc8zUoU:affine:note',
-        'yGlBdshAqN:affine:note',
-        '6lDiuDqZGL:affine:note',
-        'cauvaHOQmh:affine:note',
-        '2jwCeO8Yot:affine:note',
-        'c9MF_JiRgx:affine:note',
-        '6x7ALjUDjj:affine:surface',
-      ],
-      markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
-      ␊
-      ␊
-      ␊
-      # You own your data, with no compromises␊
-      ␊
-      ## Local-first & Real-time collaborative␊
-      ␊
-      We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊
-      ␊
-      AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊
-      ␊
-      ␊
-      ␊
-      ### Blocks that assemble your next docs, tasks kanban or whiteboard␊
-      ␊
-      There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊
-      ␊
-      We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊
-      ␊
-      If you want to learn more about the product design of AFFiNE, here goes the concepts:␊
-      ␊
-      To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊
-      ␊
-      ## A true canvas for blocks in any form␊
-      ␊
-      [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊
-      ␊
-      ␊
-      ␊
-      "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊
-      ␊
-      * Quip & Notion with their great concept of "everything is a block"␊
-      * Trello with their Kanban␊
-      * Airtable & Miro with their no-code programable datasheets␊
-      * Miro & Whimiscal with their edgeless visual whiteboard␊
-      * Remnote & Capacities with their object-based tag system␊
-      For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊
-      ␊
-      ## Self Host␊
-      ␊
-      Self host AFFiNE␊
-      ␊
-      ␊
-      ### Learning From␊
-      ||Title|Tag|␊
-      |---|---|---|␊
-      |Affine Development|Affine Development|<span data-affine-option data-value="AxSe-53xjX" data-option-color="var(--affine-tag-pink)">AFFiNE</span>|␊
-      |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|<span data-affine-option data-value="0jh9gNw4Yl" data-option-color="var(--affine-tag-orange)">Developers</span>|␊
-      |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Trello with their Kanban|Trello with their Kanban|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊
-      ␊
-      ## Affine Development␊
-      ␊
-      For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊
-      ␊
-      ␊
-      ␊
-      `,
-      title: 'Write, Draw, Plan all at Once.',
-      unknownBlocks: [],
-    }
-
-## should get doc markdown return null when doc not exists
-
-> Snapshot 1
-
-    {
-      code: 'Not Found',
-      message: 'Doc not found',
-      name: 'NOT_FOUND',
-      status: 404,
-      type: 'RESOURCE_NOT_FOUND',
-    }

packages/backend/server/src/__tests__/e2e/doc-service/controller.spec.ts

@@ -1,52 +0,0 @@
-import { randomUUID } from 'node:crypto';
-
-import { CryptoHelper } from '../../../base';
-import { app, e2e, Mockers } from '../test';
-
-const crypto = app.get(CryptoHelper);
-
-e2e('should get doc markdown success', async t => {
-  const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, {
-    owner,
-  });
-
-  const docSnapshot = await app.create(Mockers.DocSnapshot, {
-    workspaceId: workspace.id,
-    user: owner,
-  });
-
-  const path = `/rpc/workspaces/${workspace.id}/docs/${docSnapshot.id}/markdown`;
-  const res = await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect(200)
-    .expect('Content-Type', 'application/json; charset=utf-8');
-
-  const { revision, ...body } = res.body;
-  t.regex(revision, /^\d+$/);
-  t.snapshot(body);
-});
-
-e2e('should get doc markdown return null when doc not exists', async t => {
-  const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, {
-    owner,
-  });
-
-  const docId = randomUUID();
-  const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/markdown`;
-  const res = await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect(404)
-    .expect('Content-Type', 'application/json; charset=utf-8');
-
-  t.snapshot(res.body);
-});

packages/backend/server/src/__tests__/e2e/indexer/aggregate.spec.ts

@@ -1,85 +1,30 @@
-import { indexerAggregateQuery, SearchTable } from '@affine/graphql';
+import {
+  indexerAggregateQuery,
+  SearchQueryType,
+  SearchTable,
+} from '@affine/graphql';
 
+import { createDocWithMarkdown } from '../../../native';
 import { IndexerService } from '../../../plugins/indexer/service';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
 e2e('should aggregate by docId', async t => {
   const owner = await app.signup();
-
-  const workspace = await app.create(Mockers.Workspace, {
-    owner: { id: owner.id },
-  });
-
-  const indexerService = app.get(IndexerService);
-
-  await indexerService.write(
-    SearchTable.block,
-    [
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test1 hello world top2',
-        flavour: 'affine:text',
-        blockId: 'block-0',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-      },
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test2 hello hello top3',
-        flavour: 'affine:text',
-        blockId: 'block-1',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-      },
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test3 hello title top1',
-        flavour: 'affine:page',
-        blockId: 'block-2',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-      },
-      {
-        docId: 'doc-1',
-        workspaceId: workspace.id,
-        content: 'test4 hello world',
-        flavour: 'affine:text',
-        blockId: 'block-3',
-        refDocId: 'doc-0',
-        ref: ['{"foo": "bar1"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-      },
-      {
-        docId: 'doc-2',
-        workspaceId: workspace.id,
-        content: 'test5 hello',
-        flavour: 'affine:text',
-        blockId: 'block-4',
-        refDocId: 'doc-0',
-        ref: ['{"foo": "bar2"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-      },
-    ],
-    {
-      refresh: true,
-    }
-  );
+  const workspace = await app.create(Mockers.Workspace, { owner });
+  for (const [docId, markdown] of [
+    ['doc-0', 'hello world\n\nhello again'],
+    ['doc-1', 'hello world'],
+  ] as const) {
+    await app.create(Mockers.DocMeta, { workspaceId: workspace.id, docId });
+    await app.create(Mockers.DocSnapshot, {
+      workspaceId: workspace.id,
+      docId,
+      user: owner,
+      blob: createDocWithMarkdown(docId, markdown, docId),
+    });
+    await app.get(IndexerService).indexDoc(workspace.id, docId);
+  }
 
   const result = await app.gql({
     query: indexerAggregateQuery,
@@ -88,72 +33,25 @@ e2e('should aggregate by docId', async t => {
       input: {
         table: SearchTable.block,
         query: {
-          // @ts-expect-error allow to use string as enum
-          type: 'boolean',
-          // @ts-expect-error allow to use string as enum
-          occur: 'must',
-          queries: [
-            {
-              // @ts-expect-error allow to use string as enum
-              type: 'match',
-              field: 'content',
-              match: 'hello world',
-            },
-            {
-              // @ts-expect-error allow to use string as enum
-              type: 'boolean',
-              // @ts-expect-error allow to use string as enum
-              occur: 'should',
-              queries: [
-                {
-                  // @ts-expect-error allow to use string as enum
-                  type: 'match',
-                  field: 'content',
-                  match: 'hello world',
-                },
-                {
-                  // @ts-expect-error allow to use string as enum
-                  type: 'boost',
-                  boost: 1.5,
-                  query: {
-                    // @ts-expect-error allow to use string as enum
-                    type: 'match',
-                    field: 'flavour',
-                    match: 'affine:page',
-                  },
-                },
-              ],
-            },
-          ],
+          type: SearchQueryType.match,
+          field: 'content',
+          match: 'hello',
         },
         field: 'docId',
         options: {
-          pagination: {
-            limit: 50,
-            skip: 0,
-          },
+          pagination: { limit: 50, skip: 0 },
           hits: {
-            pagination: {
-              limit: 2,
-              skip: 0,
-            },
-            fields: ['blockId', 'flavour'],
-            highlights: [
-              {
-                field: 'content',
-                before: '<b>',
-                end: '</b>',
-              },
-            ],
+            pagination: { limit: 2, skip: 0 },
+            fields: ['docId', 'blockId', 'content'],
           },
         },
       },
     },
   });
 
-  t.truthy(result.workspace.aggregate, 'failed to aggregate');
-  t.is(result.workspace.aggregate.pagination.count, 5);
-  t.is(result.workspace.aggregate.pagination.hasMore, true);
-  t.truthy(result.workspace.aggregate.pagination.nextCursor);
-  t.snapshot(result.workspace.aggregate.buckets);
+  t.is(result.workspace.aggregate.pagination.count, 2);
+  t.deepEqual(
+    result.workspace.aggregate.buckets.map(bucket => bucket.key).sort(),
+    ['doc-0', 'doc-1']
+  );
 });

packages/backend/server/src/__tests__/e2e/indexer/search-docs.spec.ts

@@ -1,182 +1,57 @@
-import { indexerSearchDocsQuery, SearchTable } from '@affine/graphql';
-import { omit } from 'lodash-es';
+import { indexerSearchDocsQuery } from '@affine/graphql';
 
+import { ConfigFactory } from '../../../base';
+import { createDocWithMarkdown } from '../../../native';
+import { SearchProviderType } from '../../../plugins/indexer/config';
 import { IndexerService } from '../../../plugins/indexer/service';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
 e2e('should search docs by keyword', async t => {
   const owner = await app.signup();
+  const workspace = await app.create(Mockers.Workspace, { owner });
+  for (const docId of ['doc-0', 'doc-1', 'doc-2']) {
+    await app.create(Mockers.DocMeta, { workspaceId: workspace.id, docId });
+    await app.create(Mockers.DocSnapshot, {
+      workspaceId: workspace.id,
+      docId,
+      user: owner,
+      blob: createDocWithMarkdown(docId, `${docId} hello`, docId),
+    });
+    await app.get(IndexerService).indexDoc(workspace.id, docId);
+  }
 
-  const workspace = await app.create(Mockers.Workspace, {
-    owner,
-  });
-
-  const indexerService = app.get(IndexerService);
-
-  await indexerService.write(
-    SearchTable.block,
-    [
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test1 hello',
-        flavour: 'markdown',
-        blockId: 'block-0',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-1',
-        workspaceId: workspace.id,
-        content: 'test2 hello',
-        flavour: 'markdown',
-        blockId: 'block-1',
-        refDocId: ['doc-0'],
-        ref: ['{"foo": "bar1"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2021-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2021-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-2',
-        workspaceId: workspace.id,
-        content: 'test3 hello',
-        flavour: 'markdown',
-        blockId: 'block-2',
-        refDocId: ['doc-0', 'doc-2'],
-        ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-03-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-03-22T03:00:01.000Z'),
-      },
-    ],
-    {
-      refresh: true,
-    }
-  );
-
-  const result = await app.gql({
+  const search = app.gql({
     query: indexerSearchDocsQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        keyword: 'hello',
-      },
-    },
+    variables: { id: workspace.id, input: { keyword: 'hello', limit: 2 } },
   });
+  if (
+    app.get(ConfigFactory).config.indexer.provider.type ===
+    SearchProviderType.Manticoresearch
+  ) {
+    await t.throwsAsync(search, {
+      message: /Invalid indexer input: unsupported_query/,
+    });
+    return;
+  }
 
-  t.is(result.workspace.searchDocs.length, 3);
-  t.snapshot(
-    result.workspace.searchDocs.map(doc =>
-      omit(doc, 'createdByUser', 'updatedByUser')
-    )
-  );
-});
-
-e2e('should search docs by keyword with limit 1', async t => {
-  const owner = await app.signup();
-
-  const workspace = await app.create(Mockers.Workspace, {
-    owner,
-  });
-
-  const indexerService = app.get(IndexerService);
-
-  await indexerService.write(
-    SearchTable.block,
-    [
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test1 hello',
-        flavour: 'markdown',
-        blockId: 'block-0',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-1',
-        workspaceId: workspace.id,
-        content: 'test2 hello',
-        flavour: 'markdown',
-        blockId: 'block-1',
-        refDocId: ['doc-0'],
-        ref: ['{"foo": "bar1"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2021-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2021-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-2',
-        workspaceId: workspace.id,
-        content: 'test3 hello',
-        flavour: 'markdown',
-        blockId: 'block-2',
-        refDocId: ['doc-0', 'doc-2'],
-        ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-03-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-03-22T03:00:01.000Z'),
-      },
-    ],
-    {
-      refresh: true,
-    }
-  );
-
-  const result = await app.gql({
-    query: indexerSearchDocsQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        keyword: 'hello',
-        limit: 1,
-      },
-    },
-  });
-
-  t.is(result.workspace.searchDocs.length, 1);
-  t.snapshot(
-    result.workspace.searchDocs.map(doc =>
-      omit(doc, 'createdByUser', 'updatedByUser')
-    )
-  );
+  const result = await search;
+  t.is(result.workspace.searchDocs.length, 2);
+  t.true(result.workspace.searchDocs.every(doc => doc.highlight.length > 0));
 });
 
 e2e(
   'should search docs by keyword failed when workspace is no permission',
   async t => {
     const owner = await app.signup();
-
-    const workspace = await app.create(Mockers.Workspace, {
-      owner,
-    });
-
-    // signup another user
+    const workspace = await app.create(Mockers.Workspace, { owner });
     await app.signup();
-
     await t.throwsAsync(
       app.gql({
         query: indexerSearchDocsQuery,
-        variables: {
-          id: workspace.id,
-          input: {
-            keyword: 'hello',
-          },
-        },
+        variables: { id: workspace.id, input: { keyword: 'hello' } },
       }),
-      {
-        message: /You do not have permission to access Space/,
-      }
+      { message: /You do not have permission to access Space/ }
     );
   }
 );

packages/backend/server/src/__tests__/e2e/indexer/search.spec.ts

@@ -1,68 +1,40 @@
 import {
   indexerSearchQuery,
-  SearchQueryOccur,
   SearchQueryType,
   SearchTable,
 } from '@affine/graphql';
 
 import { DocRole } from '../../../models';
+import { createDocWithMarkdown } from '../../../native';
 import { IndexerService } from '../../../plugins/indexer/service';
 import { Mockers } from '../../mocks';
 import { app, e2e } from '../test';
 
+async function indexDoc(
+  workspaceId: string,
+  user: { id: string },
+  docId: string,
+  markdown: string,
+  defaultRole = DocRole.Manager
+) {
+  await app.create(Mockers.DocMeta, { workspaceId, docId, defaultRole });
+  await app.create(Mockers.DocSnapshot, {
+    workspaceId,
+    docId,
+    user,
+    blob: createDocWithMarkdown(docId, markdown, docId),
+  });
+  await app.get(IndexerService).indexDoc(workspaceId, docId);
+}
+
 e2e('should search with query', async t => {
   const owner = await app.signup();
-
-  const workspace = await app.create(Mockers.Workspace, {
-    owner: { id: owner.id },
-  });
-
-  const indexerService = app.get(IndexerService);
-
-  await indexerService.write(
-    SearchTable.block,
-    [
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test1',
-        flavour: 'markdown',
-        blockId: 'block-0',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-1',
-        workspaceId: workspace.id,
-        content: 'test2',
-        flavour: 'markdown',
-        blockId: 'block-1',
-        refDocId: ['doc-0'],
-        ref: ['{"foo": "bar1"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2021-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2021-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-2',
-        workspaceId: workspace.id,
-        content: 'test3',
-        flavour: 'markdown',
-        blockId: 'block-2',
-        refDocId: ['doc-0', 'doc-2'],
-        ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-03-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-03-22T00:00:00.000Z'),
-      },
-    ],
-    {
-      refresh: true,
-    }
+  const workspace = await app.create(Mockers.Workspace, { owner });
+  await indexDoc(
+    workspace.id,
+    owner,
+    'doc-0',
+    'searchable first\n\nsearchable second'
   );
 
   const result = await app.gql({
@@ -72,158 +44,95 @@ e2e('should search with query', async t => {
       input: {
         table: SearchTable.block,
         query: {
-          type: SearchQueryType.boolean,
-          occur: SearchQueryOccur.must,
-          queries: [
-            {
-              type: SearchQueryType.boolean,
-              occur: SearchQueryOccur.should,
-              queries: ['doc-0', 'doc-1', 'doc-2'].map(id => ({
-                type: SearchQueryType.match,
-                field: 'docId',
-                match: id,
-              })),
-            },
-            {
-              type: SearchQueryType.exists,
-              field: 'refDocId',
-            },
-          ],
+          type: SearchQueryType.match,
+          field: 'content',
+          match: 'searchable',
         },
         options: {
-          fields: ['refDocId', 'ref'],
-          pagination: {
-            limit: 100,
-          },
+          fields: ['docId', 'blockId', 'content'],
+          highlights: [{ field: 'content', before: '<b>', end: '</b>' }],
+          pagination: { limit: 100 },
         },
       },
     },
   });
 
-  t.truthy(result.workspace.search, 'failed to search');
-  t.is(result.workspace.search.pagination.count, 2);
-  t.is(result.workspace.search.pagination.hasMore, true);
-  t.truthy(result.workspace.search.pagination.nextCursor);
-  t.is(result.workspace.search.nodes.length, 2);
-  t.snapshot(result.workspace.search.nodes);
+  t.true(result.workspace.search.pagination.count > 0);
+  t.true(
+    result.workspace.search.nodes.every(node =>
+      node.fields.docId.includes('doc-0')
+    )
+  );
+  t.true(
+    result.workspace.search.nodes.some(node =>
+      node.highlights?.content?.some((value: string) => value.includes('<b>'))
+    )
+  );
+
+  const firstPage = await app.gql({
+    query: indexerSearchQuery,
+    variables: {
+      id: workspace.id,
+      input: {
+        table: SearchTable.block,
+        query: {
+          type: SearchQueryType.match,
+          field: 'content',
+          match: 'searchable',
+        },
+        options: {
+          fields: ['docId', 'blockId'],
+          pagination: { limit: 1 },
+        },
+      },
+    },
+  });
+  const secondPage = await app.gql({
+    query: indexerSearchQuery,
+    variables: {
+      id: workspace.id,
+      input: {
+        table: SearchTable.block,
+        query: {
+          type: SearchQueryType.match,
+          field: 'content',
+          match: 'searchable',
+        },
+        options: {
+          fields: ['docId', 'blockId'],
+          pagination: {
+            limit: 1,
+            cursor: firstPage.workspace.search.pagination.nextCursor,
+          },
+        },
+      },
+    },
+  });
+  t.not(
+    firstPage.workspace.search.nodes[0].fields.blockId[0],
+    secondPage.workspace.search.nodes[0].fields.blockId[0]
+  );
 });
 
 e2e('should filter no read permission docs on team workspace', async t => {
   const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, {
+  const workspace = await app.create(Mockers.Workspace, { owner });
+  await app.create(Mockers.TeamWorkspace, { id: workspace.id });
+  await indexDoc(
+    workspace.id,
     owner,
-  });
-  await app.create(Mockers.TeamWorkspace, {
-    id: workspace.id,
-  });
-
-  const indexerService = app.get(IndexerService);
-  await indexerService.write(
-    SearchTable.block,
-    [
-      {
-        docId: 'doc-0',
-        workspaceId: workspace.id,
-        content: 'test1',
-        flavour: 'markdown',
-        blockId: 'block-0',
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-1',
-        workspaceId: workspace.id,
-        content: 'test2',
-        flavour: 'markdown',
-        blockId: 'block-1',
-        refDocId: ['doc-0'],
-        ref: ['{"foo": "bar1"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2021-04-22T00:00:00.000Z'),
-        updatedAt: new Date('2021-04-22T00:00:00.000Z'),
-      },
-      {
-        docId: 'doc-2',
-        workspaceId: workspace.id,
-        content: 'test3',
-        flavour: 'markdown',
-        blockId: 'block-2',
-        refDocId: ['doc-0', 'doc-2'],
-        ref: ['{"foo": "bar1"}', '{"foo": "bar3"}'],
-        createdByUserId: owner.id,
-        updatedByUserId: owner.id,
-        createdAt: new Date('2025-03-22T00:00:00.000Z'),
-        updatedAt: new Date('2025-03-22T00:00:00.000Z'),
-      },
-    ],
-    {
-      refresh: true,
-    }
+    'private-doc',
+    'team secret searchable',
+    DocRole.None
   );
-  // set all docs to no access
-  await app.create(Mockers.DocMeta, {
-    workspaceId: workspace.id,
-    docId: 'doc-0',
-    defaultRole: DocRole.None,
-  });
-  await app.create(Mockers.DocMeta, {
-    workspaceId: workspace.id,
-    docId: 'doc-1',
-    defaultRole: DocRole.None,
-  });
-  await app.create(Mockers.DocMeta, {
-    workspaceId: workspace.id,
-    docId: 'doc-2',
-    defaultRole: DocRole.None,
-  });
 
-  // owner can read all docs
-  const result = await app.gql({
-    query: indexerSearchQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        table: SearchTable.block,
-        query: {
-          type: SearchQueryType.match,
-          field: 'workspaceId',
-          match: workspace.id,
-        },
-        options: {
-          fields: ['docId', 'blockId', 'refDocId', 'ref'],
-          pagination: {
-            limit: 100,
-          },
-        },
-      },
-    },
-  });
-
-  t.snapshot(result.workspace.search.nodes);
-
-  // other user can only read docs that they have read permission
-  const other = await app.signup();
+  const member = await app.signup();
   await app.create(Mockers.WorkspaceUser, {
     workspaceId: workspace.id,
-    userId: other.id,
+    userId: member.id,
   });
-  await app.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'doc-0',
-    userId: other.id,
-    type: DocRole.Reader,
-  });
-  await app.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'doc-1',
-    userId: other.id,
-    type: DocRole.Manager,
-  });
-
-  const otherResult = await app.gql({
+  await app.get(IndexerService).reconcileWorkspace(workspace.id);
+  const denied = await app.gql({
     query: indexerSearchQuery,
     variables: {
       id: workspace.id,
@@ -231,132 +140,74 @@ e2e('should filter no read permission docs on team workspace', async t => {
         table: SearchTable.block,
         query: {
           type: SearchQueryType.match,
-          field: 'workspaceId',
-          match: workspace.id,
-        },
-        options: {
-          fields: ['docId', 'blockId', 'refDocId', 'ref'],
-          pagination: {
-            limit: 100,
-          },
+          field: 'content',
+          match: 'secret',
         },
+        options: { fields: ['docId'], pagination: { limit: 10 } },
       },
     },
   });
+  t.is(denied.workspace.search.pagination.count, 0);
 
-  t.snapshot(otherResult.workspace.search.nodes);
+  await app.create(Mockers.DocUser, {
+    workspaceId: workspace.id,
+    docId: 'private-doc',
+    userId: member.id,
+    type: DocRole.Reader,
+  });
+  await app.get(IndexerService).reconcileWorkspace(workspace.id);
+  const allowed = await app.gql({
+    query: indexerSearchQuery,
+    variables: {
+      id: workspace.id,
+      input: {
+        table: SearchTable.block,
+        query: {
+          type: SearchQueryType.match,
+          field: 'content',
+          match: 'secret',
+        },
+        options: { fields: ['docId'], pagination: { limit: 10 } },
+      },
+    },
+  });
+  t.true(allowed.workspace.search.pagination.count > 0);
+
+  await app.models.docUser.delete(workspace.id, 'private-doc', member.id);
+  await app.get(IndexerService).reconcileWorkspace(workspace.id);
+  const revoked = await app.gql({
+    query: indexerSearchQuery,
+    variables: {
+      id: workspace.id,
+      input: {
+        table: SearchTable.block,
+        query: {
+          type: SearchQueryType.match,
+          field: 'content',
+          match: 'secret',
+        },
+        options: { fields: ['docId'], pagination: { limit: 10 } },
+      },
+    },
+  });
+  t.is(revoked.workspace.search.pagination.count, 0);
 });
 
 e2e('should return empty results when search not match any docs', async t => {
   const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, {
-    owner,
-  });
-
-  const result = await app.gql({
-    query: indexerSearchQuery,
-    variables: {
-      id: workspace.id,
-      input: {
-        table: SearchTable.block,
-        query: {
-          type: SearchQueryType.match,
-          field: 'workspaceId',
-          match: workspace.id,
-        },
-        options: {
-          fields: ['docId', 'blockId', 'refDocId', 'ref'],
-          pagination: {
-            limit: 100,
-          },
-        },
-      },
-    },
-  });
-
-  t.snapshot(result);
-});
-
-e2e('should return empty nodes when docId not exists', async t => {
-  const owner = await app.signup();
-  const workspace = await app.create(Mockers.Workspace, {
-    owner,
-  });
-
+  const workspace = await app.create(Mockers.Workspace, { owner });
+  await app.get(IndexerService).reconcileWorkspace(workspace.id);
   const result = await app.gql({
     query: indexerSearchQuery,
     variables: {
       id: workspace.id,
       input: {
         table: SearchTable.doc,
-        query: {
-          type: SearchQueryType.match,
-          field: 'docId',
-          match: 'not-exists-doc-id',
-        },
-        options: {
-          fields: ['summary'],
-          pagination: {
-            limit: 1,
-          },
-        },
+        query: { type: SearchQueryType.match, field: 'title', match: 'absent' },
+        options: { fields: ['docId'], pagination: { limit: 10 } },
       },
     },
   });
-
-  t.snapshot(result);
+  t.is(result.workspace.search.pagination.count, 0);
+  t.deepEqual(result.workspace.search.nodes, []);
 });
-
-e2e(
-  'should empty doc summary string when doc exists but no summary',
-  async t => {
-    const owner = await app.signup();
-    const workspace = await app.create(Mockers.Workspace, {
-      owner,
-    });
-
-    const indexerService = app.get(IndexerService);
-
-    await indexerService.write(
-      SearchTable.doc,
-      [
-        {
-          docId: 'doc-1-without-summary',
-          workspaceId: workspace.id,
-          title: 'test1',
-          summary: '',
-          createdByUserId: owner.id,
-          updatedByUserId: owner.id,
-          createdAt: new Date('2025-04-22T00:00:00.000Z'),
-          updatedAt: new Date('2025-04-22T00:00:00.000Z'),
-        },
-      ],
-      {
-        refresh: true,
-      }
-    );
-
-    const result = await app.gql({
-      query: indexerSearchQuery,
-      variables: {
-        id: workspace.id,
-        input: {
-          table: SearchTable.doc,
-          query: {
-            type: SearchQueryType.match,
-            field: 'docId',
-            match: 'doc-1-without-summary',
-          },
-          options: {
-            fields: ['summary'],
-            pagination: {
-              limit: 1,
-            },
-          },
-        },
-      },
-    });
-
-    t.snapshot(result.workspace.search.nodes);
-  }
-);

packages/backend/server/src/__tests__/e2e/notification/resolver.spec.ts

@@ -121,6 +121,51 @@ e2e('should mention user in a doc', async t => {
   t.falsy(body2.workspace!.avatarUrl);
 });
 
+e2e(
+  'notification totalCount selection does not load the notification list',
+  async t => {
+    const { member, owner, workspace } = await init();
+
+    await app.login(owner);
+    await app.gql({
+      query: mentionUserMutation,
+      variables: {
+        input: {
+          userId: member.id,
+          workspaceId: workspace.id,
+          doc: {
+            id: 'count-only-doc',
+            title: 'count-only-doc',
+            mode: DocMode.page,
+          },
+        },
+      },
+    });
+
+    await app.login(member);
+    const result = (await app.gql({
+      query: {
+        ...listNotificationsQuery,
+        op: 'CountOnlyNotifications',
+        query: `
+        query CountOnlyNotifications($pagination: PaginationInput!) {
+          currentUser {
+            notifications(pagination: $pagination) {
+              totalCount
+            }
+          }
+      }
+      `,
+      },
+      variables: { pagination: { first: 10, offset: 0 } },
+    })) as unknown as {
+      currentUser: { notifications: { totalCount: number } };
+    };
+
+    t.is(result.currentUser.notifications.totalCount, 1);
+  }
+);
+
 e2e('should mention doc mode support string value', async t => {
   const { member, owner, workspace } = await init();
 

packages/backend/server/src/__tests__/env.spec.ts

@@ -68,14 +68,20 @@ test('should read DEPLOYMENT_TYPE', t => {
 
 test('should read FLAVOR', t => {
   t.deepEqual(
-    ['allinone', 'graphql', 'sync', 'renderer', 'front', 'doc', 'script'].map(
-      envVal => {
-        process.env.SERVER_FLAVOR = envVal;
-        const env = new Env();
-        return env.FLAVOR;
-      }
-    ),
-    ['allinone', 'graphql', 'sync', 'renderer', 'front', 'doc', 'script']
+    [
+      'allinone',
+      'graphql',
+      'sync',
+      'renderer',
+      'front',
+      'worker',
+      'script',
+    ].map(envVal => {
+      process.env.SERVER_FLAVOR = envVal;
+      const env = new Env();
+      return env.FLAVOR;
+    }),
+    ['allinone', 'graphql', 'sync', 'renderer', 'front', 'worker', 'script']
   );
 
   t.throws(
@@ -85,7 +91,7 @@ test('should read FLAVOR', t => {
     },
     {
       message:
-        'Invalid value "unknown" for environment variable SERVER_FLAVOR, expected one of ["allinone","graphql","sync","renderer","front","doc","script"]',
+        'Invalid value "unknown" for environment variable SERVER_FLAVOR, expected one of ["allinone","graphql","sync","renderer","front","worker","script"]',
     }
   );
 });
@@ -113,7 +119,7 @@ test('should tell flavors correctly', t => {
     sync: true,
     renderer: true,
     front: false,
-    doc: true,
+    worker: true,
     script: false,
   });
 
@@ -123,7 +129,7 @@ test('should tell flavors correctly', t => {
     sync: false,
     renderer: false,
     front: false,
-    doc: false,
+    worker: false,
     script: false,
   });
 
@@ -133,7 +139,7 @@ test('should tell flavors correctly', t => {
     sync: false,
     renderer: false,
     front: true,
-    doc: false,
+    worker: false,
     script: false,
   });
 
@@ -143,7 +149,7 @@ test('should tell flavors correctly', t => {
     sync: false,
     renderer: false,
     front: false,
-    doc: false,
+    worker: false,
     script: true,
   });
 });

packages/backend/server/src/__tests__/event/cluster.spec.ts

@@ -63,6 +63,21 @@ test('should broadcast event to cluster instances', async t => {
   off();
 });
 
+test('should preserve encoded binary updates across cluster instances', async t => {
+  const { app1, app2 } = t.context;
+  const eventbus1 = app1.get(EventBus);
+  const eventbus2 = app2.get(EventBus);
+  const listener = Sinon.spy(app1.get(Listeners), 'onEncodedBinaryEvent');
+  const payload = {
+    updates: [Buffer.from(new Uint8Array([1, 2, 3])).toString('base64')],
+  };
+
+  eventbus2.broadcast('__test__.encodedBinary', payload);
+  await eventbus1.waitFor('__test__.encodedBinary');
+
+  t.true(listener.calledOnceWith(payload));
+});
+
 test('should continuously use the same request id', async t => {
   const { app1, app2 } = t.context;
 

packages/backend/server/src/__tests__/event/provider.ts

@@ -7,6 +7,7 @@ declare global {
   interface Events {
     '__test__.event': { count: number };
     '__test__.event2': { count: number };
+    '__test__.encodedBinary': { updates: string[] };
     '__test__.throw': { count: number };
     '__test__.suppressThrow': {};
     '__test__.requestId': {};
@@ -28,6 +29,11 @@ export class Listeners {
     return payload;
   }
 
+  @OnEvent('__test__.encodedBinary')
+  onEncodedBinaryEvent(payload: Events['__test__.encodedBinary']) {
+    return payload;
+  }
+
   @OnEvent('__test__.throw')
   onThrow() {
     throw new Error('Error in event handler');

packages/backend/server/src/__tests__/mocks/eventbus.mock.ts

@@ -6,10 +6,10 @@ import { EventName } from '../../base/event/def';
 export class MockEventBus {
   private readonly stub = Sinon.createStubInstance(EventBus);
 
-  emit = this.stub.emitAsync;
-  emitAsync = this.stub.emitAsync;
-  emitDetached = this.stub.emitAsync;
-  broadcast = this.stub.broadcast;
+  emit: Sinon.SinonStub = this.stub.emitAsync;
+  emitAsync: Sinon.SinonStub = this.stub.emitAsync;
+  emitDetached: Sinon.SinonStub = this.stub.emitAsync;
+  broadcast: Sinon.SinonStub = this.stub.broadcast;
 
   last<Event extends EventName>(
     name: Event
@@ -22,7 +22,6 @@ export class MockEventBus {
       throw new Error(`Event ${name} never called`);
     }
 
-    // @ts-expect-error allow
     return {
       name,
       payload: call.args[1],

packages/backend/server/src/__tests__/models/copilot-job.spec.ts

@@ -136,127 +136,3 @@ test('should claim job', async t => {
     'should update job status to claimed'
   );
 });
-
-test('should fence transcript dispatch generations atomically', async t => {
-  const task = await t.context.transcriptTask.create({
-    userId: user.id,
-    workspaceId: workspace.id,
-    blobId: 'transcript-blob',
-    recipeId: 'transcript.audio',
-    recipeVersion: 'v1',
-    inputSnapshot: { normalizedTranscript: 'source' },
-  });
-  const adoptions = await Promise.all([
-    t.context.transcriptTask.adoptLegacyDispatch(
-      task.id,
-      null,
-      'legacy-generation-a'
-    ),
-    t.context.transcriptTask.adoptLegacyDispatch(
-      task.id,
-      null,
-      'legacy-generation-b'
-    ),
-  ]);
-  t.is(adoptions.filter(Boolean).length, 1);
-  const adopted = await t.context.transcriptTask.get(task.id);
-  const adoptedGeneration = adopted?.dispatchGeneration;
-  if (!adoptedGeneration) {
-    t.fail('legacy dispatch should have a generation');
-    return;
-  }
-  t.true(
-    await t.context.transcriptTask.claimDispatch(
-      task.id,
-      adoptedGeneration,
-      null
-    )
-  );
-  t.true(
-    await t.context.transcriptTask.completeDispatch(
-      task.id,
-      adoptedGeneration,
-      null,
-      {
-        status: 'failed',
-        protectedResult: { normalizedTranscript: 'source' },
-        errorCode: 'provider_failed',
-      }
-    )
-  );
-
-  const claims = await Promise.all([
-    t.context.transcriptTask.claimRetry(
-      task.id,
-      user.id,
-      workspace.id,
-      null,
-      'generation-a'
-    ),
-    t.context.transcriptTask.claimRetry(
-      task.id,
-      user.id,
-      workspace.id,
-      null,
-      'generation-b'
-    ),
-  ]);
-  t.is(claims.filter(Boolean).length, 1);
-
-  const claimed = await t.context.transcriptTask.get(task.id);
-  const generation = claimed?.dispatchGeneration;
-  if (!generation) {
-    t.fail('retry should have a dispatch generation');
-    return;
-  }
-  t.false(
-    await t.context.transcriptTask.claimDispatch(
-      task.id,
-      generation === 'generation-a' ? 'generation-b' : 'generation-a',
-      null
-    )
-  );
-  t.true(
-    await t.context.transcriptTask.claimDispatch(task.id, generation, null)
-  );
-  t.true(
-    await t.context.transcriptTask.attachActionRun(
-      task.id,
-      generation,
-      null,
-      'run-next'
-    )
-  );
-  t.false(
-    await t.context.transcriptTask.attachActionRun(
-      task.id,
-      generation,
-      null,
-      'run-duplicate'
-    )
-  );
-  t.false(
-    await t.context.transcriptTask.completeDispatch(
-      task.id,
-      generation,
-      'run-duplicate',
-      { status: 'ready' }
-    )
-  );
-  t.true(
-    await t.context.transcriptTask.completeDispatch(
-      task.id,
-      generation,
-      'run-next',
-      {
-        status: 'ready',
-        protectedResult: { normalizedTranscript: 'result' },
-      }
-    )
-  );
-  t.like(await t.context.transcriptTask.get(task.id), {
-    status: 'ready',
-    dispatchGeneration: null,
-    actionRunId: 'run-next',
-  });
-});

packages/backend/server/src/__tests__/models/session.spec.ts

@@ -4,7 +4,7 @@ import ava, { TestFn } from 'ava';
 import { Config } from '../../base/config';
 import { SessionModel } from '../../models/session';
 import { UserModel } from '../../models/user';
-import { createTestingModule, type TestingModule } from '../utils';
+import { createTestingModule, sleep, type TestingModule } from '../utils';
 
 interface Context {
   config: Config;
@@ -109,6 +109,7 @@ test('should refresh exists userSession', async t => {
   t.is(userSession.userId, user.id);
   t.not(userSession.expiresAt, null);
 
+  await sleep(1);
   const existsUserSession = await t.context.session.createOrRefreshUserSession(
     user.id,
     session.id

packages/backend/server/src/__tests__/storage/blob-upload-cleanup.spec.ts

@@ -4,7 +4,11 @@ import ava, { TestFn } from 'ava';
 import Sinon from 'sinon';
 
 import { OneDay } from '../../base';
-import { StorageModule, WorkspaceBlobStorage } from '../../core/storage';
+import {
+  StorageModule,
+  StorageWorkerModule,
+  WorkspaceBlobStorage,
+} from '../../core/storage';
 import { BlobUploadCleanupJob } from '../../core/storage/job';
 import { StorageRuntimeProvider } from '../../core/storage-runtime';
 import { MockUser, MockWorkspace } from '../mocks';
@@ -25,7 +29,7 @@ test.before(async t => {
     cleanupExpiredPendingBlobs: Sinon.stub(),
   };
   t.context.module = await createTestingModule({
-    imports: [ScheduleModule.forRoot(), StorageModule],
+    imports: [ScheduleModule.forRoot(), StorageModule, StorageWorkerModule],
     tapModule: builder => {
       builder
         .overrideProvider(StorageRuntimeProvider)

packages/backend/server/src/__tests__/sync/gateway.spec.ts

@@ -3,7 +3,7 @@ import test, { type ExecutionContext } from 'ava';
 import { io, type Socket as SocketIOClient } from 'socket.io-client';
 import { Doc, encodeStateAsUpdate } from 'yjs';
 
-import { CANARY_CLIENT_VERSION_MAX_AGE_DAYS } from '../../base';
+import { CANARY_CLIENT_VERSION_MAX_AGE_DAYS, EventBus } from '../../base';
 import {
   DocRole,
   Models,
@@ -312,71 +312,7 @@ test('should reject websocket jwt auth after session deletion', async t => {
   }
 });
 
-test('clientVersion=0.25.0 should only receive space:broadcast-doc-update', async t => {
-  const { user, cookieHeader } = await login(app);
-  const spaceId = user.id;
-  const update = createYjsUpdateBase64();
-
-  const sender = createClient(url, cookieHeader);
-  const receiver = createClient(url, cookieHeader);
-
-  try {
-    await Promise.all([waitForConnect(sender), waitForConnect(receiver)]);
-
-    const receiverJoin = unwrapResponse(
-      t,
-      await emitWithAck<{ clientId: string; success: boolean }>(
-        receiver,
-        'space:join',
-        { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' }
-      )
-    );
-    t.true(receiverJoin.success);
-
-    const senderJoin = unwrapResponse(
-      t,
-      await emitWithAck<{ clientId: string; success: boolean }>(
-        sender,
-        'space:join',
-        { spaceType: 'userspace', spaceId, clientVersion: '0.26.0' }
-      )
-    );
-    t.true(senderJoin.success);
-
-    const onUpdate = waitForEvent<{
-      spaceType: string;
-      spaceId: string;
-      docId: string;
-      update: string;
-    }>(receiver, 'space:broadcast-doc-update');
-    const noUpdates = expectNoEvent(receiver, 'space:broadcast-doc-updates');
-
-    const pushRes = await emitWithAck<{ accepted: true; timestamp?: number }>(
-      sender,
-      'space:push-doc-update',
-      {
-        spaceType: 'userspace',
-        spaceId,
-        docId: 'doc-1',
-        update,
-      }
-    );
-    unwrapResponse(t, pushRes);
-
-    const message = await onUpdate;
-    t.is(message.spaceType, 'userspace');
-    t.is(message.spaceId, spaceId);
-    t.is(message.docId, 'doc-1');
-    t.is(message.update, update);
-
-    await noUpdates;
-  } finally {
-    sender.disconnect();
-    receiver.disconnect();
-  }
-});
-
-test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', async t => {
+test('clientVersion>=0.26.0 should receive legacy space:broadcast-doc-updates', async t => {
   const { user, cookieHeader } = await loginWithCookie(app);
   const spaceId = user.id;
   const update = createYjsUpdateBase64();
@@ -402,7 +338,7 @@ test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', as
       await emitWithAck<{ clientId: string; success: boolean }>(
         sender,
         'space:join',
-        { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' }
+        { spaceType: 'userspace', spaceId, clientVersion: '0.26.0' }
       )
     );
     t.true(senderJoin.success);
@@ -413,7 +349,6 @@ test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', as
       docId: string;
       updates: string[];
     }>(receiver, 'space:broadcast-doc-updates');
-    const noUpdate = expectNoEvent(receiver, 'space:broadcast-doc-update');
 
     const pushRes = await emitWithAck<{ accepted: true; timestamp?: number }>(
       sender,
@@ -432,15 +367,13 @@ test('clientVersion>=0.26.0 should only receive space:broadcast-doc-updates', as
     t.is(message.spaceId, spaceId);
     t.is(message.docId, 'doc-2');
     t.deepEqual(message.updates, [update]);
-
-    await noUpdate;
   } finally {
     sender.disconnect();
     receiver.disconnect();
   }
 });
 
-test('canary date clientVersion should use sync-026 in canary namespace', async t => {
+test('canary date clientVersion should use sync-027 in canary namespace', async t => {
   const prevNamespace = env.NAMESPACE;
   // @ts-expect-error test
   env.NAMESPACE = 'dev';
@@ -456,15 +389,18 @@ test('canary date clientVersion should use sync-026 in canary namespace', async
     try {
       await Promise.all([waitForConnect(sender), waitForConnect(receiver)]);
 
+      const canaryVersion = makeCanaryDateVersion(new Date(), '015');
       const receiverJoin = unwrapResponse(
         t,
         await emitWithAck<{ clientId: string; success: boolean }>(
           receiver,
-          'space:join',
+          'space:join-batch',
           {
-            spaceType: 'userspace',
-            spaceId,
-            clientVersion: makeCanaryDateVersion(new Date(), '015'),
+            spaces: [
+              { spaceType: 'userspace', spaceId },
+              { spaceType: 'userspace', spaceId, docId: 'doc-canary' },
+            ],
+            clientVersion: canaryVersion,
           }
         )
       );
@@ -474,8 +410,14 @@ test('canary date clientVersion should use sync-026 in canary namespace', async
         t,
         await emitWithAck<{ clientId: string; success: boolean }>(
           sender,
-          'space:join',
-          { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' }
+          'space:join-batch',
+          {
+            spaces: [
+              { spaceType: 'userspace', spaceId },
+              { spaceType: 'userspace', spaceId, docId: 'doc-canary' },
+            ],
+            clientVersion: canaryVersion,
+          }
         )
       );
       t.true(senderJoin.success);
@@ -486,7 +428,6 @@ test('canary date clientVersion should use sync-026 in canary namespace', async
         docId: string;
         updates: string[];
       }>(receiver, 'space:broadcast-doc-updates');
-      const noUpdate = expectNoEvent(receiver, 'space:broadcast-doc-update');
 
       const pushRes = await emitWithAck<{ accepted: true; timestamp?: number }>(
         sender,
@@ -505,8 +446,6 @@ test('canary date clientVersion should use sync-026 in canary namespace', async
       t.is(message.spaceId, spaceId);
       t.is(message.docId, 'doc-canary');
       t.deepEqual(message.updates, [update]);
-
-      await noUpdate;
     } finally {
       sender.disconnect();
       receiver.disconnect();
@@ -517,7 +456,7 @@ test('canary date clientVersion should use sync-026 in canary namespace', async
   }
 });
 
-test('clientVersion<0.25.0 should be rejected and disconnected', async t => {
+test('clientVersion<0.26.0 should be rejected and disconnected', async t => {
   const { user, cookieHeader } = await login(app);
   const spaceId = user.id;
 
@@ -530,7 +469,7 @@ test('clientVersion<0.25.0 should be rejected and disconnected', async t => {
       await emitWithAck<{ clientId: string; success: boolean }>(
         socket,
         'space:join',
-        { spaceType: 'userspace', spaceId, clientVersion: '0.24.4' }
+        { spaceType: 'userspace', spaceId, clientVersion: '0.25.0' }
       )
     );
     t.false(res.success);
@@ -620,7 +559,7 @@ test('canary date clientVersion should be rejected outside canary namespace', as
   }
 });
 
-test('space:join-awareness should reject clientVersion<0.25.0', async t => {
+test('space:join-awareness should reject clientVersion<0.26.0', async t => {
   const { user, cookieHeader } = await login(app);
   const spaceId = user.id;
 
@@ -637,7 +576,7 @@ test('space:join-awareness should reject clientVersion<0.25.0', async t => {
           spaceType: 'userspace',
           spaceId,
           docId: 'doc-awareness',
-          clientVersion: '0.24.4',
+          clientVersion: '0.25.0',
         }
       )
     );
@@ -649,6 +588,611 @@ test('space:join-awareness should reject clientVersion<0.25.0', async t => {
   }
 });
 
+test('new clients must use batch join endpoints on new servers', async t => {
+  const { user, cookieHeader } = await login(app);
+  const requests = [
+    {
+      event: 'space:join',
+      payload: {
+        spaceType: 'userspace',
+        spaceId: user.id,
+        clientVersion: '0.27.5',
+      },
+    },
+    {
+      event: 'space:join-awareness',
+      payload: {
+        spaceType: 'userspace',
+        spaceId: user.id,
+        docId: 'doc-awareness',
+        clientVersion: '0.27.5',
+      },
+    },
+  ] as const;
+
+  for (const request of requests) {
+    const socket = createClient(url, cookieHeader);
+    try {
+      await waitForConnect(socket);
+      const result = unwrapResponse(
+        t,
+        await emitWithAck<{ clientId: string; success: boolean }>(
+          socket,
+          request.event,
+          request.payload
+        )
+      );
+      t.false(result.success);
+      await waitForDisconnect(socket);
+    } finally {
+      socket.disconnect();
+    }
+  }
+});
+
+test('space:join-batch should validate entries before joining', async t => {
+  const { user, cookieHeader } = await login(app);
+  const socket = createClient(url, cookieHeader);
+  const spaceId = user.id;
+
+  try {
+    await waitForConnect(socket);
+
+    const invalidBatches = [
+      {
+        label: 'empty',
+        payload: { spaces: [], clientVersion: '0.27.5' },
+      },
+      {
+        label: 'missing client version',
+        payload: { spaces: [{ spaceType: 'userspace', spaceId }] },
+      },
+      {
+        label: 'cross workspace',
+        payload: {
+          spaces: [
+            { spaceType: 'userspace', spaceId },
+            { spaceType: 'userspace', spaceId: `${spaceId}-other` },
+          ],
+          clientVersion: '0.27.5',
+        },
+      },
+      {
+        label: 'duplicate',
+        payload: {
+          spaces: [
+            { spaceType: 'userspace', spaceId, docId: 'doc-1' },
+            { spaceType: 'userspace', spaceId, docId: 'doc-1' },
+          ],
+          clientVersion: '0.27.5',
+        },
+      },
+      {
+        label: 'invalid entry',
+        payload: {
+          spaces: [{ spaceType: 'invalid', spaceId }],
+          clientVersion: '0.27.5',
+        },
+      },
+      {
+        label: 'over limit',
+        payload: {
+          spaces: Array.from({ length: 101 }, (_, index) => ({
+            spaceType: 'userspace',
+            spaceId,
+            docId: `doc-${index}`,
+          })),
+          clientVersion: '0.27.5',
+        },
+      },
+    ];
+
+    for (const { label, payload } of invalidBatches) {
+      const error = getErrorResponse(
+        t,
+        await emitWithAck(socket, 'space:join-batch', payload)
+      );
+      t.is(error.name, 'BAD_REQUEST', label);
+    }
+  } finally {
+    socket.disconnect();
+  }
+});
+
+test('space:join-batch should reject clients before 0.27.5', async t => {
+  const { user, cookieHeader } = await login(app);
+  const socket = createClient(url, cookieHeader);
+
+  try {
+    await waitForConnect(socket);
+    const result = unwrapResponse(
+      t,
+      await emitWithAck<{ clientId: string; success: boolean }>(
+        socket,
+        'space:join-batch',
+        {
+          spaces: [{ spaceType: 'userspace', spaceId: user.id }],
+          clientVersion: '0.27.4',
+        }
+      )
+    );
+    t.false(result.success);
+    await waitForDisconnect(socket);
+  } finally {
+    socket.disconnect();
+  }
+});
+
+test('space:join-batch should authorize once and join all requested rooms', async t => {
+  const models = app.get(Models);
+  const { user: owner, cookieHeader: ownerCookieHeader } = await login(app);
+  const { cookieHeader: deniedCookieHeader } = await login(app);
+  const workspace = await models.workspace.create(owner.id);
+
+  const ownerSocket = createClient(url, ownerCookieHeader);
+  const receiverSocket = createClient(url, ownerCookieHeader);
+  const deniedSocket = createClient(url, deniedCookieHeader);
+
+  try {
+    await Promise.all([
+      waitForConnect(ownerSocket),
+      waitForConnect(receiverSocket),
+      waitForConnect(deniedSocket),
+    ]);
+
+    const batch = {
+      spaces: [
+        { spaceType: 'workspace', spaceId: workspace.id },
+        { spaceType: 'workspace', spaceId: workspace.id, docId: 'doc-a' },
+        { spaceType: 'workspace', spaceId: workspace.id, docId: 'doc-b' },
+      ],
+      clientVersion: '0.27.5',
+    };
+
+    for (const socket of [ownerSocket, receiverSocket]) {
+      const result = unwrapResponse(
+        t,
+        await emitWithAck<{ clientId: string; success: boolean }>(
+          socket,
+          'space:join-batch',
+          batch
+        )
+      );
+      t.true(result.success);
+    }
+
+    const awarenessOnlyResult = unwrapResponse(
+      t,
+      await emitWithAck<{ clientId: string; success: boolean }>(
+        ownerSocket,
+        'space:join-batch',
+        {
+          spaces: [
+            {
+              spaceType: 'workspace',
+              spaceId: workspace.id,
+              docId: 'doc-c',
+            },
+          ],
+          clientVersion: '0.27.5',
+        }
+      )
+    );
+    t.true(awarenessOnlyResult.success);
+
+    const timestamps = unwrapResponse(
+      t,
+      await emitWithAck<Record<string, number>>(
+        ownerSocket,
+        'space:load-doc-timestamps',
+        {
+          spaceType: 'workspace',
+          spaceId: workspace.id,
+        }
+      )
+    );
+    t.deepEqual(timestamps, {});
+
+    const deniedError = getErrorResponse(
+      t,
+      await emitWithAck(deniedSocket, 'space:join-batch', batch)
+    );
+    t.is(deniedError.name, 'SPACE_ACCESS_DENIED');
+
+    const deniedSyncRoomError = getErrorResponse(
+      t,
+      await emitWithAck(deniedSocket, 'space:load-doc-timestamps', {
+        spaceType: 'workspace',
+        spaceId: workspace.id,
+      })
+    );
+    t.is(deniedSyncRoomError.name, 'NOT_IN_SPACE');
+
+    const deniedAwarenessRoomError = getErrorResponse(
+      t,
+      await emitWithAck(deniedSocket, 'space:load-awarenesses', {
+        spaceType: 'workspace',
+        spaceId: workspace.id,
+        docId: 'doc-a',
+      })
+    );
+    t.is(deniedAwarenessRoomError.name, 'NOT_IN_SPACE');
+
+    const receivedA = waitForEvent<{
+      spaceType: string;
+      spaceId: string;
+      docId: string;
+      awarenessUpdate: string;
+    }>(receiverSocket, 'space:broadcast-awareness-update');
+    const noDeniedEvent = expectNoEvent(
+      deniedSocket,
+      'space:broadcast-awareness-update'
+    );
+
+    ownerSocket.emit('space:update-awareness', {
+      spaceType: 'workspace',
+      spaceId: workspace.id,
+      docId: 'doc-a',
+      awarenessUpdate: 'AQID',
+    });
+    const messageA = await receivedA;
+
+    const receivedB = waitForEvent<{
+      spaceType: string;
+      spaceId: string;
+      docId: string;
+      awarenessUpdate: string;
+    }>(receiverSocket, 'space:broadcast-awareness-update');
+    ownerSocket.emit('space:update-awareness', {
+      spaceType: 'workspace',
+      spaceId: workspace.id,
+      docId: 'doc-b',
+      awarenessUpdate: 'BAUG',
+    });
+    const messageB = await receivedB;
+
+    t.deepEqual(
+      new Set([messageA.docId, messageB.docId]),
+      new Set(['doc-a', 'doc-b'])
+    );
+    await noDeniedEvent;
+  } finally {
+    ownerSocket.disconnect();
+    receiverSocket.disconnect();
+    deniedSocket.disconnect();
+  }
+});
+
+test('batch doc entries require Doc.Read atomically', async t => {
+  const db = app.get(PrismaClient);
+  const models = app.get(Models);
+  const { user: owner } = await login(app);
+  const { user: collaborator, cookieHeader } = await login(app);
+  const workspace = await models.workspace.create(owner.id);
+  const docId = 'batch-private-doc';
+
+  await models.workspaceUser.set(
+    workspace.id,
+    collaborator.id,
+    WorkspaceRole.Collaborator,
+    { status: WorkspaceMemberStatus.Accepted }
+  );
+  await models.doc.setDefaultRole(workspace.id, docId, DocRole.None);
+  await createSnapshot(db, {
+    workspaceId: workspace.id,
+    docId,
+    userId: owner.id,
+  });
+
+  const socket = createClient(url, cookieHeader);
+  try {
+    await waitForConnect(socket);
+
+    const error = getErrorResponse(
+      t,
+      await emitWithAck(socket, 'space:join-batch', {
+        spaces: [
+          { spaceType: 'workspace', spaceId: workspace.id },
+          { spaceType: 'workspace', spaceId: workspace.id, docId },
+        ],
+        clientVersion: '0.27.5',
+      })
+    );
+    t.true(error.message.includes('Doc.Read'));
+
+    const timestampsError = getErrorResponse(
+      t,
+      await emitWithAck(socket, 'space:load-doc-timestamps', {
+        spaceType: 'workspace',
+        spaceId: workspace.id,
+      })
+    );
+    t.is(timestampsError.name, 'NOT_IN_SPACE');
+  } finally {
+    socket.disconnect();
+  }
+});
+
+test('batch sync routes active updates and only broadcasts invalidation to control room', async t => {
+  const { user, cookieHeader } = await login(app);
+  const spaceId = user.id;
+  const sender = createClient(url, cookieHeader);
+  const receiver = createClient(url, cookieHeader);
+  const passive = createClient(url, cookieHeader);
+
+  try {
+    await Promise.all([
+      waitForConnect(sender),
+      waitForConnect(receiver),
+      waitForConnect(passive),
+    ]);
+
+    const activeBatch = {
+      spaces: [
+        { spaceType: 'userspace', spaceId },
+        { spaceType: 'userspace', spaceId, docId: 'some-doc' },
+      ],
+      clientVersion: '0.27.5',
+    };
+    for (const socket of [sender, receiver]) {
+      const result = unwrapResponse(
+        t,
+        await emitWithAck<{ clientId: string; success: boolean }>(
+          socket,
+          'space:join-batch',
+          activeBatch
+        )
+      );
+      t.true(result.success);
+    }
+    const passiveJoin = unwrapResponse(
+      t,
+      await emitWithAck<{ clientId: string; success: boolean }>(
+        passive,
+        'space:join-batch',
+        {
+          spaces: [{ spaceType: 'userspace', spaceId }],
+          clientVersion: '0.27.5',
+        }
+      )
+    );
+    t.true(passiveJoin.success);
+
+    const receivedUpdate = waitForEvent<{
+      docId: string;
+      updates: string[];
+    }>(receiver, 'space:broadcast-doc-updates');
+    const receivedInvalidation = waitForEvent<{
+      spaceType: string;
+      spaceId: string;
+      timestamp: number;
+      docId?: string;
+      updates?: string[];
+    }>(receiver, 'space:broadcast-doc-invalidation');
+    const noPassiveUpdate = expectNoEvent(
+      passive,
+      'space:broadcast-doc-updates'
+    );
+
+    unwrapResponse(
+      t,
+      await emitWithAck(sender, 'space:push-doc-update', {
+        spaceType: 'userspace',
+        spaceId,
+        docId: 'some-doc',
+        update: createYjsUpdateBase64(),
+      })
+    );
+
+    const [update, invalidation] = await Promise.all([
+      receivedUpdate,
+      receivedInvalidation,
+    ]);
+    t.is(update.docId, 'some-doc');
+    t.deepEqual(Object.keys(invalidation).sort(), [
+      'spaceId',
+      'spaceType',
+      'timestamp',
+    ]);
+    await noPassiveUpdate;
+
+    const leave = unwrapResponse(
+      t,
+      await emitWithAck<{ clientId: string; success: boolean }>(
+        receiver,
+        'space:leave-batch',
+        {
+          spaceType: 'userspace',
+          spaceId,
+          docIds: ['some-doc'],
+        }
+      )
+    );
+    t.true(leave.success);
+
+    const noLeftUpdate = expectNoEvent(receiver, 'space:broadcast-doc-updates');
+    const receivedAfterLeave = waitForEvent(
+      receiver,
+      'space:broadcast-doc-invalidation'
+    );
+    unwrapResponse(
+      t,
+      await emitWithAck(sender, 'space:push-doc-update', {
+        spaceType: 'userspace',
+        spaceId,
+        docId: 'some-doc',
+        update: createYjsUpdateBase64(),
+      })
+    );
+    await Promise.all([noLeftUpdate, receivedAfterLeave]);
+  } finally {
+    sender.disconnect();
+    receiver.disconnect();
+    passive.disconnect();
+  }
+});
+
+test('permission revocation removes a active document subscription', async t => {
+  const db = app.get(PrismaClient);
+  const models = app.get(Models);
+  const { user: owner, cookieHeader: ownerCookie } = await login(app);
+  const { user: collaborator, cookieHeader: collaboratorCookie } =
+    await login(app);
+  const workspace = await models.workspace.create(owner.id);
+  const docId = 'revoked-doc';
+
+  await models.workspaceUser.set(
+    workspace.id,
+    collaborator.id,
+    WorkspaceRole.Collaborator,
+    { status: WorkspaceMemberStatus.Accepted }
+  );
+  await models.doc.setDefaultRole(workspace.id, docId, DocRole.None);
+  await models.docUser.set(
+    workspace.id,
+    docId,
+    collaborator.id,
+    DocRole.Reader
+  );
+  await createSnapshot(db, {
+    workspaceId: workspace.id,
+    docId,
+    userId: owner.id,
+  });
+
+  const ownerSocket = createClient(url, ownerCookie);
+  const collaboratorSocket = createClient(url, collaboratorCookie);
+  try {
+    await Promise.all([
+      waitForConnect(ownerSocket),
+      waitForConnect(collaboratorSocket),
+    ]);
+
+    for (const socket of [ownerSocket, collaboratorSocket]) {
+      const response = unwrapResponse(
+        t,
+        await emitWithAck<{ clientId: string; success: boolean }>(
+          socket,
+          'space:join-batch',
+          {
+            spaces: [
+              { spaceType: 'workspace', spaceId: workspace.id },
+              { spaceType: 'workspace', spaceId: workspace.id, docId },
+            ],
+            clientVersion: '0.27.5',
+          }
+        )
+      );
+      t.true(response.success);
+    }
+
+    await models.docUser.delete(workspace.id, docId, collaborator.id);
+    await app.get(EventBus).emitAsync('doc.grants.changed', {
+      workspaceId: workspace.id,
+      docId,
+    });
+
+    const noRevokedUpdate = expectNoEvent(
+      collaboratorSocket,
+      'space:broadcast-doc-updates'
+    );
+    unwrapResponse(
+      t,
+      await emitWithAck(ownerSocket, 'space:push-doc-update', {
+        spaceType: 'workspace',
+        spaceId: workspace.id,
+        docId,
+        update: createYjsUpdateBase64(),
+      })
+    );
+    await noRevokedUpdate;
+  } finally {
+    ownerSocket.disconnect();
+    collaboratorSocket.disconnect();
+  }
+});
+
+test('awareness requires Doc.Read but not Doc.Update', async t => {
+  const db = app.get(PrismaClient);
+  const models = app.get(Models);
+  const { user: owner, cookieHeader: ownerCookie } = await login(app);
+  const { user: reader, cookieHeader: readerCookie } = await login(app);
+  const workspace = await models.workspace.create(owner.id);
+  const docId = 'awareness-reader-doc';
+
+  await models.workspaceUser.set(
+    workspace.id,
+    reader.id,
+    WorkspaceRole.Collaborator,
+    { status: WorkspaceMemberStatus.Accepted }
+  );
+  await models.doc.setDefaultRole(workspace.id, docId, DocRole.None);
+  await models.docUser.set(workspace.id, docId, reader.id, DocRole.Reader);
+  await createSnapshot(db, {
+    workspaceId: workspace.id,
+    docId,
+    userId: owner.id,
+  });
+
+  const ownerSocket = createClient(url, ownerCookie);
+  const readerSocket = createClient(url, readerCookie);
+  try {
+    await Promise.all([
+      waitForConnect(ownerSocket),
+      waitForConnect(readerSocket),
+    ]);
+
+    for (const socket of [ownerSocket, readerSocket]) {
+      const response = unwrapResponse(
+        t,
+        await emitWithAck<{ clientId: string; success: boolean }>(
+          socket,
+          'space:join-batch',
+          {
+            spaces: [
+              { spaceType: 'workspace', spaceId: workspace.id },
+              { spaceType: 'workspace', spaceId: workspace.id, docId },
+            ],
+            clientVersion: '0.27.5',
+          }
+        )
+      );
+      t.true(response.success);
+    }
+
+    const receivedAwareness = waitForEvent<{
+      docId: string;
+      awarenessUpdate: string;
+    }>(readerSocket, 'space:broadcast-awareness-update');
+    ownerSocket.emit('space:update-awareness', {
+      spaceType: 'workspace',
+      spaceId: workspace.id,
+      docId,
+      awarenessUpdate: 'AQID',
+    });
+    t.deepEqual(await receivedAwareness, {
+      spaceType: 'workspace',
+      spaceId: workspace.id,
+      docId,
+      awarenessUpdate: 'AQID',
+    });
+
+    const updateError = getErrorResponse(
+      t,
+      await emitWithAck(readerSocket, 'space:push-doc-update', {
+        spaceType: 'workspace',
+        spaceId: workspace.id,
+        docId,
+        update: createYjsUpdateBase64(),
+      })
+    );
+    t.is(updateError.name, 'DOC_ACTION_DENIED');
+  } finally {
+    ownerSocket.disconnect();
+    readerSocket.disconnect();
+  }
+});
+
 test('active users metric should dedupe multiple sockets for one user', async t => {
   const db = app.get(PrismaClient);
   await ensureSyncActiveUsersTable(db);

packages/backend/server/src/__tests__/utils/runtime-config.ts

@@ -8,7 +8,10 @@ const testPrivateKey = privateKey
   .export({ format: 'pem', type: 'pkcs8' })
   .toString();
 
-export async function createTestRuntimeConfig(databaseUrl: string) {
+export async function createTestRuntimeConfig(
+  databaseUrl: string,
+  indexer: AppConfig['indexer']
+) {
   const directory = await mkdtemp(join(tmpdir(), 'affine-server-test-'));
   const storagePath = join(directory, 'storage');
   const storage = (bucket: string) => ({
@@ -30,6 +33,10 @@ export async function createTestRuntimeConfig(databaseUrl: string) {
         enabled: true,
         storage: storage('copilot'),
       },
+      indexer: {
+        enabled: indexer.enabled,
+        provider: indexer.provider,
+      },
     })
   );
   return {

packages/backend/server/src/__tests__/utils/testing-module.ts

@@ -75,8 +75,10 @@ export async function createTestingModule(
   moduleDef: TestingModuleMetadata = {},
   autoInitialize = true
 ): Promise<TestingModule> {
+  const config = new ConfigFactory().config;
   const runtimeConfig = await createTestRuntimeConfig(
-    new ConfigFactory().config.db.datasourceUrl
+    config.db.datasourceUrl,
+    config.indexer
   );
   // setting up
   let imports = moduleDef.imports ?? [buildAppModule(globalThis.env)];

packages/backend/server/src/__tests__/workspace/controller.spec.ts

@@ -38,12 +38,17 @@ test.before(async t => {
 
   const db = app.get(PrismaClient);
 
-  t.context.u1 = await app.signupV1('u1@affine.pro');
   t.context.db = db;
   t.context.app = app;
   t.context.storage = app.get(WorkspaceBlobStorage);
   t.context.workspace = app.get(PgWorkspaceDocStorageAdapter);
   t.context.models = app.get(Models);
+});
+
+test.beforeEach(async t => {
+  const { app, db } = t.context;
+  await app.initTestingDB();
+  t.context.u1 = await app.signupV1('u1@affine.pro');
 
   await db.workspaceDoc.create({
     data: {

packages/backend/server/src/app.module.ts

@@ -28,12 +28,16 @@ import { RedisModule } from './base/redis';
 import { RateLimiterModule } from './base/throttler';
 import { WebSocketModule } from './base/websocket';
 import { AuthModule } from './core/auth';
-import { BackendRuntimeModule } from './core/backend-runtime';
+import {
+  BackendRuntimeModule,
+  BackendRuntimeProducerModule,
+  BackendRuntimeWorkerModule,
+} from './core/backend-runtime';
 import { CommentModule } from './core/comment';
 import { ServerConfigModule, ServerConfigResolverModule } from './core/config';
 import { DocStorageModule } from './core/doc';
+import { DocJobsModule } from './core/doc-jobs';
 import { DocRendererModule } from './core/doc-renderer';
-import { DocServiceModule } from './core/doc-service';
 import { FeatureModule } from './core/features';
 import { MailModule } from './core/mail';
 import { MonitorModule } from './core/monitor';
@@ -44,20 +48,20 @@ import { QuotaModule } from './core/quota';
 import { RealtimeModule } from './core/realtime';
 import { SelfhostModule } from './core/selfhost';
 import { StaticFileModule } from './core/static-files';
-import { StorageModule } from './core/storage';
+import { StorageApiModule, StorageWorkerModule } from './core/storage';
 import { StorageRuntimeModule } from './core/storage-runtime';
 import { SyncModule } from './core/sync';
 import { TelemetryModule } from './core/telemetry';
 import { UserModule } from './core/user';
 import { VersionModule } from './core/version';
 import { WorkspaceModule } from './core/workspaces';
-import { Env } from './env';
+import { Env, ServerRole } from './env';
 import { ModelsModule } from './models';
 import { CalendarModule } from './plugins/calendar';
 import { CaptchaModule } from './plugins/captcha';
 import { CopilotModule } from './plugins/copilot';
 import { GCloudModule } from './plugins/gcloud';
-import { IndexerModule } from './plugins/indexer';
+import { IndexerModule, IndexerWorkerModule } from './plugins/indexer';
 import { LicenseModule } from './plugins/license';
 import { OAuthModule } from './plugins/oauth';
 import { PaymentModule } from './plugins/payment';
@@ -120,6 +124,7 @@ export const FunctionalityModules = [
   RealtimeModule,
   ModelsModule,
   BackendRuntimeModule,
+  BackendRuntimeProducerModule,
   StorageRuntimeModule,
   ScheduleModule.forRoot(),
   MonitorModule,
@@ -157,29 +162,27 @@ export class AppModuleBuilder {
 
 export function buildAppModule(env: Env) {
   const factor = new AppModuleBuilder();
+  const workerOnly = env.role === ServerRole.Worker;
 
   factor
     // basic
     .use(...FunctionalityModules)
 
-    // enable indexer module on graphql, doc and front service
-    .useIf(
-      () => env.flavors.graphql || env.flavors.doc || env.flavors.front,
-      IndexerModule
-    )
+    // online roles publish indexer events; only the worker registers consumers
+    .useIf(() => env.isApi || env.isFrontend, IndexerModule)
+    .useIf(() => env.isWorker, IndexerWorkerModule)
 
-    // auth
-    .use(UserModule, AuthModule, PermissionModule)
+    // the worker owns doc consumers and schedulers
+    .useIf(() => env.isWorker, DocJobsModule)
+    .useIf(() => env.isWorker, BackendRuntimeWorkerModule)
+
+    // auth and business APIs are not part of the queue worker application
+    .useIf(() => !workerOnly, UserModule, AuthModule, PermissionModule)
 
     // business modules
-    .use(
-      ServerConfigModule,
-      FeatureModule,
-      QuotaModule,
-      DocStorageModule,
-      NotificationModule,
-      MailModule
-    )
+    .use(ServerConfigModule, QuotaModule, DocStorageModule)
+    .useIf(() => env.isWorker, StorageWorkerModule)
+    .useIf(() => !workerOnly, FeatureModule, NotificationModule, MailModule)
     // renderer server and front server
     .useIf(() => env.flavors.renderer || env.flavors.front, DocRendererModule)
     // sync server and front server
@@ -197,7 +200,7 @@ export function buildAppModule(env: Env) {
       () => env.flavors.graphql,
       GqlModule,
       VersionModule,
-      StorageModule,
+      StorageApiModule,
       ServerConfigResolverModule,
       WorkspaceModule,
       LicenseModule,
@@ -210,10 +213,12 @@ export function buildAppModule(env: Env) {
       CommentModule,
       QueueDashboardModule
     )
-    // doc service and front service
-    .useIf(() => env.flavors.doc || env.flavors.front, DocServiceModule)
     // worker for and self-hosted API only for self-host and local development only
-    .useIf(() => env.dev || env.selfhosted, WorkerModule, SelfhostModule)
+    .useIf(
+      () => !workerOnly && (env.dev || env.selfhosted),
+      WorkerModule,
+      SelfhostModule
+    )
     // static frontend routes for front flavor
     .useIf(() => env.flavors.front, StaticFileModule)
 

packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts

@@ -102,24 +102,6 @@ test('should be able to safe compare', t => {
   t.false(t.context.crypto.compare('abc', 'def'));
 });
 
-test('should sign and parse internal access token', t => {
-  const token = t.context.crypto.signInternalAccessToken({
-    method: 'GET',
-    path: '/rpc/workspaces/123/docs/456',
-    now: 1700000000000,
-    nonce: 'nonce-123',
-  });
-
-  const payload = t.context.crypto.parseInternalAccessToken(token);
-  t.deepEqual(payload, {
-    v: 1,
-    ts: 1700000000000,
-    nonce: 'nonce-123',
-    m: 'GET',
-    p: '/rpc/workspaces/123/docs/456',
-  });
-});
-
 test('should be able to hash and verify password', async t => {
   const password = 'mySecurePassword';
   const hash = await t.context.crypto.encryptPassword(password);

packages/backend/server/src/base/helpers/crypto.ts

@@ -173,67 +173,6 @@ export class CryptoHelper implements OnModuleInit {
     });
   }
 
-  signInternalAccessToken(input: {
-    method: string;
-    path: string;
-    now?: number;
-    nonce?: string;
-  }) {
-    const payload = {
-      v: 1 as const,
-      ts: input.now ?? Date.now(),
-      nonce: input.nonce ?? this.randomBytes(16).toString('base64url'),
-      m: input.method.toUpperCase(),
-      p: input.path,
-    };
-    const data = Buffer.from(JSON.stringify(payload), 'utf8').toString(
-      'base64url'
-    );
-    return this.sign(data);
-  }
-
-  parseInternalAccessToken(signatureWithData: string): {
-    v: 1;
-    ts: number;
-    nonce: string;
-    m: string;
-    p: string;
-  } | null {
-    const [data, signature] = signatureWithData.split(',');
-    if (!signature) {
-      return null;
-    }
-    if (!this.verify(signatureWithData)) {
-      return null;
-    }
-    try {
-      const json = Buffer.from(data, 'base64url').toString('utf8');
-      const payload = JSON.parse(json) as unknown;
-      if (!payload || typeof payload !== 'object') {
-        return null;
-      }
-      const val = payload as {
-        v?: unknown;
-        ts?: unknown;
-        nonce?: unknown;
-        m?: unknown;
-        p?: unknown;
-      };
-      if (
-        val.v !== 1 ||
-        typeof val.ts !== 'number' ||
-        typeof val.nonce !== 'string' ||
-        typeof val.m !== 'string' ||
-        typeof val.p !== 'string'
-      ) {
-        return null;
-      }
-      return { v: 1, ts: val.ts, nonce: val.nonce, m: val.m, p: val.p };
-    } catch {
-      return null;
-    }
-  }
-
   encrypt(data: string) {
     const iv = this.randomBytes();
     const cipher = createCipheriv(

packages/backend/server/src/base/job/queue/executor.ts

@@ -2,7 +2,7 @@ import { getQueueToken, getSharedConfigToken } from '@nestjs/bullmq';
 import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import { Job, Queue as Bullmq, Worker, WorkerOptions } from 'bullmq';
-import { difference, merge } from 'lodash-es';
+import { merge } from 'lodash-es';
 import { CLS_ID, ClsServiceManager } from 'nestjs-cls';
 
 import { Config } from '../../config';
@@ -10,7 +10,8 @@ import { OnEvent } from '../../event';
 import { metrics, wrapCallMetric } from '../../metrics';
 import { QueueRedis } from '../../redis';
 import { genRequestId } from '../../utils';
-import { JOB_SIGNAL, namespace, Queue, QUEUES } from './def';
+import { JOB_SIGNAL, namespace, Queue } from './def';
+import { queuesForRole } from './owner';
 import { JobHandlerScanner } from './scanner';
 
 @Injectable()
@@ -27,18 +28,7 @@ export class JobExecutor implements OnModuleDestroy {
 
   @OnEvent('config.init')
   async onConfigInit() {
-    const queues = env.flavors.graphql
-      ? difference(QUEUES, [Queue.DOC, Queue.INDEXER])
-      : [];
-
-    // Enable doc/indexer queues in both doc and front service.
-    if (env.flavors.doc || env.flavors.front) {
-      queues.push(Queue.DOC);
-      // NOTE(@fengmk2): Once the index task cannot be processed in time, it needs to be separated from the doc service and deployed independently.
-      queues.push(Queue.INDEXER);
-    }
-
-    await this.startWorkers(queues);
+    await this.startWorkers(queuesForRole(env.role));
   }
 
   @OnEvent('config.changed')

packages/backend/server/src/base/job/queue/index.ts

@@ -55,3 +55,4 @@ export class JobModule {
 
 export { JobQueue };
 export { JOB_SIGNAL, OnJob } from './def';
+export { queuesForRole, WORKER_QUEUES } from './owner';

packages/backend/server/src/base/job/queue/owner.ts

@@ -0,0 +1,24 @@
+import { ServerRole } from '../../../env';
+import { Queue, QUEUES } from './def';
+
+export const WORKER_QUEUES = [
+  Queue.DOC,
+  Queue.INDEXER,
+  Queue.BACKENDRUNTIME,
+] as const;
+
+export function queuesForRole(role: ServerRole | undefined): Queue[] {
+  switch (role) {
+    case ServerRole.AllInOne:
+      return [...QUEUES];
+    case ServerRole.Api:
+      return QUEUES.filter(
+        queue => !(WORKER_QUEUES as readonly Queue[]).includes(queue)
+      );
+    case ServerRole.Worker:
+      return [...WORKER_QUEUES];
+    case ServerRole.Frontend:
+    case undefined:
+      return [];
+  }
+}

packages/backend/server/src/base/throttler/index.ts

@@ -15,13 +15,98 @@ import {
 import type { Request, Response } from 'express';
 
 import { Config } from '../config';
+import { CacheRedis } from '../redis';
 import { getRequestResponseFromContext } from '../utils/request';
 import { getRequestTrackerId } from '../utils/request-tracker';
 import type { ThrottlerType } from './config';
 import { THROTTLER_PROTECTED, Throttlers } from './decorators';
 
+const REDIS_THROTTLE_SCRIPT = `
+local now = redis.call("TIME")
+local nowMs = now[1] * 1000 + math.floor(now[2] / 1000)
+local blockedUntil = tonumber(redis.call("HGET", KEYS[1], "blockedUntil")) or 0
+
+if blockedUntil > nowMs then
+  return {
+    tonumber(redis.call("HGET", KEYS[1], "hits")) or 0,
+    redis.call("PTTL", KEYS[1]),
+    blockedUntil - nowMs
+  }
+end
+
+if blockedUntil > 0 then
+  redis.call("HDEL", KEYS[1], "blockedUntil")
+  redis.call("HSET", KEYS[1], "hits", 0)
+end
+
+local hits = redis.call("HINCRBY", KEYS[1], "hits", 1)
+if hits == 1 then
+  redis.call("PEXPIRE", KEYS[1], ARGV[1])
+end
+
+local blockTtl = 0
+if hits > tonumber(ARGV[2]) then
+  blockedUntil = nowMs + tonumber(ARGV[3])
+  redis.call("HSET", KEYS[1], "blockedUntil", blockedUntil)
+  if redis.call("PTTL", KEYS[1]) < tonumber(ARGV[3]) then
+    redis.call("PEXPIRE", KEYS[1], ARGV[3])
+  end
+  blockTtl = tonumber(ARGV[3])
+end
+
+return { hits, redis.call("PTTL", KEYS[1]), blockTtl }
+`;
+
 @Injectable()
-export class ThrottlerStorage extends ThrottlerStorageService {}
+export class ThrottlerStorage extends ThrottlerStorageService {
+  constructor(private readonly redis: CacheRedis) {
+    super();
+  }
+
+  override async increment(
+    key: string,
+    ttl: number,
+    limit: number,
+    blockDuration: number,
+    throttlerName: string
+  ) {
+    if (env.testing) {
+      return super.increment(key, ttl, limit, blockDuration, throttlerName);
+    }
+
+    try {
+      const result = await this.redis.eval(
+        REDIS_THROTTLE_SCRIPT,
+        1,
+        key,
+        ttl,
+        limit,
+        Math.max(blockDuration, 1)
+      );
+      if (!Array.isArray(result) || result.length !== 3) {
+        throw new Error('Unexpected Redis throttler response');
+      }
+
+      const totalHits = Number(result[0]);
+      const timeToExpire = Math.max(0, Math.ceil(Number(result[1]) / 1000));
+      const timeToBlockExpire = Math.max(
+        0,
+        Math.ceil(Number(result[2]) / 1000)
+      );
+
+      return {
+        totalHits,
+        timeToExpire,
+        isBlocked: timeToBlockExpire > 0,
+        timeToBlockExpire,
+      };
+    } catch {
+      // Preserve availability if Redis is unavailable. The inherited local
+      // storage still protects each process while the shared limiter recovers.
+      return super.increment(key, ttl, limit, blockDuration, throttlerName);
+    }
+  }
+}
 
 @Injectable()
 class CustomOptionsFactory implements ThrottlerOptionsFactory {

packages/backend/server/src/cli.ts

@@ -55,6 +55,17 @@ function buildProgram(logger: Logger) {
       });
     });
 
+  program
+    .command('admit-legacy-context-blobs')
+    .description(
+      'Admit legacy context blobs before the cleanup schema migration'
+    )
+    .action(async () => {
+      await withCliApp(logger, async app => {
+        await app.get(RunCommand).admitLegacyContextBlobs();
+      });
+    });
+
   program
     .command('revert [name]')
     .description('Revert one data migration with given name')

packages/backend/server/src/core/auth/config.ts

@@ -20,6 +20,11 @@ export interface AuthConfig {
   requireEmailVerification: boolean;
   newAccountShareActionDelay: number;
   trustedCloudflareHeaders: boolean;
+  signInRateLimit: ConfigItem<{
+    ttl: number;
+    ipLimit: number;
+    emailLimit: number;
+  }>;
   inviteQuotaShadowMode: boolean;
   inviteQuotaFailOpenOnRuntimeError: boolean;
   passwordRequirements: ConfigItem<{
@@ -61,6 +66,21 @@ defineModuleConfig('auth', {
     default: false,
     shape: z.boolean(),
   },
+  signInRateLimit: {
+    desc: 'Limits for sign-in attempts shared through Redis by source IP and email. ttl is measured in milliseconds.',
+    default: {
+      ttl: 60_000,
+      ipLimit: 20,
+      emailLimit: 5,
+    },
+    shape: z
+      .object({
+        ttl: z.number().int().positive(),
+        ipLimit: z.number().int().positive(),
+        emailLimit: z.number().int().positive(),
+      })
+      .strict(),
+  },
   inviteQuotaShadowMode: {
     desc: 'Whether workspace invite quota should record would-block decisions without rejecting requests or executing abuse actions.',
     default: false,

packages/backend/server/src/core/auth/controller.ts

@@ -118,7 +118,7 @@ export class AuthController {
   ) {
     const credential = SignInBodySchema.parse(body);
     validators.assertValidEmail(credential.email);
-    const canSignIn = await this.auth.canSignIn(credential.email);
+    const canSignIn = await this.auth.canSignIn(credential.email, req);
     if (!canSignIn) {
       throw new ActionForbidden();
     }

packages/backend/server/src/core/auth/guard.ts

@@ -11,12 +11,9 @@ import semver from 'semver';
 import { Socket } from 'socket.io';
 
 import {
-  AccessDenied,
   AuthenticationRequired,
-  Cache,
   checkCanaryDateClientVersion,
   Config,
-  CryptoHelper,
   getClientVersionFromRequest,
   getRequestResponseFromContext,
   parseCookies,
@@ -32,9 +29,6 @@ import { AuthSessionHttpError } from './session-exchange';
 import { isLikelyJwt } from './token';
 
 const PUBLIC_ENTRYPOINT_SYMBOL = Symbol('public');
-const INTERNAL_ENTRYPOINT_SYMBOL = Symbol('internal');
-const INTERNAL_ACCESS_TOKEN_TTL_MS = 5 * 60 * 1000;
-const INTERNAL_ACCESS_TOKEN_CLOCK_SKEW_MS = 30 * 1000;
 
 type AuthenticatedRequestSession =
   | { type: 'jwt'; session: Session }
@@ -50,8 +44,6 @@ export class AuthGuard implements CanActivate, OnModuleInit {
   private static readonly CANARY_REQUIRED_VERSION = 'canary (within 2 months)';
 
   constructor(
-    private readonly crypto: CryptoHelper,
-    private readonly cache: Cache,
     private readonly config: Config,
     private readonly ref: ModuleRef,
     private readonly reflector: Reflector
@@ -67,38 +59,6 @@ export class AuthGuard implements CanActivate, OnModuleInit {
     const { req, res } = getRequestResponseFromContext(context);
     const clazz = context.getClass();
     const handler = context.getHandler();
-    // rpc request is internal
-    const isInternal = this.reflector.getAllAndOverride<boolean>(
-      INTERNAL_ENTRYPOINT_SYMBOL,
-      [clazz, handler]
-    );
-    if (isInternal) {
-      const accessToken = req.get('x-access-token');
-      if (accessToken) {
-        const payload = this.crypto.parseInternalAccessToken(accessToken);
-        if (payload) {
-          const now = Date.now();
-          const method = req.method.toUpperCase();
-          const path = req.path;
-
-          const timestampInRange =
-            payload.ts <= now + INTERNAL_ACCESS_TOKEN_CLOCK_SKEW_MS &&
-            now - payload.ts <= INTERNAL_ACCESS_TOKEN_TTL_MS;
-
-          if (timestampInRange && payload.m === method && payload.p === path) {
-            const nonceKey = `rpc:nonce:${payload.nonce}`;
-            const ok = await this.cache.setnx(nonceKey, 1, {
-              ttl: INTERNAL_ACCESS_TOKEN_TTL_MS,
-            });
-            if (ok) {
-              return true;
-            }
-          }
-        }
-      }
-      throw new AccessDenied('Invalid internal request');
-    }
-
     // api is public
     const isPublic = this.reflector.getAllAndOverride<boolean>(
       PUBLIC_ENTRYPOINT_SYMBOL,
@@ -327,11 +287,6 @@ export class AuthGuard implements CanActivate, OnModuleInit {
  */
 export const Public = () => SetMetadata(PUBLIC_ENTRYPOINT_SYMBOL, true);
 
-/**
- * Mark rpc api to be internal accessible
- */
-export const Internal = () => SetMetadata(INTERNAL_ENTRYPOINT_SYMBOL, true);
-
 export const AuthWebsocketOptionsProvider: FactoryProvider = {
   provide: WEBSOCKET_OPTIONS,
   useFactory: (config: Config, guard: AuthGuard) => {

packages/backend/server/src/core/auth/service.ts

@@ -1,11 +1,18 @@
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 
 import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
 import { Transactional } from '@nestjs-cls/transactional';
 import type { CookieOptions, Request, Response } from 'express';
 import { assign, pick } from 'lodash-es';
 
-import { Config, OnEvent, SignUpForbidden } from '../../base';
+import {
+  Cache,
+  Config,
+  getRequestClientIp,
+  OnEvent,
+  SignUpForbidden,
+  TooManyRequest,
+} from '../../base';
 import { Models, type User, type UserSession } from '../../models';
 import { EntitlementService } from '../entitlement';
 import { Mailer } from '../mail/mailer';
@@ -46,7 +53,8 @@ export class AuthService implements OnApplicationBootstrap {
     private readonly models: Models,
     private readonly mailer: Mailer,
     private readonly authSessions: AuthSessionService,
-    private readonly entitlement: EntitlementService
+    private readonly entitlement: EntitlementService,
+    private readonly cache: Cache
   ) {
     this.cookieOptions = {
       sameSite: 'lax',
@@ -69,11 +77,38 @@ export class AuthService implements OnApplicationBootstrap {
     }
   }
 
-  async canSignIn(_email: string) {
+  async canSignIn(email: string, req: Request) {
+    if (!env.testing) {
+      const { ttl, ipLimit, emailLimit } = this.config.auth.signInRateLimit;
+      const normalizedEmail = email.toLowerCase();
+      const ip = getRequestClientIp(req);
+
+      const emailAttempts = this.cache.increaseWithTtl(
+        this.signInRateLimitKey('email', normalizedEmail),
+        ttl
+      );
+      const ipAttempts = ip
+        ? this.cache.increaseWithTtl(this.signInRateLimitKey('ip', ip), ttl)
+        : Promise.resolve(0);
+      const [emailCount, ipCount] = await Promise.all([
+        emailAttempts,
+        ipAttempts,
+      ]);
+
+      if (emailCount > emailLimit || ipCount > ipLimit) {
+        throw new TooManyRequest();
+      }
+    }
+
     // may add more sign-in check later
     return true;
   }
 
+  private signInRateLimitKey(scope: 'email' | 'ip', value: string) {
+    const digest = createHash('sha256').update(value).digest('hex');
+    return `auth:sign-in-rate:${scope}:${digest}`;
+  }
+
   /**
    * @deprecated
    *

packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts

@@ -16,15 +16,24 @@ import {
   CopilotSelectedSourcesUnavailable,
 } from '../../../base';
 import { Models } from '../../../models';
-import { BackendRuntimeModule, BackendRuntimeProvider } from '../index';
+import {
+  BackendRuntimeModule,
+  BackendRuntimeProducerModule,
+  BackendRuntimeProvider,
+  BackendRuntimeWorkerModule,
+} from '../index';
 import {
   BackendRuntimeEmbeddingJob,
+  BackendRuntimeEmbeddingProducer,
+  BackendRuntimeEmbeddingService,
   BackendRuntimeHousekeepingJob,
 } from '../job';
 
 interface Context {
   module: TestingModule;
   embeddingJob: BackendRuntimeEmbeddingJob;
+  embeddingProducer: BackendRuntimeEmbeddingProducer;
+  embeddingService: BackendRuntimeEmbeddingService;
   job: BackendRuntimeHousekeepingJob;
   getSnapshot: Sinon.SinonStub;
   allowEmbedding: Sinon.SinonStub;
@@ -55,7 +64,12 @@ test.before(async t => {
     syncEmbeddingState: Sinon.stub(),
   };
   t.context.module = await createTestingModule({
-    imports: [ScheduleModule.forRoot(), BackendRuntimeModule],
+    imports: [
+      ScheduleModule.forRoot(),
+      BackendRuntimeModule,
+      BackendRuntimeProducerModule,
+      BackendRuntimeWorkerModule,
+    ],
     tapModule: builder => {
       builder
         .overrideProvider(BackendRuntimeProvider)
@@ -81,6 +95,12 @@ test.before(async t => {
     'allowEmbedding'
   ).resolves(true);
   t.context.embeddingJob = t.context.module.get(BackendRuntimeEmbeddingJob);
+  t.context.embeddingProducer = t.context.module.get(
+    BackendRuntimeEmbeddingProducer
+  );
+  t.context.embeddingService = t.context.module.get(
+    BackendRuntimeEmbeddingService
+  );
   t.context.job = t.context.module.get(BackendRuntimeHousekeepingJob);
 });
 
@@ -102,7 +122,7 @@ test.after.always(async t => {
 });
 
 test('backend-runtime jobs ingest documents and clean runtime state', async t => {
-  await t.context.embeddingJob.onDocSnapshotUpdated({
+  await t.context.embeddingProducer.onDocSnapshotUpdated({
     workspaceId: 'workspace-1',
     docId: 'doc-1',
     blob: Buffer.alloc(0),
@@ -130,7 +150,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
   const documentJobCount = t.context.module.queue.count(
     'backendRuntime.syncDocumentEmbedding'
   );
-  await t.context.embeddingJob.onDocSnapshotUpdated({
+  await t.context.embeddingProducer.onDocSnapshotUpdated({
     workspaceId: 'workspace-1',
     docId: 'db$docProperties',
     blob: Buffer.alloc(0),
@@ -140,7 +160,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
     documentJobCount
   );
 
-  await t.context.embeddingJob.onDocSnapshotUpdated({
+  await t.context.embeddingProducer.onDocSnapshotUpdated({
     workspaceId: 'workspace-1',
     docId: 'workspace-1',
     blob: Buffer.alloc(0),
@@ -155,7 +175,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
     reconcileDocuments: true,
   });
 
-  await t.context.embeddingJob.prepareSelectedDocuments('workspace-1', [
+  await t.context.embeddingService.prepareSelectedDocuments('workspace-1', [
     'doc-1',
     'doc-1',
   ]);
@@ -181,7 +201,9 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
   ] as const) {
     t.context.runtime.syncEmbeddingState.rejects(new Error(nativeError));
     const error = await t.throwsAsync(() =>
-      t.context.embeddingJob.prepareSelectedDocuments('workspace-1', ['doc-1'])
+      t.context.embeddingService.prepareSelectedDocuments('workspace-1', [
+        'doc-1',
+      ])
     );
     t.true(error instanceof expectedError);
   }
@@ -189,7 +211,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
 
   await t.throwsAsync(
     () =>
-      t.context.embeddingJob.prepareSelectedDocuments(
+      t.context.embeddingService.prepareSelectedDocuments(
         'workspace-1',
         Array.from({ length: 65 }, (_, index) => `doc-${index}`)
       ),
@@ -198,7 +220,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t =>
   t.context.getSnapshot.resolves(null);
   await t.throwsAsync(
     () =>
-      t.context.embeddingJob.prepareSelectedDocuments('workspace-1', [
+      t.context.embeddingService.prepareSelectedDocuments('workspace-1', [
         'missing-doc',
       ]),
     { instanceOf: CopilotSelectedSourcesUnavailable }

packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts

@@ -11,7 +11,7 @@ const privateKey = generateKeyPairSync('ec', {
 }).privateKey.export({ format: 'pem', type: 'pkcs8' }) as string;
 const config = { crypto: { privateKey } } as Config;
 
-test('backend-runtime provider starts once, runs migrations once, and reports health', async t => {
+test('backend-runtime provider starts without migrations and exposes explicit migration', async t => {
   const provider = new BackendRuntimeProvider(config);
   const runtime = {
     start: Sinon.stub().resolves(),
@@ -27,6 +27,7 @@ test('backend-runtime provider starts once, runs migrations once, and reports he
 
   await provider.start();
   await provider.start();
+  await provider.runMigrations();
   await provider.onConfigChanged({ updates: { mailer: {} } });
   await provider.onConfigChanged({ updates: { copilot: {} } });
   await provider.onConfigChanged({ updates: { storages: {} } });
@@ -66,6 +67,106 @@ test('backend-runtime provider measures explicit typed methods', async t => {
   t.true(runtime.assertCopilotRoute.calledOnceWithExactly(routeInput));
 });
 
+test('backend-runtime provider encodes recursive search contracts at the native boundary', async t => {
+  const provider = new BackendRuntimeProvider(config);
+  const runtime = {
+    searchAuthorized: Sinon.stub().resolves({
+      ok: true,
+      value: { total: 0, nodes: [] },
+    }),
+    aggregateAuthorized: Sinon.stub().resolves({
+      ok: true,
+      value: { total: 0, buckets: [] },
+    }),
+  };
+  (provider as unknown as { runtime: typeof runtime }).runtime = runtime;
+  const query = {
+    type: 'boolean',
+    occur: 'must',
+    queries: [
+      { type: 'exists', field: 'refDocId' },
+      {
+        type: 'boost',
+        boost: 1.5,
+        query: { type: 'match', field: 'content', match: 'hello' },
+      },
+    ],
+  };
+
+  await provider.searchAuthorized('actor', 'workspace', {
+    table: 'block',
+    query,
+    options: {
+      fields: ['docId'],
+      highlights: [{ field: 'content', before: '<b>', end: '</b>' }],
+      pagination: { limit: 10, cursor: 'cursor' },
+    },
+  });
+  await provider.aggregateAuthorized('actor', 'workspace', {
+    table: 'block',
+    query,
+    field: 'docId',
+    options: {
+      hits: { fields: ['content'] },
+      pagination: { limit: 5, skip: 2 },
+    },
+  });
+
+  const search = runtime.searchAuthorized.firstCall.args[2];
+  t.is(search.rootQuery, 0);
+  t.deepEqual(search.queries, [
+    {
+      queryType: 'boolean',
+      field: undefined,
+      matchValue: undefined,
+      query: undefined,
+      queries: [1, 2],
+      occur: 'must',
+      boost: undefined,
+    },
+    {
+      queryType: 'exists',
+      field: 'refDocId',
+      matchValue: undefined,
+      query: undefined,
+      queries: undefined,
+      occur: undefined,
+      boost: undefined,
+    },
+    {
+      queryType: 'boost',
+      field: undefined,
+      matchValue: undefined,
+      query: 3,
+      queries: undefined,
+      occur: undefined,
+      boost: 1.5,
+    },
+    {
+      queryType: 'match',
+      field: 'content',
+      matchValue: 'hello',
+      query: undefined,
+      queries: undefined,
+      occur: undefined,
+      boost: undefined,
+    },
+  ]);
+  t.deepEqual(search.options, {
+    fields: ['docId'],
+    highlights: [{ field: 'content', before: '<b>', end: '</b>' }],
+    pagination: { limit: 10, cursor: 'cursor' },
+  });
+  t.deepEqual(runtime.aggregateAuthorized.firstCall.args[2].options, {
+    hits: { fields: ['content'], highlights: [], pagination: {} },
+    pagination: { limit: 5, skip: 2 },
+  });
+  t.true(runtime.searchAuthorized.calledOnce);
+  t.true(runtime.searchAuthorized.calledWithMatch('actor', 'workspace'));
+  t.true(runtime.aggregateAuthorized.calledOnce);
+  t.true(runtime.aggregateAuthorized.calledWithMatch('actor', 'workspace'));
+});
+
 test('backend-runtime provider aborts a stream handle that resolves after iterator cancellation', async t => {
   const provider = new BackendRuntimeProvider(config);
   const abort = Sinon.stub();

packages/backend/server/src/core/backend-runtime/index.ts

@@ -2,6 +2,8 @@ import { Global, Module } from '@nestjs/common';
 
 import {
   BackendRuntimeEmbeddingJob,
+  BackendRuntimeEmbeddingProducer,
+  BackendRuntimeEmbeddingService,
   BackendRuntimeHousekeepingJob,
 } from './job';
 import {
@@ -17,14 +19,30 @@ import {
       useValue: undefined,
     },
     BackendRuntimeProvider,
-    BackendRuntimeEmbeddingJob,
-    BackendRuntimeHousekeepingJob,
+    BackendRuntimeEmbeddingService,
   ],
-  exports: [BackendRuntimeProvider, BackendRuntimeEmbeddingJob],
+  exports: [BackendRuntimeProvider, BackendRuntimeEmbeddingService],
 })
 export class BackendRuntimeModule {}
 
-export { BackendRuntimeEmbeddingJob } from './job';
+@Module({
+  imports: [BackendRuntimeModule],
+  providers: [BackendRuntimeEmbeddingProducer],
+})
+export class BackendRuntimeProducerModule {}
+
+@Module({
+  imports: [BackendRuntimeModule],
+  providers: [BackendRuntimeEmbeddingJob, BackendRuntimeHousekeepingJob],
+})
+export class BackendRuntimeWorkerModule {}
+
+export {
+  BackendRuntimeEmbeddingJob,
+  BackendRuntimeEmbeddingProducer,
+  BackendRuntimeEmbeddingService,
+  BackendRuntimeHousekeepingJob,
+} from './job';
 export {
   BACKEND_RUNTIME_CONFIG_PATHS,
   BackendRuntimeProvider,

packages/backend/server/src/core/backend-runtime/job.ts

@@ -22,7 +22,7 @@ const SELECTED_DOCUMENT_WAIT_MS = 90_000;
 
 declare global {
   interface Jobs {
-    'nightly.cleanExpiredBackendRuntimeHousekeeping': {};
+    'backendRuntime.cleanExpiredHousekeeping': {};
     'backendRuntime.syncDocumentEmbedding': {
       workspaceId: string;
       docId: string;
@@ -34,54 +34,13 @@ declare global {
 }
 
 @Injectable()
-export class BackendRuntimeEmbeddingJob {
+export class BackendRuntimeEmbeddingService {
   constructor(
     private readonly rt: BackendRuntimeProvider,
-    private readonly queue: JobQueue,
     private readonly models: Models
   ) {}
 
-  @OnEvent('doc.updated')
-  async onDocUpdated({ workspaceId, docId }: Events['doc.updated']) {
-    await this.queueDocument(workspaceId, docId);
-  }
-
-  @OnEvent('doc.snapshot.updated')
-  async onDocSnapshotUpdated({
-    workspaceId,
-    docId,
-  }: Events['doc.snapshot.updated']) {
-    if (workspaceId === docId) {
-      await this.queue.add(
-        'backendRuntime.reconcileDocumentEmbeddings',
-        { workspaceId },
-        { jobId: `reconcileDocumentEmbeddings/${workspaceId}` }
-      );
-      return;
-    }
-    await this.queueDocument(workspaceId, docId);
-  }
-
-  private async queueDocument(workspaceId: string, docId: string) {
-    if (
-      workspaceId === docId ||
-      docId.startsWith('db$') ||
-      docId.startsWith('userdata$')
-    ) {
-      return;
-    }
-    await this.queue.add(
-      'backendRuntime.syncDocumentEmbedding',
-      { workspaceId, docId },
-      { jobId: `syncDocumentEmbedding/${workspaceId}/${docId}` }
-    );
-  }
-
-  @OnJob('backendRuntime.syncDocumentEmbedding')
-  async syncDocument({
-    workspaceId,
-    docId,
-  }: Jobs['backendRuntime.syncDocumentEmbedding']) {
+  async syncDocument(workspaceId: string, docId: string) {
     await this.syncDocuments(workspaceId, [docId], true);
   }
 
@@ -174,10 +133,7 @@ export class BackendRuntimeEmbeddingJob {
     });
   }
 
-  @OnJob('backendRuntime.reconcileDocumentEmbeddings')
-  async reconcileDocuments({
-    workspaceId,
-  }: Jobs['backendRuntime.reconcileDocumentEmbeddings']) {
+  async reconcileDocuments(workspaceId: string) {
     if (!(await this.rt.embeddingHealth()).enabled) return;
     await this.rt.syncEmbeddingState({
       workspaceId,
@@ -187,6 +143,67 @@ export class BackendRuntimeEmbeddingJob {
   }
 }
 
+@Injectable()
+export class BackendRuntimeEmbeddingProducer {
+  constructor(private readonly queue: JobQueue) {}
+
+  @OnEvent('doc.updated')
+  async onDocUpdated({ workspaceId, docId }: Events['doc.updated']) {
+    await this.queueDocument(workspaceId, docId);
+  }
+
+  @OnEvent('doc.snapshot.updated')
+  async onDocSnapshotUpdated({
+    workspaceId,
+    docId,
+  }: Events['doc.snapshot.updated']) {
+    if (workspaceId === docId) {
+      await this.queue.add(
+        'backendRuntime.reconcileDocumentEmbeddings',
+        { workspaceId },
+        { jobId: `reconcileDocumentEmbeddings/${workspaceId}` }
+      );
+      return;
+    }
+    await this.queueDocument(workspaceId, docId);
+  }
+
+  private async queueDocument(workspaceId: string, docId: string) {
+    if (
+      workspaceId === docId ||
+      docId.startsWith('db$') ||
+      docId.startsWith('userdata$')
+    ) {
+      return;
+    }
+    await this.queue.add(
+      'backendRuntime.syncDocumentEmbedding',
+      { workspaceId, docId },
+      { jobId: `syncDocumentEmbedding/${workspaceId}/${docId}` }
+    );
+  }
+}
+
+@Injectable()
+export class BackendRuntimeEmbeddingJob {
+  constructor(private readonly service: BackendRuntimeEmbeddingService) {}
+
+  @OnJob('backendRuntime.syncDocumentEmbedding')
+  async syncDocument({
+    workspaceId,
+    docId,
+  }: Jobs['backendRuntime.syncDocumentEmbedding']) {
+    await this.service.syncDocument(workspaceId, docId);
+  }
+
+  @OnJob('backendRuntime.reconcileDocumentEmbeddings')
+  async reconcileDocuments({
+    workspaceId,
+  }: Jobs['backendRuntime.reconcileDocumentEmbeddings']) {
+    await this.service.reconcileDocuments(workspaceId);
+  }
+}
+
 @Injectable()
 export class BackendRuntimeHousekeepingJob {
   private readonly logger = new Logger(BackendRuntimeHousekeepingJob.name);
@@ -199,7 +216,7 @@ export class BackendRuntimeHousekeepingJob {
   @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
   async nightlyJob() {
     await this.queue.add(
-      'nightly.cleanExpiredBackendRuntimeHousekeeping',
+      'backendRuntime.cleanExpiredHousekeeping',
       {},
       {
         jobId: 'nightly-backend-runtime-housekeeping',
@@ -207,7 +224,7 @@ export class BackendRuntimeHousekeepingJob {
     );
   }
 
-  @OnJob('nightly.cleanExpiredBackendRuntimeHousekeeping')
+  @OnJob('backendRuntime.cleanExpiredHousekeeping')
   async cleanExpiredRuntimeHousekeeping() {
     const states = await this.cleanBatches(() =>
       this.rt.cleanupExpiredRuntimeStates(1000)

packages/backend/server/src/core/backend-runtime/provider.ts

@@ -35,6 +35,12 @@ import {
   type RuntimeWorkspaceArtifact,
   type SyncEmbeddingStateInput,
 } from '../../native';
+import {
+  type AggregateRequestInput,
+  encodeAggregateRequest,
+  encodeSearchRequest,
+  type SearchRequestInput,
+} from './search';
 
 type RuntimeInstance = InstanceType<typeof BackendRuntime>;
 
@@ -299,11 +305,18 @@ export class BackendRuntimeProvider
 
   async start() {
     await this.runtime.start();
-    await this.runMigrationsOnce();
     const health = await this.runtime.health();
     this.logger.log(`backend runtime started: db=${health.databaseConnected}`);
   }
 
+  /**
+   * Schema changes belong to the explicit predeploy path. Runtime startup only
+   * connects services and must not mutate the database schema.
+   */
+  async runMigrations() {
+    await this.runMigrationsOnce();
+  }
+
   async stop() {
     await this.runtime.stop();
     this.logger.log('backend runtime stopped');
@@ -315,6 +328,7 @@ export class BackendRuntimeProvider
       !updates.copilot &&
       !updates.crypto &&
       !updates.db &&
+      !updates.indexer &&
       !updates.storages
     ) {
       return;
@@ -332,6 +346,74 @@ export class BackendRuntimeProvider
     );
   }
 
+  async searchAuthorized(
+    actorUserId: string,
+    workspaceId: string,
+    request: SearchRequestInput
+  ) {
+    return await this.measured('searchAuthorized', runtime =>
+      runtime.searchAuthorized(
+        actorUserId,
+        workspaceId,
+        encodeSearchRequest(request)
+      )
+    );
+  }
+
+  async aggregateAuthorized(
+    actorUserId: string,
+    workspaceId: string,
+    request: AggregateRequestInput
+  ) {
+    return await this.measured('aggregateAuthorized', runtime =>
+      runtime.aggregateAuthorized(
+        actorUserId,
+        workspaceId,
+        encodeAggregateRequest(request)
+      )
+    );
+  }
+
+  async indexSearchDocument(workspaceId: string, docId: string) {
+    await this.measured('indexSearchDocument', runtime =>
+      runtime.indexSearchDocument(workspaceId, docId)
+    );
+  }
+
+  async deleteSearchDocument(workspaceId: string, docId: string) {
+    await this.measured('deleteSearchDocument', runtime =>
+      runtime.deleteSearchDocument(workspaceId, docId)
+    );
+  }
+
+  async reconcileSearchWorkspace(workspaceId: string) {
+    await this.measured('reconcileSearchWorkspace', runtime =>
+      runtime.reconcileSearchWorkspace(workspaceId)
+    );
+  }
+
+  async deleteSearchWorkspace(workspaceId: string) {
+    await this.measured('deleteSearchWorkspace', runtime =>
+      runtime.deleteSearchWorkspace(workspaceId)
+    );
+  }
+
+  async filterReadableDocs(
+    actorUserId: string,
+    workspaceId: string,
+    docIds: string[]
+  ) {
+    return await this.measured('filterReadableDocs', runtime =>
+      runtime.filterReadableDocs(actorUserId, workspaceId, docIds)
+    );
+  }
+
+  async searchStatus() {
+    return await this.measured('searchStatus', runtime =>
+      runtime.searchStatus()
+    );
+  }
+
   async embeddingQueueCounts() {
     return await this.measured('embeddingQueueCounts', runtime =>
       runtime.embeddingQueueCounts()

packages/backend/server/src/core/backend-runtime/search.ts

@@ -0,0 +1,104 @@
+import type {
+  RuntimeAggregateRequest,
+  RuntimeSearchQuery,
+  RuntimeSearchRequest,
+} from '../../native';
+
+type SearchQueryInput = {
+  type: string;
+  field?: string;
+  match?: string;
+  query?: SearchQueryInput;
+  queries?: SearchQueryInput[];
+  occur?: string;
+  boost?: number;
+};
+
+type SearchPaginationInput = {
+  limit?: number;
+  skip?: number;
+  cursor?: string;
+};
+
+type SearchHighlightInput = {
+  field: string;
+  before: string;
+  end: string;
+};
+
+type SearchOptionsInput = {
+  fields: string[];
+  highlights?: SearchHighlightInput[];
+  pagination?: SearchPaginationInput;
+};
+
+export type SearchRequestInput = {
+  table: 'doc' | 'block';
+  query: SearchQueryInput;
+  options: SearchOptionsInput;
+};
+
+export type AggregateRequestInput = {
+  table: 'doc' | 'block';
+  query: SearchQueryInput;
+  field: string;
+  options: {
+    hits: SearchOptionsInput;
+    pagination?: SearchPaginationInput;
+  };
+};
+
+export function encodeSearchRequest(
+  request: SearchRequestInput
+): RuntimeSearchRequest {
+  const { queries, rootQuery } = encodeQuery(request.query);
+  return {
+    table: request.table,
+    queries,
+    rootQuery,
+    options: encodeOptions(request.options),
+  };
+}
+
+export function encodeAggregateRequest(
+  request: AggregateRequestInput
+): RuntimeAggregateRequest {
+  const { queries, rootQuery } = encodeQuery(request.query);
+  return {
+    table: request.table,
+    queries,
+    rootQuery,
+    field: request.field,
+    options: {
+      hits: encodeOptions(request.options.hits),
+      pagination: request.options.pagination ?? {},
+    },
+  };
+}
+
+function encodeOptions(options: SearchOptionsInput) {
+  return {
+    fields: options.fields,
+    highlights: options.highlights ?? [],
+    pagination: options.pagination ?? {},
+  };
+}
+
+function encodeQuery(root: SearchQueryInput) {
+  const nodes: RuntimeSearchQuery[] = [];
+  const visit = (query: SearchQueryInput): number => {
+    const index = nodes.length;
+    nodes.push({ queryType: query.type });
+    nodes[index] = {
+      queryType: query.type,
+      field: query.field,
+      matchValue: query.match,
+      query: query.query ? visit(query.query) : undefined,
+      queries: query.queries?.map(visit),
+      occur: query.occur,
+      boost: query.boost,
+    };
+    return index;
+  };
+  return { queries: nodes, rootQuery: visit(root) };
+}

packages/backend/server/src/core/config/service.ts

@@ -73,6 +73,25 @@ export class ServerService implements OnApplicationBootstrap {
     user: string,
     updates: Array<{ module: string; key: string; value: any }>
   ): Promise<DeepPartial<AppConfig>> {
+    const providerType = updates.find(
+      update => update.module === 'indexer' && update.key === 'provider.type'
+    );
+    if (providerType?.value === 'embedded') {
+      updates = updates.filter(update => update !== providerType);
+      updates = [
+        ...updates.filter(
+          update => !(update.module === 'indexer' && update.key === 'enabled')
+        ),
+        { module: 'indexer', key: 'enabled', value: false },
+      ];
+    } else if (providerType) {
+      updates = [
+        ...updates.filter(
+          update => !(update.module === 'indexer' && update.key === 'enabled')
+        ),
+        { module: 'indexer', key: 'enabled', value: true },
+      ];
+    }
     const errors = this.validateConfig(updates);
 
     if (errors?.length) {

packages/backend/server/src/core/doc-jobs/index.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+
+import { DocStorageModule, DocStorageWorkerModule } from '../doc';
+import { DocJobConsumer, DocJobScheduler } from './job';
+
+@Module({
+  imports: [DocStorageModule, DocStorageWorkerModule],
+  providers: [DocJobConsumer, DocJobScheduler],
+})
+export class DocJobsModule {}

packages/backend/server/src/core/doc-jobs/job.ts

@@ -24,8 +24,8 @@ declare global {
 }
 
 @Injectable()
-export class DocServiceCronJob {
-  private readonly logger = new Logger(DocServiceCronJob.name);
+export class DocJobConsumer {
+  private readonly logger = new Logger(DocJobConsumer.name);
 
   constructor(
     private readonly workspace: PgWorkspaceDocStorageAdapter,
@@ -40,7 +40,27 @@ export class DocServiceCronJob {
     workspaceId,
     docId,
   }: Jobs['doc.mergePendingDocUpdates']) {
-    await this.workspace.getDoc(workspaceId, docId);
+    const doc = await this.workspace.getDoc(workspaceId, docId);
+    if (doc) {
+      const snapshot = await this.models.doc.getSnapshot(workspaceId, docId, {
+        select: { updatedAt: true },
+      });
+      if (!snapshot) {
+        return JOB_SIGNAL.Done;
+      }
+      await this.job.add(
+        'backendRuntime.projectWorkspaceDocBlobRefs',
+        {
+          workspaceId,
+          docId,
+          sourceRevision: snapshot.updatedAt.getTime(),
+        },
+        {
+          jobId: `doc:blob-ref-projection:${workspaceId}:${docId}:${snapshot.updatedAt.getTime()}`,
+          priority: 100,
+        }
+      );
+    }
     const updatesLeft = await this.models.doc.getUpdateCount(
       workspaceId,
       docId
@@ -49,67 +69,12 @@ export class DocServiceCronJob {
     return updatesLeft > 100 ? JOB_SIGNAL.Repeat : JOB_SIGNAL.Done;
   }
 
-  @Cron(CronExpression.EVERY_30_SECONDS)
-  async schedule() {
-    const group = await this.models.doc.groupedUpdatesCount();
-
-    for (const update of group) {
-      const jobId = `doc:merge-pending-updates:${update.workspaceId}:${update.id}`;
-
-      const job = await this.job.get(jobId, 'doc.mergePendingDocUpdates');
-
-      if (job && job.opts.priority !== 0 && update._count > 100) {
-        // reschedule long pending doc with highest priority, 0 is the highest priority
-        await this.job.remove(jobId, 'doc.mergePendingDocUpdates');
-      }
-
-      await this.job.add(
-        'doc.mergePendingDocUpdates',
-        {
-          workspaceId: update.workspaceId,
-          docId: update.id,
-        },
-        {
-          jobId: `doc:merge-pending-updates:${update.workspaceId}:${update.id}`,
-          priority: update._count > 100 ? 0 : 100,
-          delay: 0,
-        }
-      );
-    }
-  }
-
   @OnJob('doc.recordPendingDocUpdatesCount')
   async recordPendingDocUpdatesCount() {
     const count = await this.prisma.update.count();
     metrics.doc.gauge('pending_updates').record(count);
   }
 
-  @Cron(CronExpression.EVERY_30_SECONDS)
-  async scheduleRecordPendingDocUpdatesCount() {
-    await this.job.add(
-      'doc.recordPendingDocUpdatesCount',
-      {},
-      {
-        // make sure only one job is running at a time
-        delay: 30 * 1000,
-        jobId: 'doc:record-pending-updates-count',
-      }
-    );
-  }
-
-  @Cron(CronExpression.EVERY_30_SECONDS)
-  async scheduleFindEmptySummaryDocs() {
-    await this.job.add(
-      'doc.findEmptySummaryDocs',
-      {},
-      {
-        // make sure only one job is running at a time
-        delay: 30 * 1000,
-        jobId: 'findEmptySummaryDocs',
-      }
-    );
-  }
-
   @OnJob('doc.findEmptySummaryDocs')
   async findEmptySummaryDocs(payload: Jobs['doc.findEmptySummaryDocs']) {
     const startSid = payload.lastFixedWorkspaceSid ?? 0;
@@ -167,3 +132,62 @@ export class DocServiceCronJob {
     return;
   }
 }
+
+@Injectable()
+export class DocJobScheduler {
+  constructor(
+    private readonly job: JobQueue,
+    private readonly models: Models
+  ) {}
+
+  @Cron(CronExpression.EVERY_30_SECONDS)
+  async schedule() {
+    const group = await this.models.doc.groupedUpdatesCount();
+
+    for (const update of group) {
+      const jobId = `doc:merge-pending-updates:${update.workspaceId}:${update.id}`;
+      const job = await this.job.get(jobId, 'doc.mergePendingDocUpdates');
+
+      if (job && job.opts.priority !== 0 && update._count > 100) {
+        await this.job.remove(jobId, 'doc.mergePendingDocUpdates');
+      }
+
+      await this.job.add(
+        'doc.mergePendingDocUpdates',
+        {
+          workspaceId: update.workspaceId,
+          docId: update.id,
+        },
+        {
+          jobId,
+          priority: update._count > 100 ? 0 : 100,
+          delay: 0,
+        }
+      );
+    }
+  }
+
+  @Cron(CronExpression.EVERY_30_SECONDS)
+  async scheduleRecordPendingDocUpdatesCount() {
+    await this.job.add(
+      'doc.recordPendingDocUpdatesCount',
+      {},
+      {
+        delay: 30 * 1000,
+        jobId: 'doc:record-pending-updates-count',
+      }
+    );
+  }
+
+  @Cron(CronExpression.EVERY_30_SECONDS)
+  async scheduleFindEmptySummaryDocs() {
+    await this.job.add(
+      'doc.findEmptySummaryDocs',
+      {},
+      {
+        delay: 30 * 1000,
+        jobId: 'findEmptySummaryDocs',
+      }
+    );
+  }
+}

packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts

@@ -7,8 +7,9 @@ import { Doc as YDoc } from 'yjs';
 
 import { MockEventBus } from '../../../__tests__/mocks';
 import { createTestingApp, type TestingApp } from '../../../__tests__/utils';
-import { ConfigFactory, EventBus } from '../../../base';
-import { Flavor } from '../../../env';
+import { buildAppModule } from '../../../app.module';
+import { EventBus } from '../../../base';
+import { Env, Flavor } from '../../../env';
 import { Models } from '../../../models';
 import { DocReader, PgWorkspaceDocStorageAdapter } from '../../doc';
 
@@ -23,9 +24,10 @@ interface Context {
 const test = ava as TestFn<Context>;
 
 test.before(async t => {
-  // @ts-expect-error testing
-  env.FLAVOR = Flavor.Renderer;
+  const rendererEnv = new Env();
+  rendererEnv.FLAVOR = Flavor.Renderer;
   const app = await createTestingApp({
+    imports: [buildAppModule(rendererEnv)],
     tapModule: m => m.overrideProvider(EventBus).useClass(MockEventBus),
   });
 
@@ -39,11 +41,6 @@ let user: User;
 let workspace: Workspace;
 
 test.beforeEach(async t => {
-  t.context.app.get(ConfigFactory).override({
-    docService: {
-      endpoint: t.context.app.url(),
-    },
-  });
   await t.context.app.initTestingDB();
   user = await t.context.models.user.create({
     email: 'test@affine.pro',
@@ -59,9 +56,7 @@ test.afterEach.always(t => {
   t.context.recordDocView?.restore();
 });
 
-test.after.always(async t => {
-  await t.context.app.close();
-});
+test.after.always(async t => t.context.app.close());
 
 async function createDoc(
   adapter: PgWorkspaceDocStorageAdapter,

packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts

@@ -1,450 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import { mock } from 'node:test';
-
-import { User, Workspace } from '@prisma/client';
-import ava, { TestFn } from 'ava';
-
-import { createTestingApp, type TestingApp } from '../../../__tests__/utils';
-import { CryptoHelper } from '../../../base';
-import { Models } from '../../../models';
-import { DatabaseDocReader } from '../../doc';
-
-const test = ava as TestFn<{
-  models: Models;
-  app: TestingApp;
-  crypto: CryptoHelper;
-  databaseDocReader: DatabaseDocReader;
-}>;
-
-test.before(async t => {
-  const app = await createTestingApp();
-
-  t.context.models = app.get(Models);
-  t.context.crypto = app.get(CryptoHelper);
-  t.context.app = app;
-  t.context.databaseDocReader = app.get(DatabaseDocReader);
-});
-
-let user: User;
-let workspace: Workspace;
-
-test.beforeEach(async t => {
-  await t.context.app.initTestingDB();
-  user = await t.context.models.user.create({
-    email: 'test@affine.pro',
-  });
-  workspace = await t.context.models.workspace.create(user.id);
-});
-
-test.afterEach.always(async () => {
-  mock.reset();
-});
-
-test.after.always(async t => {
-  await t.context.app.close();
-});
-
-test('should forbid access to rpc api without access token', async t => {
-  const { app } = t.context;
-
-  await app
-    .GET('/rpc/workspaces/123/docs/123')
-    .expect({
-      status: 403,
-      code: 'Forbidden',
-      type: 'NO_PERMISSION',
-      name: 'ACCESS_DENIED',
-      message: 'Invalid internal request',
-    })
-    .expect(403);
-  t.pass();
-});
-
-test('should forbid access to rpc api with invalid access token', async t => {
-  const { app } = t.context;
-
-  await app
-    .GET('/rpc/workspaces/123/docs/123')
-    .set('x-access-token', 'invalid,wrong-signature')
-    .expect({
-      status: 403,
-      code: 'Forbidden',
-      type: 'NO_PERMISSION',
-      name: 'ACCESS_DENIED',
-      message: 'Invalid internal request',
-    })
-    .expect(403);
-  t.pass();
-});
-
-test('should forbid replayed internal access token', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}`;
-  const token = t.context.crypto.signInternalAccessToken({
-    method: 'GET',
-    path,
-    nonce: `nonce-${randomUUID()}`,
-  });
-
-  await app.GET(path).set('x-access-token', token).expect(404);
-
-  await app
-    .GET(path)
-    .set('x-access-token', token)
-    .expect({
-      status: 403,
-      code: 'Forbidden',
-      type: 'NO_PERMISSION',
-      name: 'ACCESS_DENIED',
-      message: 'Invalid internal request',
-    })
-    .expect(403);
-  t.pass();
-});
-
-test('should forbid internal access token when method mismatched', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/diff`;
-  await app
-    .POST(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect({
-      status: 403,
-      code: 'Forbidden',
-      type: 'NO_PERMISSION',
-      name: 'ACCESS_DENIED',
-      message: 'Invalid internal request',
-    })
-    .expect(403);
-  t.pass();
-});
-
-test('should forbid internal access token when path mismatched', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const wrongPath = `/rpc/workspaces/${workspaceId}/docs/${docId}`;
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/content`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({
-        method: 'GET',
-        path: wrongPath,
-      })
-    )
-    .expect({
-      status: 403,
-      code: 'Forbidden',
-      type: 'NO_PERMISSION',
-      name: 'ACCESS_DENIED',
-      message: 'Invalid internal request',
-    })
-    .expect(403);
-  t.pass();
-});
-
-test('should forbid internal access token when expired', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({
-        method: 'GET',
-        path,
-        now: Date.now() - 10 * 60 * 1000,
-        nonce: `nonce-${randomUUID()}`,
-      })
-    )
-    .expect({
-      status: 403,
-      code: 'Forbidden',
-      type: 'NO_PERMISSION',
-      name: 'ACCESS_DENIED',
-      message: 'Invalid internal request',
-    })
-    .expect(403);
-  t.pass();
-});
-
-test('should 404 when doc not found', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect({
-      status: 404,
-      code: 'Not Found',
-      type: 'RESOURCE_NOT_FOUND',
-      name: 'NOT_FOUND',
-      message: 'Doc not found',
-    })
-    .expect(404);
-  t.pass();
-});
-
-test('should return doc when found', async t => {
-  const { app } = t.context;
-
-  const docId = randomUUID();
-  const timestamp = Date.now();
-  await t.context.models.doc.createUpdates([
-    {
-      spaceId: workspace.id,
-      docId,
-      blob: Buffer.from('blob1 data'),
-      timestamp,
-      editorId: user.id,
-    },
-  ]);
-
-  const path = `/rpc/workspaces/${workspace.id}/docs/${docId}`;
-  const res = await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .set('x-cloud-trace-context', 'test-trace-id/span-id')
-    .expect(200)
-    .expect('x-request-id', 'test-trace-id')
-    .expect('Content-Type', 'application/octet-stream');
-  const bin = res.body as Buffer;
-  t.is(bin.toString(), 'blob1 data');
-  t.is(res.headers['x-doc-timestamp'], timestamp.toString());
-  t.is(res.headers['x-doc-editor-id'], user.id);
-});
-
-test('should 404 when doc diff not found', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/diff`;
-  await app
-    .POST(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'POST', path })
-    )
-    .expect({
-      status: 404,
-      code: 'Not Found',
-      type: 'RESOURCE_NOT_FOUND',
-      name: 'NOT_FOUND',
-      message: 'Doc not found',
-    })
-    .expect(404);
-  t.pass();
-});
-
-test('should 404 when doc content not found', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/content`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect({
-      status: 404,
-      code: 'Not Found',
-      type: 'RESOURCE_NOT_FOUND',
-      name: 'NOT_FOUND',
-      message: 'Doc not found',
-    })
-    .expect(404);
-  t.pass();
-});
-
-test('should get doc content in json format', async t => {
-  const { app } = t.context;
-  mock.method(t.context.databaseDocReader, 'getDocContent', async () => {
-    return {
-      title: 'test title',
-      summary: 'test summary',
-    };
-  });
-
-  const docId = randomUUID();
-  const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/content`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect('Content-Type', 'application/json; charset=utf-8')
-    .expect({
-      title: 'test title',
-      summary: 'test summary',
-    })
-    .expect(200);
-
-  await app
-    .GET(`${path}?full=false`)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect('Content-Type', 'application/json; charset=utf-8')
-    .expect({
-      title: 'test title',
-      summary: 'test summary',
-    })
-    .expect(200);
-  t.pass();
-});
-
-test('should get full doc content in json format', async t => {
-  const { app } = t.context;
-  mock.method(t.context.databaseDocReader, 'getFullDocContent', async () => {
-    return {
-      title: 'test title',
-      summary: 'test summary full',
-    };
-  });
-
-  const docId = randomUUID();
-  const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/content`;
-  await app
-    .GET(`${path}?full=true`)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect('Content-Type', 'application/json; charset=utf-8')
-    .expect({
-      title: 'test title',
-      summary: 'test summary full',
-    })
-    .expect(200);
-  t.pass();
-});
-
-test('should 404 when workspace content not found', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/content`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect({
-      status: 404,
-      code: 'Not Found',
-      type: 'RESOURCE_NOT_FOUND',
-      name: 'NOT_FOUND',
-      message: 'Workspace not found',
-    })
-    .expect(404);
-  t.pass();
-});
-
-test('should get workspace content in json format', async t => {
-  const { app } = t.context;
-  mock.method(t.context.databaseDocReader, 'getWorkspaceContent', async () => {
-    return {
-      name: 'test name',
-      avatarKey: 'avatar key',
-    };
-  });
-
-  const workspaceId = randomUUID();
-  const path = `/rpc/workspaces/${workspaceId}/content`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect(200)
-    .expect({
-      name: 'test name',
-      avatarKey: 'avatar key',
-    });
-  t.pass();
-});
-
-test('should get doc markdown in json format', async t => {
-  const { app } = t.context;
-  mock.method(t.context.databaseDocReader, 'getDocMarkdown', async () => {
-    return {
-      title: 'test title',
-      markdown: 'test markdown',
-      knownUnsupportedBlocks: [],
-      unknownBlocks: [],
-    };
-  });
-
-  const docId = randomUUID();
-  const path = `/rpc/workspaces/${workspace.id}/docs/${docId}/markdown`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect('Content-Type', 'application/json; charset=utf-8')
-    .expect(200)
-    .expect({
-      title: 'test title',
-      markdown: 'test markdown',
-      knownUnsupportedBlocks: [],
-      unknownBlocks: [],
-    });
-  t.pass();
-});
-
-test('should 404 when doc markdown not found', async t => {
-  const { app } = t.context;
-
-  const workspaceId = '123';
-  const docId = '123';
-  const path = `/rpc/workspaces/${workspaceId}/docs/${docId}/markdown`;
-  await app
-    .GET(path)
-    .set(
-      'x-access-token',
-      t.context.crypto.signInternalAccessToken({ method: 'GET', path })
-    )
-    .expect({
-      status: 404,
-      code: 'Not Found',
-      type: 'RESOURCE_NOT_FOUND',
-      name: 'NOT_FOUND',
-      message: 'Doc not found',
-    })
-    .expect(404);
-  t.pass();
-});

packages/backend/server/src/core/doc-service/config.ts

@@ -1,17 +0,0 @@
-import { defineModuleConfig } from '../../base';
-
-declare global {
-  interface AppConfigSchema {
-    docService: {
-      endpoint: string;
-    };
-  }
-}
-
-defineModuleConfig('docService', {
-  endpoint: {
-    desc: 'The endpoint of the doc service.',
-    default: '',
-    env: 'DOC_SERVICE_ENDPOINT',
-  },
-});

packages/backend/server/src/core/doc-service/controller.ts

@@ -1,138 +0,0 @@
-import {
-  Controller,
-  Get,
-  Logger,
-  Param,
-  Post,
-  Query,
-  RawBody,
-  Res,
-} from '@nestjs/common';
-import type { Response } from 'express';
-
-import { NotFound, SkipThrottle } from '../../base';
-import { Internal } from '../auth';
-import { DatabaseDocReader } from '../doc';
-
-@Controller('/rpc')
-export class DocRpcController {
-  private readonly logger = new Logger(DocRpcController.name);
-
-  constructor(private readonly docReader: DatabaseDocReader) {}
-
-  @SkipThrottle()
-  @Internal()
-  @Get('/workspaces/:workspaceId/docs/:docId')
-  async getDoc(
-    @Param('workspaceId') workspaceId: string,
-    @Param('docId') docId: string,
-    @Res() res: Response
-  ) {
-    const doc = await this.docReader.getDoc(workspaceId, docId);
-    if (!doc) {
-      throw new NotFound('Doc not found');
-    }
-    this.logger.debug(
-      `get doc ${docId} from workspace ${workspaceId}, size: ${doc.bin.length}`
-    );
-    res.setHeader('x-doc-timestamp', doc.timestamp.toString());
-    if (doc.editor) {
-      res.setHeader('x-doc-editor-id', doc.editor);
-    }
-    res.send(doc.bin);
-  }
-
-  @SkipThrottle()
-  @Internal()
-  @Get('/workspaces/:workspaceId/docs/:docId/markdown')
-  async getDocMarkdown(
-    @Param('workspaceId') workspaceId: string,
-    @Param('docId') docId: string,
-    @Query('aiEditable') aiEditable?: string
-  ) {
-    const result = await this.docReader.getDocMarkdown(
-      workspaceId,
-      docId,
-      aiEditable === 'true'
-    );
-    if (!result) {
-      throw new NotFound('Doc not found');
-    }
-    return result;
-  }
-
-  @SkipThrottle()
-  @Internal()
-  @Post('/workspaces/:workspaceId/docs/:docId/diff')
-  async getDocDiff(
-    @Param('workspaceId') workspaceId: string,
-    @Param('docId') docId: string,
-    @RawBody() stateVector: Buffer | undefined,
-    @Res() res: Response
-  ) {
-    const diff = await this.docReader.getDocDiff(
-      workspaceId,
-      docId,
-      stateVector
-    );
-    if (!diff) {
-      throw new NotFound('Doc not found');
-    }
-    this.logger.debug(
-      `get doc diff ${docId} from workspace ${workspaceId}, missing size: ${diff.missing.length}, old state size: ${stateVector?.length}, new state size: ${diff.state.length}`
-    );
-    res.setHeader('x-doc-timestamp', diff.timestamp.toString());
-    res.setHeader('x-doc-missing-offset', `0,${diff.missing.length}`);
-    const stateOffset = diff.missing.length;
-    res.setHeader(
-      'x-doc-state-offset',
-      `${stateOffset},${stateOffset + diff.state.length}`
-    );
-    res.send(Buffer.concat([diff.missing, diff.state]));
-  }
-
-  @SkipThrottle()
-  @Internal()
-  @Get('/workspaces/:workspaceId/docs/:docId/canvas')
-  async getDocCanvas(
-    @Param('workspaceId') workspaceId: string,
-    @Param('docId') docId: string
-  ) {
-    const projection = await this.docReader.getDocCanvas(workspaceId, docId);
-    if (!projection) {
-      throw new NotFound('Doc not found');
-    }
-    return projection;
-  }
-
-  @SkipThrottle()
-  @Internal()
-  @Get('/workspaces/:workspaceId/docs/:docId/content')
-  async getDocContent(
-    @Param('workspaceId') workspaceId: string,
-    @Param('docId') docId: string,
-    @Query('full') fullContent?: string
-  ) {
-    const content =
-      fullContent === 'true'
-        ? await this.docReader.getFullDocContent(workspaceId, docId)
-        : await this.docReader.getDocContent(workspaceId, docId);
-    if (!content) {
-      throw new NotFound('Doc not found');
-    }
-    this.logger.debug(`get doc content ${docId} from workspace ${workspaceId}`);
-    return content;
-  }
-
-  @SkipThrottle()
-  @Internal()
-  @Get('/workspaces/:workspaceId/content')
-  async getWorkspaceContent(@Param('workspaceId') workspaceId: string) {
-    const content = await this.docReader.getWorkspaceContent(workspaceId);
-    if (!content) {
-      throw new NotFound('Workspace not found');
-    }
-    this.logger.debug(`get workspace content ${workspaceId}`);
-    return content;
-  }
-}

packages/backend/server/src/core/doc-service/index.ts

@@ -1,14 +0,0 @@
-import './config';
-
-import { Module } from '@nestjs/common';
-
-import { DocStorageModule } from '../doc';
-import { DocRpcController } from './controller';
-import { DocServiceCronJob } from './job';
-
-@Module({
-  imports: [DocStorageModule],
-  providers: [DocServiceCronJob],
-  controllers: [DocRpcController],
-})
-export class DocServiceModule {}

packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md

@@ -1,86 +0,0 @@
-# Snapshot report for `src/core/doc/__tests__/reader-from-rpc.spec.ts`
-
-The actual snapshot is saved in `reader-from-rpc.spec.ts.snap`.
-
-Generated by [AVA](https://avajs.dev).
-
-## should return doc markdown success
-
-> Snapshot 1
-
-    {
-      knownUnsupportedBlocks: [
-        'RX4CG2zsBk:affine:note',
-        'S1mkc8zUoU:affine:note',
-        'yGlBdshAqN:affine:note',
-        '6lDiuDqZGL:affine:note',
-        'cauvaHOQmh:affine:note',
-        '2jwCeO8Yot:affine:note',
-        'c9MF_JiRgx:affine:note',
-        '6x7ALjUDjj:affine:surface',
-      ],
-      markdown: `AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊
-      ␊
-      ␊
-      ␊
-      # You own your data, with no compromises␊
-      ␊
-      ## Local-first & Real-time collaborative␊
-      ␊
-      We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊
-      ␊
-      AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊
-      ␊
-      ␊
-      ␊
-      ### Blocks that assemble your next docs, tasks kanban or whiteboard␊
-      ␊
-      There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊
-      ␊
-      We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊
-      ␊
-      If you want to learn more about the product design of AFFiNE, here goes the concepts:␊
-      ␊
-      To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊
-      ␊
-      ## A true canvas for blocks in any form␊
-      ␊
-      [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊
-      ␊
-      ␊
-      ␊
-      "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊
-      ␊
-      * Quip & Notion with their great concept of "everything is a block"␊
-      * Trello with their Kanban␊
-      * Airtable & Miro with their no-code programable datasheets␊
-      * Miro & Whimiscal with their edgeless visual whiteboard␊
-      * Remnote & Capacities with their object-based tag system␊
-      For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊
-      ␊
-      ## Self Host␊
-      ␊
-      Self host AFFiNE␊
-      ␊
-      ␊
-      ### Learning From␊
-      ||Title|Tag|␊
-      |---|---|---|␊
-      |Affine Development|Affine Development|<span data-affine-option data-value="AxSe-53xjX" data-option-color="var(--affine-tag-pink)">AFFiNE</span>|␊
-      |For developers or installations guides, please go to AFFiNE Doc|For developers or installations guides, please go to AFFiNE Doc|<span data-affine-option data-value="0jh9gNw4Yl" data-option-color="var(--affine-tag-orange)">Developers</span>|␊
-      |Quip & Notion with their great concept of "everything is a block"|Quip & Notion with their great concept of "everything is a block"|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Trello with their Kanban|Trello with their Kanban|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Airtable & Miro with their no-code programable datasheets|Airtable & Miro with their no-code programable datasheets|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|<span data-affine-option data-value="HgHsKOUINZ" data-option-color="var(--affine-tag-blue)">Reference</span>|␊
-      |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊
-      ␊
-      ## Affine Development␊
-      ␊
-      For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊
-      ␊
-      ␊
-      ␊
-      `,
-      title: 'Write, Draw, Plan all at Once.',
-      unknownBlocks: [],
-    }

packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts

@@ -1,432 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import { mock } from 'node:test';
-
-import { User, Workspace } from '@prisma/client';
-import ava, { TestFn } from 'ava';
-import { applyUpdate, Doc as YDoc } from 'yjs';
-
-import { createModule } from '../../../__tests__/create-module';
-import { Mockers } from '../../../__tests__/mocks';
-import { createTestingApp, type TestingApp } from '../../../__tests__/utils';
-import { UserFriendlyError } from '../../../base';
-import { ConfigFactory } from '../../../base/config';
-import { Models } from '../../../models';
-import {
-  DatabaseDocReader,
-  DocReader,
-  DocStorageModule,
-  PgWorkspaceDocStorageAdapter,
-} from '../index';
-import { RpcDocReader } from '../reader';
-
-const module = await createModule({
-  imports: [DocStorageModule],
-});
-
-const test = ava as TestFn<{
-  models: Models;
-  app: TestingApp;
-  docApp: TestingApp;
-  docReader: DocReader;
-  databaseDocReader: DatabaseDocReader;
-  adapter: PgWorkspaceDocStorageAdapter;
-  config: ConfigFactory;
-}>;
-
-test.before(async t => {
-  // test key
-  process.env.AFFINE_PRIVATE_KEY = `-----BEGIN PRIVATE KEY-----
-MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgS3IAkshQuSmFWGpe
-rGTg2vwaC3LdcvBQlYHHMBYJZMyhRANCAAQXdT/TAh4neNEpd4UqpDIEqWv0XvFo
-BRJxGsC5I/fetqObdx1+KEjcm8zFU2xLaUTw9IZCu8OslloOjQv4ur0a
------END PRIVATE KEY-----`;
-  // @ts-expect-error testing
-  env.FLAVOR = 'renderer';
-  const notDocApp = await createTestingApp();
-  // @ts-expect-error testing
-  env.FLAVOR = 'doc';
-  const docApp = await createTestingApp();
-
-  t.context.models = notDocApp.get(Models);
-  t.context.docReader = notDocApp.get(DocReader);
-  t.context.databaseDocReader = docApp.get(DatabaseDocReader);
-  t.context.adapter = docApp.get(PgWorkspaceDocStorageAdapter);
-  t.context.config = notDocApp.get(ConfigFactory);
-  t.context.app = notDocApp;
-  t.context.docApp = docApp;
-});
-
-let user: User;
-let workspace: Workspace;
-
-test.beforeEach(async t => {
-  t.context.config.override({
-    docService: {
-      endpoint: t.context.docApp.url(),
-    },
-  });
-  await t.context.app.initTestingDB();
-  user = await t.context.models.user.create({
-    email: 'test@affine.pro',
-  });
-  workspace = await t.context.models.workspace.create(user.id);
-});
-
-test.afterEach.always(() => {
-  mock.reset();
-});
-
-test.after.always(async t => {
-  await t.context.app.close();
-  await t.context.docApp.close();
-  await module.close();
-});
-
-test('should be rpc reader', async t => {
-  const { docReader } = t.context;
-  t.true(docReader instanceof RpcDocReader);
-});
-
-test('should return null when doc not found', async t => {
-  const { docReader } = t.context;
-  const docId = randomUUID();
-  const doc = await docReader.getDoc(workspace.id, docId);
-  t.is(doc, null);
-});
-
-test('should throw error when doc service internal error', async t => {
-  const { docReader, adapter } = t.context;
-  const docId = randomUUID();
-  mock.method(adapter, 'getDoc', async () => {
-    throw new Error('mock doc service internal error');
-  });
-  mock.method(adapter, 'getDocBinNative', async () => {
-    throw new Error('mock doc service internal error');
-  });
-  let err = await t.throwsAsync(docReader.getDoc(workspace.id, docId), {
-    instanceOf: UserFriendlyError,
-    message: 'An internal error occurred.',
-    name: 'internal_server_error',
-  });
-  t.is(err.type, 'internal_server_error');
-  t.is(err.status, 500);
-
-  err = await t.throwsAsync(docReader.getDocDiff(workspace.id, docId), {
-    instanceOf: UserFriendlyError,
-    message: 'An internal error occurred.',
-    name: 'internal_server_error',
-  });
-  t.is(err.type, 'internal_server_error');
-  t.is(err.status, 500);
-
-  err = await t.throwsAsync(docReader.getDocContent(workspace.id, docId), {
-    instanceOf: UserFriendlyError,
-    message: 'An internal error occurred.',
-    name: 'internal_server_error',
-  });
-  t.is(err.type, 'internal_server_error');
-  t.is(err.status, 500);
-
-  err = await t.throwsAsync(docReader.getWorkspaceContent(workspace.id), {
-    instanceOf: UserFriendlyError,
-    message: 'An internal error occurred.',
-    name: 'internal_server_error',
-  });
-  t.is(err.type, 'internal_server_error');
-  t.is(err.status, 500);
-});
-
-test('should fallback to database doc reader when endpoint network error', async t => {
-  const { docReader } = t.context;
-  t.context.config.override({
-    docService: {
-      endpoint: 'http://localhost:13010',
-    },
-  });
-  const docId = randomUUID();
-  const timestamp = Date.now();
-  await t.context.models.doc.createUpdates([
-    {
-      spaceId: workspace.id,
-      docId,
-      blob: Buffer.from('blob1 data'),
-      timestamp,
-      editorId: user.id,
-    },
-  ]);
-
-  const doc = await docReader.getDoc(workspace.id, docId);
-  t.truthy(doc);
-  t.is(Buffer.from(doc!.bin).toString('utf8'), 'blob1 data');
-  t.is(doc!.timestamp, timestamp);
-  t.is(doc!.editor, user.id);
-});
-
-test('should return doc when found', async t => {
-  const { docReader } = t.context;
-
-  const docId = randomUUID();
-  const timestamp = Date.now();
-  await t.context.models.doc.createUpdates([
-    {
-      spaceId: workspace.id,
-      docId,
-      blob: Buffer.from('blob1 data'),
-      timestamp,
-      editorId: user.id,
-    },
-  ]);
-
-  const doc = await docReader.getDoc(workspace.id, docId);
-  t.truthy(doc);
-  t.is(doc!.bin.toString(), 'blob1 data');
-  t.is(doc!.timestamp, timestamp);
-  t.is(doc!.editor, user.id);
-});
-
-test('should return doc diff', async t => {
-  const { docReader } = t.context;
-  const docId = randomUUID();
-  const timestamp = Date.now();
-  let updates: Buffer[] = [];
-  const doc1 = new YDoc();
-  doc1.on('update', data => {
-    updates.push(Buffer.from(data));
-  });
-
-  const text = doc1.getText('content');
-  text.insert(0, 'hello');
-  text.insert(5, 'world');
-  text.insert(5, ' ');
-  text.insert(11, '!');
-
-  await t.context.models.doc.createUpdates(
-    updates.map((update, index) => ({
-      spaceId: workspace.id,
-      docId,
-      blob: update,
-      timestamp: timestamp + index,
-      editorId: user.id,
-    }))
-  );
-  // clear updates
-  updates.splice(0, updates.length);
-
-  const doc2 = new YDoc();
-  const diff = await docReader.getDocDiff(workspace.id, docId);
-  t.truthy(diff);
-  t.truthy(diff!.missing);
-  t.truthy(diff!.state);
-  applyUpdate(doc2, diff!.missing);
-  t.is(doc2.getText('content').toString(), 'hello world!');
-
-  // nothing changed
-  const diff2 = await docReader.getDocDiff(workspace.id, docId, diff!.state);
-  t.truthy(diff2);
-  t.truthy(diff2!.missing);
-  t.deepEqual(diff2!.missing, new Uint8Array([0, 0]));
-  t.truthy(diff2!.state);
-  applyUpdate(doc2, diff2!.missing);
-  t.is(doc2.getText('content').toString(), 'hello world!');
-
-  // add new content on doc1
-  text.insert(12, '@');
-  await t.context.models.doc.createUpdates(
-    updates.map((update, index) => ({
-      spaceId: workspace.id,
-      docId,
-      blob: update,
-      timestamp: Date.now() + index + 1000,
-      editorId: user.id,
-    }))
-  );
-
-  const diff3 = await docReader.getDocDiff(workspace.id, docId, diff2!.state);
-  t.truthy(diff3);
-  t.truthy(diff3!.missing);
-  t.truthy(diff3!.state);
-  applyUpdate(doc2, diff3!.missing);
-  t.is(doc2.getText('content').toString(), 'hello world!@');
-});
-
-test('should get doc diff fallback to database doc reader when endpoint network error', async t => {
-  const { docReader } = t.context;
-  t.context.config.override({
-    docService: {
-      endpoint: 'http://localhost:13010',
-    },
-  });
-  const docId = randomUUID();
-  const timestamp = Date.now();
-  let updates: Buffer[] = [];
-  const doc1 = new YDoc();
-  doc1.on('update', data => {
-    updates.push(Buffer.from(data));
-  });
-
-  const text = doc1.getText('content');
-  text.insert(0, 'hello');
-  text.insert(5, 'world');
-  text.insert(5, ' ');
-  text.insert(11, '!');
-
-  await t.context.models.doc.createUpdates(
-    updates.map((update, index) => ({
-      spaceId: workspace.id,
-      docId,
-      blob: update,
-      timestamp: timestamp + index,
-      editorId: user.id,
-    }))
-  );
-  // clear updates
-  updates.splice(0, updates.length);
-
-  const doc2 = new YDoc();
-  const diff = await docReader.getDocDiff(workspace.id, docId);
-  t.truthy(diff);
-  t.truthy(diff!.missing);
-  t.truthy(diff!.state);
-  applyUpdate(doc2, diff!.missing);
-  t.is(doc2.getText('content').toString(), 'hello world!');
-});
-
-test('should get doc content', async t => {
-  const docId = randomUUID();
-  const { docReader, databaseDocReader } = t.context;
-  mock.method(databaseDocReader, 'getDocContent', async () => {
-    return {
-      title: 'test title',
-      summary: 'test summary',
-    };
-  });
-  const docContent = await docReader.getDocContent(workspace.id, docId);
-  t.deepEqual(docContent, {
-    title: 'test title',
-    summary: 'test summary',
-  });
-});
-
-test('should return null when doc content not exists', async t => {
-  const docId = randomUUID();
-  const { docReader, adapter } = t.context;
-
-  const doc = new YDoc();
-  const text = doc.getText('content');
-  const updates: Buffer[] = [];
-
-  doc.on('update', update => {
-    updates.push(Buffer.from(update));
-  });
-
-  text.insert(0, 'hello');
-  text.insert(5, 'world');
-  text.insert(5, ' ');
-
-  await adapter.pushDocUpdates(workspace.id, docId, updates, user.id);
-
-  const docContent = await docReader.getDocContent(workspace.id, docId);
-  t.is(docContent, null);
-
-  const notExists = await docReader.getDocContent(workspace.id, randomUUID());
-  t.is(notExists, null);
-});
-
-test('should get workspace content from doc service rpc', async t => {
-  const { docReader, databaseDocReader } = t.context;
-  const track = mock.method(
-    databaseDocReader,
-    'getWorkspaceContent',
-    async () => {
-      return {
-        id: workspace.id,
-        name: 'test name',
-        avatarKey: '',
-      };
-    }
-  );
-
-  const workspaceContent = await docReader.getWorkspaceContent(workspace.id);
-  t.is(track.mock.callCount(), 1);
-  t.deepEqual(workspaceContent, {
-    id: workspace.id,
-    name: 'test name',
-    avatarKey: '',
-  });
-});
-
-test('should return null when workspace bin meta not exists', async t => {
-  const { docReader, adapter } = t.context;
-  const doc = new YDoc();
-  const text = doc.getText('content');
-  const updates: Buffer[] = [];
-
-  doc.on('update', update => {
-    updates.push(Buffer.from(update));
-  });
-
-  text.insert(0, 'hello');
-  text.insert(5, 'world');
-  text.insert(5, ' ');
-
-  await adapter.pushDocUpdates(workspace.id, workspace.id, updates, user.id);
-
-  const workspaceContent = await docReader.getWorkspaceContent(workspace.id);
-  t.is(workspaceContent, null);
-
-  // workspace not exists
-  const notExists = await docReader.getWorkspaceContent(randomUUID());
-  t.is(notExists, null);
-});
-
-test('should return doc markdown success', async t => {
-  const { docReader } = t.context;
-
-  const workspace = await module.create(Mockers.Workspace, {
-    owner: user,
-    name: '',
-  });
-
-  const docSnapshot = await module.create(Mockers.DocSnapshot, {
-    workspaceId: workspace.id,
-    user,
-  });
-
-  const result = await docReader.getDocMarkdown(
-    workspace.id,
-    docSnapshot.id,
-    false
-  );
-  if (result) {
-    const { revision, ...markdown } = result;
-    t.truthy(revision);
-    t.snapshot(markdown);
-  }
-  const canvas = await docReader.getDocCanvas(workspace.id, docSnapshot.id);
-  t.is(canvas?.version, 1);
-  t.is(canvas?.docId, docSnapshot.id);
-  t.truthy(canvas?.revision);
-  t.deepEqual(canvas?.counts, {
-    connector: 6,
-    group: 6,
-    shape: 7,
-    text: 7,
-  });
-});
-
-test('should read markdown return null when doc not exists', async t => {
-  const { docReader } = t.context;
-
-  const workspace = await module.create(Mockers.Workspace, {
-    owner: user,
-    name: '',
-  });
-
-  const result = await docReader.getDocMarkdown(
-    workspace.id,
-    randomUUID(),
-    false
-  );
-  t.is(result, null);
-  t.is(await docReader.getDocCanvas(workspace.id, randomUUID()), null);
-});

packages/backend/server/src/core/doc/index.ts

@@ -20,7 +20,6 @@ import { DocWriter } from './writer';
     DocStorageOptions,
     PgWorkspaceDocStorageAdapter,
     PgUserspaceDocStorageAdapter,
-    DocStorageCronJob,
     DocReaderProvider,
     DatabaseDocReader,
     DocEventsListener,
@@ -35,8 +34,14 @@ import { DocWriter } from './writer';
   ],
 })
 export class DocStorageModule {}
+
+@Module({
+  imports: [DocStorageModule],
+  providers: [DocStorageCronJob],
+})
+export class DocStorageWorkerModule {}
+
 export {
-  // only for doc-service
   DatabaseDocReader,
   DocReader,
   DocWriter,

packages/backend/server/src/core/doc/job.ts

@@ -6,7 +6,7 @@ import { BackendRuntimeProvider } from '../backend-runtime';
 
 declare global {
   interface Jobs {
-    'nightly.cleanExpiredHistories': {};
+    'doc.cleanExpiredHistories': {};
   }
 }
 
@@ -20,7 +20,7 @@ export class DocStorageCronJob {
   @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
   async nightlyJob() {
     await this.queue.add(
-      'nightly.cleanExpiredHistories',
+      'doc.cleanExpiredHistories',
       {},
       {
         jobId: 'nightly-doc-clean-expired-histories',
@@ -28,7 +28,7 @@ export class DocStorageCronJob {
     );
   }
 
-  @OnJob('nightly.cleanExpiredHistories')
+  @OnJob('doc.cleanExpiredHistories')
   async cleanExpiredHistories() {
     for (;;) {
       const count = await this.rt.cleanupExpiredSnapshotHistories(1000);

packages/backend/server/src/core/doc/reader.ts

@@ -1,21 +1,12 @@
-import { FactoryProvider, Injectable, Logger } from '@nestjs/common';
-import { ModuleRef } from '@nestjs/core';
+import { Injectable, Logger } from '@nestjs/common';
 import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';
 
-import {
-  Cache,
-  Config,
-  CryptoHelper,
-  getOrGenRequestId,
-  safeFetch,
-  UserFriendlyError,
-} from '../../base';
+import { Cache } from '../../base';
 import { Models } from '../../models';
 import { WorkspaceBlobStorage } from '../storage';
 import {
   type CanvasProjectionV1,
   type PageDocContent,
-  parseCanvasProjection,
   parseDocToMarkdownFromDocSnapshot,
   parsePageDoc,
   parseWorkspaceDoc,
@@ -288,248 +279,7 @@ export class DatabaseDocReader extends DocReader {
   }
 }
 
-@Injectable()
-export class RpcDocReader extends DatabaseDocReader {
-  protected override readonly logger = new Logger(DocReader.name);
-
-  constructor(
-    private readonly config: Config,
-    private readonly crypto: CryptoHelper,
-    protected override readonly cache: Cache,
-    protected override readonly models: Models,
-    protected override readonly blobStorage: WorkspaceBlobStorage,
-    protected override readonly workspace: PgWorkspaceDocStorageAdapter
-  ) {
-    super(cache, models, blobStorage, workspace);
-  }
-
-  private async fetch(url: string, method: 'GET' | 'POST', body?: Uint8Array) {
-    const { pathname } = new URL(url);
-    const accessToken = this.crypto.signInternalAccessToken({
-      method,
-      path: pathname,
-    });
-
-    const headers: Record<string, string> = {
-      'x-access-token': accessToken,
-      'x-cloud-trace-context': getOrGenRequestId('rpc'),
-    };
-    if (body) {
-      headers['content-type'] = 'application/octet-stream';
-    }
-    const requestInit: RequestInit = {
-      method,
-      headers,
-    };
-    if (body) {
-      requestInit.body = body;
-    }
-    const res = await safeFetch(url, requestInit, {
-      timeoutMs: 10_000,
-      maxRedirects: 0,
-      maxBytes: 50 * 1024 * 1024,
-      allowedHeaders: [
-        'content-type',
-        'x-access-token',
-        'x-cloud-trace-context',
-      ],
-      allowPrivateTargetOrigin: true,
-    });
-    if (!res.ok) {
-      if (res.status === 404) {
-        return null;
-      }
-      const body = (await res.json()) as UserFriendlyError;
-      throw UserFriendlyError.fromUserFriendlyErrorJSON(body);
-    }
-    return res;
-  }
-
-  override async getDoc(
-    workspaceId: string,
-    docId: string
-  ): Promise<DocRecord | null> {
-    const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}`;
-    try {
-      const res = await this.fetch(url, 'GET');
-      if (!res) {
-        return null;
-      }
-      const timestamp = res.headers.get('x-doc-timestamp') as string;
-      const editor = res.headers.get('x-doc-editor-id') ?? undefined;
-      const bin = await res.arrayBuffer();
-      return {
-        spaceId: workspaceId,
-        docId,
-        bin: Buffer.from(bin),
-        timestamp: parseInt(timestamp),
-        editor,
-      };
-    } catch (e) {
-      if (e instanceof UserFriendlyError) {
-        throw e;
-      }
-      const err = e as Error;
-      // other error
-      this.logger.error(
-        `Failed to fetch doc ${url}, fallback to database doc reader`,
-        err
-      );
-      // fallback to database doc reader if the error is not user friendly, like network error
-      return await super.getDoc(workspaceId, docId);
-    }
-  }
-
-  override async getDocMarkdown(
-    workspaceId: string,
-    docId: string,
-    aiEditable: boolean
-  ): Promise<DocMarkdown | null> {
-    const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/markdown?aiEditable=${aiEditable}`;
-    try {
-      const res = await this.fetch(url, 'GET');
-      if (!res) {
-        return null;
-      }
-      return (await res.json()) as DocMarkdown;
-    } catch (e) {
-      if (e instanceof UserFriendlyError) {
-        throw e;
-      }
-      const err = e as Error;
-      // other error
-      this.logger.error(
-        `Failed to fetch doc markdown ${url}, fallback to database doc reader`,
-        err
-      );
-      // fallback to database doc reader if the error is not user friendly, like network error
-      return await super.getDocMarkdown(workspaceId, docId, aiEditable);
-    }
-  }
-
-  override async getDocCanvas(
-    workspaceId: string,
-    docId: string
-  ): Promise<CanvasProjectionV1 | null> {
-    const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/canvas`;
-    try {
-      const res = await this.fetch(url, 'GET');
-      if (!res) {
-        return null;
-      }
-      return parseCanvasProjection(await res.json());
-    } catch (e) {
-      if (e instanceof UserFriendlyError) {
-        throw e;
-      }
-      this.logger.error(
-        `Failed to fetch doc canvas ${url}, fallback to database doc reader`,
-        e as Error
-      );
-      return await super.getDocCanvas(workspaceId, docId);
-    }
-  }
-
-  override async getDocDiff(
-    workspaceId: string,
-    docId: string,
-    stateVector?: Uint8Array
-  ): Promise<DocDiff | null> {
-    const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/diff`;
-    try {
-      const res = await this.fetch(url, 'POST', stateVector);
-      if (!res) {
-        return null;
-      }
-      const timestamp = res.headers.get('x-doc-timestamp') as string;
-      // blob missing data offset [0, 123]
-      // x-doc-missing-offset: 0,123
-      // blob stateVector data offset [124,789]
-      // x-doc-state-offset: 124,789
-      const missingOffset = res.headers.get('x-doc-missing-offset') as string;
-      const [missingStart, missingEnd] = missingOffset.split(',').map(Number);
-      const stateOffset = res.headers.get('x-doc-state-offset') as string;
-      const [stateStart, stateEnd] = stateOffset.split(',').map(Number);
-      const bin = await res.arrayBuffer();
-      return {
-        missing: new Uint8Array(bin, missingStart, missingEnd - missingStart),
-        state: new Uint8Array(bin, stateStart, stateEnd - stateStart),
-        timestamp: parseInt(timestamp),
-      };
-    } catch (e) {
-      if (e instanceof UserFriendlyError) {
-        throw e;
-      }
-      const err = e as Error;
-      this.logger.error(
-        `Failed to fetch doc diff ${url}, fallback to database doc reader`,
-        err
-      );
-      // fallback to database doc reader if the error is not user friendly, like network error
-      return await super.getDocDiff(workspaceId, docId, stateVector);
-    }
-  }
-
-  protected override async getDocContentWithoutCache(
-    workspaceId: string,
-    docId: string,
-    fullContent = false
-  ): Promise<PageDocContent | null> {
-    const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/content?full=${fullContent}`;
-    try {
-      const res = await this.fetch(url, 'GET');
-      if (!res) {
-        return null;
-      }
-      return (await res.json()) as PageDocContent;
-    } catch (e) {
-      if (e instanceof UserFriendlyError) {
-        throw e;
-      }
-      const err = e as Error;
-      this.logger.error(
-        `Failed to fetch doc content ${url}, fallback to database doc reader`,
-        err
-      );
-      return await super.getDocContentWithoutCache(
-        workspaceId,
-        docId,
-        fullContent
-      );
-    }
-  }
-
-  protected override async getWorkspaceContentWithoutCache(
-    workspaceId: string
-  ): Promise<WorkspaceDocInfo | null> {
-    const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/content`;
-    try {
-      const res = await this.fetch(url, 'GET');
-      if (!res) {
-        return null;
-      }
-      return (await res.json()) as WorkspaceDocInfo;
-    } catch (e) {
-      if (e instanceof UserFriendlyError) {
-        throw e;
-      }
-      const err = e as Error;
-      this.logger.error(
-        `Failed to fetch workspace content ${url}, fallback to database doc reader`,
-        err
-      );
-      return await super.getWorkspaceContentWithoutCache(workspaceId);
-    }
-  }
-}
-
-export const DocReaderProvider: FactoryProvider = {
+export const DocReaderProvider = {
   provide: DocReader,
-  useFactory: (ref: ModuleRef) => {
-    if (env.flavors.doc || env.flavors.front) {
-      return ref.create(DatabaseDocReader);
-    }
-    return ref.create(RpcDocReader);
-  },
-  inject: [ModuleRef],
+  useExisting: DatabaseDocReader,
 };

packages/backend/server/src/core/mail/__tests__/mailer.spec.ts

@@ -25,6 +25,7 @@ interface Context {
     commitMailDeliveryQuotaV1: Sinon.SinonStub;
     releaseMailDeliveryQuotaV1: Sinon.SinonStub;
     embeddingHealth: Sinon.SinonStub;
+    searchStatus: Sinon.SinonStub;
   };
 }
 
@@ -41,6 +42,7 @@ test.before(async t => {
       reason: 'test',
       workerRunning: false,
     }),
+    searchStatus: Sinon.stub().resolves({ ready: false }),
   };
   t.context.module = await createTestingModule({
     tapModule: builder => {

packages/backend/server/src/core/notification/resolver.ts

@@ -1,4 +1,17 @@
-import { Args, ID, Mutation, ResolveField, Resolver } from '@nestjs/graphql';
+import {
+  Args,
+  ID,
+  Info,
+  Mutation,
+  ResolveField,
+  Resolver,
+} from '@nestjs/graphql';
+import {
+  type FragmentDefinitionNode,
+  type GraphQLResolveInfo,
+  Kind,
+  type SelectionNode,
+} from 'graphql';
 
 import {
   MentionUserDocAccessDenied,
@@ -17,6 +30,39 @@ import {
   UnionNotificationBodyType,
 } from './types';
 
+function hasSelectedField(
+  selections: readonly SelectionNode[],
+  fieldName: string,
+  fragments: Record<string, FragmentDefinitionNode>
+): boolean {
+  for (const selection of selections) {
+    if (selection.kind === Kind.FIELD) {
+      if (selection.name.value === fieldName) return true;
+      continue;
+    }
+    if (selection.kind === Kind.INLINE_FRAGMENT) {
+      if (
+        hasSelectedField(
+          selection.selectionSet.selections,
+          fieldName,
+          fragments
+        )
+      ) {
+        return true;
+      }
+      continue;
+    }
+    const fragment = fragments[selection.name.value];
+    if (
+      fragment &&
+      hasSelectedField(fragment.selectionSet.selections, fieldName, fragments)
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
 @Resolver(() => UserType)
 export class UserNotificationResolver {
   constructor(
@@ -29,8 +75,21 @@ export class UserNotificationResolver {
   })
   async notifications(
     @CurrentUser() me: UserType,
-    @Args('pagination', PaginationInput.decode) pagination: PaginationInput
+    @Args('pagination', PaginationInput.decode) pagination: PaginationInput,
+    @Info() info: GraphQLResolveInfo
   ): Promise<PaginatedNotificationObjectType> {
+    const selections = info.fieldNodes.flatMap(node =>
+      node.selectionSet ? [...node.selectionSet.selections] : []
+    );
+    const includesList =
+      hasSelectedField(selections, 'edges', info.fragments) ||
+      hasSelectedField(selections, 'pageInfo', info.fragments);
+
+    if (!includesList) {
+      const totalCount = await this.service.countByUserId(me.id);
+      return paginate([], 'createdAt', pagination, totalCount);
+    }
+
     const [notifications, totalCount] = await Promise.all([
       this.service.findManyByUserId(me.id, pagination),
       this.service.countByUserId(me.id),

packages/backend/server/src/core/permission/__tests__/docs.spec.ts

@@ -1,426 +0,0 @@
-import { Prisma, PrismaClient } from '@prisma/client';
-import test from 'ava';
-
-import { createModule } from '../../../__tests__/create-module';
-import { Mockers } from '../../../__tests__/mocks';
-import { Models } from '../../../models';
-import { AccessControllerBuilder } from '../builder';
-import { DocRole, PermissionModule, WorkspaceRole } from '../index';
-import { PermissionSqlPredicateBuilder } from '../sql-predicate';
-import type { DocAction } from '../types';
-
-const module = await createModule({
-  imports: [PermissionModule],
-});
-
-const builder = module.get(AccessControllerBuilder);
-const models = module.get(Models);
-const db = module.get(PrismaClient);
-const sqlPredicate = module.get(PermissionSqlPredicateBuilder);
-
-test.after.always(async () => {
-  await module.close();
-});
-
-async function sqlReadableDocIds(input: {
-  workspaceId: string;
-  userId?: string;
-  action?: DocAction;
-  docIds: string[];
-}) {
-  const values = Prisma.join(
-    input.docIds.map((docId, index) => Prisma.sql`(${docId}, ${index})`)
-  );
-  const predicate = sqlPredicate.docReadableSql({
-    workspaceId: input.workspaceId,
-    userId: input.userId,
-    action: input.action ?? 'Doc.Read',
-    docIdColumn: Prisma.raw('c.doc_id'),
-  });
-  const rows = await db.$queryRaw<{ docId: string }[]>`
-    WITH candidates(doc_id, ord) AS (VALUES ${values})
-    SELECT c.doc_id AS "docId"
-    FROM candidates c
-    WHERE ${predicate}
-    ORDER BY c.ord ASC
-  `;
-  return rows.map(row => row.docId);
-}
-
-async function resetProjection(workspaceId: string) {
-  await db.$executeRaw`DELETE FROM doc_grants WHERE workspace_id = ${workspaceId}`;
-  await db.$executeRaw`DELETE FROM doc_access_policies WHERE workspace_id = ${workspaceId}`;
-  await db.$executeRaw`DELETE FROM workspace_members WHERE workspace_id = ${workspaceId}`;
-  await db.$executeRaw`
-    INSERT INTO workspace_access_policies (
-      workspace_id,
-      visibility,
-      sharing_enabled,
-      url_preview_enabled,
-      member_default_doc_role,
-      updated_at
-    )
-    VALUES (${workspaceId}, 'private', true, false, 'none', now())
-    ON CONFLICT (workspace_id)
-    DO UPDATE SET
-      visibility = EXCLUDED.visibility,
-      sharing_enabled = EXCLUDED.sharing_enabled,
-      url_preview_enabled = EXCLUDED.url_preview_enabled,
-      member_default_doc_role = EXCLUDED.member_default_doc_role,
-      updated_at = now()
-  `;
-  await setWritableRuntime(workspaceId);
-}
-
-async function setWritableRuntime(workspaceId: string) {
-  await db.effectiveWorkspaceQuotaState.upsert({
-    where: { workspaceId },
-    create: {
-      workspaceId,
-      plan: 'free',
-      usesOwnerQuota: false,
-      seatLimit: 0,
-      blobLimit: 0,
-      storageQuota: 0,
-      historyPeriodSeconds: 0,
-      readonly: false,
-      known: true,
-    },
-    update: {
-      readonly: false,
-      readonlyReasons: [],
-      known: true,
-      stale: false,
-    },
-  });
-}
-
-test('should filter docs by Doc.Read', async t => {
-  const owner = await module.create(Mockers.User);
-  const workspace = await module.create(Mockers.Workspace, {
-    owner,
-  });
-
-  const docs1 = await builder
-    .user(owner.id)
-    .workspace(workspace.id)
-    .docs(
-      [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }],
-      'Doc.Read'
-    );
-
-  t.is(docs1.length, 3);
-  t.snapshot(docs1);
-
-  // member should have access to the docs
-  const member = await module.create(Mockers.User);
-  await module.create(Mockers.WorkspaceUser, {
-    workspaceId: workspace.id,
-    userId: member.id,
-    type: WorkspaceRole.Collaborator,
-  });
-
-  await module.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'doc1',
-    userId: member.id,
-    type: DocRole.Reader,
-  });
-
-  await module.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'doc2',
-    userId: member.id,
-    type: DocRole.Manager,
-  });
-
-  const docs2 = await builder
-    .user(member.id)
-    .workspace(workspace.id)
-    .docs(
-      [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }],
-      'Doc.Read'
-    );
-
-  t.is(docs2.length, 3);
-  t.snapshot(docs2);
-
-  // other user should not have access to the docs
-  const other = await module.create(Mockers.User);
-
-  const docs3 = await builder
-    .user(other.id)
-    .workspace(workspace.id)
-    .docs(
-      [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }],
-      'Doc.Read'
-    );
-
-  t.is(docs3.length, 0);
-});
-
-test('SQL doc read predicate handles member default and public candidates', async t => {
-  const owner = await module.create(Mockers.User);
-  const member = await module.create(Mockers.User);
-  const workspace = await module.create(Mockers.Workspace, {
-    owner,
-  });
-  await resetProjection(workspace.id);
-  await db.$executeRaw`
-    UPDATE workspace_access_policies
-    SET member_default_doc_role = 'reader'
-    WHERE workspace_id = ${workspace.id}
-  `;
-  await db.$executeRaw`
-    INSERT INTO workspace_members (
-      workspace_id,
-      user_id,
-      role,
-      state,
-      source,
-      updated_at
-    )
-    VALUES (${workspace.id}, ${member.id}, 'member', 'active', 'legacy', now())
-  `;
-  await db.$executeRaw`
-    INSERT INTO doc_access_policies (
-      workspace_id,
-      doc_id,
-      visibility,
-      public_role,
-      member_default_role,
-      updated_at
-    )
-    VALUES
-      (${workspace.id}, 'member-default-none', 'private', NULL, 'none', now()),
-      (${workspace.id}, 'public-doc', 'public', 'external', NULL, now())
-  `;
-
-  const docIds = ['missing-policy', 'member-default-none', 'public-doc'];
-  const sqlReadable = await sqlReadableDocIds({
-    workspaceId: workspace.id,
-    userId: member.id,
-    docIds,
-  });
-  t.deepEqual(sqlReadable, ['missing-policy', 'public-doc']);
-});
-
-test('SQL doc read predicate handles non-member grant and sharing disabled', async t => {
-  const owner = await module.create(Mockers.User);
-  const nonMember = await module.create(Mockers.User);
-  const workspace = await module.create(Mockers.Workspace, {
-    owner,
-  });
-  await resetProjection(workspace.id);
-  await db.$executeRaw`
-    INSERT INTO doc_access_policies (
-      workspace_id,
-      doc_id,
-      visibility,
-      public_role,
-      member_default_role,
-      updated_at
-    )
-    VALUES
-      (${workspace.id}, 'public-doc', 'public', 'external', NULL, now()),
-      (${workspace.id}, 'private-doc', 'private', NULL, NULL, now()),
-      (${workspace.id}, 'explicit-grant', 'private', NULL, NULL, now()),
-      (${workspace.id}, 'explicit-owner-grant', 'private', NULL, NULL, now())
-  `;
-  await db.$executeRaw`
-    INSERT INTO doc_grants (
-      workspace_id,
-      doc_id,
-      principal_type,
-      principal_id,
-      role,
-      updated_at
-    )
-    VALUES
-      (
-        ${workspace.id},
-        'explicit-grant',
-        'user',
-        ${nonMember.id},
-        'reader',
-        now()
-      ),
-      (
-        ${workspace.id},
-        'explicit-owner-grant',
-        'user',
-        ${nonMember.id},
-        'owner',
-        now()
-      )
-  `;
-
-  const docIds = [
-    'public-doc',
-    'private-doc',
-    'explicit-grant',
-    'explicit-owner-grant',
-  ];
-  const sharingEnabledReadable = await sqlReadableDocIds({
-    workspaceId: workspace.id,
-    userId: nonMember.id,
-    docIds,
-  });
-  const sharingEnabledUpdate = await sqlReadableDocIds({
-    workspaceId: workspace.id,
-    userId: nonMember.id,
-    action: 'Doc.Update',
-    docIds,
-  });
-
-  await db.$executeRaw`
-    UPDATE workspace_access_policies
-    SET sharing_enabled = false
-    WHERE workspace_id = ${workspace.id}
-  `;
-  const sharingDisabledReadable = await sqlReadableDocIds({
-    workspaceId: workspace.id,
-    userId: nonMember.id,
-    docIds,
-  });
-
-  t.deepEqual(sharingEnabledReadable, [
-    'public-doc',
-    'explicit-grant',
-    'explicit-owner-grant',
-  ]);
-  t.deepEqual(sharingEnabledUpdate, ['explicit-owner-grant']);
-  t.deepEqual(sharingDisabledReadable, []);
-});
-
-test('SQL doc predicate suppresses member default when explicit grant exists', async t => {
-  const owner = await module.create(Mockers.User);
-  const member = await module.create(Mockers.User);
-  const workspace = await module.create(Mockers.Workspace, {
-    owner,
-  });
-  await resetProjection(workspace.id);
-  await db.$executeRaw`
-    UPDATE workspace_access_policies
-    SET member_default_doc_role = 'manager'
-    WHERE workspace_id = ${workspace.id}
-  `;
-  await db.$executeRaw`
-    INSERT INTO workspace_members (
-      workspace_id,
-      user_id,
-      role,
-      state,
-      source,
-      updated_at
-    )
-    VALUES (${workspace.id}, ${member.id}, 'member', 'active', 'legacy', now())
-  `;
-  await db.$executeRaw`
-    INSERT INTO doc_access_policies (
-      workspace_id,
-      doc_id,
-      visibility,
-      public_role,
-      member_default_role,
-      updated_at
-    )
-    VALUES
-      (${workspace.id}, 'default-manager', 'private', NULL, NULL, now()),
-      (${workspace.id}, 'explicit-reader', 'private', NULL, NULL, now())
-  `;
-  await db.$executeRaw`
-    INSERT INTO doc_grants (
-      workspace_id,
-      doc_id,
-      principal_type,
-      principal_id,
-      role,
-      updated_at
-    )
-    VALUES (
-      ${workspace.id},
-      'explicit-reader',
-      'user',
-      ${member.id},
-      'reader',
-      now()
-    )
-  `;
-
-  const docIds = ['default-manager', 'explicit-reader'];
-  const sqlUpdateAllowed = await sqlReadableDocIds({
-    workspaceId: workspace.id,
-    userId: member.id,
-    action: 'Doc.Update',
-    docIds,
-  });
-
-  t.deepEqual(sqlUpdateAllowed, ['default-manager']);
-});
-
-test('should filter docs by Doc.Publish', async t => {
-  const owner = await module.create(Mockers.User);
-  const workspace = await module.create(Mockers.Workspace, {
-    owner,
-  });
-  await models.workspace.update(workspace.id, { enableSharing: true });
-  await setWritableRuntime(workspace.id);
-
-  const docs1 = await builder
-    .user(owner.id)
-    .workspace(workspace.id)
-    .docs(
-      [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }],
-      'Doc.Publish'
-    );
-
-  t.is(docs1.length, 3);
-  t.snapshot(docs1);
-
-  // member should have access to the docs
-  const member = await module.create(Mockers.User);
-  await module.create(Mockers.WorkspaceUser, {
-    workspaceId: workspace.id,
-    userId: member.id,
-    type: WorkspaceRole.Collaborator,
-  });
-
-  await module.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'doc1',
-    userId: member.id,
-    type: DocRole.Reader,
-  });
-
-  await module.create(Mockers.DocUser, {
-    workspaceId: workspace.id,
-    docId: 'doc2',
-    userId: member.id,
-    type: DocRole.Manager,
-  });
-
-  const docs2 = await builder
-    .user(member.id)
-    .workspace(workspace.id)
-    .docs(
-      [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }],
-      'Doc.Publish'
-    );
-
-  t.is(docs2.length, 2);
-  t.snapshot(docs2);
-
-  // other user should not have access to the docs
-  const other = await module.create(Mockers.User);
-
-  const docs3 = await builder
-    .user(other.id)
-    .workspace(workspace.id)
-    .docs(
-      [{ docId: 'doc1' }, { docId: 'doc2' }, { docId: 'doc3' }],
-      'Doc.Publish'
-    );
-
-  t.is(docs3.length, 0);
-});

packages/backend/server/src/core/permission/__tests__/service.spec.ts

@@ -5,7 +5,6 @@ import { DocRole } from '../../../models';
 import { docLegacyBoundary } from '../context';
 import { PermissionContextLoader } from '../context-loader';
 import { PermissionService } from '../service';
-import { PermissionSqlPredicateBuilder } from '../sql-predicate';
 
 function createCls() {
   const store = new Map<string, unknown>();
@@ -262,68 +261,3 @@ test('PermissionService maps native validation errors to internal errors', t =>
 
   t.true(error instanceof InternalServerError);
 });
-
-test('PermissionSqlPredicateBuilder rejects unsafe raw doc id columns', t => {
-  const builder = new PermissionSqlPredicateBuilder();
-
-  t.throws(
-    () =>
-      builder.docReadable({
-        workspaceId: 'w1',
-        userId: 'u1',
-        action: 'Doc.Read',
-        docIdColumn: 'docs.id; DROP TABLE docs' as never,
-      }),
-    { message: 'Unsupported doc id column: docs.id; DROP TABLE docs' }
-  );
-});
-
-test('PermissionSqlPredicateBuilder caps non-member grants below manager', t => {
-  const builder = new PermissionSqlPredicateBuilder();
-  const update = builder.docReadable({
-    workspaceId: 'w1',
-    userId: 'u1',
-    action: 'Doc.Update',
-  });
-  const transferOwner = builder.docReadable({
-    workspaceId: 'w1',
-    userId: 'u1',
-    action: 'Doc.TransferOwner',
-  });
-
-  t.true((update.params[4] as string[]).includes('editor'));
-  t.true((update.params[4] as string[]).includes('manager'));
-  t.true((update.params[4] as string[]).includes('owner'));
-  t.deepEqual(transferOwner.params[3], ['owner']);
-  t.deepEqual(transferOwner.params[4], []);
-});
-
-test('PermissionSqlPredicateBuilder uses terminal permission tables', t => {
-  const predicate = new PermissionSqlPredicateBuilder().docReadable({
-    workspaceId: 'w1',
-    userId: 'u1',
-    action: 'Doc.Read',
-    docIdColumn: 'docs.id',
-  });
-
-  t.true(predicate.sql.includes('FROM workspace_access_policies wap'));
-  t.true(predicate.sql.includes('LEFT JOIN doc_access_policies dap'));
-  t.true(predicate.sql.includes('workspace_members'));
-  t.true(predicate.sql.includes('doc_grants'));
-  t.false(predicate.sql.includes('workspace_user_permissions'));
-  t.false(predicate.sql.includes('workspace_page_user_permissions'));
-});
-
-test('PermissionService always uses the terminal SQL predicate', t => {
-  const predicate = new PermissionService(
-    createLoader().loader
-  ).docReadableSqlPredicate({
-    workspaceId: 'w1',
-    userId: 'u1',
-    action: 'Doc.Read',
-  });
-  const sql = (predicate as unknown as { sql: string }).sql;
-
-  t.true(sql.includes('workspace_access_policies'));
-  t.false(sql.includes('workspace_user_permissions'));
-});

packages/backend/server/src/core/permission/index.ts

@@ -6,7 +6,6 @@ import { PermissionContextLoader } from './context-loader';
 import { EventsListener } from './event';
 import { WorkspacePolicyService } from './policy';
 import { PermissionService } from './service';
-import { PermissionSqlPredicateBuilder } from './sql-predicate';
 
 @Module({
   imports: [QuotaServiceModule],
@@ -14,16 +13,10 @@ import { PermissionSqlPredicateBuilder } from './sql-predicate';
     AccessControllerBuilder,
     EventsListener,
     WorkspacePolicyService,
-    PermissionSqlPredicateBuilder,
     PermissionContextLoader,
     PermissionService,
   ],
-  exports: [
-    AccessControllerBuilder,
-    WorkspacePolicyService,
-    PermissionSqlPredicateBuilder,
-    PermissionService,
-  ],
+  exports: [AccessControllerBuilder, WorkspacePolicyService, PermissionService],
 })
 export class PermissionModule {}
 
@@ -35,7 +28,6 @@ export {
 } from './permission-map';
 export { WorkspacePolicyService } from './policy';
 export { PermissionService } from './service';
-export { PermissionSqlPredicateBuilder } from './sql-predicate';
 export {
   DOC_ACTIONS,
   type DocAction,

packages/backend/server/src/core/permission/service.ts

@@ -1,5 +1,4 @@
-import { Inject, Injectable, Optional } from '@nestjs/common';
-import { Prisma } from '@prisma/client';
+import { Injectable, Optional } from '@nestjs/common';
 
 import {
   DocActionDenied,
@@ -18,8 +17,6 @@ import {
   type PermissionWorkspaceAction,
 } from './context-loader';
 import { WorkspacePolicyService } from './policy';
-import { PermissionSqlPredicateBuilder } from './sql-predicate';
-import type { DocAction } from './types';
 
 const RUNTIME_RESTRICTED_WORKSPACE_ACTIONS = new Set<PermissionWorkspaceAction>(
   [
@@ -59,21 +56,9 @@ export class PermissionService {
   constructor(
     private readonly loader: PermissionContextLoader,
     @Optional()
-    @Inject(PermissionSqlPredicateBuilder)
-    private readonly sqlPredicate = new PermissionSqlPredicateBuilder(),
-    @Optional()
     private readonly workspacePolicy?: WorkspacePolicyService
   ) {}
 
-  docReadableSqlPredicate(input: {
-    userId: string;
-    workspaceId: string;
-    action: DocAction;
-    docIdColumn?: Prisma.Sql;
-  }) {
-    return this.sqlPredicate.docReadableSql(input);
-  }
-
   evaluate(input: PermissionEvaluationInputV1) {
     try {
       return evaluatePermissionV1(input);

packages/backend/server/src/core/permission/sql-predicate.ts

@@ -1,148 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { Prisma } from '@prisma/client';
-
-import { permissionActionRoleMatrixV1 } from '../../native';
-import type { DocAction } from './types';
-
-export type PermissionSqlPredicate = {
-  sql: string;
-  params: unknown[];
-};
-
-type RawDocIdColumn = 'doc_id' | 'docs.id';
-
-@Injectable()
-export class PermissionSqlPredicateBuilder {
-  private readonly matrix = permissionActionRoleMatrixV1() as {
-    doc?: { roles?: Record<string, string[]> };
-    workspace?: { roles?: Record<string, string[]> };
-  };
-
-  private docRolesForAction(action: DocAction) {
-    return Object.entries(this.matrix.doc?.roles ?? {})
-      .filter(([, actions]) => actions.includes(action))
-      .map(([role]) => role)
-      .filter(role => role !== 'none');
-  }
-
-  private inheritedWorkspaceRolesForDocAction(action: DocAction) {
-    const docRoles = new Set(this.docRolesForAction(action));
-    return [
-      docRoles.has('owner') ? 'owner' : null,
-      docRoles.has('manager') ? 'admin' : null,
-    ].filter((role): role is string => role !== null);
-  }
-
-  private nonMemberDocGrantRolesForAction(action: DocAction) {
-    const roles = new Set(this.docRolesForAction(action));
-    roles.delete('external');
-    roles.delete('manager');
-    roles.delete('owner');
-    if (roles.has('editor')) {
-      roles.add('manager');
-      roles.add('owner');
-    }
-    return [...roles];
-  }
-
-  private rawDocIdColumn(column: RawDocIdColumn = 'doc_id') {
-    switch (column) {
-      case 'doc_id':
-      case 'docs.id':
-        return column;
-      default:
-        throw new Error(`Unsupported doc id column: ${column}`);
-    }
-  }
-
-  docReadable(input: {
-    workspaceId: string;
-    userId?: string;
-    action: DocAction;
-    docIdColumn?: RawDocIdColumn;
-  }): PermissionSqlPredicate {
-    const docRoles = this.docRolesForAction(input.action);
-    const inheritedWorkspaceRoles = this.inheritedWorkspaceRolesForDocAction(
-      input.action
-    );
-    const grantRoles = docRoles.filter(role => role !== 'external');
-    const nonMemberGrantRoles = this.nonMemberDocGrantRolesForAction(
-      input.action
-    );
-    const docIdColumn = this.rawDocIdColumn(input.docIdColumn);
-
-    return {
-      sql: [
-        `EXISTS (SELECT 1 FROM workspace_access_policies wap`,
-        `LEFT JOIN doc_access_policies dap ON dap.workspace_id = wap.workspace_id`,
-        `AND dap.doc_id = ${docIdColumn}`,
-        `LEFT JOIN workspace_members wm ON wm.workspace_id = wap.workspace_id`,
-        `AND wm.user_id = ? AND wm.state = 'active'`,
-        `LEFT JOIN doc_grants dg ON dg.workspace_id = wap.workspace_id`,
-        `AND dg.doc_id = ${docIdColumn} AND dg.principal_type = 'user' AND dg.principal_id = ?`,
-        `WHERE wap.workspace_id = ?`,
-        `AND (`,
-        `(wm.id IS NOT NULL AND dg.role = ANY(?::text[]))`,
-        `OR (wm.id IS NULL AND wap.sharing_enabled AND dg.role = ANY(?::text[]))`,
-        `OR wm.role = ANY(?::text[])`,
-        `OR (wm.id IS NOT NULL AND dg.principal_id IS NULL AND COALESCE(dap.member_default_role, wap.member_default_doc_role) = ANY(?::text[]))`,
-        `OR (wap.sharing_enabled AND dap.visibility = 'public' AND dap.public_role = ANY(?::text[]))`,
-        `))`,
-      ].join(' '),
-      params: [
-        input.userId,
-        input.userId,
-        input.workspaceId,
-        grantRoles,
-        nonMemberGrantRoles,
-        inheritedWorkspaceRoles,
-        grantRoles,
-        docRoles,
-      ],
-    };
-  }
-
-  docReadableSql(input: {
-    workspaceId: string;
-    userId?: string;
-    action: DocAction;
-    docIdColumn?: Prisma.Sql;
-  }): Prisma.Sql {
-    const docRoles = this.docRolesForAction(input.action);
-    const grantRoles = docRoles.filter(role => role !== 'external');
-    const nonMemberGrantRoles = this.nonMemberDocGrantRolesForAction(
-      input.action
-    );
-    const inheritedWorkspaceRoles = this.inheritedWorkspaceRolesForDocAction(
-      input.action
-    );
-    const docIdColumn = input.docIdColumn ?? Prisma.raw('doc_id');
-
-    return Prisma.sql`
-      EXISTS (
-        SELECT 1
-        FROM workspace_access_policies wap
-        LEFT JOIN doc_access_policies dap
-          ON dap.workspace_id = wap.workspace_id
-         AND dap.doc_id = ${docIdColumn}
-        LEFT JOIN workspace_members wm
-          ON wm.workspace_id = wap.workspace_id
-         AND wm.user_id = ${input.userId}
-         AND wm.state = 'active'
-        LEFT JOIN doc_grants dg
-          ON dg.workspace_id = wap.workspace_id
-         AND dg.doc_id = ${docIdColumn}
-         AND dg.principal_type = 'user'
-         AND dg.principal_id = ${input.userId}
-        WHERE wap.workspace_id = ${input.workspaceId}
-          AND (
-            (wm.id IS NOT NULL AND dg.role = ANY(${Prisma.sql`${grantRoles}::text[]`}))
-            OR (wm.id IS NULL AND wap.sharing_enabled AND dg.role = ANY(${Prisma.sql`${nonMemberGrantRoles}::text[]`}))
-            OR wm.role = ANY(${Prisma.sql`${inheritedWorkspaceRoles}::text[]`})
-            OR (wm.id IS NOT NULL AND dg.principal_id IS NULL AND COALESCE(dap.member_default_role, wap.member_default_doc_role) = ANY(${Prisma.sql`${grantRoles}::text[]`}))
-            OR (wap.sharing_enabled AND dap.visibility = 'public' AND dap.public_role = ANY(${Prisma.sql`${docRoles}::text[]`}))
-          )
-      )
-    `;
-  }
-}

packages/backend/server/src/core/storage-runtime/__tests__/provider.spec.ts

@@ -51,18 +51,20 @@ test('storage-runtime provider restarts on storage config changes', async t => {
   const { provider, runtime } = createProvider();
 
   await provider.start();
+  await provider.runMigrations();
   await provider.onConfigChanged({ updates: { storages: {} } });
 
   t.is(runtime.stop.callCount, 1);
   t.is(runtime.configure.callCount, 2);
   t.is(runtime.start.callCount, 2);
-  t.is(runtime.runMigrations.callCount, 2);
+  t.is(runtime.runMigrations.callCount, 1);
 });
 
 test('storage-runtime provider restarts on copilot storage config changes', async t => {
   const { provider, runtime } = createProvider();
 
   await provider.start();
+  await provider.runMigrations();
   await provider.onConfigChanged({
     updates: {
       copilot: {
@@ -78,7 +80,7 @@ test('storage-runtime provider restarts on copilot storage config changes', asyn
   t.is(runtime.stop.callCount, 1);
   t.is(runtime.configure.callCount, 2);
   t.is(runtime.start.callCount, 2);
-  t.is(runtime.runMigrations.callCount, 2);
+  t.is(runtime.runMigrations.callCount, 1);
 });
 
 test('storage-runtime provider ignores unrelated config changes', async t => {
@@ -89,5 +91,5 @@ test('storage-runtime provider ignores unrelated config changes', async t => {
 
   t.is(runtime.stop.callCount, 0);
   t.is(runtime.start.callCount, 1);
-  t.is(runtime.runMigrations.callCount, 1);
+  t.is(runtime.runMigrations.callCount, 0);
 });

packages/backend/server/src/core/storage-runtime/provider.ts

@@ -49,7 +49,6 @@ export class StorageRuntimeProvider
   async start() {
     this.configureRuntime();
     await this.runtime.start();
-    await this.runMigrationsOnce();
     const health = await this.runtime.health();
     this.logger.log(
       `storage runtime started: db=${health.databaseConnected} provider=${health.provider ?? 'none'}`
@@ -82,6 +81,10 @@ export class StorageRuntimeProvider
     return await this.runtime.health();
   }
 
+  async runMigrations() {
+    await this.runMigrationsOnce();
+  }
+
   async providerCapabilities(
     scope: string
   ): Promise<StorageProviderCapabilities> {
@@ -258,6 +261,16 @@ export class StorageRuntimeProvider
     );
   }
 
+  async rebuildDocBlobRefs(
+    workspaceId: string,
+    docId: string,
+    sourceRevision: number
+  ) {
+    return await this.measured('rebuildDocBlobRefs', rt =>
+      rt.rebuildDocBlobRefs(workspaceId, docId, sourceRevision)
+    );
+  }
+
   async reconcileWorkspaceDocuments(workspaceId: string) {
     return await this.measured('reconcileWorkspaceDocuments', rt =>
       rt.reconcileWorkspaceDocuments(workspaceId)

packages/backend/server/src/core/storage/__tests__/blob-job.spec.ts

@@ -8,6 +8,7 @@ interface Context {
     health: Sinon.SinonStub;
     reconcileWorkspaceDocuments: Sinon.SinonStub;
     backfillMissingBlobMetadata: Sinon.SinonStub;
+    rebuildDocBlobRefs: Sinon.SinonStub;
     rebuildWorkspaceDocBlobRefs: Sinon.SinonStub;
     planUnreferencedWorkspaceBlobs: Sinon.SinonStub;
     executeBlobCleanupCandidates: Sinon.SinonStub;
@@ -45,6 +46,7 @@ test.beforeEach(t => {
       recovered: 0,
     }),
     backfillMissingBlobMetadata: Sinon.stub(),
+    rebuildDocBlobRefs: Sinon.stub(),
     rebuildWorkspaceDocBlobRefs: Sinon.stub(),
     planUnreferencedWorkspaceBlobs: Sinon.stub(),
     executeBlobCleanupCandidates: Sinon.stub(),
@@ -289,6 +291,31 @@ test('storage reconciliation still refreshes document retention without object s
   t.false(t.context.runtime.planUnreferencedWorkspaceBlobs.called);
 });
 
+test('document projection worker drains metadata incrementally after a document merge', async t => {
+  t.context.runtime.rebuildDocBlobRefs.resolves({
+    scannedDocs: 1,
+    parsedDocs: 1,
+    refsWritten: 1,
+    refsDeleted: 0,
+    failedDocs: 0,
+    nextCursor: null,
+  });
+
+  await t.context.job.projectWorkspaceDocBlobRefs({
+    workspaceId: 'workspace-1',
+    docId: 'doc-1',
+    sourceRevision: 123,
+  });
+
+  t.true(
+    t.context.runtime.rebuildDocBlobRefs.calledOnceWith(
+      'workspace-1',
+      'doc-1',
+      123
+    )
+  );
+});
+
 test('document cleanup dispatches stable search effects', async t => {
   t.context.runtime.executeDocumentCleanupCandidates.resolves({
     scannedCandidates: 1,

packages/backend/server/src/core/storage/blob-job.ts

@@ -5,8 +5,8 @@ import { PrismaClient } from '@prisma/client';
 import { EventBus, JobQueue, metrics, OnJob } from '../../base';
 import { StorageRuntimeProvider } from '../storage-runtime';
 
-// Queue keys are persisted API; keep the legacy backendRuntime.* names while
-// StorageBlobJob and StorageRuntimeProvider own the implementation.
+// Queue keys are persisted API; StorageBlobJob and StorageRuntimeProvider own
+// the implementation.
 declare global {
   interface Jobs {
     'backendRuntime.backfillMissingBlobMetadata': {
@@ -23,6 +23,11 @@ declare global {
       workspaceLimit?: number;
       docLimit?: number;
     };
+    'backendRuntime.projectWorkspaceDocBlobRefs': {
+      workspaceId: string;
+      docId: string;
+      sourceRevision: number;
+    };
     'backendRuntime.executeDocumentCleanupCandidates': {
       workspaceId?: string;
       gracePeriodDays?: number;
@@ -281,6 +286,23 @@ export class StorageBlobJob {
     }
   }
 
+  @OnJob('backendRuntime.projectWorkspaceDocBlobRefs')
+  async projectWorkspaceDocBlobRefs({
+    workspaceId,
+    docId,
+    sourceRevision,
+  }: Jobs['backendRuntime.projectWorkspaceDocBlobRefs']) {
+    const result = await this.rt.rebuildDocBlobRefs(
+      workspaceId,
+      docId,
+      sourceRevision
+    );
+    this.autoLog(
+      `projected doc blob refs workspace=${workspaceId} doc=${docId} sourceRevision=${sourceRevision} parsed=${result.parsedDocs} failed=${result.failedDocs}`,
+      Boolean(result.failedDocs)
+    );
+  }
+
   @OnJob('backendRuntime.executeDocumentCleanupCandidates')
   async executeDocumentCleanupCandidates({
     workspaceId,

packages/backend/server/src/core/storage/index.ts

@@ -14,22 +14,23 @@ import {
 
 @Module({
   imports: [StorageRuntimeModule],
-  controllers: [R2UploadController],
-  providers: [
-    WorkspaceBlobStorage,
-    AvatarStorage,
-    CommentAttachmentStorage,
-    StorageBlobJob,
-    BlobUploadCleanupJob,
-  ],
-  exports: [
-    WorkspaceBlobStorage,
-    AvatarStorage,
-    CommentAttachmentStorage,
-    StorageBlobJob,
-  ],
+  providers: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
+  exports: [WorkspaceBlobStorage, AvatarStorage, CommentAttachmentStorage],
 })
 export class StorageModule {}
 
+@Module({
+  imports: [StorageModule],
+  controllers: [R2UploadController],
+})
+export class StorageApiModule {}
+
+@Module({
+  imports: [StorageModule],
+  providers: [StorageBlobJob, BlobUploadCleanupJob],
+  exports: [StorageBlobJob],
+})
+export class StorageWorkerModule {}
+
 export { StorageBlobJob } from './blob-job';
 export { AvatarStorage, CommentAttachmentStorage, WorkspaceBlobStorage };

packages/backend/server/src/core/storage/job.ts

@@ -6,7 +6,7 @@ import { StorageRuntimeProvider } from '../storage-runtime';
 
 declare global {
   interface Jobs {
-    'nightly.cleanExpiredPendingBlobs': {};
+    'backendRuntime.cleanExpiredPendingBlobs': {};
   }
 }
 
@@ -23,7 +23,7 @@ export class BlobUploadCleanupJob {
   @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
   async nightlyJob() {
     await this.queue.add(
-      'nightly.cleanExpiredPendingBlobs',
+      'backendRuntime.cleanExpiredPendingBlobs',
       {},
       {
         jobId: 'nightly-blob-clean-expired-pending',
@@ -31,7 +31,7 @@ export class BlobUploadCleanupJob {
     );
   }
 
-  @OnJob('nightly.cleanExpiredPendingBlobs')
+  @OnJob('backendRuntime.cleanExpiredPendingBlobs')
   async cleanExpiredPendingBlobs() {
     const cutoff = Date.now() - OneDay;
     let scanned = 0;

packages/backend/server/src/core/sync/gateway.ts

@@ -20,10 +20,12 @@ import semver from 'semver';
 import { type Server, Socket } from 'socket.io';
 
 import {
+  BadRequest,
   CallMetric,
   checkCanaryDateClientVersion,
   DocNotFound,
   DocUpdateBlocked,
+  EventBus,
   GatewayErrorWrapper,
   metrics,
   NotInSpace,
@@ -63,9 +65,9 @@ type EventResponse<Data = any> = Data extends never
     };
 
 // sync: shared room for space membership checks and non-protocol broadcasts.
-// sync-025: legacy 0.25 doc sync protocol (space:broadcast-doc-update).
-// sync-026: current doc sync protocol (space:broadcast-doc-updates).
-type RoomType = 'sync' | 'sync-025' | 'sync-026' | `${string}:awareness`;
+// sync-026: legacy doc sync protocol (space:broadcast-doc-updates).
+// sync-027: batch doc sync protocol (invalidation + active subscriptions).
+type RoomType = 'sync' | 'sync-026' | 'sync-027' | `${string}:awareness`;
 
 function Room(
   spaceId: string,
@@ -74,14 +76,14 @@ function Room(
   return `${spaceId}:${type}`;
 }
 
-const MIN_WS_CLIENT_VERSION = new semver.Range('>=0.25.0', {
+const MIN_WS_CLIENT_VERSION = new semver.Range('>=0.26.0', {
   includePrerelease: true,
 });
-const DOC_UPDATES_PROTOCOL_026 = new semver.Range('>=0.26.0-0', {
+const MIN_BATCH_WS_CLIENT_VERSION = new semver.Range('>=0.27.5-0', {
   includePrerelease: true,
 });
+const MAX_SPACE_JOIN_BATCH_SIZE = 100;
 
-type SyncProtocolRoomType = Extract<RoomType, 'sync-025' | 'sync-026'>;
 const SOCKET_PRESENCE_USER_ID_KEY = 'affinePresenceUserId';
 
 function normalizeWsClientVersion(clientVersion: string): string | null {
@@ -108,11 +110,9 @@ function isSupportedWsClientVersion(clientVersion: string): boolean {
   );
 }
 
-function getSyncProtocolRoomType(clientVersion: string): SyncProtocolRoomType {
+function isBatchWsClientVersion(clientVersion: string): boolean {
   const normalized = normalizeWsClientVersion(clientVersion);
-  return DOC_UPDATES_PROTOCOL_026.test(normalized ?? clientVersion)
-    ? 'sync-026'
-    : 'sync-025';
+  return Boolean(normalized && MIN_BATCH_WS_CLIENT_VERSION.test(normalized));
 }
 
 enum SpaceType {
@@ -133,11 +133,26 @@ interface JoinSpaceAwarenessMessage {
   clientVersion: string;
 }
 
+interface JoinSpaceBatchEntry {
+  spaceType: SpaceType;
+  spaceId: string;
+  docId?: string;
+}
+
+interface JoinSpaceBatchMessage {
+  spaces: [JoinSpaceBatchEntry, ...JoinSpaceBatchEntry[]];
+  clientVersion: string;
+}
+
 interface LeaveSpaceMessage {
   spaceType: SpaceType;
   spaceId: string;
 }
 
+interface LeaveSpaceBatchMessage extends LeaveSpaceMessage {
+  docIds: string[];
+}
+
 interface LeaveSpaceAwarenessMessage {
   spaceType: SpaceType;
   spaceId: string;
@@ -161,15 +176,6 @@ interface BroadcastDocUpdatesMessage {
   compressed?: boolean;
 }
 
-interface BroadcastDocUpdateMessage {
-  spaceType: SpaceType;
-  spaceId: string;
-  docId: string;
-  update: string;
-  timestamp: number;
-  editor: string;
-}
-
 interface LoadDocMessage {
   spaceType: SpaceType;
   spaceId: string;
@@ -201,6 +207,135 @@ interface UpdateAwarenessMessage {
   awarenessUpdate: string;
 }
 
+interface SyncAwarenessEvent {
+  spaceType: SpaceType;
+  spaceId: string;
+  docId: string;
+  sourceSocketId?: string;
+}
+
+interface SyncDocUpdatesPayload {
+  spaceType: SpaceType;
+  spaceId: string;
+  docId: string;
+  updates: Uint8Array[];
+  timestamp: number;
+  editor?: string;
+}
+
+declare global {
+  interface Events {
+    'sync.doc.updates.pushed': {
+      spaceType: SpaceType;
+      spaceId: string;
+      docId: string;
+      updates: string[];
+      timestamp: number;
+      editor?: string;
+    };
+    'sync.awareness.collect': SyncAwarenessEvent;
+    'sync.awareness.updated': SyncAwarenessEvent & {
+      awarenessUpdate: string;
+    };
+    'sync.permissions.changed': {
+      spaceType: SpaceType;
+      spaceId: string;
+      docId?: string;
+    };
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}
+
+function parseJoinSpaceBatchMessage(message: unknown): JoinSpaceBatchMessage {
+  if (!isRecord(message)) {
+    throw new BadRequest('Invalid space join batch payload.');
+  }
+
+  const { spaces, clientVersion } = message;
+  if (!Array.isArray(spaces) || spaces.length === 0) {
+    throw new BadRequest('Space join batch must not be empty.');
+  }
+  if (spaces.length > MAX_SPACE_JOIN_BATCH_SIZE) {
+    throw new BadRequest(
+      `Space join batch exceeds limit (${MAX_SPACE_JOIN_BATCH_SIZE}).`
+    );
+  }
+  if (typeof clientVersion !== 'string' || clientVersion.length === 0) {
+    throw new BadRequest('Space join batch requires a client version.');
+  }
+
+  const entries = spaces.map((space, index) => {
+    if (!isRecord(space)) {
+      throw new BadRequest(`Invalid space join batch entry at index ${index}.`);
+    }
+
+    const { spaceType, spaceId, docId } = space;
+    if (
+      (spaceType !== SpaceType.Userspace &&
+        spaceType !== SpaceType.Workspace) ||
+      typeof spaceId !== 'string' ||
+      spaceId.trim().length === 0 ||
+      (docId !== undefined &&
+        (typeof docId !== 'string' || docId.trim().length === 0))
+    ) {
+      throw new BadRequest(`Invalid space join batch entry at index ${index}.`);
+    }
+
+    return {
+      spaceType,
+      spaceId,
+      ...(docId === undefined ? {} : { docId }),
+    } satisfies JoinSpaceBatchEntry;
+  }) as [JoinSpaceBatchEntry, ...JoinSpaceBatchEntry[]];
+
+  const first = entries[0];
+  const duplicateKeys = new Set<string>();
+  for (const entry of entries) {
+    if (
+      entry.spaceType !== first.spaceType ||
+      entry.spaceId !== first.spaceId
+    ) {
+      throw new BadRequest(
+        'Space join batch entries must belong to one space.'
+      );
+    }
+
+    const key = JSON.stringify([
+      entry.spaceType,
+      entry.spaceId,
+      entry.docId ?? null,
+    ]);
+    if (duplicateKeys.has(key)) {
+      throw new BadRequest('Space join batch contains duplicate entries.');
+    }
+    duplicateKeys.add(key);
+  }
+
+  return { spaces: entries, clientVersion };
+}
+
+function parseLeaveSpaceBatchMessage(message: unknown): LeaveSpaceBatchMessage {
+  if (!isRecord(message)) {
+    throw new BadRequest('Invalid space leave batch payload.');
+  }
+
+  const { spaceType, spaceId, docIds } = message;
+  if (
+    (spaceType !== SpaceType.Userspace && spaceType !== SpaceType.Workspace) ||
+    typeof spaceId !== 'string' ||
+    spaceId.trim().length === 0 ||
+    !Array.isArray(docIds) ||
+    docIds.some(docId => typeof docId !== 'string' || docId.trim().length === 0)
+  ) {
+    throw new BadRequest('Invalid space leave batch payload.');
+  }
+
+  return { spaceType, spaceId, docIds };
+}
+
 @WebSocketGateway()
 @UseInterceptors(ClsInterceptor)
 export class SpaceSyncGateway
@@ -223,13 +358,16 @@ export class SpaceSyncGateway
   private activeUsersFlushTimer?: NodeJS.Timeout;
   private activeUsersFlushInFlight = false;
   private activeUsersFlushQueued = false;
+  private readonly activeDocSockets = new Map<string, Set<Socket>>();
+  private readonly activeSocketDocs = new Map<string, Set<string>>();
 
   constructor(
     private readonly ac: PermissionAccess,
     private readonly workspace: PgWorkspaceDocStorageAdapter,
     private readonly userspace: PgUserspaceDocStorageAdapter,
     private readonly docReader: DocReader,
-    private readonly models: Models
+    private readonly models: Models,
+    private readonly event: EventBus
   ) {}
 
   onModuleInit() {
@@ -345,6 +483,166 @@ export class SpaceSyncGateway
     }
   }
 
+  private activeDocKey(spaceType: SpaceType, spaceId: string, docId: string) {
+    return `${spaceType}:${spaceId}:${docId}`;
+  }
+
+  private addActiveDocSubscription(
+    client: Socket,
+    spaceType: SpaceType,
+    spaceId: string,
+    docId: string
+  ) {
+    const key = this.activeDocKey(spaceType, spaceId, docId);
+    let sockets = this.activeDocSockets.get(key);
+    if (!sockets) {
+      sockets = new Set();
+      this.activeDocSockets.set(key, sockets);
+    }
+    sockets.add(client);
+
+    let docs = this.activeSocketDocs.get(client.id);
+    if (!docs) {
+      docs = new Set();
+      this.activeSocketDocs.set(client.id, docs);
+    }
+    docs.add(key);
+  }
+
+  private removeActiveDocSubscription(
+    client: Socket,
+    spaceType: SpaceType,
+    spaceId: string,
+    docId: string
+  ) {
+    const key = this.activeDocKey(spaceType, spaceId, docId);
+    const sockets = this.activeDocSockets.get(key);
+    sockets?.delete(client);
+    if (sockets && sockets.size === 0) {
+      this.activeDocSockets.delete(key);
+    }
+
+    const docs = this.activeSocketDocs.get(client.id);
+    docs?.delete(key);
+    if (docs && docs.size === 0) {
+      this.activeSocketDocs.delete(client.id);
+    }
+  }
+
+  private removeAllActiveDocSubscriptions(client: Socket) {
+    const docs = this.activeSocketDocs.get(client.id);
+    if (!docs) {
+      return;
+    }
+
+    for (const key of docs) {
+      const sockets = this.activeDocSockets.get(key);
+      sockets?.delete(client);
+      if (sockets && sockets.size === 0) {
+        this.activeDocSockets.delete(key);
+      }
+    }
+    this.activeSocketDocs.delete(client.id);
+  }
+
+  private removeActiveDocSubscriptionsInSpace(
+    client: Socket,
+    spaceType: SpaceType,
+    spaceId: string
+  ) {
+    const prefix = `${spaceType}:${spaceId}:`;
+    for (const key of Array.from(this.activeSocketDocs.get(client.id) ?? [])) {
+      if (!key.startsWith(prefix)) {
+        continue;
+      }
+      const [, keySpaceId, docId] = key.split(':');
+      this.removeActiveDocSubscription(client, spaceType, keySpaceId, docId);
+    }
+  }
+
+  private hasActiveDocSubscription(
+    client: Socket,
+    spaceType: SpaceType,
+    spaceId: string,
+    docId: string
+  ) {
+    return Boolean(
+      this.activeSocketDocs
+        .get(client.id)
+        ?.has(this.activeDocKey(spaceType, spaceId, docId))
+    );
+  }
+
+  private emitActiveDocUpdate(
+    payload: SyncDocUpdatesPayload,
+    sourceSocketId?: string,
+    broadcastPayload?: BroadcastDocUpdatesMessage
+  ) {
+    const sockets = this.activeDocSockets.get(
+      this.activeDocKey(payload.spaceType, payload.spaceId, payload.docId)
+    );
+    if (!sockets) {
+      return;
+    }
+
+    const activeBroadcastPayload =
+      broadcastPayload ??
+      this.buildBroadcastPayload(
+        payload.spaceType,
+        payload.spaceId,
+        payload.docId,
+        payload.updates,
+        payload.timestamp,
+        payload.editor
+      );
+    for (const socket of sockets) {
+      if (socket.id !== sourceSocketId) {
+        socket.emit('space:broadcast-doc-updates', activeBroadcastPayload);
+      }
+    }
+  }
+
+  private emitActiveAwarenessCollect(event: SyncAwarenessEvent) {
+    const sockets = this.activeDocSockets.get(
+      this.activeDocKey(event.spaceType, event.spaceId, event.docId)
+    );
+    if (!sockets) {
+      return;
+    }
+
+    for (const socket of sockets) {
+      if (socket.id !== event.sourceSocketId) {
+        socket.emit('space:collect-awareness', {
+          spaceType: event.spaceType,
+          spaceId: event.spaceId,
+          docId: event.docId,
+        });
+      }
+    }
+  }
+
+  private emitActiveAwarenessUpdate(
+    event: SyncAwarenessEvent & { awarenessUpdate: string }
+  ) {
+    const sockets = this.activeDocSockets.get(
+      this.activeDocKey(event.spaceType, event.spaceId, event.docId)
+    );
+    if (!sockets) {
+      return;
+    }
+
+    for (const socket of sockets) {
+      if (socket.id !== event.sourceSocketId) {
+        socket.emit('space:broadcast-awareness-update', {
+          spaceType: event.spaceType,
+          spaceId: event.spaceId,
+          docId: event.docId,
+          awarenessUpdate: event.awarenessUpdate,
+        });
+      }
+    }
+  }
+
   handleConnection(client: Socket) {
     this.connectionCount++;
     this.logger.debug(`New connection, total: ${this.connectionCount}`);
@@ -355,6 +653,7 @@ export class SpaceSyncGateway
   }
 
   handleDisconnect(client: Socket) {
+    this.removeAllActiveDocSubscriptions(client);
     this.connectionCount = Math.max(0, this.connectionCount - 1);
     this.trackDisconnectedSocket(client.id);
     this.logger.debug(
@@ -538,39 +837,176 @@ export class SpaceSyncGateway
     timestamp,
     editor,
   }: Events['doc.updates.pushed']) {
-    if (!this.server || updates.length === 0) {
-      return;
-    }
-
-    const room025 = `${spaceType}:${Room(spaceId, 'sync-025')}`;
-    const encodedUpdates = this.encodeUpdates(updates);
-    for (const update of encodedUpdates) {
-      const payload: BroadcastDocUpdateMessage = {
-        spaceType: spaceType as SpaceType,
-        spaceId,
-        docId,
-        update,
-        timestamp,
-        editor: editor ?? '',
-      };
-      this.server.to(room025).emit('space:broadcast-doc-update', payload);
-    }
-
-    const room026 = `${spaceType}:${Room(spaceId, 'sync-026')}`;
-    const payload = this.buildBroadcastPayload(
-      spaceType as SpaceType,
+    this.publishDocUpdate({
+      spaceType: spaceType as SpaceType,
       spaceId,
       docId,
       updates,
       timestamp,
-      editor
+      editor,
+    });
+  }
+
+  @OnEvent('sync.doc.updates.pushed')
+  onClusterDocUpdatesPushed(payload: Events['sync.doc.updates.pushed']) {
+    this.emitActiveDocUpdate({
+      ...payload,
+      updates: payload.updates.map(update =>
+        Uint8Array.from(Buffer.from(update, 'base64'))
+      ),
+    });
+  }
+
+  @OnEvent('sync.awareness.collect')
+  onClusterAwarenessCollect(event: Events['sync.awareness.collect']) {
+    this.emitActiveAwarenessCollect(event);
+  }
+
+  @OnEvent('sync.awareness.updated')
+  onClusterAwarenessUpdated(event: Events['sync.awareness.updated']) {
+    this.emitActiveAwarenessUpdate(event);
+  }
+
+  @OnEvent('doc.grants.changed')
+  @OnEvent('doc.owner.changed')
+  @OnEvent('doc.default_role.changed')
+  @OnEvent('doc.public_state.changed')
+  @OnEvent('workspace.members.updated')
+  @OnEvent('workspace.members.roleChanged')
+  @OnEvent('workspace.members.removed')
+  @OnEvent('workspace.members.leave')
+  @OnEvent('workspace.owner.changed')
+  async onPermissionChanged({
+    workspaceId,
+    docId,
+  }: {
+    workspaceId: string;
+    docId?: string;
+  }) {
+    await this.publishPermissionChange({
+      spaceType: SpaceType.Workspace,
+      spaceId: workspaceId,
+      docId,
+    });
+  }
+
+  @OnEvent('sync.permissions.changed')
+  async onClusterPermissionsChanged(event: Events['sync.permissions.changed']) {
+    await this.revalidateActiveDocSubscriptions(event);
+  }
+
+  private async publishPermissionChange(
+    event: Events['sync.permissions.changed']
+  ) {
+    await this.revalidateActiveDocSubscriptions(event);
+    this.event.broadcast('sync.permissions.changed', event);
+  }
+
+  private async revalidateActiveDocSubscriptions(
+    event: Events['sync.permissions.changed']
+  ) {
+    const spacePrefix = `${event.spaceType}:${event.spaceId}:`;
+    const exactKey = event.docId
+      ? this.activeDocKey(event.spaceType, event.spaceId, event.docId)
+      : undefined;
+    const candidates = [...this.activeDocSockets.entries()].filter(
+      ([key]) => key === exactKey || (!exactKey && key.startsWith(spacePrefix))
     );
-    this.server.to(room026).emit('space:broadcast-doc-updates', payload);
+
+    for (const [key, sockets] of candidates) {
+      const [, spaceId, docId] = key.split(':');
+      for (const socket of Array.from(sockets)) {
+        const userId = this.resolvePresenceUserId(socket);
+        if (!userId) {
+          this.removeActiveDocSubscription(
+            socket,
+            event.spaceType,
+            spaceId,
+            docId
+          );
+          continue;
+        }
+
+        try {
+          this.assertUserdataSubject(event.spaceType, userId, spaceId, docId);
+          await this.assertDocActionAllowed(
+            event.spaceType,
+            userId,
+            spaceId,
+            docId,
+            'Doc.Read'
+          );
+        } catch {
+          this.removeActiveDocSubscription(
+            socket,
+            event.spaceType,
+            spaceId,
+            docId
+          );
+        }
+      }
+    }
+  }
+
+  private publishDocUpdate(
+    payload: SyncDocUpdatesPayload,
+    sourceSocket?: Socket
+  ) {
+    if (!this.server || payload.updates.length === 0) {
+      return;
+    }
+
+    const legacyRoom = `${payload.spaceType}:${Room(
+      payload.spaceId,
+      'sync-026'
+    )}`;
+    const broadcastPayload = this.buildBroadcastPayload(
+      payload.spaceType,
+      payload.spaceId,
+      payload.docId,
+      payload.updates,
+      payload.timestamp,
+      payload.editor
+    );
+    if (sourceSocket) {
+      sourceSocket
+        .to(legacyRoom)
+        .emit('space:broadcast-doc-updates', broadcastPayload);
+    } else {
+      this.server
+        .to(legacyRoom)
+        .emit('space:broadcast-doc-updates', broadcastPayload);
+    }
+
+    const batchRoom = `${payload.spaceType}:${Room(
+      payload.spaceId,
+      'sync-027'
+    )}`;
+    const invalidation = {
+      spaceType: payload.spaceType,
+      spaceId: payload.spaceId,
+      timestamp: payload.timestamp,
+    };
+    if (sourceSocket) {
+      sourceSocket
+        .to(batchRoom)
+        .emit('space:broadcast-doc-invalidation', invalidation);
+    } else {
+      this.server
+        .to(batchRoom)
+        .emit('space:broadcast-doc-invalidation', invalidation);
+    }
+
+    this.emitActiveDocUpdate(payload, sourceSocket?.id, broadcastPayload);
     metrics.socketio
       .counter('doc_updates_broadcast')
-      .add(payload.updates.length, {
-        mode: payload.compressed ? 'compressed' : 'batch',
+      .add(broadcastPayload.updates.length, {
+        mode: broadcastPayload.compressed ? 'compressed' : 'batch',
       });
+    this.event.broadcast('sync.doc.updates.pushed', {
+      ...payload,
+      updates: this.encodeUpdates(payload.updates),
+    });
   }
 
   selectAdapter(client: Socket, spaceType: SpaceType): SyncSocketAdapter {
@@ -611,21 +1047,140 @@ export class SpaceSyncGateway
       this.rejectJoin(client);
       return { data: { clientId: client.id, success: false } };
     }
+    if (isBatchWsClientVersion(clientVersion)) {
+      this.rejectJoin(client);
+      return { data: { clientId: client.id, success: false } };
+    }
 
     const adapter = this.selectAdapter(client, spaceType);
     await adapter.join(user.id, spaceId);
+    this.removeActiveDocSubscriptionsInSpace(client, spaceType, spaceId);
 
-    const protocolRoomType = getSyncProtocolRoomType(clientVersion);
-    const protocolRoom = adapter.room(spaceId, protocolRoomType);
-    const otherProtocolRoom = adapter.room(
-      spaceId,
-      protocolRoomType === 'sync-025' ? 'sync-026' : 'sync-025'
-    );
-    if (client.rooms.has(otherProtocolRoom)) {
-      await client.leave(otherProtocolRoom);
+    const legacyRoom = adapter.room(spaceId, 'sync-026');
+    const batchRoom = adapter.room(spaceId, 'sync-027');
+    if (client.rooms.has(batchRoom)) {
+      await client.leave(batchRoom);
     }
-    if (!client.rooms.has(protocolRoom)) {
-      await client.join(protocolRoom);
+    if (!client.rooms.has(legacyRoom)) {
+      await client.join(legacyRoom);
+    }
+
+    return { data: { clientId: client.id, success: true } };
+  }
+
+  @SubscribeMessage('space:join-batch')
+  async onJoinSpaceBatch(
+    @CurrentUser() user: CurrentUser,
+    @ConnectedSocket() client: Socket,
+    @MessageBody() message: unknown
+  ): Promise<EventResponse<{ clientId: string; success: boolean }>> {
+    const { spaces, clientVersion } = parseJoinSpaceBatchMessage(message);
+    if (
+      !isSupportedWsClientVersion(clientVersion) ||
+      !isBatchWsClientVersion(clientVersion)
+    ) {
+      this.rejectJoin(client);
+      return { data: { clientId: client.id, success: false } };
+    }
+
+    const [first] = spaces;
+    const adapter = this.selectAdapter(client, first.spaceType);
+
+    // Authorize the whole batch before mutating any Socket.IO room. This is
+    // intentionally separate from SyncSocketAdapter.join(), which is also
+    // used by the legacy single-room handlers.
+    await adapter.assertAccessible(first.spaceId, user.id, 'Workspace.Sync');
+
+    for (const space of spaces) {
+      if (space.docId === undefined) {
+        continue;
+      }
+      this.assertUserdataSubject(
+        space.spaceType,
+        user.id,
+        space.spaceId,
+        space.docId
+      );
+      await this.assertDocActionAllowed(
+        space.spaceType,
+        user.id,
+        space.spaceId,
+        space.docId,
+        'Doc.Read'
+      );
+    }
+
+    const rooms = new Set<string>();
+    rooms.add(adapter.room(first.spaceId));
+    rooms.add(adapter.room(first.spaceId, 'sync-027'));
+    const legacyRoom = adapter.room(first.spaceId, 'sync-026');
+
+    const roomsToJoin = [...rooms].filter(room => !client.rooms.has(room));
+    const subscriptionsToAdd = spaces.filter(
+      (space): space is JoinSpaceBatchEntry & { docId: string } =>
+        space.docId !== undefined &&
+        !this.hasActiveDocSubscription(
+          client,
+          space.spaceType,
+          space.spaceId,
+          space.docId
+        )
+    );
+    try {
+      if (roomsToJoin.length > 0) {
+        await client.join(roomsToJoin);
+      }
+      for (const space of subscriptionsToAdd) {
+        this.addActiveDocSubscription(
+          client,
+          space.spaceType,
+          space.spaceId,
+          space.docId
+        );
+      }
+      if (client.rooms.has(legacyRoom)) {
+        await client.leave(legacyRoom);
+      }
+    } catch (error) {
+      for (const space of subscriptionsToAdd) {
+        this.removeActiveDocSubscription(
+          client,
+          space.spaceType,
+          space.spaceId,
+          space.docId
+        );
+      }
+      await Promise.all(
+        roomsToJoin
+          .filter(room => client.rooms.has(room))
+          .map(async room => {
+            await client.leave(room);
+          })
+      );
+      throw error;
+    }
+
+    return { data: { clientId: client.id, success: true } };
+  }
+
+  @SubscribeMessage('space:leave-batch')
+  async onLeaveSpaceBatch(
+    @ConnectedSocket() client: Socket,
+    @MessageBody() message: unknown
+  ): Promise<EventResponse<{ clientId: string; success: true }>> {
+    const { spaceType, spaceId, docIds } = parseLeaveSpaceBatchMessage(message);
+    for (const docId of docIds) {
+      this.removeActiveDocSubscription(client, spaceType, spaceId, docId);
+    }
+
+    const activeDocs = this.activeSocketDocs.get(client.id);
+    const hasActiveDocsInSpace = Array.from(activeDocs ?? []).some(key =>
+      key.startsWith(`${spaceType}:${spaceId}:`)
+    );
+    if (docIds.length === 0 && !hasActiveDocsInSpace) {
+      const adapter = this.selectAdapter(client, spaceType);
+      await adapter.leave(spaceId, 'sync-027');
+      await adapter.leave(spaceId);
     }
 
     return { data: { clientId: client.id, success: true } };
@@ -636,7 +1191,11 @@ export class SpaceSyncGateway
     @ConnectedSocket() client: Socket,
     @MessageBody() { spaceType, spaceId }: LeaveSpaceMessage
   ): Promise<EventResponse<{ clientId: string; success: true }>> {
-    await this.selectAdapter(client, spaceType).leave(spaceId);
+    const adapter = this.selectAdapter(client, spaceType);
+    this.removeActiveDocSubscriptionsInSpace(client, spaceType, spaceId);
+    await adapter.leave(spaceId);
+    await adapter.leave(spaceId, 'sync-026');
+    await adapter.leave(spaceId, 'sync-027');
 
     return { data: { clientId: client.id, success: true } };
   }
@@ -729,33 +1288,17 @@ export class SpaceSyncGateway
       user.id
     );
 
-    const payload = this.buildBroadcastPayload(
-      spaceType,
-      spaceId,
-      docId,
-      [Buffer.from(update, 'base64')],
-      timestamp,
-      user.id
-    );
-    client
-      .to(adapter.room(spaceId, 'sync-026'))
-      .emit('space:broadcast-doc-updates', payload);
-    metrics.socketio
-      .counter('doc_updates_broadcast')
-      .add(payload.updates.length, {
-        mode: payload.compressed ? 'compressed' : 'batch',
-      });
-
-    client
-      .to(adapter.room(spaceId, 'sync-025'))
-      .emit('space:broadcast-doc-update', {
+    this.publishDocUpdate(
+      {
         spaceType,
         spaceId,
         docId,
-        update,
+        updates: [Buffer.from(update, 'base64')],
         timestamp,
         editor: user.id,
-      } satisfies BroadcastDocUpdateMessage);
+      },
+      client
+    );
 
     return {
       data: {
@@ -813,6 +1356,10 @@ export class SpaceSyncGateway
       this.rejectJoin(client);
       return { data: { clientId: client.id, success: false } };
     }
+    if (isBatchWsClientVersion(clientVersion)) {
+      this.rejectJoin(client);
+      return { data: { clientId: client.id, success: false } };
+    }
 
     await this.selectAdapter(client, spaceType).join(
       user.id,
@@ -845,11 +1392,18 @@ export class SpaceSyncGateway
   ) {
     const adapter = this.selectAdapter(client, spaceType);
 
-    const roomType = `${docId}:awareness` as const;
-    adapter.assertIn(spaceId, roomType);
-    client
-      .to(adapter.room(spaceId, roomType))
-      .emit('space:collect-awareness', { spaceType, spaceId, docId });
+    if (this.hasActiveDocSubscription(client, spaceType, spaceId, docId)) {
+      adapter.assertIn(spaceId);
+      const event = { spaceType, spaceId, docId, sourceSocketId: client.id };
+      this.emitActiveAwarenessCollect(event);
+      this.event.broadcast('sync.awareness.collect', event);
+    } else {
+      const roomType = `${docId}:awareness` as const;
+      adapter.assertIn(spaceId, roomType);
+      client
+        .to(adapter.room(spaceId, roomType))
+        .emit('space:collect-awareness', { spaceType, spaceId, docId });
+    }
 
     return { data: { clientId: client.id } };
   }
@@ -862,11 +1416,18 @@ export class SpaceSyncGateway
     const { spaceType, spaceId, docId } = message;
     const adapter = this.selectAdapter(client, spaceType);
 
-    const roomType = `${docId}:awareness` as const;
-    adapter.assertIn(spaceId, roomType);
-    client
-      .to(adapter.room(spaceId, roomType))
-      .emit('space:broadcast-awareness-update', message);
+    if (this.hasActiveDocSubscription(client, spaceType, spaceId, docId)) {
+      adapter.assertIn(spaceId);
+      const event = { ...message, sourceSocketId: client.id };
+      this.emitActiveAwarenessUpdate(event);
+      this.event.broadcast('sync.awareness.updated', event);
+    } else {
+      const roomType = `${docId}:awareness` as const;
+      adapter.assertIn(spaceId, roomType);
+      client
+        .to(adapter.room(spaceId, roomType))
+        .emit('space:broadcast-awareness-update', message);
+    }
 
     return {};
   }

packages/backend/server/src/core/workspaces/resolvers/doc.ts

@@ -44,7 +44,6 @@ import {
   type DotToUnderline,
   mapPermissionsToGraphqlPermissions,
   PermissionAccess,
-  PermissionService,
 } from '../../permission';
 import { PublicUserType, WorkspaceUserType } from '../../user';
 import { canUserExecuteLimitedActions } from '../abuse';
@@ -300,7 +299,6 @@ export class WorkspaceDocResolver {
      */
     private readonly prisma: PrismaClient,
     private readonly ac: PermissionAccess,
-    private readonly permission: PermissionService,
     private readonly models: Models,
     private readonly cache: Cache,
     private readonly event: EventBus,
@@ -409,12 +407,14 @@ export class WorkspaceDocResolver {
     @Parent() workspace: WorkspaceType,
     @Args('pagination', PaginationInput.decode) pagination: PaginationInput
   ): Promise<PaginatedDocType> {
-    const predicate = this.permission.docReadableSqlPredicate({
-      userId: me.id,
-      workspaceId: workspace.id,
-      action: 'Doc.Read',
-      docIdColumn: Prisma.raw('"workspace_pages"."page_id"'),
-    });
+    const readable = await this.runtime.filterReadableDocs(
+      me.id,
+      workspace.id,
+      await this.models.doc.listWorkspaceDocIds(workspace.id)
+    );
+    const predicate = readable.length
+      ? Prisma.sql`"workspace_pages"."page_id" IN (${Prisma.join(readable)})`
+      : Prisma.sql`FALSE`;
     const [count, rows] = await this.models.doc.paginateDocInfoByUpdatedAt(
       workspace.id,
       pagination,

packages/backend/server/src/data/__tests__/migrations.spec.ts

@@ -1,12 +1,16 @@
+import { randomUUID } from 'node:crypto';
+
 import { ModuleRef } from '@nestjs/core';
 import { PrismaClient } from '@prisma/client';
 import ava, { TestFn } from 'ava';
 
 import { createTestingModule, type TestingModule } from '../../__tests__/utils';
+import { BackendRuntimeProvider } from '../../core/backend-runtime';
 import { Models } from '../../models';
 import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection';
 import { BackfillTranscriptStorageKeys1786805802350 } from '../migrations/1786805802350-backfill-transcript-storage-keys';
 import { ConvergeManagedProviderProfiles1786810000000 } from '../migrations/1786810000000-converge-managed-provider-profiles';
+import { MigrateLegacyContextBlobArtifacts1786820000000 } from '../migrations/1786820000000-migrate-legacy-context-blob-artifacts';
 
 interface Context {
   module: TestingModule;
@@ -207,3 +211,107 @@ test('managed provider migration preserves explicit profiles and converts legacy
     })
   );
 });
+
+test('legacy context blob migration admits each blob once through the artifact runtime', async t => {
+  const user = await t.context.models.user.create({
+    email: 'legacy-context@affine.pro',
+  });
+  const workspace = await t.context.db.workspace.create({
+    data: { accessPolicy: { create: {} } },
+  });
+  const session = await t.context.db.aiSession.create({
+    data: {
+      userId: user.id,
+      workspaceId: workspace.id,
+      promptName: 'copilot',
+    },
+  });
+  const blobId = 'legacy-context-blob';
+  const legacyTable = await t.context.db.$queryRaw<{ exists: boolean }[]>`
+    SELECT to_regclass('public.ai_contexts') IS NOT NULL AS exists
+  `;
+  const createdLegacyTable = !legacyTable[0]?.exists;
+  if (createdLegacyTable) {
+    const ref = {
+      get() {
+        throw new Error(
+          'legacy context runtime should not be resolved without source tables'
+        );
+      },
+    } as unknown as ModuleRef;
+    await MigrateLegacyContextBlobArtifacts1786820000000.up(t.context.db, ref);
+  }
+  if (createdLegacyTable) {
+    await t.context.db.$executeRaw`
+      CREATE TABLE ai_contexts (
+        id VARCHAR PRIMARY KEY,
+        session_id VARCHAR NOT NULL,
+        config JSON NOT NULL,
+        created_at TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMPTZ(3) NOT NULL
+      )
+    `;
+  }
+  await t.context.db.blob.create({
+    data: {
+      workspaceId: workspace.id,
+      key: blobId,
+      size: 12,
+      mime: 'text/plain',
+      status: 'completed',
+    },
+  });
+  await t.context.db.$executeRaw`
+    INSERT INTO ai_contexts (id, session_id, config, created_at, updated_at)
+    VALUES (${randomUUID()}, ${session.id}, ${JSON.stringify({ blobs: [blobId] })}::jsonb, now(), now())
+  `;
+
+  const calls: Array<{
+    workspaceId: string;
+    blobId: string;
+    mimeType: string;
+    libraryOwned?: boolean;
+  }> = [];
+  const runtime = {
+    async ensureWorkspaceBlobArtifact(input: (typeof calls)[number]) {
+      calls.push(input);
+      await t.context.db.$executeRaw`
+        INSERT INTO workspace_artifacts (
+          id, workspace_id, content_hash, canonical_media_type, size_bytes,
+          storage_scope, storage_key, status, ready_at
+        ) VALUES (
+          ${randomUUID()}::uuid, ${input.workspaceId}, ${`hash-${input.blobId}`},
+          ${input.mimeType}, 12, 'blob',
+          ${`${input.workspaceId}/${input.blobId}`}, 'ready', now()
+        )
+      `;
+      return {};
+    },
+  };
+  const ref = {
+    get(token: unknown) {
+      if (token === BackendRuntimeProvider) {
+        return runtime;
+      }
+      throw new Error('unexpected migration dependency');
+    },
+  } as unknown as ModuleRef;
+
+  try {
+    await MigrateLegacyContextBlobArtifacts1786820000000.up(t.context.db, ref);
+    await MigrateLegacyContextBlobArtifacts1786820000000.up(t.context.db, ref);
+  } finally {
+    if (createdLegacyTable) {
+      await t.context.db.$executeRaw`DROP TABLE ai_contexts`;
+    }
+  }
+
+  t.deepEqual(calls, [
+    {
+      workspaceId: workspace.id,
+      blobId,
+      mimeType: 'text/plain',
+      libraryOwned: false,
+    },
+  ]);
+});

packages/backend/server/src/data/app.ts

@@ -1,13 +1,12 @@
 import { Module } from '@nestjs/common';
 
 import { FunctionalityModules } from '../app.module';
-import { IndexerModule } from '../plugins/indexer';
 import { CreateCommand } from './commands/create';
 import { ImportConfigCommand } from './commands/import';
 import { RevertCommand, RunCommand } from './commands/run';
 
 @Module({
-  imports: [...FunctionalityModules, IndexerModule],
+  imports: FunctionalityModules,
   providers: [CreateCommand, RunCommand, RevertCommand, ImportConfigCommand],
 })
 export class CliAppModule {}

packages/backend/server/src/data/commands/run.ts

@@ -3,6 +3,8 @@ import { ModuleRef } from '@nestjs/core';
 import { PrismaClient } from '@prisma/client';
 import { once } from 'lodash-es';
 
+import { BackendRuntimeProvider } from '../../core/backend-runtime';
+import { StorageRuntimeProvider } from '../../core/storage-runtime';
 import * as migrationImports from '../migrations';
 
 interface Migration {
@@ -13,6 +15,9 @@ interface Migration {
   order: number;
 }
 
+const LEGACY_CONTEXT_BLOB_ARTIFACT_MIGRATION =
+  'MigrateLegacyContextBlobArtifacts1786820000000';
+
 export const collectMigrations = once(() => {
   const migrations = Object.values(migrationImports).map(migration => {
     const order = Number(migration.name.match(/([\d]+)$/)?.[1]);
@@ -43,6 +48,12 @@ export class RunCommand {
   ) {}
 
   async execute(): Promise<void> {
+    await this.injector
+      .get(BackendRuntimeProvider, { strict: false })
+      .runMigrations();
+    await this.injector
+      .get(StorageRuntimeProvider, { strict: false })
+      .runMigrations();
     const migrations = collectMigrations();
     const done: Migration[] = [];
     for (const migration of migrations) {
@@ -85,6 +96,33 @@ export class RunCommand {
     await this.runMigration(migration);
   }
 
+  async admitLegacyContextBlobs(): Promise<void> {
+    const tables = await this.db.$queryRaw<
+      Array<{
+        contexts: string | null;
+        sessions: string | null;
+        blobs: string | null;
+        artifacts: string | null;
+      }>
+    >`
+      SELECT
+        to_regclass('public.ai_contexts')::text AS contexts,
+        to_regclass('public.ai_sessions_metadata')::text AS sessions,
+        to_regclass('public.blobs')::text AS blobs,
+        to_regclass('public.workspace_artifacts')::text AS artifacts
+    `;
+
+    const schemaExists = Object.values(tables[0] ?? {}).every(Boolean);
+    if (!schemaExists) {
+      this.logger.log(
+        'Skipping legacy context blob admission because its source schema is not present.'
+      );
+      return;
+    }
+
+    await this.runOne(LEGACY_CONTEXT_BLOB_ARTIFACT_MIGRATION);
+  }
+
   private async runMigration(migration: Migration) {
     this.logger.log(`Running ${migration.name}...`);
     const record = await this.db.dataMigration.upsert({

packages/backend/server/src/data/migrations/1745211351719-create-indexer-tables.ts

@@ -1,16 +0,0 @@
-import { ModuleRef } from '@nestjs/core';
-import { PrismaClient } from '@prisma/client';
-
-import { IndexerService } from '../../plugins/indexer';
-
-export class CreateIndexerTables1745211351719 {
-  static always = true;
-
-  // do the migration
-  static async up(_db: PrismaClient, ref: ModuleRef) {
-    await ref.get(IndexerService, { strict: false }).createTables();
-  }
-
-  // revert the migration
-  static async down(_db: PrismaClient) {}
-}

packages/backend/server/src/data/migrations/1763800000000-rebuild-manticore-mixed-script-indexes.ts

@@ -1,12 +0,0 @@
-import { ModuleRef } from '@nestjs/core';
-import { PrismaClient } from '@prisma/client';
-
-import { IndexerService } from '../../plugins/indexer';
-
-export class RebuildManticoreMixedScriptIndexes1763800000000 {
-  static async up(_db: PrismaClient, ref: ModuleRef) {
-    await ref.get(IndexerService, { strict: false }).rebuildManticoreIndexes();
-  }
-
-  static async down(_db: PrismaClient) {}
-}

packages/backend/server/src/data/migrations/1786820000000-migrate-legacy-context-blob-artifacts.ts

@@ -0,0 +1,79 @@
+import { ModuleRef } from '@nestjs/core';
+import { PrismaClient } from '@prisma/client';
+
+import { BackendRuntimeProvider } from '../../core/backend-runtime';
+
+type LegacyContextBlob = {
+  workspaceId: string;
+  blobId: string;
+  mimeType: string;
+};
+
+/**
+ * Convert the last product-owned references to workspace blobs into the
+ * artifact retention fact before the legacy context tables are removed.
+ *
+ * The runtime admission path is intentional here: a database row alone does
+ * not prove that the object still exists or that its metadata is truthful.
+ */
+export class MigrateLegacyContextBlobArtifacts1786820000000 {
+  static async up(db: PrismaClient, injector: ModuleRef) {
+    const tables = await db.$queryRaw<
+      Array<{
+        contexts: string | null;
+        sessions: string | null;
+        blobs: string | null;
+        artifacts: string | null;
+      }>
+    >`
+      SELECT
+        to_regclass('public.ai_contexts')::text AS contexts,
+        to_regclass('public.ai_sessions_metadata')::text AS sessions,
+        to_regclass('public.blobs')::text AS blobs,
+        to_regclass('public.workspace_artifacts')::text AS artifacts
+    `;
+
+    if (!Object.values(tables[0] ?? {}).every(Boolean)) {
+      return;
+    }
+
+    const runtime = injector.get(BackendRuntimeProvider, { strict: false });
+    const blobs = await db.$queryRaw<LegacyContextBlob[]>`
+      SELECT DISTINCT
+        session.workspace_id AS "workspaceId",
+        blob.key AS "blobId",
+        blob.mime AS "mimeType"
+      FROM ai_contexts context
+      JOIN ai_sessions_metadata session ON session.id = context.session_id
+      JOIN blobs blob
+        ON blob.workspace_id = session.workspace_id
+       AND blob.deleted_at IS NULL
+       AND blob.status = 'completed'
+      WHERE jsonb_path_exists(
+        context.config::jsonb,
+        '$.** ? (@ == $blobKey)',
+        jsonb_build_object('blobKey', to_jsonb(blob.key::text))
+      )
+        AND NOT EXISTS (
+          SELECT 1
+          FROM workspace_artifacts artifact
+          WHERE artifact.workspace_id = session.workspace_id
+            AND artifact.storage_scope = 'blob'
+            AND artifact.storage_key = concat(session.workspace_id, '/', blob.key)
+            AND artifact.status = 'ready'
+        )
+      ORDER BY session.workspace_id, blob.key
+    `;
+
+    for (const blob of blobs) {
+      await runtime.ensureWorkspaceBlobArtifact({
+        workspaceId: blob.workspaceId,
+        blobId: blob.blobId,
+        mimeType: blob.mimeType,
+        libraryOwned: false,
+      });
+    }
+  }
+
+  static async down(_db: PrismaClient, _injector: ModuleRef) {}
+}

packages/backend/server/src/data/migrations/index.ts

@@ -1,10 +1,9 @@
 export * from './1698398506533-guid';
 export * from './1703756315970-unamed-account';
 export * from './1721299086340-refresh-unnamed-user';
-export * from './1745211351719-create-indexer-tables';
 export * from './1751966744168-correct-session-update-time';
-export * from './1763800000000-rebuild-manticore-mixed-script-indexes';
 export * from './1765500000000-backfill-permission-projection';
 export * from './1765600000000-backfill-entitlement-projection';
 export * from './1786805802350-backfill-transcript-storage-keys';
 export * from './1786810000000-converge-managed-provider-profiles';
+export * from './1786820000000-migrate-legacy-context-blob-artifacts';

packages/backend/server/src/env.ts

@@ -24,10 +24,17 @@ export enum Flavor {
   Sync = 'sync',
   Renderer = 'renderer',
   Front = 'front',
-  Doc = 'doc',
+  Worker = 'worker',
   Script = 'script',
 }
 
+export enum ServerRole {
+  Frontend = 'frontend',
+  Api = 'api',
+  Worker = 'worker',
+  AllInOne = 'allinone',
+}
+
 export enum Namespace {
   Dev = 'dev',
   Beta = 'beta',
@@ -101,6 +108,39 @@ export class Env implements AppEnv {
     return this.DEPLOYMENT_TYPE === DeploymentType.Selfhosted;
   }
 
+  get role(): ServerRole | undefined {
+    switch (this.FLAVOR) {
+      case Flavor.AllInOne:
+        return ServerRole.AllInOne;
+      case Flavor.Graphql:
+        return ServerRole.Api;
+      case Flavor.Worker:
+        return ServerRole.Worker;
+      case Flavor.Front:
+      case Flavor.Sync:
+      case Flavor.Renderer:
+        return ServerRole.Frontend;
+      case Flavor.Script:
+        return undefined;
+    }
+  }
+
+  get isApi() {
+    return this.FLAVOR === Flavor.Graphql || this.FLAVOR === Flavor.AllInOne;
+  }
+
+  get isWorker() {
+    return this.FLAVOR === Flavor.Worker || this.FLAVOR === Flavor.AllInOne;
+  }
+
+  get isFrontend() {
+    return (
+      this.FLAVOR === Flavor.Front ||
+      this.FLAVOR === Flavor.Sync ||
+      this.FLAVOR === Flavor.Renderer
+    );
+  }
+
   isFlavor(flavor: Flavor) {
     return this.FLAVOR === flavor || this.FLAVOR === Flavor.AllInOne;
   }
@@ -111,7 +151,7 @@ export class Env implements AppEnv {
       sync: this.isFlavor(Flavor.Sync),
       renderer: this.isFlavor(Flavor.Renderer),
       front: this.FLAVOR === Flavor.Front,
-      doc: this.isFlavor(Flavor.Doc),
+      worker: this.isFlavor(Flavor.Worker),
       // Script in a special flavor, return true only when it is set explicitly
       script: this.FLAVOR === Flavor.Script,
     };

packages/backend/server/src/models/__tests__/comment.spec.ts

@@ -100,6 +100,7 @@ test('should update a comment', async t => {
     userId: owner.id,
   });
 
+  await waitNextMillisecond();
   const comment2 = await models.comment.update({
     id: comment1.id,
     content: {

packages/backend/server/src/models/copilot-transcript-task.ts

@@ -125,23 +125,6 @@ export class CopilotTranscriptTaskModel extends BaseModel {
     return count === 1;
   }
 
-  async adoptLegacyDispatch(
-    id: string,
-    actionRunId: string | null,
-    dispatchGeneration: string
-  ) {
-    const { count } = await this.db.aiTranscriptTask.updateMany({
-      where: {
-        id,
-        status: 'pending',
-        dispatchGeneration: null,
-        actionRunId,
-      },
-      data: { dispatchGeneration },
-    });
-    return count === 1;
-  }
-
   async attachActionRun(
     id: string,
     dispatchGeneration: string,

packages/backend/server/src/models/doc.ts

@@ -803,6 +803,14 @@ export class DocModel extends BaseModel {
     ] as const;
   }
 
+  async listWorkspaceDocIds(workspaceId: string) {
+    const rows = await this.db.workspaceDoc.findMany({
+      where: { workspaceId },
+      select: { docId: true },
+    });
+    return rows.map(row => row.docId);
+  }
+
   async findEmptySummaryDocIds(workspaceId: string) {
     const rows = await this.db.workspaceDoc.findMany({
       where: {

packages/backend/server/src/native.ts

@@ -45,6 +45,7 @@ import serverNativeModule, {
   type RemoteMimeTypeRequest,
   type ResolvedEntitlement,
   type ResolveEntitlementInput,
+  type RuntimeAggregateRequest,
   type RuntimeBlobCleanupExecuteResult,
   type RuntimeBlobCleanupPlanResult,
   type RuntimeBlobCleanupResult,
@@ -64,6 +65,8 @@ import serverNativeModule, {
   type RuntimeObjectStoragePutOptions,
   type RuntimePresignedObjectRequest,
   type RuntimeRetrievalScope,
+  type RuntimeSearchQuery,
+  type RuntimeSearchRequest,
   type RuntimeTurnScopeSnapshot,
   type RuntimeVerificationTokenRecord,
   type RuntimeWorkspaceArtifact,
@@ -144,6 +147,7 @@ export type {
   RemoteMimeTypeRequest,
   ResolvedEntitlement,
   ResolveEntitlementInput,
+  RuntimeAggregateRequest,
   RuntimeBlobCleanupExecuteResult,
   RuntimeBlobCleanupPlanResult,
   RuntimeBlobCleanupResult,
@@ -163,6 +167,8 @@ export type {
   RuntimeObjectStoragePutOptions,
   RuntimePresignedObjectRequest,
   RuntimeRetrievalScope,
+  RuntimeSearchQuery,
+  RuntimeSearchRequest,
   RuntimeTurnScopeSnapshot,
   RuntimeVerificationTokenRecord,
   RuntimeWorkspaceArtifact,

packages/backend/server/src/plugins/calendar/__tests__/service.spec.ts

@@ -479,6 +479,32 @@ test('syncSubscription invalidates account when refresh token is invalid', async
   t.is(events.length, 0);
 });
 
+test('syncSubscription does not disable a calendar when token refresh returns 404', async t => {
+  const user = await module.create(Mockers.User);
+  const account = await createAccount(user.id, {
+    accessToken: 'expired-access-token',
+    expiresAt: new Date(Date.now() - 5 * 60 * 1000),
+  });
+  const subscription = await createSubscription(account.id, {
+    syncToken: 'sync-token',
+  });
+
+  const provider = new MockCalendarProvider();
+  mock.method(provider, 'refreshTokens', async () => {
+    throw new CalendarProviderRequestError({
+      status: 404,
+      message: 'Token endpoint not found',
+    });
+  });
+  mock.method(providerFactory, 'get', () => provider);
+
+  await calendarService.syncSubscription(subscription.id);
+
+  const updated = await models.calendarSubscription.get(subscription.id);
+  t.is(updated?.enabled, true);
+  t.is(updated?.syncRetryCount, 1);
+});
+
 test('syncSubscription disables subscription on provider 404', async t => {
   const user = await module.create(Mockers.User);
   const account = await createAccount(user.id);
@@ -673,6 +699,44 @@ test('syncSubscription renews webhook channel when expiring', async t => {
   t.truthy(updated?.channelExpiration);
 });
 
+test('syncSubscription replaces a webhook channel that is already gone', async t => {
+  const user = await module.create(Mockers.User);
+  const account = await createAccount(user.id);
+  const subscription = await createSubscription(account.id, {
+    syncToken: 'sync-token',
+    customChannelId: 'missing-channel',
+    customResourceId: 'missing-resource',
+    channelExpiration: new Date(Date.now() + 60 * 60 * 1000),
+  });
+
+  const provider = new MockCalendarProvider();
+  mock.method(provider, 'listEvents', async () => ({
+    events: [],
+    nextSyncToken: 'next-sync',
+  }));
+  const stopMock = mock.method(provider, 'stopChannel', async () => {
+    throw new CalendarProviderRequestError({
+      status: 404,
+      message: 'Channel not found',
+    });
+  });
+  const watchMock = mock.method(provider, 'watchCalendar', async () => ({
+    channelId: 'replacement-channel',
+    resourceId: 'replacement-resource',
+    expiration: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000),
+  }));
+  mock.method(providerFactory, 'get', () => provider);
+
+  await calendarService.syncSubscription(subscription.id);
+
+  t.is(stopMock.mock.callCount(), 1);
+  t.is(watchMock.mock.callCount(), 1);
+  const updated = await models.calendarSubscription.get(subscription.id);
+  t.is(updated?.customChannelId, 'replacement-channel');
+  t.is(updated?.customResourceId, 'replacement-resource');
+  t.is(updated?.syncRetryCount, 0);
+});
+
 test('syncSubscription falls back to polling when push is unsupported', async t => {
   const user = await module.create(Mockers.User);
   const account = await createAccount(user.id);

packages/backend/server/src/plugins/calendar/service.ts

@@ -403,6 +403,7 @@ export class CalendarService {
             account,
             provider,
             accessToken,
+            disableOnNotFound: true,
           });
           return;
         }
@@ -413,6 +414,7 @@ export class CalendarService {
           account,
           provider,
           accessToken,
+          disableOnNotFound: true,
         });
         return;
       }
@@ -933,10 +935,22 @@ export class CalendarService {
       subscription.customChannelId &&
       subscription.customResourceId
     ) {
-      await provider.stopChannel({
-        accessToken,
-        channelId: subscription.customChannelId,
-        resourceId: subscription.customResourceId,
+      try {
+        await provider.stopChannel({
+          accessToken,
+          channelId: subscription.customChannelId,
+          resourceId: subscription.customResourceId,
+        });
+      } catch (error) {
+        if (!this.isNotFoundError(error)) {
+          throw error;
+        }
+      }
+
+      await this.models.calendarSubscription.updateChannel(subscription.id, {
+        customChannelId: null,
+        customResourceId: null,
+        channelExpiration: null,
       });
     }
 
@@ -986,8 +1000,9 @@ export class CalendarService {
     account: CalendarAccount;
     provider: CalendarProvider;
     accessToken?: string;
+    disableOnNotFound?: boolean;
   }) {
-    if (this.isSubscriptionMissingError(params.error)) {
+    if (params.disableOnNotFound && this.isNotFoundError(params.error)) {
       await this.disableSubscription({
         subscriptionId: params.subscription.id,
         provider: params.provider,
@@ -1021,7 +1036,7 @@ export class CalendarService {
     );
   }
 
-  private isSubscriptionMissingError(error: unknown) {
+  private isNotFoundError(error: unknown) {
     if (!(error instanceof CalendarProviderRequestError)) {
       return false;
     }

packages/backend/server/src/plugins/copilot/retrieval/document.ts

@@ -1,6 +1,6 @@
 import { Inject, Injectable } from '@nestjs/common';
 
-import { Config, SearchProviderNotFound } from '../../../base';
+import { SearchProviderNotFound } from '../../../base';
 import { PermissionAccess } from '../../../core/permission';
 import type { DocVisibility } from '../../../core/utils/blocksuite';
 import { type DocChunkSimilarity, Models } from '../../../models';
@@ -70,7 +70,6 @@ function hasVectorProjectionMetadata(hit: DocChunkSimilarity) {
 @Injectable()
 export class DocumentRetrievalService {
   constructor(
-    private readonly config: Config,
     private readonly ac: PermissionAccess,
     private readonly indexer: IndexerService,
     @Inject(DOCUMENT_VECTOR_SEARCH)
@@ -96,17 +95,15 @@ export class DocumentRetrievalService {
       byokLeaseId: options.byokLeaseId,
     };
     const [lexicalAttempt, vectorAttempt] = await Promise.allSettled([
-      this.config.indexer.enabled
-        ? this.indexer
-            .searchDocsByKeyword(workspaceId, query, {
-              limit: Math.max(limit * 3, 20),
-              docIds,
-            })
-            .catch(error => {
-              if (error instanceof SearchProviderNotFound) return null;
-              throw error;
-            })
-        : null,
+      this.indexer
+        .searchDocsByKeyword(userId, workspaceId, query, {
+          limit: Math.max(limit * 3, 20),
+          docIds,
+        })
+        .catch(error => {
+          if (error instanceof SearchProviderNotFound) return null;
+          throw error;
+        }),
       this.context.canEmbedding
         ? this.context.matchWorkspaceDocCandidates(
             workspaceId,

packages/backend/server/src/plugins/copilot/runtime/turn-orchestrator.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 
-import { BackendRuntimeEmbeddingJob } from '../../../core/backend-runtime';
+import { BackendRuntimeEmbeddingService } from '../../../core/backend-runtime';
 import { type Turn } from '../core';
 import {
   type ModelConditions,
@@ -23,7 +23,7 @@ export class TurnOrchestrator {
     private readonly runtime: CapabilityRuntime,
     private readonly imageResults: ImageResultHost,
     private readonly turnPersistence: TurnPersistence,
-    private readonly embeddings: BackendRuntimeEmbeddingJob
+    private readonly embeddings: BackendRuntimeEmbeddingService
   ) {}
 
   private buildPromptParams(latestTurn?: Turn): Record<string, unknown> {

packages/backend/server/src/plugins/copilot/transcript/service.ts

@@ -512,7 +512,7 @@ export class CopilotTranscriptionService {
   async transcriptTask({
     taskId,
     payload,
-    generation: queuedGeneration,
+    generation,
     retryOf,
   }: Jobs['copilot.transcript.task.submit']) {
     const task = await this.models.copilotTranscriptTask.get(taskId);
@@ -520,17 +520,6 @@ export class CopilotTranscriptionService {
       throw new CopilotTranscriptionJobNotFound();
     }
     let actionRunId = retryOf ?? null;
-    const generation = queuedGeneration ?? randomUUID();
-    if (
-      !queuedGeneration &&
-      !(await this.models.copilotTranscriptTask.adoptLegacyDispatch(
-        taskId,
-        actionRunId,
-        generation
-      ))
-    ) {
-      return;
-    }
     const claimed = await this.models.copilotTranscriptTask.claimDispatch(
       taskId,
       generation,

packages/backend/server/src/plugins/copilot/transcript/types.ts

@@ -53,7 +53,7 @@ declare global {
     'copilot.transcript.task.submit': {
       taskId: string;
       payload: TranscriptionPayloadV2;
-      generation?: string;
+      generation: string;
       retryOf?: string;
     };
   }

packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-blocks.json

@@ -1,13 +0,0 @@
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId1", "content" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "flavour" : "title", "blob" : "blob1", "ref_doc_id" : "refDocId1", "ref" : "ref1", "parent_flavour" : "parentFlavour1", "parent_block_id" : "parentBlockId1", "additional" : "additional1", "markdown_preview" : "markdownPreview1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId2", "content" : "title2 world, test searching morphology", "flavour" : "flavour2", "blob" : "blob2", "ref_doc_id" : "refDocId2", "ref" : "ref2", "parent_flavour" : "parentFlavour2", "parent_block_id" : "parentBlockId2", "additional" : "additional2", "markdown_preview" : "markdownPreview2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId3", "content" : "title3 hello update", "flavour" : "flavour3", "blob" : "blob3", "ref_doc_id" : "refDocId3", "ref" : "ref3", "parent_flavour" : "parentFlavour3", "parent_block_id" : "parentBlockId3", "additional" : "additional3", "markdown_preview" : "markdownPreview3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId4", "content" : "title4 hello", "flavour" : "flavour4", "blob" : "blob4", "ref_doc_id" : "refDocId4", "ref" : "ref4", "parent_flavour" : "parentFlavour4", "parent_block_id" : "parentBlockId4", "additional" : "additional4", "markdown_preview" : "markdownPreview4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId5", "content" : "title5 hello", "flavour" : "flavour5", "blob" : "blob5", "ref_doc_id" : "refDocId5", "ref" : "ref5", "parent_flavour" : "parentFlavour5", "parent_block_id" : "parentBlockId5", "additional" : "additional5", "markdown_preview" : "markdownPreview5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "block_id" : "blockId6", "content" : "title6 hello", "flavour" : "flavour6", "blob" : "blob6", "ref_doc_id" : "refDocId6", "ref" : "ref6", "parent_flavour" : "parentFlavour6", "parent_block_id" : "parentBlockId6", "additional" : "additional6", "markdown_preview" : "markdownPreview6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId2", "doc_id" : "docId1", "block_id" : "blockId7", "content" : "title7 hello", "flavour" : "flavour7", "blob" : "blob7", "ref_doc_id" : "refDocId7", "ref" : "ref7", "parent_flavour" : "parentFlavour7", "parent_block_id" : "parentBlockId7", "additional" : "additional7", "markdown_preview" : "markdownPreview7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId9", "block_id" : "blockId9", "content" : "title9 hello affine issue hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour9", "parent_block_id" : "parentBlockId9", "additional" : "additional9", "markdown_preview" : "markdownPreview9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId10", "content" : "this is docId2 title content hello", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour10", "parent_block_id" : "parentBlockId10", "additional" : "additional10", "markdown_preview" : "markdownPreview10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId11", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour11", "parent_block_id" : "parentBlockId11", "additional" : "additional11", "markdown_preview" : "markdownPreview11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "block_id" : "blockId12", "content" : "this is docId2 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour12", "parent_block_id" : "parentBlockId12", "additional" : "additional12", "markdown_preview" : "markdownPreview12", "created_by_user_id" : "userId12", "updated_by_user_id" : "userId12", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId13", "content" : "this is docId3 title content world", "flavour" : "affine:page", "flavour_indexed": "affine:page", "parent_flavour": "parentFlavour13", "parent_block_id" : "parentBlockId13", "additional" : "additional13", "markdown_preview" : "markdownPreview13", "created_by_user_id" : "userId13", "updated_by_user_id" : "userId13", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "block_id" : "blockId14", "content" : "this is docId3 title content world", "flavour" : "affine:database", "parent_flavour": "affine:database", "parent_block_id" : "parentBlockId14", "additional" : "additional14", "markdown_preview" : "markdownPreview14", "created_by_user_id" : "userId14", "updated_by_user_id" : "userId14", "created_at" : "2023-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z", "ref_doc_id" : "docId2"}

packages/backend/server/src/plugins/indexer/__tests__/__fixtures__/test-docs.json

@@ -1,11 +0,0 @@
-{"workspace_id" : "workspaceId1", "doc_id" : "docId1", "title" : "title1 hello, 这是一段包含中文的标题，hello 你好😄", "summary" : "summary1", "journal" : "journal1", "created_by_user_id" : "userId1", "updated_by_user_id" : "userId1", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-10T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId2", "title" : "title2 world, test searching morphology", "summary" : "summary2", "journal" : "journal2", "created_by_user_id" : "userId2", "updated_by_user_id" : "userId2", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId3", "title" : "title3 hello update", "summary" : "summary3", "journal" : "journal3", "created_by_user_id" : "userId3", "updated_by_user_id" : "userId3", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-09T06:04:13.278Z"}
-{"workspace_id" : "workspaceId2", "doc_id" : "docId4", "title" : "title4 hello", "summary" : "summary4", "journal" : "journal4", "created_by_user_id" : "userId4", "updated_by_user_id" : "userId4", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId5", "title" : "title5 hello", "summary" : "summary5", "journal" : "journal5", "created_by_user_id" : "userId5", "updated_by_user_id" : "userId5", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId6", "title" : "title6 hello", "summary" : "summary6", "journal" : "journal6", "created_by_user_id" : "userId6", "updated_by_user_id" : "userId6", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId7", "title" : "title7 hello", "summary" : "summary7", "journal" : "journal7", "created_by_user_id" : "userId7", "updated_by_user_id" : "userId7", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId8", "title" : "title8 hello", "summary" : "summary8", "journal" : "journal8", "created_by_user_id" : "userId8", "updated_by_user_id" : "userId8", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId9", "title" : "title9 hello", "summary" : "summary9", "journal" : "journal9", "created_by_user_id" : "userId9", "updated_by_user_id" : "userId9", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId10", "title" : "title10 hello", "summary" : "summary10", "journal" : "journal10", "created_by_user_id" : "userId10", "updated_by_user_id" : "userId10", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2025-04-08T06:04:13.278Z"}
-{"workspace_id" : "workspaceId1", "doc_id" : "docId11", "title" : "title11 hello, old value", "summary" : "summary11", "journal" : "journal11", "created_by_user_id" : "userId11", "updated_by_user_id" : "userId11", "created_at" : "2025-03-08T06:04:13.278Z", "updated_at" : "2024-04-08T06:04:13.278Z"}

packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md

@@ -1,567 +0,0 @@
-# Snapshot report for `src/plugins/indexer/__tests__/service.spec.ts`
-
-The actual snapshot is saved in `service.spec.ts.snap`.
-
-Generated by [AVA](https://avajs.dev).
-
-## should write block with array content work
-
-> Snapshot 1
-
-    [
-      {
-        fields: {
-          content: [
-            'hello world',
-          ],
-        },
-      },
-    ]
-
-## should parse all query work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      fields: [
-        'flavour',
-        'doc_id',
-        'ref_doc_id',
-      ],
-      query: {
-        match_all: {},
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should parse exists query work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      fields: [
-        'flavour',
-        'doc_id',
-        'ref_doc_id',
-      ],
-      query: {
-        exists: {
-          field: 'ref_doc_id',
-        },
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should parse boost query work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      fields: [
-        'flavour',
-        'doc_id',
-        'ref_doc_id',
-      ],
-      query: {
-        term: {
-          flavour: {
-            boost: 1.5,
-            value: 'affine:page',
-          },
-        },
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should parse match query work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      fields: [
-        'flavour',
-        'doc_id',
-        'ref_doc_id',
-        'parent_flavour',
-        'parent_block_id',
-        'additional',
-        'markdown_preview',
-        'created_by_user_id',
-        'updated_by_user_id',
-        'created_at',
-        'updated_at',
-      ],
-      query: {
-        term: {
-          flavour: {
-            value: 'affine:page',
-          },
-        },
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should parse boolean query work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      fields: [
-        'flavour',
-        'doc_id',
-        'ref_doc_id',
-        'parent_flavour',
-        'parent_block_id',
-        'additional',
-        'markdown_preview',
-        'created_by_user_id',
-        'updated_by_user_id',
-        'created_at',
-        'updated_at',
-      ],
-      query: {
-        bool: {
-          must: [
-            {
-              term: {
-                workspace_id: {
-                  value: 'workspaceId1',
-                },
-              },
-            },
-            {
-              match: {
-                content: {
-                  query: 'hello',
-                },
-              },
-            },
-            {
-              bool: {
-                should: [
-                  {
-                    match: {
-                      content: {
-                        query: 'hello',
-                      },
-                    },
-                  },
-                  {
-                    term: {
-                      flavour: {
-                        boost: 1.5,
-                        value: 'affine:page',
-                      },
-                    },
-                  },
-                ],
-              },
-            },
-          ],
-        },
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should parse search input highlight work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      fields: [
-        'flavour',
-        'doc_id',
-        'ref_doc_id',
-      ],
-      highlight: {
-        fields: {
-          content: {
-            post_tags: [
-              '</b>',
-            ],
-            pre_tags: [
-              '<b>',
-            ],
-          },
-        },
-      },
-      query: {
-        match_all: {},
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should parse aggregate input highlight work
-
-> Snapshot 1
-
-    {
-      _source: [
-        'workspace_id',
-        'doc_id',
-      ],
-      aggs: {
-        result: {
-          aggs: {
-            max_score: {
-              max: {
-                script: {
-                  source: '_score',
-                },
-              },
-            },
-            result: {
-              top_hits: {
-                _source: [
-                  'workspace_id',
-                  'doc_id',
-                ],
-                fields: [
-                  'flavour',
-                  'doc_id',
-                  'ref_doc_id',
-                ],
-                highlight: {
-                  fields: {
-                    content: {
-                      post_tags: [
-                        '</b>',
-                      ],
-                      pre_tags: [
-                        '<b>',
-                      ],
-                    },
-                  },
-                },
-              },
-            },
-          },
-          terms: {
-            field: 'flavour',
-            order: {
-              max_score: 'desc',
-            },
-            size: undefined,
-          },
-        },
-      },
-      query: {
-        match_all: {},
-      },
-      sort: [
-        '_score',
-        {
-          updated_at: 'desc',
-        },
-        'id',
-      ],
-    }
-
-## should search work
-
-> Snapshot 1
-
-    [
-      {
-        fields: {
-          summary: [
-            'this is a test',
-          ],
-          title: [
-            'hello world',
-          ],
-        },
-        highlights: {
-          title: [
-            '<b>hello</b> world',
-          ],
-        },
-      },
-    ]
-
-> Snapshot 2
-
-    [
-      {
-        fields: {
-          summary: [
-            '这是测试',
-          ],
-          title: [
-            '你好世界',
-          ],
-        },
-        highlights: {
-          title: [
-            '<b>你好</b> 世界',
-          ],
-        },
-      },
-    ]
-
-## should search with exists query work
-
-> Snapshot 1
-
-    [
-      {
-        fields: {
-          blockId: [
-            'blockId1',
-          ],
-          parentBlockId: [
-            'blockId2',
-          ],
-        },
-      },
-    ]
-
-## should search a doc summary work
-
-> Snapshot 1
-
-    [
-      {
-        fields: {
-          summary: [
-            'hello world, this is a summary',
-          ],
-        },
-      },
-    ]
-
-## should aggregate with bool must_not query work
-
-> Snapshot 1
-
-    [
-      {
-        count: 2,
-        hits: [
-          {
-            fields: {
-              additional: [
-                '{"foo": "bar3"}',
-              ],
-              markdownPreview: [
-                'hello world, this is a title',
-              ],
-              parentBlockId: [
-                'parentBlockId1',
-              ],
-              parentFlavour: [
-                'affine:database',
-              ],
-            },
-          },
-          {
-            fields: {
-              additional: [
-                '{"foo": "bar3"}',
-              ],
-              markdownPreview: [
-                'hello world, this is a title',
-              ],
-              parentBlockId: [
-                'parentBlockId2',
-              ],
-              parentFlavour: [
-                'affine:database',
-              ],
-            },
-          },
-        ],
-      },
-      {
-        count: 1,
-        hits: [
-          {
-            fields: {
-              additional: [
-                '{"foo": "bar3"}',
-              ],
-              markdownPreview: [
-                'hello world, this is a title',
-              ],
-              parentBlockId: [
-                'parentBlockId3',
-              ],
-              parentFlavour: [
-                'affine:database',
-              ],
-            },
-          },
-        ],
-      },
-    ]
-
-## should index doc work
-
-> Snapshot 1
-
-    {
-      summary: [
-        `We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊
-        Airtable & Miro with their no-code programable datasheets␊
-        ␊
-        For developer or installation guides, please go to AFFiNE Development␊
-        Blocks that assemble your next docs, tasks kanban or whiteboard␊
-        ␊
-        Trello with their Kanban␊
-        Remnote & Capacities with their object-based tag system␊
-        AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊
-        There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step fu`,
-      ],
-      title: [
-        'Write, Draw, Plan all at Once.',
-      ],
-    }
-
-> Snapshot 2
-
-    [
-      {
-        blockId: [
-          'VMx9lHw3TR',
-        ],
-        content: [
-          'For developers or installations guides, please go to AFFiNE Doc',
-        ],
-        flavour: [
-          'affine:paragraph',
-        ],
-      },
-      {
-        blockId: [
-          '9-K49otbCv',
-        ],
-        content: [
-          'For developer or installation guides, please go to AFFiNE Development',
-        ],
-        flavour: [
-          'affine:paragraph',
-        ],
-      },
-    ]
-
-## should search blob names from doc snapshot work
-
-> Snapshot 1
-
-    Map {
-      'ldZMrM4PDlsNG4Q4YvCsz623h6TKu4qI9_FpTqIypfw=' => 'test file name here.txt',
-    }
-
-## should search blob names work
-
-> Snapshot 1
-
-    [
-      [
-        'blob1',
-        'blob1 name.txt',
-      ],
-      [
-        'blob2',
-        'blob2 name.md',
-      ],
-      [
-        'blob3',
-        'blob3 name.docx',
-      ],
-    ]
-
-## should search docs by keyword work
-
-> Snapshot 1
-
-    [
-      {
-        blockId: 'block1',
-        createdAt: Date 2025-06-20 00:00:00 UTC {},
-        highlight: '<b>hello</b> world',
-        title: 'hello world',
-        updatedAt: Date 2025-06-20 00:00:00 UTC {},
-      },
-      {
-        blockId: 'block2',
-        createdAt: Date 2025-06-20 00:00:01 UTC {},
-        highlight: '<b>hello</b> world 2',
-        title: 'hello world 2',
-        updatedAt: Date 2025-06-20 00:00:01 UTC {},
-      },
-      {
-        blockId: 'block3',
-        createdAt: Date 2025-06-20 00:00:02 UTC {},
-        highlight: '<b>hello</b> world 3',
-        title: 'hello world 3',
-        updatedAt: Date 2025-06-20 00:00:02 UTC {},
-      },
-      {
-        blockId: 'block4',
-        createdAt: Date 2025-06-20 00:00:03 UTC {},
-        highlight: '<b>hello</b> world 4',
-        title: '',
-        updatedAt: Date 2025-06-20 00:00:03 UTC {},
-      },
-    ]

packages/backend/server/src/plugins/indexer/__tests__/event.spec.ts

@@ -1,11 +1,11 @@
 import test from 'ava';
-import Sinon from 'sinon';
 
 import { createModule } from '../../../__tests__/create-module';
-import { Config } from '../../../base';
+import { JobQueue } from '../../../base';
 import { ConfigModule } from '../../../base/config';
 import { IndexerEvent } from '../event';
 import { IndexerModule } from '../index';
+import { IndexerScheduler } from '../scheduler';
 
 const module = await createModule({
   imports: [
@@ -18,29 +18,12 @@ const module = await createModule({
   ],
 });
 const indexerEvent = module.get(IndexerEvent);
-const config = module.get(Config);
+const indexerScheduler = new IndexerScheduler(module.get(JobQueue));
 
 test.after.always(async () => {
   await module.close();
 });
 
-test.afterEach.always(() => {
-  Sinon.restore();
-});
-
-test('should not index workspace if indexer is disabled', async t => {
-  Sinon.stub(config.indexer, 'enabled').value(false);
-  const count = module.queue.count('indexer.indexWorkspace');
-
-  // @ts-expect-error ignore missing fields
-  await indexerEvent.indexWorkspace({
-    workspaceId: 'test-workspace',
-    docId: 'test-workspace',
-  });
-
-  t.is(module.queue.count('indexer.indexWorkspace'), count);
-});
-
 test('should index workspace when root snapshot is updated', async t => {
   // @ts-expect-error ignore missing fields
   await indexerEvent.indexWorkspace({
@@ -64,19 +47,19 @@ test('should not index workspace when non-root snapshot is updated', async t =>
   t.is(module.queue.count('indexer.indexWorkspace'), count);
 });
 
-test('should not delete workspace if indexer is disabled', async t => {
-  Sinon.stub(config.indexer, 'enabled').value(false);
-  const count = module.queue.count('indexer.deleteWorkspace');
-
-  // @ts-expect-error ignore missing fields
-  await indexerEvent.deleteUserWorkspaces({
-    ownedWorkspaces: ['test-workspace'],
+test('should reindex documents after document access changes', async t => {
+  await indexerEvent.reindexDocOnGrantChange({
+    workspaceId: 'test-workspace',
+    docId: 'test-doc',
+  });
+  const { payload } = await module.queue.waitFor('indexer.indexDoc');
+  t.deepEqual(payload, {
+    workspaceId: 'test-workspace',
+    docId: 'test-doc',
   });
-
-  t.is(module.queue.count('indexer.deleteWorkspace'), count);
 });
 
-test('should delete workspace if indexer is enabled', async t => {
+test('should delete workspace', async t => {
   // @ts-expect-error ignore missing fields
   await indexerEvent.deleteUserWorkspaces({
     ownedWorkspaces: ['test-workspace'],
@@ -86,17 +69,8 @@ test('should delete workspace if indexer is enabled', async t => {
   t.is(payload.workspaceId, 'test-workspace');
 });
 
-test('should not schedule auto index workspaces if indexer is disabled', async t => {
-  Sinon.stub(config.indexer, 'enabled').value(false);
-  const count = module.queue.count('indexer.autoIndexWorkspaces');
-
-  await indexerEvent.autoIndexWorkspaces();
-
-  t.is(module.queue.count('indexer.autoIndexWorkspaces'), count);
-});
-
 test('should schedule auto index workspaces', async t => {
-  await indexerEvent.autoIndexWorkspaces();
+  await indexerScheduler.autoIndexWorkspaces();
 
   const { payload } = await module.queue.waitFor('indexer.autoIndexWorkspaces');
   t.is(payload.lastIndexedWorkspaceSid, undefined);

packages/backend/server/src/plugins/indexer/__tests__/job.spec.ts

@@ -1,26 +1,23 @@
 import { randomUUID } from 'node:crypto';
-import { mock } from 'node:test';
 
 import test from 'ava';
 import Sinon from 'sinon';
 
 import { createModule } from '../../../__tests__/create-module';
 import { Mockers } from '../../../__tests__/mocks';
-import { Config, JOB_SIGNAL } from '../../../base';
+import { JOB_SIGNAL } from '../../../base';
 import { ConfigModule } from '../../../base/config';
 import { ServerConfigModule } from '../../../core/config';
 import { DocReader } from '../../../core/doc';
 import { Models } from '../../../models';
 import { addDocToRootDoc } from '../../../native';
-import { SearchProviderFactory } from '../factory';
-import { IndexerModule, IndexerService } from '../index';
+import { IndexerModule, IndexerService, IndexerWorkerModule } from '../index';
 import { IndexerJob } from '../job';
-import { ManticoresearchProvider } from '../providers';
-import { blockSQL, docSQL, SearchTable } from '../tables';
 
 const module = await createModule({
   imports: [
     IndexerModule,
+    IndexerWorkerModule,
     ServerConfigModule,
     ConfigModule.override({
       indexer: {
@@ -32,11 +29,8 @@ const module = await createModule({
 });
 const indexerService = module.get(IndexerService);
 const indexerJob = module.get(IndexerJob);
-const searchProviderFactory = module.get(SearchProviderFactory);
-const manticoresearch = module.get(ManticoresearchProvider);
 const models = module.get(Models);
 const docReader = module.get(DocReader);
-const config = module.get(Config);
 
 const user = await module.create(Mockers.User);
 const workspace = await module.create(Mockers.Workspace, {
@@ -44,24 +38,12 @@ const workspace = await module.create(Mockers.Workspace, {
   owner: user,
 });
 
-test.before(async () => {
-  await manticoresearch.recreateTable(SearchTable.block, blockSQL);
-  await manticoresearch.recreateTable(SearchTable.doc, docSQL);
-});
-
 test.after.always(async () => {
   await module.close();
 });
 
 test.afterEach.always(() => {
   Sinon.restore();
-  mock.reset();
-});
-
-test.beforeEach(() => {
-  mock.method(searchProviderFactory, 'get', () => {
-    return manticoresearch;
-  });
 });
 
 test('should handle indexer.indexDoc job', async t => {
@@ -83,54 +65,19 @@ test('should handle indexer.deleteDoc job', async t => {
 });
 
 test('should handle indexer.indexWorkspace job', async t => {
-  const count = module.queue.count('indexer.deleteDoc');
-  const spy = Sinon.spy(indexerService, 'listDocIds');
+  const spy = Sinon.stub(indexerService, 'reconcileWorkspace').resolves();
 
   await indexerJob.indexWorkspace({
     workspaceId: workspace.id,
   });
 
-  t.is(spy.callCount, 1);
-  const { payload } = await module.queue.waitFor('indexer.indexDoc');
-  t.is(payload.workspaceId, workspace.id);
-  t.is(payload.docId, '5nS9BSp3Px');
-  // no delete job
-  t.is(module.queue.count('indexer.deleteDoc'), count);
+  t.true(spy.calledOnceWith(workspace.id));
 
   // workspace should be indexed
   const ws = await models.workspace.get(workspace.id);
   t.is(ws!.indexed, true);
 });
 
-test('should not sync existing doc', async t => {
-  const count = module.queue.count('indexer.indexDoc');
-  mock.method(indexerService, 'listDocIds', async () => {
-    return ['5nS9BSp3Px'];
-  });
-
-  await indexerJob.indexWorkspace({
-    workspaceId: workspace.id,
-  });
-
-  t.is(module.queue.count('indexer.indexDoc'), count);
-});
-
-test('should delete dangling indexed docs absent from the root live set', async t => {
-  const count = module.queue.count('indexer.deleteDoc');
-  mock.method(indexerService, 'listDocIds', async () => {
-    return ['mock-doc-id1', 'mock-doc-id2'];
-  });
-
-  await indexerJob.indexWorkspace({
-    workspaceId: workspace.id,
-  });
-
-  const { payload } = await module.queue.waitFor('indexer.indexDoc');
-  t.is(payload.workspaceId, workspace.id);
-  t.is(payload.docId, '5nS9BSp3Px');
-  t.is(module.queue.count('indexer.deleteDoc'), count + 2);
-});
-
 test('document cleanup reconcile deletes missing search state before ack', async t => {
   const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
   const indexSpy = Sinon.spy(indexerService, 'indexDoc');
@@ -203,32 +150,6 @@ test('document cleanup reconcile reindexes restored doc before ack', async t =>
   });
 });
 
-test('document cleanup reconcile only acknowledges when indexer is disabled', async t => {
-  Sinon.stub(config.indexer, 'enabled').value(false);
-  const deleteSpy = Sinon.spy(indexerService, 'deleteDoc');
-  const indexSpy = Sinon.spy(indexerService, 'indexDoc');
-  const getDocSpy = Sinon.spy(docReader, 'getDoc');
-
-  await indexerJob.reconcileDocumentCleanup({
-    workspaceId: workspace.id,
-    docId: 'disabled-doc',
-    cleanupVersion: 'version-disabled',
-  });
-
-  t.false(deleteSpy.called);
-  t.false(indexSpy.called);
-  t.false(getDocSpy.called);
-  const { payload } = await module.queue.waitFor(
-    'backendRuntime.ackDocumentCleanupEffect'
-  );
-  t.deepEqual(payload, {
-    workspaceId: workspace.id,
-    docId: 'disabled-doc',
-    cleanupVersion: 'version-disabled',
-    effect: 'search',
-  });
-});
-
 test('should handle indexer.deleteWorkspace job', async t => {
   const spy = Sinon.spy(indexerService, 'deleteWorkspace');
 

packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md

@@ -1,727 +0,0 @@
-# Snapshot report for `src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts`
-
-The actual snapshot is saved in `elasticsearch.spec.ts.snap`.
-
-Generated by [AVA](https://avajs.dev).
-
-## should batch write bugfix
-
-> Snapshot 1
-
-    [
-      {
-        _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b1',
-        _source: {
-          doc_id: 'a',
-          workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch',
-        },
-        fields: {
-          block_id: [
-            'b1',
-          ],
-          doc_id: [
-            'a',
-          ],
-          workspace_id: [
-            'workspaceId-batch-write-bugfix-for-elasticsearch',
-          ],
-        },
-        highlights: undefined,
-      },
-      {
-        _id: 'workspaceId-batch-write-bugfix-for-elasticsearch/a/b2',
-        _source: {
-          doc_id: 'a',
-          workspace_id: 'workspaceId-batch-write-bugfix-for-elasticsearch',
-        },
-        fields: {
-          block_id: [
-            'b2',
-          ],
-          doc_id: [
-            'a',
-          ],
-          workspace_id: [
-            'workspaceId-batch-write-bugfix-for-elasticsearch',
-          ],
-        },
-        highlights: undefined,
-      },
-    ]
-
-## should search block table query match url work
-
-> Snapshot 1
-
-    {
-      _id: 'workspaceId1/docId2/blockId8',
-      _source: {
-        doc_id: 'docId2',
-        workspace_id: 'workspaceId1',
-      },
-      fields: {
-        additional: [
-          'additional8',
-        ],
-        content: [
-          'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2',
-        ],
-        created_at: [
-          Date 2025-03-08 06:04:13 278ms UTC {},
-        ],
-        doc_id: [
-          'docId2',
-        ],
-        markdown_preview: [
-          'markdownPreview8',
-        ],
-        parent_block_id: [
-          'parentBlockId8',
-        ],
-        parent_flavour: [
-          'parentFlavour8',
-        ],
-        ref: [
-          '{"docId":"docId1","mode":"page"}',
-          '{"docId":"docId2","mode":"page"}',
-        ],
-        ref_doc_id: [
-          'docId1',
-        ],
-        updated_at: [
-          Date 2025-03-08 06:04:13 278ms UTC {},
-        ],
-      },
-      highlights: {
-        content: [
-          'hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some <b>link</b>',
-          '<b>https</b>://<b>linear.app</b>/<b>affine</b>-<b>design</b>/<b>issue</b>/<b>AF</b>-<b>1379</b>/<b>slash</b>-<b>commands</b>-%<b>E6</b>%<b>BF</b>%<b>80</b>%<b>E6</b>%<b>B4</b>%<b>BB</b>%<b>E6</b>%<b>8F</b>%<b>92</b>%<b>E5</b>%<b>85</b>%<b>A5</b>-<b>link</b>',
-          '-%<b>E7</b>%<b>9A</b>%<b>84</b>%<b>E5</b>%<b>BC</b>%<b>B9</b>%<b>E7</b>%<b>AA</b>%<b>97</b>%<b>E9</b>%<b>87</b>%<b>8C</b>%<b>EF</b>%<b>BC</b>%<b>8C</b>%<b>E8</b>%<b>BE</b>%<b>93</b>%<b>E5</b>%<b>85</b>%<b>A5</b>%<b>E9</b>%<b>93</b>%<b>BE</b>%<b>E6</b>%<b>8E</b>%<b>A5</b>%<b>E4</b>%<b>B9</b>%<b>8B</b>%<b>E5</b>%<b>90</b>%<b>8E</b>%',
-          '<b>E4</b>%<b>B8</b>%<b>8D</b>%<b>E5</b>%<b>BA</b>%<b>94</b>%<b>E8</b>%<b>AF</b>%<b>A5</b>%<b>E7</b>%<b>9B</b>%<b>B4</b>%<b>E6</b>%<b>8E</b>%<b>A5</b>%<b>E5</b>%<b>AF</b>%<b>B9</b>%<b>E9</b>%<b>93</b>%<b>BE</b>%<b>E6</b>%<b>8E</b>%<b>A5</b>%<b>E8</b>%<b>BF</b>%<b>9B</b>%<b>E8</b>%<b>A1</b>%<b>8C</b>%<b>E5</b>%<b>88</b>%<b>86</b>%<b>E8</b>%',
-          '<b>AF</b>%<b>8D</b>%<b>E6</b>%<b>90</b>%<b>9C</b>%<b>E7</b>%<b>B4</b>%<b>A2</b>',
-        ],
-      },
-    }
-
-> Snapshot 2
-
-    {
-      _id: 'workspaceId1/docId2/blockId8',
-      _source: {
-        doc_id: 'docId2',
-        workspace_id: 'workspaceId1',
-      },
-      fields: {
-        additional: [
-          'additional8',
-        ],
-        content: [
-          'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2',
-        ],
-        created_at: [
-          Date 2025-03-08 06:04:13 278ms UTC {},
-        ],
-        doc_id: [
-          'docId2',
-        ],
-        markdown_preview: [
-          'markdownPreview8',
-        ],
-        parent_block_id: [
-          'parentBlockId8',
-        ],
-        parent_flavour: [
-          'parentFlavour8',
-        ],
-        ref: [
-          '{"docId":"docId1","mode":"page"}',
-          '{"docId":"docId2","mode":"page"}',
-        ],
-        ref_doc_id: [
-          'docId1',
-        ],
-        updated_at: [
-          Date 2025-03-08 06:04:13 278ms UTC {},
-        ],
-      },
-      highlights: {
-        content: [
-          'hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link <b>https</b>',
-          '://<b>linear.app</b>/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%',
-        ],
-      },
-    }
-
-## should search block table query content match cjk work
-
-> Snapshot 1
-
-    {
-      _id: 'workspaceId1/docId2-affine/blockId8',
-      _source: {
-        doc_id: 'docId2-affine',
-        workspace_id: 'workspaceId1',
-      },
-      fields: {
-        content: [
-          'AFFiNE 是一个基于云端的笔记应用',
-        ],
-        doc_id: [
-          'docId2-affine',
-        ],
-        flavour: [
-          'flavour8',
-        ],
-      },
-      highlights: {
-        content: [
-          'AFFiNE 是一个基于云端的<b>笔记应用</b>',
-        ],
-      },
-    }
-
-> Snapshot 2
-
-    {
-      _id: 'workspaceId1/docId2-affine/blockId8',
-      _source: {
-        doc_id: 'docId2-affine',
-        workspace_id: 'workspaceId1',
-      },
-      fields: {
-        content: [
-          'AFFiNE 是一个基于云端的笔记应用',
-        ],
-        doc_id: [
-          'docId2-affine',
-        ],
-        flavour: [
-          'flavour8',
-        ],
-      },
-      highlights: {
-        content: [
-          'AFFiNE 是一个基于云端的笔<b>记</b>应用',
-        ],
-      },
-    }
-
-## should search doc table query title match cjk work
-
-> Snapshot 1
-
-    {
-      _id: 'workspace-test-doc-title-cjk/doc-0',
-      _source: {
-        doc_id: 'doc-0',
-        workspace_id: 'workspace-test-doc-title-cjk',
-      },
-      fields: {
-        doc_id: [
-          'doc-0',
-        ],
-        title: [
-          'AFFiNE 是一个基于云端的笔记应用',
-        ],
-      },
-      highlights: {
-        title: [
-          'AFFiNE 是一个基于云端的<b>笔记应</b>用',
-        ],
-      },
-    }
-
-> Snapshot 2
-
-    {
-      _id: 'workspace-test-doc-title-cjk/doc-0',
-      _source: {
-        doc_id: 'doc-0',
-        workspace_id: 'workspace-test-doc-title-cjk',
-      },
-      fields: {
-        doc_id: [
-          'doc-0',
-        ],
-        title: [
-          'AFFiNE 是一个基于云端的笔记应用',
-        ],
-      },
-      highlights: {
-        title: [
-          'AFFiNE 是一个基于云端的<b>笔</b>记应用',
-        ],
-      },
-    }
-
-## should search doc table query title.autocomplete work
-
-> Snapshot 1
-
-    {
-      _id: 'workspace-test-doc-title-autocomplete/doc-0',
-      _source: {
-        doc_id: 'doc-0',
-        workspace_id: 'workspace-test-doc-title-autocomplete',
-      },
-      fields: {
-        doc_id: [
-          'doc-0',
-        ],
-        title: [
-          'AFFiNE 是一个基于云端的笔记应用',
-        ],
-      },
-      highlights: {
-        'title.autocomplete': [
-          '<b>AFF</b>iNE 是一个基于云端的笔记应用',
-        ],
-      },
-    }
-
-## should search query match ref_doc_id work
-
-> Snapshot 1
-
-    [
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar0"}',
-          ],
-          block_id: [
-            'blockId1',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId1',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-1',
-          ],
-        },
-      },
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar1"}',
-          ],
-          block_id: [
-            'blockId-all',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId2',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-2',
-            'doc-3',
-            'doc-4',
-            'doc-5',
-            'doc-6',
-            'doc-7',
-            'doc-8',
-            'doc-9',
-            'doc-10',
-            'doc-1',
-          ],
-        },
-      },
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar1"}',
-          ],
-          block_id: [
-            'blockId1-2',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId2',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-1',
-            'doc-2',
-          ],
-        },
-      },
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar1"}',
-          ],
-          block_id: [
-            'blockId2-1',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId2',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-2',
-            'doc-1',
-          ],
-        },
-      },
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar1"}',
-          ],
-          block_id: [
-            'blockId3-2-1-4',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId2',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-3',
-            'doc-2',
-            'doc-1',
-            'doc-4',
-          ],
-        },
-      },
-    ]
-
-> Snapshot 2
-
-    [
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar1"}',
-          ],
-          block_id: [
-            'blockId-all',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId2',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-2',
-            'doc-3',
-            'doc-4',
-            'doc-5',
-            'doc-6',
-            'doc-7',
-            'doc-8',
-            'doc-9',
-            'doc-10',
-            'doc-1',
-          ],
-        },
-      },
-      {
-        fields: {
-          additional: [
-            '{"foo": "bar3"}',
-          ],
-          block_id: [
-            'blockId4',
-          ],
-          doc_id: [
-            'doc-0',
-          ],
-          parent_block_id: [
-            'parentBlockId4',
-          ],
-          parent_flavour: [
-            'affine:database',
-          ],
-          ref_doc_id: [
-            'doc-10',
-          ],
-        },
-      },
-    ]
-
-## should search doc title support stemmer filter
-
-> Snapshot 1
-
-    {
-      _id: 'workspace-test-doc-title-stemmer-filter/doc-0',
-      _source: {
-        doc_id: 'doc-0',
-        workspace_id: 'workspace-test-doc-title-stemmer-filter',
-      },
-      fields: {
-        doc_id: [
-          'doc-0',
-        ],
-        title: [
-          'Deploy on Windows by a designer',
-        ],
-      },
-      highlights: {
-        title: [
-          'Deploy on <b>Windows</b> by a designer',
-        ],
-      },
-    }
-
-> Snapshot 2
-
-    {
-      _id: 'workspace-test-doc-title-stemmer-filter/doc-0',
-      _source: {
-        doc_id: 'doc-0',
-        workspace_id: 'workspace-test-doc-title-stemmer-filter',
-      },
-      fields: {
-        doc_id: [
-          'doc-0',
-        ],
-        title: [
-          'Deploy on Windows by a designer',
-        ],
-      },
-      highlights: {
-        title: [
-          'Deploy on <b>Windows</b> by a designer',
-        ],
-      },
-    }
-
-> Snapshot 3
-
-    {
-      _id: 'workspace-test-doc-title-stemmer-filter/doc-0',
-      _source: {
-        doc_id: 'doc-0',
-        workspace_id: 'workspace-test-doc-title-stemmer-filter',
-      },
-      fields: {
-        doc_id: [
-          'doc-0',
-        ],
-        title: [
-          'Deploy on Windows by a designer',
-        ],
-      },
-      highlights: {
-        title: [
-          'Deploy on Windows by a <b>designer</b>',
-        ],
-      },
-    }
-
-## should return empty string field:summary value
-
-> Snapshot 1
-
-    [
-      {
-        _id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch/doc0',
-        _source: {
-          doc_id: 'doc0',
-          workspace_id: 'workspaceId-search-query-return-empty-string-field-summary-value-for-elasticsearch',
-        },
-        fields: {
-          doc_id: [
-            'doc0',
-          ],
-          summary: [
-            '',
-          ],
-          title: [
-            '',
-          ],
-        },
-        highlights: undefined,
-      },
-    ]
-
-## should not return not exists field:ref_doc_id
-
-> Snapshot 1
-
-    [
-      {
-        _id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch/doc0/block0',
-        _source: {
-          doc_id: 'doc0',
-          workspace_id: 'workspaceId-search-query-not-return-not-exists-field-ref_doc_id-for-elasticsearch',
-        },
-        fields: {
-          block_id: [
-            'block0',
-          ],
-          doc_id: [
-            'doc0',
-          ],
-        },
-        highlights: undefined,
-      },
-    ]
-
-## should aggregate query work
-
-> Snapshot 1
-
-    [
-      {
-        _id: 'workspaceId1/docId9/blockId9',
-        _source: {
-          doc_id: 'docId9',
-          workspace_id: 'workspaceId1',
-        },
-        fields: {
-          block_id: [
-            'blockId9',
-          ],
-          flavour: [
-            'affine:page',
-          ],
-        },
-        highlights: {
-          content: [
-            'title9 <b>hello</b> affine issue <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b>, <b>hello</b> <b>hello</b> <b>hello</b>',
-            '<b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b> <b>hello</b>',
-          ],
-        },
-      },
-    ]
-
-## should aggregate query return top score first
-
-> Snapshot 1
-
-    [
-      {
-        count: 1,
-        hits: [
-          {
-            _id: 'aggregate-test-workspace-top-score-max-first/doc-0/block-0',
-            _source: {
-              doc_id: 'doc-0',
-              workspace_id: 'aggregate-test-workspace-top-score-max-first',
-            },
-            fields: {
-              block_id: [
-                'block-0',
-              ],
-              flavour: [
-                'affine:page',
-              ],
-            },
-            highlights: {
-              content: [
-                '<b>0.15</b> - <b>week</b>.<b>1</b>进度',
-              ],
-            },
-          },
-        ],
-        key: 'doc-0',
-      },
-      {
-        count: 2,
-        hits: [
-          {
-            _id: 'aggregate-test-workspace-top-score-max-first/doc-10/block-10-1',
-            _source: {
-              doc_id: 'doc-10',
-              workspace_id: 'aggregate-test-workspace-top-score-max-first',
-            },
-            fields: {
-              block_id: [
-                'block-10-1',
-              ],
-              flavour: [
-                'affine:paragraph',
-              ],
-            },
-            highlights: {
-              content: [
-                'Example <b>1</b>',
-              ],
-            },
-          },
-          {
-            _id: 'aggregate-test-workspace-top-score-max-first/doc-10/block-10-2',
-            _source: {
-              doc_id: 'doc-10',
-              workspace_id: 'aggregate-test-workspace-top-score-max-first',
-            },
-            fields: {
-              block_id: [
-                'block-10-2',
-              ],
-              flavour: [
-                'affine:paragraph',
-              ],
-            },
-            highlights: {
-              content: [
-                'Single substitution format <b>1</b>',
-              ],
-            },
-          },
-        ],
-        key: 'doc-10',
-      },
-    ]
-
-> Snapshot 2
-
-    [
-      {
-        count: 1,
-        hits: [
-          {
-            _id: 'aggregate-test-workspace-top-score-max-first/doc-0/block-0',
-            _source: {
-              doc_id: 'doc-0',
-              workspace_id: 'aggregate-test-workspace-top-score-max-first',
-            },
-            fields: {
-              block_id: [
-                'block-0',
-              ],
-              flavour: [
-                'affine:page',
-              ],
-            },
-            highlights: {
-              content: [
-                '<b>0.15</b> - <b>week</b>.<b>1</b>进度',
-              ],
-            },
-          },
-        ],
-        key: 'doc-0',
-      },
-    ]