feat(server): improve context management (#15448)

#### PR Dependency Tree


* **PR #15448** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added workspace artifact upload, browsing, removal, deduplication, and
library ownership support.
* Copilot now supports scoped document and artifact search, canvas
reading, live editor context, and frontend tools.
* Added scope and focus selectors with source-resolution receipts in
chat.
* Added embedding health, progress, synchronization, and retrieval
capabilities.
* Added BYOK policy visibility, provider restrictions, endpoint dialect
selection, and validation.
* Added delegated editor interactions and userdata document
authorization.

* **Bug Fixes**
* Improved attachment handling, cancellation, access control, retrieval
fallbacks, workspace synchronization, and configuration validation.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/plugins/indexer/__tests__/__snapshots__/service.spec.ts.md

@@ -461,7 +461,16 @@ Generated by [AVA](https://avajs.dev).
 
     {
       summary: [
-        'AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. You own your data, with no compromisesLocal-first & Real-time collaborativeWe love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.Blocks that assemble your next docs, tasks kanban or whiteboardThere is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further. ',
+        `We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊
+        Airtable & Miro with their no-code programable datasheets␊
+        ␊
+        For developer or installation guides, please go to AFFiNE Development␊
+        Blocks that assemble your next docs, tasks kanban or whiteboard␊
+        ␊
+        Trello with their Kanban␊
+        Remnote & Capacities with their object-based tag system␊
+        AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro. ␊
+        There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step fu`,
       ],
       title: [
         'Write, Draw, Plan all at Once.',

packages/backend/server/src/plugins/indexer/__tests__/service.spec.ts

@@ -2044,7 +2044,6 @@ test('should list doc ids work', async t => {
 // #region indexDoc()
 
 test('should index doc work', async t => {
-  const count = module.queue.count('copilot.embedding.updateDoc');
   const docSnapshot = await module.create(Mockers.DocSnapshot, {
     workspaceId: workspace.id,
     user,
@@ -2092,7 +2091,16 @@ test('should index doc work', async t => {
       ],
     },
     options: {
-      fields: ['workspaceId', 'docId', 'blockId', 'content', 'flavour'],
+      fields: [
+        'workspaceId',
+        'docId',
+        'blockId',
+        'unitId',
+        'projectionVersion',
+        'sourceHash',
+        'content',
+        'flavour',
+      ],
       highlights: [
         {
           field: 'content',
@@ -2107,10 +2115,26 @@ test('should index doc work', async t => {
   });
 
   t.is(result2.nodes.length, 2);
-  t.snapshot(
-    result2.nodes.map(node => omit(node.fields, ['workspaceId', 'docId']))
+  t.true(
+    result2.nodes.every(
+      node =>
+        node.fields.unitId.length === 1 &&
+        node.fields.projectionVersion[0] === 1 &&
+        node.fields.sourceHash.length === 1
+    )
+  );
+  t.is(new Set(result2.nodes.map(node => node.fields.sourceHash[0])).size, 1);
+  t.snapshot(
+    result2.nodes.map(node =>
+      omit(node.fields, [
+        'workspaceId',
+        'docId',
+        'unitId',
+        'projectionVersion',
+        'sourceHash',
+      ])
+    )
   );
-  t.is(module.queue.count('copilot.embedding.updateDoc'), count + 1);
 });
 // #endregion
 

packages/backend/server/src/plugins/indexer/service.ts

@@ -6,7 +6,7 @@ import {
   JobQueue,
   SearchProviderNotFound,
 } from '../../base';
-import { readAllBlocksFromDocSnapshot } from '../../core/utils/blocksuite';
+import { projectDocSearch } from '../../core/utils/blocksuite';
 import { Models } from '../../models';
 import { SearchProviderType } from './config';
 import { SearchProviderFactory } from './factory';
@@ -284,9 +284,10 @@ export class IndexerService {
     };
 
     try {
-      const result = await readAllBlocksFromDocSnapshot(
+      const projection = projectDocSearch(
+        docSnapshot.blob,
         docId,
-        docSnapshot.blob
+        docSnapshot.updatedAt.getTime().toString()
       );
       await this.write(
         SearchTable.doc,
@@ -294,8 +295,11 @@ export class IndexerService {
           {
             workspaceId,
             docId,
-            title: result.title,
-            summary: result.summary,
+            title: projection.title,
+            summary: projection.units
+              .map(unit => unit.text)
+              .join('\n')
+              .slice(0, 1000),
             // NOTE(@fengmk): journal is not supported yet
             // journal: result.journal,
             createdByUserId: docSnapshot.createdBy ?? '',
@@ -309,20 +313,25 @@ export class IndexerService {
       await this.deleteBlocksByDocId(workspaceId, docId, options);
       await this.write(
         SearchTable.block,
-        result.blocks.map(block => ({
+        projection.units.map(unit => ({
           workspaceId,
           docId,
-          blockId: block.blockId,
-          content: block.content ?? '',
-          flavour: block.flavour,
-          blob: block.blob,
-          refDocId: block.refDocId,
-          ref: block.ref,
-          parentFlavour: block.parentFlavour,
-          parentBlockId: block.parentBlockId,
-          additional: block.additional
-            ? JSON.stringify(block.additional)
-            : undefined,
+          blockId: unit.blockId ?? unit.unitId,
+          unitId: unit.unitId,
+          projectionVersion: projection.version,
+          sourceHash: projection.sourceHash,
+          visibility: unit.visibility,
+          elementId: unit.elementId,
+          frameId: unit.frameId,
+          sourceBlockId: unit.blockId,
+          blob: unit.blobId,
+          refDocId: unit.refDocIds.length ? unit.refDocIds : undefined,
+          ref: unit.refs.length ? unit.refs : undefined,
+          content: unit.text,
+          flavour: `affine:${unit.type}`,
+          parentFlavour: unit.parentFlavour,
+          parentBlockId: unit.parentBlockId,
+          additional: unit.additional,
           markdownPreview: undefined,
           createdByUserId: docSnapshot.createdBy ?? '',
           updatedByUserId: docSnapshot.updatedBy ?? '',
@@ -332,12 +341,8 @@ export class IndexerService {
         options
       );
 
-      await this.queue.add('copilot.embedding.updateDoc', {
-        workspaceId,
-        docId,
-      });
       this.logger.verbose(
-        `synced doc ${workspaceId}/${docId} with ${result.blocks.length} blocks`
+        `synced doc ${workspaceId}/${docId} with ${projection.units.length} search units`
       );
     } catch (err) {
       this.logger.warn(
@@ -559,6 +564,13 @@ export class IndexerService {
         hits: {
           fields: [
             'blockId',
+            'unitId',
+            'projectionVersion',
+            'sourceHash',
+            'visibility',
+            'elementId',
+            'frameId',
+            'sourceBlockId',
             'flavour',
             'content',
             'createdAt',
@@ -590,6 +602,20 @@ export class IndexerService {
     for (const bucket of result.buckets) {
       const docId = bucket.key;
       const blockId = bucket.hits.nodes[0].fields.blockId[0] as string;
+      const unitId = bucket.hits.nodes[0].fields.unitId[0] as string;
+      const projectionVersion = bucket.hits.nodes[0].fields
+        .projectionVersion[0] as number;
+      const sourceHash = bucket.hits.nodes[0].fields.sourceHash[0] as string;
+      const visibility = bucket.hits.nodes[0].fields.visibility[0] as string;
+      const elementId = bucket.hits.nodes[0].fields.elementId?.[0] as
+        | string
+        | undefined;
+      const frameId = bucket.hits.nodes[0].fields.frameId?.[0] as
+        | string
+        | undefined;
+      const sourceBlockId = bucket.hits.nodes[0].fields.sourceBlockId?.[0] as
+        | string
+        | undefined;
       const flavour = bucket.hits.nodes[0].fields.flavour[0] as string;
       const content = bucket.hits.nodes[0].fields.content[0] as string;
       const createdAt = bucket.hits.nodes[0].fields.createdAt[0] as Date;
@@ -611,7 +637,13 @@ export class IndexerService {
 
       docs.push({
         docId,
-        blockId,
+        blockId: sourceBlockId || blockId,
+        ...(unitId ? { unitId } : {}),
+        ...(projectionVersion ? { projectionVersion } : {}),
+        ...(sourceHash ? { sourceHash } : {}),
+        ...(visibility ? { visibility } : {}),
+        ...(elementId ? { elementId } : {}),
+        ...(frameId ? { frameId } : {}),
         title,
         highlight,
         createdAt,

packages/backend/server/src/plugins/indexer/tables/block.ts

@@ -4,6 +4,13 @@ export const BlockSchema = z.object({
   workspace_id: z.string(),
   doc_id: z.string(),
   block_id: z.string(),
+  unit_id: z.string().optional(),
+  projection_version: z.number().int().optional(),
+  source_hash: z.string().optional(),
+  visibility: z.string().optional(),
+  element_id: z.string().optional(),
+  frame_id: z.string().optional(),
+  source_block_id: z.string().optional(),
   content: z.union([z.string(), z.string().array()]),
   flavour: z.string(),
   blob: z.union([z.string(), z.string().array()]).optional(),
@@ -75,6 +82,13 @@ export const blockMapping = {
       block_id: {
         type: 'keyword',
       },
+      unit_id: { type: 'keyword' },
+      projection_version: { type: 'integer' },
+      source_hash: { type: 'keyword' },
+      visibility: { type: 'keyword' },
+      element_id: { type: 'keyword' },
+      frame_id: { type: 'keyword' },
+      source_block_id: { type: 'keyword' },
       content: {
         type: 'text',
         analyzer: 'standard_with_cjk',
@@ -128,6 +142,13 @@ CREATE TABLE IF NOT EXISTS block (
   workspace_id string attribute,
   doc_id string attribute,
   block_id string attribute,
+  unit_id string attribute,
+  projection_version int,
+  source_hash string attribute,
+  visibility string attribute,
+  element_id string attribute,
+  frame_id string attribute,
+  source_block_id string attribute,
   content text,
   flavour string attribute,
   -- use flavour_indexed to match with boost

packages/backend/server/src/plugins/indexer/types.ts

@@ -45,6 +45,12 @@ registerEnumType(SearchQueryOccur, {
 export interface SearchDoc {
   docId: string;
   blockId: string;
+  unitId?: string;
+  projectionVersion?: number;
+  sourceHash?: string;
+  visibility?: string;
+  elementId?: string;
+  frameId?: string;
   title: string;
   highlight: string;
   createdAt: Date;