feat(core): extract md & snapshot & attachments from selected (#13312)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary by CodeRabbit

* **New Features**
* Enhanced extraction of selected content in the editor to include
document snapshots, markdown summaries, and attachments for both
edgeless and page modes.
* Attachments related to selected content are now available in chat and
input contexts, providing additional metadata.
* Added utility to identify and retrieve selected attachments in editor
content.

* **Bug Fixes**
* Improved consistency in attachment retrieval when extracting selected
content.

* **Chores**
* Updated dependencies and workspace references to include new block
suite components.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

> CLOSE AF-2770

blocksuite/affine/blocks/root/src/edgeless/index.ts

@@ -4,6 +4,6 @@ export * from './clipboard/command';
 export * from './edgeless-root-block.js';
 export { EdgelessRootService } from './edgeless-root-service.js';
 export * from './utils/clipboard-utils.js';
-export { sortEdgelessElements } from './utils/clone-utils.js';
+export { getElementProps, sortEdgelessElements } from './utils/clone-utils.js';
 export { isCanvasElement } from './utils/query.js';
 export { EDGELESS_BLOCK_CHILD_PADDING } from '@blocksuite/affine-shared/consts';

packages/frontend/core/package.json

@@ -20,6 +20,7 @@
     "@affine/templates": "workspace:*",
     "@affine/track": "workspace:*",
     "@blocksuite/affine": "workspace:*",
+    "@blocksuite/affine-block-root": "workspace:*",
     "@blocksuite/affine-components": "workspace:*",
     "@blocksuite/affine-shared": "workspace:*",
     "@blocksuite/global": "workspace:*",

packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/ai-chat-content.ts

@@ -48,6 +48,9 @@ const DEFAULT_CHAT_CONTEXT_VALUE: ChatContextValue = {
   status: 'idle',
   error: null,
   markdown: '',
+  snapshot: null,
+  attachments: [],
+  combinedElementsMarkdown: null,
 };
 
 export class AIChatContent extends SignalWatcher(

packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/type.ts

@@ -12,5 +12,11 @@ export type ChatContextValue = {
   markdown: string;
   // images of the selected content or user uploaded
   images: File[];
+  // snapshot of the selected content
+  snapshot: string | null;
+  // attachments of the selected content
+  attachments: { sourceId: string; name: string }[];
+  // combined markdown of the selected elements
+  combinedElementsMarkdown: string | null;
   abortController: AbortController | null;
 };

packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/type.ts

@@ -1,6 +1,7 @@
 import type { Signal } from '@preact/signals-core';
 
 import type { AIError } from '../../provider';
+import type { ChatContextValue } from '../ai-chat-content';
 import type { ChatStatus, HistoryMessage } from '../ai-chat-messages';
 
 export interface AINetworkSearchConfig {
@@ -27,4 +28,7 @@ export type AIChatInputContext = {
   markdown?: string;
   images: File[];
   abortController: AbortController | null;
-};
+} & Pick<
+  ChatContextValue,
+  'snapshot' | 'combinedElementsMarkdown' | 'attachments'
+>;

packages/frontend/core/src/blocksuite/ai/components/playground/chat.ts

@@ -45,6 +45,9 @@ const DEFAULT_CHAT_CONTEXT_VALUE: ChatContextValue = {
   status: 'idle',
   error: null,
   markdown: '',
+  snapshot: null,
+  attachments: [],
+  combinedElementsMarkdown: null,
 };
 
 export class PlaygroundChat extends SignalWatcher(

packages/frontend/core/src/blocksuite/ai/utils/attachment.ts

@@ -0,0 +1,9 @@
+import { AttachmentBlockModel } from '@blocksuite/affine/model';
+import type { BlockModel } from '@blocksuite/affine/store';
+import type { GfxModel } from '@blocksuite/std/gfx';
+
+export function isAttachment(
+  model: GfxModel | BlockModel
+): model is AttachmentBlockModel {
+  return model instanceof AttachmentBlockModel;
+}

packages/frontend/core/src/blocksuite/ai/utils/extract.ts

@@ -1,7 +1,9 @@
+import { WorkspaceImpl } from '@affine/core/modules/workspace/impls/workspace';
+import { getSurfaceBlock } from '@blocksuite/affine/blocks/surface';
 import {
   DatabaseBlockModel,
   ImageBlockModel,
-  type NoteBlockModel,
+  NoteBlockModel,
   NoteDisplayMode,
 } from '@blocksuite/affine/model';
 import {
@@ -15,16 +17,25 @@ import {
 } from '@blocksuite/affine/shared/commands';
 import { DocModeProvider } from '@blocksuite/affine/shared/services';
 import {
+  getBlockProps,
   isInsideEdgelessEditor,
   matchModels,
 } from '@blocksuite/affine/shared/utils';
 import { BlockStdScope, type EditorHost } from '@blocksuite/affine/std';
-import type { BlockModel, Store } from '@blocksuite/affine/store';
+import {
+  GfxControllerIdentifier,
+  GfxPrimitiveElementModel,
+} from '@blocksuite/affine/std/gfx';
+import type { BlockModel, DocSnapshot, Store } from '@blocksuite/affine/store';
 import { Slice, toDraftModel } from '@blocksuite/affine/store';
+import { getElementProps } from '@blocksuite/affine-block-root';
+import { Doc as YDoc } from 'yjs';
 
 import { getStoreManager } from '../../manager/store';
 import type { ChatContextValue } from '../components/ai-chat-content';
+import { isAttachment } from './attachment';
 import {
+  getSelectedAttachments,
   getSelectedImagesAsBlobs,
   getSelectedTextContent,
   selectedToCanvas,
@@ -46,6 +57,69 @@ async function extractEdgelessSelected(
   host: EditorHost
 ): Promise<Partial<ChatContextValue> | null> {
   if (!isInsideEdgelessEditor(host)) return null;
+  const gfx = host.std.get(GfxControllerIdentifier);
+  const selectedElements = gfx.selection.selectedElements;
+
+  let snapshot: DocSnapshot | null = null;
+  let markdown = '';
+  const attachments: ChatContextValue['attachments'] = [];
+
+  if (selectedElements.length) {
+    const transformer = host.store.getTransformer();
+    const markdownAdapter = new MarkdownAdapter(
+      transformer,
+      host.store.provider
+    );
+    const collection = new WorkspaceImpl({
+      id: 'AI_EXTRACT',
+      rootDoc: new YDoc({ guid: 'AI_EXTRACT' }),
+    });
+    collection.meta.initialize();
+
+    let needSnapshot = false;
+    let needMarkdown = false;
+    try {
+      const fragmentDoc = collection.createDoc();
+      const fragment = fragmentDoc.getStore();
+      fragmentDoc.load();
+
+      const rootId = fragment.addBlock('affine:page');
+      fragment.addBlock('affine:surface', {}, rootId);
+      const noteId = fragment.addBlock('affine:note', {}, rootId);
+      const surface = getSurfaceBlock(fragment);
+      if (!surface) {
+        throw new Error('Failed to get surface block');
+      }
+
+      for (const element of selectedElements) {
+        if (element instanceof NoteBlockModel) {
+          needMarkdown = true;
+          for (const child of element.children) {
+            const props = getBlockProps(child);
+            fragment.addBlock(child.flavour, props, noteId);
+          }
+        } else if (isAttachment(element)) {
+          const { name, sourceId } = element.props;
+          if (name && sourceId) {
+            attachments.push({ name, sourceId });
+          }
+        } else if (element instanceof GfxPrimitiveElementModel) {
+          needSnapshot = true;
+          const props = getElementProps(element, new Map());
+          surface.addElement(props);
+        }
+      }
+
+      if (needSnapshot) {
+        snapshot = transformer.docToSnapshot(fragment) ?? null;
+      }
+      if (needMarkdown) {
+        markdown = (await markdownAdapter.fromDoc(fragment))?.file ?? '';
+      }
+    } finally {
+      collection.dispose();
+    }
+  }
 
   const canvas = await selectedToCanvas(host);
   if (!canvas) return null;
@@ -57,6 +131,9 @@ async function extractEdgelessSelected(
 
   return {
     images: [new File([blob], 'selected.png')],
+    snapshot: snapshot ? JSON.stringify(snapshot) : null,
+    combinedElementsMarkdown: markdown.length ? markdown : null,
+    attachments,
   };
 }
 
@@ -65,6 +142,7 @@ async function extractPageSelected(
 ): Promise<Partial<ChatContextValue> | null> {
   const text = await getSelectedTextContent(host, 'plain-text');
   const images = await getSelectedImagesAsBlobs(host);
+  const attachments = await getSelectedAttachments(host);
   const hasText = text.length > 0;
   const hasImages = images.length > 0;
 
@@ -73,6 +151,7 @@ async function extractPageSelected(
     return {
       quote: text,
       markdown: markdown,
+      attachments,
     };
   } else if (!hasText && hasImages && images.length === 1) {
     host.command
@@ -83,6 +162,7 @@ async function extractPageSelected(
       })
       .run();
     return {
+      attachments,
       images,
     };
   } else {
@@ -92,6 +172,7 @@ async function extractPageSelected(
       quote: text,
       markdown,
       images,
+      attachments,
     };
   }
 }

packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts

@@ -29,6 +29,7 @@ import {
 
 import { getContentFromSlice } from '../../utils';
 import type { CopilotTool } from '../tool/copilot-tool';
+import { isAttachment } from './attachment';
 import { getEdgelessCopilotWidget } from './get-edgeless-copilot-widget';
 
 export async function selectedToCanvas(host: EditorHost) {
@@ -232,6 +233,26 @@ export const getSelectedImagesAsBlobs = async (host: EditorHost) => {
   return blobs.filter((blob): blob is File => !!blob);
 };
 
+export const getSelectedAttachments = async (host: EditorHost) => {
+  const [_, data] = host.command.exec(getSelectedBlocksCommand, {
+    types: ['block'],
+  });
+
+  const blocks = data.selectedBlocks ?? [];
+  const attachments: { sourceId: string; name: string }[] = [];
+
+  for (const block of blocks) {
+    if (isAttachment(block.model)) {
+      const { sourceId, name } = block.model.props;
+      if (sourceId && name) {
+        attachments.push({ sourceId, name });
+      }
+    }
+  }
+
+  return attachments;
+};
+
 export const getSelectedNoteAnchor = (host: EditorHost, id: string) => {
   return host.querySelector(`affine-edgeless-note[data-block-id="${id}"]`);
 };

packages/frontend/core/tsconfig.json

@@ -18,6 +18,7 @@
     { "path": "../../common/reader" },
     { "path": "../track" },
     { "path": "../../../blocksuite/affine/all" },
+    { "path": "../../../blocksuite/affine/blocks/root" },
     { "path": "../../../blocksuite/affine/components" },
     { "path": "../../../blocksuite/affine/shared" },
     { "path": "../../../blocksuite/framework/global" },

tools/utils/src/workspace.gen.ts

@@ -1351,6 +1351,7 @@ export const PackageList = [
       'packages/frontend/templates',
       'packages/frontend/track',
       'blocksuite/affine/all',
+      'blocksuite/affine/blocks/root',
       'blocksuite/affine/components',
       'blocksuite/affine/shared',
       'blocksuite/framework/global',

yarn.lock

@@ -404,6 +404,7 @@ __metadata:
     "@affine/templates": "workspace:*"
     "@affine/track": "workspace:*"
     "@blocksuite/affine": "workspace:*"
+    "@blocksuite/affine-block-root": "workspace:*"
     "@blocksuite/affine-components": "workspace:*"
     "@blocksuite/affine-ext-loader": "workspace:*"
     "@blocksuite/affine-shared": "workspace:*"