diff --git a/blocksuite/affine/blocks/root/src/edgeless/index.ts b/blocksuite/affine/blocks/root/src/edgeless/index.ts index b0dd5fe3bf..06ec028d95 100644 --- a/blocksuite/affine/blocks/root/src/edgeless/index.ts +++ b/blocksuite/affine/blocks/root/src/edgeless/index.ts @@ -4,6 +4,6 @@ export * from './clipboard/command'; export * from './edgeless-root-block.js'; export { EdgelessRootService } from './edgeless-root-service.js'; export * from './utils/clipboard-utils.js'; -export { sortEdgelessElements } from './utils/clone-utils.js'; +export { getElementProps, sortEdgelessElements } from './utils/clone-utils.js'; export { isCanvasElement } from './utils/query.js'; export { EDGELESS_BLOCK_CHILD_PADDING } from '@blocksuite/affine-shared/consts'; diff --git a/packages/frontend/core/package.json b/packages/frontend/core/package.json index af1227b269..d424ba4ec8 100644 --- a/packages/frontend/core/package.json +++ b/packages/frontend/core/package.json @@ -20,6 +20,7 @@ "@affine/templates": "workspace:*", "@affine/track": "workspace:*", "@blocksuite/affine": "workspace:*", + "@blocksuite/affine-block-root": "workspace:*", "@blocksuite/affine-components": "workspace:*", "@blocksuite/affine-shared": "workspace:*", "@blocksuite/global": "workspace:*", diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/ai-chat-content.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/ai-chat-content.ts index 812b4596d8..7f15b89d9a 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/ai-chat-content.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/ai-chat-content.ts @@ -48,6 +48,9 @@ const DEFAULT_CHAT_CONTEXT_VALUE: ChatContextValue = { status: 'idle', error: null, markdown: '', + snapshot: null, + attachments: [], + combinedElementsMarkdown: null, }; export class AIChatContent extends SignalWatcher( diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/type.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/type.ts index cd7a456ee7..712bfd5e31 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/type.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-content/type.ts @@ -12,5 +12,11 @@ export type ChatContextValue = { markdown: string; // images of the selected content or user uploaded images: File[]; + // snapshot of the selected content + snapshot: string | null; + // attachments of the selected content + attachments: { sourceId: string; name: string }[]; + // combined markdown of the selected elements + combinedElementsMarkdown: string | null; abortController: AbortController | null; }; diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/type.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/type.ts index 22b09806b9..f1a8f4b343 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/type.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/type.ts @@ -1,6 +1,7 @@ import type { Signal } from '@preact/signals-core'; import type { AIError } from '../../provider'; +import type { ChatContextValue } from '../ai-chat-content'; import type { ChatStatus, HistoryMessage } from '../ai-chat-messages'; export interface AINetworkSearchConfig { @@ -27,4 +28,7 @@ export type AIChatInputContext = { markdown?: string; images: File[]; abortController: AbortController | null; -}; +} & Pick< + ChatContextValue, + 'snapshot' | 'combinedElementsMarkdown' | 'attachments' +>; diff --git a/packages/frontend/core/src/blocksuite/ai/components/playground/chat.ts b/packages/frontend/core/src/blocksuite/ai/components/playground/chat.ts index 90eea06a26..5e051cfbc2 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/playground/chat.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/playground/chat.ts @@ -45,6 +45,9 @@ const DEFAULT_CHAT_CONTEXT_VALUE: ChatContextValue = { status: 'idle', error: null, markdown: '', + snapshot: null, + attachments: [], + combinedElementsMarkdown: null, }; export class PlaygroundChat extends SignalWatcher( diff --git a/packages/frontend/core/src/blocksuite/ai/utils/attachment.ts b/packages/frontend/core/src/blocksuite/ai/utils/attachment.ts new file mode 100644 index 0000000000..6f3d3ab652 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/utils/attachment.ts @@ -0,0 +1,9 @@ +import { AttachmentBlockModel } from '@blocksuite/affine/model'; +import type { BlockModel } from '@blocksuite/affine/store'; +import type { GfxModel } from '@blocksuite/std/gfx'; + +export function isAttachment( + model: GfxModel | BlockModel +): model is AttachmentBlockModel { + return model instanceof AttachmentBlockModel; +} diff --git a/packages/frontend/core/src/blocksuite/ai/utils/extract.ts b/packages/frontend/core/src/blocksuite/ai/utils/extract.ts index bb9e198972..d5bb263bdd 100644 --- a/packages/frontend/core/src/blocksuite/ai/utils/extract.ts +++ b/packages/frontend/core/src/blocksuite/ai/utils/extract.ts @@ -1,7 +1,9 @@ +import { WorkspaceImpl } from '@affine/core/modules/workspace/impls/workspace'; +import { getSurfaceBlock } from '@blocksuite/affine/blocks/surface'; import { DatabaseBlockModel, ImageBlockModel, - type NoteBlockModel, + NoteBlockModel, NoteDisplayMode, } from '@blocksuite/affine/model'; import { @@ -15,16 +17,25 @@ import { } from '@blocksuite/affine/shared/commands'; import { DocModeProvider } from '@blocksuite/affine/shared/services'; import { + getBlockProps, isInsideEdgelessEditor, matchModels, } from '@blocksuite/affine/shared/utils'; import { BlockStdScope, type EditorHost } from '@blocksuite/affine/std'; -import type { BlockModel, Store } from '@blocksuite/affine/store'; +import { + GfxControllerIdentifier, + GfxPrimitiveElementModel, +} from '@blocksuite/affine/std/gfx'; +import type { BlockModel, DocSnapshot, Store } from '@blocksuite/affine/store'; import { Slice, toDraftModel } from '@blocksuite/affine/store'; +import { getElementProps } from '@blocksuite/affine-block-root'; +import { Doc as YDoc } from 'yjs'; import { getStoreManager } from '../../manager/store'; import type { ChatContextValue } from '../components/ai-chat-content'; +import { isAttachment } from './attachment'; import { + getSelectedAttachments, getSelectedImagesAsBlobs, getSelectedTextContent, selectedToCanvas, @@ -46,6 +57,69 @@ async function extractEdgelessSelected( host: EditorHost ): Promise | null> { if (!isInsideEdgelessEditor(host)) return null; + const gfx = host.std.get(GfxControllerIdentifier); + const selectedElements = gfx.selection.selectedElements; + + let snapshot: DocSnapshot | null = null; + let markdown = ''; + const attachments: ChatContextValue['attachments'] = []; + + if (selectedElements.length) { + const transformer = host.store.getTransformer(); + const markdownAdapter = new MarkdownAdapter( + transformer, + host.store.provider + ); + const collection = new WorkspaceImpl({ + id: 'AI_EXTRACT', + rootDoc: new YDoc({ guid: 'AI_EXTRACT' }), + }); + collection.meta.initialize(); + + let needSnapshot = false; + let needMarkdown = false; + try { + const fragmentDoc = collection.createDoc(); + const fragment = fragmentDoc.getStore(); + fragmentDoc.load(); + + const rootId = fragment.addBlock('affine:page'); + fragment.addBlock('affine:surface', {}, rootId); + const noteId = fragment.addBlock('affine:note', {}, rootId); + const surface = getSurfaceBlock(fragment); + if (!surface) { + throw new Error('Failed to get surface block'); + } + + for (const element of selectedElements) { + if (element instanceof NoteBlockModel) { + needMarkdown = true; + for (const child of element.children) { + const props = getBlockProps(child); + fragment.addBlock(child.flavour, props, noteId); + } + } else if (isAttachment(element)) { + const { name, sourceId } = element.props; + if (name && sourceId) { + attachments.push({ name, sourceId }); + } + } else if (element instanceof GfxPrimitiveElementModel) { + needSnapshot = true; + const props = getElementProps(element, new Map()); + surface.addElement(props); + } + } + + if (needSnapshot) { + snapshot = transformer.docToSnapshot(fragment) ?? null; + } + if (needMarkdown) { + markdown = (await markdownAdapter.fromDoc(fragment))?.file ?? ''; + } + } finally { + collection.dispose(); + } + } const canvas = await selectedToCanvas(host); if (!canvas) return null; @@ -57,6 +131,9 @@ async function extractEdgelessSelected( return { images: [new File([blob], 'selected.png')], + snapshot: snapshot ? JSON.stringify(snapshot) : null, + combinedElementsMarkdown: markdown.length ? markdown : null, + attachments, }; } @@ -65,6 +142,7 @@ async function extractPageSelected( ): Promise | null> { const text = await getSelectedTextContent(host, 'plain-text'); const images = await getSelectedImagesAsBlobs(host); + const attachments = await getSelectedAttachments(host); const hasText = text.length > 0; const hasImages = images.length > 0; @@ -73,6 +151,7 @@ async function extractPageSelected( return { quote: text, markdown: markdown, + attachments, }; } else if (!hasText && hasImages && images.length === 1) { host.command @@ -83,6 +162,7 @@ async function extractPageSelected( }) .run(); return { + attachments, images, }; } else { @@ -92,6 +172,7 @@ async function extractPageSelected( quote: text, markdown, images, + attachments, }; } } diff --git a/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts b/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts index 2ae7670d36..a7d1846959 100644 --- a/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts +++ b/packages/frontend/core/src/blocksuite/ai/utils/selection-utils.ts @@ -29,6 +29,7 @@ import { import { getContentFromSlice } from '../../utils'; import type { CopilotTool } from '../tool/copilot-tool'; +import { isAttachment } from './attachment'; import { getEdgelessCopilotWidget } from './get-edgeless-copilot-widget'; export async function selectedToCanvas(host: EditorHost) { @@ -232,6 +233,26 @@ export const getSelectedImagesAsBlobs = async (host: EditorHost) => { return blobs.filter((blob): blob is File => !!blob); }; +export const getSelectedAttachments = async (host: EditorHost) => { + const [_, data] = host.command.exec(getSelectedBlocksCommand, { + types: ['block'], + }); + + const blocks = data.selectedBlocks ?? []; + const attachments: { sourceId: string; name: string }[] = []; + + for (const block of blocks) { + if (isAttachment(block.model)) { + const { sourceId, name } = block.model.props; + if (sourceId && name) { + attachments.push({ sourceId, name }); + } + } + } + + return attachments; +}; + export const getSelectedNoteAnchor = (host: EditorHost, id: string) => { return host.querySelector(`affine-edgeless-note[data-block-id="${id}"]`); }; diff --git a/packages/frontend/core/tsconfig.json b/packages/frontend/core/tsconfig.json index 5daf2c28ca..ef207575bf 100644 --- a/packages/frontend/core/tsconfig.json +++ b/packages/frontend/core/tsconfig.json @@ -18,6 +18,7 @@ { "path": "../../common/reader" }, { "path": "../track" }, { "path": "../../../blocksuite/affine/all" }, + { "path": "../../../blocksuite/affine/blocks/root" }, { "path": "../../../blocksuite/affine/components" }, { "path": "../../../blocksuite/affine/shared" }, { "path": "../../../blocksuite/framework/global" }, diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 3b6d565842..0c7f4cb52a 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -1351,6 +1351,7 @@ export const PackageList = [ 'packages/frontend/templates', 'packages/frontend/track', 'blocksuite/affine/all', + 'blocksuite/affine/blocks/root', 'blocksuite/affine/components', 'blocksuite/affine/shared', 'blocksuite/framework/global', diff --git a/yarn.lock b/yarn.lock index abd6f2c6c3..46597c2e73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -404,6 +404,7 @@ __metadata: "@affine/templates": "workspace:*" "@affine/track": "workspace:*" "@blocksuite/affine": "workspace:*" + "@blocksuite/affine-block-root": "workspace:*" "@blocksuite/affine-components": "workspace:*" "@blocksuite/affine-ext-loader": "workspace:*" "@blocksuite/affine-shared": "workspace:*"