diff --git a/packages/frontend/core/src/blocksuite/presets/ai/_common/config.ts b/packages/frontend/core/src/blocksuite/presets/ai/_common/config.ts index f2b7d4f8e9..3afcdb99b2 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/_common/config.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/_common/config.ts @@ -36,6 +36,7 @@ import { AISearchIcon, AIStarIconWithAnimation, ChatWithAIIcon, + CommentIcon, ExplainIcon, ImproveWritingIcon, LanguageIcon, @@ -395,6 +396,15 @@ const GenerateWithAIGroup: AIItemGroupConfig = { const OthersAIGroup: AIItemGroupConfig = { name: 'Others', items: [ + { + name: 'Continue with AI', + icon: CommentIcon, + handler: host => { + const panel = getAIPanel(host); + AIProvider.slots.requestContinueWithAIInChat.emit({ host }); + panel.hide(); + }, + }, { name: 'Open AI Chat', icon: ChatWithAIIcon, diff --git a/packages/frontend/core/src/blocksuite/presets/ai/_common/icons.ts b/packages/frontend/core/src/blocksuite/presets/ai/_common/icons.ts index bf0029bd59..8acf92d977 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/_common/icons.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/_common/icons.ts @@ -1061,3 +1061,17 @@ export const MoreIcon = html` `; + +export const CommentIcon = html` + +`; diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-cards.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-cards.ts index dee2d6157f..4d36762d31 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-cards.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-cards.ts @@ -1,12 +1,10 @@ -import type { BaseSelection, EditorHost } from '@blocksuite/block-std'; +import type { EditorHost } from '@blocksuite/block-std'; import { WithDisposable } from '@blocksuite/block-std'; import { - type CopilotSelectionController, type ImageBlockModel, type NoteBlockModel, NoteDisplayMode, } from '@blocksuite/blocks'; -import { debounce } from '@blocksuite/global/utils'; import type { BlockModel } from '@blocksuite/store'; import { css, html, LitElement, nothing, type PropertyValues } from 'lit'; import { customElement, property, state } from 'lit/decorators.js'; @@ -18,8 +16,8 @@ import { DocIcon, SmallImageIcon, } from '../_common/icons'; +import { AIProvider } from '../provider'; import { - getEdgelessRootFromEditor, getSelectedImagesAsBlobs, getSelectedTextContent, getTextContentFromBlockModels, @@ -54,15 +52,68 @@ const cardsStyles = css` } `; -const ChatCardsConfig = [ - { - name: 'current-selection', - render: (text?: string, _?: File, __?: string) => { - if (!text) return nothing; +enum CardType { + Text, + Image, + Block, + Doc, +} - const lines = text.split('\n'); +type CardBase = { + id: number; +}; - return html`
+type CardText = CardBase & { + type: CardType.Text; + text: string; + markdown: string; +}; + +type CardImage = CardBase & { + type: CardType.Image; + image: File; + caption?: string; +}; + +type CardBlock = CardBase & { + type: CardType.Block | CardType.Doc; + text?: string; + markdown?: string; + images?: File[]; +}; + +type Card = CardText | CardImage | CardBlock; + +const MAX_CARDS = 3; + +@customElement('chat-cards') +export class ChatCards extends WithDisposable(LitElement) { + static override styles = css` + :host { + display: flex; + flex-direction: column; + gap: 12px; + } + + ${cardsStyles} + `; + + @property({ attribute: false }) + accessor host!: EditorHost; + + @property({ attribute: false }) + accessor updateContext!: (context: Partial) => void; + + @state() + accessor cards: Card[] = []; + + private _selectedCardId: number = 0; + + static renderText({ text }: CardText) { + const lines = text.split('\n'); + + return html` +
${CurrentSelectionIcon}
Start with current selection
@@ -71,8 +122,8 @@ const ChatCardsConfig = [ ${repeat( lines.slice(0, 2), line => line, - line => { - return html`
html` +
${line} -
`; - } +
+ ` )}
-
`; - }, - handler: ( - updateContext: (context: Partial) => void, - text: string, - markdown: string, - images?: File[] - ) => { - const value: Partial = { - quote: text, - markdown: markdown, - }; - if (images) { - value.images = images; - } - updateContext(value); - }, - }, - { - name: 'image', - render: (_?: string, image?: File, caption?: string) => { - if (!image) return nothing; +
+ `; + } - return html`
-
`; - }, - handler: ( - updateContext: (context: Partial) => void, - _: string, - __: string, - images?: File[] - ) => { - const value: Partial = {}; - if (images) { - value.images = images; - } - updateContext(value); - }, - }, - { - name: 'doc', - render: () => { - return html` -
-
- ${DocIcon} -
Start with this doc
-
-
you've chosen within the doc
+
+ `; + } + + static renderDoc(_: CardBlock) { + return html` +
+
+ ${DocIcon} +
Start with this doc
- `; - }, - handler: ( - updateContext: (context: Partial) => void, - text: string, - markdown: string, - images?: File[] - ) => { - const value: Partial = { - quote: text, - markdown: markdown, - }; - if (images) { - value.images = images; +
you've chosen within the doc
+
+ `; + } + + private _renderCard(card: Card) { + if (card.type === CardType.Text) { + return ChatCards.renderText(card); + } + + if (card.type === CardType.Image) { + return ChatCards.renderImage(card); + } + + if (card.type === CardType.Doc) { + return ChatCards.renderDoc(card); + } + + return nothing; + } + + private _updateCards(card: Card) { + this.cards.unshift(card); + + if (this.cards.length > MAX_CARDS) { + this.cards.pop(); + } + + this.requestUpdate(); + } + + private async _handleDocSelection(card: CardBlock) { + const { text, markdown, images } = await this._extractAll(); + + card.text = text; + card.markdown = markdown; + card.images = images; + } + + private async _handleClick(card: Card) { + AIProvider.slots.toggleChatCards.emit({ visible: false }); + + this._selectedCardId = card.id; + + switch (card.type) { + case CardType.Text: { + this.updateContext({ + quote: card.text, + markdown: card.markdown, + }); + break; + } + case CardType.Image: { + this.updateContext({ + images: [card.image], + }); + break; + } + case CardType.Doc: { + await this._handleDocSelection(card); + this.updateContext({ + quote: card.text, + markdown: card.markdown, + images: card.images, + }); + break; } - updateContext(value); - }, - }, -]; - -@customElement('chat-cards') -export class ChatCards extends WithDisposable(LitElement) { - static override styles = css` - ${cardsStyles} - .cards-container { - display: flex; - flex-direction: column; - gap: 12px; } - `; - - @property({ attribute: false }) - accessor host!: EditorHost; - - @property({ attribute: false }) - accessor chatContextValue!: ChatContextValue; - - @property({ attribute: false }) - accessor updateContext!: (context: Partial) => void; - - @property({ attribute: false }) - accessor selectionValue: BaseSelection[] = []; - - @state() - accessor text: string = ''; - - @state() - accessor markdown: string = ''; - - @state() - accessor images: File[] = []; - - @state() - accessor caption: string = ''; - - private _onEdgelessCopilotAreaUpdated() { - if (!this.host.closest('edgeless-editor')) return; - const edgeless = getEdgelessRootFromEditor(this.host); - - const copilotSelectionTool = edgeless.tools.controllers - .copilot as CopilotSelectionController; - - this._disposables.add( - copilotSelectionTool.draggingAreaUpdated.on( - debounce(() => { - selectedToCanvas(this.host) - .then(canvas => { - canvas?.toBlob(blob => { - if (!blob) return; - const file = new File([blob], 'selected.png'); - this.images = [file]; - }); - }) - .catch(console.error); - }, 300) - ) - ); } - private async _updateState() { - if ( - this.selectionValue.some( - selection => selection.is('text') || selection.is('image') - ) - ) + private async _extract() { + const text = await getSelectedTextContent(this.host, 'plain-text'); + const images = await getSelectedImagesAsBlobs(this.host); + const hasText = text.length > 0; + const hasImages = images.length > 0; + + if (hasText && !hasImages) { + const markdown = await getSelectedTextContent(this.host, 'markdown'); + + this._updateCards({ + id: Date.now(), + type: CardType.Text, + text, + markdown, + }); + + return; + } + + if (!hasText && hasImages && images.length === 1) { + const [_, data] = this.host.command + .chain() + .tryAll(chain => [chain.getImageSelections()]) + .getSelectedBlocks({ + types: ['image'], + }) + .run(); + let caption = ''; + + if (data.currentImageSelections?.[0]) { + caption = + ( + this.host.doc.getBlock(data.currentImageSelections[0].blockId) + ?.model as ImageBlockModel + ).caption ?? ''; + } + + this._updateCards({ + id: Date.now(), + type: CardType.Image, + image: images[0], + caption, + }); + return; - this.text = await getSelectedTextContent(this.host, 'plain-text'); - this.markdown = await getSelectedTextContent(this.host, 'markdown'); - this.images = await getSelectedImagesAsBlobs(this.host); - const [_, data] = this.host.command - .chain() - .tryAll(chain => [ - chain.getTextSelection(), - chain.getBlockSelections(), - chain.getImageSelections(), - ]) - .getSelectedBlocks({ - types: ['image'], - }) - .run(); - if (data.currentBlockSelections?.[0]) { - this.caption = - ( - this.host.doc.getBlock(data.currentBlockSelections[0].blockId) - ?.model as ImageBlockModel - ).caption ?? ''; } } - private async _handleDocSelection() { + private async _extractOnEdgeless() { + if (!this.host.closest('edgeless-editor')) return; + + const canvas = await selectedToCanvas(this.host); + if (!canvas) return; + + const blob: Blob | null = await new Promise(resolve => + canvas.toBlob(resolve) + ); + if (!blob) return; + + this._updateCards({ + id: Date.now(), + type: CardType.Image, + image: new File([blob], 'selected.png'), + }); + } + + private async _extractAll() { const notes = this.host.doc .getBlocksByFlavour('affine:note') .filter( @@ -305,50 +351,68 @@ export class ChatCards extends WithDisposable(LitElement) { }) ?? [] ); const images = blobs.filter((blob): blob is File => !!blob); - this.text = text; - this.markdown = markdown; - this.images = images; + + return { + text, + markdown, + images, + }; } - protected override async updated(_changedProperties: PropertyValues) { - if (_changedProperties.has('selectionValue')) { - await this._updateState(); - } + protected override async updated(changedProperties: PropertyValues) { + if (changedProperties.has('host')) { + const { text, images } = await this._extractAll(); + const hasText = text.length > 0; + const hasImages = images.length > 0; - if (_changedProperties.has('host')) { - this._onEdgelessCopilotAreaUpdated(); + // Currently only supports checking on first load + if ( + (hasText || hasImages) && + !this.cards.some(card => card.type === CardType.Doc) + ) { + this._updateCards({ + id: Date.now(), + type: CardType.Doc, + }); + } } } + override async connectedCallback() { + super.connectedCallback(); + + this._disposables.add( + AIProvider.slots.requestContinueWithAIInChat.on(async ({ mode }) => { + if (mode === 'edgeless') { + await this._extractOnEdgeless(); + } else { + await this._extract(); + } + }) + ); + + this._disposables.add( + AIProvider.slots.toggleChatCards.on(({ visible, ok }) => { + if (visible && ok && this._selectedCardId > 0) { + this.cards = this.cards.filter( + card => card.id !== this._selectedCardId + ); + this._selectedCardId = 0; + } + }) + ); + } + protected override render() { - return html`
- ${repeat( - ChatCardsConfig, - card => card.name, - card => { - if ( - card.render(this.text, this.images[0], this.caption) !== nothing - ) { - return html`
{ - if (card.name === 'doc') { - await this._handleDocSelection(); - } - card.handler( - this.updateContext, - this.text, - this.markdown, - this.images - ); - }} - > - ${card.render(this.text, this.images[0], this.caption)} -
`; - } - return nothing; - } - )} -
`; + return repeat( + this.cards, + card => card.id, + card => html` +
this._handleClick(card)}> + ${this._renderCard(card)} +
+ ` + ); } } diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts index a946f23a12..2f477e74fb 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts @@ -268,6 +268,8 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
{ + AIProvider.slots.toggleChatCards.emit({ visible: true }); + if (this.curIndex >= 0 && this.curIndex < images.length) { const newImages = [...images]; newImages.splice(this.curIndex, 1); @@ -315,6 +317,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
{ + AIProvider.slots.toggleChatCards.emit({ visible: true }); this.updateContext({ quote: '', markdown: '' }); }} > diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-messages.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-messages.ts index 12ad20a414..4354d73a3d 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-messages.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-messages.ts @@ -27,6 +27,7 @@ import { } from '@blocksuite/blocks'; import { css, html, nothing, type PropertyValues } from 'lit'; import { customElement, property, query, state } from 'lit/decorators.js'; +import { cache } from 'lit/directives/cache.js'; import { repeat } from 'lit/directives/repeat.js'; import { styleMap } from 'lit/directives/style-map.js'; @@ -144,7 +145,8 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) { } `; - private _selectionValue: BaseSelection[] = []; + @state() + accessor _selectionValue: BaseSelection[] = []; @state() accessor showDownIndicator = false; @@ -167,14 +169,16 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) { @query('.chat-panel-messages') accessor messagesContainer!: HTMLDivElement; - protected override updated(_changedProperties: PropertyValues) { - if (_changedProperties.has('host')) { + @state() + accessor showChatCards = true; + + protected override updated(changedProperties: PropertyValues) { + if (changedProperties.has('host')) { const { disposables } = this; disposables.add( this.host.selection.slots.changed.on(() => { this._selectionValue = this.host.selection.value; - this.requestUpdate(); }) ); const { docModeService } = this.host.spec.getService('affine:page'); @@ -226,12 +230,16 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) { : 'What can I help you with?'}
- ` + ${cache( + this.showChatCards + ? html` + + ` + : nothing + )}` : repeat(filteredItems, (item, index) => { const isLast = index === filteredItems.length - 1; return html`
@@ -265,6 +273,12 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) { } }) ); + + this.disposables.add( + AIProvider.slots.toggleChatCards.on(({ visible }) => { + this.showChatCards = visible; + }) + ); } renderError() { diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts index 703146e8ca..98d57820ae 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts @@ -190,6 +190,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { AIProvider.slots.actions.on(({ action, event }) => { const { status } = this.chatContextValue; + if ( action !== 'chat' && event === 'finished' && @@ -197,6 +198,13 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { ) { this._resetItems(); } + + if (action === 'chat' && event === 'finished') { + AIProvider.slots.toggleChatCards.emit({ + visible: true, + ok: status === 'success', + }); + } }); AIProvider.slots.userInfo.on(userInfo => { diff --git a/packages/frontend/core/src/blocksuite/presets/ai/entries/edgeless/actions-config.ts b/packages/frontend/core/src/blocksuite/presets/ai/entries/edgeless/actions-config.ts index 2414415ef6..e17bd4e0b7 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/entries/edgeless/actions-config.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/entries/edgeless/actions-config.ts @@ -19,6 +19,7 @@ import { AIPresentationIconWithAnimation, AISearchIcon, ChatWithAIIcon, + CommentIcon, ExplainIcon, ImproveWritingIcon, LanguageIcon, @@ -98,6 +99,19 @@ export const imageProcessingSubItem = imageProcessingTypes.map(type => { const othersGroup: AIItemGroupConfig = { name: 'others', items: [ + { + name: 'Continue with AI', + icon: CommentIcon, + showWhen: () => true, + handler: host => { + const panel = getAIPanel(host); + AIProvider.slots.requestContinueWithAIInChat.emit({ + host, + mode: 'edgeless', + }); + panel.hide(); + }, + }, { name: 'Open AI Chat', icon: ChatWithAIIcon, diff --git a/packages/frontend/core/src/blocksuite/presets/ai/provider.ts b/packages/frontend/core/src/blocksuite/presets/ai/provider.ts index d6b54edd97..4b3ec14f5e 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/provider.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/provider.ts @@ -81,6 +81,10 @@ export class AIProvider { // use case: when user selects "continue in chat" in an ask ai result panel // do we need to pass the context to the chat panel? requestContinueInChat: new Slot<{ host: EditorHost; show: boolean }>(), + requestContinueWithAIInChat: new Slot<{ + host: EditorHost; + mode?: 'page' | 'edgeless'; + }>(), requestLogin: new Slot<{ host: EditorHost }>(), requestUpgradePlan: new Slot<{ host: EditorHost }>(), // when an action is requested to run in edgeless mode (show a toast in affine) @@ -94,6 +98,10 @@ export class AIProvider { // downstream can emit this slot to notify ai presets that user info has been updated userInfo: new Slot(), // add more if needed + toggleChatCards: new Slot<{ + visible: boolean; + ok?: boolean; + }>(), }; // track the history of triggered actions (in memory only)