From 58fed5928b1646099c1c4a37712e83ace3b1ad68 Mon Sep 17 00:00:00 2001 From: akumatus Date: Wed, 12 Feb 2025 08:33:06 +0000 Subject: [PATCH] feat: add doc copilot context api (#10103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What Changed? - Add graphql APIs. - Provide context and session service in `AIProvider`. - Rename the state from `embedding` to `processing`. - Reafctor front-end session create, update and save logic. Persist the document selected by the user: [录屏2025-02-08 11.04.40.mov (uploaded via Graphite) ](https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/195a85f2-43c4-4e49-88d9-6b5fc4f235ca.mov) --- .../src/plugins/copilot/context/resolver.ts | 10 +- packages/backend/server/src/schema.gql | 6 +- .../presets/ai/actions/doc-handler.ts | 9 +- .../blocksuite/presets/ai/actions/types.ts | 53 +++++- .../presets/ai/chat-panel/chat-context.ts | 15 +- .../presets/ai/chat-panel/chat-panel-chips.ts | 78 +++++++-- .../presets/ai/chat-panel/chat-panel-input.ts | 69 ++++++-- .../ai/chat-panel/chat-panel-messages.ts | 14 +- .../ai/chat-panel/components/add-popover.ts | 2 +- .../ai/chat-panel/components/doc-chip.ts | 20 ++- .../ai/chat-panel/components/file-chip.ts | 2 +- .../presets/ai/chat-panel/components/utils.ts | 24 ++- .../blocksuite/presets/ai/chat-panel/index.ts | 163 ++++++++++++------ .../src/blocksuite/presets/ai/provider.ts | 28 +++ .../block-suite-editor/ai/copilot-client.ts | 73 ++++++++ .../block-suite-editor/ai/request.ts | 18 +- .../block-suite-editor/ai/setup-provider.tsx | 129 +++++++------- .../providers/workspace-side-effects.tsx | 6 +- .../frontend/graphql/src/graphql/index.ts | 22 ++- packages/frontend/graphql/src/schema.ts | 63 +++++-- .../affine-cloud-copilot/e2e/copilot.spec.ts | 28 +-- 21 files changed, 588 insertions(+), 244 deletions(-) diff --git a/packages/backend/server/src/plugins/copilot/context/resolver.ts b/packages/backend/server/src/plugins/copilot/context/resolver.ts index 14bd94071e..7ba20a2367 100644 --- a/packages/backend/server/src/plugins/copilot/context/resolver.ts +++ b/packages/backend/server/src/plugins/copilot/context/resolver.ts @@ -36,12 +36,12 @@ class AddContextDocInput { } @InputType() -class RemoveContextFileInput { +class RemoveContextDocInput { @Field(() => String) contextId!: string; @Field(() => String) - fileId!: string; + docId!: string; } @ObjectType('CopilotContext') @@ -227,8 +227,8 @@ export class CopilotContextResolver { }) @CallMetric('ai', 'context_doc_remove') async removeContextDoc( - @Args({ name: 'options', type: () => RemoveContextFileInput }) - options: RemoveContextFileInput + @Args({ name: 'options', type: () => RemoveContextDocInput }) + options: RemoveContextDocInput ) { const lockFlag = `${COPILOT_LOCKER}:context:${options.contextId}`; await using lock = await this.mutex.acquire(lockFlag); @@ -238,7 +238,7 @@ export class CopilotContextResolver { const session = await this.context.get(options.contextId); try { - return await session.removeDocRecord(options.fileId); + return await session.removeDocRecord(options.docId); } catch (e: any) { throw new CopilotFailedToModifyContext({ contextId: options.contextId, diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index a188d2c96f..65186f2f07 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -713,7 +713,7 @@ type Mutation { removeAvatar: RemoveAvatar! """remove a doc from context""" - removeContextDoc(options: RemoveContextFileInput!): Boolean! + removeContextDoc(options: RemoveContextDocInput!): Boolean! removeWorkspaceFeature(feature: FeatureType!, workspaceId: String!): Boolean! resumeSubscription(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, workspaceId: String): SubscriptionType! revoke(userId: String!, workspaceId: String!): Boolean! @@ -891,9 +891,9 @@ type RemoveAvatar { success: Boolean! } -input RemoveContextFileInput { +input RemoveContextDocInput { contextId: String! - fileId: String! + docId: String! } input RevokeDocUserRoleInput { diff --git a/packages/frontend/core/src/blocksuite/presets/ai/actions/doc-handler.ts b/packages/frontend/core/src/blocksuite/presets/ai/actions/doc-handler.ts index 260bcb399f..e437cf7456 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/actions/doc-handler.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/actions/doc-handler.ts @@ -236,9 +236,14 @@ export function handleInlineAskAIAction( host.selection.set([selection]); selectAboveBlocks(host) - .then(context => { - assertExists(AIProvider.actions.chat); + .then(async context => { + if (!AIProvider.session || !AIProvider.actions.chat) return; + const sessionId = await AIProvider.session.createSession( + host.doc.workspace.id, + host.doc.id + ); const stream = AIProvider.actions.chat({ + sessionId, input: `${context}\n${input}`, stream: true, host, diff --git a/packages/frontend/core/src/blocksuite/presets/ai/actions/types.ts b/packages/frontend/core/src/blocksuite/presets/ai/actions/types.ts index 17dfa361e6..2bc38ac97f 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/actions/types.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/actions/types.ts @@ -1,4 +1,9 @@ -import type { getCopilotHistoriesQuery, RequestOptions } from '@affine/graphql'; +import type { + CopilotContextDoc, + CopilotContextFile, + getCopilotHistoriesQuery, + RequestOptions, +} from '@affine/graphql'; import type { EditorHost } from '@blocksuite/affine/block-std'; import type { GfxModel } from '@blocksuite/affine/block-std/gfx'; import type { BlockModel } from '@blocksuite/affine/store'; @@ -105,10 +110,9 @@ declare global { T['stream'] extends true ? TextStream : Promise; interface ChatOptions extends AITextActionOptions { - // related documents - docs?: DocContext[]; sessionId?: string; isRootSession?: boolean; + docs?: DocContext[]; } interface TranslateOptions extends AITextActionOptions { @@ -232,6 +236,40 @@ declare global { ): AIActionTextResponse; } + interface AIContextService { + createContext: ( + workspaceId: string, + sessionId: string + ) => Promise; + getContextId: ( + workspaceId: string, + sessionId: string + ) => Promise; + addContextDoc: (options: { + contextId: string; + docId: string; + }) => Promise>; + removeContextDoc: (options: { + contextId: string; + docId: string; + }) => Promise; + addContextFile: (options: { + contextId: string; + fileId: string; + }) => Promise; + removeContextFile: (options: { + contextId: string; + fileId: string; + }) => Promise; + getContextDocsAndFiles: ( + workspaceId: string, + sessionId: string, + contextId: string + ) => Promise< + { docs: CopilotContextDoc[]; files: CopilotContextFile[] } | undefined + >; + } + // TODO(@Peng): should be refactored to get rid of implement details (like messages, action, role, etc.) interface AIHistory { sessionId: string; @@ -256,6 +294,15 @@ declare global { >[]; }; + interface AISessionService { + createSession: ( + workspaceId: string, + docId: string, + promptName?: string + ) => Promise; + updateSession: (sessionId: string, promptName: string) => Promise; + } + interface AIHistoryService { // non chat histories actions: ( diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts index 11a4550d99..c93d03efe1 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts @@ -46,7 +46,6 @@ export type ChatContextValue = { // chips of workspace doc or user uploaded file chips: ChatChip[]; abortController: AbortController | null; - chatSessionId: string | null; }; export type ChatBlockMessage = ChatMessage & { @@ -55,20 +54,14 @@ export type ChatBlockMessage = ChatMessage & { avatarUrl?: string; }; -export type ChipState = - | 'candidate' - | 'uploading' - | 'embedding' - | 'success' - | 'failed'; +export type ChipState = 'candidate' | 'processing' | 'success' | 'failed'; export interface BaseChip { /** * candidate: the chip is a candidate for the chat - * uploading: the chip is uploading - * embedding: the chip is embedding - * success: the chip is successfully embedded - * failed: the chip is failed to embed + * processing: the chip is processing + * success: the chip is successfully processed + * failed: the chip is failed to process */ state: ChipState; tooltip?: string; diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts index 6f6809a411..f2773f33f5 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts @@ -10,6 +10,7 @@ import { css, html } from 'lit'; import { property, query } from 'lit/decorators.js'; import { repeat } from 'lit/directives/repeat.js'; +import { AIProvider } from '../provider'; import type { DocDisplayConfig, DocSearchMenuConfig } from './chat-config'; import type { BaseChip, ChatChip, ChatContextValue } from './chat-context'; import { getChipKey, isDocChip, isFileChip } from './components/utils'; @@ -45,6 +46,9 @@ export class ChatPanelChips extends WithDisposable(ShadowlessElement) { @property({ attribute: false }) accessor chatContextValue!: ChatContextValue; + @property({ attribute: false }) + accessor chatContextId!: string | undefined; + @property({ attribute: false }) accessor updateContext!: (context: Partial) => void; @@ -69,6 +73,7 @@ export class ChatPanelChips extends WithDisposable(ShadowlessElement) { if (isDocChip(chip)) { return html` { + private readonly _addChip = async (chip: ChatChip) => { if ( this.chatContextValue.chips.length === 1 && this.chatContextValue.chips[0].state === 'candidate' ) { + await this._addToContext(chip); this.updateContext({ chips: [chip], }); @@ -132,12 +138,16 @@ export class ChatPanelChips extends WithDisposable(ShadowlessElement) { } // remove the chip if it already exists const chips = this.chatContextValue.chips.filter(item => { - if (isDocChip(item)) { - return !isDocChip(chip) || item.docId !== chip.docId; + if (isDocChip(chip)) { + return !isDocChip(item) || item.docId !== chip.docId; } else { - return !isFileChip(chip) || item.fileId !== chip.fileId; + return !isFileChip(item) || item.fileId !== chip.fileId; } }); + if (chips.length < this.chatContextValue.chips.length) { + await this._removeFromContext(chip); + } + await this._addToContext(chip); this.updateContext({ chips: [...chips, chip], }); @@ -167,15 +177,55 @@ export class ChatPanelChips extends WithDisposable(ShadowlessElement) { }); }; - private readonly _removeChip = (chip: ChatChip) => { - this.updateContext({ - chips: this.chatContextValue.chips.filter(item => { - if (isDocChip(item)) { - return !isDocChip(chip) || item.docId !== chip.docId; - } else { - return !isFileChip(chip) || item.fileId !== chip.fileId; - } - }), - }); + private readonly _removeChip = async (chip: ChatChip) => { + if (isDocChip(chip)) { + await this._removeFromContext(chip); + this.updateContext({ + chips: this.chatContextValue.chips.filter(item => { + return !isDocChip(item) || item.docId !== chip.docId; + }), + }); + } else { + await this._removeFromContext(chip); + this.updateContext({ + chips: this.chatContextValue.chips.filter(item => { + return !isFileChip(item) || item.fileId !== chip.fileId; + }), + }); + } + }; + + private readonly _addToContext = async (chip: ChatChip) => { + if (!AIProvider.context || !this.chatContextId) { + return; + } + if (isDocChip(chip)) { + await AIProvider.context.addContextDoc({ + contextId: this.chatContextId, + docId: chip.docId, + }); + } else { + await AIProvider.context.addContextFile({ + contextId: this.chatContextId, + fileId: chip.fileId, + }); + } + }; + + private readonly _removeFromContext = async (chip: ChatChip) => { + if (!AIProvider.context || !this.chatContextId) { + return; + } + if (isDocChip(chip)) { + await AIProvider.context.removeContextDoc({ + contextId: this.chatContextId, + docId: chip.docId, + }); + } else { + await AIProvider.context.removeContextFile({ + contextId: this.chatContextId, + fileId: chip.fileId, + }); + } }; } diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts index 62a98b2f1d..615603dded 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts @@ -258,6 +258,9 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) { @property({ attribute: false }) accessor chatContextValue!: ChatContextValue; + @property({ attribute: false }) + accessor chatSessionId!: string | undefined; + @property({ attribute: false }) accessor updateContext!: (context: Partial) => void; @@ -267,6 +270,44 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) { @property({ attribute: false }) accessor networkSearchConfig!: AINetworkSearchConfig; + private _lastPromptName: string | null = null; + + private get _isNetworkActive() { + return ( + !!this.networkSearchConfig.visible.value && + !!this.networkSearchConfig.enabled.value + ); + } + + private get _isNetworkDisabled() { + return ( + !!this.chatContextValue.images.length || + !!this.chatContextValue.chips.filter(chip => chip.state !== 'candidate') + .length + ); + } + + private get _promptName() { + if (this._isNetworkDisabled) { + return 'Chat With AFFiNE AI'; + } + return this._isNetworkActive + ? 'Search With AFFiNE AI' + : 'Chat With AFFiNE AI'; + } + + private async _updatePromptName() { + if (this._lastPromptName !== this._promptName) { + this._lastPromptName = this._promptName; + if (this.chatSessionId) { + await AIProvider.session?.updateSession( + this.chatSessionId, + this._promptName + ); + } + } + } + private _addImages(images: File[]) { const oldImages = this.chatContextValue.images; this.updateContext({ @@ -363,12 +404,7 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) { const { images, status } = this.chatContextValue; const hasImages = images.length > 0; const maxHeight = hasImages ? 272 + 2 : 200 + 2; - const networkDisabled = - !!this.chatContextValue.images.length || - !!this.chatContextValue.chips.filter(chip => chip.state !== 'candidate') - .length; - const networkActive = !!this.networkSearchConfig.enabled.value; - const uploadDisabled = networkActive && !networkDisabled; + const uploadDisabled = this._isNetworkActive && !this._isNetworkDisabled; return html`