feat(core): support ai network search (#9357)

### What Changed? - Add `PerplexityProvider` in backend. - Update session prompt name if user toggle network search mode in chat panel. - Add experimental flag for AI network search feature. - Add unit tests and e2e tests. Search results are streamed and appear word for word: <div class='graphite__hidden'> <div>🎥 Video uploaded on Graphite:</div> <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov"> <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov"> </a> </div> <video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov">录屏2024-12-27 18.58.40.mov</video> Click the little globe icon to manually turn on/off Internet search: <div class='graphite__hidden'> <div>🎥 Video uploaded on Graphite:</div> <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov"> <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov"> </a> </div> <video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov">录屏2024-12-27 19.01.16.mov</video> When there is an image, it will automatically switch to the openai model: <div class='graphite__hidden'> <div>🎥 Video uploaded on Graphite:</div> <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov"> <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov"> </a> </div> <video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov">录屏2024-12-27 19.02.13.mov</video>
2026-02-22 00:37:05 +08:00 · 2025-01-09 04:00:58 +00:00
parent 4f10457815
commit 58ce86533e
49 changed files with 1274 additions and 169 deletions
--- a/packages/frontend/core/src/blocksuite/presets/ai/ai-panel.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/ai-panel.ts
@@ -1,3 +1,4 @@
+import { AINetworkSearchService } from '@affine/core/modules/ai-button/services/network-search';
 import type { EditorHost } from '@blocksuite/affine/block-std';
 import {
  type AffineAIPanelWidget,
@@ -9,6 +10,7 @@ import {
  NoteDisplayMode,
 } from '@blocksuite/affine/blocks';
 import { assertExists, Bound } from '@blocksuite/affine/global/utils';
+import type { FrameworkProvider } from '@toeverything/infra';
 import type { TemplateResult } from 'lit';

 import { createTextRenderer, insertFromMarkdown } from '../_common';
@@ -287,14 +289,21 @@ export function buildCopyConfig(panel: AffineAIPanelWidget) {
 }

 export function buildAIPanelConfig(
-  panel: AffineAIPanelWidget
+  panel: AffineAIPanelWidget,
+  framework: FrameworkProvider
 ): AffineAIPanelWidgetConfig {
  const ctx = new AIContext();
+  const searchService = framework.get(AINetworkSearchService);
  return {
    answerRenderer: createTextRenderer(panel.host, { maxHeight: 320 }),
    finishStateConfig: buildFinishConfig(panel, 'chat', ctx),
    generatingStateConfig: buildGeneratingConfig(),
    errorStateConfig: buildErrorConfig(panel),
    copy: buildCopyConfig(panel),
+    networkSearchConfig: {
+      visible: searchService.visible,
+      enabled: searchService.enabled,
+      setEnabled: searchService.setEnabled,
+    },
  };
 }
--- a/packages/frontend/core/src/blocksuite/presets/ai/ai-spec.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/ai-spec.ts
@@ -23,6 +23,7 @@ import {
 } from '@blocksuite/affine/blocks';
 import { assertInstanceOf } from '@blocksuite/affine/global/utils';
 import type { ExtensionType } from '@blocksuite/affine/store';
+import type { FrameworkProvider } from '@toeverything/infra';
 import { literal, unsafeStatic } from 'lit/static-html.js';

 import { buildAIPanelConfig } from './ai-panel';
@@ -36,96 +37,110 @@ import { setupImageToolbarAIEntry } from './entries/image-toolbar/setup-image-to
 import { setupSlashMenuAIEntry } from './entries/slash-menu/setup-slash-menu';
 import { setupSpaceAIEntry } from './entries/space/setup-space';

-class AIPageRootWatcher extends BlockServiceWatcher {
-  static override readonly flavour = 'affine:page';
+function getAIPageRootWatcher(framework: FrameworkProvider) {
+  class AIPageRootWatcher extends BlockServiceWatcher {
+    static override readonly flavour = 'affine:page';

-  override mounted() {
-    super.mounted();
-    this.blockService.specSlots.widgetConnected.on(view => {
-      if (view.component instanceof AffineAIPanelWidget) {
-        view.component.style.width = '630px';
-        view.component.config = buildAIPanelConfig(view.component);
-        setupSpaceAIEntry(view.component);
-      }
+    override mounted() {
+      super.mounted();
+      this.blockService.specSlots.widgetConnected.on(view => {
+        if (view.component instanceof AffineAIPanelWidget) {
+          view.component.style.width = '630px';
+          view.component.config = buildAIPanelConfig(view.component, framework);
+          setupSpaceAIEntry(view.component);
+        }

-      if (view.component instanceof AffineFormatBarWidget) {
-        setupFormatBarAIEntry(view.component);
-      }
+        if (view.component instanceof AffineFormatBarWidget) {
+          setupFormatBarAIEntry(view.component);
+        }

-      if (view.component instanceof AffineSlashMenuWidget) {
-        setupSlashMenuAIEntry(view.component);
-      }
-    });
+        if (view.component instanceof AffineSlashMenuWidget) {
+          setupSlashMenuAIEntry(view.component);
+        }
+      });
+    }
  }
+  return AIPageRootWatcher;
 }

-export const AIPageRootBlockSpec: ExtensionType[] = [
-  ...PageRootBlockSpec,
-  AIPageRootWatcher,
-  {
-    setup: di => {
-      di.override(WidgetViewMapIdentifier('affine:page'), () => {
-        return {
-          ...pageRootWidgetViewMap,
-          [AFFINE_AI_PANEL_WIDGET]: literal`${unsafeStatic(
-            AFFINE_AI_PANEL_WIDGET
-          )}`,
-        };
-      });
+export function createAIPageRootBlockSpec(
+  framework: FrameworkProvider
+): ExtensionType[] {
+  return [
+    ...PageRootBlockSpec,
+    getAIPageRootWatcher(framework),
+    {
+      setup: di => {
+        di.override(WidgetViewMapIdentifier('affine:page'), () => {
+          return {
+            ...pageRootWidgetViewMap,
+            [AFFINE_AI_PANEL_WIDGET]: literal`${unsafeStatic(
+              AFFINE_AI_PANEL_WIDGET
+            )}`,
+          };
+        });
+      },
    },
-  },
-];
-
-class AIEdgelessRootWatcher extends BlockServiceWatcher {
-  static override readonly flavour = 'affine:page';
-
-  override mounted() {
-    super.mounted();
-    this.blockService.specSlots.widgetConnected.on(view => {
-      if (view.component instanceof AffineAIPanelWidget) {
-        view.component.style.width = '430px';
-        view.component.config = buildAIPanelConfig(view.component);
-        setupSpaceAIEntry(view.component);
-      }
-
-      if (view.component instanceof EdgelessCopilotWidget) {
-        setupEdgelessCopilot(view.component);
-      }
-
-      if (view.component instanceof EdgelessElementToolbarWidget) {
-        setupEdgelessElementToolbarAIEntry(view.component);
-      }
-
-      if (view.component instanceof AffineFormatBarWidget) {
-        setupFormatBarAIEntry(view.component);
-      }
-
-      if (view.component instanceof AffineSlashMenuWidget) {
-        setupSlashMenuAIEntry(view.component);
-      }
-    });
-  }
+  ];
 }

-export const AIEdgelessRootBlockSpec: ExtensionType[] = [
-  ...EdgelessRootBlockSpec,
-  AIEdgelessRootWatcher,
-  {
-    setup: di => {
-      di.override(WidgetViewMapIdentifier('affine:page'), () => {
-        return {
-          ...edgelessRootWidgetViewMap,
-          [AFFINE_EDGELESS_COPILOT_WIDGET]: literal`${unsafeStatic(
-            AFFINE_EDGELESS_COPILOT_WIDGET
-          )}`,
-          [AFFINE_AI_PANEL_WIDGET]: literal`${unsafeStatic(
-            AFFINE_AI_PANEL_WIDGET
-          )}`,
-        };
+function getAIEdgelessRootWatcher(framework: FrameworkProvider) {
+  class AIEdgelessRootWatcher extends BlockServiceWatcher {
+    static override readonly flavour = 'affine:page';
+
+    override mounted() {
+      super.mounted();
+      this.blockService.specSlots.widgetConnected.on(view => {
+        if (view.component instanceof AffineAIPanelWidget) {
+          view.component.style.width = '430px';
+          view.component.config = buildAIPanelConfig(view.component, framework);
+          setupSpaceAIEntry(view.component);
+        }
+
+        if (view.component instanceof EdgelessCopilotWidget) {
+          setupEdgelessCopilot(view.component);
+        }
+
+        if (view.component instanceof EdgelessElementToolbarWidget) {
+          setupEdgelessElementToolbarAIEntry(view.component);
+        }
+
+        if (view.component instanceof AffineFormatBarWidget) {
+          setupFormatBarAIEntry(view.component);
+        }
+
+        if (view.component instanceof AffineSlashMenuWidget) {
+          setupSlashMenuAIEntry(view.component);
+        }
      });
+    }
+  }
+  return AIEdgelessRootWatcher;
+}
+
+export function createAIEdgelessRootBlockSpec(
+  framework: FrameworkProvider
+): ExtensionType[] {
+  return [
+    ...EdgelessRootBlockSpec,
+    getAIEdgelessRootWatcher(framework),
+    {
+      setup: di => {
+        di.override(WidgetViewMapIdentifier('affine:page'), () => {
+          return {
+            ...edgelessRootWidgetViewMap,
+            [AFFINE_EDGELESS_COPILOT_WIDGET]: literal`${unsafeStatic(
+              AFFINE_EDGELESS_COPILOT_WIDGET
+            )}`,
+            [AFFINE_AI_PANEL_WIDGET]: literal`${unsafeStatic(
+              AFFINE_AI_PANEL_WIDGET
+            )}`,
+          };
+        });
+      },
    },
-  },
-];
+  ];
+}

 class AIParagraphBlockWatcher extends BlockServiceWatcher {
  static override readonly flavour = 'affine:paragraph';
--- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts
@@ -1,6 +1,14 @@
+import { stopPropagation } from '@affine/core/utils';
 import type { EditorHost } from '@blocksuite/affine/block-std';
 import { type AIError, openFileOrFiles } from '@blocksuite/affine/blocks';
-import { assertExists, WithDisposable } from '@blocksuite/affine/global/utils';
+import {
+  assertExists,
+  SignalWatcher,
+  WithDisposable,
+} from '@blocksuite/affine/global/utils';
+import { unsafeCSSVarV2 } from '@blocksuite/affine-shared/theme';
+import { ImageIcon, PublishIcon } from '@blocksuite/icons/lit';
+import type { Signal } from '@preact/signals-core';
 import { css, html, LitElement, nothing } from 'lit';
 import { property, query, state } from 'lit/decorators.js';
 import { repeat } from 'lit/directives/repeat.js';
@@ -10,7 +18,6 @@ import {
  ChatClearIcon,
  ChatSendIcon,
  CloseIcon,
-  ImageIcon,
 } from '../_common/icons';
 import { AIProvider } from '../provider';
 import { reportResponse } from '../utils/action-reporter';
@@ -24,7 +31,13 @@ function getFirstTwoLines(text: string) {
  return lines.slice(0, 2);
 }

-export class ChatPanelInput extends WithDisposable(LitElement) {
+export interface AINetworkSearchConfig {
+  visible: Signal<boolean | undefined>;
+  enabled: Signal<boolean | undefined>;
+  setEnabled: (state: boolean) => void;
+}
+
+export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) {
  static override styles = css`
    .chat-panel-input {
      display: flex;
@@ -104,10 +117,28 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
        margin-left: auto;
      }

-      .image-upload {
+      .image-upload,
+      .chat-network-search {
        display: flex;
        justify-content: center;
        align-items: center;
+        svg {
+          width: 20px;
+          height: 20px;
+          color: ${unsafeCSSVarV2('icon/primary')};
+        }
+      }
+      .chat-network-search[data-active='true'] svg {
+        color: ${unsafeCSSVarV2('icon/activated')};
+      }
+
+      .image-upload[aria-disabled='true'],
+      .chat-network-search[aria-disabled='true'] {
+        cursor: not-allowed;
+      }
+      .image-upload[aria-disabled='true'] svg,
+      .chat-network-search[aria-disabled='true'] svg {
+        color: var(--affine-text-disable-color) !important;
      }
    }

@@ -235,6 +266,9 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
  @property({ attribute: false })
  accessor cleanupHistories!: () => Promise<void>;

+  @property({ attribute: false })
+  accessor networkSearchConfig!: AINetworkSearchConfig;
+
  private _addImages(images: File[]) {
    const oldImages = this.chatContextValue.images;
    this.updateContext({
@@ -296,6 +330,23 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
    `;
  }

+  private readonly _toggleNetworkSearch = (e: MouseEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    const enable = this.networkSearchConfig.enabled.value;
+    this.networkSearchConfig.setEnabled(!enable);
+  };
+
+  private readonly _uploadImageFiles = async (_e: MouseEvent) => {
+    const images = await openFileOrFiles({
+      acceptType: 'Images',
+      multiple: true,
+    });
+    if (!images) return;
+    this._addImages(images);
+  };
+
  override connectedCallback() {
    super.connectedCallback();

@@ -305,7 +356,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
          if (this.host === host) {
            context && this.updateContext(context);
            await this.updateComplete;
-            await this.send(input);
+            input && (await this.send(input));
          }
        }
      )
@@ -316,7 +367,9 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
    const { images, status } = this.chatContextValue;
    const hasImages = images.length > 0;
    const maxHeight = hasImages ? 272 + 2 : 200 + 2;
-
+    const networkDisabled = !!this.chatContextValue.images.length;
+    const networkActive = !!this.networkSearchConfig.enabled.value;
+    const uploadDisabled = networkActive && !networkDisabled;
    return html`<style>
        .chat-panel-input {
          border-color: ${this.focused
@@ -364,8 +417,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
          }}
          @keydown=${async (evt: KeyboardEvent) => {
            if (evt.key === 'Enter' && !evt.shiftKey && !evt.isComposing) {
-              evt.preventDefault();
-              await this.send();
+              this._onTextareaSend(evt);
            }
          }}
          @focus=${() => {
@@ -399,19 +451,29 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
          >
            ${ChatClearIcon}
          </div>
+          ${this.networkSearchConfig.visible.value
+            ? html`
+                <div
+                  class="chat-network-search"
+                  data-testid="chat-network-search"
+                  aria-disabled=${networkDisabled}
+                  data-active=${networkActive}
+                  @click=${networkDisabled
+                    ? undefined
+                    : this._toggleNetworkSearch}
+                  @pointerdown=${stopPropagation}
+                >
+                  ${PublishIcon()}
+                </div>
+              `
+            : nothing}
          ${images.length < MaximumImageCount
            ? html`<div
                class="image-upload"
-                @click=${async () => {
-                  const images = await openFileOrFiles({
-                    acceptType: 'Images',
-                    multiple: true,
-                  });
-                  if (!images) return;
-                  this._addImages(images);
-                }}
+                aria-disabled=${uploadDisabled}
+                @click=${uploadDisabled ? undefined : this._uploadImageFiles}
              >
-                ${ImageIcon}
+                ${ImageIcon()}
              </div>`
            : nothing}
          ${status === 'transmitting'
@@ -425,7 +487,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
                ${ChatAbortIcon}
              </div>`
            : html`<div
-                @click="${this.send}"
+                @click="${this._onTextareaSend}"
                class="chat-panel-send"
                aria-disabled=${this.isInputEmpty}
                data-testid="chat-panel-send"
@@ -436,19 +498,30 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
      </div>`;
  }

-  send = async (input?: string) => {
+  private readonly _onTextareaSend = (e: MouseEvent | KeyboardEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    const value = this.textarea.value.trim();
+    if (value.length === 0) return;
+
+    this.textarea.value = '';
+    this.isInputEmpty = true;
+    this.textarea.style.height = 'unset';
+
+    this.send(value).catch(console.error);
+  };
+
+  send = async (text: string) => {
    const { status, markdown } = this.chatContextValue;
    if (status === 'loading' || status === 'transmitting') return;

-    const text = input || this.textarea.value;
    const { images } = this.chatContextValue;
    if (!text && images.length === 0) {
      return;
    }
    const { doc } = this.host;
-    this.textarea.value = '';
-    this.isInputEmpty = true;
-    this.textarea.style.height = 'unset';
+
    this.updateContext({
      images: [],
      status: 'loading',
--- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts
@@ -17,6 +17,7 @@ import {
  getSelectedTextContent,
 } from '../utils/selection-utils';
 import type { ChatAction, ChatContextValue, ChatItem } from './chat-context';
+import type { AINetworkSearchConfig } from './chat-panel-input';
 import type { ChatPanelMessages } from './chat-panel-messages';

 export class ChatPanel extends WithDisposable(ShadowlessElement) {
@@ -143,6 +144,9 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
  @property({ attribute: false })
  accessor doc!: Store;

+  @property({ attribute: false })
+  accessor networkSearchConfig!: AINetworkSearchConfig;
+
  @state()
  accessor isLoading = false;

@@ -280,6 +284,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
      ></chat-panel-messages>
      <chat-panel-input
        .chatContextValue=${this.chatContextValue}
+        .networkSearchConfig=${this.networkSearchConfig}
        .updateContext=${this.updateContext}
        .host=${this.host}
        .cleanupHistories=${this._cleanupHistories}