feat(server): init gemini provider & transcript action (#10731)

.github/actions/copilot-test/action.yml

@@ -8,6 +8,9 @@ inputs:
   openai-key:
     description: 'OpenAI secret key'
     required: true
+  google-key:
+    description: 'Google secret key'
+    required: true
   fal-key:
     description: 'Fal secret key'
     required: true
@@ -28,6 +31,7 @@ runs:
         COPILOT: true
         DEV_SERVER_URL: http://localhost:8080
         COPILOT_OPENAI_API_KEY: ${{ inputs.openai-key }}
+        COPILOT_GOOGLE_API_KEY: ${{ inputs.google-key }}
         COPILOT_FAL_API_KEY: ${{ inputs.fal-key }}
         COPILOT_PERPLEXITY_API_KEY: ${{ inputs.perplexity-key }}
 

.github/actions/deploy/deploy.mjs

@@ -17,6 +17,7 @@ const {
   METRICS_CUSTOMER_IO_TOKEN,
   COPILOT_OPENAI_API_KEY,
   COPILOT_FAL_API_KEY,
+  COPILOT_GOOGLE_API_KEY,
   COPILOT_PERPLEXITY_API_KEY,
   COPILOT_UNSPLASH_API_KEY,
   MAILER_SENDER,
@@ -160,6 +161,7 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set        graphql.app.copilot.enabled=true`,
     `--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`,
     `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
+    `--set-string graphql.app.copilot.google.key="${COPILOT_GOOGLE_API_KEY}"`,
     `--set-string graphql.app.copilot.perplexity.key="${COPILOT_PERPLEXITY_API_KEY}"`,
     `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
     `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,

.github/helm/affine/charts/graphql/templates/copilot-secret.yaml

@@ -7,6 +7,7 @@ type: Opaque
 data:
   openaiSecret: {{ .Values.app.copilot.openai.key | b64enc }}
   falSecret: {{ .Values.app.copilot.fal.key | b64enc }}
+  googleSecret: {{ .Values.app.copilot.google.key | b64enc }}
   perplexitySecret: {{ .Values.app.copilot.perplexity.key | b64enc }}
   unsplashSecret: {{ .Values.app.copilot.unsplash.key | b64enc }}
 {{- end }}

.github/workflows/build-test.yml

@@ -507,6 +507,7 @@ jobs:
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: 'use_fake_openai_api_key'
+          COPILOT_GOOGLE_API_KEY: 'use_fake_google_api_key'
           CI_NODE_INDEX: ${{ matrix.node_index }}
           CI_NODE_TOTAL: ${{ matrix.total_nodes }}
 
@@ -679,6 +680,7 @@ jobs:
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          COPILOT_GOOGLE_API_KEY: ${{ secrets.COPILOT_GOOGLE_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
           COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 
@@ -767,6 +769,7 @@ jobs:
         with:
           script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          google-key: ${{ secrets.COPILOT_GOOGLE_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
           perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 

.github/workflows/copilot-test.yml

@@ -147,6 +147,7 @@ jobs:
         with:
           script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          google-key: ${{ secrets.COPILOT_GOOGLE_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
           perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 

packages/backend/server/package.json

@@ -24,6 +24,7 @@
     "postinstall": "prisma generate"
   },
   "dependencies": {
+    "@ai-sdk/google": "^1.1.19",
     "@apollo/server": "^4.11.2",
     "@aws-sdk/client-s3": "^3.709.0",
     "@fal-ai/serverless-client": "^0.15.0",
@@ -64,6 +65,7 @@
     "@prisma/instrumentation": "^5.22.0",
     "@react-email/components": "0.0.33",
     "@socket.io/redis-adapter": "^8.3.0",
+    "ai": "^4.1.51",
     "bullmq": "^5.40.2",
     "cookie-parser": "^1.4.7",
     "date-fns": "^4.0.0",

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -1,5 +1,6 @@
 import type { ExecutionContext, TestFn } from 'ava';
 import ava from 'ava';
+import { z } from 'zod';
 
 import { ConfigModule } from '../base/config';
 import { AuthService } from '../core/auth';
@@ -269,6 +270,36 @@ test('should validate markdown list', t => {
 // ==================== action ====================
 
 const actions = [
+  {
+    promptName: ['Transcript audio'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: '',
+        attachments: [
+          'https://cdn.affine.pro/copilot-test/MP9qDGuYgnY+ILoEAmHpp3h9Npuw2403EAYMEA.mp3',
+        ],
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      // cleanup json markdown wrap
+      const cleaned = result
+        .replace(/```[\w\s]+\n/g, '')
+        .replace(/\n```/g, '')
+        .trim();
+      t.notThrows(() => {
+        z.object({
+          speaker: z.string(),
+          start: z.string(),
+          end: z.string(),
+          transcription: z.string(),
+        })
+          .array()
+          .parse(JSON.parse(cleaned));
+      });
+    },
+    type: 'text' as const,
+  },
   {
     promptName: [
       'Summary',
@@ -401,6 +432,7 @@ const actions = [
     type: 'image' as const,
   },
 ];
+
 for (const { promptName, messages, verifier, type } of actions) {
   const prompts = Array.isArray(promptName) ? promptName : [promptName];
   for (const promptName of prompts) {

packages/backend/server/src/config/affine.env.ts

@@ -28,6 +28,7 @@ AFFiNE.ENV_MAP = {
   CAPTCHA_TURNSTILE_SECRET: ['plugins.captcha.turnstile.secret', 'string'],
   COPILOT_OPENAI_API_KEY: 'plugins.copilot.openai.apiKey',
   COPILOT_FAL_API_KEY: 'plugins.copilot.fal.apiKey',
+  COPILOT_GOOGLE_API_KEY: 'plugins.copilot.google.apiKey',
   COPILOT_PERPLEXITY_API_KEY: 'plugins.copilot.perplexity.apiKey',
   COPILOT_UNSPLASH_API_KEY: 'plugins.copilot.unsplashKey',
   REDIS_SERVER_HOST: 'redis.host',

packages/backend/server/src/plugins/copilot/config.ts

@@ -3,11 +3,13 @@ import type { ClientOptions as OpenAIClientOptions } from 'openai';
 import { defineStartupConfig, ModuleConfig } from '../../base/config';
 import { StorageConfig } from '../../base/storage/config';
 import type { FalConfig } from './providers/fal';
+import { GoogleConfig } from './providers/google';
 import { PerplexityConfig } from './providers/perplexity';
 
 export interface CopilotStartupConfigurations {
   openai?: OpenAIClientOptions;
   fal?: FalConfig;
+  google?: GoogleConfig;
   perplexity?: PerplexityConfig;
   test?: never;
   unsplashKey?: string;

packages/backend/server/src/plugins/copilot/index.ts

@@ -23,6 +23,7 @@ import {
   PerplexityProvider,
   registerCopilotProvider,
 } from './providers';
+import { GoogleProvider } from './providers/google';
 import {
   CopilotResolver,
   PromptsManagementResolver,
@@ -34,6 +35,7 @@ import { CopilotWorkflowExecutors, CopilotWorkflowService } from './workflow';
 
 registerCopilotProvider(FalProvider);
 registerCopilotProvider(OpenAIProvider);
+registerCopilotProvider(GoogleProvider);
 registerCopilotProvider(PerplexityProvider);
 
 @Plugin({

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -317,6 +317,62 @@ const actions: Prompt[] = [
     model: 'face-to-sticker',
     messages: [],
   },
+  {
+    name: 'Transcript audio',
+    action: 'Transcript audio',
+    model: 'gemini-2.0-flash-001',
+    messages: [
+      {
+        role: 'system',
+        content: `
+Convert a multi-speaker audio recording into a structured JSON format by transcribing the speech and identifying individual speakers.
+
+1. Analyze the audio to detect the presence of multiple speakers using distinct microphone inputs.
+2. Transcribe the audio content for each speaker and note the time intervals of speech.
+
+# Output Format
+
+The output should be a JSON array, with each element containing:
+- "speaker": A label identifying the speaker, such as "A", "B", etc.
+- "start": The start time of the transcribed segment in the format "HH:MM:SS".
+- "end": The end time of the transcribed segment in the format "HH:MM:SS".
+- "transcription": The transcribed text for the speaker's segment.
+
+# Examples
+
+**Example Input:**
+- A multi-speaker audio file
+
+**Example Output:**
+
+[
+  {
+    "speaker": "A",
+    "start": "00:00:30",
+    "end": "00:00:45",
+    "transcription": "Hello, everyone."
+  },
+  {
+    "speaker": "B",
+    "start": "00:00:46",
+    "end": "00:01:10",
+    "transcription": "Hi, thank you for joining the meeting today."
+  }
+]
+
+# Notes
+
+- Ensure the accurate differentiation of speakers even if multiple speakers overlap slightly or switch rapidly.
+- Maintain a consistent speaker labeling system throughout the transcription.
+`,
+      },
+    ],
+    config: {
+      audioTimestamp: true,
+      jsonMode: true,
+    },
+  },
+
   {
     name: 'Generate a caption',
     action: 'Generate a caption',

packages/backend/server/src/plugins/copilot/providers/google.ts

@@ -0,0 +1,292 @@
+import {
+  createGoogleGenerativeAI,
+  type GoogleGenerativeAIProvider,
+} from '@ai-sdk/google';
+import { Logger } from '@nestjs/common';
+import {
+  AISDKError,
+  type CoreAssistantMessage,
+  type CoreUserMessage,
+  FilePart,
+  generateText,
+  streamText,
+  TextPart,
+} from 'ai';
+
+import {
+  CopilotPromptInvalid,
+  CopilotProviderSideError,
+  metrics,
+  UserFriendlyError,
+} from '../../../base';
+import {
+  ChatMessageRole,
+  CopilotCapability,
+  CopilotChatOptions,
+  CopilotProviderType,
+  CopilotTextToTextProvider,
+  PromptMessage,
+} from '../types';
+
+export const DEFAULT_DIMENSIONS = 256;
+
+const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
+const FORMAT_INFER_MAP: Record<string, string> = {
+  pdf: 'application/pdf',
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  png: 'image/png',
+  jpeg: 'image/jpeg',
+  jpg: 'image/jpeg',
+  webp: 'image/webp',
+  txt: 'text/plain',
+  md: 'text/plain',
+  mov: 'video/mov',
+  mpeg: 'video/mpeg',
+  mp4: 'video/mp4',
+  avi: 'video/avi',
+  wmv: 'video/wmv',
+  flv: 'video/flv',
+};
+
+export type GoogleConfig = {
+  apiKey: string;
+  baseUrl?: string;
+};
+
+type ChatMessage = CoreUserMessage | CoreAssistantMessage;
+
+export class GoogleProvider implements CopilotTextToTextProvider {
+  static readonly type = CopilotProviderType.Google;
+  static readonly capabilities = [CopilotCapability.TextToText];
+
+  readonly availableModels = [
+    // text to text
+    'gemini-2.0-flash-001',
+    // embeddings
+    'text-embedding-004',
+  ];
+
+  private readonly logger = new Logger(GoogleProvider.name);
+  private readonly instance: GoogleGenerativeAIProvider;
+
+  constructor(config: GoogleConfig) {
+    this.instance = createGoogleGenerativeAI(config);
+  }
+
+  static assetsConfig(config: GoogleConfig) {
+    return !!config?.apiKey;
+  }
+
+  get type(): CopilotProviderType {
+    return GoogleProvider.type;
+  }
+
+  getCapabilities(): CopilotCapability[] {
+    return GoogleProvider.capabilities;
+  }
+
+  async isModelAvailable(model: string): Promise<boolean> {
+    return this.availableModels.includes(model);
+  }
+
+  private inferMimeType(url: string) {
+    if (url.startsWith('data:')) {
+      return url.split(';')[0].split(':')[1];
+    }
+    const extension = url.split('.').pop();
+    if (extension) {
+      return FORMAT_INFER_MAP[extension];
+    }
+    return undefined;
+  }
+
+  protected chatToGPTMessage(
+    messages: PromptMessage[]
+  ): [string | undefined, ChatMessage[]] {
+    let system =
+      messages[0]?.role === 'system' ? messages.shift()?.content : undefined;
+
+    // filter redundant fields
+    const msgs = messages
+      .filter(m => m.role !== 'system')
+      .map(({ role, content, attachments, params }) => {
+        content = content.trim();
+        role = role as 'user' | 'assistant';
+        const mimetype = params?.mimetype;
+        if (Array.isArray(attachments)) {
+          const contents: (TextPart | FilePart)[] = [];
+          if (content.length) {
+            contents.push({
+              type: 'text',
+              text: content,
+            });
+          }
+          contents.push(
+            ...attachments
+              .map(url => {
+                if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
+                  const mimeType =
+                    typeof mimetype === 'string'
+                      ? mimetype
+                      : this.inferMimeType(url);
+                  if (mimeType) {
+                    const data = url.startsWith('data:') ? url : new URL(url);
+                    return {
+                      type: 'file' as const,
+                      data,
+                      mimeType,
+                    };
+                  }
+                }
+                return undefined;
+              })
+              .filter(c => !!c)
+          );
+          return { role, content: contents } as ChatMessage;
+        } else {
+          return { role, content } as ChatMessage;
+        }
+      });
+    return [system, msgs];
+  }
+
+  protected async checkParams({
+    messages,
+    embeddings,
+    model,
+    options = {},
+  }: {
+    messages?: PromptMessage[];
+    embeddings?: string[];
+    model: string;
+    options: CopilotChatOptions;
+  }) {
+    if (!(await this.isModelAvailable(model))) {
+      throw new CopilotPromptInvalid(`Invalid model: ${model}`);
+    }
+    if (Array.isArray(messages) && messages.length > 0) {
+      if (
+        messages.some(
+          m =>
+            // check non-object
+            typeof m !== 'object' ||
+            !m ||
+            // check content
+            typeof m.content !== 'string' ||
+            // content and attachments must exist at least one
+            ((!m.content || !m.content.trim()) &&
+              (!Array.isArray(m.attachments) || !m.attachments.length))
+        )
+      ) {
+        throw new CopilotPromptInvalid('Empty message content');
+      }
+      if (
+        messages.some(
+          m =>
+            typeof m.role !== 'string' ||
+            !m.role ||
+            !ChatMessageRole.includes(m.role)
+        )
+      ) {
+        throw new CopilotPromptInvalid('Invalid message role');
+      }
+      // json mode need 'json' keyword in content
+      // ref: https://platform.openai.com/docs/api-reference/chat/create#chat-create-response_format
+      if (
+        options.jsonMode &&
+        !messages.some(m => m.content.toLowerCase().includes('json'))
+      ) {
+        throw new CopilotPromptInvalid('Prompt not support json mode');
+      }
+    } else if (
+      Array.isArray(embeddings) &&
+      embeddings.some(e => typeof e !== 'string' || !e || !e.trim())
+    ) {
+      throw new CopilotPromptInvalid('Invalid embedding');
+    }
+  }
+
+  private handleError(e: any) {
+    if (e instanceof UserFriendlyError) {
+      return e;
+    } else if (e instanceof AISDKError) {
+      this.logger.error('Throw error from ai sdk:', e);
+      return new CopilotProviderSideError({
+        provider: this.type,
+        kind: e.name || 'unknown',
+        message: e.message,
+      });
+    } else {
+      return new CopilotProviderSideError({
+        provider: this.type,
+        kind: 'unexpected_response',
+        message: e?.message || 'Unexpected google response',
+      });
+    }
+  }
+
+  // ====== text to text ======
+  async generateText(
+    messages: PromptMessage[],
+    model: string = 'gemini-2.0-flash-001',
+    options: CopilotChatOptions = {}
+  ): Promise<string> {
+    await this.checkParams({ messages, model, options });
+
+    try {
+      metrics.ai.counter('chat_text_calls').add(1, { model });
+
+      const [system, msgs] = this.chatToGPTMessage(messages);
+
+      const { text } = await generateText({
+        model: this.instance(model, {
+          audioTimestamp: Boolean(options.audioTimestamp),
+          structuredOutputs: Boolean(options.jsonMode),
+        }),
+        system,
+        messages: msgs,
+        abortSignal: options.signal,
+      });
+
+      if (!text) throw new Error('Failed to generate text');
+      return text.trim();
+    } catch (e: any) {
+      metrics.ai.counter('chat_text_errors').add(1, { model });
+      throw this.handleError(e);
+    }
+  }
+
+  async *generateTextStream(
+    messages: PromptMessage[],
+    model: string = 'gpt-4o-mini',
+    options: CopilotChatOptions = {}
+  ): AsyncIterable<string> {
+    await this.checkParams({ messages, model, options });
+
+    try {
+      metrics.ai.counter('chat_text_stream_calls').add(1, { model });
+      const [system, msgs] = this.chatToGPTMessage(messages);
+
+      const { textStream } = streamText({
+        model: this.instance(model),
+        system,
+        messages: msgs,
+        abortSignal: options.signal,
+      });
+
+      for await (const message of textStream) {
+        if (message) {
+          yield message;
+          if (options.signal?.aborted) {
+            await textStream.cancel();
+            break;
+          }
+        }
+      }
+    } catch (e: any) {
+      metrics.ai.counter('chat_text_stream_errors').add(1, { model });
+      throw this.handleError(e);
+    }
+  }
+}

packages/backend/server/src/plugins/copilot/types.ts

@@ -77,6 +77,8 @@ export const PromptConfigStrictSchema = z.object({
     )
     .nullable()
     .optional(),
+  // google
+  audioTimestamp: z.boolean().nullable().optional(),
 });
 
 export const PromptConfigSchema =
@@ -156,6 +158,7 @@ export type ListHistoriesOptions = {
 
 export enum CopilotProviderType {
   FAL = 'fal',
+  Google = 'google',
   OpenAI = 'openai',
   Perplexity = 'perplexity',
   // only for test

yarn.lock

@@ -845,6 +845,7 @@ __metadata:
     "@affine-tools/cli": "workspace:*"
     "@affine-tools/utils": "workspace:*"
     "@affine/server-native": "workspace:*"
+    "@ai-sdk/google": "npm:^1.1.19"
     "@apollo/server": "npm:^4.11.2"
     "@aws-sdk/client-s3": "npm:^3.709.0"
     "@faker-js/faker": "npm:^9.6.0"
@@ -901,6 +902,7 @@ __metadata:
     "@types/semver": "npm:^7.5.8"
     "@types/sinon": "npm:^17.0.3"
     "@types/supertest": "npm:^6.0.2"
+    ai: "npm:^4.1.51"
     ava: "npm:^6.2.0"
     bullmq: "npm:^5.40.2"
     c8: "npm:^10.1.3"
@@ -1000,6 +1002,80 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@ai-sdk/google@npm:^1.1.19":
+  version: 1.1.19
+  resolution: "@ai-sdk/google@npm:1.1.19"
+  dependencies:
+    "@ai-sdk/provider": "npm:1.0.9"
+    "@ai-sdk/provider-utils": "npm:2.1.10"
+  peerDependencies:
+    zod: ^3.0.0
+  checksum: 10/c49ec5b6e17f37b0beeb540b95ed424e347fa96f7d028f16610cdd2933d8cc6eea9df584fcee50db029ac0fe50b094b8efef44e6fc5c9d05cbd6a2d9c904008e
+  languageName: node
+  linkType: hard
+
+"@ai-sdk/provider-utils@npm:2.1.10":
+  version: 2.1.10
+  resolution: "@ai-sdk/provider-utils@npm:2.1.10"
+  dependencies:
+    "@ai-sdk/provider": "npm:1.0.9"
+    eventsource-parser: "npm:^3.0.0"
+    nanoid: "npm:^3.3.8"
+    secure-json-parse: "npm:^2.7.0"
+  peerDependencies:
+    zod: ^3.0.0
+  peerDependenciesMeta:
+    zod:
+      optional: true
+  checksum: 10/600a732d5e0b02b873234921b8aa873a9c78b23a4d058a68e30e0783b1be0ba5067621cf119a3a676d082d1fbacad6951533ab7bb55bdb3a7d29b63a0bcd9b18
+  languageName: node
+  linkType: hard
+
+"@ai-sdk/provider@npm:1.0.9":
+  version: 1.0.9
+  resolution: "@ai-sdk/provider@npm:1.0.9"
+  dependencies:
+    json-schema: "npm:^0.4.0"
+  checksum: 10/5f399ded99da304821010ef189df3f7da72fc4c28c73d570bdc27dc8157dfa35cef8552099c450bef51b1c74c80a6029b36a515a035d4426fae8d5c5269fb82c
+  languageName: node
+  linkType: hard
+
+"@ai-sdk/react@npm:1.1.20":
+  version: 1.1.20
+  resolution: "@ai-sdk/react@npm:1.1.20"
+  dependencies:
+    "@ai-sdk/provider-utils": "npm:2.1.10"
+    "@ai-sdk/ui-utils": "npm:1.1.16"
+    swr: "npm:^2.2.5"
+    throttleit: "npm:2.1.0"
+  peerDependencies:
+    react: ^18 || ^19 || ^19.0.0-rc
+    zod: ^3.0.0
+  peerDependenciesMeta:
+    react:
+      optional: true
+    zod:
+      optional: true
+  checksum: 10/85e948ab30f0ea15deea480fc25799feb14540c0b3a28591ba2021f1ce6cc2b2cb1ab95b0edb18ef8791c556a15142b504c8825f32dd66fa3363d504a254f11c
+  languageName: node
+  linkType: hard
+
+"@ai-sdk/ui-utils@npm:1.1.16":
+  version: 1.1.16
+  resolution: "@ai-sdk/ui-utils@npm:1.1.16"
+  dependencies:
+    "@ai-sdk/provider": "npm:1.0.9"
+    "@ai-sdk/provider-utils": "npm:2.1.10"
+    zod-to-json-schema: "npm:^3.24.1"
+  peerDependencies:
+    zod: ^3.0.0
+  peerDependenciesMeta:
+    zod:
+      optional: true
+  checksum: 10/406c4e76d131df6ddd49d4dae2d10745fb41ac1881f71c15facb333550a5eb636e295f499ef3e972e5af3d5e87c7b2685fe8f4b41a424b419f90f82aa554fc93
+  languageName: node
+  linkType: hard
+
 "@alloc/quick-lru@npm:^5.2.0":
   version: 5.2.0
   resolution: "@alloc/quick-lru@npm:5.2.0"
@@ -8948,7 +9024,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.8, @opentelemetry/api@npm:^1.9.0":
+"@opentelemetry/api@npm:1.9.0, @opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.8, @opentelemetry/api@npm:^1.9.0":
   version: 1.9.0
   resolution: "@opentelemetry/api@npm:1.9.0"
   checksum: 10/a607f0eef971893c4f2ee2a4c2069aade6ec3e84e2a1f5c2aac19f65c5d9eeea41aa72db917c1029faafdd71789a1a040bdc18f40d63690e22ccae5d7070f194
@@ -14101,6 +14177,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/diff-match-patch@npm:^1.0.36":
+  version: 1.0.36
+  resolution: "@types/diff-match-patch@npm:1.0.36"
+  checksum: 10/7d7ce03422fcc3e79d0cda26e4748aeb176b75ca4b4e5f38459b112bf24660d628424bdb08d330faefa69039d19a5316e7a102a8ab68b8e294c8346790e55113
+  languageName: node
+  linkType: hard
+
 "@types/doctrine@npm:^0.0.9":
   version: 0.0.9
   resolution: "@types/doctrine@npm:0.0.9"
@@ -15778,6 +15861,28 @@ __metadata:
   languageName: node
   linkType: hard
 
+"ai@npm:^4.1.51":
+  version: 4.1.51
+  resolution: "ai@npm:4.1.51"
+  dependencies:
+    "@ai-sdk/provider": "npm:1.0.9"
+    "@ai-sdk/provider-utils": "npm:2.1.10"
+    "@ai-sdk/react": "npm:1.1.20"
+    "@ai-sdk/ui-utils": "npm:1.1.16"
+    "@opentelemetry/api": "npm:1.9.0"
+    jsondiffpatch: "npm:0.6.0"
+  peerDependencies:
+    react: ^18 || ^19 || ^19.0.0-rc
+    zod: ^3.0.0
+  peerDependenciesMeta:
+    react:
+      optional: true
+    zod:
+      optional: true
+  checksum: 10/52b8e94929b77a38c7005767a0bf2334aa6820555b0ecb70f89b35b03eb9c3c8478af826526c734bff2baba7245426b366b34c63c09ca749fcb78af602aedd19
+  languageName: node
+  linkType: hard
+
 "ajv-formats@npm:^2.1.1":
   version: 2.1.1
   resolution: "ajv-formats@npm:2.1.1"
@@ -19061,6 +19166,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"diff-match-patch@npm:^1.0.5":
+  version: 1.0.5
+  resolution: "diff-match-patch@npm:1.0.5"
+  checksum: 10/fd1ab417eba9559bda752a4dfc9a8ac73fa2ca8b146d29d153964b437168e301c09d8a688fae0cd81d32dc6508a4918a94614213c85df760793f44e245173bb6
+  languageName: node
+  linkType: hard
+
 "diff@npm:^4.0.1":
   version: 4.0.2
   resolution: "diff@npm:4.0.2"
@@ -23825,6 +23937,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"json-schema@npm:^0.4.0":
+  version: 0.4.0
+  resolution: "json-schema@npm:0.4.0"
+  checksum: 10/8b3b64eff4a807dc2a3045b104ed1b9335cd8d57aa74c58718f07f0f48b8baa3293b00af4dcfbdc9144c3aafea1e97982cc27cc8e150fc5d93c540649507a458
+  languageName: node
+  linkType: hard
+
 "json-stable-stringify-without-jsonify@npm:^1.0.1":
   version: 1.0.1
   resolution: "json-stable-stringify-without-jsonify@npm:1.0.1"
@@ -23865,6 +23984,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"jsondiffpatch@npm:0.6.0":
+  version: 0.6.0
+  resolution: "jsondiffpatch@npm:0.6.0"
+  dependencies:
+    "@types/diff-match-patch": "npm:^1.0.36"
+    chalk: "npm:^5.3.0"
+    diff-match-patch: "npm:^1.0.5"
+  bin:
+    jsondiffpatch: bin/jsondiffpatch.js
+  checksum: 10/124b9797c266c693e69f8d23216e64d5ca4b21a4ec10e3a769a7b8cb19602ba62522f9a3d0c55299c1bfbe5ad955ca9ad2852439ca2c6b6316b8f91a5c218e94
+  languageName: node
+  linkType: hard
+
 "jsonfile@npm:^4.0.0":
   version: 4.0.0
   resolution: "jsonfile@npm:4.0.0"
@@ -30128,6 +30260,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"secure-json-parse@npm:^2.7.0":
+  version: 2.7.0
+  resolution: "secure-json-parse@npm:2.7.0"
+  checksum: 10/974386587060b6fc5b1ac06481b2f9dbbb0d63c860cc73dc7533f27835fdb67b0ef08762dbfef25625c15bc0a0c366899e00076cb0d556af06b71e22f1dede4c
+  languageName: node
+  linkType: hard
+
 "selderee@npm:^0.11.0":
   version: 0.11.0
   resolution: "selderee@npm:0.11.0"
@@ -31820,6 +31959,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"throttleit@npm:2.1.0":
+  version: 2.1.0
+  resolution: "throttleit@npm:2.1.0"
+  checksum: 10/a2003947aafc721c4a17e6f07db72dc88a64fa9bba0f9c659f7997d30f9590b3af22dadd6a41851e0e8497d539c33b2935c2c7919cf4255922509af6913c619b
+  languageName: node
+  linkType: hard
+
 "through2@npm:^4.0.2":
   version: 4.0.2
   resolution: "through2@npm:4.0.2"
@@ -33973,6 +34119,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"zod-to-json-schema@npm:^3.24.1":
+  version: 3.24.3
+  resolution: "zod-to-json-schema@npm:3.24.3"
+  peerDependencies:
+    zod: ^3.24.1
+  checksum: 10/9dc6fafc3a9b5e088a92e2296bee6bc7b1c149f1d762c535a17626ce471721ef0c472d287f137f5408dad6368469621fe538ed8f0bf79811c7a69d67a1a7908b
+  languageName: node
+  linkType: hard
+
 "zod@npm:^3.23.8, zod@npm:^3.24.1":
   version: 3.24.2
   resolution: "zod@npm:3.24.2"