feat: adapt cloudflare worker ai (#14732)

#### PR Dependency Tree * **PR #14732** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal)  ## Summary by CodeRabbit * **New Features** * Cloudflare Workers AI added as a Copilot provider (configurable in admin settings). * Gemini 3.1 Flash Lite Preview model made available. * **Behavior Changes** * Reranking now uses a different default model identifier (affects relevancy scores). * **Tests** * Rerank tests adjusted to focus on the updated model and expected results.
2026-07-16 01:26:37 +08:00 · 2026-03-27 02:15:49 +08:00
parent 5b05c5a1b2
commit a41c5e4366
18 changed files with 502 additions and 30 deletions
@@ -10,6 +10,7 @@ import {
  AnthropicOfficialConfig,
  AnthropicVertexConfig,
 } from './providers/anthropic';
+import { CloudflareWorkersAIConfig } from './providers/cloudflare';
 import type { FalConfig } from './providers/fal';
 import { GeminiGenerativeConfig, GeminiVertexConfig } from './providers/gemini';
 import { MorphConfig } from './providers/morph';
@@ -23,6 +24,7 @@ import {

 export type CopilotProviderConfigMap = {
  [CopilotProviderType.OpenAI]: OpenAIConfig;
+  [CopilotProviderType.CloudflareWorkersAi]: CloudflareWorkersAIConfig;
  [CopilotProviderType.FAL]: FalConfig;
  [CopilotProviderType.Gemini]: GeminiGenerativeConfig;
  [CopilotProviderType.GeminiVertex]: GeminiVertexConfig;
@@ -117,6 +119,12 @@ const FalConfigShape = z.object({
  apiKey: z.string(),
 });

+const CloudflareWorkersAIConfigShape = z.object({
+  apiToken: z.string(),
+  accountId: z.string().optional(),
+  baseURL: z.string().optional(),
+});
+
 const GeminiGenerativeConfigShape = z.object({
  apiKey: z.string(),
  baseURL: z.string().optional(),
@@ -153,6 +161,10 @@ const CopilotProviderProfileShape = z.discriminatedUnion('type', [
    type: z.literal(CopilotProviderType.FAL),
    config: FalConfigShape,
  }),
+  CopilotProviderProfileBaseShape.extend({
+    type: z.literal(CopilotProviderType.CloudflareWorkersAi),
+    config: CloudflareWorkersAIConfigShape,
+  }),
  CopilotProviderProfileBaseShape.extend({
    type: z.literal(CopilotProviderType.Gemini),
    config: GeminiGenerativeConfigShape,
@@ -205,6 +217,7 @@ declare global {
        profiles: ConfigItem<CopilotProviderProfile[]>;
        defaults: ConfigItem<CopilotProviderDefaults>;
        openai: ConfigItem<OpenAIConfig>;
+        cloudflareWorkersAi: ConfigItem<CloudflareWorkersAIConfig>;
        fal: ConfigItem<FalConfig>;
        gemini: ConfigItem<GeminiGenerativeConfig>;
        geminiVertex: ConfigItem<GeminiVertexConfig>;
@@ -257,6 +270,13 @@ defineModuleConfig('copilot', {
    },
    link: 'https://github.com/openai/openai-node',
  },
+  'providers.cloudflareWorkersAi': {
+    desc: 'The config for the Cloudflare Workers AI provider.',
+    default: {
+      apiToken: '',
+      accountId: '',
+    },
+  },
  'providers.fal': {
    desc: 'The config for the fal provider.',
    default: {
@@ -19,7 +19,7 @@ import {
 import { EmbeddingClient, type ReRankResult } from './types';

 const EMBEDDING_MODEL = 'gemini-embedding-001';
-const RERANK_MODEL = 'gpt-5.2';
+const RERANK_MODEL = 'gpt-4o-mini';
 class ProductionEmbeddingClient extends EmbeddingClient {
  private readonly logger = new Logger(ProductionEmbeddingClient.name);

@@ -0,0 +1,305 @@
+import {
+  CopilotProviderSideError,
+  metrics,
+  UserFriendlyError,
+} from '../../../base';
+import {
+  llmDispatchStream,
+  llmRerankDispatch,
+  type NativeLlmBackendConfig,
+  type NativeLlmRequest,
+  type NativeLlmRerankRequest,
+  type NativeLlmRerankResponse,
+} from '../../../native';
+import type { NodeTextMiddleware } from '../config';
+import type { CopilotTool, CopilotToolSet } from '../tools';
+import {
+  buildNativeRequest,
+  buildNativeRerankRequest,
+  NativeProviderAdapter,
+} from './native';
+import { CopilotProvider } from './provider';
+import type {
+  CopilotChatOptions,
+  CopilotChatTools,
+  CopilotProviderModel,
+  CopilotRerankRequest,
+  ModelConditions,
+  PromptMessage,
+  StreamObject,
+} from './types';
+import { CopilotProviderType, ModelInputType, ModelOutputType } from './types';
+
+export type CloudflareWorkersAIConfig = {
+  apiToken: string;
+  accountId?: string;
+  baseURL?: string;
+};
+
+function rerankOnlyModel(
+  id: string,
+  name: string,
+  defaultForOutputType = false
+): CopilotProviderModel {
+  return {
+    name,
+    id,
+    capabilities: [
+      {
+        input: [ModelInputType.Text],
+        output: [ModelOutputType.Rerank],
+        ...(defaultForOutputType ? { defaultForOutputType } : {}),
+      },
+    ],
+  };
+}
+
+function chatAndRerankModel(
+  id: string,
+  name: string,
+  defaultForRerank = false
+): CopilotProviderModel {
+  return {
+    name,
+    id,
+    capabilities: [
+      {
+        input: [ModelInputType.Text],
+        output: [
+          ModelOutputType.Text,
+          ModelOutputType.Object,
+          ModelOutputType.Rerank,
+        ],
+        ...(defaultForRerank ? { defaultForOutputType: true } : {}),
+      },
+    ],
+  };
+}
+
+export class CloudflareWorkersAIProvider extends CopilotProvider<CloudflareWorkersAIConfig> {
+  override readonly type = CopilotProviderType.CloudflareWorkersAi;
+
+  override readonly models = [
+    rerankOnlyModel('@cf/baai/bge-reranker-base', 'BGE Reranker Base', true),
+    chatAndRerankModel('@cf/moonshotai/kimi-k2.5', 'Kimi K2.5'),
+    chatAndRerankModel(
+      '@cf/ibm-granite/granite-4.0-h-micro',
+      'Granite 4.0 H Micro'
+    ),
+    chatAndRerankModel(
+      '@cf/aisingapore/gemma-sea-lion-v4-27b-it',
+      'Gemma Sea Lion V4 27B IT'
+    ),
+    chatAndRerankModel(
+      '@cf/nvidia/nemotron-3-120b-a12b',
+      'Nemotron 3 120B A12B'
+    ),
+    chatAndRerankModel('@cf/zai-org/glm-4.7-flash', 'GLM 4.7 Flash'),
+    chatAndRerankModel('@cf/qwen/qwen3-30b-a3b-fp8', 'Qwen3 30B A3B FP8'),
+  ];
+
+  override configured(): boolean {
+    return (
+      !!this.config.apiToken &&
+      (!!this.config.accountId || !!this.config.baseURL)
+    );
+  }
+
+  override async refreshOnlineModels() {}
+
+  override getProviderSpecificTools(
+    toolName: CopilotChatTools,
+    _model: string
+  ): [string, CopilotTool?] | undefined {
+    if (toolName === 'docEdit') {
+      return ['doc_edit', undefined];
+    }
+    return;
+  }
+
+  private handleError(e: any) {
+    if (e instanceof UserFriendlyError) {
+      return e;
+    }
+    return new CopilotProviderSideError({
+      provider: this.type,
+      kind: 'unexpected_response',
+      message: e?.message || 'Unexpected cloudflare workers ai response',
+    });
+  }
+
+  private createNativeConfig(): NativeLlmBackendConfig {
+    return {
+      base_url: this.resolveBaseUrl(),
+      auth_token: this.config.apiToken,
+      request_layer: 'cloudflare_workers_ai',
+    };
+  }
+
+  private createNativeDispatch(
+    backendConfig: NativeLlmBackendConfig,
+    tools: CopilotToolSet,
+    nodeTextMiddleware?: NodeTextMiddleware[]
+  ) {
+    return new NativeProviderAdapter(
+      (request: NativeLlmRequest, signal?: AbortSignal) =>
+        llmDispatchStream('openai_chat', backendConfig, request, signal),
+      tools,
+      this.MAX_STEPS,
+      { nodeTextMiddleware }
+    );
+  }
+
+  private createNativeRerankDispatch(backendConfig: NativeLlmBackendConfig) {
+    return (
+      request: NativeLlmRerankRequest
+    ): Promise<NativeLlmRerankResponse> =>
+      llmRerankDispatch('openai_chat', backendConfig, request);
+  }
+
+  private resolveBaseUrl() {
+    if (this.config.baseURL) {
+      return this.config.baseURL.replace(/\/v1\/?$/, '').replace(/\/$/, '');
+    }
+    const accountId = this.config.accountId ?? '';
+    return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai`;
+  }
+
+  override async text(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotChatOptions = {}
+  ): Promise<string> {
+    const normalizedCond = await this.checkParams({
+      messages,
+      cond: { ...cond, outputType: ModelOutputType.Text },
+      options,
+    });
+    const model = this.selectModel(normalizedCond);
+
+    try {
+      metrics.ai.counter('chat_text_calls').add(1, this.metricLabels(model.id));
+      const tools = await this.getTools(options, model.id);
+      const middleware = this.getActiveProviderMiddleware();
+      const { request } = await buildNativeRequest({
+        model: model.id,
+        messages,
+        options,
+        tools,
+        middleware,
+      });
+      return await this.createNativeDispatch(
+        this.createNativeConfig(),
+        tools,
+        middleware.node?.text
+      ).text(request, options.signal, messages);
+    } catch (e: any) {
+      metrics.ai
+        .counter('chat_text_errors')
+        .add(1, this.metricLabels(model.id));
+      throw this.handleError(e);
+    }
+  }
+
+  override async *streamText(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotChatOptions = {}
+  ): AsyncIterable<string> {
+    const normalizedCond = await this.checkParams({
+      messages,
+      cond: { ...cond, outputType: ModelOutputType.Text },
+      options,
+    });
+    const model = this.selectModel(normalizedCond);
+
+    try {
+      metrics.ai
+        .counter('chat_text_stream_calls')
+        .add(1, this.metricLabels(model.id));
+      const tools = await this.getTools(options, model.id);
+      const middleware = this.getActiveProviderMiddleware();
+      const { request } = await buildNativeRequest({
+        model: model.id,
+        messages,
+        options,
+        tools,
+        middleware,
+      });
+      for await (const chunk of this.createNativeDispatch(
+        this.createNativeConfig(),
+        tools,
+        middleware.node?.text
+      ).streamText(request, options.signal, messages)) {
+        yield chunk;
+      }
+    } catch (e: any) {
+      metrics.ai
+        .counter('chat_text_stream_errors')
+        .add(1, this.metricLabels(model.id));
+      throw this.handleError(e);
+    }
+  }
+
+  override async *streamObject(
+    cond: ModelConditions,
+    messages: PromptMessage[],
+    options: CopilotChatOptions = {}
+  ): AsyncIterable<StreamObject> {
+    const normalizedCond = await this.checkParams({
+      messages,
+      cond: { ...cond, outputType: ModelOutputType.Object },
+      options,
+    });
+    const model = this.selectModel(normalizedCond);
+
+    try {
+      metrics.ai
+        .counter('chat_object_stream_calls')
+        .add(1, this.metricLabels(model.id));
+      const tools = await this.getTools(options, model.id);
+      const middleware = this.getActiveProviderMiddleware();
+      const { request } = await buildNativeRequest({
+        model: model.id,
+        messages,
+        options,
+        tools,
+        middleware,
+      });
+      for await (const chunk of this.createNativeDispatch(
+        this.createNativeConfig(),
+        tools,
+        middleware.node?.text
+      ).streamObject(request, options.signal, messages)) {
+        yield chunk;
+      }
+    } catch (e: any) {
+      metrics.ai
+        .counter('chat_object_stream_errors')
+        .add(1, this.metricLabels(model.id));
+      throw this.handleError(e);
+    }
+  }
+
+  override async rerank(
+    cond: ModelConditions,
+    request: CopilotRerankRequest,
+    options: CopilotChatOptions = {}
+  ): Promise<number[]> {
+    const normalizedCond = await this.checkParams({
+      messages: [],
+      cond: { ...cond, outputType: ModelOutputType.Rerank },
+      options,
+    });
+    const model = this.selectModel(normalizedCond);
+
+    try {
+      const response = await this.createNativeRerankDispatch(
+        this.createNativeConfig()
+      )(buildNativeRerankRequest(model.id, request));
+      return response.scores;
+    } catch (e: any) {
+      throw this.handleError(e);
+    }
+  }
+}
@@ -81,6 +81,27 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
        },
      ],
    },
+    {
+      name: 'Gemini 3.1 Flash Lite Preview',
+      id: 'gemini-3.1-flash-lite-preview',
+      capabilities: [
+        {
+          input: [
+            ModelInputType.Text,
+            ModelInputType.Image,
+            ModelInputType.Audio,
+            ModelInputType.File,
+          ],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
+          attachments: GEMINI_ATTACHMENT_CAPABILITY,
+          structuredAttachments: GEMINI_ATTACHMENT_CAPABILITY,
+        },
+      ],
+    },
    {
      name: 'Gemini Embedding',
      id: 'gemini-embedding-001',
@@ -77,6 +77,27 @@ export class GeminiVertexProvider extends GeminiProvider<GeminiVertexConfig> {
        },
      ],
    },
+    {
+      name: 'Gemini 3.1 Flash Lite Preview',
+      id: 'gemini-3.1-flash-lite-preview',
+      capabilities: [
+        {
+          input: [
+            ModelInputType.Text,
+            ModelInputType.Image,
+            ModelInputType.Audio,
+            ModelInputType.File,
+          ],
+          output: [
+            ModelOutputType.Text,
+            ModelOutputType.Object,
+            ModelOutputType.Structured,
+          ],
+          attachments: GEMINI_ATTACHMENT_CAPABILITY,
+          structuredAttachments: GEMINI_ATTACHMENT_CAPABILITY,
+        },
+      ],
+    },
    {
      name: 'Gemini Embedding',
      id: 'gemini-embedding-001',
@@ -2,6 +2,7 @@ import {
  AnthropicOfficialProvider,
  AnthropicVertexProvider,
 } from './anthropic';
+import { CloudflareWorkersAIProvider } from './cloudflare';
 import { FalProvider } from './fal';
 import { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini';
 import { MorphProvider } from './morph';
@@ -10,6 +11,7 @@ import { PerplexityProvider } from './perplexity';

 export const CopilotProviders = [
  OpenAIProvider,
+  CloudflareWorkersAIProvider,
  FalProvider,
  GeminiGenerativeProvider,
  GeminiVertexProvider,
@@ -23,6 +25,7 @@ export {
  AnthropicOfficialProvider,
  AnthropicVertexProvider,
 } from './anthropic';
+export { CloudflareWorkersAIProvider } from './cloudflare';
 export { CopilotProviderFactory } from './factory';
 export { FalProvider } from './fal';
 export { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini';
@@ -6,6 +6,7 @@ import type {
  NativeLlmCoreMessage,
  NativeLlmEmbeddingRequest,
  NativeLlmRequest,
+  NativeLlmRerankRequest,
  NativeLlmStreamEvent,
  NativeLlmStructuredRequest,
  NativeLlmStructuredResponse,
@@ -19,6 +20,7 @@ import {
 import { NativeDispatchFn, ToolCallLoop, ToolSchemaExtractor } from './loop';
 import type {
  CopilotChatOptions,
+  CopilotRerankRequest,
  CopilotStructuredOptions,
  ModelAttachmentCapability,
  PromptMessage,
@@ -194,6 +196,21 @@ export function parseNativeStructuredOutput(
  );
 }

+export function buildNativeRerankRequest(
+  model: string,
+  request: CopilotRerankRequest
+): NativeLlmRerankRequest {
+  return {
+    model,
+    query: request.query,
+    candidates: request.candidates.map(candidate => ({
+      ...(candidate.id ? { id: candidate.id } : {}),
+      text: candidate.text,
+    })),
+    ...(request.topK ? { top_n: request.topK } : {}),
+  };
+}
+
 async function toCoreContents(
  message: PromptMessage,
  withAttachment: boolean,
@@ -27,6 +27,7 @@ import { IMAGE_ATTACHMENT_CAPABILITY } from './attachments';
 import {
  buildNativeEmbeddingRequest,
  buildNativeRequest,
+  buildNativeRerankRequest,
  buildNativeStructuredRequest,
  NativeProviderAdapter,
  parseNativeStructuredOutput,
@@ -133,21 +134,6 @@ function normalizeImageResponseData(
    .filter((value): value is string => typeof value === 'string');
 }

-function buildOpenAIRerankRequest(
-  model: string,
-  request: CopilotRerankRequest
-): NativeLlmRerankRequest {
-  return {
-    model,
-    query: request.query,
-    candidates: request.candidates.map(candidate => ({
-      ...(candidate.id ? { id: candidate.id } : {}),
-      text: candidate.text,
-    })),
-    ...(request.topK ? { top_n: request.topK } : {}),
-  };
-}
-
 function createOpenAIMultimodalCapability(
  output: ModelCapability['output'],
  options: Pick<ModelCapability, 'defaultForOutputType'> = {}
@@ -194,6 +180,7 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
        createOpenAIMultimodalCapability([
          ModelOutputType.Text,
          ModelOutputType.Object,
+          ModelOutputType.Rerank,
        ]),
      ],
    },
@@ -444,7 +431,7 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
    return;
  }

-  private createNativeConfig(): NativeLlmBackendConfig {
+  protected createNativeConfig(): NativeLlmBackendConfig {
    const baseUrl = this.config.baseURL || 'https://api.openai.com/v1';
    return {
      base_url: baseUrl.replace(/\/v1\/?$/, ''),
@@ -452,7 +439,7 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
    };
  }

-  private getNativeProtocol() {
+  protected getNativeProtocol() {
    return this.config.oldApiStyle ? 'openai_chat' : 'openai_responses';
  }

@@ -709,7 +696,7 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {

    try {
      const backendConfig = this.createNativeConfig();
-      const nativeRequest = buildOpenAIRerankRequest(model.id, request);
+      const nativeRequest = buildNativeRerankRequest(model.id, request);
      const response =
        await this.createNativeRerankDispatch(backendConfig)(nativeRequest);
      return response.scores;
@@ -14,6 +14,15 @@ const DEFAULT_MIDDLEWARE_BY_TYPE: Record<
      text: ['citation_footnote', 'callout'],
    },
  },
+  [CopilotProviderType.CloudflareWorkersAi]: {
+    rust: {
+      request: ['normalize_messages'],
+      stream: ['stream_event_normalize', 'citation_indexing'],
+    },
+    node: {
+      text: ['citation_footnote', 'callout'],
+    },
+  },
  [CopilotProviderType.Anthropic]: {
    rust: {
      request: ['normalize_messages', 'tool_schema_rewrite'],
@@ -11,6 +11,7 @@ const PROVIDER_ID_PATTERN = /^[a-zA-Z0-9-_]+$/;

 const LEGACY_PROVIDER_ORDER: CopilotProviderType[] = [
  CopilotProviderType.OpenAI,
+  CopilotProviderType.CloudflareWorkersAi,
  CopilotProviderType.FAL,
  CopilotProviderType.Gemini,
  CopilotProviderType.GeminiVertex,
@@ -8,6 +8,7 @@ import { JSONSchema } from '../../../base';
 export enum CopilotProviderType {
  Anthropic = 'anthropic',
  AnthropicVertex = 'anthropicVertex',
+  CloudflareWorkersAi = 'cloudflareWorkersAi',
  FAL = 'fal',
  Gemini = 'gemini',
  GeminiVertex = 'geminiVertex',