diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index c625f9e778..362e29f0b6 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -1005,6 +1005,14 @@ "baseURL": "https://api.openai.com/v1" } }, + "providers.cloudflareWorkersAi": { + "type": "object", + "description": "The config for the Cloudflare Workers AI provider.\n@default {\"apiToken\":\"\",\"accountId\":\"\"}", + "default": { + "apiToken": "", + "accountId": "" + } + }, "providers.fal": { "type": "object", "description": "The config for the fal provider.\n@default {\"apiKey\":\"\"}", diff --git a/Cargo.lock b/Cargo.lock index 0396cbb6d9..9a9c7ee7a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3371,9 +3371,9 @@ dependencies = [ [[package]] name = "llm_adapter" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98485dda5180cc89b993a001688bed93307be6bd8fedcde445b69bbca4f554d" +checksum = "cd95a9dd20745f3d80d47460e6cf6131921bef928c38fcd961b10b574d749305" dependencies = [ "base64", "serde", diff --git a/Cargo.toml b/Cargo.toml index ee89683f57..391b99b989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ resolver = "3" libc = "0.2" libwebp-sys = "0.14.2" little_exif = "0.6.23" - llm_adapter = { version = "0.1.3", default-features = false } + llm_adapter = { version = "0.1.4", default-features = false } log = "0.4" loom = { version = "0.7", features = ["checkpoint"] } lru = "0.16" diff --git a/packages/backend/server/src/__tests__/copilot/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot/copilot-provider.spec.ts index a27c51796e..e0052d46f2 100644 --- a/packages/backend/server/src/__tests__/copilot/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot/copilot-provider.spec.ts @@ -964,11 +964,11 @@ test( 'The stock market is experiencing significant fluctuations.', ]; - const provider = (await factory.getProviderByModel('gpt-5.2'))!; + const provider = (await factory.getProviderByModel('gpt-4o-mini'))!; t.assert(provider, 'should have provider for rerank'); const scores = await provider.rerank( - { modelId: 'gpt-5.2' }, + { modelId: 'gpt-4o-mini' }, { query, candidates: embeddings.map((text, index) => ({ @@ -990,8 +990,8 @@ test( t.log('Rerank scores:', scores); t.is( scores.filter(s => s > 0.5).length, - 5, - 'should have 5 related chunks' + 4, + 'should have 4 related chunks' ); }); } diff --git a/packages/backend/server/src/__tests__/copilot/provider-native.spec.ts b/packages/backend/server/src/__tests__/copilot/provider-native.spec.ts index 58e04d4f63..dc4f0f467f 100644 --- a/packages/backend/server/src/__tests__/copilot/provider-native.spec.ts +++ b/packages/backend/server/src/__tests__/copilot/provider-native.spec.ts @@ -3,6 +3,7 @@ import test from 'ava'; import type { NativeLlmRerankRequest } from '../../native'; import { ProviderMiddlewareConfig } from '../../plugins/copilot/config'; +import { CloudflareWorkersAIProvider } from '../../plugins/copilot/providers/cloudflare'; import { normalizeOpenAIOptionsForModel, OpenAIProvider, @@ -53,7 +54,7 @@ class TestOpenAIProvider extends CopilotProvider<{ apiKey: string }> { class NativeRerankProtocolProvider extends OpenAIProvider { override readonly models = [ { - id: 'gpt-5.2', + id: 'gpt-4o-mini', capabilities: [ { input: [ModelInputType.Text], @@ -77,6 +78,19 @@ class NativeRerankProtocolProvider extends OpenAIProvider { } } +class NativeCloudflareRerankProtocolProvider extends CloudflareWorkersAIProvider { + override get config() { + return { + apiToken: 'test-key', + accountId: 'account-1', + }; + } + + override configured() { + return true; + } +} + function createProvider(profileMiddleware?: ProviderMiddlewareConfig) { const provider = new TestOpenAIProvider(); (provider as any).AFFiNEConfig = { @@ -170,14 +184,14 @@ test('OpenAI rerank should always use chat-completions native protocol', async t ) => { capturedProtocol = protocol; capturedRequest = JSON.parse(requestJson) as NativeLlmRerankRequest; - return JSON.stringify({ model: 'gpt-5.2', scores: [0.9, 0.1] }); + return JSON.stringify({ model: 'gpt-4o-mini', scores: [0.9, 0.1] }); }; t.teardown(() => { (serverNativeModule as any).llmRerankDispatch = original; }); const scores = await provider.rerank( - { modelId: 'gpt-5.2' }, + { modelId: 'gpt-4o-mini' }, { query: 'programming', candidates: [ @@ -190,7 +204,7 @@ test('OpenAI rerank should always use chat-completions native protocol', async t t.deepEqual(scores, [0.9, 0.1]); t.is(capturedProtocol, 'openai_chat'); t.deepEqual(capturedRequest, { - model: 'gpt-5.2', + model: 'gpt-4o-mini', query: 'programming', candidates: [ { id: 'react', text: 'React is a UI library.' }, @@ -198,3 +212,63 @@ test('OpenAI rerank should always use chat-completions native protocol', async t ], }); }); + +test('Cloudflare rerank should keep native protocol details behind provider', async t => { + const provider = new NativeCloudflareRerankProtocolProvider(); + let capturedProtocol: string | undefined; + let capturedRequest: NativeLlmRerankRequest | undefined; + let capturedBackendConfig: Record | undefined; + + const original = (serverNativeModule as any).llmRerankDispatch; + (serverNativeModule as any).llmRerankDispatch = ( + protocol: string, + backendConfigJson: string, + requestJson: string + ) => { + capturedProtocol = protocol; + capturedBackendConfig = JSON.parse(backendConfigJson) as Record< + string, + unknown + >; + capturedRequest = JSON.parse(requestJson) as NativeLlmRerankRequest; + return JSON.stringify({ + model: '@cf/qwen/qwen3-30b-a3b-fp8', + scores: [0.9, 0.1], + }); + }; + t.teardown(() => { + (serverNativeModule as any).llmRerankDispatch = original; + }); + + for (const modelId of [ + '@cf/qwen/qwen3-30b-a3b-fp8', + '@cf/baai/bge-reranker-base', + ]) { + const scores = await provider.rerank( + { modelId }, + { + query: 'programming', + candidates: [ + { id: 'react', text: 'React is a UI library.' }, + { id: 'weather', text: 'The weather is sunny today.' }, + ], + } + ); + + t.deepEqual(scores, [0.9, 0.1]); + t.is(capturedProtocol, 'openai_chat'); + t.deepEqual(capturedBackendConfig, { + base_url: 'https://api.cloudflare.com/client/v4/accounts/account-1/ai', + auth_token: 'test-key', + request_layer: 'cloudflare_workers_ai', + }); + t.deepEqual(capturedRequest, { + model: modelId, + query: 'programming', + candidates: [ + { id: 'react', text: 'React is a UI library.' }, + { id: 'weather', text: 'The weather is sunny today.' }, + ], + }); + } +}); diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index 1eb481df31..0565c191bf 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -103,6 +103,7 @@ export type NativeLlmBackendConfig = { request_layer?: | 'anthropic' | 'chat_completions' + | 'cloudflare_workers_ai' | 'responses' | 'vertex' | 'vertex_anthropic' diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts index aa9e0feffe..ee04b9a9ae 100644 --- a/packages/backend/server/src/plugins/copilot/config.ts +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -10,6 +10,7 @@ import { AnthropicOfficialConfig, AnthropicVertexConfig, } from './providers/anthropic'; +import { CloudflareWorkersAIConfig } from './providers/cloudflare'; import type { FalConfig } from './providers/fal'; import { GeminiGenerativeConfig, GeminiVertexConfig } from './providers/gemini'; import { MorphConfig } from './providers/morph'; @@ -23,6 +24,7 @@ import { export type CopilotProviderConfigMap = { [CopilotProviderType.OpenAI]: OpenAIConfig; + [CopilotProviderType.CloudflareWorkersAi]: CloudflareWorkersAIConfig; [CopilotProviderType.FAL]: FalConfig; [CopilotProviderType.Gemini]: GeminiGenerativeConfig; [CopilotProviderType.GeminiVertex]: GeminiVertexConfig; @@ -117,6 +119,12 @@ const FalConfigShape = z.object({ apiKey: z.string(), }); +const CloudflareWorkersAIConfigShape = z.object({ + apiToken: z.string(), + accountId: z.string().optional(), + baseURL: z.string().optional(), +}); + const GeminiGenerativeConfigShape = z.object({ apiKey: z.string(), baseURL: z.string().optional(), @@ -153,6 +161,10 @@ const CopilotProviderProfileShape = z.discriminatedUnion('type', [ type: z.literal(CopilotProviderType.FAL), config: FalConfigShape, }), + CopilotProviderProfileBaseShape.extend({ + type: z.literal(CopilotProviderType.CloudflareWorkersAi), + config: CloudflareWorkersAIConfigShape, + }), CopilotProviderProfileBaseShape.extend({ type: z.literal(CopilotProviderType.Gemini), config: GeminiGenerativeConfigShape, @@ -205,6 +217,7 @@ declare global { profiles: ConfigItem; defaults: ConfigItem; openai: ConfigItem; + cloudflareWorkersAi: ConfigItem; fal: ConfigItem; gemini: ConfigItem; geminiVertex: ConfigItem; @@ -257,6 +270,13 @@ defineModuleConfig('copilot', { }, link: 'https://github.com/openai/openai-node', }, + 'providers.cloudflareWorkersAi': { + desc: 'The config for the Cloudflare Workers AI provider.', + default: { + apiToken: '', + accountId: '', + }, + }, 'providers.fal': { desc: 'The config for the fal provider.', default: { diff --git a/packages/backend/server/src/plugins/copilot/embedding/client.ts b/packages/backend/server/src/plugins/copilot/embedding/client.ts index 34e5ccfa5f..d68ca535bf 100644 --- a/packages/backend/server/src/plugins/copilot/embedding/client.ts +++ b/packages/backend/server/src/plugins/copilot/embedding/client.ts @@ -19,7 +19,7 @@ import { import { EmbeddingClient, type ReRankResult } from './types'; const EMBEDDING_MODEL = 'gemini-embedding-001'; -const RERANK_MODEL = 'gpt-5.2'; +const RERANK_MODEL = 'gpt-4o-mini'; class ProductionEmbeddingClient extends EmbeddingClient { private readonly logger = new Logger(ProductionEmbeddingClient.name); diff --git a/packages/backend/server/src/plugins/copilot/providers/cloudflare.ts b/packages/backend/server/src/plugins/copilot/providers/cloudflare.ts new file mode 100644 index 0000000000..382c23e3cd --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/providers/cloudflare.ts @@ -0,0 +1,305 @@ +import { + CopilotProviderSideError, + metrics, + UserFriendlyError, +} from '../../../base'; +import { + llmDispatchStream, + llmRerankDispatch, + type NativeLlmBackendConfig, + type NativeLlmRequest, + type NativeLlmRerankRequest, + type NativeLlmRerankResponse, +} from '../../../native'; +import type { NodeTextMiddleware } from '../config'; +import type { CopilotTool, CopilotToolSet } from '../tools'; +import { + buildNativeRequest, + buildNativeRerankRequest, + NativeProviderAdapter, +} from './native'; +import { CopilotProvider } from './provider'; +import type { + CopilotChatOptions, + CopilotChatTools, + CopilotProviderModel, + CopilotRerankRequest, + ModelConditions, + PromptMessage, + StreamObject, +} from './types'; +import { CopilotProviderType, ModelInputType, ModelOutputType } from './types'; + +export type CloudflareWorkersAIConfig = { + apiToken: string; + accountId?: string; + baseURL?: string; +}; + +function rerankOnlyModel( + id: string, + name: string, + defaultForOutputType = false +): CopilotProviderModel { + return { + name, + id, + capabilities: [ + { + input: [ModelInputType.Text], + output: [ModelOutputType.Rerank], + ...(defaultForOutputType ? { defaultForOutputType } : {}), + }, + ], + }; +} + +function chatAndRerankModel( + id: string, + name: string, + defaultForRerank = false +): CopilotProviderModel { + return { + name, + id, + capabilities: [ + { + input: [ModelInputType.Text], + output: [ + ModelOutputType.Text, + ModelOutputType.Object, + ModelOutputType.Rerank, + ], + ...(defaultForRerank ? { defaultForOutputType: true } : {}), + }, + ], + }; +} + +export class CloudflareWorkersAIProvider extends CopilotProvider { + override readonly type = CopilotProviderType.CloudflareWorkersAi; + + override readonly models = [ + rerankOnlyModel('@cf/baai/bge-reranker-base', 'BGE Reranker Base', true), + chatAndRerankModel('@cf/moonshotai/kimi-k2.5', 'Kimi K2.5'), + chatAndRerankModel( + '@cf/ibm-granite/granite-4.0-h-micro', + 'Granite 4.0 H Micro' + ), + chatAndRerankModel( + '@cf/aisingapore/gemma-sea-lion-v4-27b-it', + 'Gemma Sea Lion V4 27B IT' + ), + chatAndRerankModel( + '@cf/nvidia/nemotron-3-120b-a12b', + 'Nemotron 3 120B A12B' + ), + chatAndRerankModel('@cf/zai-org/glm-4.7-flash', 'GLM 4.7 Flash'), + chatAndRerankModel('@cf/qwen/qwen3-30b-a3b-fp8', 'Qwen3 30B A3B FP8'), + ]; + + override configured(): boolean { + return ( + !!this.config.apiToken && + (!!this.config.accountId || !!this.config.baseURL) + ); + } + + override async refreshOnlineModels() {} + + override getProviderSpecificTools( + toolName: CopilotChatTools, + _model: string + ): [string, CopilotTool?] | undefined { + if (toolName === 'docEdit') { + return ['doc_edit', undefined]; + } + return; + } + + private handleError(e: any) { + if (e instanceof UserFriendlyError) { + return e; + } + return new CopilotProviderSideError({ + provider: this.type, + kind: 'unexpected_response', + message: e?.message || 'Unexpected cloudflare workers ai response', + }); + } + + private createNativeConfig(): NativeLlmBackendConfig { + return { + base_url: this.resolveBaseUrl(), + auth_token: this.config.apiToken, + request_layer: 'cloudflare_workers_ai', + }; + } + + private createNativeDispatch( + backendConfig: NativeLlmBackendConfig, + tools: CopilotToolSet, + nodeTextMiddleware?: NodeTextMiddleware[] + ) { + return new NativeProviderAdapter( + (request: NativeLlmRequest, signal?: AbortSignal) => + llmDispatchStream('openai_chat', backendConfig, request, signal), + tools, + this.MAX_STEPS, + { nodeTextMiddleware } + ); + } + + private createNativeRerankDispatch(backendConfig: NativeLlmBackendConfig) { + return ( + request: NativeLlmRerankRequest + ): Promise => + llmRerankDispatch('openai_chat', backendConfig, request); + } + + private resolveBaseUrl() { + if (this.config.baseURL) { + return this.config.baseURL.replace(/\/v1\/?$/, '').replace(/\/$/, ''); + } + const accountId = this.config.accountId ?? ''; + return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai`; + } + + override async text( + cond: ModelConditions, + messages: PromptMessage[], + options: CopilotChatOptions = {} + ): Promise { + const normalizedCond = await this.checkParams({ + messages, + cond: { ...cond, outputType: ModelOutputType.Text }, + options, + }); + const model = this.selectModel(normalizedCond); + + try { + metrics.ai.counter('chat_text_calls').add(1, this.metricLabels(model.id)); + const tools = await this.getTools(options, model.id); + const middleware = this.getActiveProviderMiddleware(); + const { request } = await buildNativeRequest({ + model: model.id, + messages, + options, + tools, + middleware, + }); + return await this.createNativeDispatch( + this.createNativeConfig(), + tools, + middleware.node?.text + ).text(request, options.signal, messages); + } catch (e: any) { + metrics.ai + .counter('chat_text_errors') + .add(1, this.metricLabels(model.id)); + throw this.handleError(e); + } + } + + override async *streamText( + cond: ModelConditions, + messages: PromptMessage[], + options: CopilotChatOptions = {} + ): AsyncIterable { + const normalizedCond = await this.checkParams({ + messages, + cond: { ...cond, outputType: ModelOutputType.Text }, + options, + }); + const model = this.selectModel(normalizedCond); + + try { + metrics.ai + .counter('chat_text_stream_calls') + .add(1, this.metricLabels(model.id)); + const tools = await this.getTools(options, model.id); + const middleware = this.getActiveProviderMiddleware(); + const { request } = await buildNativeRequest({ + model: model.id, + messages, + options, + tools, + middleware, + }); + for await (const chunk of this.createNativeDispatch( + this.createNativeConfig(), + tools, + middleware.node?.text + ).streamText(request, options.signal, messages)) { + yield chunk; + } + } catch (e: any) { + metrics.ai + .counter('chat_text_stream_errors') + .add(1, this.metricLabels(model.id)); + throw this.handleError(e); + } + } + + override async *streamObject( + cond: ModelConditions, + messages: PromptMessage[], + options: CopilotChatOptions = {} + ): AsyncIterable { + const normalizedCond = await this.checkParams({ + messages, + cond: { ...cond, outputType: ModelOutputType.Object }, + options, + }); + const model = this.selectModel(normalizedCond); + + try { + metrics.ai + .counter('chat_object_stream_calls') + .add(1, this.metricLabels(model.id)); + const tools = await this.getTools(options, model.id); + const middleware = this.getActiveProviderMiddleware(); + const { request } = await buildNativeRequest({ + model: model.id, + messages, + options, + tools, + middleware, + }); + for await (const chunk of this.createNativeDispatch( + this.createNativeConfig(), + tools, + middleware.node?.text + ).streamObject(request, options.signal, messages)) { + yield chunk; + } + } catch (e: any) { + metrics.ai + .counter('chat_object_stream_errors') + .add(1, this.metricLabels(model.id)); + throw this.handleError(e); + } + } + + override async rerank( + cond: ModelConditions, + request: CopilotRerankRequest, + options: CopilotChatOptions = {} + ): Promise { + const normalizedCond = await this.checkParams({ + messages: [], + cond: { ...cond, outputType: ModelOutputType.Rerank }, + options, + }); + const model = this.selectModel(normalizedCond); + + try { + const response = await this.createNativeRerankDispatch( + this.createNativeConfig() + )(buildNativeRerankRequest(model.id, request)); + return response.scores; + } catch (e: any) { + throw this.handleError(e); + } + } +} diff --git a/packages/backend/server/src/plugins/copilot/providers/gemini/generative.ts b/packages/backend/server/src/plugins/copilot/providers/gemini/generative.ts index d29c5a98ba..2f649e055a 100644 --- a/packages/backend/server/src/plugins/copilot/providers/gemini/generative.ts +++ b/packages/backend/server/src/plugins/copilot/providers/gemini/generative.ts @@ -81,6 +81,27 @@ export class GeminiGenerativeProvider extends GeminiProvider { }, ], }, + { + name: 'Gemini 3.1 Flash Lite Preview', + id: 'gemini-3.1-flash-lite-preview', + capabilities: [ + { + input: [ + ModelInputType.Text, + ModelInputType.Image, + ModelInputType.Audio, + ModelInputType.File, + ], + output: [ + ModelOutputType.Text, + ModelOutputType.Object, + ModelOutputType.Structured, + ], + attachments: GEMINI_ATTACHMENT_CAPABILITY, + structuredAttachments: GEMINI_ATTACHMENT_CAPABILITY, + }, + ], + }, { name: 'Gemini Embedding', id: 'gemini-embedding-001', diff --git a/packages/backend/server/src/plugins/copilot/providers/index.ts b/packages/backend/server/src/plugins/copilot/providers/index.ts index 97cb547a4d..0d9c019cfa 100644 --- a/packages/backend/server/src/plugins/copilot/providers/index.ts +++ b/packages/backend/server/src/plugins/copilot/providers/index.ts @@ -2,6 +2,7 @@ import { AnthropicOfficialProvider, AnthropicVertexProvider, } from './anthropic'; +import { CloudflareWorkersAIProvider } from './cloudflare'; import { FalProvider } from './fal'; import { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini'; import { MorphProvider } from './morph'; @@ -10,6 +11,7 @@ import { PerplexityProvider } from './perplexity'; export const CopilotProviders = [ OpenAIProvider, + CloudflareWorkersAIProvider, FalProvider, GeminiGenerativeProvider, GeminiVertexProvider, @@ -23,6 +25,7 @@ export { AnthropicOfficialProvider, AnthropicVertexProvider, } from './anthropic'; +export { CloudflareWorkersAIProvider } from './cloudflare'; export { CopilotProviderFactory } from './factory'; export { FalProvider } from './fal'; export { GeminiGenerativeProvider, GeminiVertexProvider } from './gemini'; diff --git a/packages/backend/server/src/plugins/copilot/providers/native.ts b/packages/backend/server/src/plugins/copilot/providers/native.ts index 355e68d735..0298a4b5a0 100644 --- a/packages/backend/server/src/plugins/copilot/providers/native.ts +++ b/packages/backend/server/src/plugins/copilot/providers/native.ts @@ -6,6 +6,7 @@ import type { NativeLlmCoreMessage, NativeLlmEmbeddingRequest, NativeLlmRequest, + NativeLlmRerankRequest, NativeLlmStreamEvent, NativeLlmStructuredRequest, NativeLlmStructuredResponse, @@ -19,6 +20,7 @@ import { import { NativeDispatchFn, ToolCallLoop, ToolSchemaExtractor } from './loop'; import type { CopilotChatOptions, + CopilotRerankRequest, CopilotStructuredOptions, ModelAttachmentCapability, PromptMessage, @@ -194,6 +196,21 @@ export function parseNativeStructuredOutput( ); } +export function buildNativeRerankRequest( + model: string, + request: CopilotRerankRequest +): NativeLlmRerankRequest { + return { + model, + query: request.query, + candidates: request.candidates.map(candidate => ({ + ...(candidate.id ? { id: candidate.id } : {}), + text: candidate.text, + })), + ...(request.topK ? { top_n: request.topK } : {}), + }; +} + async function toCoreContents( message: PromptMessage, withAttachment: boolean, diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 1c77626cfa..0f6c7ae0e1 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -27,6 +27,7 @@ import { IMAGE_ATTACHMENT_CAPABILITY } from './attachments'; import { buildNativeEmbeddingRequest, buildNativeRequest, + buildNativeRerankRequest, buildNativeStructuredRequest, NativeProviderAdapter, parseNativeStructuredOutput, @@ -133,21 +134,6 @@ function normalizeImageResponseData( .filter((value): value is string => typeof value === 'string'); } -function buildOpenAIRerankRequest( - model: string, - request: CopilotRerankRequest -): NativeLlmRerankRequest { - return { - model, - query: request.query, - candidates: request.candidates.map(candidate => ({ - ...(candidate.id ? { id: candidate.id } : {}), - text: candidate.text, - })), - ...(request.topK ? { top_n: request.topK } : {}), - }; -} - function createOpenAIMultimodalCapability( output: ModelCapability['output'], options: Pick = {} @@ -194,6 +180,7 @@ export class OpenAIProvider extends CopilotProvider { createOpenAIMultimodalCapability([ ModelOutputType.Text, ModelOutputType.Object, + ModelOutputType.Rerank, ]), ], }, @@ -444,7 +431,7 @@ export class OpenAIProvider extends CopilotProvider { return; } - private createNativeConfig(): NativeLlmBackendConfig { + protected createNativeConfig(): NativeLlmBackendConfig { const baseUrl = this.config.baseURL || 'https://api.openai.com/v1'; return { base_url: baseUrl.replace(/\/v1\/?$/, ''), @@ -452,7 +439,7 @@ export class OpenAIProvider extends CopilotProvider { }; } - private getNativeProtocol() { + protected getNativeProtocol() { return this.config.oldApiStyle ? 'openai_chat' : 'openai_responses'; } @@ -709,7 +696,7 @@ export class OpenAIProvider extends CopilotProvider { try { const backendConfig = this.createNativeConfig(); - const nativeRequest = buildOpenAIRerankRequest(model.id, request); + const nativeRequest = buildNativeRerankRequest(model.id, request); const response = await this.createNativeRerankDispatch(backendConfig)(nativeRequest); return response.scores; diff --git a/packages/backend/server/src/plugins/copilot/providers/provider-middleware.ts b/packages/backend/server/src/plugins/copilot/providers/provider-middleware.ts index d43d30c5bc..db5a570714 100644 --- a/packages/backend/server/src/plugins/copilot/providers/provider-middleware.ts +++ b/packages/backend/server/src/plugins/copilot/providers/provider-middleware.ts @@ -14,6 +14,15 @@ const DEFAULT_MIDDLEWARE_BY_TYPE: Record< text: ['citation_footnote', 'callout'], }, }, + [CopilotProviderType.CloudflareWorkersAi]: { + rust: { + request: ['normalize_messages'], + stream: ['stream_event_normalize', 'citation_indexing'], + }, + node: { + text: ['citation_footnote', 'callout'], + }, + }, [CopilotProviderType.Anthropic]: { rust: { request: ['normalize_messages', 'tool_schema_rewrite'], diff --git a/packages/backend/server/src/plugins/copilot/providers/provider-registry.ts b/packages/backend/server/src/plugins/copilot/providers/provider-registry.ts index 73d1bbdc8b..2189d19e05 100644 --- a/packages/backend/server/src/plugins/copilot/providers/provider-registry.ts +++ b/packages/backend/server/src/plugins/copilot/providers/provider-registry.ts @@ -11,6 +11,7 @@ const PROVIDER_ID_PATTERN = /^[a-zA-Z0-9-_]+$/; const LEGACY_PROVIDER_ORDER: CopilotProviderType[] = [ CopilotProviderType.OpenAI, + CopilotProviderType.CloudflareWorkersAi, CopilotProviderType.FAL, CopilotProviderType.Gemini, CopilotProviderType.GeminiVertex, diff --git a/packages/backend/server/src/plugins/copilot/providers/types.ts b/packages/backend/server/src/plugins/copilot/providers/types.ts index 9da70b97a4..f669c7ac77 100644 --- a/packages/backend/server/src/plugins/copilot/providers/types.ts +++ b/packages/backend/server/src/plugins/copilot/providers/types.ts @@ -8,6 +8,7 @@ import { JSONSchema } from '../../../base'; export enum CopilotProviderType { Anthropic = 'anthropic', AnthropicVertex = 'anthropicVertex', + CloudflareWorkersAi = 'cloudflareWorkersAi', FAL = 'fal', Gemini = 'gemini', GeminiVertex = 'geminiVertex', diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index 32c0e45b89..b3af7a15f5 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -326,6 +326,10 @@ "desc": "The config for the openai provider.", "link": "https://github.com/openai/openai-node" }, + "providers.cloudflareWorkersAi": { + "type": "Object", + "desc": "The config for the Cloudflare Workers AI provider." + }, "providers.fal": { "type": "Object", "desc": "The config for the fal provider."