mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-19 11:06:25 +08:00
feat(server): refactor provider interface (#11665)
fix AI-4 fix AI-18 better provider/model choose to allow fallback to similar models (e.g., self-hosted) when the provider is not fully configured split functions of different output types
This commit is contained in:
darkskygit
@@ -13,13 +13,17 @@ import {
|
||||
} from '../../../base';
|
||||
import { createExaCrawlTool, createExaSearchTool } from '../tools';
|
||||
import { CopilotProvider } from './provider';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
import {
|
||||
ChatMessageRole,
|
||||
CopilotCapability,
|
||||
CopilotChatOptions,
|
||||
CopilotProviderType,
|
||||
CopilotTextToTextProvider,
|
||||
PromptMessage,
|
||||
ModelInputType,
|
||||
ModelOutputType,
|
||||
} from './types';
|
||||
import { chatToGPTMessage } from './utils';
|
||||
|
||||
@@ -28,15 +32,28 @@ export type AnthropicConfig = {
|
||||
baseUrl?: string;
|
||||
};
|
||||
|
||||
export class AnthropicProvider
|
||||
extends CopilotProvider<AnthropicConfig>
|
||||
implements CopilotTextToTextProvider
|
||||
{
|
||||
export class AnthropicProvider extends CopilotProvider<AnthropicConfig> {
|
||||
override readonly type = CopilotProviderType.Anthropic;
|
||||
override readonly capabilities = [CopilotCapability.TextToText];
|
||||
override readonly models = [
|
||||
'claude-3-7-sonnet-20250219',
|
||||
'claude-3-5-sonnet-20241022',
|
||||
{
|
||||
id: 'claude-3-7-sonnet-20250219',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'claude-3-5-sonnet-20241022',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
private readonly MAX_STEPS = 20;
|
||||
@@ -58,14 +75,16 @@ export class AnthropicProvider
|
||||
}
|
||||
|
||||
protected async checkParams({
|
||||
cond,
|
||||
messages,
|
||||
model,
|
||||
}: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
model: string;
|
||||
embeddings?: string[];
|
||||
options?: CopilotChatOptions;
|
||||
}) {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
if (!(await this.match(cond))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${cond.modelId}`);
|
||||
}
|
||||
if (Array.isArray(messages) && messages.length > 0) {
|
||||
if (
|
||||
@@ -115,27 +134,28 @@ export class AnthropicProvider
|
||||
}
|
||||
}
|
||||
|
||||
// ====== text to text ======
|
||||
async generateText(
|
||||
async text(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'claude-3-7-sonnet-20250219',
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<string> {
|
||||
await this.checkParams({ messages, model });
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Text };
|
||||
await this.checkParams({ cond: fullCond, messages });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
const [system, msgs] = await chatToGPTMessage(messages);
|
||||
|
||||
const modelInstance = this.#instance(model);
|
||||
const modelInstance = this.#instance(model.id);
|
||||
const { text, reasoning } = await generateText({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
abortSignal: options.signal,
|
||||
providerOptions: {
|
||||
anthropic: this.getAnthropicOptions(options, model),
|
||||
anthropic: this.getAnthropicOptions(options, model.id),
|
||||
},
|
||||
tools: this.getTools(),
|
||||
maxSteps: this.MAX_STEPS,
|
||||
@@ -146,28 +166,30 @@ export class AnthropicProvider
|
||||
|
||||
return reasoning ? `${reasoning}\n${text}` : text;
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateTextStream(
|
||||
async *streamText(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'claude-3-7-sonnet-20250219',
|
||||
options: CopilotChatOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
await this.checkParams({ messages, model });
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Text };
|
||||
await this.checkParams({ cond: fullCond, messages });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
|
||||
const [system, msgs] = await chatToGPTMessage(messages);
|
||||
const { fullStream } = streamText({
|
||||
model: this.#instance(model),
|
||||
model: this.#instance(model.id),
|
||||
system,
|
||||
messages: msgs,
|
||||
abortSignal: options.signal,
|
||||
providerOptions: {
|
||||
anthropic: this.getAnthropicOptions(options, model),
|
||||
anthropic: this.getAnthropicOptions(options, model.id),
|
||||
},
|
||||
tools: this.getTools(),
|
||||
maxSteps: this.MAX_STEPS,
|
||||
@@ -244,7 +266,7 @@ export class AnthropicProvider
|
||||
lastType = chunk.type;
|
||||
}
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,25 +1,8 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { ServerFeature, ServerService } from '../../../core';
|
||||
import type { AnthropicProvider } from './anthropic';
|
||||
import type { FalProvider } from './fal';
|
||||
import type { GeminiProvider } from './gemini';
|
||||
import type { OpenAIProvider } from './openai';
|
||||
import type { PerplexityProvider } from './perplexity';
|
||||
import type { CopilotProvider } from './provider';
|
||||
import {
|
||||
CapabilityToCopilotProvider,
|
||||
CopilotCapability,
|
||||
CopilotProviderType,
|
||||
} from './types';
|
||||
|
||||
type TypedProvider = {
|
||||
[CopilotProviderType.Anthropic]: AnthropicProvider;
|
||||
[CopilotProviderType.Gemini]: GeminiProvider;
|
||||
[CopilotProviderType.OpenAI]: OpenAIProvider;
|
||||
[CopilotProviderType.Perplexity]: PerplexityProvider;
|
||||
[CopilotProviderType.FAL]: FalProvider;
|
||||
};
|
||||
import { CopilotProviderType, ModelFullConditions } from './types';
|
||||
|
||||
@Injectable()
|
||||
export class CopilotProviderFactory {
|
||||
@@ -29,68 +12,54 @@ export class CopilotProviderFactory {
|
||||
|
||||
readonly #providers = new Map<CopilotProviderType, CopilotProvider>();
|
||||
|
||||
getProvider<P extends CopilotProviderType>(provider: P): TypedProvider[P] {
|
||||
return this.#providers.get(provider) as TypedProvider[P];
|
||||
}
|
||||
|
||||
async getProviderByCapability<C extends CopilotCapability>(
|
||||
capability: C,
|
||||
async getProvider(
|
||||
cond: ModelFullConditions,
|
||||
filter: {
|
||||
model?: string;
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<CapabilityToCopilotProvider[C] | null> {
|
||||
): Promise<CopilotProvider | null> {
|
||||
this.logger.debug(
|
||||
`Resolving copilot provider for capability: ${capability}`
|
||||
`Resolving copilot provider for output type: ${cond.outputType}`
|
||||
);
|
||||
let candidate: CopilotProvider | null = null;
|
||||
for (const [type, provider] of this.#providers.entries()) {
|
||||
// we firstly match by capability
|
||||
if (provider.capabilities.includes(capability)) {
|
||||
// use the first match if no filter provided
|
||||
if (!filter.model && !filter.prefer) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
break;
|
||||
}
|
||||
if (filter.prefer && filter.prefer !== type) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
(!filter.model || (await provider.isModelAvailable(filter.model))) &&
|
||||
(!filter.prefer || filter.prefer === type)
|
||||
) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
break;
|
||||
}
|
||||
const isMatched = await provider.match(cond);
|
||||
|
||||
if (isMatched) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return candidate as CapabilityToCopilotProvider[C] | null;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
async getProviderByModel<C extends CopilotCapability>(
|
||||
model: string,
|
||||
async getProviderByModel(
|
||||
modelId: string,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<CapabilityToCopilotProvider[C] | null> {
|
||||
this.logger.debug(`Resolving copilot provider for model: ${model}`);
|
||||
): Promise<CopilotProvider | null> {
|
||||
this.logger.debug(`Resolving copilot provider for model: ${modelId}`);
|
||||
|
||||
let candidate: CopilotProvider | null = null;
|
||||
for (const [type, provider] of this.#providers.entries()) {
|
||||
// we firstly match by model
|
||||
if (await provider.isModelAvailable(model)) {
|
||||
if (filter.prefer && filter.prefer !== type) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await provider.match({ modelId })) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
|
||||
// then we match by prefer filter
|
||||
if (!filter.prefer || filter.prefer === type) {
|
||||
candidate = provider;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidate as CapabilityToCopilotProvider[C] | null;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
register(provider: CopilotProvider) {
|
||||
|
||||
@@ -12,15 +12,13 @@ import {
|
||||
UserFriendlyError,
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
CopilotCapability,
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotImageOptions,
|
||||
CopilotImageToImageProvider,
|
||||
CopilotProviderType,
|
||||
CopilotTextToImageProvider,
|
||||
ModelConditions,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
import { CopilotProviderType, ModelInputType, ModelOutputType } from './types';
|
||||
|
||||
export type FalConfig = {
|
||||
apiKey: string;
|
||||
@@ -72,30 +70,66 @@ type FalPrompt = {
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class FalProvider
|
||||
extends CopilotProvider<FalConfig>
|
||||
implements CopilotTextToImageProvider, CopilotImageToImageProvider
|
||||
{
|
||||
export class FalProvider extends CopilotProvider<FalConfig> {
|
||||
override type = CopilotProviderType.FAL;
|
||||
override readonly capabilities = [
|
||||
CopilotCapability.TextToImage,
|
||||
CopilotCapability.ImageToImage,
|
||||
CopilotCapability.ImageToText,
|
||||
];
|
||||
|
||||
override readonly models = [
|
||||
// text to image
|
||||
'fast-turbo-diffusion',
|
||||
// image to image
|
||||
'lcm-sd15-i2i',
|
||||
'clarity-upscaler',
|
||||
'face-to-sticker',
|
||||
'imageutils/rembg',
|
||||
'fast-sdxl/image-to-image',
|
||||
'workflowutils/teed',
|
||||
'lora/image-to-image',
|
||||
// image to text
|
||||
'llava-next',
|
||||
// image to image models
|
||||
{
|
||||
id: 'lcm-sd15-i2i',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'clarity-upscaler',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'face-to-sticker',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'imageutils/rembg',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'workflowutils/teed',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'lora/image-to-image',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
override configured(): boolean {
|
||||
@@ -204,20 +238,20 @@ export class FalProvider
|
||||
});
|
||||
}
|
||||
|
||||
async generateText(
|
||||
async text(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'llava-next',
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<string> {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
}
|
||||
const model = this.selectModel(cond);
|
||||
|
||||
// by default, image prompt assumes there is only one message
|
||||
const prompt = this.extractPrompt(messages.pop());
|
||||
try {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model });
|
||||
const response = await fetch(`https://fal.run/fal-ai/${model}`, {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
// by default, image prompt assumes there is only one message
|
||||
const prompt = this.extractPrompt(messages[messages.length - 1]);
|
||||
|
||||
const response = await fetch(`https://fal.run/fal-ai/${model.id}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `key ${this.config.apiKey}`,
|
||||
@@ -237,112 +271,101 @@ export class FalProvider
|
||||
}
|
||||
return data.output;
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateTextStream(
|
||||
async *streamText(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'llava-next',
|
||||
options: CopilotChatOptions = {}
|
||||
options: CopilotChatOptions | CopilotImageOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
|
||||
const result = await this.generateText(messages, model, options);
|
||||
const model = this.selectModel(cond);
|
||||
|
||||
for (const content of result) {
|
||||
if (content) {
|
||||
yield content;
|
||||
if (options.signal?.aborted) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
|
||||
const result = await this.text(cond, messages, options);
|
||||
|
||||
yield result;
|
||||
} catch (e) {
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model: model.id });
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private async buildResponse(
|
||||
override async *streamImages(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = this.models[0],
|
||||
options: CopilotImageOptions = {}
|
||||
) {
|
||||
// by default, image prompt assumes there is only one message
|
||||
const prompt = this.extractPrompt(messages.pop(), options);
|
||||
if (model.startsWith('workflows/')) {
|
||||
const stream = await falStream(model, { input: prompt });
|
||||
return this.parseSchema(FalStreamOutputSchema, await stream.done())
|
||||
.output;
|
||||
} else {
|
||||
const response = await fetch(`https://fal.run/fal-ai/${model}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `key ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
...prompt,
|
||||
sync_mode: true,
|
||||
seed: options.seed || 42,
|
||||
enable_safety_checks: false,
|
||||
}),
|
||||
signal: options.signal,
|
||||
});
|
||||
return this.parseSchema(FalResponseSchema, await response.json());
|
||||
}
|
||||
}
|
||||
|
||||
// ====== image to image ======
|
||||
async generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string = this.models[0],
|
||||
options: CopilotImageOptions = {}
|
||||
): Promise<Array<string>> {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
}
|
||||
): AsyncIterable<string> {
|
||||
const model = this.selectModel({
|
||||
...cond,
|
||||
outputType: ModelOutputType.Image,
|
||||
});
|
||||
|
||||
try {
|
||||
metrics.ai.counter('generate_images_calls').add(1, { model });
|
||||
metrics.ai
|
||||
.counter('generate_images_stream_calls')
|
||||
.add(1, { model: model.id });
|
||||
|
||||
const data = await this.buildResponse(messages, model, options);
|
||||
// by default, image prompt assumes there is only one message
|
||||
const prompt = this.extractPrompt(
|
||||
messages[messages.length - 1],
|
||||
options as CopilotImageOptions
|
||||
);
|
||||
|
||||
let data: FalResponse;
|
||||
if (model.id.startsWith('workflows/')) {
|
||||
const stream = await falStream(model.id, { input: prompt });
|
||||
data = this.parseSchema(
|
||||
FalStreamOutputSchema,
|
||||
await stream.done()
|
||||
).output;
|
||||
} else {
|
||||
const response = await fetch(`https://fal.run/fal-ai/${model.id}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `key ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
...prompt,
|
||||
sync_mode: true,
|
||||
seed: (options as CopilotImageOptions)?.seed || 42,
|
||||
enable_safety_checks: false,
|
||||
}),
|
||||
signal: options.signal,
|
||||
});
|
||||
data = this.parseSchema(FalResponseSchema, await response.json());
|
||||
}
|
||||
|
||||
if (!data.images?.length && !data.image?.url) {
|
||||
throw this.extractFalError(data, 'Failed to generate images');
|
||||
}
|
||||
|
||||
if (data.image?.url) {
|
||||
return [data.image.url];
|
||||
yield data.image.url;
|
||||
return;
|
||||
}
|
||||
|
||||
return (
|
||||
const imageUrls =
|
||||
data.images
|
||||
?.filter((image): image is NonNullable<FalImage> => !!image)
|
||||
.map(image => image.url) || []
|
||||
);
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('generate_images_errors').add(1, { model });
|
||||
.map(image => image.url) || [];
|
||||
|
||||
for (const url of imageUrls) {
|
||||
yield url;
|
||||
if (options.signal?.aborted) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return;
|
||||
} catch (e) {
|
||||
metrics.ai
|
||||
.counter('generate_images_stream_errors')
|
||||
.add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model: string = this.models[0],
|
||||
options: CopilotImageOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
try {
|
||||
metrics.ai.counter('generate_images_stream_calls').add(1, { model });
|
||||
const ret = await this.generateImages(messages, model, options);
|
||||
for (const url of ret) {
|
||||
yield url;
|
||||
}
|
||||
} catch (e) {
|
||||
metrics.ai.counter('generate_images_stream_errors').add(1, { model });
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,13 +17,18 @@ import {
|
||||
UserFriendlyError,
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotImageOptions,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
import {
|
||||
ChatMessageRole,
|
||||
CopilotCapability,
|
||||
CopilotChatOptions,
|
||||
CopilotProviderType,
|
||||
CopilotTextToTextProvider,
|
||||
PromptMessage,
|
||||
ModelInputType,
|
||||
ModelOutputType,
|
||||
} from './types';
|
||||
import { chatToGPTMessage } from './utils';
|
||||
|
||||
@@ -34,18 +39,49 @@ export type GeminiConfig = {
|
||||
baseUrl?: string;
|
||||
};
|
||||
|
||||
export class GeminiProvider
|
||||
extends CopilotProvider<GeminiConfig>
|
||||
implements CopilotTextToTextProvider
|
||||
{
|
||||
export class GeminiProvider extends CopilotProvider<GeminiConfig> {
|
||||
override readonly type = CopilotProviderType.Gemini;
|
||||
override readonly capabilities = [CopilotCapability.TextToText];
|
||||
override readonly models = [
|
||||
// text to text
|
||||
'gemini-2.0-flash-001',
|
||||
'gemini-2.5-pro-preview-03-25',
|
||||
// embeddings
|
||||
'text-embedding-004',
|
||||
|
||||
readonly models = [
|
||||
{
|
||||
name: 'Gemini 2.0 Flash',
|
||||
id: 'gemini-2.0-flash-001',
|
||||
capabilities: [
|
||||
{
|
||||
input: [
|
||||
ModelInputType.Text,
|
||||
ModelInputType.Image,
|
||||
ModelInputType.Audio,
|
||||
],
|
||||
output: [ModelOutputType.Text, ModelOutputType.Structured],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Gemini 2.5 Pro',
|
||||
id: 'gemini-2.5-pro-preview-03-25',
|
||||
capabilities: [
|
||||
{
|
||||
input: [
|
||||
ModelInputType.Text,
|
||||
ModelInputType.Image,
|
||||
ModelInputType.Audio,
|
||||
],
|
||||
output: [ModelOutputType.Text, ModelOutputType.Structured],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Text Embedding 004',
|
||||
id: 'text-embedding-004',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Embedding],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
#instance!: GoogleGenerativeAIProvider;
|
||||
@@ -63,16 +99,17 @@ export class GeminiProvider
|
||||
}
|
||||
|
||||
protected async checkParams({
|
||||
cond,
|
||||
messages,
|
||||
embeddings,
|
||||
model,
|
||||
}: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
model: string;
|
||||
options?: CopilotChatOptions;
|
||||
}) {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
if (!(await this.match(cond))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${cond.modelId}`);
|
||||
}
|
||||
if (Array.isArray(messages) && messages.length > 0) {
|
||||
if (
|
||||
@@ -127,72 +164,100 @@ export class GeminiProvider
|
||||
}
|
||||
}
|
||||
|
||||
// ====== text to text ======
|
||||
async generateText(
|
||||
override async text(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'gemini-2.0-flash-001',
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<string> {
|
||||
await this.checkParams({ messages, model });
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Text };
|
||||
await this.checkParams({ cond: fullCond, messages, options });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
const [system, msgs, schema] = await chatToGPTMessage(messages);
|
||||
const [system, msgs] = await chatToGPTMessage(messages);
|
||||
|
||||
const modelInstance = this.#instance(model, {
|
||||
structuredOutputs: Boolean(options.jsonMode),
|
||||
const modelInstance = this.#instance(model.id);
|
||||
const { text } = await generateText({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
abortSignal: options.signal,
|
||||
});
|
||||
const { text } = schema
|
||||
? await generateObject({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
schema,
|
||||
abortSignal: options.signal,
|
||||
experimental_repairText: async ({ text, error }) => {
|
||||
if (error instanceof JSONParseError) {
|
||||
// strange fixed response, temporarily replace it
|
||||
const ret = text.replaceAll(/^ny\n/g, ' ').trim();
|
||||
if (ret.startsWith('```') || ret.endsWith('```')) {
|
||||
return ret
|
||||
.replace(/```[\w\s]+\n/g, '')
|
||||
.replace(/\n```/g, '')
|
||||
.trim();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
}).then(r => ({ text: JSON.stringify(r.object) }))
|
||||
: await generateText({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
abortSignal: options.signal,
|
||||
});
|
||||
|
||||
if (!text) throw new Error('Failed to generate text');
|
||||
return text.trim();
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateTextStream(
|
||||
override async structure(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'gemini-2.0-flash-001',
|
||||
options: CopilotChatOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
await this.checkParams({ messages, model });
|
||||
): Promise<string> {
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Structured };
|
||||
await this.checkParams({ cond: fullCond, messages });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
const [system, msgs, schema] = await chatToGPTMessage(messages);
|
||||
if (!schema) {
|
||||
throw new CopilotPromptInvalid('Schema is required');
|
||||
}
|
||||
|
||||
const modelInstance = this.#instance(model.id, {
|
||||
structuredOutputs: true,
|
||||
});
|
||||
const { object } = await generateObject({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
schema,
|
||||
abortSignal: options.signal,
|
||||
experimental_repairText: async ({ text, error }) => {
|
||||
if (error instanceof JSONParseError) {
|
||||
// strange fixed response, temporarily replace it
|
||||
const ret = text.replaceAll(/^ny\n/g, ' ').trim();
|
||||
if (ret.startsWith('```') || ret.endsWith('```')) {
|
||||
return ret
|
||||
.replace(/```[\w\s]+\n/g, '')
|
||||
.replace(/\n```/g, '')
|
||||
.trim();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
return JSON.stringify(object);
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
override async *streamText(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotChatOptions | CopilotImageOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Text };
|
||||
await this.checkParams({ cond: fullCond, messages });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
|
||||
const [system, msgs] = await chatToGPTMessage(messages);
|
||||
|
||||
const { textStream } = streamText({
|
||||
model: this.#instance(model),
|
||||
model: this.#instance(model.id),
|
||||
system,
|
||||
messages: msgs,
|
||||
abortSignal: options.signal,
|
||||
@@ -208,7 +273,7 @@ export class GeminiProvider
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,4 +18,5 @@ export { FalProvider } from './fal';
|
||||
export { GeminiProvider } from './gemini';
|
||||
export { OpenAIProvider } from './openai';
|
||||
export { PerplexityProvider } from './perplexity';
|
||||
export type { CopilotProvider } from './provider';
|
||||
export * from './types';
|
||||
|
||||
@@ -21,19 +21,21 @@ import {
|
||||
} from '../../../base';
|
||||
import { createExaCrawlTool, createExaSearchTool } from '../tools';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
ChatMessageRole,
|
||||
CopilotCapability,
|
||||
import type {
|
||||
CopilotChatOptions,
|
||||
CopilotEmbeddingOptions,
|
||||
CopilotImageOptions,
|
||||
CopilotImageToTextProvider,
|
||||
CopilotProviderType,
|
||||
CopilotTextToEmbeddingProvider,
|
||||
CopilotTextToImageProvider,
|
||||
CopilotTextToTextProvider,
|
||||
CopilotStructuredOptions,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
import {
|
||||
ChatMessageRole,
|
||||
CopilotProviderType,
|
||||
ModelInputType,
|
||||
ModelOutputType,
|
||||
} from './types';
|
||||
import { chatToGPTMessage, CitationParser } from './utils';
|
||||
|
||||
export const DEFAULT_DIMENSIONS = 256;
|
||||
@@ -49,44 +51,144 @@ type OpenAITools = {
|
||||
web_crawl_exa: ReturnType<typeof createExaCrawlTool>;
|
||||
};
|
||||
|
||||
export class OpenAIProvider
|
||||
extends CopilotProvider<OpenAIConfig>
|
||||
implements
|
||||
CopilotTextToTextProvider,
|
||||
CopilotTextToEmbeddingProvider,
|
||||
CopilotTextToImageProvider,
|
||||
CopilotImageToTextProvider
|
||||
{
|
||||
export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
|
||||
readonly type = CopilotProviderType.OpenAI;
|
||||
readonly capabilities = [
|
||||
CopilotCapability.TextToText,
|
||||
CopilotCapability.TextToEmbedding,
|
||||
CopilotCapability.TextToImage,
|
||||
CopilotCapability.ImageToText,
|
||||
];
|
||||
|
||||
readonly models = [
|
||||
// text to text
|
||||
'gpt-4o',
|
||||
'gpt-4o-2024-08-06',
|
||||
'gpt-4o-mini',
|
||||
'gpt-4o-mini-2024-07-18',
|
||||
'gpt-4.1',
|
||||
'gpt-4.1-2025-04-14',
|
||||
'gpt-4.1-mini',
|
||||
'o1',
|
||||
'o3',
|
||||
'o4-mini',
|
||||
// embeddings
|
||||
'text-embedding-3-large',
|
||||
'text-embedding-3-small',
|
||||
'text-embedding-ada-002',
|
||||
// moderation
|
||||
'text-moderation-latest',
|
||||
'text-moderation-stable',
|
||||
// text to image
|
||||
'dall-e-3',
|
||||
'gpt-image-1',
|
||||
// Text to Text models
|
||||
{
|
||||
id: 'gpt-4o',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
// FIXME(@darkskygit): deprecated
|
||||
{
|
||||
id: 'gpt-4o-2024-08-06',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'gpt-4o-mini',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
// FIXME(@darkskygit): deprecated
|
||||
{
|
||||
id: 'gpt-4o-mini-2024-07-18',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'gpt-4.1',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'gpt-4.1-2025-04-14',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'gpt-4.1-mini',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'o1',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'o3',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'o4-mini',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
// Embedding models
|
||||
{
|
||||
id: 'text-embedding-3-large',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Embedding],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'text-embedding-3-small',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Embedding],
|
||||
},
|
||||
],
|
||||
},
|
||||
// Image generation models
|
||||
{
|
||||
id: 'dall-e-3',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Image],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'gpt-image-1',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text, ModelInputType.Image],
|
||||
output: [ModelOutputType.Image],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
private readonly MAX_STEPS = 20;
|
||||
@@ -108,18 +210,17 @@ export class OpenAIProvider
|
||||
}
|
||||
|
||||
protected async checkParams({
|
||||
cond,
|
||||
messages,
|
||||
embeddings,
|
||||
model,
|
||||
options = {},
|
||||
}: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
model: string;
|
||||
options: CopilotChatOptions;
|
||||
options?: CopilotChatOptions;
|
||||
}) {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
if (!(await this.match(cond))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${cond.modelId}`);
|
||||
}
|
||||
if (Array.isArray(messages) && messages.length > 0) {
|
||||
if (
|
||||
@@ -147,14 +248,6 @@ export class OpenAIProvider
|
||||
) {
|
||||
throw new CopilotPromptInvalid('Invalid message role');
|
||||
}
|
||||
// json mode need 'json' keyword in content
|
||||
// ref: https://platform.openai.com/docs/api-reference/chat/create#chat-create-response_format
|
||||
if (
|
||||
options.jsonMode &&
|
||||
!messages.some(m => m.content.toLowerCase().includes('json'))
|
||||
) {
|
||||
throw new CopilotPromptInvalid('Prompt not support json mode');
|
||||
}
|
||||
} else if (
|
||||
Array.isArray(embeddings) &&
|
||||
embeddings.some(e => typeof e !== 'string' || !e || !e.trim())
|
||||
@@ -215,82 +308,77 @@ export class OpenAIProvider
|
||||
return tools;
|
||||
}
|
||||
|
||||
// ====== text to text ======
|
||||
async generateText(
|
||||
async text(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'gpt-4.1-mini',
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<string> {
|
||||
await this.checkParams({ messages, model, options });
|
||||
const fullCond = {
|
||||
...cond,
|
||||
outputType: ModelOutputType.Text,
|
||||
};
|
||||
await this.checkParams({ messages, cond: fullCond, options });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
const [system, msgs, schema] = await chatToGPTMessage(messages);
|
||||
const [system, msgs] = await chatToGPTMessage(messages);
|
||||
|
||||
const modelInstance = this.#instance(model, {
|
||||
structuredOutputs: Boolean(options.jsonMode),
|
||||
user: options.user,
|
||||
});
|
||||
const modelInstance = this.#instance.responses(model.id);
|
||||
|
||||
const commonParams = {
|
||||
const { text } = await generateText({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
temperature: options.temperature || 0,
|
||||
maxTokens: options.maxTokens || 4096,
|
||||
providerOptions: {
|
||||
openai: this.getOpenAIOptions(options, model.id),
|
||||
},
|
||||
tools: this.getTools(options, model.id),
|
||||
maxSteps: this.MAX_STEPS,
|
||||
abortSignal: options.signal,
|
||||
};
|
||||
|
||||
const { text } = schema
|
||||
? await generateObject({
|
||||
...commonParams,
|
||||
schema,
|
||||
}).then(r => ({ text: JSON.stringify(r.object) }))
|
||||
: await generateText({
|
||||
...commonParams,
|
||||
providerOptions: {
|
||||
openai: this.getOpenAIOptions(options, model),
|
||||
},
|
||||
tools: this.getTools(options, model),
|
||||
maxSteps: this.MAX_STEPS,
|
||||
});
|
||||
});
|
||||
|
||||
return text.trim();
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model });
|
||||
throw this.handleError(e, model, options);
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e, model.id, options);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateTextStream(
|
||||
async *streamText(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'gpt-4.1-mini',
|
||||
options: CopilotChatOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
await this.checkParams({ messages, model, options });
|
||||
const fullCond = {
|
||||
...cond,
|
||||
outputType: ModelOutputType.Text,
|
||||
};
|
||||
await this.checkParams({ messages, cond: fullCond });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
|
||||
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
|
||||
const [system, msgs] = await chatToGPTMessage(messages);
|
||||
|
||||
const modelInstance = this.#instance.responses(model);
|
||||
const modelInstance = this.#instance.responses(model.id);
|
||||
|
||||
const tools = this.getTools(options, model);
|
||||
const { fullStream } = streamText({
|
||||
model: modelInstance,
|
||||
system,
|
||||
messages: msgs,
|
||||
providerOptions: {
|
||||
openai: this.getOpenAIOptions(options, model),
|
||||
},
|
||||
tools: tools as OpenAITools,
|
||||
maxSteps: this.MAX_STEPS,
|
||||
frequencyPenalty: options.frequencyPenalty || 0,
|
||||
presencePenalty: options.presencePenalty || 0,
|
||||
temperature: options.temperature || 0,
|
||||
maxTokens: options.maxTokens || 4096,
|
||||
providerOptions: {
|
||||
openai: this.getOpenAIOptions(options, model.id),
|
||||
},
|
||||
tools: this.getTools(options, model.id) as OpenAITools,
|
||||
maxSteps: this.MAX_STEPS,
|
||||
abortSignal: options.signal,
|
||||
});
|
||||
|
||||
@@ -368,54 +456,68 @@ export class OpenAIProvider
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model });
|
||||
throw this.handleError(e, model, options);
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e, model.id, options);
|
||||
}
|
||||
}
|
||||
|
||||
// ====== text to embedding ======
|
||||
|
||||
async generateEmbedding(
|
||||
messages: string | string[],
|
||||
model: string,
|
||||
options: CopilotEmbeddingOptions = { dimensions: DEFAULT_DIMENSIONS }
|
||||
): Promise<number[][]> {
|
||||
messages = Array.isArray(messages) ? messages : [messages];
|
||||
await this.checkParams({ embeddings: messages, model, options });
|
||||
override async structure(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options: CopilotStructuredOptions = {}
|
||||
): Promise<string> {
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Structured };
|
||||
await this.checkParams({ messages, cond: fullCond, options });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('generate_embedding_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
const modelInstance = this.#instance.embedding(model, {
|
||||
dimensions: options.dimensions || DEFAULT_DIMENSIONS,
|
||||
user: options.user,
|
||||
});
|
||||
const [system, msgs, schema] = await chatToGPTMessage(messages);
|
||||
if (!schema) {
|
||||
throw new CopilotPromptInvalid('Schema is required');
|
||||
}
|
||||
|
||||
const { embeddings } = await embedMany({
|
||||
const modelInstance = this.#instance.responses(model.id);
|
||||
|
||||
const { object } = await generateObject({
|
||||
model: modelInstance,
|
||||
values: messages,
|
||||
system,
|
||||
messages: msgs,
|
||||
temperature: ('temperature' in options && options.temperature) || 0,
|
||||
maxTokens: ('maxTokens' in options && options.maxTokens) || 4096,
|
||||
schema,
|
||||
providerOptions: {
|
||||
openai: options.user ? { user: options.user } : {},
|
||||
},
|
||||
abortSignal: options.signal,
|
||||
});
|
||||
|
||||
return embeddings.filter(v => v && Array.isArray(v));
|
||||
return JSON.stringify(object);
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('generate_embedding_errors').add(1, { model });
|
||||
throw this.handleError(e, model, options);
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e, model.id, options);
|
||||
}
|
||||
}
|
||||
|
||||
// ====== text to image ======
|
||||
async generateImages(
|
||||
override async *streamImages(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'dall-e-3',
|
||||
options: CopilotImageOptions = {}
|
||||
): Promise<Array<string>> {
|
||||
const { content: prompt } = messages.pop() || {};
|
||||
) {
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Image };
|
||||
await this.checkParams({ messages, cond: fullCond });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
metrics.ai
|
||||
.counter('generate_images_stream_calls')
|
||||
.add(1, { model: model.id });
|
||||
|
||||
const { content: prompt } = [...messages].pop() || {};
|
||||
if (!prompt) throw new CopilotPromptInvalid('Prompt is required');
|
||||
|
||||
try {
|
||||
metrics.ai.counter('generate_images_calls').add(1, { model });
|
||||
|
||||
const modelInstance = this.#instance.image(model);
|
||||
const modelInstance = this.#instance.image(model.id);
|
||||
|
||||
const result = await generateImage({
|
||||
model: modelInstance,
|
||||
@@ -427,29 +529,54 @@ export class OpenAIProvider
|
||||
},
|
||||
});
|
||||
|
||||
return result.images.map(
|
||||
const imageUrls = result.images.map(
|
||||
image => `data:image/png;base64,${image.base64}`
|
||||
);
|
||||
|
||||
for (const imageUrl of imageUrls) {
|
||||
yield imageUrl;
|
||||
if (options.signal?.aborted) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return;
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('generate_images_errors').add(1, { model });
|
||||
throw this.handleError(e, model, options);
|
||||
metrics.ai.counter('generate_images_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e, model.id, options);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model: string = 'dall-e-3',
|
||||
options: CopilotImageOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
override async embedding(
|
||||
cond: ModelConditions,
|
||||
messages: string | string[],
|
||||
options: CopilotEmbeddingOptions = { dimensions: DEFAULT_DIMENSIONS }
|
||||
): Promise<number[][]> {
|
||||
messages = Array.isArray(messages) ? messages : [messages];
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Embedding };
|
||||
await this.checkParams({ embeddings: messages, cond: fullCond, options });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('generate_images_stream_calls').add(1, { model });
|
||||
const ret = await this.generateImages(messages, model, options);
|
||||
for (const url of ret) {
|
||||
yield url;
|
||||
}
|
||||
} catch (e) {
|
||||
metrics.ai.counter('generate_images_stream_errors').add(1, { model });
|
||||
throw e;
|
||||
metrics.ai
|
||||
.counter('generate_embedding_calls')
|
||||
.add(1, { model: model.id });
|
||||
|
||||
const modelInstance = this.#instance.embedding(model.id, {
|
||||
dimensions: options.dimensions || DEFAULT_DIMENSIONS,
|
||||
user: options.user,
|
||||
});
|
||||
|
||||
const { embeddings } = await embedMany({
|
||||
model: modelInstance,
|
||||
values: messages,
|
||||
});
|
||||
|
||||
return embeddings.filter(v => v && Array.isArray(v));
|
||||
} catch (e: any) {
|
||||
metrics.ai
|
||||
.counter('generate_embedding_errors')
|
||||
.add(1, { model: model.id });
|
||||
throw this.handleError(e, model.id, options);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,10 +12,12 @@ import {
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
CopilotCapability,
|
||||
CopilotChatOptions,
|
||||
CopilotProviderType,
|
||||
CopilotTextToTextProvider,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
ModelInputType,
|
||||
ModelOutputType,
|
||||
PromptMessage,
|
||||
} from './types';
|
||||
import { chatToGPTMessage, CitationParser } from './utils';
|
||||
@@ -46,17 +48,51 @@ const PerplexityErrorSchema = z.union([
|
||||
|
||||
type PerplexityError = z.infer<typeof PerplexityErrorSchema>;
|
||||
|
||||
export class PerplexityProvider
|
||||
extends CopilotProvider<PerplexityConfig>
|
||||
implements CopilotTextToTextProvider
|
||||
{
|
||||
export class PerplexityProvider extends CopilotProvider<PerplexityConfig> {
|
||||
readonly type = CopilotProviderType.Perplexity;
|
||||
readonly capabilities = [CopilotCapability.TextToText];
|
||||
|
||||
readonly models = [
|
||||
'sonar',
|
||||
'sonar-pro',
|
||||
'sonar-reasoning',
|
||||
'sonar-reasoning-pro',
|
||||
{
|
||||
name: 'Sonar',
|
||||
id: 'sonar',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Text],
|
||||
defaultForOutputType: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Sonar Pro',
|
||||
id: 'sonar-pro',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Sonar Reasoning',
|
||||
id: 'sonar-reasoning',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Sonar Reasoning Pro',
|
||||
id: 'sonar-reasoning-pro',
|
||||
capabilities: [
|
||||
{
|
||||
input: [ModelInputType.Text],
|
||||
output: [ModelOutputType.Text],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
#instance!: VercelPerplexityProvider;
|
||||
@@ -73,18 +109,21 @@ export class PerplexityProvider
|
||||
});
|
||||
}
|
||||
|
||||
async generateText(
|
||||
async text(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'sonar',
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<string> {
|
||||
await this.checkParams({ messages, model, options });
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Text };
|
||||
await this.checkParams({ cond: fullCond, messages });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
|
||||
|
||||
const [system, msgs] = await chatToGPTMessage(messages, false);
|
||||
|
||||
const modelInstance = this.#instance(model);
|
||||
const modelInstance = this.#instance(model.id);
|
||||
|
||||
const { text, sources } = await generateText({
|
||||
model: modelInstance,
|
||||
@@ -105,23 +144,26 @@ export class PerplexityProvider
|
||||
result += parser.end();
|
||||
return result;
|
||||
} catch (e: any) {
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_errors').add(1, { model: model.id });
|
||||
throw this.handleError(e);
|
||||
}
|
||||
}
|
||||
|
||||
async *generateTextStream(
|
||||
async *streamText(
|
||||
cond: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
model: string = 'sonar',
|
||||
options: CopilotChatOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
await this.checkParams({ messages, model, options });
|
||||
const fullCond = { ...cond, outputType: ModelOutputType.Text };
|
||||
await this.checkParams({ cond: fullCond, messages });
|
||||
const model = this.selectModel(fullCond);
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
|
||||
|
||||
const [system, msgs] = await chatToGPTMessage(messages, false);
|
||||
|
||||
const modelInstance = this.#instance(model);
|
||||
const modelInstance = this.#instance(model.id);
|
||||
|
||||
const stream = streamText({
|
||||
model: modelInstance,
|
||||
@@ -168,21 +210,21 @@ export class PerplexityProvider
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model });
|
||||
metrics.ai.counter('chat_text_stream_errors').add(1, { model: model.id });
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
protected async checkParams({
|
||||
model,
|
||||
cond,
|
||||
}: {
|
||||
cond: ModelFullConditions;
|
||||
messages?: PromptMessage[];
|
||||
embeddings?: string[];
|
||||
model: string;
|
||||
options: CopilotChatOptions;
|
||||
options?: CopilotChatOptions;
|
||||
}) {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
if (!(await this.match(cond))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${cond.modelId}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,15 +1,30 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { Config, OnEvent } from '../../../base';
|
||||
import {
|
||||
Config,
|
||||
CopilotPromptInvalid,
|
||||
CopilotProviderNotSupported,
|
||||
OnEvent,
|
||||
} from '../../../base';
|
||||
import { CopilotProviderFactory } from './factory';
|
||||
import { CopilotCapability, CopilotProviderType } from './types';
|
||||
import {
|
||||
type CopilotChatOptions,
|
||||
type CopilotEmbeddingOptions,
|
||||
type CopilotImageOptions,
|
||||
CopilotProviderModel,
|
||||
CopilotProviderType,
|
||||
CopilotStructuredOptions,
|
||||
ModelCapability,
|
||||
ModelConditions,
|
||||
ModelFullConditions,
|
||||
type PromptMessage,
|
||||
} from './types';
|
||||
|
||||
@Injectable()
|
||||
export abstract class CopilotProvider<C = any> {
|
||||
protected readonly logger = new Logger(this.constructor.name);
|
||||
abstract readonly type: CopilotProviderType;
|
||||
abstract readonly capabilities: CopilotCapability[];
|
||||
abstract readonly models: string[];
|
||||
abstract readonly models: CopilotProviderModel[];
|
||||
abstract configured(): boolean;
|
||||
|
||||
@Inject() protected readonly AFFiNEConfig!: Config;
|
||||
@@ -19,10 +34,6 @@ export abstract class CopilotProvider<C = any> {
|
||||
return this.AFFiNEConfig.copilot.providers[this.type] as C;
|
||||
}
|
||||
|
||||
isModelAvailable(model: string): Promise<boolean> | boolean {
|
||||
return this.models.includes(model);
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
this.setup();
|
||||
@@ -42,4 +53,88 @@ export abstract class CopilotProvider<C = any> {
|
||||
this.factory.unregister(this);
|
||||
}
|
||||
}
|
||||
|
||||
private findValidModel(
|
||||
cond: ModelFullConditions
|
||||
): CopilotProviderModel | undefined {
|
||||
const { modelId, outputType, inputTypes } = cond;
|
||||
const matcher = (cap: ModelCapability) =>
|
||||
(!outputType || cap.output.includes(outputType)) &&
|
||||
(!inputTypes || inputTypes.every(type => cap.input.includes(type)));
|
||||
|
||||
if (modelId) {
|
||||
return this.models.find(
|
||||
m => m.id === modelId && m.capabilities.some(matcher)
|
||||
);
|
||||
}
|
||||
if (!outputType) return undefined;
|
||||
|
||||
return this.models.find(m =>
|
||||
m.capabilities.some(c => matcher(c) && c.defaultForOutputType)
|
||||
);
|
||||
}
|
||||
|
||||
// make it async to allow dynamic check available models in some providers
|
||||
async match(cond: ModelFullConditions = {}): Promise<boolean> {
|
||||
return this.configured() && !!this.findValidModel(cond);
|
||||
}
|
||||
|
||||
protected selectModel(cond: ModelFullConditions): CopilotProviderModel {
|
||||
const model = this.findValidModel(cond);
|
||||
if (model) return model;
|
||||
|
||||
const { modelId, outputType, inputTypes } = cond;
|
||||
throw new CopilotPromptInvalid(
|
||||
modelId
|
||||
? `Model ${modelId} does not support ${outputType ?? '<any>'} output with ${inputTypes ?? '<any>'} input`
|
||||
: outputType
|
||||
? `No model supports ${outputType} output with ${inputTypes ?? '<any>'} input for provider ${this.type}`
|
||||
: 'Output type is required when modelId is not provided'
|
||||
);
|
||||
}
|
||||
|
||||
abstract text(
|
||||
model: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
|
||||
abstract streamText(
|
||||
model: ModelConditions,
|
||||
messages: PromptMessage[],
|
||||
options?: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
|
||||
structure(
|
||||
_cond: ModelConditions,
|
||||
_messages: PromptMessage[],
|
||||
_options: CopilotStructuredOptions
|
||||
): Promise<string> {
|
||||
throw new CopilotProviderNotSupported({
|
||||
provider: this.type,
|
||||
kind: 'structure',
|
||||
});
|
||||
}
|
||||
|
||||
streamImages(
|
||||
_model: ModelConditions,
|
||||
_messages: PromptMessage[],
|
||||
_options?: CopilotImageOptions
|
||||
): AsyncIterable<string> {
|
||||
throw new CopilotProviderNotSupported({
|
||||
provider: this.type,
|
||||
kind: 'image',
|
||||
});
|
||||
}
|
||||
|
||||
embedding(
|
||||
_model: ModelConditions,
|
||||
_text: string,
|
||||
_options?: CopilotEmbeddingOptions
|
||||
): Promise<number[][]> {
|
||||
throw new CopilotProviderNotSupported({
|
||||
provider: this.type,
|
||||
kind: 'embedding',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { type CopilotProvider } from './provider';
|
||||
|
||||
export enum CopilotProviderType {
|
||||
Anthropic = 'anthropic',
|
||||
FAL = 'fal',
|
||||
@@ -11,18 +9,16 @@ export enum CopilotProviderType {
|
||||
Perplexity = 'perplexity',
|
||||
}
|
||||
|
||||
export enum CopilotCapability {
|
||||
TextToText = 'text-to-text',
|
||||
TextToEmbedding = 'text-to-embedding',
|
||||
TextToImage = 'text-to-image',
|
||||
ImageToImage = 'image-to-image',
|
||||
ImageToText = 'image-to-text',
|
||||
}
|
||||
export const CopilotProviderSchema = z.object({
|
||||
type: z.nativeEnum(CopilotProviderType),
|
||||
});
|
||||
|
||||
export const PromptConfigStrictSchema = z.object({
|
||||
tools: z.enum(['webSearch']).array().nullable().optional(),
|
||||
// params requirements
|
||||
requireContent: z.boolean().nullable().optional(),
|
||||
requireAttachment: z.boolean().nullable().optional(),
|
||||
// openai
|
||||
jsonMode: z.boolean().nullable().optional(),
|
||||
frequencyPenalty: z.number().nullable().optional(),
|
||||
presencePenalty: z.number().nullable().optional(),
|
||||
temperature: z.number().nullable().optional(),
|
||||
@@ -87,13 +83,11 @@ export const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
|
||||
export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
|
||||
|
||||
export const CopilotEmbeddingOptionsSchema =
|
||||
CopilotProviderOptionsSchema.extend({
|
||||
dimensions: z.number(),
|
||||
}).optional();
|
||||
export const CopilotStructuredOptionsSchema =
|
||||
CopilotProviderOptionsSchema.merge(PromptConfigStrictSchema).optional();
|
||||
|
||||
export type CopilotEmbeddingOptions = z.infer<
|
||||
typeof CopilotEmbeddingOptionsSchema
|
||||
export type CopilotStructuredOptions = z.infer<
|
||||
typeof CopilotStructuredOptionsSchema
|
||||
>;
|
||||
|
||||
export const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
@@ -107,81 +101,44 @@ export const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
|
||||
export type CopilotImageOptions = z.infer<typeof CopilotImageOptionsSchema>;
|
||||
|
||||
export interface CopilotTextToTextProvider extends CopilotProvider {
|
||||
generateText(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
generateTextStream(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
export const CopilotEmbeddingOptionsSchema =
|
||||
CopilotProviderOptionsSchema.extend({
|
||||
dimensions: z.number(),
|
||||
}).optional();
|
||||
|
||||
export type CopilotEmbeddingOptions = z.infer<
|
||||
typeof CopilotEmbeddingOptionsSchema
|
||||
>;
|
||||
|
||||
export enum ModelInputType {
|
||||
Text = 'text',
|
||||
Image = 'image',
|
||||
Audio = 'audio',
|
||||
}
|
||||
|
||||
export interface CopilotTextToEmbeddingProvider extends CopilotProvider {
|
||||
generateEmbedding(
|
||||
messages: string[] | string,
|
||||
model: string,
|
||||
options?: CopilotEmbeddingOptions
|
||||
): Promise<number[][]>;
|
||||
export enum ModelOutputType {
|
||||
Text = 'text',
|
||||
Embedding = 'embedding',
|
||||
Image = 'image',
|
||||
Structured = 'structured',
|
||||
}
|
||||
|
||||
export interface CopilotTextToImageProvider extends CopilotProvider {
|
||||
generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): Promise<Array<string>>;
|
||||
generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): AsyncIterable<string>;
|
||||
export interface ModelCapability {
|
||||
input: ModelInputType[];
|
||||
output: ModelOutputType[];
|
||||
defaultForOutputType?: boolean;
|
||||
}
|
||||
|
||||
export interface CopilotImageToTextProvider extends CopilotProvider {
|
||||
generateText(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
generateTextStream(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
export interface CopilotProviderModel {
|
||||
id: string;
|
||||
capabilities: ModelCapability[];
|
||||
}
|
||||
|
||||
export interface CopilotImageToImageProvider extends CopilotProvider {
|
||||
generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): Promise<Array<string>>;
|
||||
generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export type CapabilityToCopilotProvider = {
|
||||
[CopilotCapability.TextToText]: CopilotTextToTextProvider;
|
||||
[CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
|
||||
[CopilotCapability.TextToImage]: CopilotTextToImageProvider;
|
||||
[CopilotCapability.ImageToText]: CopilotImageToTextProvider;
|
||||
[CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
|
||||
export type ModelConditions = {
|
||||
inputTypes?: ModelInputType[];
|
||||
modelId?: string;
|
||||
};
|
||||
|
||||
export type CopilotTextProvider =
|
||||
| CopilotTextToTextProvider
|
||||
| CopilotImageToTextProvider;
|
||||
export type CopilotImageProvider =
|
||||
| CopilotTextToImageProvider
|
||||
| CopilotImageToImageProvider;
|
||||
export type CopilotAllProvider =
|
||||
| CopilotTextProvider
|
||||
| CopilotImageProvider
|
||||
| CopilotTextToEmbeddingProvider;
|
||||
export type ModelFullConditions = ModelConditions & {
|
||||
outputType?: ModelOutputType;
|
||||
};
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
ImagePart,
|
||||
TextPart,
|
||||
} from 'ai';
|
||||
import { ZodType } from 'zod';
|
||||
|
||||
import { PromptMessage } from './types';
|
||||
|
||||
@@ -61,9 +62,12 @@ export async function chatToGPTMessage(
|
||||
messages: PromptMessage[],
|
||||
// TODO(@darkskygit): move this logic in interface refactoring
|
||||
withAttachment: boolean = true
|
||||
): Promise<[string | undefined, ChatMessage[], any]> {
|
||||
): Promise<[string | undefined, ChatMessage[], ZodType?]> {
|
||||
const system = messages[0]?.role === 'system' ? messages.shift() : undefined;
|
||||
const schema = system?.params?.schema;
|
||||
const schema =
|
||||
system?.params?.schema && system.params.schema instanceof ZodType
|
||||
? system.params.schema
|
||||
: undefined;
|
||||
|
||||
// filter redundant fields
|
||||
const msgs: ChatMessage[] = [];
|
||||
|
||||
Reference in New Issue
Block a user