Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-08-09 05:05:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): adapt gemini3.1 preview (#14583)

Browse Source 
#### PR Dependency Tree


* **PR #14583** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added Gemini 3.1 Pro Preview support (text, image, audio) and new
GPT‑5 variants as defaults; centralized persistent telemetry state for
more reliable client identity.

* **UX**
  * Improved model submenu placement in chat preferences.
* More robust mindmap parsing, preview, regeneration and replace
behavior.

* **Chores**
  * Bumped AI SDK and related dependencies.

* **Tests**
  * Expanded/updated tests and increased timeouts for flaky flows.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-03-08 00:53:16 +08:00
committed by GitHub
parent 9742e9735e
commit 9c55edeb62
36 changed files with 980 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts
+43 -8
View File
@@ -5,6 +5,7 @@ import type {
import type { GoogleVertexProvider } from '@ai-sdk/google-vertex';
import {
AISDKError,
type EmbeddingModel,
embedMany,
generateObject,
generateText,
@@ -43,6 +44,34 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
| GoogleGenerativeAIProvider
| GoogleVertexProvider;
private getThinkingConfig(
model: string,
options: { includeThoughts: boolean; useDynamicBudget?: boolean }
): NonNullable<GoogleGenerativeAIProviderOptions['thinkingConfig']> {
if (this.isGemini3Model(model)) {
return {
includeThoughts: options.includeThoughts,
thinkingLevel: 'high',
};
}
return {
includeThoughts: options.includeThoughts,
thinkingBudget: options.useDynamicBudget ? -1 : 12000,
};
}
private getEmbeddingModel(model: string) {
const provider = this.instance as typeof this.instance & {
embeddingModel?: (modelId: string) => EmbeddingModel;
textEmbeddingModel?: (modelId: string) => EmbeddingModel;
};
return (
provider.embeddingModel?.(model) ?? provider.textEmbeddingModel?.(model)
);
}
private handleError(e: any) {
if (e instanceof UserFriendlyError) {
return e;
@@ -122,10 +151,10 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
schema,
providerOptions: {
google: {
thinkingConfig: {
thinkingBudget: -1,
thinkingConfig: this.getThinkingConfig(model.id, {
includeThoughts: false,
},
useDynamicBudget: true,
}),
},
},
abortSignal: options.signal,
@@ -234,7 +263,10 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
.counter('generate_embedding_calls')
.add(1, { model: model.id });
const modelInstance = this.instance.textEmbeddingModel(model.id);
const modelInstance = this.getEmbeddingModel(model.id);
if (!modelInstance) {
throw new Error(`Embedding model is not available for ${model.id}`);
}
const embeddings = await Promise.allSettled(
messages.map(m =>
@@ -286,15 +318,18 @@ export abstract class GeminiProvider<T> extends CopilotProvider<T> {
private getGeminiOptions(options: CopilotChatOptions, model: string) {
const result: GoogleGenerativeAIProviderOptions = {};
if (options?.reasoning && this.isReasoningModel(model)) {
result.thinkingConfig = {
thinkingBudget: 12000,
result.thinkingConfig = this.getThinkingConfig(model, {
includeThoughts: true,
};
});
}
return result;
}
private isGemini3Model(model: string) {
return model.startsWith('gemini-3');
}
private isReasoningModel(model: string) {
return model.startsWith('gemini-2.5');
return model.startsWith('gemini-2.5') || this.isGemini3Model(model);
}
}
packages/backend/server/src/plugins/copilot/providers/gemini/generative.ts
+18 -19
View File
@@ -20,25 +20,6 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
override readonly type = CopilotProviderType.Gemini;
readonly models = [
{
name: 'Gemini 2.0 Flash',
id: 'gemini-2.0-flash-001',
capabilities: [
{
input: [
ModelInputType.Text,
ModelInputType.Image,
ModelInputType.Audio,
],
output: [
ModelOutputType.Text,
ModelOutputType.Object,
ModelOutputType.Structured,
],
defaultForOutputType: true,
},
],
},
{
name: 'Gemini 2.5 Flash',
id: 'gemini-2.5-flash',
@@ -75,6 +56,24 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
},
],
},
{
name: 'Gemini 3.1 Pro Preview',
id: 'gemini-3.1-pro-preview',
capabilities: [
{
input: [
ModelInputType.Text,
ModelInputType.Image,
ModelInputType.Audio,
],
output: [
ModelOutputType.Text,
ModelOutputType.Object,
ModelOutputType.Structured,
],
},
],
},
{
name: 'Gemini Embedding',
id: 'gemini-embedding-001',
packages/backend/server/src/plugins/copilot/providers/gemini/vertex.ts
+18
View File
@@ -50,6 +50,24 @@ export class GeminiVertexProvider extends GeminiProvider<GeminiVertexConfig> {
},
],
},
{
name: 'Gemini 3.1 Pro Preview',
id: 'gemini-3.1-pro-preview',
capabilities: [
{
input: [
ModelInputType.Text,
ModelInputType.Image,
ModelInputType.Audio,
],
output: [
ModelOutputType.Text,
ModelOutputType.Object,
ModelOutputType.Structured,
],
},
],
},
{
name: 'Gemini Embedding',
id: 'gemini-embedding-001',
packages/backend/server/src/plugins/copilot/providers/openai.ts
+87 -7
View File
@@ -18,6 +18,12 @@ import {
import type { NodeTextMiddleware } from '../config';
import { buildNativeRequest, NativeProviderAdapter } from './native';
import { CopilotProvider } from './provider';
import {
normalizeRerankModel,
OPENAI_RERANK_MAX_COMPLETION_TOKENS,
OPENAI_RERANK_TOP_LOGPROBS_LIMIT,
usesRerankReasoning,
} from './rerank';
import type {
CopilotChatOptions,
CopilotChatTools,
@@ -33,6 +39,30 @@ import { chatToGPTMessage } from './utils';
export const DEFAULT_DIMENSIONS = 256;
const GPT_5_SAMPLING_UNSUPPORTED_MODELS = /^(gpt-5(?:$|[.-]))/;
export function normalizeOpenAIOptionsForModel<
T extends {
frequencyPenalty?: number | null;
presencePenalty?: number | null;
temperature?: number | null;
topP?: number | null;
},
>(options: T, model: string): T {
if (!GPT_5_SAMPLING_UNSUPPORTED_MODELS.test(model)) {
return options;
}
const normalizedOptions = { ...options };
delete normalizedOptions.frequencyPenalty;
delete normalizedOptions.presencePenalty;
delete normalizedOptions.temperature;
delete normalizedOptions.topP;
return normalizedOptions;
}
export type OpenAIConfig = {
apiKey: string;
baseURL?: string;
@@ -252,6 +282,34 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
},
],
},
{
name: 'GPT 5.2',
id: 'gpt-5.2',
capabilities: [
{
input: [ModelInputType.Text, ModelInputType.Image],
output: [
ModelOutputType.Text,
ModelOutputType.Object,
ModelOutputType.Structured,
],
},
],
},
{
name: 'GPT 5.2 2025-12-11',
id: 'gpt-5.2-2025-12-11',
capabilities: [
{
input: [ModelInputType.Text, ModelInputType.Image],
output: [
ModelOutputType.Text,
ModelOutputType.Object,
ModelOutputType.Structured,
],
},
],
},
{
name: 'GPT 5 Nano',
id: 'gpt-5-nano',
@@ -435,10 +493,14 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
metrics.ai.counter('chat_text_calls').add(1, this.metricLabels(model.id));
const tools = await this.getTools(options, model.id);
const middleware = this.getActiveProviderMiddleware();
const normalizedOptions = normalizeOpenAIOptionsForModel(
options,
model.id
);
const { request } = await buildNativeRequest({
model: model.id,
messages,
options,
options: normalizedOptions,
tools,
include: options.webSearch ? ['citations'] : undefined,
reasoning: this.getReasoning(options, model.id),
@@ -472,10 +534,14 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
.add(1, this.metricLabels(model.id));
const tools = await this.getTools(options, model.id);
const middleware = this.getActiveProviderMiddleware();
const normalizedOptions = normalizeOpenAIOptionsForModel(
options,
model.id
);
const { request } = await buildNativeRequest({
model: model.id,
messages,
options,
options: normalizedOptions,
tools,
include: options.webSearch ? ['citations'] : undefined,
reasoning: this.getReasoning(options, model.id),
@@ -508,10 +574,14 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
.add(1, this.metricLabels(model.id));
const tools = await this.getTools(options, model.id);
const middleware = this.getActiveProviderMiddleware();
const normalizedOptions = normalizeOpenAIOptionsForModel(
options,
model.id
);
const { request } = await buildNativeRequest({
model: model.id,
messages,
options,
options: normalizedOptions,
tools,
include: options.webSearch ? ['citations'] : undefined,
reasoning: this.getReasoning(options, model.id),
@@ -542,10 +612,14 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
const tools = await this.getTools(options, model.id);
const middleware = this.getActiveProviderMiddleware();
const normalizedOptions = normalizeOpenAIOptionsForModel(
options,
model.id
);
const { request, schema } = await buildNativeRequest({
model: model.id,
messages,
options,
options: normalizedOptions,
tools,
reasoning: this.getReasoning(options, model.id),
middleware,
@@ -576,15 +650,21 @@ export class OpenAIProvider extends CopilotProvider<OpenAIConfig> {
const scores = await Promise.all(
chunkMessages.map(async messages => {
const [system, msgs] = await chatToGPTMessage(messages);
const rerankModel = normalizeRerankModel(model.id);
const response = await this.requestOpenAIJson(
'/chat/completions',
{
model: model.id,
model: rerankModel,
messages: this.toOpenAIChatMessages(system, msgs),
temperature: 0,
max_tokens: 16,
logprobs: true,
top_logprobs: 16,
top_logprobs: OPENAI_RERANK_TOP_LOGPROBS_LIMIT,
...(usesRerankReasoning(rerankModel)
? {
reasoning_effort: 'none' as const,
max_completion_tokens: OPENAI_RERANK_MAX_COMPLETION_TOKENS,
}
: { max_tokens: OPENAI_RERANK_MAX_COMPLETION_TOKENS }),
},
options.signal
);
packages/backend/server/src/plugins/copilot/providers/rerank.ts
+23
View File
@@ -0,0 +1,23 @@
const GPT_4_RERANK_MODELS = /^(gpt-4(?:$|[.-]))/;
const GPT_5_RERANK_LOGPROBS_MODELS = /^(gpt-5\.2(?:$|-))/;
export const DEFAULT_RERANK_MODEL = 'gpt-5.2';
export const OPENAI_RERANK_TOP_LOGPROBS_LIMIT = 5;
export const OPENAI_RERANK_MAX_COMPLETION_TOKENS = 16;
export function supportsRerankModel(model: string): boolean {
return (
GPT_4_RERANK_MODELS.test(model) || GPT_5_RERANK_LOGPROBS_MODELS.test(model)
);
}
export function usesRerankReasoning(model: string): boolean {
return GPT_5_RERANK_LOGPROBS_MODELS.test(model);
}
export function normalizeRerankModel(model?: string | null): string {
if (model && supportsRerankModel(model)) {
return model;
}
return DEFAULT_RERANK_MODEL;
}
packages/backend/server/src/plugins/copilot/providers/utils.ts
+3 -3
View File
@@ -2,12 +2,12 @@ import { GoogleVertexProviderSettings } from '@ai-sdk/google-vertex';
import { GoogleVertexAnthropicProviderSettings } from '@ai-sdk/google-vertex/anthropic';
import { Logger } from '@nestjs/common';
import {
CoreAssistantMessage,
CoreUserMessage,
AssistantModelMessage,
FilePart,
ImagePart,
TextPart,
TextStreamPart,
UserModelMessage,
} from 'ai';
import { GoogleAuth, GoogleAuthOptions } from 'google-auth-library';
import z, { ZodType } from 'zod';
@@ -23,7 +23,7 @@ import {
import { CustomAITools } from '../tools';
import { PromptMessage, StreamObject } from './types';
type ChatMessage = CoreUserMessage | CoreAssistantMessage;
type ChatMessage = UserModelMessage | AssistantModelMessage;
const ATTACHMENT_MAX_BYTES = 20 * 1024 * 1024;
const ATTACH_HEAD_PARAMS = { timeoutMs: OneMinute / 12, maxRedirects: 3 };