diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index 37497bf976..333fb42f6c 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -669,20 +669,20 @@ }, "scenarios": { "type": "object", - "description": "The models used in the scene for the copilot, will use this config if enabled.\n@default {\"enabled\":false,\"scenarios\":{\"audio\":\"gemini-2.5-flash\",\"chat\":\"claude-sonnet-4@20250514\",\"embedding\":\"gemini-embedding-001\",\"image\":\"gpt-image-1\",\"rerank\":\"gpt-4.1\",\"brainstorm\":\"gpt-4o-2024-08-06\",\"coding\":\"claude-sonnet-4@20250514\",\"quick_decision\":\"gpt-4.1-mini\",\"quick_written\":\"gemini-2.5-flash\",\"summary_inspection\":\"gemini-2.5-flash\"}}", + "description": "Use custom models in scenarios and override default settings.\n@default {\"override_enabled\":false,\"scenarios\":{\"audio_transcribing\":\"gemini-2.5-flash\",\"chat\":\"claude-sonnet-4@20250514\",\"embedding\":\"gemini-embedding-001\",\"image\":\"gpt-image-1\",\"rerank\":\"gpt-4.1\",\"coding\":\"claude-sonnet-4@20250514\",\"complex_text_generation\":\"gpt-4o-2024-08-06\",\"quick_decision_making\":\"gpt-4.1-mini\",\"quick_text_generation\":\"gemini-2.5-flash\",\"polish_and_summarize\":\"gemini-2.5-flash\"}}", "default": { - "enabled": false, + "override_enabled": false, "scenarios": { - "audio": "gemini-2.5-flash", + "audio_transcribing": "gemini-2.5-flash", "chat": "claude-sonnet-4@20250514", "embedding": "gemini-embedding-001", "image": "gpt-image-1", "rerank": "gpt-4.1", - "brainstorm": "gpt-4o-2024-08-06", "coding": "claude-sonnet-4@20250514", - "quick_decision": "gpt-4.1-mini", - "quick_written": "gemini-2.5-flash", - "summary_inspection": "gemini-2.5-flash" + "complex_text_generation": "gpt-4o-2024-08-06", + "quick_decision_making": "gpt-4.1-mini", + "quick_text_generation": "gemini-2.5-flash", + "polish_and_summarize": "gemini-2.5-flash" } } }, diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index b7a1da5adf..25d78264da 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -117,13 +117,13 @@ test.serial.before(async t => { value: { enabled: true, scenarios: { - image: 'lcm', + image: 'flux-1/schnell', rerank: 'gpt-4.1-mini', - brainstorm: 'gpt-4.1-mini', + complex_text_generation: 'gpt-4.1-mini', coding: 'gpt-4.1-mini', - quick_decision: 'gpt-4.1-mini', - quick_written: 'gpt-4.1-mini', - summary_inspection: 'gemini-2.5-flash', + quick_decision_making: 'gpt-4.1-mini', + quick_text_generation: 'gpt-4.1-mini', + polish_and_summarize: 'gemini-2.5-flash', }, }, }, diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts index dfafeea803..1517ab23b5 100644 --- a/packages/backend/server/src/plugins/copilot/config.ts +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -46,20 +46,20 @@ defineModuleConfig('copilot', { default: false, }, scenarios: { - desc: 'The models used in the scene for the copilot, will use this config if enabled.', + desc: 'Use custom models in scenarios and override default settings.', default: { - enabled: false, + override_enabled: false, scenarios: { - audio: 'gemini-2.5-flash', + audio_transcribing: 'gemini-2.5-flash', chat: 'claude-sonnet-4@20250514', embedding: 'gemini-embedding-001', image: 'gpt-image-1', rerank: 'gpt-4.1', - brainstorm: 'gpt-4o-2024-08-06', coding: 'claude-sonnet-4@20250514', - quick_decision: 'gpt-4.1-mini', - quick_written: 'gemini-2.5-flash', - summary_inspection: 'gemini-2.5-flash', + complex_text_generation: 'gpt-4o-2024-08-06', + quick_decision_making: 'gpt-4.1-mini', + quick_text_generation: 'gemini-2.5-flash', + polish_and_summarize: 'gemini-2.5-flash', }, }, }, diff --git a/packages/backend/server/src/plugins/copilot/embedding/client.ts b/packages/backend/server/src/plugins/copilot/embedding/client.ts index 1126892e5b..be002143fc 100644 --- a/packages/backend/server/src/plugins/copilot/embedding/client.ts +++ b/packages/backend/server/src/plugins/copilot/embedding/client.ts @@ -38,7 +38,7 @@ class ProductionEmbeddingClient extends EmbeddingClient { override async configured(): Promise { const embedding = await this.providerFactory.getProvider({ - modelId: this.config.copilot?.scenarios?.enabled + modelId: this.config.copilot?.scenarios?.override_enabled ? this.config.copilot.scenarios.scenarios?.embedding || EMBEDDING_MODEL : EMBEDDING_MODEL, outputType: ModelOutputType.Embedding, diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index fcc6f80d0f..8d0f394f7f 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -19,24 +19,9 @@ type Prompt = Omit< config?: PromptConfig; }; -export const Scenario: Record = { - audio: ['Transcript audio'], - brainstorm: [ - 'Brainstorm mindmap', - 'Create a presentation', - 'Expand mind map', - 'workflow:brainstorm:step2', - 'workflow:presentation:step2', - 'workflow:presentation:step4', - ], +export const Scenario = { + audio_transcribing: ['Transcript audio'], chat: ['Chat With AFFiNE AI'], - coding: [ - 'Apply Updates', - 'Code Artifact', - 'Make it real', - 'Make it real with text', - 'Section Edit', - ], // no prompt needed, just a placeholder embedding: [], image: [ @@ -49,7 +34,23 @@ export const Scenario: Record = { 'Remove background', 'Upscale image', ], - quick_decision: [ + rerank: ['Rerank results'], + coding: [ + 'Apply Updates', + 'Code Artifact', + 'Make it real', + 'Make it real with text', + 'Section Edit', + ], + complex_text_generation: [ + 'Brainstorm mindmap', + 'Create a presentation', + 'Expand mind map', + 'workflow:brainstorm:step2', + 'workflow:presentation:step2', + 'workflow:presentation:step4', + ], + quick_decision_making: [ 'Create headings', 'Generate a caption', 'Translate to', @@ -60,7 +61,7 @@ export const Scenario: Record = { 'workflow:image-pixel:step2', 'workflow:image-sketch:step2', ], - quick_written: [ + quick_text_generation: [ 'Brainstorm ideas about this', 'Continue writing', 'Explain this code', @@ -73,8 +74,7 @@ export const Scenario: Record = { 'Write an article about this', 'Write outline', ], - rerank: ['Rerank results'], - summary_inspection: [ + polish_and_summarize: [ 'Change tone to', 'Check code error', 'Conversation Summary', @@ -92,7 +92,7 @@ export const Scenario: Record = { }; export type CopilotPromptScenario = { - enabled?: boolean; + override_enabled?: boolean; scenarios?: Partial>; }; diff --git a/packages/backend/server/src/plugins/copilot/prompt/service.ts b/packages/backend/server/src/plugins/copilot/prompt/service.ts index 99d3e7bc28..cc894f2c35 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/service.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/service.ts @@ -45,10 +45,11 @@ export class PromptService implements OnApplicationBootstrap { } protected async setup(scenarios?: CopilotPromptScenario) { - if (!!scenarios && scenarios.enabled && scenarios.scenarios) { + if (!!scenarios && scenarios.override_enabled && scenarios.scenarios) { this.logger.log('Updating prompts based on scenarios...'); for (const [scenario, model] of Object.entries(scenarios.scenarios)) { - const promptNames = Scenario[scenario]; + const promptNames = Scenario[scenario as keyof typeof Scenario] || []; + if (!promptNames.length) continue; for (const name of promptNames) { const prompt = prompts.find(p => p.name === name); if (prompt && model) { diff --git a/packages/backend/server/src/plugins/copilot/providers/fal.ts b/packages/backend/server/src/plugins/copilot/providers/fal.ts index b6a5dc5a78..486d3e91fd 100644 --- a/packages/backend/server/src/plugins/copilot/providers/fal.ts +++ b/packages/backend/server/src/plugins/copilot/providers/fal.ts @@ -75,7 +75,7 @@ export class FalProvider extends CopilotProvider { override readonly models = [ { - id: 'lcm', + id: 'flux-1/schnell', capabilities: [ { input: [ModelInputType.Text], diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index 6edec27e28..273d88b306 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -258,7 +258,7 @@ }, "scenarios": { "type": "Object", - "desc": "The models used in the scene for the copilot, will use this config if enabled." + "desc": "Use custom models in scenarios and override default settings." }, "providers.openai": { "type": "Object",