From 743e2eb8d2a0168234570735adb4ef34b753a760 Mon Sep 17 00:00:00 2001 From: darkskygit Date: Thu, 24 Apr 2025 09:04:29 +0000 Subject: [PATCH] feat(server): adapt 4o image (#11956) fix AI-88 ## Summary by CodeRabbit - **New Features** - Added support for the new image generation model "gpt-image-1" in both backend and frontend. - Updated available prompt options to include "gpt-image-1" for image creation actions. - **Improvements** - Set a 5-minute timeout for image generation requests to improve reliability. - **Tests** - Expanded test coverage to validate image generation using the new "gpt-image-1" prompt. --- packages/backend/server/package.json | 2 +- .../server/src/__tests__/copilot-provider.spec.ts | 2 +- .../server/src/plugins/copilot/prompt/prompts.ts | 6 ++++++ .../server/src/plugins/copilot/providers/openai.ts | 1 + .../frontend/core/src/blocksuite/ai/provider/prompt.ts | 1 + .../core/src/blocksuite/ai/provider/setup-provider.tsx | 4 +++- yarn.lock | 10 +++++----- 7 files changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 3ff76ab9b5..190f11121a 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -26,7 +26,7 @@ }, "dependencies": { "@ai-sdk/google": "^1.2.10", - "@ai-sdk/openai": "^1.3.9", + "@ai-sdk/openai": "^1.3.18", "@ai-sdk/perplexity": "^1.1.6", "@apollo/server": "^4.11.3", "@aws-sdk/client-s3": "^3.779.0", diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index a03d685915..0868eaa4a6 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -514,7 +514,7 @@ const actions = [ type: 'image' as const, }, { - promptName: ['debug:action:dalle3'], + promptName: ['debug:action:dalle3', 'debug:action:gpt-image-1'], messages: [ { role: 'user' as const, diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index eeed0ceba4..97db163239 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -288,6 +288,12 @@ const actions: Prompt[] = [ model: 'dall-e-3', messages: [], }, + { + name: 'debug:action:gpt-image-1', + action: 'image', + model: 'gpt-image-1', + messages: [], + }, { name: 'debug:action:fal-sd15', action: 'image', diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 8d6c5510dc..edd0fb5d5a 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -76,6 +76,7 @@ export class OpenAIProvider 'text-moderation-stable', // text to image 'dall-e-3', + 'gpt-image-1', ]; #instance!: VercelOpenAIProvider; diff --git a/packages/frontend/core/src/blocksuite/ai/provider/prompt.ts b/packages/frontend/core/src/blocksuite/ai/provider/prompt.ts index 00cc036eac..52d844737b 100644 --- a/packages/frontend/core/src/blocksuite/ai/provider/prompt.ts +++ b/packages/frontend/core/src/blocksuite/ai/provider/prompt.ts @@ -3,6 +3,7 @@ export const promptKeys = [ 'debug:chat:gpt4', 'debug:action:dalle3', + 'debug:action:gpt-image-1', 'debug:action:fal-sd15', 'debug:action:fal-upscaler', 'debug:action:fal-remove-bg', diff --git a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx index 7d8b21aead..b9cce2d7cf 100644 --- a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx +++ b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx @@ -492,7 +492,7 @@ Could you make a new website based on these notes and send back just the html fi AIProvider.provide('createImage', async options => { // test to image - let promptName: PromptKey = 'debug:action:dalle3'; + let promptName: PromptKey = 'debug:action:gpt-image-1'; // image to image if (options.attachments?.length) { promptName = 'debug:action:fal-sd15'; @@ -507,6 +507,8 @@ Could you make a new website based on these notes and send back just the html fi client, sessionId, content: options.input, + // 5 minutes + timeout: 300000, }); }); diff --git a/yarn.lock b/yarn.lock index 3e2b371c0b..355019b31b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -870,7 +870,7 @@ __metadata: "@affine/graphql": "workspace:*" "@affine/server-native": "workspace:*" "@ai-sdk/google": "npm:^1.2.10" - "@ai-sdk/openai": "npm:^1.3.9" + "@ai-sdk/openai": "npm:^1.3.18" "@ai-sdk/perplexity": "npm:^1.1.6" "@apollo/server": "npm:^4.11.3" "@aws-sdk/client-s3": "npm:^3.779.0" @@ -1044,15 +1044,15 @@ __metadata: languageName: node linkType: hard -"@ai-sdk/openai@npm:^1.3.9": - version: 1.3.12 - resolution: "@ai-sdk/openai@npm:1.3.12" +"@ai-sdk/openai@npm:^1.3.18": + version: 1.3.18 + resolution: "@ai-sdk/openai@npm:1.3.18" dependencies: "@ai-sdk/provider": "npm:1.1.3" "@ai-sdk/provider-utils": "npm:2.2.7" peerDependencies: zod: ^3.0.0 - checksum: 10/067e6ce7a59bda062ea5198f928809d7cad9aae994c786b611f104515f3fcf3cb93f370ce3cb58c223ebc18da633d8f934beec4e879d26d071a8da81013369fb + checksum: 10/5d6e8ea5b3a6afc237d3220bdb7f307b6b82b1fd2511d9627f09b1be70e36c15060e807381148c4203d61a317acf87091b3b42edc55da7b424f2c2caf11c5a19 languageName: node linkType: hard