feat(server): switch i2i to gpt (#12238)

fix AI-14
fix AI-17
fix AI-39
fix AI-112

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Expanded and reorganized prompt options for text and image actions, adding new prompts for image generation, style conversions, upscaling, background removal, and sticker creation.
  - Enhanced image editing capabilities with direct support for image attachments in prompts.

- **Improvements**
  - Updated prompt names and descriptions to be more user-friendly and descriptive.
  - Simplified and clarified prompt selection and image processing workflows with improved default behaviors.
  - Better organization of prompts through clear grouping and categorization.

- **Bug Fixes**
  - Improved validation and handling of image attachments during editing requests.

- **Refactor**
  - Internal code restructuring of prompts and provider logic for clarity and maintainability without affecting user workflows.
  - Refined message handling and content merging logic to ensure consistent prompt processing.
  - Adjusted image attachment rendering logic for improved display consistency.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
darkskygit
2025-05-27 11:36:47 +00:00
parent 1e9cbdb65d
commit 3c0fa429c5
10 changed files with 342 additions and 145 deletions

View File

@@ -21,7 +21,8 @@ export class ActionImage extends WithDisposable(ShadowlessElement) {
accessor testId = 'action-image';
protected override render() {
const images = this.item.messages[0].attachments;
const images =
this.item.messages[1]?.attachments ?? this.item.messages[0].attachments;
return html`<action-wrapper .host=${this.host} .item=${this.item}>
<div style=${styleMap({ marginBottom: '12px' })}>

View File

@@ -1,12 +1,7 @@
// manually synced with packages/backend/server/src/data/migrations/utils/prompts.ts
// TODO(@Peng): automate this
export const promptKeys = [
'debug:action:dalle3',
'debug:action:gpt-image-1',
'debug:action:fal-sd15',
'debug:action:fal-upscaler',
'debug:action:fal-remove-bg',
'debug:action:fal-face-to-sticker',
// text actions
'Chat With AFFiNE AI',
'Search With AFFiNE AI',
'Summary',
@@ -36,12 +31,18 @@ export const promptKeys = [
'Make it longer',
'Make it shorter',
'Continue writing',
// image actions
'Generate image',
'Convert to Anime style',
'Convert to Clay style',
'Convert to Pixel style',
'Convert to Sketch style',
'Convert to sticker',
'Upscale image',
'Remove background',
// workflows
'workflow:presentation',
'workflow:brainstorm',
'workflow:image-sketch',
'workflow:image-clay',
'workflow:image-anime',
'workflow:image-pixel',
] as const;
export type PromptKey = (typeof promptKeys)[number];

View File

@@ -27,18 +27,18 @@ function toAIUserInfo(account: AuthAccountInfo | null) {
const filterStyleToPromptName = new Map<string, PromptKey>(
Object.entries({
'Clay style': 'workflow:image-clay',
'Pixel style': 'workflow:image-pixel',
'Sketch style': 'workflow:image-sketch',
'Anime style': 'workflow:image-anime',
'Clay style': 'Convert to Clay style',
'Pixel style': 'Convert to Pixel style',
'Sketch style': 'Convert to Sketch style',
'Anime style': 'Convert to Anime style',
})
);
const processTypeToPromptName = new Map<string, PromptKey>(
Object.entries({
Clearer: 'debug:action:fal-upscaler',
'Remove background': 'debug:action:fal-remove-bg',
'Convert to sticker': 'debug:action:fal-face-to-sticker',
Clearer: 'Upscale image',
'Remove background': 'Remove background',
'Convert to sticker': 'Convert to sticker',
})
);
@@ -486,22 +486,18 @@ Could you make a new website based on these notes and send back just the html fi
});
AIProvider.provide('createImage', async options => {
// test to image
let promptName: PromptKey = 'debug:action:gpt-image-1';
// image to image
if (options.attachments?.length) {
promptName = 'debug:action:fal-sd15';
}
const sessionId = await createSession({
promptName,
promptName: 'Generate image',
...options,
});
return toImage({
...options,
client,
sessionId,
content: options.input,
content:
!options.input && options.attachments
? 'Make the image more detailed.'
: options.input,
// 5 minutes
timeout: 300000,
});