From 887a496f8b38ace9b42fb673fcba866d89b1a999 Mon Sep 17 00:00:00 2001 From: darkskygit Date: Fri, 30 May 2025 08:39:32 +0000 Subject: [PATCH] feat(server): add attachment fallback for ai sdk (#12639) fix AI-161 --- packages/backend/server/package.json | 8 ++--- .../copilot/providers/anthropic/anthropic.ts | 2 +- .../src/plugins/copilot/providers/utils.ts | 12 +++++--- yarn.lock | 30 +++++++++---------- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index f6f9245655..3349970674 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -28,11 +28,11 @@ "dependencies": { "@affine/reader": "workspace:*", "@affine/server-native": "workspace:*", - "@ai-sdk/anthropic": "^1.2.10", + "@ai-sdk/anthropic": "^1.2.12", "@ai-sdk/google": "^1.2.18", - "@ai-sdk/google-vertex": "^2.2.22", - "@ai-sdk/openai": "^1.3.21", - "@ai-sdk/perplexity": "^1.1.6", + "@ai-sdk/google-vertex": "^2.2.23", + "@ai-sdk/openai": "^1.3.22", + "@ai-sdk/perplexity": "^1.1.9", "@apollo/server": "^4.11.3", "@aws-sdk/client-s3": "^3.779.0", "@aws-sdk/s3-request-presigner": "^3.779.0", diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts index eb96e2d095..7fd0032d28 100644 --- a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts +++ b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts @@ -58,7 +58,7 @@ export abstract class AnthropicProvider extends CopilotProvider { try { metrics.ai.counter('chat_text_calls').add(1, { model: model.id }); - const [system, msgs] = await chatToGPTMessage(messages); + const [system, msgs] = await chatToGPTMessage(messages, true, true); const modelInstance = this.instance(model.id); const { text, reasoning } = await generateText({ diff --git a/packages/backend/server/src/plugins/copilot/providers/utils.ts b/packages/backend/server/src/plugins/copilot/providers/utils.ts index b5c8055800..ddee828892 100644 --- a/packages/backend/server/src/plugins/copilot/providers/utils.ts +++ b/packages/backend/server/src/plugins/copilot/providers/utils.ts @@ -64,7 +64,10 @@ export async function inferMimeType(url: string) { export async function chatToGPTMessage( messages: PromptMessage[], // TODO(@darkskygit): move this logic in interface refactoring - withAttachment: boolean = true + withAttachment: boolean = true, + // NOTE: some providers in vercel ai sdk are not able to handle url attachments yet + // so we need to use base64 encoded attachments instead + useBase64Attachment: boolean = false ): Promise<[string | undefined, ChatMessage[], ZodType?]> { const system = messages[0]?.role === 'system' ? messages.shift() : undefined; const schema = @@ -101,9 +104,10 @@ export async function chatToGPTMessage( if (mimeType.startsWith('image/')) { contents.push({ type: 'image', image: attachment, mimeType }); } else { - const data = attachment.startsWith('data:') - ? await fetch(attachment).then(r => r.arrayBuffer()) - : new URL(attachment); + const data = + attachment.startsWith('data:') || useBase64Attachment + ? await fetch(attachment).then(r => r.arrayBuffer()) + : new URL(attachment); contents.push({ type: 'file' as const, data, mimeType }); } } diff --git a/yarn.lock b/yarn.lock index eb65621007..312e3943bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -908,11 +908,11 @@ __metadata: "@affine/graphql": "workspace:*" "@affine/reader": "workspace:*" "@affine/server-native": "workspace:*" - "@ai-sdk/anthropic": "npm:^1.2.10" + "@ai-sdk/anthropic": "npm:^1.2.12" "@ai-sdk/google": "npm:^1.2.18" - "@ai-sdk/google-vertex": "npm:^2.2.22" - "@ai-sdk/openai": "npm:^1.3.21" - "@ai-sdk/perplexity": "npm:^1.1.6" + "@ai-sdk/google-vertex": "npm:^2.2.23" + "@ai-sdk/openai": "npm:^1.3.22" + "@ai-sdk/perplexity": "npm:^1.1.9" "@apollo/server": "npm:^4.11.3" "@aws-sdk/client-s3": "npm:^3.779.0" "@aws-sdk/s3-request-presigner": "npm:^3.779.0" @@ -1075,30 +1075,30 @@ __metadata: languageName: unknown linkType: soft -"@ai-sdk/anthropic@npm:1.2.11, @ai-sdk/anthropic@npm:^1.2.10": - version: 1.2.11 - resolution: "@ai-sdk/anthropic@npm:1.2.11" +"@ai-sdk/anthropic@npm:1.2.12, @ai-sdk/anthropic@npm:^1.2.12": + version: 1.2.12 + resolution: "@ai-sdk/anthropic@npm:1.2.12" dependencies: "@ai-sdk/provider": "npm:1.1.3" "@ai-sdk/provider-utils": "npm:2.2.8" peerDependencies: zod: ^3.0.0 - checksum: 10/89f48e23f406bef946d2b9153e67eb8417ba01693e291566693f4d49ca4eb9e41e7e49ec83cbb738c7ac2eaf8c3aa572327a0ab6ae7f8f8bf50f08d3be0c2471 + checksum: 10/ee09f00328c88954fba8d00795f6c379091f95b7717bb90292acf32f41b072b2c89b178c1c18dfc23fcec7532e04fcd13233b9a813ce4881e0a16cae20a878c8 languageName: node linkType: hard -"@ai-sdk/google-vertex@npm:^2.2.22": - version: 2.2.22 - resolution: "@ai-sdk/google-vertex@npm:2.2.22" +"@ai-sdk/google-vertex@npm:^2.2.23": + version: 2.2.23 + resolution: "@ai-sdk/google-vertex@npm:2.2.23" dependencies: - "@ai-sdk/anthropic": "npm:1.2.11" + "@ai-sdk/anthropic": "npm:1.2.12" "@ai-sdk/google": "npm:1.2.18" "@ai-sdk/provider": "npm:1.1.3" "@ai-sdk/provider-utils": "npm:2.2.8" google-auth-library: "npm:^9.15.0" peerDependencies: zod: ^3.0.0 - checksum: 10/852928d35797cc14802d8c4a7dba2794197f019507a24124e7f87e456ba19f2b4e50438af626dd30e3a2a0f9f10819d6534a31e23c08f7f3d148d56b68167d0a + checksum: 10/19b7120879662e3597e95b84d30a587ada24d5fbd4306e4551f88605ae4e7c9d3f52bc60b6ea3bfbfc487a58c0a0ad35d1e058f7408140ea84e2ff86e6a1d052 languageName: node linkType: hard @@ -1114,7 +1114,7 @@ __metadata: languageName: node linkType: hard -"@ai-sdk/openai@npm:^1.3.21": +"@ai-sdk/openai@npm:^1.3.22": version: 1.3.22 resolution: "@ai-sdk/openai@npm:1.3.22" dependencies: @@ -1126,7 +1126,7 @@ __metadata: languageName: node linkType: hard -"@ai-sdk/perplexity@npm:^1.1.6": +"@ai-sdk/perplexity@npm:^1.1.9": version: 1.1.9 resolution: "@ai-sdk/perplexity@npm:1.1.9" dependencies: