Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-12 04:18:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): use backend prompts (#6542)

Browse Source
This commit is contained in:
pengx17
2024-04-12 11:15:38 +00:00
committed by Peng Xiao
parent 2336638996
commit 9b620ecbc9
9 changed files with 482 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/copilot-client.ts Normal file
View File

@@ -0,0 +1,140 @@
import {
createCopilotMessageMutation,
createCopilotSessionMutation,
fetcher,
getBaseUrl,
getCopilotHistoriesQuery,
getCopilotSessionsQuery,
type GraphQLQuery,
type RequestOptions,
} from '@affine/graphql';
type OptionsField<T extends GraphQLQuery> =
RequestOptions<T>['variables'] extends { options: infer U } ? U : never;
export class CopilotClient {
readonly backendUrl = getBaseUrl();
async createSession(
options: OptionsField<typeof createCopilotSessionMutation>
) {
const res = await fetcher({
query: createCopilotSessionMutation,
variables: {
options,
},
});
return res.createCopilotSession;
}
async createMessage(
options: OptionsField<typeof createCopilotMessageMutation>
) {
const res = await fetcher({
query: createCopilotMessageMutation,
variables: {
options,
},
});
return res.createCopilotMessage;
}
async getSessions(workspaceId: string) {
const res = await fetcher({
query: getCopilotSessionsQuery,
variables: {
workspaceId,
},
});
return res.currentUser?.copilot;
}
async getHistories(
workspaceId: string,
docId?: string,
options?: OptionsField<typeof getCopilotHistoriesQuery>
) {
const res = await fetcher({
query: getCopilotHistoriesQuery,
variables: {
workspaceId,
docId,
options,
},
});
return res.currentUser?.copilot?.histories;
}
async textToText(message: string, sessionId: string) {
const res = await fetch(
`${this.backendUrl}/api/copilot/chat/${sessionId}?message=${encodeURIComponent(message)}`
);
if (!res.ok) return;
return res.text();
}
textToTextStream(message: string, sessionId: string) {
return new EventSource(
`${this.backendUrl}/api/copilot/chat/${sessionId}/stream?message=${encodeURIComponent(message)}`
);
}
chatText({
sessionId,
messageId,
message,
}: {
sessionId: string;
messageId?: string;
message?: string;
}) {
if (messageId && message) {
throw new Error('Only one of messageId or message can be provided');
} else if (!messageId && !message) {
throw new Error('Either messageId or message must be provided');
}
const url = new URL(`${this.backendUrl}/api/copilot/chat/${sessionId}`);
if (messageId) {
url.searchParams.set('messageId', messageId);
}
if (message) {
url.searchParams.set('message', message);
}
return fetch(url.toString());
}
// Text or image to text
chatTextStream({
sessionId,
messageId,
message,
}: {
sessionId: string;
messageId?: string;
message?: string;
}) {
if (messageId && message) {
throw new Error('Only one of messageId or message can be provided');
} else if (!messageId && !message) {
throw new Error('Either messageId or message must be provided');
}
const url = new URL(
`${this.backendUrl}/api/copilot/chat/${sessionId}/stream`
);
if (messageId) {
url.searchParams.set('messageId', messageId);
}
if (message) {
url.searchParams.set('message', message);
}
return new EventSource(url.toString());
}
// Text or image to images
imagesStream(messageId: string, sessionId: string) {
return new EventSource(
`${this.backendUrl}/api/copilot/chat/${sessionId}/images?messageId=${messageId}`
);
}
}

35
packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/prompt.ts Normal file
View File

@@ -0,0 +1,35 @@
// manually synced with packages/backend/server/src/data/migrations/utils/prompts.ts
// todo: automate this
export const promptKeys = [
'debug:chat:gpt4',
'debug:action:gpt4',
'debug:action:vision4',
'debug:action:dalle3',
'debug:action:fal-sd15',
'Summary',
'Summary the webpage',
'Explain this',
'Explain this image',
'Explain this code',
'Translate to',
'Write an article about this',
'Write a twitter about this',
'Write a poem about this',
'Write a blog post about this',
'Write outline',
'Change tone to',
'Brainstorm ideas about this',
'Brainstorm mindmap',
'Improve writing for it',
'Improve grammar for it',
'Fix spelling for it',
'Find action items from it',
'Check code error',
'Create a presentation',
'Create headings',
'Make it real',
'Make it longer',
'Make it shorter',
] as const;
export type PromptKey = (typeof promptKeys)[number];

283
packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts
View File

@@ -1,284 +1,193 @@
import { assertExists } from '@blocksuite/global/utils';
import { AIProvider } from '@blocksuite/presets';
import { imageToTextStream, textToTextStream } from './request';
import { textToText } from './request';
export function setupAIProvider() {
AIProvider.provideAction('chat', options => {
assertExists(options.stream);
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt: options.input,
return textToText({
...options,
content: options.input,
promptName: 'debug:chat:gpt4',
});
});
AIProvider.provideAction('summary', options => {
assertExists(options.stream);
const prompt = `
Summarize the key points from the following content in a clear and concise manner,
suitable for a reader who is seeking a quick understanding of the original content.
Ensure to capture the main ideas and any significant details without unnecessary elaboration:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Summary',
});
});
AIProvider.provideAction('translate', options => {
assertExists(options.stream);
const prompt = `Please translate the following content into ${options.lang} and return it to us, adhering to the original format of the content：
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
promptName: 'Translate to',
content: options.input,
params: {
language: options.lang,
},
});
});
AIProvider.provideAction('changeTone', options => {
assertExists(options.stream);
const prompt = `Change the tone of the following content to ${options.tone}: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Change tone to',
});
});
AIProvider.provideAction('improveWriting', options => {
assertExists(options.stream);
const prompt = `Improve the writing of the following content: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Improve writing for it',
});
});
AIProvider.provideAction('improveGrammar', options => {
assertExists(options.stream);
const prompt = `Improve the grammar of the following content: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Improve grammar for it',
});
});
AIProvider.provideAction('fixSpelling', options => {
assertExists(options.stream);
const prompt = `Fix the spelling of the following content: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Fix spelling for it',
});
});
AIProvider.provideAction('createHeadings', options => {
assertExists(options.stream);
const prompt = `Create headings for the following content: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Create headings',
});
});
AIProvider.provideAction('makeLonger', options => {
assertExists(options.stream);
const prompt = `Make the following content longer: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Make it longer',
});
});
AIProvider.provideAction('makeShorter', options => {
assertExists(options.stream);
const prompt = `Make the following content shorter: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Make it shorter',
});
});
AIProvider.provideAction('checkCodeErrors', options => {
assertExists(options.stream);
const prompt = `Check the code errors in the following content and provide the corrected version:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Check code error',
});
});
AIProvider.provideAction('explainCode', options => {
assertExists(options.stream);
const prompt = `Explain the code in the following content, focusing on the logic, functions, and expected outcomes:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Explain this code',
});
});
AIProvider.provideAction('writeArticle', options => {
assertExists(options.stream);
const prompt = `Write an article based on the following content, focusing on the main ideas, structure, and flow:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Write an article about this',
});
});
AIProvider.provideAction('writeTwitterPost', options => {
assertExists(options.stream);
const prompt = `Write a Twitter post based on the following content, keeping it concise and engaging:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Write a twitter about this',
});
});
AIProvider.provideAction('writePoem', options => {
assertExists(options.stream);
const prompt = `Write a poem based on the following content, focusing on the emotions, imagery, and rhythm:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Write a poem about this',
});
});
AIProvider.provideAction('writeOutline', options => {
assertExists(options.stream);
const prompt = `Write an outline from the following content in Markdown: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Write outline',
});
});
AIProvider.provideAction('writeBlogPost', options => {
assertExists(options.stream);
const prompt = `Write a blog post based on the following content, focusing on the insights, analysis, and personal perspective:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Write a blog post about this',
});
});
AIProvider.provideAction('brainstorm', options => {
assertExists(options.stream);
const prompt = `Brainstorm ideas based on the following content, exploring different angles, perspectives, and approaches:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Brainstorm ideas about this',
});
});
AIProvider.provideAction('findActions', options => {
assertExists(options.stream);
const prompt = `Find actions related to the following content and return content in markdown: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
});
});
AIProvider.provideAction('writeOutline', options => {
assertExists(options.stream);
const prompt = `Write an outline based on the following content, organizing the main points, subtopics, and structure:
${options.input}
`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Find action items from it',
});
});
AIProvider.provideAction('brainstormMindmap', options => {
assertExists(options.stream);
const prompt = `Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Brainstorm mindmap',
});
});
AIProvider.provideAction('explain', options => {
assertExists(options.stream);
const prompt = `Explain the following content in Markdown: ${options.input}`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
return textToText({
...options,
content: options.input,
promptName: 'Explain this',
});
});
AIProvider.provideAction('explainImage', options => {
assertExists(options.stream);
const prompt = `Describe the scene captured in this image, focusing on the details, colors, emotions, and any interactions between subjects or objects present.`;
return textToTextStream({
docId: options.docId,
workspaceId: options.workspaceId,
prompt,
attachments: options.attachments,
return textToText({
...options,
content: options.input,
promptName: 'Explain this image',
});
});
AIProvider.provideAction('makeItReal', options => {
assertExists(options.stream);
const promptName = 'Make it real';
return imageToTextStream({
promptName,
docId: options.docId,
workspaceId: options.workspaceId,
return textToText({
...options,
promptName: 'Make it real',
// @ts-expect-error todo: fix this after blocksuite bump
params: options.params,
attachments: options.attachments,
content:
options.content ||
'Here are the latest wireframes. Could you make a new website based on these wireframes and notes and send back just the html file?',

218
packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts
View File

@@ -1,107 +1,135 @@
import { getBaseUrl } from '@affine/graphql';
import { CopilotClient, toTextStream } from '@blocksuite/presets';
import { toTextStream } from '@blocksuite/presets';
const TIMEOUT = 500000;
import { CopilotClient } from './copilot-client';
import type { PromptKey } from './prompt';
export function textToTextStream({
docId,
workspaceId,
prompt,
attachments,
params,
}: {
const TIMEOUT = 50000;
const client = new CopilotClient();
function readBlobAsURL(blob: Blob) {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = e => {
if (typeof e.target?.result === 'string') {
resolve(e.target.result);
} else {
reject();
}
};
reader.onerror = reject;
reader.readAsDataURL(blob);
});
}
export type TextToTextOptions = {
docId: string;
workspaceId: string;
prompt: string;
attachments?: string[];
params?: string;
}): BlockSuitePresets.TextStream {
const client = new CopilotClient(getBaseUrl());
return {
[Symbol.asyncIterator]: async function* () {
const hasAttachments = attachments && attachments.length > 0;
const session = await client.createSession({
workspaceId,
docId,
promptName: hasAttachments ? 'debug:action:vision4' : 'Summary',
});
if (hasAttachments) {
const messageId = await client.createMessage({
sessionId: session,
content: prompt,
promptName: PromptKey;
content?: string;
attachments?: (string | Blob)[];
params?: Record<string, string>;
timeout?: number;
stream?: boolean;
};
async function createSessionMessage({
docId,
workspaceId,
promptName,
content,
attachments,
params,
}: TextToTextOptions) {
const hasAttachments = attachments && attachments.length > 0;
const session = await client.createSession({
workspaceId,
docId,
promptName,
});
if (hasAttachments) {
const normalizedAttachments = await Promise.all(
attachments.map(async attachment => {
if (typeof attachment === 'string') {
return attachment;
}
const url = await readBlobAsURL(attachment);
return url;
})
);
const messageId = await client.createMessage({
sessionId: session,
content,
attachments: normalizedAttachments,
params,
});
return {
messageId,
session,
};
} else if (content) {
return {
message: content,
session,
};
} else {
throw new Error('No content or attachments provided');
}
}
export function textToText({
docId,
workspaceId,
promptName,
content,
attachments,
params,
stream,
timeout = TIMEOUT,
}: TextToTextOptions) {
if (stream) {
return {
[Symbol.asyncIterator]: async function* () {
const message = await createSessionMessage({
docId,
workspaceId,
promptName,
content,
attachments,
params,
});
const eventSource = client.textStream(messageId, session);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
} else {
const eventSource = client.textToTextStream(prompt, session);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
}
},
};
}
// Image to text(html)
export function imageToTextStream({
docId,
workspaceId,
promptName,
...options
}: {
docId: string;
workspaceId: string;
promptName: string;
params?: string;
content: string;
attachments?: string[];
}) {
const client = new CopilotClient(getBaseUrl());
return {
[Symbol.asyncIterator]: async function* () {
const sessionId = await client.createSession({
workspaceId,
const eventSource = client.chatTextStream({
sessionId: message.session,
messageId: message.messageId,
message: message.message,
});
yield* toTextStream(eventSource, { timeout: timeout });
},
};
} else {
return Promise.race([
timeout
? new Promise((_res, rej) => {
setTimeout(() => {
rej(new Error('Timeout'));
}, timeout);
})
: null,
createSessionMessage({
docId,
promptName,
});
const messageId = await client.createMessage({
sessionId,
...options,
});
const eventSource = client.textStream(messageId, sessionId);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
},
};
}
// Image to images
export function imageToImagesStream({
docId,
workspaceId,
promptName,
...options
}: {
docId: string;
workspaceId: string;
promptName: string;
content: string;
params?: string;
attachments?: string[];
}) {
const client = new CopilotClient(getBaseUrl());
return {
[Symbol.asyncIterator]: async function* () {
const sessionId = await client.createSession({
workspaceId,
docId,
promptName,
});
const messageId = await client.createMessage({
sessionId,
...options,
});
const eventSource = client.imagesStream(messageId, sessionId);
yield* toTextStream(eventSource, { timeout: TIMEOUT });
},
};
content,
attachments,
params,
}).then(message => {
return client.chatText({
sessionId: message.session,
messageId: message.messageId,
message: message.message,
});
}),
]);
}
}

2
packages/frontend/graphql/codegen.yml
View File

@@ -16,7 +16,7 @@ config:
Decimal: number
UUID: string
ID: string
JSON: string
JSON: Record<string, string>
Upload: File
SafeInt: number
overwrite: true

3
packages/frontend/graphql/src/graphql/create-copilot-message.gql Normal file
View File

@@ -0,0 +1,3 @@
mutation createCopilotMessage($options: CreateChatMessageInput!) {
createCopilotMessage(options: $options)
}

11
packages/frontend/graphql/src/graphql/index.ts
View File

@@ -144,6 +144,17 @@ mutation createCheckoutSession($input: CreateCheckoutSessionInput!) {
}`,
};
export const createCopilotMessageMutation = {
id: 'createCopilotMessageMutation' as const,
operationName: 'createCopilotMessage',
definitionName: 'createCopilotMessage',
containsFile: false,
query: `
mutation createCopilotMessage($options: CreateChatMessageInput!) {
createCopilotMessage(options: $options)
}`,
};
export const createCopilotSessionMutation = {
id: 'createCopilotSessionMutation' as const,
operationName: 'createCopilotSession',

16
packages/frontend/graphql/src/schema.ts
View File

@@ -29,7 +29,7 @@ export interface Scalars {
/** A date-time string at UTC, such as 2019-12-03T09:54:33Z, compliant with the date-time format. */
DateTime: { input: string; output: string };
/** The `JSON` scalar type represents JSON values as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). */
JSON: { input: string; output: string };
JSON: { input: Record<string, string>; output: Record<string, string> };
/** The `SafeInt` scalar type represents non-fractional signed whole numeric values that are considered safe as defined by the ECMAScript specification. */
SafeInt: { input: number; output: number };
/** The `Upload` scalar type represents a file upload. */
@@ -240,6 +240,15 @@ export type CreateCheckoutSessionMutation = {
createCheckoutSession: string;
};
export type CreateCopilotMessageMutationVariables = Exact<{
options: CreateChatMessageInput;
}>;
export type CreateCopilotMessageMutation = {
__typename?: 'Mutation';
createCopilotMessage: string;
};
export type CreateCopilotSessionMutationVariables = Exact<{
options: CreateChatSessionInput;
}>;
@@ -1214,6 +1223,11 @@ export type Mutations =
variables: CreateCheckoutSessionMutationVariables;
response: CreateCheckoutSessionMutation;
}
| {
name: 'createCopilotMessageMutation';
variables: CreateCopilotMessageMutationVariables;
response: CreateCopilotMessageMutation;
}
| {
name: 'createCopilotSessionMutation';
variables: CreateCopilotSessionMutationVariables;