feat(core): ai images (#6506)

2026-07-29 16:19:43 +08:00 · 2024-04-12 03:58:32 +00:00
parent 1697cd76fe
commit 13b39fc5f3
4 changed files with 292 additions and 7 deletions
@@ -1,7 +1,7 @@
 import { assertExists } from '@blocksuite/global/utils';
 import { AIProvider } from '@blocksuite/presets';

-import { textToTextStream } from './request';
+import { imageToTextStream, textToTextStream } from './request';

 export function setupAIProvider() {
  AIProvider.provideAction('chat', options => {
@@ -31,7 +31,10 @@ export function setupAIProvider() {

  AIProvider.provideAction('translate', options => {
    assertExists(options.stream);
-    const prompt = `Translate the following content to ${options.lang}: ${options.input}`;
+    const prompt = `Please translate the following content into ${options.lang} and return it to us, adhering to the original format of the content：
+
+    ${options.input}
+    `;
    return textToTextStream({
      docId: options.docId,
      workspaceId: options.workspaceId,
@@ -111,11 +114,174 @@ export function setupAIProvider() {

  AIProvider.provideAction('checkCodeErrors', options => {
    assertExists(options.stream);
-    const prompt = `Check the code errors in the following content: ${options.input}`;
+    const prompt = `Check the code errors in the following content and provide the corrected version:
+
+    ${options.input}
+    `;
    return textToTextStream({
      docId: options.docId,
      workspaceId: options.workspaceId,
      prompt,
    });
  });
+
+  AIProvider.provideAction('explainCode', options => {
+    assertExists(options.stream);
+    const prompt = `Explain the code in the following content, focusing on the logic, functions, and expected outcomes:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeArticle', options => {
+    assertExists(options.stream);
+    const prompt = `Write an article based on the following content, focusing on the main ideas, structure, and flow:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeTwitterPost', options => {
+    assertExists(options.stream);
+    const prompt = `Write a Twitter post based on the following content, keeping it concise and engaging:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writePoem', options => {
+    assertExists(options.stream);
+    const prompt = `Write a poem based on the following content, focusing on the emotions, imagery, and rhythm:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeOutline', options => {
+    assertExists(options.stream);
+    const prompt = `Write an outline from the following content in Markdown: ${options.input}`;
+
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeBlogPost', options => {
+    assertExists(options.stream);
+    const prompt = `Write a blog post based on the following content, focusing on the insights, analysis, and personal perspective:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('brainstorm', options => {
+    assertExists(options.stream);
+    const prompt = `Brainstorm ideas based on the following content, exploring different angles, perspectives, and approaches:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('findActions', options => {
+    assertExists(options.stream);
+    const prompt = `Find actions related to the following content and return content in markdown: ${options.input}`;
+
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeOutline', options => {
+    assertExists(options.stream);
+    const prompt = `Write an outline based on the following content, organizing the main points, subtopics, and structure:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('brainstormMindmap', options => {
+    assertExists(options.stream);
+    const prompt = `Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: ${options.input}`;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('explain', options => {
+    assertExists(options.stream);
+    const prompt = `Explain the following content in Markdown: ${options.input}`;
+
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('explainImage', options => {
+    assertExists(options.stream);
+    const prompt = `Describe the scene captured in this image, focusing on the details, colors, emotions, and any interactions between subjects or objects present.`;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+      attachments: options.attachments,
+    });
+  });
+
+  AIProvider.provideAction('makeItReal', options => {
+    assertExists(options.stream);
+    const promptName = 'Make it real';
+    return imageToTextStream({
+      promptName,
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      params: options.params,
+      attachments: options.attachments,
+      content:
+        options.content ||
+        'Here are the latest wireframes. Could you make a new website based on these wireframes and notes and send back just the html file?',
+    });
+  });
 }
@@ -1,26 +1,106 @@
 import { getBaseUrl } from '@affine/graphql';
 import { CopilotClient, toTextStream } from '@blocksuite/presets';

-const TIMEOUT = 5000;
+const TIMEOUT = 500000;

 export function textToTextStream({
  docId,
  workspaceId,
  prompt,
+  attachments,
+  params,
 }: {
  docId: string;
  workspaceId: string;
  prompt: string;
+  attachments?: string[];
+  params?: string;
 }): BlockSuitePresets.TextStream {
  const client = new CopilotClient(getBaseUrl());
  return {
    [Symbol.asyncIterator]: async function* () {
+      const hasAttachments = attachments && attachments.length > 0;
      const session = await client.createSession({
        workspaceId,
        docId,
-        promptName: 'Summary', // placeholder
+        promptName: hasAttachments ? 'debug:action:vision4' : 'Summary',
      });
-      const eventSource = client.textToTextStream(prompt, session);
+      if (hasAttachments) {
+        const messageId = await client.createMessage({
+          sessionId: session,
+          content: prompt,
+          attachments,
+          params,
+        });
+        const eventSource = client.textStream(messageId, session);
+        yield* toTextStream(eventSource, { timeout: TIMEOUT });
+      } else {
+        const eventSource = client.textToTextStream(prompt, session);
+        yield* toTextStream(eventSource, { timeout: TIMEOUT });
+      }
+    },
+  };
+}
+
+// Image to text(html)
+export function imageToTextStream({
+  docId,
+  workspaceId,
+  promptName,
+  ...options
+}: {
+  docId: string;
+  workspaceId: string;
+  promptName: string;
+  params?: string;
+  content: string;
+  attachments?: string[];
+}) {
+  const client = new CopilotClient(getBaseUrl());
+  return {
+    [Symbol.asyncIterator]: async function* () {
+      const sessionId = await client.createSession({
+        workspaceId,
+        docId,
+        promptName,
+      });
+      const messageId = await client.createMessage({
+        sessionId,
+        ...options,
+      });
+      const eventSource = client.textStream(messageId, sessionId);
+      yield* toTextStream(eventSource, { timeout: TIMEOUT });
+    },
+  };
+}
+
+// Image to images
+export function imageToImagesStream({
+  docId,
+  workspaceId,
+  promptName,
+  ...options
+}: {
+  docId: string;
+  workspaceId: string;
+  promptName: string;
+  content: string;
+  params?: string;
+  attachments?: string[];
+}) {
+  const client = new CopilotClient(getBaseUrl());
+  return {
+    [Symbol.asyncIterator]: async function* () {
+      const sessionId = await client.createSession({
+        workspaceId,
+        docId,
+        promptName,
+      });
+      const messageId = await client.createMessage({
+        sessionId,
+        ...options,
+      });
+      const eventSource = client.imagesStream(messageId, sessionId);
      yield* toTextStream(eventSource, { timeout: TIMEOUT });
    },
  };