feat(core): add summary to transcription block (#11753)

fix AF-2523

blocksuite/affine/blocks/callout/src/callout-block.ts

@@ -21,16 +21,21 @@ export class CalloutBlockComponent extends CaptionedBlockComponent<CalloutBlockM
 
     .affine-callout-block-container {
       display: flex;
-      padding: 12px 16px;
+      padding: 5px 10px;
       border-radius: 8px;
       background-color: ${unsafeCSSVarV2('block/callout/background/grey')};
     }
 
     .affine-callout-emoji-container {
-      margin-right: 12px;
+      margin-right: 10px;
-      margin-top: 10px;
+      margin-top: 14px;
       user-select: none;
       font-size: 1.2em;
+      width: 24px;
+      height: 24px;
+      display: flex;
+      align-items: center;
+      justify-content: center;
     }
     .affine-callout-emoji:hover {
       cursor: pointer;
@@ -40,6 +45,7 @@ export class CalloutBlockComponent extends CaptionedBlockComponent<CalloutBlockM
     .affine-callout-children {
       flex: 1;
       min-width: 0;
+      padding-left: 10px;
     }
   `;
 

packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts

@@ -10,24 +10,15 @@ export class LitTranscriptionBlock extends BlockComponent<TranscriptionBlockMode
     css`
       transcription-block {
         outline: none;
+        display: flex;
+        flex-direction: column;
+        gap: 4px;
       }
     `,
   ];
 
-  get lastCalloutBlock() {
-    for (const child of this.model.children.toReversed()) {
-      if (child.flavour === 'affine:callout') {
-        return child;
-      }
-    }
-    return null;
-  }
-
   override render() {
-    return this.std.host.renderChildren(this.model, model => {
+    return this.std.host.renderChildren(this.model);
-      // if model is the last transcription block, we should render it
-      return model === this.lastCalloutBlock;
-    });
   }
 
   @property({ type: String, attribute: 'data-block-id' })

packages/frontend/core/src/blocksuite/ai/slides/index.ts

@@ -56,7 +56,7 @@ export const PPTBuilder = (host: EditorHost) => {
   return {
     process: async (text: string) => {
       try {
-        const snapshot = await markdownToSnapshot(text, host);
+        const snapshot = await markdownToSnapshot(text, host.doc, host);
 
         const block = snapshot.snapshot?.content[0];
         if (!block) {
@@ -73,7 +73,7 @@ export const PPTBuilder = (host: EditorHost) => {
       return { contents, images: allImages };
     },
     done: async (text: string) => {
-      const snapshot = await markdownToSnapshot(text, host);
+      const snapshot = await markdownToSnapshot(text, host.doc, host);
       const block = snapshot.snapshot?.content[0];
       if (!block) {
         return;

packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts

@@ -110,7 +110,11 @@ export const replace = async (
 
   if (textSelection) {
     host.std.command.exec(deleteTextCommand, { textSelection });
-    const { snapshot, transformer } = await markdownToSnapshot(content, host);
+    const { snapshot, transformer } = await markdownToSnapshot(
+      content,
+      host.doc,
+      host
+    );
     if (snapshot) {
       await transformer.snapshotToSlice(
         snapshot,

packages/frontend/core/src/blocksuite/utils/markdown-utils.ts

@@ -102,18 +102,19 @@ export async function getContentFromSlice(
 
 export const markdownToSnapshot = async (
   markdown: string,
-  host: EditorHost
+  store: Store,
+  host?: EditorHost
 ) => {
-  const transformer = host.std.store.getTransformer([
+  const middlewares = host
-    defaultImageProxyMiddleware,
+    ? [defaultImageProxyMiddleware, pasteMiddleware(host.std)]
-    pasteMiddleware(host.std),
+    : [defaultImageProxyMiddleware];
-  ]);
+  const transformer = store.getTransformer(middlewares);
-  const markdownAdapter = new MixTextAdapter(transformer, host.std.provider);
+  const markdownAdapter = new MixTextAdapter(transformer, store.provider);
   const payload = {
     file: markdown,
     assets: transformer.assetsManager,
-    workspaceId: host.std.store.workspace.id,
+    workspaceId: store.workspace.id,
-    pageId: host.std.store.id,
+    pageId: store.id,
   };
 
   const snapshot = await markdownAdapter.toSliceSnapshot(payload);
@@ -125,13 +126,17 @@ export const markdownToSnapshot = async (
 };
 
 export async function insertFromMarkdown(
-  host: EditorHost,
+  host: EditorHost | undefined,
   markdown: string,
   doc: Store,
   parent?: string,
   index?: number
 ) {
-  const { snapshot, transformer } = await markdownToSnapshot(markdown, host);
+  const { snapshot, transformer } = await markdownToSnapshot(
+    markdown,
+    doc,
+    host
+  );
 
   const snapshots = snapshot?.content.flatMap(x => x.children) ?? [];
 

packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts

@@ -1,3 +1,4 @@
+import { insertFromMarkdown } from '@affine/core/blocksuite/utils';
 import { encodeAudioBlobToOpusSlices } from '@affine/core/utils/webm-encoding';
 import { DebugLogger } from '@affine/debug';
 import { AiJobStatus } from '@affine/graphql';
@@ -26,6 +27,18 @@ function sanitizeText(text: string) {
   return text.replace(/\r/g, '');
 }
 
+const colorOptions = [
+  cssVarV2.text.highlight.fg.red,
+  cssVarV2.text.highlight.fg.green,
+  cssVarV2.text.highlight.fg.blue,
+  cssVarV2.text.highlight.fg.yellow,
+  cssVarV2.text.highlight.fg.purple,
+  cssVarV2.text.highlight.fg.orange,
+  cssVarV2.text.highlight.fg.teal,
+  cssVarV2.text.highlight.fg.grey,
+  cssVarV2.text.highlight.fg.magenta,
+];
+
 export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
   private readonly refCount$ = new LiveData<number>(0);
   readonly audioMedia: AudioMedia;
@@ -142,7 +155,7 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
       }
       const status = await this.transcriptionJob.start();
       if (status.status === AiJobStatus.claimed) {
-        this.fillTranscriptionResult(status.result);
+        await this.fillTranscriptionResult(status.result);
       }
     } catch (error) {
       track.doc.editor.audioBlock.transcribeRecording({
@@ -154,55 +167,77 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
     }
   };
 
-  private readonly fillTranscriptionResult = (result: TranscriptionResult) => {
+  private readonly fillTranscriptionResult = async (
+    result: TranscriptionResult
+  ) => {
     this.props.props.caption = result.title ?? '';
 
-    const calloutId = this.props.doc.addBlock(
+    const addCalloutBlock = (emoji: string, title: string) => {
-      'affine:callout',
+      const calloutId = this.props.doc.addBlock(
-      {
+        'affine:callout',
-        emoji: '💬',
-      },
-      this.transcriptionBlock$.value?.id
-    );
-
-    // todo: refactor
-    const speakerToColors = new Map<string, string>();
-    for (const segment of result.segments) {
-      let color = speakerToColors.get(segment.speaker);
-      const colorOptions = [
-        cssVarV2.text.highlight.fg.red,
-        cssVarV2.text.highlight.fg.green,
-        cssVarV2.text.highlight.fg.blue,
-        cssVarV2.text.highlight.fg.yellow,
-        cssVarV2.text.highlight.fg.purple,
-        cssVarV2.text.highlight.fg.orange,
-        cssVarV2.text.highlight.fg.teal,
-        cssVarV2.text.highlight.fg.grey,
-        cssVarV2.text.highlight.fg.magenta,
-      ];
-      if (!color) {
-        color = colorOptions[speakerToColors.size % colorOptions.length];
-        speakerToColors.set(segment.speaker, color);
-      }
-      const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [
         {
-          insert: sanitizeText(segment.start + ' ' + segment.speaker),
+          emoji,
-          attributes: {
-            color,
-            bold: true,
-          },
         },
-        {
+        this.transcriptionBlock$.value?.id
-          insert: ': ' + sanitizeText(segment.transcription),
+      );
-        },
-      ];
       this.props.doc.addBlock(
         'affine:paragraph',
         {
-          text: new Text(deltaInserts),
+          type: 'h6',
+          text: new Text([
+            {
+              insert: title,
+            },
+          ]),
         },
         calloutId
       );
-    }
+      return calloutId;
+    };
+    const fillTranscription = (segments: TranscriptionResult['segments']) => {
+      const calloutId = addCalloutBlock('💬', 'Transcript');
+
+      const speakerToColors = new Map<string, string>();
+      for (const segment of segments) {
+        let color = speakerToColors.get(segment.speaker);
+        if (!color) {
+          color = colorOptions[speakerToColors.size % colorOptions.length];
+          speakerToColors.set(segment.speaker, color);
+        }
+        const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [
+          {
+            insert: sanitizeText(segment.start + ' ' + segment.speaker),
+            attributes: {
+              color,
+              bold: true,
+            },
+          },
+          {
+            insert: ': ' + sanitizeText(segment.transcription),
+          },
+        ];
+        this.props.doc.addBlock(
+          'affine:paragraph',
+          {
+            text: new Text(deltaInserts),
+          },
+          calloutId
+        );
+      }
+    };
+
+    const fillSummary = async (summary: TranscriptionResult['summary']) => {
+      const calloutId = addCalloutBlock('📑', 'Summary');
+      await insertFromMarkdown(
+        undefined,
+        summary,
+        this.props.doc,
+        calloutId,
+        1
+      );
+    };
+
+    fillTranscription(result.segments);
+    await fillSummary(result.summary);
   };
 }