From ebf1d5476dd0542b94099d900ed32b757e4da2f3 Mon Sep 17 00:00:00 2001 From: pengx17 Date: Thu, 17 Apr 2025 06:33:04 +0000 Subject: [PATCH] feat(core): add summary to transcription block (#11753) fix AF-2523 --- .../blocks/callout/src/callout-block.ts | 12 +- .../ai-chat-block/ai-transcription-block.ts | 17 +-- .../core/src/blocksuite/ai/slides/index.ts | 4 +- .../src/blocksuite/ai/utils/editor-actions.ts | 6 +- .../src/blocksuite/utils/markdown-utils.ts | 25 ++-- .../media/entities/audio-attachment-block.ts | 117 ++++++++++++------ 6 files changed, 111 insertions(+), 70 deletions(-) diff --git a/blocksuite/affine/blocks/callout/src/callout-block.ts b/blocksuite/affine/blocks/callout/src/callout-block.ts index 5e09c52368..dd42e39ddd 100644 --- a/blocksuite/affine/blocks/callout/src/callout-block.ts +++ b/blocksuite/affine/blocks/callout/src/callout-block.ts @@ -21,16 +21,21 @@ export class CalloutBlockComponent extends CaptionedBlockComponent { - // if model is the last transcription block, we should render it - return model === this.lastCalloutBlock; - }); + return this.std.host.renderChildren(this.model); } @property({ type: String, attribute: 'data-block-id' }) diff --git a/packages/frontend/core/src/blocksuite/ai/slides/index.ts b/packages/frontend/core/src/blocksuite/ai/slides/index.ts index bf2cf6e9b9..2788c6d373 100644 --- a/packages/frontend/core/src/blocksuite/ai/slides/index.ts +++ b/packages/frontend/core/src/blocksuite/ai/slides/index.ts @@ -56,7 +56,7 @@ export const PPTBuilder = (host: EditorHost) => { return { process: async (text: string) => { try { - const snapshot = await markdownToSnapshot(text, host); + const snapshot = await markdownToSnapshot(text, host.doc, host); const block = snapshot.snapshot?.content[0]; if (!block) { @@ -73,7 +73,7 @@ export const PPTBuilder = (host: EditorHost) => { return { contents, images: allImages }; }, done: async (text: string) => { - const snapshot = await markdownToSnapshot(text, host); + const snapshot = await markdownToSnapshot(text, host.doc, host); const block = snapshot.snapshot?.content[0]; if (!block) { return; diff --git a/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts b/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts index 9f6ab7c2e1..08d68578f7 100644 --- a/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts +++ b/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts @@ -110,7 +110,11 @@ export const replace = async ( if (textSelection) { host.std.command.exec(deleteTextCommand, { textSelection }); - const { snapshot, transformer } = await markdownToSnapshot(content, host); + const { snapshot, transformer } = await markdownToSnapshot( + content, + host.doc, + host + ); if (snapshot) { await transformer.snapshotToSlice( snapshot, diff --git a/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts b/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts index 5e2c64e737..c73236bbf4 100644 --- a/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts +++ b/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts @@ -102,18 +102,19 @@ export async function getContentFromSlice( export const markdownToSnapshot = async ( markdown: string, - host: EditorHost + store: Store, + host?: EditorHost ) => { - const transformer = host.std.store.getTransformer([ - defaultImageProxyMiddleware, - pasteMiddleware(host.std), - ]); - const markdownAdapter = new MixTextAdapter(transformer, host.std.provider); + const middlewares = host + ? [defaultImageProxyMiddleware, pasteMiddleware(host.std)] + : [defaultImageProxyMiddleware]; + const transformer = store.getTransformer(middlewares); + const markdownAdapter = new MixTextAdapter(transformer, store.provider); const payload = { file: markdown, assets: transformer.assetsManager, - workspaceId: host.std.store.workspace.id, - pageId: host.std.store.id, + workspaceId: store.workspace.id, + pageId: store.id, }; const snapshot = await markdownAdapter.toSliceSnapshot(payload); @@ -125,13 +126,17 @@ export const markdownToSnapshot = async ( }; export async function insertFromMarkdown( - host: EditorHost, + host: EditorHost | undefined, markdown: string, doc: Store, parent?: string, index?: number ) { - const { snapshot, transformer } = await markdownToSnapshot(markdown, host); + const { snapshot, transformer } = await markdownToSnapshot( + markdown, + doc, + host + ); const snapshots = snapshot?.content.flatMap(x => x.children) ?? []; diff --git a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts index 531432947c..45bbf37bfa 100644 --- a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts +++ b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts @@ -1,3 +1,4 @@ +import { insertFromMarkdown } from '@affine/core/blocksuite/utils'; import { encodeAudioBlobToOpusSlices } from '@affine/core/utils/webm-encoding'; import { DebugLogger } from '@affine/debug'; import { AiJobStatus } from '@affine/graphql'; @@ -26,6 +27,18 @@ function sanitizeText(text: string) { return text.replace(/\r/g, ''); } +const colorOptions = [ + cssVarV2.text.highlight.fg.red, + cssVarV2.text.highlight.fg.green, + cssVarV2.text.highlight.fg.blue, + cssVarV2.text.highlight.fg.yellow, + cssVarV2.text.highlight.fg.purple, + cssVarV2.text.highlight.fg.orange, + cssVarV2.text.highlight.fg.teal, + cssVarV2.text.highlight.fg.grey, + cssVarV2.text.highlight.fg.magenta, +]; + export class AudioAttachmentBlock extends Entity { private readonly refCount$ = new LiveData(0); readonly audioMedia: AudioMedia; @@ -142,7 +155,7 @@ export class AudioAttachmentBlock extends Entity { } const status = await this.transcriptionJob.start(); if (status.status === AiJobStatus.claimed) { - this.fillTranscriptionResult(status.result); + await this.fillTranscriptionResult(status.result); } } catch (error) { track.doc.editor.audioBlock.transcribeRecording({ @@ -154,55 +167,77 @@ export class AudioAttachmentBlock extends Entity { } }; - private readonly fillTranscriptionResult = (result: TranscriptionResult) => { + private readonly fillTranscriptionResult = async ( + result: TranscriptionResult + ) => { this.props.props.caption = result.title ?? ''; - const calloutId = this.props.doc.addBlock( - 'affine:callout', - { - emoji: '💬', - }, - this.transcriptionBlock$.value?.id - ); - - // todo: refactor - const speakerToColors = new Map(); - for (const segment of result.segments) { - let color = speakerToColors.get(segment.speaker); - const colorOptions = [ - cssVarV2.text.highlight.fg.red, - cssVarV2.text.highlight.fg.green, - cssVarV2.text.highlight.fg.blue, - cssVarV2.text.highlight.fg.yellow, - cssVarV2.text.highlight.fg.purple, - cssVarV2.text.highlight.fg.orange, - cssVarV2.text.highlight.fg.teal, - cssVarV2.text.highlight.fg.grey, - cssVarV2.text.highlight.fg.magenta, - ]; - if (!color) { - color = colorOptions[speakerToColors.size % colorOptions.length]; - speakerToColors.set(segment.speaker, color); - } - const deltaInserts: DeltaInsert[] = [ + const addCalloutBlock = (emoji: string, title: string) => { + const calloutId = this.props.doc.addBlock( + 'affine:callout', { - insert: sanitizeText(segment.start + ' ' + segment.speaker), - attributes: { - color, - bold: true, - }, + emoji, }, - { - insert: ': ' + sanitizeText(segment.transcription), - }, - ]; + this.transcriptionBlock$.value?.id + ); this.props.doc.addBlock( 'affine:paragraph', { - text: new Text(deltaInserts), + type: 'h6', + text: new Text([ + { + insert: title, + }, + ]), }, calloutId ); - } + return calloutId; + }; + const fillTranscription = (segments: TranscriptionResult['segments']) => { + const calloutId = addCalloutBlock('💬', 'Transcript'); + + const speakerToColors = new Map(); + for (const segment of segments) { + let color = speakerToColors.get(segment.speaker); + if (!color) { + color = colorOptions[speakerToColors.size % colorOptions.length]; + speakerToColors.set(segment.speaker, color); + } + const deltaInserts: DeltaInsert[] = [ + { + insert: sanitizeText(segment.start + ' ' + segment.speaker), + attributes: { + color, + bold: true, + }, + }, + { + insert: ': ' + sanitizeText(segment.transcription), + }, + ]; + this.props.doc.addBlock( + 'affine:paragraph', + { + text: new Text(deltaInserts), + }, + calloutId + ); + } + }; + + const fillSummary = async (summary: TranscriptionResult['summary']) => { + const calloutId = addCalloutBlock('📑', 'Summary'); + await insertFromMarkdown( + undefined, + summary, + this.props.doc, + calloutId, + 1 + ); + }; + + fillTranscription(result.segments); + await fillSummary(result.summary); }; }