From 3794f8aa8085937130ff2c847338b8f92d7d25f6 Mon Sep 17 00:00:00 2001 From: darkskygit Date: Tue, 22 Apr 2025 09:23:24 +0000 Subject: [PATCH] feat(server): find transcript actions (#11890) --- .../__tests__/__snapshots__/copilot.e2e.ts.md | 2 + .../__snapshots__/copilot.e2e.ts.snap | Bin 987 -> 1011 bytes .../server/src/__tests__/utils/copilot.ts | 2 + .../src/plugins/copilot/prompt/prompts.ts | 59 ++++++++++++++---- .../plugins/copilot/transcript/resolver.ts | 5 ++ .../src/plugins/copilot/transcript/service.ts | 30 ++++++++- .../src/plugins/copilot/transcript/types.ts | 4 ++ packages/backend/server/src/schema.gql | 1 + .../copilot-jobs-transcription-claim.gql | 1 + packages/common/graphql/src/graphql/index.ts | 1 + packages/common/graphql/src/schema.ts | 2 + 11 files changed, 94 insertions(+), 13 deletions(-) diff --git a/packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.md b/packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.md index 7a37e06810..448c65cc1e 100644 --- a/packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.md +++ b/packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.md @@ -69,6 +69,7 @@ Generated by [AVA](https://avajs.dev). [ { + actions: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]', status: 'claimed', summary: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]', title: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]', @@ -101,6 +102,7 @@ Generated by [AVA](https://avajs.dev). [ { + actions: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]', status: 'claimed', summary: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]', title: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]', diff --git a/packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.snap b/packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.snap index 747545f2ce8780fe4ec1e6112082f13cf481e922..53e83ab6981c6c8c3e959e6dc342f9929271c841 100644 GIT binary patch literal 1011 zcmVANH_tLXJ)KO=npxSE zU1aAnubTI|zxTbbzIVGDBr+VRx9qP7_)t5yQu|O~$B!X$la-s7Pdo$L3#)omN zGCov&snxAJQN%TWjf9OQjDOT1s|nx&fOF)XC3iE=`G{y5t*)+umO%tsA$nn2WcL81 z0N$MzdfUN{2sK%CxVE+iIt{WwXNW!p@EF-3_bfG1p%m{41zN>9vZl{HEe9e~<AX_4?iB)iYe4=9P1X8l-cnU5InYLQQ7b~2x1KA(TSb1BbwPp}f+uHNI$ ztsHcwt^`m3_yoXR0KYGutz%9wNMx`T)oga-L~KQEBkNSyFS5bQ@mr!E^(>zmPisDx z6;FqWSRY1R^J7YLU8p*7n9EQo6=%JQ`KKxxOS&uO*cg(UkoM~P2j$EP0mO@WA_KY= z*Bp6~47Z|o1^LOnAV)G(Q)xUsFAbgZOoe$od`K4WR2_bFR2>?_S;~tndoY>vRTruV z(pWQIMDa`?GGjY3sHE-OK4=r30x&8a26cuqoGc2Of^G)m`0q{f5u|j=$+Q zedTv9wwk{1I~$jpzV|a(}ad z`!>*UfM*?G+c8Sqg%)j#toEKptJP_>IvbbfWHJjpn&oZ! zbj_UJ{$RE{ZS(zeFt6UIDC literal 987 zcmV<110?)GRzV%#P za>@TpXCI3Q00000000B+Rl$oJRT%$$Z)URDUAnuaiV77UgGG?ICf#&}Ae1ds+>?R_ z74h&g^ODSFGB3<~(Zte=6@{XB6@-GtKcFWs-t^Q{@#s}g9u#k0C6k%dysVo@SGKg9 zOTKUTzM0?m`+e{CzG*dM6?FBD+tNfPmP4*PY7mFKDYuD5m6CsZf=6sKm=MRdbMbB+5mb0 zJ}VkaZ!`~NV2Dk_t*tH484v?)5Pb{a335Q*St|FW)}k#nXcOm%70*jXb!DQn)SZC4 z3*epw&N&?1&r@(N0c!+YCJUT<7`Pb-D}hRkOiYnK*_Me+g^}FIp&64q{DL zR)yA)HX<=H7EKusUz`^xjO25)E%Idkg| zs;Q>{3;=u$;CleSEY{Yw6>P?;xgSp1Y(=r$4{KJ{YS0-Z-M6BTTK^0}r( zuOG|pe%P`s(HIb#8UXfaP@L1)pF zqf)4X{jiope)=%Tp$c@d8t3O%!=xh7K^pb{u@;}?6MlJ86I$Y|q=O{6ugryf3O#7{ zBE!WXjArIg2|rNHe6?M6tgF3Cz&+a;mRN0Tpdc1ePlmRC=s~Y1(ovCLd+(Yr{D!~h zSA6X^cB&O$`i=Ufif{ae|GJE0RpIgjnT}KqL?v)4xpKR}8qtfFBs}D+5-Yqwj+2DLwNb4$>X+u$q%TERrbSMBwpiMT)L5_CsajRDycJKkZ1t9dS!>kn_s_w+vJA}n z?ou!-|7B}(-*`UvjqLCpZ5xk-^?84uTHoVlecp2nxWIsy8PH(B)kPNQW|F@-wZI#> z1+Ei)Hx4Hj`P(>jvM-!>0N(*#bAYx3e6iRVe{_Jm4)D7JY`8$(J=z${v z7x>8q{<2$2b1g4WvJ?Eym%xaC String, { nullable: true }) summary!: string | null; + @Field(() => String, { nullable: true }) + actions!: string | null; + @Field(() => [TranscriptionItemType], { nullable: true }) transcription!: TranscriptionItemType[] | null; @@ -84,11 +87,13 @@ export class CopilotTranscriptionResolver { status, title: null, summary: null, + actions: null, transcription: null, }; if (FinishedStatus.has(finalJob.status)) { finalJob.title = ret?.title || null; finalJob.summary = ret?.summary || null; + finalJob.actions = ret?.actions || null; finalJob.transcription = ret?.transcription || null; } return finalJob; diff --git a/packages/backend/server/src/plugins/copilot/transcript/service.ts b/packages/backend/server/src/plugins/copilot/transcript/service.ts index c4650b0111..25efbf4cb7 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/service.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts @@ -283,7 +283,7 @@ export class CopilotTranscriptionService { .trim(); if (content.length) { - payload.summary = await this.chatWithPrompt('Summary', { + payload.summary = await this.chatWithPrompt('Summarize the meeting', { content, }); await this.models.copilotJob.update(jobId, { @@ -328,7 +328,7 @@ export class CopilotTranscriptionService { await this.models.copilotJob.update(jobId, { payload, }); - this.event.emit('workspace.file.transcript.finished', { + await this.job.add('copilot.transcript.findAction.submit', { jobId, }); return; @@ -346,6 +346,32 @@ export class CopilotTranscriptionService { } } + @OnJob('copilot.transcript.findAction.submit') + async transcriptFindAction({ + jobId, + }: Jobs['copilot.transcript.findAction.submit']) { + try { + const payload = await this.models.copilotJob.getPayload( + jobId, + TranscriptPayloadSchema + ); + if (payload.summary) { + const actions = await this.chatWithPrompt('Find action for summary', { + content: payload.summary, + }).then(a => a.trim()); + if (actions) { + payload.actions = actions; + await this.models.copilotJob.update(jobId, { + payload, + }); + } + } + } catch {} // finish even if failed + this.event.emit('workspace.file.transcript.finished', { + jobId, + }); + } + @OnEvent('workspace.file.transcript.finished') async onFileTranscriptFinish({ jobId, diff --git a/packages/backend/server/src/plugins/copilot/transcript/types.ts b/packages/backend/server/src/plugins/copilot/transcript/types.ts index 1c5c320568..77e2a7ca51 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/types.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/types.ts @@ -33,6 +33,7 @@ export const TranscriptPayloadSchema = z.object({ infos: AudioBlobInfosSchema.nullable().optional(), title: z.string().nullable().optional(), summary: z.string().nullable().optional(), + actions: z.string().nullable().optional(), transcription: TranscriptionSchema.nullable().optional(), }); @@ -66,6 +67,9 @@ declare global { 'copilot.transcript.title.submit': { jobId: string; }; + 'copilot.transcript.findAction.submit': { + jobId: string; + }; } } diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 7f429a5c23..f8efd8f56d 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -1430,6 +1430,7 @@ type TranscriptionItemType { } type TranscriptionResultType { + actions: String id: ID! status: AiJobStatus! summary: String diff --git a/packages/common/graphql/src/graphql/copilot-jobs-transcription-claim.gql b/packages/common/graphql/src/graphql/copilot-jobs-transcription-claim.gql index 437e99824e..aa60ebe8a0 100644 --- a/packages/common/graphql/src/graphql/copilot-jobs-transcription-claim.gql +++ b/packages/common/graphql/src/graphql/copilot-jobs-transcription-claim.gql @@ -4,6 +4,7 @@ mutation claimAudioTranscription($jobId: String!) { status title summary + actions transcription { speaker start diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index b7ecf013bb..61348b9b4b 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -624,6 +624,7 @@ export const claimAudioTranscriptionMutation = { status title summary + actions transcription { speaker start diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index 528e28a8d5..424785551c 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -1935,6 +1935,7 @@ export interface TranscriptionItemType { export interface TranscriptionResultType { __typename?: 'TranscriptionResultType'; + actions: Maybe; id: Scalars['ID']['output']; status: AiJobStatus; summary: Maybe; @@ -3029,6 +3030,7 @@ export type ClaimAudioTranscriptionMutation = { status: AiJobStatus; title: string | null; summary: string | null; + actions: string | null; transcription: Array<{ __typename?: 'TranscriptionItemType'; speaker: string;