mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-17 10:06:17 +08:00
feat(server): refactor copilot (#14892)
#### PR Dependency Tree * **PR #14892** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal)
This commit is contained in:
@@ -152,7 +152,6 @@ declare global {
|
||||
selectedMarkdown?: string;
|
||||
html?: string;
|
||||
};
|
||||
postfix?: (text: string) => string;
|
||||
}
|
||||
|
||||
interface TranslateOptions extends AITextActionOptions {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* @vitest-environment happy-dom
|
||||
*/
|
||||
import { describe, expect, test, vi } from 'vitest';
|
||||
|
||||
import { CopilotClient, Endpoint } from './copilot-client';
|
||||
|
||||
describe('CopilotClient action streams', () => {
|
||||
test('routes action endpoint outside the deprecated workflow path', () => {
|
||||
const eventSource = vi.fn(
|
||||
() =>
|
||||
({
|
||||
close: vi.fn(),
|
||||
}) as unknown as EventSource
|
||||
);
|
||||
const client = new CopilotClient(vi.fn(), eventSource);
|
||||
|
||||
client.chatTextStream(
|
||||
{
|
||||
sessionId: 'session-1',
|
||||
messageId: 'message-1',
|
||||
actionId: 'mindmap.generate',
|
||||
actionVersion: 'v1',
|
||||
retry: true,
|
||||
runId: 'run-1',
|
||||
},
|
||||
Endpoint.Action
|
||||
);
|
||||
|
||||
expect(eventSource).toHaveBeenCalledWith(
|
||||
'/api/copilot/actions/session-1/stream?messageId=message-1&actionId=mindmap.generate&actionVersion=v1&runId=run-1&retry=true'
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -42,8 +42,8 @@ import {
|
||||
} from './error';
|
||||
|
||||
export enum Endpoint {
|
||||
Action = 'action',
|
||||
StreamObject = 'stream-object',
|
||||
Workflow = 'workflow',
|
||||
Images = 'images',
|
||||
}
|
||||
|
||||
@@ -469,21 +469,36 @@ export class CopilotClient {
|
||||
reasoning,
|
||||
modelId,
|
||||
toolsConfig,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
retry,
|
||||
}: {
|
||||
sessionId: string;
|
||||
messageId?: string;
|
||||
reasoning?: boolean;
|
||||
modelId?: string;
|
||||
toolsConfig?: AIToolsConfig;
|
||||
actionId?: string;
|
||||
actionVersion?: string;
|
||||
runId?: string;
|
||||
retry?: boolean;
|
||||
},
|
||||
endpoint = Endpoint.StreamObject
|
||||
) {
|
||||
let url = `/api/copilot/chat/${sessionId}/${endpoint}`;
|
||||
let url =
|
||||
endpoint === Endpoint.Action
|
||||
? `/api/copilot/actions/${sessionId}/stream`
|
||||
: `/api/copilot/chat/${sessionId}/${endpoint}`;
|
||||
const queryString = this.paramsToQueryString({
|
||||
messageId,
|
||||
reasoning,
|
||||
modelId,
|
||||
toolsConfig,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
retry,
|
||||
});
|
||||
if (queryString) {
|
||||
url += `?${queryString}`;
|
||||
|
||||
@@ -39,9 +39,13 @@ export const promptKeys = [
|
||||
'Convert to sticker',
|
||||
'Upscale image',
|
||||
'Remove background',
|
||||
// workflows
|
||||
'workflow:presentation',
|
||||
'workflow:brainstorm',
|
||||
// actions
|
||||
'mindmap.generate',
|
||||
'slides.outline',
|
||||
'image.filter.sketch',
|
||||
'image.filter.clay',
|
||||
'image.filter.anime',
|
||||
'image.filter.pixel',
|
||||
] as const;
|
||||
|
||||
export type PromptKey = (typeof promptKeys)[number];
|
||||
|
||||
@@ -18,8 +18,10 @@ export type TextToTextOptions = {
|
||||
signal?: AbortSignal;
|
||||
retry?: boolean;
|
||||
endpoint?: Endpoint;
|
||||
actionId?: string;
|
||||
actionVersion?: string;
|
||||
runId?: string;
|
||||
isRootSession?: boolean;
|
||||
postfix?: (text: string) => string;
|
||||
reasoning?: boolean;
|
||||
modelId?: string;
|
||||
toolsConfig?: AIToolsConfig;
|
||||
@@ -120,7 +122,9 @@ export function textToText({
|
||||
timeout = TIMEOUT,
|
||||
retry = false,
|
||||
endpoint = Endpoint.StreamObject,
|
||||
postfix,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
reasoning,
|
||||
modelId,
|
||||
toolsConfig,
|
||||
@@ -148,6 +152,10 @@ export function textToText({
|
||||
reasoning,
|
||||
modelId,
|
||||
toolsConfig,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
retry,
|
||||
},
|
||||
endpoint
|
||||
);
|
||||
@@ -166,25 +174,12 @@ export function textToText({
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
if (postfix) {
|
||||
const messages: string[] = [];
|
||||
for await (const event of toTextStream(eventSource, {
|
||||
timeout,
|
||||
signal,
|
||||
})) {
|
||||
if (event.type === 'message') {
|
||||
messages.push(event.data);
|
||||
}
|
||||
}
|
||||
yield postfix(messages.join(''));
|
||||
} else {
|
||||
for await (const event of toTextStream(eventSource, {
|
||||
timeout,
|
||||
signal,
|
||||
})) {
|
||||
if (event.type === 'message') {
|
||||
yield event.data;
|
||||
}
|
||||
for await (const event of toTextStream(eventSource, {
|
||||
timeout,
|
||||
signal,
|
||||
})) {
|
||||
if (event.type === 'message') {
|
||||
yield event.data;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
@@ -215,6 +210,10 @@ export function textToText({
|
||||
reasoning,
|
||||
modelId,
|
||||
toolsConfig,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
retry,
|
||||
},
|
||||
endpoint
|
||||
);
|
||||
@@ -244,7 +243,7 @@ export function textToText({
|
||||
}
|
||||
|
||||
const result = messages.join('');
|
||||
return postfix ? postfix(result) : result;
|
||||
return result;
|
||||
} finally {
|
||||
eventSource.close();
|
||||
if (signal && onAbort) {
|
||||
@@ -266,6 +265,9 @@ export function toImage({
|
||||
timeout = TIMEOUT,
|
||||
retry = false,
|
||||
endpoint,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
client,
|
||||
}: ToImageOptions) {
|
||||
let messageId: string | undefined;
|
||||
@@ -282,12 +284,20 @@ export function toImage({
|
||||
signal,
|
||||
});
|
||||
}
|
||||
const eventSource = client.imagesStream(
|
||||
sessionId,
|
||||
messageId,
|
||||
seed,
|
||||
endpoint
|
||||
);
|
||||
const eventSource =
|
||||
endpoint === Endpoint.Action
|
||||
? client.chatTextStream(
|
||||
{
|
||||
sessionId,
|
||||
messageId,
|
||||
actionId,
|
||||
actionVersion,
|
||||
runId,
|
||||
retry,
|
||||
},
|
||||
Endpoint.Action
|
||||
)
|
||||
: client.imagesStream(sessionId, messageId, seed, endpoint);
|
||||
AIProvider.LAST_ACTION_SESSIONID = sessionId;
|
||||
|
||||
for await (const event of toTextStream(eventSource, {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* @vitest-environment happy-dom
|
||||
*/
|
||||
import { BehaviorSubject } from 'rxjs';
|
||||
import { describe, expect, test, vi } from 'vitest';
|
||||
|
||||
import { AIProvider } from './ai-provider';
|
||||
import { CopilotClient, Endpoint } from './copilot-client';
|
||||
import { setupAIProvider } from './setup-provider';
|
||||
|
||||
Object.defineProperty(globalThis, 'EventSource', {
|
||||
configurable: true,
|
||||
value: {
|
||||
CLOSED: 2,
|
||||
},
|
||||
});
|
||||
|
||||
type SetupAIProviderArgs = Parameters<typeof setupAIProvider>;
|
||||
type ActionInput<T extends keyof BlockSuitePresets.AIActions> = Parameters<
|
||||
NonNullable<BlockSuitePresets.AIActions[T]>
|
||||
>[0];
|
||||
|
||||
async function drain(stream: AsyncIterable<unknown>) {
|
||||
for await (const chunk of stream) {
|
||||
void chunk;
|
||||
}
|
||||
}
|
||||
|
||||
async function drainActionResult(
|
||||
stream: string | AsyncIterable<unknown> | undefined
|
||||
) {
|
||||
expect(stream).toBeDefined();
|
||||
expect(typeof stream).not.toBe('string');
|
||||
await drain(stream as AsyncIterable<unknown>);
|
||||
}
|
||||
|
||||
function createClosedEventSource(): EventSource {
|
||||
return {
|
||||
readyState: EventSource.CLOSED,
|
||||
addEventListener: vi.fn(),
|
||||
close: vi.fn(),
|
||||
} as unknown as EventSource;
|
||||
}
|
||||
|
||||
describe('setupAIProvider action migrations', () => {
|
||||
test('routes mindmap, slides and image filter through action API', async () => {
|
||||
const createdSessions: unknown[] = [];
|
||||
const textStreams: unknown[] = [];
|
||||
const client = new CopilotClient(
|
||||
vi.fn(),
|
||||
vi.fn(() => createClosedEventSource())
|
||||
);
|
||||
vi.spyOn(client, 'createSession').mockImplementation(async options => {
|
||||
createdSessions.push(options);
|
||||
return `session:${options.promptName}`;
|
||||
});
|
||||
vi.spyOn(client, 'createMessage').mockResolvedValue('message-1');
|
||||
vi.spyOn(client, 'chatTextStream').mockImplementation(
|
||||
(options, endpoint) => {
|
||||
textStreams.push({ options, endpoint });
|
||||
return createClosedEventSource();
|
||||
}
|
||||
);
|
||||
vi.spyOn(client, 'imagesStream').mockReturnValue(createClosedEventSource());
|
||||
|
||||
setupAIProvider(
|
||||
client,
|
||||
{ open: vi.fn() } as unknown as SetupAIProviderArgs[1],
|
||||
{
|
||||
session: {
|
||||
account$: new BehaviorSubject(null),
|
||||
},
|
||||
} as unknown as SetupAIProviderArgs[2]
|
||||
);
|
||||
|
||||
await drainActionResult(
|
||||
await AIProvider.actions.brainstormMindmap?.({
|
||||
workspaceId: 'workspace-1',
|
||||
input: 'make a map',
|
||||
stream: true,
|
||||
} satisfies ActionInput<'brainstormMindmap'>)
|
||||
);
|
||||
await drainActionResult(
|
||||
await AIProvider.actions.createSlides?.({
|
||||
workspaceId: 'workspace-1',
|
||||
input: 'make slides',
|
||||
stream: true,
|
||||
} satisfies ActionInput<'createSlides'>)
|
||||
);
|
||||
await drainActionResult(
|
||||
await AIProvider.actions.filterImage?.({
|
||||
workspaceId: 'workspace-1',
|
||||
input: 'convert',
|
||||
attachments: ['blob-1'],
|
||||
style: 'Sketch style',
|
||||
} satisfies ActionInput<'filterImage'>)
|
||||
);
|
||||
|
||||
expect(createdSessions).toEqual([
|
||||
expect.objectContaining({ promptName: 'mindmap.generate' }),
|
||||
expect.objectContaining({ promptName: 'slides.outline' }),
|
||||
expect.objectContaining({ promptName: 'image.filter.sketch' }),
|
||||
]);
|
||||
expect(textStreams).toEqual([
|
||||
expect.objectContaining({
|
||||
endpoint: Endpoint.Action,
|
||||
options: expect.objectContaining({ actionId: 'mindmap.generate' }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
endpoint: Endpoint.Action,
|
||||
options: expect.objectContaining({ actionId: 'slides.outline' }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
endpoint: Endpoint.Action,
|
||||
options: expect.objectContaining({ actionId: 'image.filter.sketch' }),
|
||||
}),
|
||||
]);
|
||||
expect(client.imagesStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
type RequestOptions,
|
||||
type UpdateChatSessionInput,
|
||||
} from '@affine/graphql';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { AIProvider } from './ai-provider';
|
||||
import { type CopilotClient, Endpoint } from './copilot-client';
|
||||
@@ -30,10 +29,10 @@ function toAIUserInfo(account: AuthAccountInfo | null) {
|
||||
|
||||
const filterStyleToPromptName = new Map<string, PromptKey>(
|
||||
Object.entries({
|
||||
'Clay style': 'Convert to Clay style',
|
||||
'Pixel style': 'Convert to Pixel style',
|
||||
'Sketch style': 'Convert to Sketch style',
|
||||
'Anime style': 'Convert to Anime style',
|
||||
'Clay style': 'image.filter.clay',
|
||||
'Pixel style': 'image.filter.pixel',
|
||||
'Sketch style': 'image.filter.sketch',
|
||||
'Anime style': 'image.filter.anime',
|
||||
})
|
||||
);
|
||||
|
||||
@@ -350,7 +349,7 @@ export function setupAIProvider(
|
||||
|
||||
AIProvider.provide('brainstormMindmap', async options => {
|
||||
const sessionId = await createSession({
|
||||
promptName: 'workflow:brainstorm',
|
||||
promptName: 'mindmap.generate',
|
||||
...options,
|
||||
});
|
||||
return textToText({
|
||||
@@ -360,7 +359,9 @@ export function setupAIProvider(
|
||||
content: options.input,
|
||||
// 3 minutes
|
||||
timeout: 180000,
|
||||
endpoint: Endpoint.Workflow,
|
||||
endpoint: Endpoint.Action,
|
||||
actionId: 'mindmap.generate',
|
||||
actionVersion: 'v1',
|
||||
});
|
||||
});
|
||||
|
||||
@@ -439,44 +440,8 @@ Could you make a new website based on these notes and send back just the html fi
|
||||
});
|
||||
|
||||
AIProvider.provide('createSlides', async options => {
|
||||
const SlideSchema = z.object({
|
||||
page: z.number(),
|
||||
type: z.enum(['name', 'title', 'content']),
|
||||
content: z.string(),
|
||||
});
|
||||
type Slide = z.infer<typeof SlideSchema>;
|
||||
const parseJson = (json: string) => {
|
||||
try {
|
||||
return SlideSchema.parse(JSON.parse(json));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
// TODO(@darkskygit): move this to backend's workflow after workflow support custom code action
|
||||
const postfix = (text: string): string => {
|
||||
const slides = text
|
||||
.split('\n')
|
||||
.map(parseJson)
|
||||
.filter((v): v is Slide => !!v);
|
||||
return slides
|
||||
.map(slide => {
|
||||
if (slide.type === 'name') {
|
||||
return `- ${slide.content}`;
|
||||
} else if (slide.type === 'title') {
|
||||
return ` - ${slide.content}`;
|
||||
} else if (slide.content.includes('\n')) {
|
||||
return slide.content
|
||||
.split('\n')
|
||||
.map(c => ` - ${c}`)
|
||||
.join('\n');
|
||||
} else {
|
||||
return ` - ${slide.content}`;
|
||||
}
|
||||
})
|
||||
.join('\n');
|
||||
};
|
||||
const sessionId = await createSession({
|
||||
promptName: 'workflow:presentation',
|
||||
promptName: 'slides.outline',
|
||||
...options,
|
||||
});
|
||||
return textToText({
|
||||
@@ -486,8 +451,9 @@ Could you make a new website based on these notes and send back just the html fi
|
||||
content: options.input,
|
||||
// 3 minutes
|
||||
timeout: 180000,
|
||||
endpoint: Endpoint.Workflow,
|
||||
postfix,
|
||||
endpoint: Endpoint.Action,
|
||||
actionId: 'slides.outline',
|
||||
actionVersion: 'v1',
|
||||
});
|
||||
});
|
||||
|
||||
@@ -521,14 +487,15 @@ Could you make a new website based on these notes and send back just the html fi
|
||||
promptName,
|
||||
...options,
|
||||
});
|
||||
const isWorkflow = !!promptName?.startsWith('workflow:');
|
||||
return toImage({
|
||||
...options,
|
||||
client,
|
||||
sessionId,
|
||||
content: options.input,
|
||||
timeout: 180000,
|
||||
endpoint: isWorkflow ? Endpoint.Workflow : Endpoint.Images,
|
||||
endpoint: Endpoint.Action,
|
||||
actionId: promptName,
|
||||
actionVersion: 'v1',
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import {
|
||||
import { insertFromMarkdown } from '@affine/core/blocksuite/utils';
|
||||
import { preprocessAudioBlobForTranscription } from '@affine/core/utils/opus-encoding';
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import { AiJobStatus } from '@affine/graphql';
|
||||
import track from '@affine/track';
|
||||
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
|
||||
import type { AffineTextAttributes } from '@blocksuite/affine/shared/types';
|
||||
@@ -175,7 +174,7 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
|
||||
return;
|
||||
}
|
||||
const status = await this.transcriptionJob.start();
|
||||
if (status.status === AiJobStatus.claimed) {
|
||||
if (status.status === 'settled') {
|
||||
await this.fillTranscriptionResult(status.result);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
import {
|
||||
getTranscriptTaskQuery,
|
||||
retryTranscriptTaskMutation,
|
||||
settleTranscriptTaskMutation,
|
||||
submitTranscriptTaskMutation,
|
||||
} from '@affine/graphql';
|
||||
import { Framework } from '@toeverything/infra';
|
||||
import { describe, expect, test, vi } from 'vitest';
|
||||
|
||||
import { DefaultServerService } from '../../cloud/services/default-server';
|
||||
import { GraphQLService } from '../../cloud/services/graphql';
|
||||
import { WorkspaceServerService } from '../../cloud/services/workspace-server';
|
||||
import { WorkspaceService } from '../../workspace';
|
||||
import { AudioTranscriptionJobStore } from './audio-transcription-job-store';
|
||||
|
||||
type AudioTranscriptionInput = {
|
||||
files: File[];
|
||||
input?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function createStore(
|
||||
gql: ReturnType<typeof vi.fn>,
|
||||
getAudioTranscriptionInput: () => Promise<AudioTranscriptionInput> = async () => ({
|
||||
files: [],
|
||||
})
|
||||
) {
|
||||
const framework = new Framework();
|
||||
const server = {
|
||||
scope: {
|
||||
get: (key: unknown) => (key === GraphQLService ? { gql } : null),
|
||||
},
|
||||
};
|
||||
framework
|
||||
.service(WorkspaceService, {
|
||||
workspace: { id: 'workspace-1' },
|
||||
} as WorkspaceService)
|
||||
.service(WorkspaceServerService, {
|
||||
server: {
|
||||
scope: server.scope,
|
||||
},
|
||||
} as WorkspaceServerService)
|
||||
.service(DefaultServerService, {
|
||||
server: null,
|
||||
} as unknown as DefaultServerService)
|
||||
.entity(AudioTranscriptionJobStore, [
|
||||
WorkspaceService,
|
||||
WorkspaceServerService,
|
||||
DefaultServerService,
|
||||
]);
|
||||
return framework.provider().createEntity(AudioTranscriptionJobStore, {
|
||||
blobId: 'blob-1',
|
||||
getAudioTranscriptionInput,
|
||||
});
|
||||
}
|
||||
|
||||
describe('AudioTranscriptionJobStore transcript task API', () => {
|
||||
test('uses new transcript task mutations and query', async () => {
|
||||
const file = new File(['audio'], 'audio.webm', { type: 'audio/webm' });
|
||||
const gql = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ submitTranscriptTask: { id: 'task-1' } })
|
||||
.mockResolvedValueOnce({ retryTranscriptTask: { id: 'task-2' } })
|
||||
.mockResolvedValueOnce({
|
||||
currentUser: {
|
||||
copilot: {
|
||||
transcriptTask: { id: 'task-2' },
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({ settleTranscriptTask: { id: 'task-2' } });
|
||||
const store = createStore(gql, async () => ({
|
||||
files: [file],
|
||||
input: { strategy: 'gemini' },
|
||||
}));
|
||||
|
||||
await store.submitTranscriptTask();
|
||||
await store.retryTranscriptTask('task-1');
|
||||
await store.getTranscriptTask('blob-1', 'task-2');
|
||||
await store.settleTranscriptTask('task-2');
|
||||
|
||||
expect(gql).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
query: submitTranscriptTaskMutation,
|
||||
variables: {
|
||||
workspaceId: 'workspace-1',
|
||||
blobId: 'blob-1',
|
||||
blobs: [file],
|
||||
input: { strategy: 'gemini' },
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(gql).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
query: retryTranscriptTaskMutation,
|
||||
variables: {
|
||||
workspaceId: 'workspace-1',
|
||||
taskId: 'task-1',
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(gql).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
query: getTranscriptTaskQuery,
|
||||
variables: {
|
||||
workspaceId: 'workspace-1',
|
||||
taskId: 'task-2',
|
||||
blobId: 'blob-1',
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(gql).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
expect.objectContaining({
|
||||
query: settleTranscriptTaskMutation,
|
||||
variables: {
|
||||
workspaceId: 'workspace-1',
|
||||
taskId: 'task-2',
|
||||
},
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
+25
-24
@@ -1,8 +1,8 @@
|
||||
import {
|
||||
claimAudioTranscriptionMutation,
|
||||
getAudioTranscriptionQuery,
|
||||
retryAudioTranscriptionMutation,
|
||||
submitAudioTranscriptionMutation,
|
||||
getTranscriptTaskQuery,
|
||||
retryTranscriptTaskMutation,
|
||||
settleTranscriptTaskMutation,
|
||||
submitTranscriptTaskMutation,
|
||||
} from '@affine/graphql';
|
||||
import { Entity } from '@toeverything/infra';
|
||||
|
||||
@@ -39,7 +39,7 @@ export class AudioTranscriptionJobStore extends Entity<{
|
||||
return this.workspaceService.workspace.id;
|
||||
}
|
||||
|
||||
submitAudioTranscription = async () => {
|
||||
submitTranscriptTask = async () => {
|
||||
const graphqlService = this.graphqlService;
|
||||
if (!graphqlService) {
|
||||
throw new Error('No graphql service available');
|
||||
@@ -47,7 +47,7 @@ export class AudioTranscriptionJobStore extends Entity<{
|
||||
const { files, input } = await this.props.getAudioTranscriptionInput();
|
||||
const response = await graphqlService.gql({
|
||||
timeout: 0, // default 15s is too short for audio transcription
|
||||
query: submitAudioTranscriptionMutation,
|
||||
query: submitTranscriptTaskMutation,
|
||||
variables: {
|
||||
workspaceId: this.currentWorkspaceId,
|
||||
blobId: this.props.blobId,
|
||||
@@ -55,31 +55,31 @@ export class AudioTranscriptionJobStore extends Entity<{
|
||||
input,
|
||||
},
|
||||
});
|
||||
if (!response.submitAudioTranscription?.id) {
|
||||
if (!response.submitTranscriptTask?.id) {
|
||||
throw new Error('Failed to submit audio transcription');
|
||||
}
|
||||
return response.submitAudioTranscription;
|
||||
return response.submitTranscriptTask;
|
||||
};
|
||||
|
||||
retryAudioTranscription = async (jobId: string) => {
|
||||
retryTranscriptTask = async (taskId: string) => {
|
||||
const graphqlService = this.graphqlService;
|
||||
if (!graphqlService) {
|
||||
throw new Error('No graphql service available');
|
||||
}
|
||||
const response = await graphqlService.gql({
|
||||
query: retryAudioTranscriptionMutation,
|
||||
query: retryTranscriptTaskMutation,
|
||||
variables: {
|
||||
jobId,
|
||||
taskId,
|
||||
workspaceId: this.currentWorkspaceId,
|
||||
},
|
||||
});
|
||||
if (!response.retryAudioTranscription) {
|
||||
if (!response.retryTranscriptTask) {
|
||||
throw new Error('Failed to retry audio transcription');
|
||||
}
|
||||
return response.retryAudioTranscription;
|
||||
return response.retryTranscriptTask;
|
||||
};
|
||||
|
||||
getAudioTranscription = async (blobId: string, jobId?: string) => {
|
||||
getTranscriptTask = async (blobId: string, taskId?: string) => {
|
||||
const graphqlService = this.graphqlService;
|
||||
if (!graphqlService) {
|
||||
throw new Error('No graphql service available');
|
||||
@@ -89,32 +89,33 @@ export class AudioTranscriptionJobStore extends Entity<{
|
||||
throw new Error('No current workspace id');
|
||||
}
|
||||
const response = await graphqlService.gql({
|
||||
query: getAudioTranscriptionQuery,
|
||||
query: getTranscriptTaskQuery,
|
||||
variables: {
|
||||
workspaceId: currentWorkspaceId,
|
||||
jobId,
|
||||
taskId,
|
||||
blobId,
|
||||
},
|
||||
});
|
||||
if (!response.currentUser?.copilot?.audioTranscription) {
|
||||
if (!response.currentUser?.copilot?.transcriptTask) {
|
||||
return null;
|
||||
}
|
||||
return response.currentUser.copilot.audioTranscription;
|
||||
return response.currentUser.copilot.transcriptTask;
|
||||
};
|
||||
claimAudioTranscription = async (jobId: string) => {
|
||||
settleTranscriptTask = async (taskId: string) => {
|
||||
const graphqlService = this.graphqlService;
|
||||
if (!graphqlService) {
|
||||
throw new Error('No graphql service available');
|
||||
}
|
||||
const response = await graphqlService.gql({
|
||||
query: claimAudioTranscriptionMutation,
|
||||
query: settleTranscriptTaskMutation,
|
||||
variables: {
|
||||
jobId,
|
||||
taskId,
|
||||
workspaceId: this.currentWorkspaceId,
|
||||
},
|
||||
});
|
||||
if (!response.claimAudioTranscription) {
|
||||
throw new Error('Failed to claim transcription result');
|
||||
if (!response.settleTranscriptTask) {
|
||||
throw new Error('Failed to settle transcription result');
|
||||
}
|
||||
return response.claimAudioTranscription;
|
||||
return response.settleTranscriptTask;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -13,37 +13,37 @@ import type { TranscriptionResult } from './types';
|
||||
|
||||
// The UI status of the transcription job
|
||||
export type TranscriptionStatus =
|
||||
| {
|
||||
status: 'waiting-for-job';
|
||||
}
|
||||
| {
|
||||
status: 'started';
|
||||
}
|
||||
| {
|
||||
status: AiJobStatus.pending;
|
||||
}
|
||||
| {
|
||||
status: AiJobStatus.running;
|
||||
}
|
||||
| { status: 'waiting-for-job' }
|
||||
| { status: 'started' }
|
||||
| { status: AiJobStatus.pending }
|
||||
| { status: AiJobStatus.running }
|
||||
| {
|
||||
status: AiJobStatus.failed;
|
||||
error: UserFriendlyError; // <<- this is not visible on UI yet
|
||||
}
|
||||
| {
|
||||
status: AiJobStatus.finished; // ready to be claimed, but may be rejected because of insufficient credits
|
||||
}
|
||||
| {
|
||||
status: AiJobStatus.claimed;
|
||||
result: TranscriptionResult;
|
||||
};
|
||||
| { status: AiJobStatus.finished }
|
||||
| { status: 'settled'; result: TranscriptionResult };
|
||||
|
||||
const logger = new DebugLogger('audio-transcription-job');
|
||||
|
||||
function hasSettledTranscriptResult(
|
||||
job: {
|
||||
status: AiJobStatus;
|
||||
normalizedTranscript?: string | null;
|
||||
transcription?: unknown[] | null;
|
||||
} | null
|
||||
) {
|
||||
return (
|
||||
job?.status === AiJobStatus.finished &&
|
||||
(!!job.normalizedTranscript || !!job.transcription?.length)
|
||||
);
|
||||
}
|
||||
|
||||
// facts on transcription job ownership
|
||||
// 1. jobid + blobid is unique for a given user
|
||||
// 2. only the creator can claim the job
|
||||
// 3. all users can query the claimed job result
|
||||
// 4. claim a job requires AI credits
|
||||
// 2. only the creator can settle/unlock the task result
|
||||
// 3. all users can query the settled result
|
||||
// 4. settlement requires AI credits
|
||||
export class AudioTranscriptionJob extends Entity<{
|
||||
readonly blockProps: TranscriptionBlockProps;
|
||||
readonly blobId: string;
|
||||
@@ -97,12 +97,12 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
readonly preflightCheck = async () => {
|
||||
// if the job id is given, check if the job exists
|
||||
if (this.props.blockProps.jobId) {
|
||||
const existingJob = await this.store.getAudioTranscription(
|
||||
const existingJob = await this.store.getTranscriptTask(
|
||||
this.props.blobId,
|
||||
this.props.blockProps.jobId
|
||||
);
|
||||
|
||||
if (existingJob?.status === AiJobStatus.claimed) {
|
||||
if (hasSettledTranscriptResult(existingJob)) {
|
||||
// if job exists, anyone can query it
|
||||
return;
|
||||
}
|
||||
@@ -142,25 +142,28 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
let job: {
|
||||
id: string;
|
||||
status: AiJobStatus;
|
||||
} | null = await this.store.getAudioTranscription(
|
||||
} | null = await this.store.getTranscriptTask(
|
||||
this.props.blobId,
|
||||
this.props.blockProps.jobId
|
||||
);
|
||||
|
||||
if (!job) {
|
||||
logger.debug('No existing job found, submitting new transcription job');
|
||||
job = await this.store.submitAudioTranscription();
|
||||
job = await this.store.submitTranscriptTask();
|
||||
} else if (job.status === AiJobStatus.failed) {
|
||||
logger.debug('Found existing failed job, retrying', {
|
||||
jobId: job.id,
|
||||
});
|
||||
job = await this.store.retryAudioTranscription(job.id);
|
||||
job = await this.store.retryTranscriptTask(job.id);
|
||||
} else {
|
||||
logger.debug('Found existing job', {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
});
|
||||
}
|
||||
if (!job) {
|
||||
throw UserFriendlyError.fromAny('failed to submit transcription');
|
||||
}
|
||||
|
||||
this.props.blockProps.jobId = job.id;
|
||||
this.props.blockProps.createdBy = this.currentUserId;
|
||||
@@ -174,8 +177,8 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
throw UserFriendlyError.fromAny('failed to submit transcription');
|
||||
}
|
||||
|
||||
await this.untilJobFinishedOrClaimed();
|
||||
await this.claim();
|
||||
await this.untilTaskReadyOrSettled();
|
||||
await this.settle();
|
||||
} catch (err) {
|
||||
logger.debug('Error during job submission', { error: err });
|
||||
this._status$.value = {
|
||||
@@ -186,7 +189,7 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
return this.status$.value;
|
||||
}
|
||||
|
||||
private async untilJobFinishedOrClaimed() {
|
||||
private async untilTaskReadyOrSettled() {
|
||||
while (
|
||||
!this.disposed &&
|
||||
this.props.blockProps.jobId &&
|
||||
@@ -195,7 +198,7 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
logger.debug('Polling job status', {
|
||||
jobId: this.props.blockProps.jobId,
|
||||
});
|
||||
const job = await this.store.getAudioTranscription(
|
||||
const job = await this.store.getTranscriptTask(
|
||||
this.props.blobId,
|
||||
this.props.blockProps.jobId
|
||||
);
|
||||
@@ -207,8 +210,8 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
throw UserFriendlyError.fromAny('Transcription job failed');
|
||||
}
|
||||
|
||||
if (job?.status === 'finished' || job?.status === 'claimed') {
|
||||
logger.debug('Job finished, ready to claim', {
|
||||
if (job?.status === AiJobStatus.finished) {
|
||||
logger.debug('Transcript task is ready to settle', {
|
||||
jobId: this.props.blockProps.jobId,
|
||||
});
|
||||
this._status$.value = {
|
||||
@@ -222,37 +225,37 @@ export class AudioTranscriptionJob extends Entity<{
|
||||
}
|
||||
}
|
||||
|
||||
async claim() {
|
||||
async settle() {
|
||||
if (this.disposed) {
|
||||
logger.debug('Job already disposed, cannot claim');
|
||||
logger.debug('Job already disposed, cannot settle');
|
||||
throw new Error('Job already disposed');
|
||||
}
|
||||
|
||||
logger.debug('Attempting to claim job', {
|
||||
logger.debug('Attempting to settle transcript task', {
|
||||
jobId: this.props.blockProps.jobId,
|
||||
});
|
||||
|
||||
if (!this.props.blockProps.jobId) {
|
||||
logger.debug('No job id found, cannot claim');
|
||||
logger.debug('No job id found, cannot settle');
|
||||
throw new Error('No job id found');
|
||||
}
|
||||
|
||||
const claimedJob = await this.store.claimAudioTranscription(
|
||||
const settledTask = await this.store.settleTranscriptTask(
|
||||
this.props.blockProps.jobId
|
||||
);
|
||||
|
||||
if (claimedJob) {
|
||||
logger.debug('Successfully claimed job', {
|
||||
if (settledTask) {
|
||||
logger.debug('Successfully settled transcript task', {
|
||||
jobId: this.props.blockProps.jobId,
|
||||
});
|
||||
const result: TranscriptionResult = buildTranscriptionResult(claimedJob);
|
||||
const result: TranscriptionResult = buildTranscriptionResult(settledTask);
|
||||
|
||||
this._status$.value = {
|
||||
status: AiJobStatus.claimed,
|
||||
status: 'settled',
|
||||
result,
|
||||
};
|
||||
} else {
|
||||
throw new Error('Failed to claim transcription result');
|
||||
throw new Error('Failed to settle transcription result');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user