chore(server): improve transcript stability (#13821)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Enhanced audio/video detection for MP4 files to better distinguish
audio-only vs. video.

* **Dependencies**
* Added MP4 parsing dependency and updated AI provider libraries
(Anthropic, Google, OpenAI, etc.).

* **Bug Fixes**
  * Tightened authentication state validation for magic-link/OTP flows.
* Stricter space-join validation to reject invalid client
types/versions.
  * Improved transcript entry deduplication and data handling.

* **API**
* Transcript submit payload now requires infos and removes deprecated
url/mimeType fields.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2025-10-29 17:48:15 +08:00
committed by GitHub
parent d74087fdc5
commit b7ac7caab4
11 changed files with 176 additions and 126 deletions

View File

@@ -270,14 +270,12 @@ export class AuthController {
validators.assertValidEmail(email);
const cacheKey = OTP_CACHE_KEY(otp);
const cachedToken = await this.cache.get<
{ token: string; clientNonce: string } | string
>(cacheKey);
const cachedToken = await this.cache.get<{
token: string;
clientNonce: string;
}>(cacheKey);
let token: string | undefined;
// TODO(@fengmk2): this is a temporary compatible with cache token is string value, should be removed in 0.22
if (typeof cachedToken === 'string') {
token = cachedToken;
} else if (cachedToken) {
if (cachedToken && typeof cachedToken === 'object') {
token = cachedToken.token;
if (cachedToken.clientNonce && cachedToken.clientNonce !== clientNonce) {
throw new InvalidAuthState();

View File

@@ -194,14 +194,12 @@ export class SpaceSyncGateway
@ConnectedSocket() client: Socket,
@MessageBody()
{ spaceType, spaceId, clientVersion }: JoinSpaceMessage
): Promise<EventResponse<{ clientId: string; success: true }>> {
// TODO(@forehalo): remove this after 0.19 goes out of life
// simple match 0.19.x
if (/^0.19.[\d]$/.test(clientVersion)) {
const room = Room(spaceId, 'sync-019');
if (!client.rooms.has(room)) {
await client.join(room);
}
): Promise<EventResponse<{ clientId: string; success: boolean }>> {
if (
![SpaceType.Userspace, SpaceType.Workspace].includes(spaceType) ||
/^0.1/.test(clientVersion)
) {
return { data: { clientId: client.id, success: false } };
} else {
if (spaceType === SpaceType.Workspace) {
this.event.emit('workspace.embedding', { workspaceId: spaceId });

View File

@@ -112,9 +112,7 @@ export class GeminiGenerativeProvider extends GeminiProvider<GeminiGenerativeCon
`${baseUrl}/models?key=${this.config.apiKey}`
)
.then(r => r.json())
.then(
r => (console.log(JSON.stringify(r)), ModelListSchema.parse(r))
);
.then(r => ModelListSchema.parse(r));
this.onlineModelList = models.map(model =>
model.name.replace('models/', '')
);

View File

@@ -171,11 +171,11 @@ export class CopilotTranscriptionService {
if (payload.success) {
let { url, mimeType, infos } = payload.data;
infos = infos || [];
if (url && mimeType) {
if (url && mimeType && !infos.find(i => i.url === url)) {
infos.push({ url, mimeType });
}
ret.infos = this.mergeInfos(infos, url, mimeType);
ret.infos = infos;
if (job.status === AiJobStatus.claimed) {
ret.transcription = payload.data;
}
@@ -239,22 +239,6 @@ export class CopilotTranscriptionService {
}
}
// TODO(@darkskygit): remove after old server down
private mergeInfos(
infos?: AudioBlobInfos | null,
url?: string | null,
mimeType?: string | null
) {
if (url && mimeType) {
if (infos) {
infos.push({ url, mimeType });
} else {
infos = [{ url, mimeType }];
}
}
return infos || [];
}
private convertTime(time: number, offset = 0) {
time = time + offset;
const minutes = Math.floor(time / 60);
@@ -298,14 +282,10 @@ export class CopilotTranscriptionService {
jobId,
infos,
modelId,
// @deprecated
url,
mimeType,
}: Jobs['copilot.transcript.submit']) {
try {
const blobInfos = this.mergeInfos(infos, url, mimeType);
const transcriptions = await Promise.all(
Array.from(blobInfos.entries()).map(([idx, { url, mimeType }]) =>
Array.from(infos.entries()).map(([idx, { url, mimeType }]) =>
this.callTranscript(url, mimeType, idx * 10 * 60, modelId)
)
);

View File

@@ -55,12 +55,8 @@ declare global {
interface Jobs {
'copilot.transcript.submit': {
jobId: string;
infos?: AudioBlobInfos;
infos: AudioBlobInfos;
modelId?: string;
/// @deprecated use `infos` instead
url?: string;
/// @deprecated use `infos` instead
mimeType?: string;
};
'copilot.transcript.summary.submit': {
jobId: string;