From bcf2a51d4147e88c0f8711ccd4e3cfaf6f735fab Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Sun, 22 Mar 2026 02:50:14 +0800 Subject: [PATCH] feat(native): record encoding (#14188) fix #13784 ## Summary by CodeRabbit * **New Features** * Start/stop system or meeting recordings with Ogg/Opus artifacts and native start/stop APIs; workspace backup recovery. * **Refactor** * Simplified recording lifecycle and UI flows; native runtime now orchestrates recording/processing and reporting. * **Bug Fixes** * Stronger path validation, safer import/export dialogs, consistent error handling/logging, and retry-safe recording processing. * **Chores** * Added cross-platform native audio capture and Ogg/Opus encoding support. * **Tests** * New unit, integration, and e2e tests for recording, path guards, dialogs, and workspace recovery. --- .github/workflows/build-test.yml | 11 +- Cargo.lock | 33 + Cargo.toml | 1 + .../linked-doc/src/transformers/obsidian.ts | 210 +++- .../src/base/storage/__tests__/fs.spec.ts | 17 + .../server/src/base/storage/providers/fs.ts | 53 +- .../apps/android/src/plugins/nbstore/index.ts | 4 +- .../src/app/effects/recording.ts | 310 ++++-- .../src/popup/recording/index.tsx | 150 +-- .../apps/electron/src/helper/dialog/dialog.ts | 190 ++-- .../apps/electron/src/helper/dialog/index.ts | 16 +- .../electron/src/helper/workspace/handlers.ts | 88 +- .../electron/src/helper/workspace/index.ts | 2 + .../electron/src/helper/workspace/meta.ts | 8 +- .../apps/electron/src/main/protocol.ts | 51 +- .../electron/src/main/recording/feature.ts | 501 ++++------ .../apps/electron/src/main/recording/index.ts | 45 +- .../src/main/recording/state-machine.ts | 162 ++- .../src/main/recording/state-transitions.md | 103 +- .../apps/electron/src/main/recording/types.ts | 41 +- .../apps/electron/src/main/tray/index.ts | 16 +- .../apps/electron/src/shared/utils.ts | 127 ++- .../apps/electron/test/dialog/dialog.spec.ts | 110 +- .../apps/electron/test/helper/utils.spec.ts | 107 ++ .../test/main/recording-effect.spec.ts | 256 +++++ .../test/main/recording-state.spec.ts | 116 +++ .../electron/test/workspace/handlers.spec.ts | 48 + .../apps/ios/src/plugins/nbstore/index.ts | 4 +- .../ui/date-picker/calendar/day-picker.tsx | 1 - .../ui/date-picker/calendar/month-picker.tsx | 2 - .../setting/general-setting/backup/index.tsx | 4 +- .../core/src/modules/backup/services/index.ts | 6 +- .../frontend/core/src/utils/opus-encoding.ts | 95 -- packages/frontend/native/index.d.ts | 31 + packages/frontend/native/index.js | 2 + .../frontend/native/media_capture/Cargo.toml | 21 +- .../media_capture/src/audio_callback.rs | 31 + .../frontend/native/media_capture/src/lib.rs | 2 + .../src/macos/screen_capture_kit.rs | 38 +- .../media_capture/src/macos/tap_audio.rs | 19 +- .../native/media_capture/src/recording.rs | 942 ++++++++++++++++++ .../src/windows/audio_capture.rs | 17 +- .../src/windows/screen_capture_kit.rs | 32 +- tests/affine-desktop/e2e/workspace.spec.ts | 41 +- 44 files changed, 2921 insertions(+), 1143 deletions(-) create mode 100644 packages/frontend/apps/electron/test/helper/utils.spec.ts create mode 100644 packages/frontend/apps/electron/test/main/recording-effect.spec.ts create mode 100644 packages/frontend/apps/electron/test/main/recording-state.spec.ts create mode 100644 packages/frontend/native/media_capture/src/audio_callback.rs create mode 100644 packages/frontend/native/media_capture/src/recording.rs diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index dde4f784bb..9d843ac71b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -269,10 +269,13 @@ jobs: - name: Run playground build run: yarn workspace @blocksuite/playground build - - name: Run playwright tests - run: | - yarn workspace @blocksuite/integration-test test:unit - yarn workspace @affine-test/blocksuite test "cross-platform/" --forbid-only + - name: Run integration browser tests + timeout-minutes: 10 + run: yarn workspace @blocksuite/integration-test test:unit + + - name: Run cross-platform playwright tests + timeout-minutes: 10 + run: yarn workspace @affine-test/blocksuite test "cross-platform/" --forbid-only - name: Upload test results if: always() diff --git a/Cargo.lock b/Cargo.lock index 8504c43732..677697ee44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,6 +92,9 @@ dependencies = [ "napi-derive", "objc2", "objc2-foundation", + "ogg", + "opus-codec", + "rand 0.9.2", "rubato", "screencapturekit", "symphonia", @@ -621,6 +624,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease", "proc-macro2", "quote", "regex", @@ -1083,6 +1088,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "cobs" version = "0.3.0" @@ -3994,6 +4008,15 @@ dependencies = [ "cc", ] +[[package]] +name = "ogg" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdab8dcd8d4052eaacaf8fb07a3ccd9a6e26efadb42878a413c68fc4af1dee2b" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -4018,6 +4041,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus-codec" +version = "0.1.2" +source = "git+https://github.com/toeverything/opus-codec?rev=c2afef2#c2afef20773c3afb06395a26a4f054ca90ba9078" +dependencies = [ + "bindgen", + "cmake", + "pkg-config", +] + [[package]] name = "ordered-float" version = "5.1.0" diff --git a/Cargo.toml b/Cargo.toml index b3691d2834..ee89683f57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ resolver = "3" notify = { version = "8", features = ["serde"] } objc2 = "0.6" objc2-foundation = "0.3" + ogg = "0.9" once_cell = "1" ordered-float = "5" parking_lot = "0.12" diff --git a/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts b/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts index a528f71f54..c5688caa70 100644 --- a/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts +++ b/blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts @@ -183,6 +183,32 @@ function createTextFootnoteDefinition(content: string): string { }); } +function parseFootnoteDefLine(line: string): { + identifier: string; + content: string; +} | null { + if (!line.startsWith('[^')) return null; + + const closeBracketIndex = line.indexOf(']:', 2); + if (closeBracketIndex <= 2) return null; + + const identifier = line.slice(2, closeBracketIndex); + if (!identifier || identifier.includes(']')) return null; + + let contentStart = closeBracketIndex + 2; + while ( + contentStart < line.length && + (line[contentStart] === ' ' || line[contentStart] === '\t') + ) { + contentStart += 1; + } + + return { + identifier, + content: line.slice(contentStart), + }; +} + function extractObsidianFootnotes(markdown: string): { content: string; footnotes: string[]; @@ -193,14 +219,14 @@ function extractObsidianFootnotes(markdown: string): { for (let index = 0; index < lines.length; index += 1) { const line = lines[index]; - const match = line.match(/^\[\^([^\]]+)\]:\s*(.*)$/); - if (!match) { + const definition = parseFootnoteDefLine(line); + if (!definition) { output.push(line); continue; } - const identifier = match[1]; - const contentLines = [match[2]]; + const { identifier } = definition; + const contentLines = [definition.content]; while (index + 1 < lines.length) { const nextLine = lines[index + 1]; @@ -392,49 +418,119 @@ function parseObsidianAttach(value: string): ObsidianAttachmentEmbed | null { } } +function parseWikiLinkAt( + source: string, + startIdx: number, + embedded: boolean +): { + raw: string; + rawTarget: string; + rawAlias?: string; + endIdx: number; +} | null { + const opener = embedded ? '![[' : '[['; + if (!source.startsWith(opener, startIdx)) return null; + + const contentStart = startIdx + opener.length; + const closeIndex = source.indexOf(']]', contentStart); + if (closeIndex === -1) return null; + + const inner = source.slice(contentStart, closeIndex); + const separatorIdx = inner.indexOf('|'); + const rawTarget = separatorIdx === -1 ? inner : inner.slice(0, separatorIdx); + const rawAlias = + separatorIdx === -1 ? undefined : inner.slice(separatorIdx + 1); + + if ( + rawTarget.length === 0 || + rawTarget.includes(']') || + rawTarget.includes('|') || + rawAlias?.includes(']') + ) { + return null; + } + + return { + raw: source.slice(startIdx, closeIndex + 2), + rawTarget, + rawAlias, + endIdx: closeIndex + 2, + }; +} + +function replaceWikiLinks( + source: string, + embedded: boolean, + replacer: (match: { + raw: string; + rawTarget: string; + rawAlias?: string; + }) => string +): string { + const opener = embedded ? '![[' : '[['; + let cursor = 0; + let output = ''; + + while (cursor < source.length) { + const matchStart = source.indexOf(opener, cursor); + if (matchStart === -1) { + output += source.slice(cursor); + break; + } + + output += source.slice(cursor, matchStart); + const match = parseWikiLinkAt(source, matchStart, embedded); + if (!match) { + output += source.slice(matchStart, matchStart + opener.length); + cursor = matchStart + opener.length; + continue; + } + + output += replacer(match); + cursor = match.endIdx; + } + + return output; +} + function preprocessObsidianEmbeds( markdown: string, filePath: string, pageLookupMap: ReadonlyMap, pathBlobIdMap: ReadonlyMap ): string { - return markdown.replace( - /!\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g, - (match, rawTarget: string, rawAlias?: string) => { - const targetPageId = resolvePageIdFromLookup( - pageLookupMap, - rawTarget, - filePath - ); - if (targetPageId) { - return `[[${rawTarget}${rawAlias ? `|${rawAlias}` : ''}]]`; - } - - const { path } = parseObsidianTarget(rawTarget); - if (!path) { - return match; - } - - const assetPath = getImageFullPath(filePath, path); - const encodedPath = encodeMarkdownPath(assetPath); - - if (isImageAssetPath(path)) { - const alt = getEmbedLabel(rawAlias, path, false); - return `![${escapeMarkdownLabel(alt)}](${encodedPath})`; - } - - const label = getEmbedLabel(rawAlias, path, true); - const blobId = pathBlobIdMap.get(assetPath); - if (!blobId) return `[${escapeMarkdownLabel(label)}](${encodedPath})`; - - const extension = path.split('.').at(-1)?.toLowerCase() ?? ''; - return createObsidianAttach({ - blobId, - fileName: basename(path), - fileType: extMimeMap.get(extension) ?? '', - }); + return replaceWikiLinks(markdown, true, ({ raw, rawTarget, rawAlias }) => { + const targetPageId = resolvePageIdFromLookup( + pageLookupMap, + rawTarget, + filePath + ); + if (targetPageId) { + return `[[${rawTarget}${rawAlias ? `|${rawAlias}` : ''}]]`; } - ); + + const { path } = parseObsidianTarget(rawTarget); + if (!path) return raw; + + const assetPath = getImageFullPath(filePath, path); + const encodedPath = encodeMarkdownPath(assetPath); + + if (isImageAssetPath(path)) { + const alt = getEmbedLabel(rawAlias, path, false); + return `![${escapeMarkdownLabel(alt)}](${encodedPath})`; + } + + const label = getEmbedLabel(rawAlias, path, true); + const blobId = pathBlobIdMap.get(assetPath); + if (!blobId) return `[${escapeMarkdownLabel(label)}](${encodedPath})`; + + const extension = path.split('.').at(-1)?.toLowerCase() ?? ''; + return createObsidianAttach({ + blobId, + fileName: basename(path), + fileType: extMimeMap.get(extension) ?? '', + }); + }); } function preprocessObsidianMarkdown( @@ -521,21 +617,31 @@ export const obsidianWikilinkToDeltaMatcher = MarkdownASTToDeltaExtension({ } const nodeContent = textNode.value; - const wikilinkRegex = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g; const deltas: DeltaInsert[] = []; + let cursor = 0; - let lastProcessedIndex = 0; - let linkMatch; + while (cursor < nodeContent.length) { + const matchStart = nodeContent.indexOf('[[', cursor); + if (matchStart === -1) { + deltas.push({ insert: nodeContent.substring(cursor) }); + break; + } - while ((linkMatch = wikilinkRegex.exec(nodeContent)) !== null) { - if (linkMatch.index > lastProcessedIndex) { + if (matchStart > cursor) { deltas.push({ - insert: nodeContent.substring(lastProcessedIndex, linkMatch.index), + insert: nodeContent.substring(cursor, matchStart), }); } - const targetPageName = linkMatch[1].trim(); - const alias = linkMatch[2]?.trim(); + const linkMatch = parseWikiLinkAt(nodeContent, matchStart, false); + if (!linkMatch) { + deltas.push({ insert: '[[' }); + cursor = matchStart + 2; + continue; + } + + const targetPageName = linkMatch.rawTarget.trim(); + const alias = linkMatch.rawAlias?.trim(); const currentFilePath = context.configs.get(FULL_FILE_PATH_KEY); const targetPageId = resolvePageIdFromLookup( { get: key => context.configs.get(`obsidian:pageId:${key}`) }, @@ -560,14 +666,10 @@ export const obsidianWikilinkToDeltaMatcher = MarkdownASTToDeltaExtension({ }, }); } else { - deltas.push({ insert: linkMatch[0] }); + deltas.push({ insert: linkMatch.raw }); } - lastProcessedIndex = wikilinkRegex.lastIndex; - } - - if (lastProcessedIndex < nodeContent.length) { - deltas.push({ insert: nodeContent.substring(lastProcessedIndex) }); + cursor = linkMatch.endIdx; } return deltas; diff --git a/packages/backend/server/src/base/storage/__tests__/fs.spec.ts b/packages/backend/server/src/base/storage/__tests__/fs.spec.ts index 84ac4c4881..9077adfd5e 100644 --- a/packages/backend/server/src/base/storage/__tests__/fs.spec.ts +++ b/packages/backend/server/src/base/storage/__tests__/fs.spec.ts @@ -111,3 +111,20 @@ test('delete', async t => { await t.throwsAsync(() => fs.access(join(config.path, provider.bucket, key))); }); + +test('rejects unsafe object keys', async t => { + const provider = createProvider(); + + await t.throwsAsync(() => provider.put('../escape', Buffer.from('nope'))); + await t.throwsAsync(() => provider.get('nested/../escape')); + await t.throwsAsync(() => provider.head('./escape')); + t.throws(() => provider.delete('nested//escape')); +}); + +test('rejects unsafe list prefixes', async t => { + const provider = createProvider(); + + await t.throwsAsync(() => provider.list('../escape')); + await t.throwsAsync(() => provider.list('nested/../../escape')); + await t.throwsAsync(() => provider.list('/absolute')); +}); diff --git a/packages/backend/server/src/base/storage/providers/fs.ts b/packages/backend/server/src/base/storage/providers/fs.ts index fc6252f72c..6483931ed2 100644 --- a/packages/backend/server/src/base/storage/providers/fs.ts +++ b/packages/backend/server/src/base/storage/providers/fs.ts @@ -25,9 +25,47 @@ import { } from './provider'; import { autoMetadata, toBuffer } from './utils'; -function escapeKey(key: string): string { - // avoid '../' and './' in key - return key.replace(/\.?\.[/\\]/g, '%'); +function normalizeStorageKey(key: string): string { + const normalized = key.replaceAll('\\', '/'); + const segments = normalized.split('/'); + + if ( + !normalized || + normalized.startsWith('/') || + segments.some(segment => !segment || segment === '.' || segment === '..') + ) { + throw new Error(`Invalid storage key: ${key}`); + } + + return segments.join('/'); +} + +function normalizeStoragePrefix(prefix: string): string { + const normalized = prefix.replaceAll('\\', '/'); + if (!normalized) { + return normalized; + } + if (normalized.startsWith('/')) { + throw new Error(`Invalid storage prefix: ${prefix}`); + } + + const segments = normalized.split('/'); + const lastSegment = segments.pop(); + + if ( + lastSegment === undefined || + segments.some(segment => !segment || segment === '.' || segment === '..') || + lastSegment === '.' || + lastSegment === '..' + ) { + throw new Error(`Invalid storage prefix: ${prefix}`); + } + + if (lastSegment === '') { + return `${segments.join('/')}/`; + } + + return [...segments, lastSegment].join('/'); } export interface FsStorageConfig { @@ -57,7 +95,7 @@ export class FsStorageProvider implements StorageProvider { body: BlobInputType, metadata: PutObjectMetadata = {} ): Promise { - key = escapeKey(key); + key = normalizeStorageKey(key); const blob = await toBuffer(body); // write object @@ -68,6 +106,7 @@ export class FsStorageProvider implements StorageProvider { } async head(key: string) { + key = normalizeStorageKey(key); const metadata = this.readMetadata(key); if (!metadata) { this.logger.verbose(`Object \`${key}\` not found`); @@ -80,7 +119,7 @@ export class FsStorageProvider implements StorageProvider { body?: Readable; metadata?: GetObjectMetadata; }> { - key = escapeKey(key); + key = normalizeStorageKey(key); try { const metadata = this.readMetadata(key); @@ -105,7 +144,7 @@ export class FsStorageProvider implements StorageProvider { // read dir recursively and filter out '.metadata.json' files let dir = this.path; if (prefix) { - prefix = escapeKey(prefix); + prefix = normalizeStoragePrefix(prefix); const parts = prefix.split(/[/\\]/); // for prefix `a/b/c`, move `a/b` to dir and `c` to key prefix if (parts.length > 1) { @@ -152,7 +191,7 @@ export class FsStorageProvider implements StorageProvider { } delete(key: string): Promise { - key = escapeKey(key); + key = normalizeStorageKey(key); try { rmSync(this.join(key), { force: true }); diff --git a/packages/frontend/apps/android/src/plugins/nbstore/index.ts b/packages/frontend/apps/android/src/plugins/nbstore/index.ts index 659b5f3657..e20a10f522 100644 --- a/packages/frontend/apps/android/src/plugins/nbstore/index.ts +++ b/packages/frontend/apps/android/src/plugins/nbstore/index.ts @@ -433,7 +433,9 @@ export const NbStoreNativeDBApis: NativeDBApis = { id: string, docId: string ): Promise { - return NbStore.getDocIndexedClock({ id, docId }); + return NbStore.getDocIndexedClock({ id, docId }).then(clock => + clock ? { ...clock, timestamp: new Date(clock.timestamp) } : null + ); }, setDocIndexedClock: function ( id: string, diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts b/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts index 79f07d06ac..554b5b58d9 100644 --- a/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts +++ b/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts @@ -13,6 +13,19 @@ import type { FrameworkProvider } from '@toeverything/infra'; import { getCurrentWorkspace, isAiEnabled } from './utils'; const logger = new DebugLogger('electron-renderer:recording'); +const RECORDING_PROCESS_RETRY_MS = 1000; +const NATIVE_RECORDING_MIME_TYPE = 'audio/ogg'; + +type ProcessingRecordingStatus = { + id: number; + status: 'processing'; + appName?: string; + blockCreationStatus?: undefined; + filepath: string; + startTime: number; +}; + +type WorkspaceHandle = NonNullable>; async function readRecordingFile(filepath: string) { if (apis?.recording?.readRecordingFile) { @@ -45,118 +58,217 @@ async function saveRecordingBlob(blobEngine: BlobEngine, filepath: string) { logger.debug('Saving recording', filepath); const opusBuffer = await readRecordingFile(filepath); const blob = new Blob([opusBuffer], { - type: 'audio/mp4', + type: NATIVE_RECORDING_MIME_TYPE, }); const blobId = await blobEngine.set(blob); logger.debug('Recording saved', blobId); return { blob, blobId }; } -export function setupRecordingEvents(frameworkProvider: FrameworkProvider) { - events?.recording.onRecordingStatusChanged(status => { - (async () => { - if ((await apis?.ui.isActiveTab()) && status?.status === 'ready') { - using currentWorkspace = getCurrentWorkspace(frameworkProvider); - if (!currentWorkspace) { - // maybe the workspace is not ready yet, eg. for shared workspace view - await apis?.recording.handleBlockCreationFailed(status.id); - return; - } - const { workspace } = currentWorkspace; - const docsService = workspace.scope.get(DocsService); - const aiEnabled = isAiEnabled(frameworkProvider); +function shouldProcessRecording( + status: unknown +): status is ProcessingRecordingStatus { + return ( + !!status && + typeof status === 'object' && + 'status' in status && + status.status === 'processing' && + 'filepath' in status && + typeof status.filepath === 'string' && + !('blockCreationStatus' in status && status.blockCreationStatus) + ); +} - const timestamp = i18nTime(status.startTime, { - absolute: { - accuracy: 'minute', - noYear: true, - }, - }); +async function createRecordingDoc( + frameworkProvider: FrameworkProvider, + workspace: WorkspaceHandle['workspace'], + status: ProcessingRecordingStatus +) { + const docsService = workspace.scope.get(DocsService); + const aiEnabled = isAiEnabled(frameworkProvider); + const recordingFilepath = status.filepath; - const docProps: DocProps = { - onStoreLoad: (doc, { noteId }) => { - (async () => { - if (status.filepath) { - // it takes a while to save the blob, so we show the attachment first - const { blobId, blob } = await saveRecordingBlob( - doc.workspace.blobSync, - status.filepath - ); + const timestamp = i18nTime(status.startTime, { + absolute: { + accuracy: 'minute', + noYear: true, + }, + }); - // name + timestamp(readable) + extension - const attachmentName = - (status.appName ?? 'System Audio') + - ' ' + - timestamp + - '.opus'; + await new Promise((resolve, reject) => { + const docProps: DocProps = { + onStoreLoad: (doc, { noteId }) => { + void (async () => { + // it takes a while to save the blob, so we show the attachment first + const { blobId, blob } = await saveRecordingBlob( + doc.workspace.blobSync, + recordingFilepath + ); - // add size and sourceId to the attachment later - const attachmentId = doc.addBlock( - 'affine:attachment', - { - name: attachmentName, - type: 'audio/opus', - size: blob.size, - sourceId: blobId, - embed: true, - }, - noteId - ); + // name + timestamp(readable) + extension + const attachmentName = + (status.appName ?? 'System Audio') + ' ' + timestamp + '.opus'; - const model = doc.getBlock(attachmentId) - ?.model as AttachmentBlockModel; + const attachmentId = doc.addBlock( + 'affine:attachment', + { + name: attachmentName, + type: NATIVE_RECORDING_MIME_TYPE, + size: blob.size, + sourceId: blobId, + embed: true, + }, + noteId + ); - if (!aiEnabled) { - return; - } + const model = doc.getBlock(attachmentId) + ?.model as AttachmentBlockModel; - using currentWorkspace = getCurrentWorkspace(frameworkProvider); - if (!currentWorkspace) { - return; - } - const { workspace } = currentWorkspace; - using audioAttachment = workspace.scope - .get(AudioAttachmentService) - .get(model); - audioAttachment?.obj - .transcribe() - .then(() => { - track.doc.editor.audioBlock.transcribeRecording({ - type: 'Meeting record', - method: 'success', - option: 'Auto transcribing', - }); - }) - .catch(err => { - logger.error('Failed to transcribe recording', err); - }); - } else { - throw new Error('No attachment model found'); - } - })() - .then(async () => { - await apis?.recording.handleBlockCreationSuccess(status.id); - }) - .catch(error => { - logger.error('Failed to transcribe recording', error); - return apis?.recording.handleBlockCreationFailed( - status.id, - error - ); - }) - .catch(error => { - console.error('unknown error', error); + if (!aiEnabled) { + return; + } + + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + return; + } + const { workspace } = currentWorkspace; + using audioAttachment = workspace.scope + .get(AudioAttachmentService) + .get(model); + audioAttachment?.obj + .transcribe() + .then(() => { + track.doc.editor.audioBlock.transcribeRecording({ + type: 'Meeting record', + method: 'success', + option: 'Auto transcribing', }); - }, - }; - const page = docsService.createDoc({ - docProps, - title: - 'Recording ' + (status.appName ?? 'System Audio') + ' ' + timestamp, - primaryMode: 'page', - }); - workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); - } - })().catch(console.error); + }) + .catch(err => { + logger.error('Failed to transcribe recording', err); + }); + })().then(resolve, reject); + }, + }; + + const page = docsService.createDoc({ + docProps, + title: + 'Recording ' + (status.appName ?? 'System Audio') + ' ' + timestamp, + primaryMode: 'page', + }); + workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); + }); +} + +export function setupRecordingEvents(frameworkProvider: FrameworkProvider) { + let pendingStatus: ProcessingRecordingStatus | null = null; + let retryTimer: ReturnType | null = null; + let processingStatusId: number | null = null; + + const clearRetry = () => { + if (retryTimer !== null) { + clearTimeout(retryTimer); + retryTimer = null; + } + }; + + const clearPending = (id?: number) => { + if (id === undefined || pendingStatus?.id === id) { + pendingStatus = null; + clearRetry(); + } + if (id === undefined || processingStatusId === id) { + processingStatusId = null; + } + }; + + const scheduleRetry = () => { + if (!pendingStatus || retryTimer !== null) { + return; + } + retryTimer = setTimeout(() => { + retryTimer = null; + void processPendingStatus().catch(console.error); + }, RECORDING_PROCESS_RETRY_MS); + }; + + const processPendingStatus = async () => { + const status = pendingStatus; + if (!status || processingStatusId === status.id) { + return; + } + + let isActiveTab = false; + try { + isActiveTab = !!(await apis?.ui.isActiveTab()); + } catch (error) { + logger.error('Failed to probe active recording tab', error); + scheduleRetry(); + return; + } + + if (!isActiveTab) { + scheduleRetry(); + return; + } + + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + // Workspace can lag behind the post-recording status update for a short + // time; keep retrying instead of permanently failing the import. + scheduleRetry(); + return; + } + + processingStatusId = status.id; + + try { + await createRecordingDoc( + frameworkProvider, + currentWorkspace.workspace, + status + ); + await apis?.recording.setRecordingBlockCreationStatus( + status.id, + 'success' + ); + clearPending(status.id); + } catch (error) { + logger.error('Failed to create recording block', error); + try { + await apis?.recording.setRecordingBlockCreationStatus( + status.id, + 'failed', + error instanceof Error ? error.message : undefined + ); + } finally { + clearPending(status.id); + } + } finally { + if (pendingStatus?.id === status.id) { + processingStatusId = null; + scheduleRetry(); + } + } + }; + + events?.recording.onRecordingStatusChanged(status => { + if (shouldProcessRecording(status)) { + pendingStatus = status; + clearRetry(); + void processPendingStatus().catch(console.error); + return; + } + + if (!status) { + clearPending(); + return; + } + + if (pendingStatus?.id === status.id) { + clearPending(status.id); + } }); } diff --git a/packages/frontend/apps/electron-renderer/src/popup/recording/index.tsx b/packages/frontend/apps/electron-renderer/src/popup/recording/index.tsx index 3930f6b6d9..207cfb661c 100644 --- a/packages/frontend/apps/electron-renderer/src/popup/recording/index.tsx +++ b/packages/frontend/apps/electron-renderer/src/popup/recording/index.tsx @@ -1,28 +1,17 @@ import { Button } from '@affine/component'; import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks'; import { appIconMap } from '@affine/core/utils'; -import { - createStreamEncoder, - encodeRawBufferToOpus, - type OpusStreamEncoder, -} from '@affine/core/utils/opus-encoding'; import { apis, events } from '@affine/electron-api'; import { useI18n } from '@affine/i18n'; import track from '@affine/track'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import * as styles from './styles.css'; type Status = { id: number; - status: - | 'new' - | 'recording' - | 'paused' - | 'stopped' - | 'ready' - | 'create-block-success' - | 'create-block-failed'; + status: 'new' | 'recording' | 'processing' | 'ready'; + blockCreationStatus?: 'success' | 'failed'; appName?: string; appGroupId?: number; icon?: Buffer; @@ -58,6 +47,7 @@ const appIcon = appIconMap[BUILD_CONFIG.appBuildType]; export function Recording() { const status = useRecordingStatus(); + const trackedNewRecordingIdsRef = useRef>(new Set()); const t = useI18n(); const textElement = useMemo(() => { @@ -66,14 +56,19 @@ export function Recording() { } if (status.status === 'new') { return t['com.affine.recording.new'](); - } else if (status.status === 'create-block-success') { + } else if ( + status.status === 'ready' && + status.blockCreationStatus === 'success' + ) { return t['com.affine.recording.success.prompt'](); - } else if (status.status === 'create-block-failed') { + } else if ( + status.status === 'ready' && + status.blockCreationStatus === 'failed' + ) { return t['com.affine.recording.failed.prompt'](); } else if ( status.status === 'recording' || - status.status === 'ready' || - status.status === 'stopped' + status.status === 'processing' ) { if (status.appName) { return t['com.affine.recording.recording']({ @@ -105,106 +100,16 @@ export function Recording() { await apis?.recording?.stopRecording(status.id); }, [status]); - const handleProcessStoppedRecording = useAsyncCallback( - async (currentStreamEncoder?: OpusStreamEncoder) => { - let id: number | undefined; - try { - const result = await apis?.recording?.getCurrentRecording(); - - if (!result) { - return; - } - - id = result.id; - - const { filepath, sampleRate, numberOfChannels } = result; - if (!filepath || !sampleRate || !numberOfChannels) { - return; - } - const [buffer] = await Promise.all([ - currentStreamEncoder - ? currentStreamEncoder.finish() - : encodeRawBufferToOpus({ - filepath, - sampleRate, - numberOfChannels, - }), - new Promise(resolve => { - setTimeout(() => { - resolve(); - }, 500); // wait at least 500ms for better user experience - }), - ]); - await apis?.recording.readyRecording(result.id, buffer); - } catch (error) { - console.error('Failed to stop recording', error); - await apis?.popup?.dismissCurrentRecording(); - if (id) { - await apis?.recording.removeRecording(id); - } - } - }, - [] - ); - useEffect(() => { - let removed = false; - let currentStreamEncoder: OpusStreamEncoder | undefined; + if (!status || status.status !== 'new') return; + if (trackedNewRecordingIdsRef.current.has(status.id)) return; - apis?.recording - .getCurrentRecording() - .then(status => { - if (status) { - return handleRecordingStatusChanged(status); - } - return; - }) - .catch(console.error); - - const handleRecordingStatusChanged = async (status: Status) => { - if (removed) { - return; - } - if (status?.status === 'new') { - track.popup.$.recordingBar.toggleRecordingBar({ - type: 'Meeting record', - appName: status.appName || 'System Audio', - }); - } - - if ( - status?.status === 'recording' && - status.sampleRate && - status.numberOfChannels && - (!currentStreamEncoder || currentStreamEncoder.id !== status.id) - ) { - currentStreamEncoder?.close(); - currentStreamEncoder = createStreamEncoder(status.id, { - sampleRate: status.sampleRate, - numberOfChannels: status.numberOfChannels, - }); - currentStreamEncoder.poll().catch(console.error); - } - - if (status?.status === 'stopped') { - handleProcessStoppedRecording(currentStreamEncoder); - currentStreamEncoder = undefined; - } - }; - - // allow processing stopped event in tray menu as well: - const unsubscribe = events?.recording.onRecordingStatusChanged(status => { - if (status) { - handleRecordingStatusChanged(status).catch(console.error); - } + trackedNewRecordingIdsRef.current.add(status.id); + track.popup.$.recordingBar.toggleRecordingBar({ + type: 'Meeting record', + appName: status.appName || 'System Audio', }); - - return () => { - removed = true; - unsubscribe?.(); - currentStreamEncoder?.close(); - }; - }, [handleProcessStoppedRecording]); + }, [status]); const handleStartRecording = useAsyncCallback(async () => { if (!status) { @@ -249,7 +154,10 @@ export function Recording() { {t['com.affine.recording.stop']()} ); - } else if (status.status === 'stopped' || status.status === 'ready') { + } else if ( + status.status === 'processing' || + (status.status === 'ready' && !status.blockCreationStatus) + ) { return ( ); - } else if (status.status === 'create-block-failed') { + } else if ( + status.status === 'ready' && + status.blockCreationStatus === 'failed' + ) { return ( <>