Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-13 21:05:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(editor): audio block (#10947)

Browse Source 
AudioMedia entity for loading & controlling a single audio media
AudioMediaManagerService: Global audio state synchronization across tabs
AudioAttachmentService + AudioAttachmentBlock for manipulating AttachmentBlock in affine - e.g., filling transcription (using mock endpoint for now)
Added AudioBlock + AudioPlayer for rendering audio block in affine (new transcription block whose renderer is provided in affine)

fix AF-2292
fix AF-2337
This commit is contained in:
pengx17
2025-03-20 12:46:14 +00:00
parent 8a5393ea50
commit fad49bb070
120 changed files with 5407 additions and 950 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/frontend/apps/electron-renderer/package.json
View File

@@ -10,6 +10,7 @@
"dependencies": {
"@affine/component": "workspace:*",
"@affine/core": "workspace:*",
"@affine/debug": "workspace:*",
"@affine/electron-api": "workspace:*",
"@affine/i18n": "workspace:*",
"@affine/nbstore": "workspace:*",
@@ -24,7 +25,8 @@
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^6.28.0",
"uuid": "^11.0.3"
"uuid": "^11.0.3",
"webm-muxer": "^5.0.3"
},
"devDependencies": {
"@affine-tools/utils": "workspace:*",

331
packages/frontend/apps/electron-renderer/src/app.tsx
View File

@@ -1,331 +0,0 @@
import type { DocProps } from '@affine/core/blocksuite/initialization';
import { AffineContext } from '@affine/core/components/context';
import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls';
import { AppContainer } from '@affine/core/desktop/components/app-container';
import { router } from '@affine/core/desktop/router';
import { configureCommonModules } from '@affine/core/modules';
import { configureAppTabsHeaderModule } from '@affine/core/modules/app-tabs-header';
import { configureDesktopBackupModule } from '@affine/core/modules/backup';
import { ValidatorProvider } from '@affine/core/modules/cloud';
import {
configureDesktopApiModule,
DesktopApiService,
} from '@affine/core/modules/desktop-api';
import { WorkspaceDialogService } from '@affine/core/modules/dialogs';
import { DocsService } from '@affine/core/modules/doc';
import {
configureSpellCheckSettingModule,
EditorSettingService,
} from '@affine/core/modules/editor-setting';
import { configureFindInPageModule } from '@affine/core/modules/find-in-page';
import { GlobalContextService } from '@affine/core/modules/global-context';
import { I18nProvider } from '@affine/core/modules/i18n';
import { JournalService } from '@affine/core/modules/journal';
import { LifecycleService } from '@affine/core/modules/lifecycle';
import {
configureElectronStateStorageImpls,
NbstoreProvider,
} from '@affine/core/modules/storage';
import {
ClientSchemeProvider,
PopupWindowProvider,
} from '@affine/core/modules/url';
import {
configureDesktopWorkbenchModule,
WorkbenchService,
} from '@affine/core/modules/workbench';
import { WorkspacesService } from '@affine/core/modules/workspace';
import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
import createEmotionCache from '@affine/core/utils/create-emotion-cache';
import { apis, events } from '@affine/electron-api';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import type { AttachmentBlockProps } from '@blocksuite/affine/model';
import { Text } from '@blocksuite/affine/store';
import { CacheProvider } from '@emotion/react';
import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
import { Suspense } from 'react';
import { RouterProvider } from 'react-router-dom';
import { v4 as uuid } from 'uuid';
import { DesktopThemeSync } from './theme-sync';
const storeManagerClient = createStoreManagerClient();
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
const desktopWhiteList = [
'/open-app/signin-redirect',
'/open-app/url',
'/upgrade-success',
'/ai-upgrade-success',
'/share',
'/oauth',
'/magic-link',
];
if (
!BUILD_CONFIG.isElectron &&
BUILD_CONFIG.debug &&
desktopWhiteList.every(path => !location.pathname.startsWith(path))
) {
document.body.innerHTML = `<h1 style="color:red;font-size:5rem;text-align:center;">Don't run electron entry in browser.</h1>`;
throw new Error('Wrong distribution');
}
const cache = createEmotionCache();
const future = {
v7_startTransition: true,
} as const;
const framework = new Framework();
configureCommonModules(framework);
configureElectronStateStorageImpls(framework);
configureBrowserWorkspaceFlavours(framework);
configureDesktopWorkbenchModule(framework);
configureAppTabsHeaderModule(framework);
configureFindInPageModule(framework);
configureDesktopApiModule(framework);
configureSpellCheckSettingModule(framework);
configureDesktopBackupModule(framework);
framework.impl(NbstoreProvider, {
openStore(key, options) {
const { store, dispose } = storeManagerClient.open(key, options);
return {
store,
dispose: () => {
dispose();
},
};
},
});
framework.impl(PopupWindowProvider, p => {
const apis = p.get(DesktopApiService).api;
return {
open: (url: string) => {
apis.handler.ui.openExternal(url).catch(e => {
console.error('Failed to open external URL', e);
});
},
};
});
framework.impl(ClientSchemeProvider, p => {
const appInfo = p.get(DesktopApiService).appInfo;
return {
getClientScheme() {
return appInfo?.scheme;
},
};
});
framework.impl(ValidatorProvider, p => {
const apis = p.get(DesktopApiService).api;
return {
async validate(_challenge, resource) {
const token = await apis.handler.ui.getChallengeResponse(resource);
if (!token) {
throw new Error('Challenge failed');
}
return token;
},
};
});
const frameworkProvider = framework.provider();
// setup application lifecycle events, and emit application start event
window.addEventListener('focus', () => {
frameworkProvider.get(LifecycleService).applicationFocus();
});
frameworkProvider.get(LifecycleService).applicationStart();
window.addEventListener('unload', () => {
frameworkProvider
.get(DesktopApiService)
.api.handler.ui.pingAppLayoutReady(false)
.catch(console.error);
});
function getCurrentWorkspace() {
const currentWorkspaceId = frameworkProvider
.get(GlobalContextService)
.globalContext.workspaceId.get();
const workspacesService = frameworkProvider.get(WorkspacesService);
const workspaceRef = currentWorkspaceId
? workspacesService.openByWorkspaceId(currentWorkspaceId)
: null;
if (!workspaceRef) {
return;
}
const { workspace, dispose } = workspaceRef;
return {
workspace,
dispose,
};
}
events?.applicationMenu.openAboutPageInSettingModal(() => {
const currentWorkspace = getCurrentWorkspace();
if (!currentWorkspace) {
return;
}
const { workspace, dispose } = currentWorkspace;
workspace.scope.get(WorkspaceDialogService).open('setting', {
activeTab: 'about',
});
dispose();
});
events?.applicationMenu.onNewPageAction(type => {
apis?.ui
.isActiveTab()
.then(isActive => {
if (!isActive) {
return;
}
const currentWorkspace = getCurrentWorkspace();
if (!currentWorkspace) {
return;
}
const { workspace, dispose } = currentWorkspace;
const editorSettingService = frameworkProvider.get(EditorSettingService);
const docsService = workspace.scope.get(DocsService);
const editorSetting = editorSettingService.editorSetting;
const docProps = {
note: editorSetting.get('affine:note'),
};
const page = docsService.createDoc({ docProps, primaryMode: type });
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
dispose();
})
.catch(err => {
console.error(err);
});
});
events?.recording.onRecordingStatusChanged(status => {
(async () => {
if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
const currentWorkspace = getCurrentWorkspace();
if (!currentWorkspace) {
return;
}
const { workspace, dispose } = currentWorkspace;
const editorSettingService = frameworkProvider.get(EditorSettingService);
const docsService = workspace.scope.get(DocsService);
const editorSetting = editorSettingService.editorSetting;
const docProps: DocProps = {
note: editorSetting.get('affine:note'),
page: {
title: new Text(
'Recording ' +
(status.appGroup?.name ?? 'System Audio') +
' ' +
new Date(status.startTime).toISOString()
),
},
onStoreLoad: (doc, { noteId }) => {
(async () => {
const data = await apis?.recording.saveRecording(status.id);
if (!data) {
return;
}
const blob = new Blob([data], { type: 'audio/mp3' });
const blobId = await doc.workspace.blobSync.set(blob);
const attachmentProps: Partial<AttachmentBlockProps> = {
name: 'Recording',
size: blob.size,
type: 'audio/mp3',
sourceId: blobId,
embed: true,
};
doc.addBlock('affine:attachment', attachmentProps, noteId);
})().catch(console.error);
},
};
const page = docsService.createDoc({ docProps, primaryMode: 'page' });
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
dispose();
}
})().catch(console.error);
});
events?.applicationMenu.onOpenJournal(() => {
const currentWorkspace = getCurrentWorkspace();
if (!currentWorkspace) {
return;
}
const { workspace, dispose } = currentWorkspace;
const workbench = workspace.scope.get(WorkbenchService).workbench;
const journalService = workspace.scope.get(JournalService);
const docId = journalService.ensureJournalByDate(new Date()).id;
workbench.openDoc(docId);
dispose();
});
export function App() {
return (
<Suspense>
<FrameworkRoot framework={frameworkProvider}>
<CacheProvider value={cache}>
<I18nProvider>
<AffineContext store={getCurrentStore()}>
<DesktopThemeSync />
<RouterProvider
fallbackElement={<AppContainer fallback />}
router={router}
future={future}
/>
{environment.isWindows && (
<div style={{ position: 'fixed', right: 0, top: 0, zIndex: 5 }}>
<WindowsAppControls />
</div>
)}
</AffineContext>
</I18nProvider>
</CacheProvider>
</FrameworkRoot>
</Suspense>
);
}
function createStoreManagerClient() {
const { port1: portForOpClient, port2: portForWorker } = new MessageChannel();
let portFromWorker: MessagePort | null = null;
let portId = uuid();
const handleMessage = (ev: MessageEvent) => {
if (
ev.data.type === 'electron:worker-connect' &&
ev.data.portId === portId
) {
portFromWorker = ev.ports[0];
// connect portForWorker and portFromWorker
portFromWorker.addEventListener('message', ev => {
portForWorker.postMessage(ev.data, [...ev.ports]);
});
portForWorker.addEventListener('message', ev => {
// oxlint-disable-next-line no-non-null-assertion
portFromWorker!.postMessage(ev.data, [...ev.ports]);
});
portForWorker.start();
portFromWorker.start();
}
};
window.addEventListener('message', handleMessage);
// oxlint-disable-next-line no-non-null-assertion
apis!.worker.connectWorker('affine-shared-worker', portId).catch(err => {
console.error('failed to connect worker', err);
});
const storeManager = new StoreManagerClient(new OpClient(portForOpClient));
portForOpClient.start();
return storeManager;
}

0
packages/frontend/apps/electron-renderer/src/app.css.ts → packages/frontend/apps/electron-renderer/src/app/app.css.ts
View File

65
packages/frontend/apps/electron-renderer/src/app/app.tsx Normal file
View File

@@ -0,0 +1,65 @@
import { AffineContext } from '@affine/core/components/context';
import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls';
import { AppContainer } from '@affine/core/desktop/components/app-container';
import { router } from '@affine/core/desktop/router';
import { I18nProvider } from '@affine/core/modules/i18n';
import createEmotionCache from '@affine/core/utils/create-emotion-cache';
import { CacheProvider } from '@emotion/react';
import { FrameworkRoot, getCurrentStore } from '@toeverything/infra';
import { Suspense } from 'react';
import { RouterProvider } from 'react-router-dom';
import { setupEffects } from './effects';
import { DesktopThemeSync } from './theme-sync';
const { frameworkProvider } = setupEffects();
const desktopWhiteList = [
'/open-app/signin-redirect',
'/open-app/url',
'/upgrade-success',
'/ai-upgrade-success',
'/share',
'/oauth',
'/magic-link',
];
if (
!BUILD_CONFIG.isElectron &&
BUILD_CONFIG.debug &&
desktopWhiteList.every(path => !location.pathname.startsWith(path))
) {
document.body.innerHTML = `<h1 style="color:red;font-size:5rem;text-align:center;">Don't run electron entry in browser.</h1>`;
throw new Error('Wrong distribution');
}
const cache = createEmotionCache();
const future = {
v7_startTransition: true,
} as const;
export function App() {
return (
<Suspense>
<FrameworkRoot framework={frameworkProvider}>
<CacheProvider value={cache}>
<I18nProvider>
<AffineContext store={getCurrentStore()}>
<DesktopThemeSync />
<RouterProvider
fallbackElement={<AppContainer fallback />}
router={router}
future={future}
/>
{environment.isWindows && (
<div style={{ position: 'fixed', right: 0, top: 0, zIndex: 5 }}>
<WindowsAppControls />
</div>
)}
</AffineContext>
</I18nProvider>
</CacheProvider>
</FrameworkRoot>
</Suspense>
);
}

82
packages/frontend/apps/electron-renderer/src/app/effects/events.ts Normal file
View File

@@ -0,0 +1,82 @@
import { DesktopApiService } from '@affine/core/modules/desktop-api';
import { WorkspaceDialogService } from '@affine/core/modules/dialogs';
import { DocsService } from '@affine/core/modules/doc';
import { EditorSettingService } from '@affine/core/modules/editor-setting';
import { JournalService } from '@affine/core/modules/journal';
import { LifecycleService } from '@affine/core/modules/lifecycle';
import { WorkbenchService } from '@affine/core/modules/workbench';
import { apis, events } from '@affine/electron-api';
import type { FrameworkProvider } from '@toeverything/infra';
import { setupRecordingEvents } from './recording';
import { getCurrentWorkspace } from './utils';
export function setupEvents(frameworkProvider: FrameworkProvider) {
// setup application lifecycle events, and emit application start event
window.addEventListener('focus', () => {
frameworkProvider.get(LifecycleService).applicationFocus();
});
frameworkProvider.get(LifecycleService).applicationStart();
window.addEventListener('unload', () => {
frameworkProvider
.get(DesktopApiService)
.api.handler.ui.pingAppLayoutReady(false)
.catch(console.error);
});
events?.applicationMenu.openAboutPageInSettingModal(() => {
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace } = currentWorkspace;
workspace.scope.get(WorkspaceDialogService).open('setting', {
activeTab: 'about',
});
});
events?.applicationMenu.onNewPageAction(type => {
apis?.ui
.isActiveTab()
.then(isActive => {
if (!isActive) {
return;
}
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace } = currentWorkspace;
const editorSettingService =
frameworkProvider.get(EditorSettingService);
const docsService = workspace.scope.get(DocsService);
const editorSetting = editorSettingService.editorSetting;
const docProps = {
note: editorSetting.get('affine:note'),
};
const page = docsService.createDoc({ docProps, primaryMode: type });
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
})
.catch(err => {
console.error(err);
});
});
events?.applicationMenu.onOpenJournal(() => {
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace, dispose } = currentWorkspace;
const workbench = workspace.scope.get(WorkbenchService).workbench;
const journalService = workspace.scope.get(JournalService);
const docId = journalService.ensureJournalByDate(new Date()).id;
workbench.openDoc(docId);
dispose();
});
setupRecordingEvents(frameworkProvider);
}

10
packages/frontend/apps/electron-renderer/src/app/effects/index.ts Normal file
View File

@@ -0,0 +1,10 @@
import { setupEvents } from './events';
import { setupModules } from './modules';
import { setupStoreManager } from './store-manager';
export function setupEffects() {
const { framework, frameworkProvider } = setupModules();
setupStoreManager(framework);
setupEvents(frameworkProvider);
return { framework, frameworkProvider };
}

66
packages/frontend/apps/electron-renderer/src/app/effects/modules.ts Normal file
View File

@@ -0,0 +1,66 @@
import { configureCommonModules } from '@affine/core/modules';
import { configureAppTabsHeaderModule } from '@affine/core/modules/app-tabs-header';
import { configureDesktopBackupModule } from '@affine/core/modules/backup';
import { ValidatorProvider } from '@affine/core/modules/cloud';
import {
configureDesktopApiModule,
DesktopApiService,
} from '@affine/core/modules/desktop-api';
import { configureSpellCheckSettingModule } from '@affine/core/modules/editor-setting';
import { configureFindInPageModule } from '@affine/core/modules/find-in-page';
import { configureElectronStateStorageImpls } from '@affine/core/modules/storage';
import {
ClientSchemeProvider,
PopupWindowProvider,
} from '@affine/core/modules/url';
import { configureDesktopWorkbenchModule } from '@affine/core/modules/workbench';
import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
import { Framework } from '@toeverything/infra';
export function setupModules() {
const framework = new Framework();
configureCommonModules(framework);
configureElectronStateStorageImpls(framework);
configureBrowserWorkspaceFlavours(framework);
configureDesktopWorkbenchModule(framework);
configureAppTabsHeaderModule(framework);
configureFindInPageModule(framework);
configureDesktopApiModule(framework);
configureSpellCheckSettingModule(framework);
configureDesktopBackupModule(framework);
framework.impl(PopupWindowProvider, p => {
const apis = p.get(DesktopApiService).api;
return {
open: (url: string) => {
apis.handler.ui.openExternal(url).catch(e => {
console.error('Failed to open external URL', e);
});
},
};
});
framework.impl(ClientSchemeProvider, p => {
const appInfo = p.get(DesktopApiService).appInfo;
return {
getClientScheme() {
return appInfo?.scheme;
},
};
});
framework.impl(ValidatorProvider, p => {
const apis = p.get(DesktopApiService).api;
return {
async validate(_challenge, resource) {
const token = await apis.handler.ui.getChallengeResponse(resource);
if (!token) {
throw new Error('Challenge failed');
}
return token;
},
};
});
const frameworkProvider = framework.provider();
return { framework, frameworkProvider };
}

225
packages/frontend/apps/electron-renderer/src/app/effects/recording.ts Normal file
View File

@@ -0,0 +1,225 @@
import type { DocProps } from '@affine/core/blocksuite/initialization';
import { DocsService } from '@affine/core/modules/doc';
import { EditorSettingService } from '@affine/core/modules/editor-setting';
import { AudioAttachmentService } from '@affine/core/modules/media/services/audio-attachment';
import { WorkbenchService } from '@affine/core/modules/workbench';
import { DebugLogger } from '@affine/debug';
import { apis, events } from '@affine/electron-api';
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
import { Text } from '@blocksuite/affine/store';
import type { BlobEngine } from '@blocksuite/affine/sync';
import type { FrameworkProvider } from '@toeverything/infra';
import { ArrayBufferTarget, Muxer } from 'webm-muxer';
import { getCurrentWorkspace } from './utils';
const logger = new DebugLogger('electron-renderer:recording');
/**
* Encodes raw audio data to Opus in WebM container.
*/
async function encodeRawBufferToOpus({
filepath,
sampleRate,
numberOfChannels,
}: {
filepath: string;
sampleRate: number;
numberOfChannels: number;
}): Promise<Uint8Array> {
// Use streams to process audio data incrementally
const response = await fetch(new URL(filepath, location.origin));
if (!response.body) {
throw new Error('Response body is null');
}
// Setup Opus encoder
const encodedChunks: EncodedAudioChunk[] = [];
const encoder = new AudioEncoder({
output: chunk => {
encodedChunks.push(chunk);
},
error: err => {
throw new Error(`Encoding error: ${err}`);
},
});
// Configure Opus encoder
encoder.configure({
codec: 'opus',
sampleRate: sampleRate,
numberOfChannels: numberOfChannels,
bitrate: 96000, // 96 kbps is good for stereo audio
});
// Process the stream
const reader = response.body.getReader();
let offset = 0;
const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
// Convert the chunk to Float32Array
const float32Data = new Float32Array(value.buffer);
// Process in smaller chunks to avoid large frames
for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
const chunk = float32Data.subarray(i, i + chunkSize);
// Create and encode frame
const frame = new AudioData({
format: 'f32',
sampleRate: sampleRate,
numberOfFrames: chunk.length / numberOfChannels,
numberOfChannels: numberOfChannels,
timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
data: chunk,
});
encoder.encode(frame);
frame.close();
offset += chunk.length / numberOfChannels;
}
}
} finally {
await encoder.flush();
encoder.close();
}
if (encodedChunks.length === 0) {
throw new Error('No chunks were produced during encoding');
}
// Initialize WebM muxer
const target = new ArrayBufferTarget();
const muxer = new Muxer({
target,
audio: {
codec: 'A_OPUS',
sampleRate: sampleRate,
numberOfChannels: numberOfChannels,
},
});
// Add all chunks to the muxer
for (const chunk of encodedChunks) {
muxer.addAudioChunk(chunk, {});
}
// Finalize and get WebM container
muxer.finalize();
const { buffer: webmBuffer } = target;
return new Uint8Array(webmBuffer);
}
async function saveRecordingBlob(
blobEngine: BlobEngine,
recording: {
id: number;
filepath: string;
sampleRate: number;
numberOfChannels: number;
}
) {
logger.debug('Saving recording', recording.id);
const opusBuffer = await encodeRawBufferToOpus({
filepath: recording.filepath,
sampleRate: recording.sampleRate,
numberOfChannels: recording.numberOfChannels,
});
const blob = new Blob([opusBuffer], {
type: 'audio/webm',
});
const blobId = await blobEngine.set(blob);
logger.debug('Recording saved', blobId);
return { blob, blobId };
}
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
events?.recording.onRecordingStatusChanged(status => {
(async () => {
if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace } = currentWorkspace;
const editorSettingService =
frameworkProvider.get(EditorSettingService);
const docsService = workspace.scope.get(DocsService);
const editorSetting = editorSettingService.editorSetting;
const docProps: DocProps = {
note: editorSetting.get('affine:note'),
page: {
title: new Text(
'Recording ' +
(status.appGroup?.name ?? 'System Audio') +
' ' +
new Date(status.startTime).toISOString()
),
},
onStoreLoad: (doc, { noteId }) => {
(async () => {
const recording = await apis?.recording.getRecording(status.id);
if (!recording) {
logger.error('Failed to save recording');
return;
}
// name + timestamp(readable) + extension
const attachmentName =
(status.appGroup?.name ?? 'System Audio') +
' ' +
new Date(status.startTime).toISOString() +
'.webm';
// add size and sourceId to the attachment later
const attachmentId = doc.addBlock(
'affine:attachment',
{
name: attachmentName,
type: 'audio/webm',
},
noteId
);
const model = doc.getBlock(attachmentId)
?.model as AttachmentBlockModel;
if (model) {
// it takes a while to save the blob, so we show the attachment first
const { blobId, blob } = await saveRecordingBlob(
doc.workspace.blobSync,
recording
);
model.props.size = blob.size;
model.props.sourceId = blobId;
model.props.embed = true;
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
if (!currentWorkspace) {
return;
}
const { workspace } = currentWorkspace;
using audioAttachment = workspace.scope
.get(AudioAttachmentService)
.get(model);
audioAttachment?.obj.transcribe();
}
})().catch(console.error);
},
};
const page = docsService.createDoc({ docProps, primaryMode: 'page' });
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
}
})().catch(console.error);
});
}

62
packages/frontend/apps/electron-renderer/src/app/effects/store-manager.ts Normal file
View File

@@ -0,0 +1,62 @@
import { NbstoreProvider } from '@affine/core/modules/storage';
import { apis } from '@affine/electron-api';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
import type { Framework } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
import { v4 as uuid } from 'uuid';
function createStoreManagerClient() {
const { port1: portForOpClient, port2: portForWorker } = new MessageChannel();
let portFromWorker: MessagePort | null = null;
let portId = uuid();
const handleMessage = (ev: MessageEvent) => {
if (
ev.data.type === 'electron:worker-connect' &&
ev.data.portId === portId
) {
portFromWorker = ev.ports[0];
// connect portForWorker and portFromWorker
portFromWorker.addEventListener('message', ev => {
portForWorker.postMessage(ev.data, [...ev.ports]);
});
portForWorker.addEventListener('message', ev => {
// oxlint-disable-next-line no-non-null-assertion
portFromWorker!.postMessage(ev.data, [...ev.ports]);
});
portForWorker.start();
portFromWorker.start();
}
};
window.addEventListener('message', handleMessage);
// oxlint-disable-next-line no-non-null-assertion
apis!.worker.connectWorker('affine-shared-worker', portId).catch(err => {
console.error('failed to connect worker', err);
});
const storeManager = new StoreManagerClient(new OpClient(portForOpClient));
portForOpClient.start();
return storeManager;
}
export function setupStoreManager(framework: Framework) {
const storeManagerClient = createStoreManagerClient();
window.addEventListener('beforeunload', () => {
storeManagerClient.dispose();
});
framework.impl(NbstoreProvider, {
openStore(key, options) {
const { store, dispose } = storeManagerClient.open(key, options);
return {
store,
dispose: () => {
dispose();
},
};
},
});
}

23
packages/frontend/apps/electron-renderer/src/app/effects/utils.ts Normal file
View File

@@ -0,0 +1,23 @@
import { GlobalContextService } from '@affine/core/modules/global-context';
import { WorkspacesService } from '@affine/core/modules/workspace';
import type { FrameworkProvider } from '@toeverything/infra';
export function getCurrentWorkspace(frameworkProvider: FrameworkProvider) {
const currentWorkspaceId = frameworkProvider
.get(GlobalContextService)
.globalContext.workspaceId.get();
const workspacesService = frameworkProvider.get(WorkspacesService);
const workspaceRef = currentWorkspaceId
? workspacesService.openByWorkspaceId(currentWorkspaceId)
: null;
if (!workspaceRef) {
return;
}
const { workspace, dispose } = workspaceRef;
return {
workspace,
dispose,
[Symbol.dispose]: dispose,
};
}

0
packages/frontend/apps/electron-renderer/src/global.css → packages/frontend/apps/electron-renderer/src/app/global.css
View File

0
packages/frontend/apps/electron-renderer/src/index.tsx → packages/frontend/apps/electron-renderer/src/app/index.tsx
View File

0
packages/frontend/apps/electron-renderer/src/setup.ts → packages/frontend/apps/electron-renderer/src/app/setup.ts
View File

0
packages/frontend/apps/electron-renderer/src/theme-sync.ts → packages/frontend/apps/electron-renderer/src/app/theme-sync.ts
View File

2
packages/frontend/apps/electron-renderer/src/shell/setup.ts
View File

@@ -1,3 +1,3 @@
import '@affine/core/bootstrap/electron';
import '@affine/component/theme';
import '../global.css';
import '../app/global.css';

1
packages/frontend/apps/electron-renderer/tsconfig.json
View File

@@ -10,6 +10,7 @@
"references": [
{ "path": "../../component" },
{ "path": "../../core" },
{ "path": "../../../common/debug" },
{ "path": "../../electron-api" },
{ "path": "../../i18n" },
{ "path": "../../../common/nbstore" },

2
packages/frontend/apps/electron-renderer/webpack.config.ts
View File

@@ -1,6 +1,6 @@
export const config = {
entry: {
app: './src/index.tsx',
app: './src/app/index.tsx',
shell: './src/shell/index.tsx',
backgroundWorker: './src/background-worker/index.ts',
},

4
packages/frontend/apps/electron/src/helper/index.ts
View File

@@ -12,7 +12,7 @@ function setupRendererConnection(rendererPort: Electron.MessagePortMain) {
try {
const start = performance.now();
const result = await handler(...args);
logger.info(
logger.debug(
'[async-api]',
`${namespace}.${name}`,
args.filter(
@@ -74,7 +74,7 @@ function main() {
if (e.data.channel === 'renderer-connect' && e.ports.length === 1) {
const rendererPort = e.ports[0];
setupRendererConnection(rendererPort);
logger.info('[helper] renderer connected');
logger.debug('[helper] renderer connected');
}
});
}

1
packages/frontend/apps/electron/src/helper/logger.ts
View File

@@ -3,3 +3,4 @@ import log from 'electron-log/main';
export const logger = log.scope('helper');
log.transports.file.level = 'info';
log.transports.console.level = 'info';

4
packages/frontend/apps/electron/src/helper/main-rpc.ts
View File

@@ -2,7 +2,6 @@ import { AsyncCall } from 'async-call-rpc';
import type { HelperToMain, MainToHelper } from '../shared/type';
import { exposed } from './provide';
import { encodeToMp3 } from './recording/encode';
const helperToMainServer: HelperToMain = {
getMeta: () => {
@@ -11,8 +10,6 @@ const helperToMainServer: HelperToMain = {
}
return exposed;
},
// allow main process encode audio samples to mp3 buffer (because it is slow and blocking)
encodeToMp3,
};
export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {
@@ -33,4 +30,5 @@ export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {
process.parentPort.postMessage(data);
},
},
log: false,
});

16
packages/frontend/apps/electron/src/helper/recording/encode.ts
View File

@@ -1,16 +0,0 @@
import { Mp3Encoder } from '@affine/native';
// encode audio samples to mp3 buffer
export function encodeToMp3(
samples: Float32Array,
opts: {
channels?: number;
sampleRate?: number;
} = {}
): Uint8Array {
const mp3Encoder = new Mp3Encoder({
channels: opts.channels ?? 2,
sampleRate: opts.sampleRate ?? 44100,
});
return mp3Encoder.encode(samples);
}

42
packages/frontend/apps/electron/src/main/cleanup.ts
View File

@@ -1,15 +1,22 @@
import type { MediaStats } from '@toeverything/infra';
import { app } from 'electron';
import { logger } from './logger';
import { globalStateStorage } from './shared-storage/storage';
const cleanupRegistry: (() => void)[] = [];
const beforeAppQuitRegistry: (() => void)[] = [];
const beforeTabCloseRegistry: ((tabId: string) => void)[] = [];
export function beforeAppQuit(fn: () => void) {
cleanupRegistry.push(fn);
beforeAppQuitRegistry.push(fn);
}
export function beforeTabClose(fn: (tabId: string) => void) {
beforeTabCloseRegistry.push(fn);
}
app.on('before-quit', () => {
cleanupRegistry.forEach(fn => {
beforeAppQuitRegistry.forEach(fn => {
// some cleanup functions might throw on quit and crash the app
try {
fn();
@@ -18,3 +25,32 @@ app.on('before-quit', () => {
}
});
});
export function onTabClose(tabId: string) {
beforeTabCloseRegistry.forEach(fn => {
try {
fn(tabId);
} catch (err) {
logger.warn('cleanup error on tab close', err);
}
});
}
app.on('ready', () => {
globalStateStorage.set('media:playback-state', null);
globalStateStorage.set('media:stats', null);
});
beforeAppQuit(() => {
globalStateStorage.set('media:playback-state', null);
globalStateStorage.set('media:stats', null);
});
// set audio play state
beforeTabClose(tabId => {
const stats = globalStateStorage.get<MediaStats | null>('media:stats');
if (stats && stats.tabId === tabId) {
globalStateStorage.set('media:playback-state', null);
globalStateStorage.set('media:stats', null);
}
});

1
packages/frontend/apps/electron/src/main/helper-process.ts
View File

@@ -115,6 +115,7 @@ class HelperProcessManager {
unknownMessage: false,
},
channel: new MessageEventChannel(this.#process),
log: false,
});
}
}

4
packages/frontend/apps/electron/src/main/index.ts
View File

@@ -15,7 +15,7 @@ import { registerHandlers } from './handlers';
import { logger } from './logger';
import { registerProtocol } from './protocol';
import { setupRecording } from './recording';
import { getTrayState } from './tray';
import { setupTrayState } from './tray';
import { registerUpdater } from './updater';
import { launch } from './windows-manager/launcher';
import { launchStage } from './windows-manager/stage';
@@ -89,7 +89,7 @@ app
.then(launch)
.then(setupRecording)
.then(createApplicationMenu)
.then(getTrayState)
.then(setupTrayState)
.then(registerUpdater)
.catch(e => console.error('Failed create window:', e));

1
packages/frontend/apps/electron/src/main/logger.ts
View File

@@ -7,6 +7,7 @@ log.initialize({
});
log.transports.file.level = 'info';
log.transports.console.level = 'info';
export function getLogFilePath() {
return log.transports.file.getFile().path;

38
packages/frontend/apps/electron/src/main/protocol.ts
View File

@@ -1,6 +1,6 @@
import { join } from 'node:path';
import { net, protocol, session } from 'electron';
import { app, net, protocol, session } from 'electron';
import cookieParser from 'set-cookie-parser';
import { resourcesPath } from '../shared/utils';
@@ -43,8 +43,10 @@ function isNetworkResource(pathname: string) {
async function handleFileRequest(request: Request) {
const urlObject = new URL(request.url);
const isAbsolutePath = urlObject.host !== '.';
// Redirect to webpack dev server if defined
if (process.env.DEV_SERVER_URL) {
if (process.env.DEV_SERVER_URL && !isAbsolutePath) {
const devServerUrl = new URL(
urlObject.pathname,
process.env.DEV_SERVER_URL
@@ -56,20 +58,30 @@ async function handleFileRequest(request: Request) {
});
// this will be file types (in the web-static folder)
let filepath = '';
// if is a file type, load the file in resources
if (urlObject.pathname.split('/').at(-1)?.includes('.')) {
// Sanitize pathname to prevent path traversal attacks
const decodedPath = decodeURIComponent(urlObject.pathname);
const normalizedPath = join(webStaticDir, decodedPath).normalize();
if (!normalizedPath.startsWith(webStaticDir)) {
// Attempted path traversal - reject by using empty path
filepath = join(webStaticDir, '');
// for relative path, load the file in resources
if (!isAbsolutePath) {
if (urlObject.pathname.split('/').at(-1)?.includes('.')) {
// Sanitize pathname to prevent path traversal attacks
const decodedPath = decodeURIComponent(urlObject.pathname);
const normalizedPath = join(webStaticDir, decodedPath).normalize();
if (!normalizedPath.startsWith(webStaticDir)) {
// Attempted path traversal - reject by using empty path
filepath = join(webStaticDir, '');
} else {
filepath = normalizedPath;
}
} else {
filepath = normalizedPath;
// else, fallback to load the index.html instead
filepath = join(webStaticDir, 'index.html');
}
} else {
// else, fallback to load the index.html instead
filepath = join(webStaticDir, 'index.html');
filepath = decodeURIComponent(urlObject.pathname);
// security check if the filepath is within app.getPath('sessionData')
const sessionDataPath = app.getPath('sessionData');
if (!filepath.startsWith(sessionDataPath)) {
throw new Error('Invalid filepath');
}
}
return net.fetch('file://' + filepath, clonedRequest);
}

94
packages/frontend/apps/electron/src/main/recording/index.ts
View File

@@ -1,11 +1,13 @@
import path from 'node:path';
import { ShareableContent } from '@affine/native';
import { nativeImage, Notification } from 'electron';
import { app, nativeImage, Notification } from 'electron';
import fs from 'fs-extra';
import { debounce } from 'lodash-es';
import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs';
import { isMacOS } from '../../shared/utils';
import { beforeAppQuit } from '../cleanup';
import { ensureHelperProcess } from '../helper-process';
import { logger } from '../logger';
import type { NamespaceHandlers } from '../type';
import { getMainWindow } from '../windows-manager';
@@ -18,6 +20,11 @@ import type {
const subscribers: Subscriber[] = [];
const SAVED_RECORDINGS_DIR = path.join(
app.getPath('sessionData'),
'recordings'
);
beforeAppQuit(() => {
subscribers.forEach(subscriber => {
try {
@@ -134,7 +141,9 @@ function setupNewRunningAppGroup() {
recordingStatus$.value?.appGroup?.processGroupId ===
currentGroup.processGroupId
) {
stopRecording();
stopRecording().catch(err => {
logger.error('failed to stop recording', err);
});
}
}
})
@@ -142,7 +151,13 @@ function setupNewRunningAppGroup() {
}
function createRecording(status: RecordingStatus) {
const buffers: Float32Array[] = [];
const bufferedFilePath = path.join(
SAVED_RECORDINGS_DIR,
`${status.appGroup?.bundleIdentifier ?? 'unknown'}-${status.id}-${status.startTime}.raw`
);
fs.ensureDirSync(SAVED_RECORDINGS_DIR);
const file = fs.createWriteStream(bufferedFilePath);
function tapAudioSamples(err: Error | null, samples: Float32Array) {
const recordingStatus = recordingStatus$.getValue();
@@ -157,7 +172,9 @@ function createRecording(status: RecordingStatus) {
if (err) {
logger.error('failed to get audio samples', err);
} else {
buffers.push(new Float32Array(samples));
// Writing raw Float32Array samples directly to file
// For stereo audio, samples are interleaved [L,R,L,R,...]
file.write(Buffer.from(samples.buffer));
}
}
@@ -170,44 +187,29 @@ function createRecording(status: RecordingStatus) {
startTime: status.startTime,
app: status.app,
appGroup: status.appGroup,
buffers,
file,
stream,
};
return recording;
}
function concatBuffers(buffers: Float32Array[]): Float32Array {
const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0);
const buffer = new Float32Array(totalSamples);
let offset = 0;
buffers.forEach(buf => {
buffer.set(buf, offset);
offset += buf.length;
});
return buffer;
}
export async function saveRecording(id: number) {
export async function getRecording(id: number) {
const recording = recordings.get(id);
if (!recording) {
logger.error(`Recording ${id} not found`);
return;
}
const { buffers } = recording;
const helperProcessManager = await ensureHelperProcess();
const buffer = concatBuffers(buffers);
const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, {
channels: recording.stream.channels,
const rawFilePath = String(recording.file.path);
return {
id,
appGroup: recording.appGroup,
app: recording.app,
startTime: recording.startTime,
filepath: rawFilePath,
sampleRate: recording.stream.sampleRate,
});
if (!mp3Buffer) {
logger.error('failed to encode audio samples to mp3');
return;
}
recordings.delete(recording.id);
return mp3Buffer;
numberOfChannels: recording.stream.channels,
};
}
function setupRecordingListeners() {
@@ -386,9 +388,15 @@ export function resumeRecording() {
});
}
export function stopRecording() {
export async function stopRecording() {
const recordingStatus = recordingStatus$.value;
if (!recordingStatus) {
logger.error('No recording status to stop');
return;
}
const recording = recordings.get(recordingStatus?.id);
if (!recording) {
logger.error(`Recording ${recordingStatus?.id} not found`);
return;
}
@@ -398,6 +406,15 @@ export function stopRecording() {
status: 'stopped',
});
const { file } = recording;
file.end();
await new Promise<void>(resolve => {
file.on('finish', () => {
resolve();
});
});
// bring up the window
getMainWindow()
.then(mainWindow => {
@@ -411,8 +428,17 @@ export function stopRecording() {
}
export const recordingHandlers = {
saveRecording: async (_, id: number) => {
return saveRecording(id);
getRecording: async (_, id: number) => {
return getRecording(id);
},
deleteCachedRecording: async (_, id: number) => {
const recording = recordings.get(id);
if (recording) {
recording.stream.stop();
recordings.delete(id);
await fs.unlink(recording.file.path);
}
return true;
},
} satisfies NamespaceHandlers;

6
packages/frontend/apps/electron/src/main/recording/types.ts
View File

@@ -1,3 +1,5 @@
import type { WriteStream } from 'node:fs';
import type { AudioTapStream, TappableApplication } from '@affine/native';
export interface TappableAppInfo {
@@ -23,8 +25,8 @@ export interface Recording {
// the app may not be available if the user choose to record system audio
app?: TappableAppInfo;
appGroup?: AppGroupInfo;
// the raw audio buffers that are already accumulated
buffers: Float32Array[];
// the buffered file that is being recorded streamed to
file: WriteStream;
stream: AudioTapStream;
startTime: number;
}

6
packages/frontend/apps/electron/src/main/tray/index.ts
View File

@@ -198,7 +198,9 @@ class TrayState {
label: 'Stop',
click: () => {
logger.info('User action: Stop Recording');
stopRecording();
stopRecording().catch(err => {
logger.error('Failed to stop recording:', err);
});
},
},
],
@@ -290,7 +292,7 @@ class TrayState {
let _trayState: TrayState | undefined;
export const getTrayState = () => {
export const setupTrayState = () => {
if (!_trayState) {
_trayState = new TrayState();
_trayState.init();

6
packages/frontend/apps/electron/src/main/ui/events.ts
View File

@@ -35,6 +35,12 @@ export const uiEvents = {
},
onTabsStatusChange,
onActiveTabChanged,
onTabGoToRequest: (fn: (opts: { tabId: string; to: string }) => void) => {
const sub = uiSubjects.tabGoToRequest$.subscribe(fn);
return () => {
sub.unsubscribe();
};
},
onTabShellViewActiveChange,
onAuthenticationRequest: (fn: (state: AuthenticationRequest) => void) => {
const sub = uiSubjects.authenticationRequest$.subscribe(fn);

7
packages/frontend/apps/electron/src/main/ui/handlers.ts
View File

@@ -10,6 +10,7 @@ import {
activateView,
addTab,
closeTab,
ensureTabLoaded,
getMainWindow,
getOnboardingWindow,
getTabsStatus,
@@ -187,6 +188,12 @@ export const uiHandlers = {
showTab: async (_, ...args: Parameters<typeof showTab>) => {
await showTab(...args);
},
tabGoTo: async (_, tabId: string, to: string) => {
uiSubjects.tabGoToRequest$.next({ tabId, to });
},
ensureTabLoaded: async (_, ...args: Parameters<typeof ensureTabLoaded>) => {
await ensureTabLoaded(...args);
},
closeTab: async (_, ...args: Parameters<typeof closeTab>) => {
await closeTab(...args);
},

1
packages/frontend/apps/electron/src/main/ui/subject.ts
View File

@@ -6,6 +6,7 @@ export const uiSubjects = {
onMaximized$: new Subject<boolean>(),
onFullScreen$: new Subject<boolean>(),
onToggleRightSidebar$: new Subject<string>(),
tabGoToRequest$: new Subject<{ tabId: string; to: string }>(),
authenticationRequest$: new Subject<AuthenticationRequest>(),
// via menu -> close view (CMD+W)
onCloseView$: new Subject<void>(),

11
packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts
View File

@@ -24,7 +24,7 @@ import {
} from 'rxjs';
import { isMacOS } from '../../shared/utils';
import { beforeAppQuit } from '../cleanup';
import { beforeAppQuit, onTabClose } from '../cleanup';
import { mainWindowOrigin, shellViewUrl } from '../constants';
import { ensureHelperProcess } from '../helper-process';
import { logger } from '../logger';
@@ -400,6 +400,8 @@ export class WebContentViewsManager {
});
}
}, 500); // delay a bit to get rid of the flicker
onTabClose(id);
};
undoCloseTab = async () => {
@@ -1052,6 +1054,13 @@ export const loadUrlInActiveTab = async (_url: string) => {
// todo: implement
throw new Error('loadUrlInActiveTab not implemented');
};
export const ensureTabLoaded = async (tabId: string) => {
const tab = WebContentViewsManager.instance.tabViewsMap.get(tabId);
if (tab) {
return tab;
}
return WebContentViewsManager.instance.loadTab(tabId);
};
export const showTab = WebContentViewsManager.instance.showTab;
export const closeTab = WebContentViewsManager.instance.closeTab;
export const undoCloseTab = WebContentViewsManager.instance.undoCloseTab;

7
packages/frontend/apps/electron/src/shared/type.ts
View File

@@ -17,13 +17,6 @@ export interface HelperToRenderer {
// helper <-> main
export interface HelperToMain {
getMeta: () => ExposedMeta;
encodeToMp3: (
samples: Float32Array,
opts?: {
channels?: number;
sampleRate?: number;
}
) => Uint8Array;
}
export type MainToHelper = Pick<