mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-13 12:55:00 +00:00
feat(editor): audio block (#10947)
AudioMedia entity for loading & controlling a single audio media AudioMediaManagerService: Global audio state synchronization across tabs AudioAttachmentService + AudioAttachmentBlock for manipulating AttachmentBlock in affine - e.g., filling transcription (using mock endpoint for now) Added AudioBlock + AudioPlayer for rendering audio block in affine (new transcription block whose renderer is provided in affine) fix AF-2292 fix AF-2337
This commit is contained in:
pengx17
@@ -1,331 +0,0 @@
|
||||
import type { DocProps } from '@affine/core/blocksuite/initialization';
|
||||
import { AffineContext } from '@affine/core/components/context';
|
||||
import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls';
|
||||
import { AppContainer } from '@affine/core/desktop/components/app-container';
|
||||
import { router } from '@affine/core/desktop/router';
|
||||
import { configureCommonModules } from '@affine/core/modules';
|
||||
import { configureAppTabsHeaderModule } from '@affine/core/modules/app-tabs-header';
|
||||
import { configureDesktopBackupModule } from '@affine/core/modules/backup';
|
||||
import { ValidatorProvider } from '@affine/core/modules/cloud';
|
||||
import {
|
||||
configureDesktopApiModule,
|
||||
DesktopApiService,
|
||||
} from '@affine/core/modules/desktop-api';
|
||||
import { WorkspaceDialogService } from '@affine/core/modules/dialogs';
|
||||
import { DocsService } from '@affine/core/modules/doc';
|
||||
import {
|
||||
configureSpellCheckSettingModule,
|
||||
EditorSettingService,
|
||||
} from '@affine/core/modules/editor-setting';
|
||||
import { configureFindInPageModule } from '@affine/core/modules/find-in-page';
|
||||
import { GlobalContextService } from '@affine/core/modules/global-context';
|
||||
import { I18nProvider } from '@affine/core/modules/i18n';
|
||||
import { JournalService } from '@affine/core/modules/journal';
|
||||
import { LifecycleService } from '@affine/core/modules/lifecycle';
|
||||
import {
|
||||
configureElectronStateStorageImpls,
|
||||
NbstoreProvider,
|
||||
} from '@affine/core/modules/storage';
|
||||
import {
|
||||
ClientSchemeProvider,
|
||||
PopupWindowProvider,
|
||||
} from '@affine/core/modules/url';
|
||||
import {
|
||||
configureDesktopWorkbenchModule,
|
||||
WorkbenchService,
|
||||
} from '@affine/core/modules/workbench';
|
||||
import { WorkspacesService } from '@affine/core/modules/workspace';
|
||||
import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
|
||||
import createEmotionCache from '@affine/core/utils/create-emotion-cache';
|
||||
import { apis, events } from '@affine/electron-api';
|
||||
import { StoreManagerClient } from '@affine/nbstore/worker/client';
|
||||
import type { AttachmentBlockProps } from '@blocksuite/affine/model';
|
||||
import { Text } from '@blocksuite/affine/store';
|
||||
import { CacheProvider } from '@emotion/react';
|
||||
import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
|
||||
import { OpClient } from '@toeverything/infra/op';
|
||||
import { Suspense } from 'react';
|
||||
import { RouterProvider } from 'react-router-dom';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
import { DesktopThemeSync } from './theme-sync';
|
||||
|
||||
const storeManagerClient = createStoreManagerClient();
|
||||
window.addEventListener('beforeunload', () => {
|
||||
storeManagerClient.dispose();
|
||||
});
|
||||
|
||||
const desktopWhiteList = [
|
||||
'/open-app/signin-redirect',
|
||||
'/open-app/url',
|
||||
'/upgrade-success',
|
||||
'/ai-upgrade-success',
|
||||
'/share',
|
||||
'/oauth',
|
||||
'/magic-link',
|
||||
];
|
||||
if (
|
||||
!BUILD_CONFIG.isElectron &&
|
||||
BUILD_CONFIG.debug &&
|
||||
desktopWhiteList.every(path => !location.pathname.startsWith(path))
|
||||
) {
|
||||
document.body.innerHTML = `<h1 style="color:red;font-size:5rem;text-align:center;">Don't run electron entry in browser.</h1>`;
|
||||
throw new Error('Wrong distribution');
|
||||
}
|
||||
|
||||
const cache = createEmotionCache();
|
||||
|
||||
const future = {
|
||||
v7_startTransition: true,
|
||||
} as const;
|
||||
|
||||
const framework = new Framework();
|
||||
configureCommonModules(framework);
|
||||
configureElectronStateStorageImpls(framework);
|
||||
configureBrowserWorkspaceFlavours(framework);
|
||||
configureDesktopWorkbenchModule(framework);
|
||||
configureAppTabsHeaderModule(framework);
|
||||
configureFindInPageModule(framework);
|
||||
configureDesktopApiModule(framework);
|
||||
configureSpellCheckSettingModule(framework);
|
||||
configureDesktopBackupModule(framework);
|
||||
framework.impl(NbstoreProvider, {
|
||||
openStore(key, options) {
|
||||
const { store, dispose } = storeManagerClient.open(key, options);
|
||||
|
||||
return {
|
||||
store,
|
||||
dispose: () => {
|
||||
dispose();
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
framework.impl(PopupWindowProvider, p => {
|
||||
const apis = p.get(DesktopApiService).api;
|
||||
return {
|
||||
open: (url: string) => {
|
||||
apis.handler.ui.openExternal(url).catch(e => {
|
||||
console.error('Failed to open external URL', e);
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
framework.impl(ClientSchemeProvider, p => {
|
||||
const appInfo = p.get(DesktopApiService).appInfo;
|
||||
return {
|
||||
getClientScheme() {
|
||||
return appInfo?.scheme;
|
||||
},
|
||||
};
|
||||
});
|
||||
framework.impl(ValidatorProvider, p => {
|
||||
const apis = p.get(DesktopApiService).api;
|
||||
return {
|
||||
async validate(_challenge, resource) {
|
||||
const token = await apis.handler.ui.getChallengeResponse(resource);
|
||||
if (!token) {
|
||||
throw new Error('Challenge failed');
|
||||
}
|
||||
return token;
|
||||
},
|
||||
};
|
||||
});
|
||||
const frameworkProvider = framework.provider();
|
||||
|
||||
// setup application lifecycle events, and emit application start event
|
||||
window.addEventListener('focus', () => {
|
||||
frameworkProvider.get(LifecycleService).applicationFocus();
|
||||
});
|
||||
frameworkProvider.get(LifecycleService).applicationStart();
|
||||
window.addEventListener('unload', () => {
|
||||
frameworkProvider
|
||||
.get(DesktopApiService)
|
||||
.api.handler.ui.pingAppLayoutReady(false)
|
||||
.catch(console.error);
|
||||
});
|
||||
|
||||
function getCurrentWorkspace() {
|
||||
const currentWorkspaceId = frameworkProvider
|
||||
.get(GlobalContextService)
|
||||
.globalContext.workspaceId.get();
|
||||
const workspacesService = frameworkProvider.get(WorkspacesService);
|
||||
const workspaceRef = currentWorkspaceId
|
||||
? workspacesService.openByWorkspaceId(currentWorkspaceId)
|
||||
: null;
|
||||
if (!workspaceRef) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = workspaceRef;
|
||||
|
||||
return {
|
||||
workspace,
|
||||
dispose,
|
||||
};
|
||||
}
|
||||
|
||||
events?.applicationMenu.openAboutPageInSettingModal(() => {
|
||||
const currentWorkspace = getCurrentWorkspace();
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = currentWorkspace;
|
||||
workspace.scope.get(WorkspaceDialogService).open('setting', {
|
||||
activeTab: 'about',
|
||||
});
|
||||
dispose();
|
||||
});
|
||||
|
||||
events?.applicationMenu.onNewPageAction(type => {
|
||||
apis?.ui
|
||||
.isActiveTab()
|
||||
.then(isActive => {
|
||||
if (!isActive) {
|
||||
return;
|
||||
}
|
||||
const currentWorkspace = getCurrentWorkspace();
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = currentWorkspace;
|
||||
const editorSettingService = frameworkProvider.get(EditorSettingService);
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const editorSetting = editorSettingService.editorSetting;
|
||||
|
||||
const docProps = {
|
||||
note: editorSetting.get('affine:note'),
|
||||
};
|
||||
const page = docsService.createDoc({ docProps, primaryMode: type });
|
||||
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
|
||||
dispose();
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(err);
|
||||
});
|
||||
});
|
||||
|
||||
events?.recording.onRecordingStatusChanged(status => {
|
||||
(async () => {
|
||||
if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
|
||||
const currentWorkspace = getCurrentWorkspace();
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = currentWorkspace;
|
||||
const editorSettingService = frameworkProvider.get(EditorSettingService);
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const editorSetting = editorSettingService.editorSetting;
|
||||
|
||||
const docProps: DocProps = {
|
||||
note: editorSetting.get('affine:note'),
|
||||
page: {
|
||||
title: new Text(
|
||||
'Recording ' +
|
||||
(status.appGroup?.name ?? 'System Audio') +
|
||||
' ' +
|
||||
new Date(status.startTime).toISOString()
|
||||
),
|
||||
},
|
||||
onStoreLoad: (doc, { noteId }) => {
|
||||
(async () => {
|
||||
const data = await apis?.recording.saveRecording(status.id);
|
||||
if (!data) {
|
||||
return;
|
||||
}
|
||||
const blob = new Blob([data], { type: 'audio/mp3' });
|
||||
const blobId = await doc.workspace.blobSync.set(blob);
|
||||
const attachmentProps: Partial<AttachmentBlockProps> = {
|
||||
name: 'Recording',
|
||||
size: blob.size,
|
||||
type: 'audio/mp3',
|
||||
sourceId: blobId,
|
||||
embed: true,
|
||||
};
|
||||
doc.addBlock('affine:attachment', attachmentProps, noteId);
|
||||
})().catch(console.error);
|
||||
},
|
||||
};
|
||||
const page = docsService.createDoc({ docProps, primaryMode: 'page' });
|
||||
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
|
||||
|
||||
dispose();
|
||||
}
|
||||
})().catch(console.error);
|
||||
});
|
||||
|
||||
events?.applicationMenu.onOpenJournal(() => {
|
||||
const currentWorkspace = getCurrentWorkspace();
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = currentWorkspace;
|
||||
|
||||
const workbench = workspace.scope.get(WorkbenchService).workbench;
|
||||
const journalService = workspace.scope.get(JournalService);
|
||||
const docId = journalService.ensureJournalByDate(new Date()).id;
|
||||
workbench.openDoc(docId);
|
||||
|
||||
dispose();
|
||||
});
|
||||
|
||||
export function App() {
|
||||
return (
|
||||
<Suspense>
|
||||
<FrameworkRoot framework={frameworkProvider}>
|
||||
<CacheProvider value={cache}>
|
||||
<I18nProvider>
|
||||
<AffineContext store={getCurrentStore()}>
|
||||
<DesktopThemeSync />
|
||||
<RouterProvider
|
||||
fallbackElement={<AppContainer fallback />}
|
||||
router={router}
|
||||
future={future}
|
||||
/>
|
||||
{environment.isWindows && (
|
||||
<div style={{ position: 'fixed', right: 0, top: 0, zIndex: 5 }}>
|
||||
<WindowsAppControls />
|
||||
</div>
|
||||
)}
|
||||
</AffineContext>
|
||||
</I18nProvider>
|
||||
</CacheProvider>
|
||||
</FrameworkRoot>
|
||||
</Suspense>
|
||||
);
|
||||
}
|
||||
|
||||
function createStoreManagerClient() {
|
||||
const { port1: portForOpClient, port2: portForWorker } = new MessageChannel();
|
||||
let portFromWorker: MessagePort | null = null;
|
||||
let portId = uuid();
|
||||
|
||||
const handleMessage = (ev: MessageEvent) => {
|
||||
if (
|
||||
ev.data.type === 'electron:worker-connect' &&
|
||||
ev.data.portId === portId
|
||||
) {
|
||||
portFromWorker = ev.ports[0];
|
||||
// connect portForWorker and portFromWorker
|
||||
portFromWorker.addEventListener('message', ev => {
|
||||
portForWorker.postMessage(ev.data, [...ev.ports]);
|
||||
});
|
||||
portForWorker.addEventListener('message', ev => {
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
portFromWorker!.postMessage(ev.data, [...ev.ports]);
|
||||
});
|
||||
portForWorker.start();
|
||||
portFromWorker.start();
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('message', handleMessage);
|
||||
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
apis!.worker.connectWorker('affine-shared-worker', portId).catch(err => {
|
||||
console.error('failed to connect worker', err);
|
||||
});
|
||||
|
||||
const storeManager = new StoreManagerClient(new OpClient(portForOpClient));
|
||||
portForOpClient.start();
|
||||
return storeManager;
|
||||
}
|
||||
65
packages/frontend/apps/electron-renderer/src/app/app.tsx
Normal file
65
packages/frontend/apps/electron-renderer/src/app/app.tsx
Normal file
@@ -0,0 +1,65 @@
|
||||
import { AffineContext } from '@affine/core/components/context';
|
||||
import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls';
|
||||
import { AppContainer } from '@affine/core/desktop/components/app-container';
|
||||
import { router } from '@affine/core/desktop/router';
|
||||
import { I18nProvider } from '@affine/core/modules/i18n';
|
||||
import createEmotionCache from '@affine/core/utils/create-emotion-cache';
|
||||
import { CacheProvider } from '@emotion/react';
|
||||
import { FrameworkRoot, getCurrentStore } from '@toeverything/infra';
|
||||
import { Suspense } from 'react';
|
||||
import { RouterProvider } from 'react-router-dom';
|
||||
|
||||
import { setupEffects } from './effects';
|
||||
import { DesktopThemeSync } from './theme-sync';
|
||||
|
||||
const { frameworkProvider } = setupEffects();
|
||||
|
||||
const desktopWhiteList = [
|
||||
'/open-app/signin-redirect',
|
||||
'/open-app/url',
|
||||
'/upgrade-success',
|
||||
'/ai-upgrade-success',
|
||||
'/share',
|
||||
'/oauth',
|
||||
'/magic-link',
|
||||
];
|
||||
if (
|
||||
!BUILD_CONFIG.isElectron &&
|
||||
BUILD_CONFIG.debug &&
|
||||
desktopWhiteList.every(path => !location.pathname.startsWith(path))
|
||||
) {
|
||||
document.body.innerHTML = `<h1 style="color:red;font-size:5rem;text-align:center;">Don't run electron entry in browser.</h1>`;
|
||||
throw new Error('Wrong distribution');
|
||||
}
|
||||
|
||||
const cache = createEmotionCache();
|
||||
|
||||
const future = {
|
||||
v7_startTransition: true,
|
||||
} as const;
|
||||
|
||||
export function App() {
|
||||
return (
|
||||
<Suspense>
|
||||
<FrameworkRoot framework={frameworkProvider}>
|
||||
<CacheProvider value={cache}>
|
||||
<I18nProvider>
|
||||
<AffineContext store={getCurrentStore()}>
|
||||
<DesktopThemeSync />
|
||||
<RouterProvider
|
||||
fallbackElement={<AppContainer fallback />}
|
||||
router={router}
|
||||
future={future}
|
||||
/>
|
||||
{environment.isWindows && (
|
||||
<div style={{ position: 'fixed', right: 0, top: 0, zIndex: 5 }}>
|
||||
<WindowsAppControls />
|
||||
</div>
|
||||
)}
|
||||
</AffineContext>
|
||||
</I18nProvider>
|
||||
</CacheProvider>
|
||||
</FrameworkRoot>
|
||||
</Suspense>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { DesktopApiService } from '@affine/core/modules/desktop-api';
|
||||
import { WorkspaceDialogService } from '@affine/core/modules/dialogs';
|
||||
import { DocsService } from '@affine/core/modules/doc';
|
||||
import { EditorSettingService } from '@affine/core/modules/editor-setting';
|
||||
import { JournalService } from '@affine/core/modules/journal';
|
||||
import { LifecycleService } from '@affine/core/modules/lifecycle';
|
||||
import { WorkbenchService } from '@affine/core/modules/workbench';
|
||||
import { apis, events } from '@affine/electron-api';
|
||||
import type { FrameworkProvider } from '@toeverything/infra';
|
||||
|
||||
import { setupRecordingEvents } from './recording';
|
||||
import { getCurrentWorkspace } from './utils';
|
||||
|
||||
export function setupEvents(frameworkProvider: FrameworkProvider) {
|
||||
// setup application lifecycle events, and emit application start event
|
||||
window.addEventListener('focus', () => {
|
||||
frameworkProvider.get(LifecycleService).applicationFocus();
|
||||
});
|
||||
frameworkProvider.get(LifecycleService).applicationStart();
|
||||
window.addEventListener('unload', () => {
|
||||
frameworkProvider
|
||||
.get(DesktopApiService)
|
||||
.api.handler.ui.pingAppLayoutReady(false)
|
||||
.catch(console.error);
|
||||
});
|
||||
|
||||
events?.applicationMenu.openAboutPageInSettingModal(() => {
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
workspace.scope.get(WorkspaceDialogService).open('setting', {
|
||||
activeTab: 'about',
|
||||
});
|
||||
});
|
||||
|
||||
events?.applicationMenu.onNewPageAction(type => {
|
||||
apis?.ui
|
||||
.isActiveTab()
|
||||
.then(isActive => {
|
||||
if (!isActive) {
|
||||
return;
|
||||
}
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
const editorSettingService =
|
||||
frameworkProvider.get(EditorSettingService);
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const editorSetting = editorSettingService.editorSetting;
|
||||
|
||||
const docProps = {
|
||||
note: editorSetting.get('affine:note'),
|
||||
};
|
||||
const page = docsService.createDoc({ docProps, primaryMode: type });
|
||||
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(err);
|
||||
});
|
||||
});
|
||||
|
||||
events?.applicationMenu.onOpenJournal(() => {
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = currentWorkspace;
|
||||
|
||||
const workbench = workspace.scope.get(WorkbenchService).workbench;
|
||||
const journalService = workspace.scope.get(JournalService);
|
||||
const docId = journalService.ensureJournalByDate(new Date()).id;
|
||||
workbench.openDoc(docId);
|
||||
|
||||
dispose();
|
||||
});
|
||||
|
||||
setupRecordingEvents(frameworkProvider);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { setupEvents } from './events';
|
||||
import { setupModules } from './modules';
|
||||
import { setupStoreManager } from './store-manager';
|
||||
|
||||
export function setupEffects() {
|
||||
const { framework, frameworkProvider } = setupModules();
|
||||
setupStoreManager(framework);
|
||||
setupEvents(frameworkProvider);
|
||||
return { framework, frameworkProvider };
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { configureCommonModules } from '@affine/core/modules';
|
||||
import { configureAppTabsHeaderModule } from '@affine/core/modules/app-tabs-header';
|
||||
import { configureDesktopBackupModule } from '@affine/core/modules/backup';
|
||||
import { ValidatorProvider } from '@affine/core/modules/cloud';
|
||||
import {
|
||||
configureDesktopApiModule,
|
||||
DesktopApiService,
|
||||
} from '@affine/core/modules/desktop-api';
|
||||
import { configureSpellCheckSettingModule } from '@affine/core/modules/editor-setting';
|
||||
import { configureFindInPageModule } from '@affine/core/modules/find-in-page';
|
||||
import { configureElectronStateStorageImpls } from '@affine/core/modules/storage';
|
||||
import {
|
||||
ClientSchemeProvider,
|
||||
PopupWindowProvider,
|
||||
} from '@affine/core/modules/url';
|
||||
import { configureDesktopWorkbenchModule } from '@affine/core/modules/workbench';
|
||||
import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine';
|
||||
import { Framework } from '@toeverything/infra';
|
||||
|
||||
export function setupModules() {
|
||||
const framework = new Framework();
|
||||
configureCommonModules(framework);
|
||||
configureElectronStateStorageImpls(framework);
|
||||
configureBrowserWorkspaceFlavours(framework);
|
||||
configureDesktopWorkbenchModule(framework);
|
||||
configureAppTabsHeaderModule(framework);
|
||||
configureFindInPageModule(framework);
|
||||
configureDesktopApiModule(framework);
|
||||
configureSpellCheckSettingModule(framework);
|
||||
configureDesktopBackupModule(framework);
|
||||
|
||||
framework.impl(PopupWindowProvider, p => {
|
||||
const apis = p.get(DesktopApiService).api;
|
||||
return {
|
||||
open: (url: string) => {
|
||||
apis.handler.ui.openExternal(url).catch(e => {
|
||||
console.error('Failed to open external URL', e);
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
framework.impl(ClientSchemeProvider, p => {
|
||||
const appInfo = p.get(DesktopApiService).appInfo;
|
||||
return {
|
||||
getClientScheme() {
|
||||
return appInfo?.scheme;
|
||||
},
|
||||
};
|
||||
});
|
||||
framework.impl(ValidatorProvider, p => {
|
||||
const apis = p.get(DesktopApiService).api;
|
||||
return {
|
||||
async validate(_challenge, resource) {
|
||||
const token = await apis.handler.ui.getChallengeResponse(resource);
|
||||
if (!token) {
|
||||
throw new Error('Challenge failed');
|
||||
}
|
||||
return token;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const frameworkProvider = framework.provider();
|
||||
|
||||
return { framework, frameworkProvider };
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import type { DocProps } from '@affine/core/blocksuite/initialization';
|
||||
import { DocsService } from '@affine/core/modules/doc';
|
||||
import { EditorSettingService } from '@affine/core/modules/editor-setting';
|
||||
import { AudioAttachmentService } from '@affine/core/modules/media/services/audio-attachment';
|
||||
import { WorkbenchService } from '@affine/core/modules/workbench';
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import { apis, events } from '@affine/electron-api';
|
||||
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
|
||||
import { Text } from '@blocksuite/affine/store';
|
||||
import type { BlobEngine } from '@blocksuite/affine/sync';
|
||||
import type { FrameworkProvider } from '@toeverything/infra';
|
||||
import { ArrayBufferTarget, Muxer } from 'webm-muxer';
|
||||
|
||||
import { getCurrentWorkspace } from './utils';
|
||||
|
||||
const logger = new DebugLogger('electron-renderer:recording');
|
||||
|
||||
/**
|
||||
* Encodes raw audio data to Opus in WebM container.
|
||||
*/
|
||||
async function encodeRawBufferToOpus({
|
||||
filepath,
|
||||
sampleRate,
|
||||
numberOfChannels,
|
||||
}: {
|
||||
filepath: string;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
}): Promise<Uint8Array> {
|
||||
// Use streams to process audio data incrementally
|
||||
const response = await fetch(new URL(filepath, location.origin));
|
||||
if (!response.body) {
|
||||
throw new Error('Response body is null');
|
||||
}
|
||||
|
||||
// Setup Opus encoder
|
||||
const encodedChunks: EncodedAudioChunk[] = [];
|
||||
const encoder = new AudioEncoder({
|
||||
output: chunk => {
|
||||
encodedChunks.push(chunk);
|
||||
},
|
||||
error: err => {
|
||||
throw new Error(`Encoding error: ${err}`);
|
||||
},
|
||||
});
|
||||
|
||||
// Configure Opus encoder
|
||||
encoder.configure({
|
||||
codec: 'opus',
|
||||
sampleRate: sampleRate,
|
||||
numberOfChannels: numberOfChannels,
|
||||
bitrate: 96000, // 96 kbps is good for stereo audio
|
||||
});
|
||||
|
||||
// Process the stream
|
||||
const reader = response.body.getReader();
|
||||
let offset = 0;
|
||||
const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
// Convert the chunk to Float32Array
|
||||
const float32Data = new Float32Array(value.buffer);
|
||||
|
||||
// Process in smaller chunks to avoid large frames
|
||||
for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) {
|
||||
const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i);
|
||||
const chunk = float32Data.subarray(i, i + chunkSize);
|
||||
|
||||
// Create and encode frame
|
||||
const frame = new AudioData({
|
||||
format: 'f32',
|
||||
sampleRate: sampleRate,
|
||||
numberOfFrames: chunk.length / numberOfChannels,
|
||||
numberOfChannels: numberOfChannels,
|
||||
timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds
|
||||
data: chunk,
|
||||
});
|
||||
|
||||
encoder.encode(frame);
|
||||
frame.close();
|
||||
|
||||
offset += chunk.length / numberOfChannels;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await encoder.flush();
|
||||
encoder.close();
|
||||
}
|
||||
|
||||
if (encodedChunks.length === 0) {
|
||||
throw new Error('No chunks were produced during encoding');
|
||||
}
|
||||
|
||||
// Initialize WebM muxer
|
||||
const target = new ArrayBufferTarget();
|
||||
const muxer = new Muxer({
|
||||
target,
|
||||
audio: {
|
||||
codec: 'A_OPUS',
|
||||
sampleRate: sampleRate,
|
||||
numberOfChannels: numberOfChannels,
|
||||
},
|
||||
});
|
||||
|
||||
// Add all chunks to the muxer
|
||||
for (const chunk of encodedChunks) {
|
||||
muxer.addAudioChunk(chunk, {});
|
||||
}
|
||||
|
||||
// Finalize and get WebM container
|
||||
muxer.finalize();
|
||||
const { buffer: webmBuffer } = target;
|
||||
|
||||
return new Uint8Array(webmBuffer);
|
||||
}
|
||||
|
||||
async function saveRecordingBlob(
|
||||
blobEngine: BlobEngine,
|
||||
recording: {
|
||||
id: number;
|
||||
filepath: string;
|
||||
sampleRate: number;
|
||||
numberOfChannels: number;
|
||||
}
|
||||
) {
|
||||
logger.debug('Saving recording', recording.id);
|
||||
const opusBuffer = await encodeRawBufferToOpus({
|
||||
filepath: recording.filepath,
|
||||
sampleRate: recording.sampleRate,
|
||||
numberOfChannels: recording.numberOfChannels,
|
||||
});
|
||||
const blob = new Blob([opusBuffer], {
|
||||
type: 'audio/webm',
|
||||
});
|
||||
const blobId = await blobEngine.set(blob);
|
||||
logger.debug('Recording saved', blobId);
|
||||
return { blob, blobId };
|
||||
}
|
||||
|
||||
export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {
|
||||
events?.recording.onRecordingStatusChanged(status => {
|
||||
(async () => {
|
||||
if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
const editorSettingService =
|
||||
frameworkProvider.get(EditorSettingService);
|
||||
const docsService = workspace.scope.get(DocsService);
|
||||
const editorSetting = editorSettingService.editorSetting;
|
||||
|
||||
const docProps: DocProps = {
|
||||
note: editorSetting.get('affine:note'),
|
||||
page: {
|
||||
title: new Text(
|
||||
'Recording ' +
|
||||
(status.appGroup?.name ?? 'System Audio') +
|
||||
' ' +
|
||||
new Date(status.startTime).toISOString()
|
||||
),
|
||||
},
|
||||
onStoreLoad: (doc, { noteId }) => {
|
||||
(async () => {
|
||||
const recording = await apis?.recording.getRecording(status.id);
|
||||
if (!recording) {
|
||||
logger.error('Failed to save recording');
|
||||
return;
|
||||
}
|
||||
|
||||
// name + timestamp(readable) + extension
|
||||
const attachmentName =
|
||||
(status.appGroup?.name ?? 'System Audio') +
|
||||
' ' +
|
||||
new Date(status.startTime).toISOString() +
|
||||
'.webm';
|
||||
|
||||
// add size and sourceId to the attachment later
|
||||
const attachmentId = doc.addBlock(
|
||||
'affine:attachment',
|
||||
{
|
||||
name: attachmentName,
|
||||
type: 'audio/webm',
|
||||
},
|
||||
noteId
|
||||
);
|
||||
|
||||
const model = doc.getBlock(attachmentId)
|
||||
?.model as AttachmentBlockModel;
|
||||
|
||||
if (model) {
|
||||
// it takes a while to save the blob, so we show the attachment first
|
||||
const { blobId, blob } = await saveRecordingBlob(
|
||||
doc.workspace.blobSync,
|
||||
recording
|
||||
);
|
||||
|
||||
model.props.size = blob.size;
|
||||
model.props.sourceId = blobId;
|
||||
model.props.embed = true;
|
||||
|
||||
using currentWorkspace = getCurrentWorkspace(frameworkProvider);
|
||||
if (!currentWorkspace) {
|
||||
return;
|
||||
}
|
||||
const { workspace } = currentWorkspace;
|
||||
using audioAttachment = workspace.scope
|
||||
.get(AudioAttachmentService)
|
||||
.get(model);
|
||||
audioAttachment?.obj.transcribe();
|
||||
}
|
||||
})().catch(console.error);
|
||||
},
|
||||
};
|
||||
const page = docsService.createDoc({ docProps, primaryMode: 'page' });
|
||||
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
|
||||
}
|
||||
})().catch(console.error);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { NbstoreProvider } from '@affine/core/modules/storage';
|
||||
import { apis } from '@affine/electron-api';
|
||||
import { StoreManagerClient } from '@affine/nbstore/worker/client';
|
||||
import type { Framework } from '@toeverything/infra';
|
||||
import { OpClient } from '@toeverything/infra/op';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
function createStoreManagerClient() {
|
||||
const { port1: portForOpClient, port2: portForWorker } = new MessageChannel();
|
||||
let portFromWorker: MessagePort | null = null;
|
||||
let portId = uuid();
|
||||
|
||||
const handleMessage = (ev: MessageEvent) => {
|
||||
if (
|
||||
ev.data.type === 'electron:worker-connect' &&
|
||||
ev.data.portId === portId
|
||||
) {
|
||||
portFromWorker = ev.ports[0];
|
||||
// connect portForWorker and portFromWorker
|
||||
portFromWorker.addEventListener('message', ev => {
|
||||
portForWorker.postMessage(ev.data, [...ev.ports]);
|
||||
});
|
||||
portForWorker.addEventListener('message', ev => {
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
portFromWorker!.postMessage(ev.data, [...ev.ports]);
|
||||
});
|
||||
portForWorker.start();
|
||||
portFromWorker.start();
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('message', handleMessage);
|
||||
|
||||
// oxlint-disable-next-line no-non-null-assertion
|
||||
apis!.worker.connectWorker('affine-shared-worker', portId).catch(err => {
|
||||
console.error('failed to connect worker', err);
|
||||
});
|
||||
|
||||
const storeManager = new StoreManagerClient(new OpClient(portForOpClient));
|
||||
portForOpClient.start();
|
||||
return storeManager;
|
||||
}
|
||||
|
||||
export function setupStoreManager(framework: Framework) {
|
||||
const storeManagerClient = createStoreManagerClient();
|
||||
window.addEventListener('beforeunload', () => {
|
||||
storeManagerClient.dispose();
|
||||
});
|
||||
|
||||
framework.impl(NbstoreProvider, {
|
||||
openStore(key, options) {
|
||||
const { store, dispose } = storeManagerClient.open(key, options);
|
||||
|
||||
return {
|
||||
store,
|
||||
dispose: () => {
|
||||
dispose();
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { GlobalContextService } from '@affine/core/modules/global-context';
|
||||
import { WorkspacesService } from '@affine/core/modules/workspace';
|
||||
import type { FrameworkProvider } from '@toeverything/infra';
|
||||
|
||||
export function getCurrentWorkspace(frameworkProvider: FrameworkProvider) {
|
||||
const currentWorkspaceId = frameworkProvider
|
||||
.get(GlobalContextService)
|
||||
.globalContext.workspaceId.get();
|
||||
const workspacesService = frameworkProvider.get(WorkspacesService);
|
||||
const workspaceRef = currentWorkspaceId
|
||||
? workspacesService.openByWorkspaceId(currentWorkspaceId)
|
||||
: null;
|
||||
if (!workspaceRef) {
|
||||
return;
|
||||
}
|
||||
const { workspace, dispose } = workspaceRef;
|
||||
|
||||
return {
|
||||
workspace,
|
||||
dispose,
|
||||
[Symbol.dispose]: dispose,
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,3 @@
|
||||
import '@affine/core/bootstrap/electron';
|
||||
import '@affine/component/theme';
|
||||
import '../global.css';
|
||||
import '../app/global.css';
|
||||
|
||||
Reference in New Issue
Block a user