From 9e0cae58d74b2baece8fbe74ed48c1a055a5f2c6 Mon Sep 17 00:00:00 2001 From: pengx17 Date: Thu, 27 Feb 2025 15:02:38 +0000 Subject: [PATCH] fix(native): split application & tappable application (#10491) A listening tappable app's info should inherit from its group process's name/icon. However the group process may not be listed as a tappable application. --- .../apps/electron-renderer/src/app.tsx | 34 +- .../electron/resources/icons/doc-edgeless.png | Bin 0 -> 938 bytes .../electron/resources/icons/doc-page.png | Bin 0 -> 636 bytes .../resources/icons/journal-today.png | Bin 0 -> 748 bytes .../electron/resources/icons/tray-icon.png | Bin 0 -> 2512 bytes .../src/main/application-menu/create.ts | 2 +- .../src/main/application-menu/index.ts | 8 +- .../src/main/application-menu/subject.ts | 3 +- .../frontend/apps/electron/src/main/index.ts | 6 +- .../apps/electron/src/main/protocol.ts | 3 +- .../apps/electron/src/main/recording/index.ts | 237 +++++++++ .../apps/electron/src/main/tray/index.ts | 214 ++++++++ .../src/main/windows-manager/main-window.ts | 4 +- .../apps/electron/src/shared/utils.ts | 39 ++ .../media-capture-playground/server/main.ts | 172 ++++--- .../web/components/app-item.tsx | 4 +- .../web/components/app-list.tsx | 7 +- .../media-capture-playground/web/types.ts | 2 +- packages/frontend/native/index.d.ts | 25 +- packages/frontend/native/index.js | 1 + .../src/macos/screen_capture_kit.rs | 487 ++++++++++-------- .../media_capture/src/macos/tap_audio.rs | 40 +- 22 files changed, 975 insertions(+), 313 deletions(-) create mode 100644 packages/frontend/apps/electron/resources/icons/doc-edgeless.png create mode 100644 packages/frontend/apps/electron/resources/icons/doc-page.png create mode 100644 packages/frontend/apps/electron/resources/icons/journal-today.png create mode 100644 packages/frontend/apps/electron/resources/icons/tray-icon.png create mode 100644 packages/frontend/apps/electron/src/main/recording/index.ts create mode 100644 packages/frontend/apps/electron/src/main/tray/index.ts diff --git a/packages/frontend/apps/electron-renderer/src/app.tsx b/packages/frontend/apps/electron-renderer/src/app.tsx index 45e11d4c4b..8424fbd7bb 100644 --- a/packages/frontend/apps/electron-renderer/src/app.tsx +++ b/packages/frontend/apps/electron-renderer/src/app.tsx @@ -19,6 +19,7 @@ import { import { configureFindInPageModule } from '@affine/core/modules/find-in-page'; import { GlobalContextService } from '@affine/core/modules/global-context'; import { I18nProvider } from '@affine/core/modules/i18n'; +import { JournalService } from '@affine/core/modules/journal'; import { LifecycleService } from '@affine/core/modules/lifecycle'; import { configureElectronStateStorageImpls, @@ -146,7 +147,8 @@ events?.applicationMenu.openAboutPageInSettingModal(() => activeTab: 'about', }) ); -events?.applicationMenu.onNewPageAction(() => { + +function getCurrentWorkspace() { const currentWorkspaceId = frameworkProvider .get(GlobalContextService) .globalContext.workspaceId.get(); @@ -158,6 +160,19 @@ events?.applicationMenu.onNewPageAction(() => { return; } const { workspace, dispose } = workspaceRef; + + return { + workspace, + dispose, + }; +} + +events?.applicationMenu.onNewPageAction(type => { + const currentWorkspace = getCurrentWorkspace(); + if (!currentWorkspace) { + return; + } + const { workspace, dispose } = currentWorkspace; const editorSettingService = frameworkProvider.get(EditorSettingService); const docsService = workspace.scope.get(DocsService); const editorSetting = editorSettingService.editorSetting; @@ -171,7 +186,7 @@ events?.applicationMenu.onNewPageAction(() => { if (!isActive) { return; } - const page = docsService.createDoc({ docProps }); + const page = docsService.createDoc({ docProps, primaryMode: type }); workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); }) .catch(err => { @@ -181,6 +196,21 @@ events?.applicationMenu.onNewPageAction(() => { dispose(); }); +events?.applicationMenu.onOpenJournal(() => { + const currentWorkspace = getCurrentWorkspace(); + if (!currentWorkspace) { + return; + } + const { workspace, dispose } = currentWorkspace; + + const workbench = workspace.scope.get(WorkbenchService).workbench; + const journalService = workspace.scope.get(JournalService); + const docId = journalService.ensureJournalByDate(new Date()).id; + workbench.openDoc(docId); + + dispose(); +}); + export function App() { return ( diff --git a/packages/frontend/apps/electron/resources/icons/doc-edgeless.png b/packages/frontend/apps/electron/resources/icons/doc-edgeless.png new file mode 100644 index 0000000000000000000000000000000000000000..53a206f60135b6f87fa336a098e0064d9fa9e343 GIT binary patch literal 938 zcmV;b16BNqP)@qiKZqUgIlH4GiAjz_;&=ZvIx`Ujc=?S2_?n+OP=??LuP9cpqNt?Xg#k1MAXz zILu$9Lu1z}89sQ+SodVHHWsm7>;k!5?unn`Bv;VJRA>r}xiW>jY!s(a})}8lA8~Go@u>AI*ifDExnx z;mz1AYBU;5YqL;;D|=ifjD$cmZop~%=~5z{V|1+eEk(|qC;qb96I z^69iX;q{>BGk|&5gLc1ycCJzY9_I1S2EnM^K~#7F?U-Fo z!axwmr%;|CkpxJ1TuvaIzuu)hShwErx#yED5pB}z$2QDW(cgb>BqY}VRrHm4MmOeW)_>1VUq?r=DiQTj*& zOQq7KWm#=p7AZl{r^Lq{jYb`%PW7-_t%~JxISAbH(Wy@{z$)-SKL?Bc8sqW!HB4`l zg4Jqu6Zm;PnM}Hr0F_FG-Q}LI(cYqIhG{bqeL36qDJkO^%$SFnd*ExdO23XW@B@tU zjShsCfrrE<1B$@5?MEz+2$6KKj4maB(G2EtIf2H((uTyg^_JC#At~Zi-q=f5jO)jnn7&5K4!6oazLnbY@BKI?lAVjZAB(& z&;0o)P7MqLr^|_sF3P|jf5F*XTq;sRVC!+{tJLhOw93LAM%6LWm%y#W3wLBhHPtH+ zomU@CBn5Xr$zC+^(Wi?u-BXFQULNtuffUQZWgT^hmhUM4)Ux`8LcvR_Zb?b`*SrJB W|Kjr98+mU40000LDL)16ErzNd;`b{s=P4+%;?D@Sjk#RyIKjNf`WHQv)MfFcDruY_O9zXolfVR@}S*r4_d8Oh~G`t zK4?2peSVN7WwIm;LzmN1spMwugSL*!I%G*HO@P_@NQU@%&-t(g0VeQsKA%rbLZgCg zFWc?*ALR&kX0Ti?|0e5q6m&qXR-5ovVrByAAEZ+I&=(|Z@RDav*#`ej54&)MKG#~k zgYtzk8`)?ydO`@f_ham#%AiA&r=otuGC0eGGJ>}M)i*~5nHP=64w#hhI7zu&j_qwW zn~9{nTCI+e1eo>CcBH|(faFT`eL|Y;#p97BO6fD3K8wZT63Hb!1M6cZZ}zSr+YaRb zOdwTOJYyM z3*k+y*XyZvC%mOoDY!3WLDo0btI%kaW#Pkl@eo@zP(@j<*UzH+8^1fWZ4BVWnLYJr z3IbpE*41joULB~S7sXU4>eG>U^IL4CAXersQP!{LY55mnr6BBTfFq;FKfE74>9=8i zPed}dO*#_RH)<)RQ0bF;c3O@crbjX)}D1*}j7 zq9kr81x%1Of)*4ig0jeyB_b-+0xgOn2*})U6bQH`hgQ2;=xtONvQN!87-!M#thaQaH*ez$t?S!jxD(6PuBe%;u3YoN*r* zBoHgz1RVAQg`eb%3-Te~xO?MBfwZqag@Yq!TmqlZArT1a z>FM}%JA5iPp0J%rBob_F3AVP@Aj6uM$>uXNtl7LRA4UGFLyP6ba9JEaE0v8^>N2BK z)A-IfoHEhp_HmtjR@}dt*t}0=fdvwjGlcDU8^UL8Fw{xuCGFy}V!_NxeV6S{ADI8= z`;^CtpiKThi}_gT2QOHv3*3qDdD~pz`Vk`<0H6`zMWZk>Ad`xhyi@L&){#Fn16~tafE!$L!Tv^aDN$y?iyQbY`TWzNf8CzL3Q`I5Z!b zgB~8us3RM$zu8=NxR?=K^0s?KrsMh-kITafUz3k2 zRTt&WYjy?W10_=jk&8c59hY_TK~tqFZEDB$PuN)qAbcA`z?K+C(Cj z%g0EE*LXVo3AiL*ykqX}qN!{B+sU??3KdAi2FonnEn-HGTBGKGN&-2m0iCqXyxf(T1st!w_32rv7zr7Gq9VV{Z zrPdeLf2DIGA24`^W~{!TZ2cl$tE*CtyB1Oib+Jt<_(j2W7+|ZU{#js{{^Aj2NM*vt z`Xq*Qr&w&<8=a;(Jl*_Q-pv0JiE6A&Ar$P}u|qP1?2dOb$U7XnCmdR2P<}og_hr`R zLoZ7st8tg}ZFEp>XPJ*vWHnm(adsl|M${MX(wcg^zA$rStReoJ7SpS(@;WWY5WT9_wM%tp z%k_~~CufgluG;YO>hPn8`l+Hs>o7!R2EmLfJa}h_ra)aRwZEaKju+_2N@kf{;~0b3 z>32)Rx%bWN!fK0Qj{;z?&M{Ba*6hnL39hK{d_g1Zrc)c=z;*7lqslsmuTRC^z~ntV zepD(GSJO41Hch{Ty*yx?haLRMko6lJ+daCJk6jBzl%Kd!y@1(|zj5nbblJ?Vkg5fA zzxGLH=Ub_BnASjc=pR4&=Fdi>-^{VP2Od)c*(4+LzT@9c`Esro&EwswQ1@aFg@jcC z+~=o_92|7l3j>#v6D65+REKKNlWpgZwe!r$`h!`ggH@5bbA0kL^1z9&G8na&;J2^# zQC71y@_}oi-BV#2A4OJTJ&>mmSpYDN1auYF{B*jmX!IluM$4Y`18*ZsrfB5n&(nrKUj<` zR?oe$5oIMrbcCH}C4FgJwM7fobY>=V@9Q`v-`DI8639BZBty+0u5G~% zeGQ>J+k!Tlpl#Yh9KUp|aLpbWdW^Vo2_vg=dsPx;bk#q%OZphC;~K2#L%eE1T&X0M zco@W;4Whh$QiKxZMI~iF8knOCk4hQiGX$Q|@2jvZ;t>34BK-;ar!^L4eE!IVk*F)B zr!wo>MZRA8k_pDeY@68g-iqVW@H>CcNB5f;LUPxlcT`*e_iTEvS*>hRl1LxxF|&f^ zshUhMx-^76M~QDYeQVV{;tGuL~#I+WYG@t*S*GzJ(hNn)jZ8 z2X-`^ohz1GmZUZGiCVXwMORvx8z4G2B>Tc)0dj{fHe$aUYya7~A`i*uMb(n6|gM9Cq^cMy{M3vY+tgz{aWYH zn<6?rKUh2q*YA&39e!X%3Hf&UXY3{#rJh7CN}A?{b1XM&Om@dPx_dvHXiqMDPZ!O! z(mS5Nv|qxz&vRRIv4Y++*ZW~oeJIQWGvDHE{nx!A#5kiP4L@nvdTBQ9y?AH$Z*2uQ z;NFSNdREyp?Kha!+TsLQfg^LDF{@`WZ1tVznJ~TSozw+nf7>DBI?`owB%B=`b7u*D zo-}1qn!|YCCA_Bc{JlcD8zOSKH1wZF7(1~b2RzD4=FOXogOmDZL-MbPCIpC{7llyy niz7vB(#kF4&ljFk4}4jpswVOIczGFF`A+e2r_*YvQQ7|htg0K& literal 0 HcmV?d00001 diff --git a/packages/frontend/apps/electron/src/main/application-menu/create.ts b/packages/frontend/apps/electron/src/main/application-menu/create.ts index 32b140f1cd..20b30ea9d7 100644 --- a/packages/frontend/apps/electron/src/main/application-menu/create.ts +++ b/packages/frontend/apps/electron/src/main/application-menu/create.ts @@ -64,7 +64,7 @@ export function createApplicationMenu() { click: async () => { await initAndShowMainWindow(); // fixme: if the window is just created, the new page action will not be triggered - applicationMenuSubjects.newPageAction$.next(); + applicationMenuSubjects.newPageAction$.next('page'); }, }, ], diff --git a/packages/frontend/apps/electron/src/main/application-menu/index.ts b/packages/frontend/apps/electron/src/main/application-menu/index.ts index 61c88f2085..22cbc86403 100644 --- a/packages/frontend/apps/electron/src/main/application-menu/index.ts +++ b/packages/frontend/apps/electron/src/main/application-menu/index.ts @@ -11,7 +11,7 @@ export const applicationMenuEvents = { /** * File -> New Doc */ - onNewPageAction: (fn: () => void) => { + onNewPageAction: (fn: (type: 'page' | 'edgeless') => void) => { const sub = applicationMenuSubjects.newPageAction$.subscribe(fn); return () => { sub.unsubscribe(); @@ -24,4 +24,10 @@ export const applicationMenuEvents = { sub.unsubscribe(); }; }, + onOpenJournal: (fn: () => void) => { + const sub = applicationMenuSubjects.openJournal$.subscribe(fn); + return () => { + sub.unsubscribe(); + }; + }, } satisfies Record; diff --git a/packages/frontend/apps/electron/src/main/application-menu/subject.ts b/packages/frontend/apps/electron/src/main/application-menu/subject.ts index bee518184a..fc420c3cff 100644 --- a/packages/frontend/apps/electron/src/main/application-menu/subject.ts +++ b/packages/frontend/apps/electron/src/main/application-menu/subject.ts @@ -1,6 +1,7 @@ import { Subject } from 'rxjs'; export const applicationMenuSubjects = { - newPageAction$: new Subject(), + newPageAction$: new Subject<'page' | 'edgeless'>(), + openJournal$: new Subject(), openAboutPageInSettingModal$: new Subject(), }; diff --git a/packages/frontend/apps/electron/src/main/index.ts b/packages/frontend/apps/electron/src/main/index.ts index 70a68a6016..912f4ea40f 100644 --- a/packages/frontend/apps/electron/src/main/index.ts +++ b/packages/frontend/apps/electron/src/main/index.ts @@ -14,8 +14,9 @@ import { registerEvents } from './events'; import { registerHandlers } from './handlers'; import { logger } from './logger'; import { registerProtocol } from './protocol'; +import { getShareableContent } from './recording'; +import { getTrayState } from './tray'; import { isOnline } from './ui'; -import { registerUpdater } from './updater'; import { launch } from './windows-manager/launcher'; import { launchStage } from './windows-manager/stage'; @@ -86,8 +87,9 @@ app .then(registerHandlers) .then(registerEvents) .then(launch) + .then(getShareableContent) .then(createApplicationMenu) - .then(registerUpdater) + .then(getTrayState) .catch(e => console.error('Failed create window:', e)); if (process.env.SENTRY_RELEASE) { diff --git a/packages/frontend/apps/electron/src/main/protocol.ts b/packages/frontend/apps/electron/src/main/protocol.ts index 63dadb0141..b9e6060a70 100644 --- a/packages/frontend/apps/electron/src/main/protocol.ts +++ b/packages/frontend/apps/electron/src/main/protocol.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { net, protocol, session } from 'electron'; import cookieParser from 'set-cookie-parser'; +import { resourcesPath } from '../shared/utils'; import { logger } from './logger'; protocol.registerSchemesAsPrivileged([ @@ -33,7 +34,7 @@ protocol.registerSchemesAsPrivileged([ ]); const NETWORK_REQUESTS = ['/api', '/ws', '/socket.io', '/graphql']; -const webStaticDir = join(__dirname, '../resources/web-static'); +const webStaticDir = join(resourcesPath, 'web-static'); function isNetworkResource(pathname: string) { return NETWORK_REQUESTS.some(opt => pathname.startsWith(opt)); diff --git a/packages/frontend/apps/electron/src/main/recording/index.ts b/packages/frontend/apps/electron/src/main/recording/index.ts new file mode 100644 index 0000000000..abcc9bd24a --- /dev/null +++ b/packages/frontend/apps/electron/src/main/recording/index.ts @@ -0,0 +1,237 @@ +import { ShareableContent, TappableApplication } from '@affine/native'; +import { Notification } from 'electron'; +import { + BehaviorSubject, + distinctUntilChanged, + pairwise, + startWith, +} from 'rxjs'; + +import { isMacOS } from '../../shared/utils'; +import { beforeAppQuit } from '../cleanup'; +import { logger } from '../logger'; + +interface TappableAppInfo { + rawInstance: TappableApplication; + isRunning: boolean; + processId: number; + processGroupId: number; + bundleIdentifier: string; + name: string; +} + +interface AppGroupInfo { + processGroupId: number; + apps: TappableAppInfo[]; + name: string; + icon: Buffer | undefined; + isRunning: boolean; +} + +const subscribers: Subscriber[] = []; + +beforeAppQuit(() => { + subscribers.forEach(subscriber => { + subscriber.unsubscribe(); + }); +}); + +let shareableContent: ShareableContent | null = null; + +export const applications$ = new BehaviorSubject([]); +export const appGroups$ = new BehaviorSubject([]); + +if (isMacOS()) { + // Update appGroups$ whenever applications$ changes + subscribers.push( + applications$.pipe(distinctUntilChanged()).subscribe(apps => { + const appGroups: AppGroupInfo[] = []; + apps.forEach(app => { + let appGroup = appGroups.find( + group => group.processGroupId === app.processGroupId + ); + + if (!appGroup) { + const groupProcess = shareableContent?.applicationWithProcessId( + app.processGroupId + ); + if (!groupProcess) { + return; + } + appGroup = { + processGroupId: app.processGroupId, + apps: [], + name: groupProcess.name, + // icon will be lazy loaded + get icon() { + try { + return groupProcess.icon; + } catch (error) { + logger.error( + `Failed to get icon for ${groupProcess.name}`, + error + ); + return undefined; + } + }, + get isRunning() { + return this.apps.some(app => app.rawInstance.isRunning); + }, + }; + appGroups.push(appGroup); + } + if (appGroup) { + appGroup.apps.push(app); + } + }); + appGroups$.next(appGroups); + }) + ); + + subscribers.push( + appGroups$ + .pipe(startWith([] as AppGroupInfo[]), pairwise()) + .subscribe(([previousGroups, currentGroups]) => { + currentGroups.forEach(currentGroup => { + const previousGroup = previousGroups.find( + group => group.processGroupId === currentGroup.processGroupId + ); + if (previousGroup?.isRunning !== currentGroup.isRunning) { + console.log( + 'appgroup running changed', + currentGroup.name, + currentGroup.isRunning + ); + if (currentGroup.isRunning) { + new Notification({ + title: 'Recording Meeting', + body: `Recording meeting with ${currentGroup.name}`, + }).show(); + } + } + }); + }) + ); +} + +async function getAllApps(): Promise { + if (!shareableContent) { + return []; + } + + const apps = shareableContent.applications().map(app => { + try { + return { + rawInstance: app, + processId: app.processId, + processGroupId: app.processGroupId, + bundleIdentifier: app.bundleIdentifier, + name: app.name, + isRunning: app.isRunning, + }; + } catch (error) { + logger.error('failed to get app info', error); + return null; + } + }); + + const filteredApps = apps.filter( + (v): v is TappableAppInfo => + v !== null && + !v.bundleIdentifier.startsWith('com.apple') && + v.processId !== process.pid + ); + + return filteredApps; +} + +type Subscriber = { + unsubscribe: () => void; +}; + +function setupMediaListeners() { + subscribers.push( + ShareableContent.onApplicationListChanged(() => { + getAllApps() + .then(apps => { + applications$.next(apps); + }) + .catch(err => { + logger.error('failed to get apps', err); + }); + }) + ); + + getAllApps() + .then(apps => { + applications$.next(apps); + }) + .catch(err => { + logger.error('failed to get apps', err); + }); + + let appStateSubscribers: Subscriber[] = []; + + subscribers.push( + applications$.subscribe(apps => { + appStateSubscribers.forEach(subscriber => { + subscriber.unsubscribe(); + }); + const _appStateSubscribers: Subscriber[] = []; + + apps.forEach(app => { + try { + // Try to create a TappableApplication with a default audio object ID + // In a real implementation, you would need to get the actual audio object ID + // This is just a placeholder value that seems to work for testing + const tappableApp = TappableApplication.fromApplication( + app.rawInstance, + 1 + ); + + if (tappableApp) { + _appStateSubscribers.push( + ShareableContent.onAppStateChanged(tappableApp, () => { + setTimeout(() => { + const apps = applications$.getValue(); + applications$.next( + apps.map(_app => { + if (_app.processId === app.processId) { + return { ..._app, isRunning: tappableApp.isRunning }; + } + return _app; + }) + ); + }, 10); + }) + ); + } + } catch (error) { + logger.error( + `Failed to convert app ${app.name} to TappableApplication`, + error + ); + } + }); + + appStateSubscribers = _appStateSubscribers; + return () => { + _appStateSubscribers.forEach(subscriber => { + subscriber.unsubscribe(); + }); + }; + }) + ); +} + +export function getShareableContent() { + if (!shareableContent && isMacOS()) { + try { + shareableContent = new ShareableContent(); + setupMediaListeners(); + } catch (error) { + logger.error('failed to get shareable content', error); + } + } + return shareableContent; +} diff --git a/packages/frontend/apps/electron/src/main/tray/index.ts b/packages/frontend/apps/electron/src/main/tray/index.ts new file mode 100644 index 0000000000..249c441b7b --- /dev/null +++ b/packages/frontend/apps/electron/src/main/tray/index.ts @@ -0,0 +1,214 @@ +import { join } from 'node:path'; + +import { + app, + Menu, + MenuItem, + type NativeImage, + nativeImage, + Tray, +} from 'electron'; + +import { isMacOS, resourcesPath } from '../../shared/utils'; +import { applicationMenuSubjects } from '../application-menu'; +import { beforeAppQuit } from '../cleanup'; +import { appGroups$ } from '../recording'; +import { getMainWindow } from '../windows-manager'; + +export interface TrayMenuConfigItem { + label: string; + click?: () => void; + icon?: NativeImage | string | Buffer; + disabled?: boolean; +} + +export type TrayMenuConfig = Array; + +// each provider has a unique key and provides a menu config (a group of menu items) +interface TrayMenuProvider { + key: string; + getConfig(): TrayMenuConfig; +} + +function showMainWindow() { + getMainWindow() + .then(w => { + w.show(); + }) + .catch(err => console.error(err)); +} + +class TrayState { + tray: Tray | null = null; + + // tray's icon + icon: NativeImage = nativeImage + .createFromPath(join(resourcesPath, 'icons/tray-icon.png')) + .resize({ width: 16, height: 16 }); + + // tray's tooltip + tooltip: string = 'AFFiNE'; + + constructor() { + this.icon.setTemplateImage(true); + } + + // sorry, no idea on better naming + getPrimaryMenuProvider(): TrayMenuProvider { + return { + key: 'primary', + getConfig: () => [ + { + label: 'Open Journal', + icon: join(resourcesPath, 'icons/journal-today.png'), + click: () => { + showMainWindow(); + applicationMenuSubjects.openJournal$.next(); + }, + }, + { + label: 'New Page', + icon: join(resourcesPath, 'icons/doc-page.png'), + click: () => { + showMainWindow(); + applicationMenuSubjects.newPageAction$.next('page'); + }, + }, + { + label: 'New Edgeless', + icon: join(resourcesPath, 'icons/doc-edgeless.png'), + click: () => { + showMainWindow(); + applicationMenuSubjects.newPageAction$.next('edgeless'); + }, + }, + ], + }; + } + + getRecordingMenuProvider(): TrayMenuProvider { + const appGroups = appGroups$.value; + const runningAppGroups = appGroups.filter(appGroup => appGroup.isRunning); + return { + key: 'recording', + getConfig: () => [ + { + label: 'Start Recording Meeting', + disabled: true, + }, + ...runningAppGroups.map(appGroup => ({ + label: appGroup.name, + icon: appGroup.icon || undefined, + click: () => { + console.log(appGroup); + }, + })), + ], + }; + } + + getSecondaryMenuProvider(): TrayMenuProvider { + return { + key: 'secondary', + getConfig: () => [ + { + label: 'Open AFFiNE', + click: () => { + getMainWindow() + .then(w => { + w.show(); + }) + .catch(err => { + console.error(err); + }); + }, + }, + 'separator', + { + label: 'Quit AFFiNE Completely...', + click: () => { + app.quit(); + }, + }, + ], + }; + } + + buildMenu(providers: TrayMenuProvider[]) { + const menu = new Menu(); + providers.forEach((provider, index) => { + provider.getConfig().forEach(item => { + if (item === 'separator') { + menu.append(new MenuItem({ type: 'separator' })); + } else { + const { icon, disabled, ...rest } = item; + let nativeIcon: NativeImage | undefined; + if (typeof icon === 'string') { + nativeIcon = nativeImage.createFromPath(icon); + } else if (Buffer.isBuffer(icon)) { + try { + nativeIcon = nativeImage.createFromBuffer(icon); + } catch (error) { + console.error('Failed to create icon from buffer', error); + } + } + if (nativeIcon) { + nativeIcon = nativeIcon.resize({ width: 20, height: 20 }); + } + menu.append( + new MenuItem({ + ...rest, + enabled: !disabled, + icon: nativeIcon, + }) + ); + } + }); + if (index !== providers.length - 1) { + menu.append(new MenuItem({ type: 'separator' })); + } + }); + return menu; + } + + update() { + if (!this.tray) { + this.tray = new Tray(this.icon); + this.tray.setToolTip(this.tooltip); + const clickHandler = () => { + this.update(); + if (!isMacOS()) { + this.tray?.popUpContextMenu(); + } + }; + this.tray.on('click', clickHandler); + beforeAppQuit(() => { + this.tray?.off('click', clickHandler); + this.tray?.destroy(); + }); + } + + const providers = [ + this.getPrimaryMenuProvider(), + isMacOS() ? this.getRecordingMenuProvider() : null, + this.getSecondaryMenuProvider(), + ].filter(p => p !== null); + + const menu = this.buildMenu(providers); + this.tray.setContextMenu(menu); + } + + init() { + this.update(); + } +} + +let _trayState: TrayState | undefined; + +export const getTrayState = () => { + if (!_trayState) { + _trayState = new TrayState(); + _trayState.init(); + } + return _trayState; +}; diff --git a/packages/frontend/apps/electron/src/main/windows-manager/main-window.ts b/packages/frontend/apps/electron/src/main/windows-manager/main-window.ts index 7c57406b78..fe903aff46 100644 --- a/packages/frontend/apps/electron/src/main/windows-manager/main-window.ts +++ b/packages/frontend/apps/electron/src/main/windows-manager/main-window.ts @@ -4,7 +4,7 @@ import { BrowserWindow, nativeTheme } from 'electron'; import electronWindowState from 'electron-window-state'; import { BehaviorSubject } from 'rxjs'; -import { isLinux, isMacOS, isWindows } from '../../shared/utils'; +import { isLinux, isMacOS, isWindows, resourcesPath } from '../../shared/utils'; import { beforeAppQuit } from '../cleanup'; import { buildType } from '../config'; import { mainWindowOrigin } from '../constants'; @@ -92,7 +92,7 @@ export class MainWindowManager { if (isLinux()) { browserWindow.setIcon( // __dirname is `packages/frontend/apps/electron/dist` (the bundled output directory) - join(__dirname, `../resources/icons/icon_${buildType}_64x64.png`) + join(resourcesPath, `icons/icon_${buildType}_64x64.png`) ); } diff --git a/packages/frontend/apps/electron/src/shared/utils.ts b/packages/frontend/apps/electron/src/shared/utils.ts index 51aa4f52ba..4535258f23 100644 --- a/packages/frontend/apps/electron/src/shared/utils.ts +++ b/packages/frontend/apps/electron/src/shared/utils.ts @@ -1,3 +1,5 @@ +import { join } from 'node:path'; + import type { EventBasedChannel } from 'async-call-rpc'; export function getTime() { @@ -42,3 +44,40 @@ export class MessageEventChannel implements EventBasedChannel { this.worker.postMessage(data); } } + +export const resourcesPath = join(__dirname, `../resources`); + +// credit: https://github.com/facebook/fbjs/blob/main/packages/fbjs/src/core/shallowEqual.js +export function shallowEqual(objA: any, objB: any) { + if (Object.is(objA, objB)) { + return true; + } + + if ( + typeof objA !== 'object' || + objA === null || + typeof objB !== 'object' || + objB === null + ) { + return false; + } + + const keysA = Object.keys(objA); + const keysB = Object.keys(objB); + + if (keysA.length !== keysB.length) { + return false; + } + + // Test for A's keys different from B. + for (const key of keysA) { + if ( + !Object.prototype.hasOwnProperty.call(objB, key) || + !Object.is(objA[key], objB[key]) + ) { + return false; + } + } + + return true; +} diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.ts index 2d8a647785..51133ee1ac 100644 --- a/packages/frontend/media-capture-playground/server/main.ts +++ b/packages/frontend/media-capture-playground/server/main.ts @@ -5,6 +5,7 @@ import { type Application, type AudioTapStream, ShareableContent, + type TappableApplication, } from '@affine/native'; import type { FSWatcher } from 'chokidar'; import chokidar from 'chokidar'; @@ -26,7 +27,7 @@ console.log(`📁 Ensuring recordings directory exists at ${RECORDING_DIR}`); // Types interface Recording { - app: Application; + app: TappableApplication; appGroup: Application | null; buffers: Float32Array[]; stream: AudioTapStream; @@ -54,12 +55,12 @@ interface RecordingMetadata { } interface AppInfo { - app: Application; + app?: TappableApplication; processId: number; processGroupId: number; bundleIdentifier: string; name: string; - running: boolean; + isRunning: boolean; } interface TranscriptionMetadata { @@ -216,7 +217,7 @@ function emitRecordingStatus() { io.emit('apps:recording', { recordings: getRecordingStatus() }); } -async function startRecording(app: Application) { +async function startRecording(app: TappableApplication) { if (recordingMap.has(app.processId)) { console.log( `⚠ī¸ Recording already in progress for ${app.name} (PID: ${app.processId})` @@ -224,40 +225,44 @@ async function startRecording(app: Application) { return; } - const processGroupId = app.processGroupId; - const rootApp = processGroupId - ? (shareableContent - .applications() - .find(a => a.processId === processGroupId) ?? app) - : app; - - console.log( - `🎙ī¸ Starting recording for ${rootApp.name} (PID: ${rootApp.processId})` - ); - - const buffers: Float32Array[] = []; - const stream = app.tapAudio((err, samples) => { - if (err) { - console.error(`❌ Audio stream error for ${rootApp.name}:`, err); + try { + const processGroupId = app.processGroupId; + const rootApp = shareableContent.applicationWithProcessId(processGroupId); + if (!rootApp) { + console.error(`❌ App group not found for ${app.name}`); return; } - const recording = recordingMap.get(app.processId); - if (recording && !recording.isWriting) { - buffers.push(new Float32Array(samples)); - } - }); - recordingMap.set(app.processId, { - app, - appGroup: rootApp, - buffers, - stream, - startTime: Date.now(), - isWriting: false, - }); + console.log( + `🎙ī¸ Starting recording for ${rootApp.name} (PID: ${rootApp.processId})` + ); - console.log(`✅ Recording started successfully for ${rootApp.name}`); - emitRecordingStatus(); + const buffers: Float32Array[] = []; + const stream = app.tapAudio((err, samples) => { + if (err) { + console.error(`❌ Audio stream error for ${rootApp.name}:`, err); + return; + } + const recording = recordingMap.get(app.processId); + if (recording && !recording.isWriting) { + buffers.push(new Float32Array(samples)); + } + }); + + recordingMap.set(app.processId, { + app, + appGroup: rootApp, + buffers, + stream, + startTime: Date.now(), + isWriting: false, + }); + + console.log(`✅ Recording started successfully for ${rootApp.name}`); + emitRecordingStatus(); + } catch (error) { + console.error(`❌ Error starting recording for ${app.name}:`, error); + } } async function stopRecording(processId: number) { @@ -432,7 +437,7 @@ async function setupRecordingsWatcher() { const shareableContent = new ShareableContent(); async function getAllApps(): Promise { - const apps = shareableContent.applications().map(app => { + const apps: (AppInfo | null)[] = shareableContent.applications().map(app => { try { return { app, @@ -440,7 +445,7 @@ async function getAllApps(): Promise { processGroupId: app.processGroupId, bundleIdentifier: app.bundleIdentifier, name: app.name, - running: app.isRunning, + isRunning: app.isRunning, }; } catch (error) { console.error(error); @@ -453,11 +458,30 @@ async function getAllApps(): Promise { v !== null && !v.bundleIdentifier.startsWith('com.apple') ); + for (const app of filteredApps) { + if (filteredApps.some(a => a.processId === app.processGroupId)) { + continue; + } + const appGroup = shareableContent.applicationWithProcessId( + app.processGroupId + ); + if (!appGroup) { + continue; + } + filteredApps.push({ + processId: appGroup.processId, + processGroupId: appGroup.processGroupId, + bundleIdentifier: appGroup.bundleIdentifier, + name: appGroup.name, + isRunning: false, + }); + } + // Stop recording if app is not listed await Promise.all( - filteredApps.map(async ({ app }) => { - if (!filteredApps.some(a => a.processId === app.processId)) { - await stopRecording(app.processId); + Array.from(recordingMap.keys()).map(async processId => { + if (!filteredApps.some(a => a.processId === processId)) { + await stopRecording(processId); } }) ); @@ -467,24 +491,36 @@ async function getAllApps(): Promise { function listenToAppStateChanges(apps: AppInfo[]) { const subscribers = apps.map(({ app }) => { - return ShareableContent.onAppStateChanged(app, () => { - setTimeout(() => { - console.log( - `🔄 Application state changed: ${app.name} (PID: ${app.processId}) is now ${ - app.isRunning ? 'â–ļī¸ running' : '⏚ī¸ stopped' - }` - ); - io.emit('apps:state-changed', { - processId: app.processId, - running: app.isRunning, - }); - if (!app.isRunning) { - stopRecording(app.processId).catch(error => { - console.error('❌ Error stopping recording:', error); + try { + if (!app) { + return { unsubscribe: () => {} }; + } + return ShareableContent.onAppStateChanged(app, () => { + setTimeout(() => { + console.log( + `🔄 Application state changed: ${app.name} (PID: ${app.processId}) is now ${ + app.isRunning ? 'â–ļī¸ running' : '⏚ī¸ stopped' + }` + ); + io.emit('apps:state-changed', { + processId: app.processId, + isRunning: app.isRunning, }); - } - }, 50); - }); + + if (!app.isRunning) { + stopRecording(app.processId).catch(error => { + console.error('❌ Error stopping recording:', error); + }); + } + }, 100); + }); + } catch (error) { + console.error( + `Failed to listen to app state changes for ${app?.name}:`, + error + ); + return { unsubscribe: () => {} }; + } }); appsSubscriber(); @@ -505,7 +541,7 @@ io.on('connection', async socket => { console.log(`📤 Sending ${files.length} saved recordings to new client`); socket.emit('apps:saved', { recordings: files }); - listenToAppStateChanges(initialApps); + listenToAppStateChanges(initialApps.map(app => app.app).filter(app => !!app)); socket.on('disconnect', () => { console.log('🔌 Client disconnected'); @@ -614,19 +650,33 @@ app.get('/apps/:process_id/icon', (req, res) => { const processId = parseInt(req.params.process_id); try { const app = shareableContent.applicationWithProcessId(processId); + if (!app) { + res.status(404).json({ error: 'App not found' }); + return; + } const icon = app.icon; res.set('Content-Type', 'image/png'); res.send(icon); - } catch { + } catch (error) { + console.error(`Error getting icon for process ${processId}:`, error); res.status(404).json({ error: 'App icon not found' }); } }); app.post('/apps/:process_id/record', async (req, res) => { const processId = parseInt(req.params.process_id); - const app = shareableContent.applicationWithProcessId(processId); - await startRecording(app); - res.json({ success: true }); + try { + const app = shareableContent.tappableApplicationWithProcessId(processId); + if (!app) { + res.status(404).json({ error: 'App not found' }); + return; + } + await startRecording(app); + res.json({ success: true }); + } catch (error) { + console.error(`Error starting recording for process ${processId}:`, error); + res.status(500).json({ error: 'Failed to start recording' }); + } }); app.post('/apps/:process_id/stop', async (req, res) => { diff --git a/packages/frontend/media-capture-playground/web/components/app-item.tsx b/packages/frontend/media-capture-playground/web/components/app-item.tsx index cde56c9778..e3cab1aa85 100644 --- a/packages/frontend/media-capture-playground/web/components/app-item.tsx +++ b/packages/frontend/media-capture-playground/web/components/app-item.tsx @@ -15,14 +15,14 @@ export function AppItem({ app, recordings }: AppItemProps) { const appName = app.rootApp.name || ''; const bundleId = app.rootApp.bundleIdentifier || ''; const firstLetter = appName.charAt(0).toUpperCase(); - const isRunning = app.apps.some(a => a.running); + const isRunning = app.apps.some(a => a.isRunning); const recording = recordings?.find((r: RecordingStatus) => app.apps.some(a => a.processId === r.processId) ); const handleRecordClick = React.useCallback(() => { - const recordingApp = app.apps.find(a => a.running); + const recordingApp = app.apps.find(a => a.isRunning); if (!recordingApp) { return; } diff --git a/packages/frontend/media-capture-playground/web/components/app-list.tsx b/packages/frontend/media-capture-playground/web/components/app-list.tsx index d5d3e91162..4b222fffc8 100644 --- a/packages/frontend/media-capture-playground/web/components/app-list.tsx +++ b/packages/frontend/media-capture-playground/web/components/app-list.tsx @@ -24,12 +24,13 @@ export function AppList() { }); socket.on('apps:state-changed', data => { const index = apps.findIndex(a => a.processId === data.processId); + console.log('apps:state-changed', data, index); if (index !== -1) { next( null, apps.toSpliced(index, 1, { ...apps[index], - running: data.running, + isRunning: data.isRunning, }) ); } @@ -83,10 +84,10 @@ export function AppList() { }, [apps]); const runningApps = (appGroups || []).filter(app => - app.apps.some(a => a.running) + app.apps.some(a => a.isRunning) ); const notRunningApps = (appGroups || []).filter( - app => !app.apps.some(a => a.running) + app => !app.apps.some(a => a.isRunning) ); return ( diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.ts index e1ec56a4b0..8e8776b582 100644 --- a/packages/frontend/media-capture-playground/web/types.ts +++ b/packages/frontend/media-capture-playground/web/types.ts @@ -3,7 +3,7 @@ export interface App { processGroupId: number; bundleIdentifier: string; name: string; - running: boolean; + isRunning: boolean; } export interface AppGroup { diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index 2c6afc574e..43a7fadaeb 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -1,14 +1,12 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ export declare class Application { - static tapGlobalAudio(excludedProcesses: Array | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioTapStream + constructor(processId: number) get processId(): number get processGroupId(): number get bundleIdentifier(): string get name(): string get icon(): Buffer - get isRunning(): boolean - tapAudio(audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioTapStream } export declare class ApplicationListChangedSubscriber { @@ -73,11 +71,13 @@ export declare class RecordingPermissions { export declare class ShareableContent { static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber - static onAppStateChanged(app: Application, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber + static onAppStateChanged(app: TappableApplication, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber constructor() - applications(): Array - applicationWithProcessId(processId: number): Application + applications(): Array + applicationWithProcessId(processId: number): Application | null + tappableApplicationWithProcessId(processId: number): TappableApplication | null checkRecordingPermissions(): RecordingPermissions + static tapGlobalAudio(excludedProcesses: Array | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioTapStream } export declare class SqliteConnection { @@ -118,6 +118,19 @@ export declare class SqliteConnection { checkpoint(): Promise } +export declare class TappableApplication { + constructor(objectId: AudioObjectID) + static fromApplication(app: Application, objectId: AudioObjectID): TappableApplication + get processId(): number + get processGroupId(): number + get bundleIdentifier(): string + get name(): string + get objectId(): number + get icon(): Buffer + get isRunning(): boolean + tapAudio(audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioTapStream +} + /**Enumeration of valid values for `set_brate` */ export declare enum Bitrate { /**8_000 */ diff --git a/packages/frontend/native/index.js b/packages/frontend/native/index.js index 2a5fc54f5a..786274f9a4 100644 --- a/packages/frontend/native/index.js +++ b/packages/frontend/native/index.js @@ -380,6 +380,7 @@ module.exports.Mp3Encoder = nativeBinding.Mp3Encoder module.exports.RecordingPermissions = nativeBinding.RecordingPermissions module.exports.ShareableContent = nativeBinding.ShareableContent module.exports.SqliteConnection = nativeBinding.SqliteConnection +module.exports.TappableApplication = nativeBinding.TappableApplication module.exports.Bitrate = nativeBinding.Bitrate module.exports.decodeAudio = nativeBinding.decodeAudio module.exports.decodeAudioSync = nativeBinding.decodeAudioSync diff --git a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs index 0c8f5a728c..1157501163 100644 --- a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs +++ b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs @@ -36,7 +36,6 @@ use screencapturekit::shareable_content::SCShareableContent; use uuid::Uuid; use crate::{ - error::CoreAudioError, pid::{audio_process_list, get_process_property}, tap_audio::{AggregateDevice, AudioTapStream}, }; @@ -94,133 +93,161 @@ static AVCAPTUREDEVICE_CLASS: LazyLock> = static SCSTREAM_CLASS: LazyLock> = LazyLock::new(|| AnyClass::get(c"SCStream")); -struct TappableApplication { - object_id: AudioObjectID, +#[napi] +pub struct Application { + pub(crate) process_id: i32, + pub(crate) name: String, } -impl TappableApplication { - fn new(object_id: AudioObjectID) -> Self { - Self { object_id } - } +#[napi] +impl Application { + #[napi(constructor)] + pub fn new(process_id: i32) -> Result { + // Default values for when we can't get information + let mut app = Self { + process_id, + name: String::new(), + }; - fn process_id(&self) -> std::result::Result { - get_process_property(&self.object_id, kAudioProcessPropertyPID) - } + // Try to populate fields using NSRunningApplication + if process_id > 0 { + // Get NSRunningApplication class + if let Some(running_app_class) = NSRUNNING_APPLICATION_CLASS.as_ref() { + // Get running application with PID + let running_app: *mut AnyObject = unsafe { + msg_send![ + *running_app_class, + runningApplicationWithProcessIdentifier: process_id + ] + }; - fn bundle_identifier(&self) -> Result { - let bundle_id: CFStringRef = - get_process_property(&self.object_id, kAudioProcessPropertyBundleID)?; - Ok(unsafe { CFString::wrap_under_get_rule(bundle_id) }.to_string()) - } + if !running_app.is_null() { + // Get name + unsafe { + let name_ptr: *mut NSString = msg_send![running_app, localizedName]; + if !name_ptr.is_null() { + let length: usize = msg_send![name_ptr, length]; + let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String]; - fn name(&self) -> Result { - // Use catch_unwind to prevent any panics - let name_result = std::panic::catch_unwind(|| { - // Get process ID with error handling - let pid = match self.process_id() { - Ok(pid) => pid, - Err(_) => { - return Ok(String::new()); - } - }; - - // Get NSRunningApplication class with error handling - let running_app_class = match NSRUNNING_APPLICATION_CLASS.as_ref() { - Some(class) => class, - None => { - return Ok(String::new()); - } - }; - - // Get running application with PID - let running_app: *mut AnyObject = - unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] }; - - if running_app.is_null() { - return Ok(String::new()); - } - - // Instead of using Retained::from_raw which takes ownership, - // we'll just copy the string value and let the Objective-C runtime - // handle the memory management of the original object - unsafe { - // Get localized name - let name_ptr: *mut NSString = msg_send![running_app, localizedName]; - if name_ptr.is_null() { - return Ok(String::new()); - } - - // Create a copy of the string without taking ownership of the NSString - let length: usize = msg_send![name_ptr, length]; - let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String]; - - if utf8_ptr.is_null() { - return Ok(String::new()); - } - - let bytes = std::slice::from_raw_parts(utf8_ptr, length); - match std::str::from_utf8(bytes) { - Ok(s) => Ok(s.to_string()), - Err(_) => Ok(String::new()), + if !utf8_ptr.is_null() { + let bytes = std::slice::from_raw_parts(utf8_ptr, length); + if let Ok(s) = std::str::from_utf8(bytes) { + app.name = s.to_string(); + } + } + } + } } } - }); - - // Handle any panics that might have occurred - match name_result { - Ok(result) => result, - Err(_) => Ok(String::new()), } + + Ok(app) } - fn icon(&self) -> Result> { + #[napi(getter)] + pub fn process_id(&self) -> i32 { + self.process_id + } + + #[napi(getter)] + pub fn process_group_id(&self) -> i32 { + if self.process_id > 0 { + let pgid = unsafe { libc::getpgid(self.process_id) }; + if pgid != -1 { + return pgid; + } + // Fall back to process_id if getpgid fails + return self.process_id; + } + -1 + } + + #[napi(getter)] + pub fn bundle_identifier(&self) -> String { + if self.process_id <= 0 { + return String::new(); + } + + // Try to get bundle identifier using NSRunningApplication + if let Some(running_app_class) = NSRUNNING_APPLICATION_CLASS.as_ref() { + let running_app: *mut AnyObject = unsafe { + msg_send![ + *running_app_class, + runningApplicationWithProcessIdentifier: self.process_id + ] + }; + + if !running_app.is_null() { + unsafe { + let bundle_id_ptr: *mut NSString = msg_send![running_app, bundleIdentifier]; + if !bundle_id_ptr.is_null() { + let length: usize = msg_send![bundle_id_ptr, length]; + let utf8_ptr: *const u8 = msg_send![bundle_id_ptr, UTF8String]; + + if !utf8_ptr.is_null() { + let bytes = std::slice::from_raw_parts(utf8_ptr, length); + if let Ok(s) = std::str::from_utf8(bytes) { + return s.to_string(); + } + } + } + } + } + } + + String::new() + } + + #[napi(getter)] + pub fn name(&self) -> String { + self.name.clone() + } + + #[napi(getter)] + pub fn icon(&self) -> Result { // Use catch_unwind to prevent any panics let icon_result = std::panic::catch_unwind(|| { - // Get process ID with error handling - let pid = match self.process_id() { - Ok(pid) => pid, - Err(_) => { - return Ok(Vec::new()); - } - }; - // Get NSRunningApplication class with error handling let running_app_class = match NSRUNNING_APPLICATION_CLASS.as_ref() { Some(class) => class, None => { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } }; // Get running application with PID - let running_app: *mut AnyObject = - unsafe { msg_send![*running_app_class, runningApplicationWithProcessIdentifier: pid] }; + let running_app: *mut AnyObject = unsafe { + msg_send![ + *running_app_class, + runningApplicationWithProcessIdentifier: self.process_id + ] + }; if running_app.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } unsafe { // Get original icon let icon: *mut AnyObject = msg_send![running_app, icon]; if icon.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } // Create a new NSImage with 64x64 size let nsimage_class = match AnyClass::get(c"NSImage") { Some(class) => class, - None => return Ok(Vec::new()), + None => return Ok(Buffer::from(Vec::::new())), }; let resized_image: *mut AnyObject = msg_send![nsimage_class, alloc]; if resized_image.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } let resized_image: *mut AnyObject = msg_send![resized_image, initWithSize: NSSize { width: 64.0, height: 64.0 }]; if resized_image.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } let _: () = msg_send![resized_image, lockFocus]; @@ -241,41 +268,41 @@ impl TappableApplication { // Get TIFF representation from the downsized image let tiff_data: *mut AnyObject = msg_send![resized_image, TIFFRepresentation]; if tiff_data.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } // Create bitmap image rep from TIFF let bitmap_class = match AnyClass::get(c"NSBitmapImageRep") { Some(class) => class, - None => return Ok(Vec::new()), + None => return Ok(Buffer::from(Vec::::new())), }; let bitmap: *mut AnyObject = msg_send![bitmap_class, imageRepWithData: tiff_data]; if bitmap.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } // Create properties dictionary with compression factor let dict_class = match AnyClass::get(c"NSMutableDictionary") { Some(class) => class, - None => return Ok(Vec::new()), + None => return Ok(Buffer::from(Vec::::new())), }; let properties: *mut AnyObject = msg_send![dict_class, dictionary]; if properties.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } // Add compression properties let compression_key = NSString::from_str("NSImageCompressionFactor"); let number_class = match AnyClass::get(c"NSNumber") { Some(class) => class, - None => return Ok(Vec::new()), + None => return Ok(Buffer::from(Vec::::new())), }; let compression_value: *mut AnyObject = msg_send![number_class, numberWithDouble: 0.8]; if compression_value.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } let _: () = msg_send![properties, setObject: compression_value, forKey: &*compression_key]; @@ -285,7 +312,7 @@ impl TappableApplication { msg_send![bitmap, representationUsingType: 4, properties: properties]; // 4 = PNG if png_data.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } // Get bytes from NSData @@ -293,129 +320,101 @@ impl TappableApplication { let length: usize = msg_send![png_data, length]; if bytes.is_null() { - return Ok(Vec::new()); + return Ok(Buffer::from(Vec::::new())); } // Copy bytes into a Vec instead of using the original memory let data = std::slice::from_raw_parts(bytes, length).to_vec(); - Ok(data) + Ok(Buffer::from(data)) } }); // Handle any panics that might have occurred match icon_result { Ok(result) => result, - Err(_) => Ok(Vec::new()), - } - } - - fn process_group_id(&self) -> Result { - // Use catch_unwind to prevent any panics - let pgid_result = std::panic::catch_unwind(|| { - // First get the process ID - let pid = match self.process_id() { - Ok(pid) => pid, - Err(_) => { - return Ok(-1); // Return -1 for error cases - } - }; - - // Call libc's getpgid function to get the process group ID - let pgid = unsafe { libc::getpgid(pid) }; - - // getpgid returns -1 on error - if pgid == -1 { - return Ok(-1); - } - - Ok(pgid) - }); - - // Handle any panics - match pgid_result { - Ok(result) => result, - Err(_) => Ok(-1), + Err(_) => Ok(Buffer::from(Vec::::new())), } } } #[napi] -pub struct Application { - inner: TappableApplication, +pub struct TappableApplication { + pub(crate) app: Application, pub(crate) object_id: AudioObjectID, - pub(crate) process_id: i32, - pub(crate) process_group_id: i32, - pub(crate) bundle_identifier: String, - pub(crate) name: String, } #[napi] -impl Application { - fn new(app: TappableApplication) -> Result { - let object_id = app.object_id; - let bundle_identifier = app.bundle_identifier()?; - let name = app.name()?; - let process_id = app.process_id()?; - let process_group_id = app.process_group_id()?; +impl TappableApplication { + #[napi(constructor)] + pub fn new(object_id: AudioObjectID) -> Result { + // Get process ID from object_id + let process_id = match get_process_property(&object_id, kAudioProcessPropertyPID) { + Ok(pid) => pid, + Err(_) => -1, + }; - Ok(Self { - inner: app, - object_id, - process_id, - process_group_id, - bundle_identifier, - name, - }) + // Create base Application + let app = Application::new(process_id)?; + + Ok(Self { app, object_id }) } - #[napi] - pub fn tap_global_audio( - excluded_processes: Option>, - audio_stream_callback: Arc>, - ) -> Result { - let mut device = AggregateDevice::create_global_tap_but_exclude_processes( - &excluded_processes - .unwrap_or_default() - .iter() - .map(|app| app.object_id) - .collect::>(), - )?; - device.start(audio_stream_callback) + #[napi(factory)] + pub fn from_application(app: &Application, object_id: AudioObjectID) -> Self { + Self { + app: Application { + process_id: app.process_id, + name: app.name.clone(), + }, + object_id, + } } #[napi(getter)] pub fn process_id(&self) -> i32 { - self.process_id + self.app.process_id } #[napi(getter)] pub fn process_group_id(&self) -> i32 { - self.process_group_id + self.app.process_group_id() } #[napi(getter)] pub fn bundle_identifier(&self) -> String { - self.bundle_identifier.clone() + // First try to get from the Application + let app_bundle_id = self.app.bundle_identifier(); + if !app_bundle_id.is_empty() { + return app_bundle_id; + } + + // If not available, try to get from the audio process property + match get_process_property::(&self.object_id, kAudioProcessPropertyBundleID) { + Ok(bundle_id) => { + // Safely convert CFStringRef to Rust String + let cf_string = unsafe { CFString::wrap_under_create_rule(bundle_id) }; + cf_string.to_string() + } + Err(_) => { + // Return empty string if we couldn't get the bundle ID + String::new() + } + } } #[napi(getter)] pub fn name(&self) -> String { - self.name.clone() + self.app.name.clone() + } + + #[napi(getter)] + pub fn object_id(&self) -> u32 { + self.object_id } #[napi(getter)] pub fn icon(&self) -> Result { - // Use catch_unwind to prevent any panics - let result = std::panic::catch_unwind(|| match self.inner.icon() { - Ok(icon) => Ok(Buffer::from(icon)), - Err(_) => Ok(Buffer::from(Vec::::new())), - }); - - // Handle any panics - match result { - Ok(result) => result, - Err(_) => Ok(Buffer::from(Vec::::new())), - } + self.app.icon() } #[napi(getter)] @@ -424,20 +423,14 @@ impl Application { let result = std::panic::catch_unwind(|| { match get_process_property(&self.object_id, kAudioProcessPropertyIsRunningInput) { Ok(is_running) => Ok(is_running), - Err(_) => { - // Default to true to avoid potential issues - Ok(true) - } + Err(_) => Ok(false), } }); // Handle any panics match result { Ok(result) => result, - Err(_) => { - // Default to true to avoid potential issues - Ok(true) - } + Err(_) => Ok(false), } } @@ -446,6 +439,7 @@ impl Application { &self, audio_stream_callback: Arc>, ) -> Result { + // Use the new method that takes a TappableApplication directly let mut device = AggregateDevice::new(self)?; device.start(audio_stream_callback) } @@ -585,20 +579,22 @@ impl ShareableContent { #[napi] pub fn on_app_state_changed( - app: &Application, + app: &TappableApplication, callback: Arc>, ) -> Result { let id = Uuid::new_v4(); + let object_id = app.object_id; + let mut lock = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write().map_err(|_| { Error::new( Status::GenericFailure, "Poisoned RwLock while writing ApplicationStateChangedSubscribers", ) })?; - if let Some(subscribers) = lock.get_mut(&app.object_id) { + + if let Some(subscribers) = lock.get_mut(&object_id) { subscribers.insert(id, callback); } else { - let object_id = app.object_id; let list_change: RcBlock = RcBlock::new(move |in_number_addresses, in_addresses: *mut c_void| { let addresses = unsafe { @@ -630,7 +626,7 @@ impl ShareableContent { let listener_block = &*list_change as *const Block; let status = unsafe { AudioObjectAddPropertyListenerBlock( - app.object_id, + object_id, &address, ptr::null_mut(), listener_block.cast_mut().cast(), @@ -647,12 +643,9 @@ impl ShareableContent { map.insert(id, callback); map }; - lock.insert(app.object_id, subscribers); + lock.insert(object_id, subscribers); } - Ok(ApplicationStateChangedSubscriber { - id, - object_id: app.object_id, - }) + Ok(ApplicationStateChangedSubscriber { id, object_id }) } #[napi(constructor)] @@ -663,8 +656,8 @@ impl ShareableContent { } #[napi] - pub fn applications(&self) -> Result> { - RUNNING_APPLICATIONS + pub fn applications(&self) -> Result> { + let app_list = RUNNING_APPLICATIONS .read() .map_err(|_| { Error::new( @@ -674,46 +667,73 @@ impl ShareableContent { })? .iter() .filter_map(|id| { - let app = TappableApplication::new(*id); - if !app.bundle_identifier().ok()?.is_empty() { - Some(Application::new(app)) + let tappable_app = match TappableApplication::new(*id) { + Ok(app) => app, + Err(_) => return None, + }; + + if !tappable_app.bundle_identifier().is_empty() { + Some(tappable_app) } else { None } }) - .collect() + .collect::>(); + + Ok(app_list) } #[napi] - pub fn application_with_process_id(&self, process_id: u32) -> Result { - // Find the AudioObjectID for the given process ID - let audio_object_id = { - let running_apps = RUNNING_APPLICATIONS.read().map_err(|_| { - Error::new( - Status::GenericFailure, - "Poisoned RwLock while reading RunningApplications", - ) - })?; - - *running_apps - .iter() - .find(|&&id| { - let app = TappableApplication::new(id); - app - .process_id() - .map(|pid| pid as u32 == process_id) - .unwrap_or(false) - }) - .ok_or_else(|| { - Error::new( - Status::GenericFailure, - format!("No application found with process ID {}", process_id), - ) - })? + pub fn application_with_process_id(&self, process_id: u32) -> Option { + // Get NSRunningApplication class + let running_app_class = match NSRUNNING_APPLICATION_CLASS.as_ref() { + Some(class) => class, + None => return None, }; - let app = TappableApplication::new(audio_object_id); - Application::new(app) + // Get running application with PID + let running_app: *mut AnyObject = unsafe { + msg_send![ + *running_app_class, + runningApplicationWithProcessIdentifier: process_id as i32 + ] + }; + + if running_app.is_null() { + return None; + } + + // Create an Application directly + match Application::new(process_id as i32) { + Ok(app) => Some(app), + Err(_) => None, + } + } + + #[napi] + pub fn tappable_application_with_process_id( + &self, + process_id: u32, + ) -> Option { + // Find the TappableApplication with this process ID in the list of running + // applications + match self.applications() { + Ok(apps) => { + for app in apps { + if app.process_id() == process_id as i32 { + return Some(app); + } + } + + // If we couldn't find a TappableApplication with this process ID, create a new + // one with a default object_id of 0 (which won't be able to tap audio) + match Application::new(process_id as i32) { + Ok(app) => Some(TappableApplication::from_application(&app, 0)), + Err(_) => None, + } + } + Err(_) => None, + } } #[napi] @@ -743,4 +763,19 @@ impl ShareableContent { screen: screen_status, }) } + + #[napi] + pub fn tap_global_audio( + excluded_processes: Option>, + audio_stream_callback: Arc>, + ) -> Result { + let mut device = AggregateDevice::create_global_tap_but_exclude_processes( + &excluded_processes + .unwrap_or_default() + .iter() + .map(|app| app.object_id) + .collect::>(), + )?; + device.start(audio_stream_callback) + } } diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs index 093346ac35..60ca00ab25 100644 --- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs +++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs @@ -30,7 +30,7 @@ use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode}; use crate::{ ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError, - queue::create_audio_tap_queue, screen_capture_kit::Application, + queue::create_audio_tap_queue, screen_capture_kit::TappableApplication, }; extern "C" { @@ -88,10 +88,12 @@ pub struct AggregateDevice { } impl AggregateDevice { - pub fn new(app: &Application) -> Result { + pub fn new(app: &TappableApplication) -> Result { + let object_id = app.object_id; + + let tap_description = CATapDescription::init_stereo_mixdown_of_processes(object_id)?; let mut tap_id: AudioObjectID = 0; - let tap_description = CATapDescription::init_stereo_mixdown_of_processes(app.object_id)?; let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) }; if status != 0 { @@ -109,7 +111,37 @@ impl AggregateDevice { ) }; - // Check the status and return the appropriate result + if status != 0 { + return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into()); + } + + Ok(Self { + tap_id, + id: aggregate_device_id, + }) + } + + pub fn new_from_object_id(object_id: AudioObjectID) -> Result { + let mut tap_id: AudioObjectID = 0; + + let tap_description = CATapDescription::init_stereo_mixdown_of_processes(object_id)?; + let status = unsafe { AudioHardwareCreateProcessTap(tap_description.inner, &mut tap_id) }; + + if status != 0 { + return Err(CoreAudioError::CreateProcessTapFailed(status).into()); + } + + let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?; + + let mut aggregate_device_id: AudioObjectID = 0; + + let status = unsafe { + AudioHardwareCreateAggregateDevice( + description_dict.as_concrete_TypeRef().cast(), + &mut aggregate_device_id, + ) + }; + if status != 0 { return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into()); }