feat(editor): audio block (#10947)

AudioMedia entity for loading & controlling a single audio media AudioMediaManagerService: Global audio state synchronization across tabs AudioAttachmentService + AudioAttachmentBlock for manipulating AttachmentBlock in affine - e.g., filling transcription (using mock endpoint for now) Added AudioBlock + AudioPlayer for rendering audio block in affine (new transcription block whose renderer is provided in affine) fix AF-2292 fix AF-2337
2026-08-31 05:39:28 +08:00 · 2025-03-20 12:46:14 +00:00
parent 8a5393ea50
commit fad49bb070
120 changed files with 5407 additions and 950 deletions
@@ -5,14 +5,28 @@ import { type PropsWithChildren } from 'react';

 import * as styles from './index.css';

-export function SidebarContainer({ children }: PropsWithChildren) {
-  return <div className={clsx([styles.baseContainer])}>{children}</div>;
+interface SidebarContainerProps extends PropsWithChildren {
+  className?: string;
 }

-export function SidebarScrollableContainer({ children }: PropsWithChildren) {
+export function SidebarContainer({
+  children,
+  className,
+}: SidebarContainerProps) {
+  return (
+    <div className={clsx([styles.baseContainer, className])}>{children}</div>
+  );
+}
+
+export function SidebarScrollableContainer({
+  children,
+  className,
+}: SidebarContainerProps) {
  const [setContainer, hasScrollTop] = useHasScrollTop();
  return (
-    <ScrollArea.Root className={styles.scrollableContainerRoot}>
+    <ScrollArea.Root
+      className={clsx([styles.scrollableContainerRoot, className])}
+    >
      <div
        data-has-scroll-top={hasScrollTop}
        className={styles.scrollTopBorder}
@@ -1,6 +1,7 @@
 import { notify } from '@affine/component';
 import { I18n } from '@affine/i18n';
 import { OnEvent, Service } from '@toeverything/infra';
+import type { To } from 'history';
 import { debounce } from 'lodash-es';

 import { AuthService, DefaultServerService, ServersService } from '../../cloud';
@@ -32,6 +33,25 @@ export class DesktopApiService extends Service {
    return this.api.sharedStorage;
  }

+  async showTab(tabId: string, to?: To) {
+    if (to) {
+      const url = new URL(to.toString());
+      const tabs = await this.api.handler.ui.getTabViewsMeta();
+      const tab = tabs.workbenches.find(t => t.id === tabId);
+      if (tab) {
+        const basename = tab.basename;
+        if (url.pathname.startsWith(basename)) {
+          const pathname = url.pathname.slice(basename.length);
+          await this.api.handler.ui.tabGoTo(
+            tabId,
+            pathname + url.search + url.hash
+          );
+        }
+      }
+    }
+    await this.api.handler.ui.showTab(tabId);
+  }
+
  private setupStartListener() {
    this.setupCommonUIEvents();
    this.setupAuthRequestEvent();
@@ -243,6 +243,15 @@ export const AFFINE_FLAGS = {
    configurable: !isMobile,
    defaultState: false,
  },
+  enable_audio_block: {
+    category: 'affine',
+    displayName:
+      'com.affine.settings.workspace.experimental-features.enable-audio-block.name',
+    description:
+      'com.affine.settings.workspace.experimental-features.enable-audio-block.description',
+    configurable: !isMobile,
+    defaultState: false,
+  },
  enable_editor_rtl: {
    category: 'affine',
    displayName:
@@ -30,6 +30,7 @@ import { configureImportTemplateModule } from './import-template';
 import { configureIntegrationModule } from './integration';
 import { configureJournalModule } from './journal';
 import { configureLifecycleModule } from './lifecycle';
+import { configureMediaModule } from './media';
 import { configureNavigationModule } from './navigation';
 import { configureNotificationModule } from './notification';
 import { configureOpenInApp } from './open-in-app';
@@ -103,6 +104,7 @@ export function configureCommonModules(framework: Framework) {
  configureAIButtonModule(framework);
  configureTemplateDocModule(framework);
  configureBlobManagementModule(framework);
+  configureMediaModule(framework);
  configureImportClipperModule(framework);
  configureNotificationModule(framework);
  configureIntegrationModule(framework);
@@ -0,0 +1,35 @@
+# Cross-Tab Audio State Synchronization
+
+## How Cross-Tab Audio Synchronization Works
+
+1. **Global State**:
+
+   - Shared between all tabs via Electron's global state or browser storage
+   - Contains `PlaybackState` and `MediaStats` objects
+   - Each state update includes a timestamp (`updateTime`) to track recency
+
+2. **Tab 1 - Playing Audio**:
+
+   - User initiates playback in Tab 1
+   - `AudioMediaManagerService` updates global state with new playback info
+   - Global state includes the tab ID that initiated playback
+
+3. **Tab 2 - Responding to Changes**:
+
+   - Observes changes to global state via `observeGlobalPlaybackState`
+   - Detects that audio is playing in another tab (different tabId)
+   - Automatically stops any playing audio in Tab 2
+   - Does not attempt to play the audio from Tab 1
+
+4. **State Synchronization**:
+
+   - All state changes include `updateTime` to prevent race conditions
+   - `distinctUntilChanged` ensures only meaningful state changes trigger updates
+   - `skipUpdate` parameter prevents circular update loops
+
+5. **Exclusive Playback**:
+   - `ensureExclusivePlayback` ensures only one audio plays at a time
+   - When a tab starts playing, all other tabs stop their playback
+   - Global state maintains a single source of truth
+
+This architecture ensures that audio playback is synchronized across tabs, with only one audio playing at any time, while maintaining a consistent user experience.
@@ -0,0 +1,191 @@
+import {
+  type AttachmentBlockModel,
+  TranscriptionBlockFlavour,
+  type TranscriptionBlockModel,
+} from '@blocksuite/affine/model';
+import type { AffineTextAttributes } from '@blocksuite/affine/shared/types';
+import { type DeltaInsert, Text } from '@blocksuite/affine/store';
+import { computed } from '@preact/signals-core';
+import {
+  catchErrorInto,
+  effect,
+  Entity,
+  fromPromise,
+  LiveData,
+  onComplete,
+  onStart,
+} from '@toeverything/infra';
+import { cssVarV2 } from '@toeverything/theme/v2';
+import { EMPTY, mergeMap, switchMap } from 'rxjs';
+
+import type { AudioMediaManagerService } from '../services/audio-media-manager';
+import type { AudioMedia } from './audio-media';
+
+export interface TranscriptionResult {
+  title: string;
+  summary: string;
+  segments: {
+    speaker: string;
+    start_time: string;
+    end_time: string;
+    transcription: string;
+  }[];
+}
+
+// BlockSuiteError: yText must not contain "\r" because it will break the range synchronization
+function sanitizeText(text: string) {
+  return text.replace(/\r/g, '');
+}
+
+export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
+  private readonly refCount$ = new LiveData<number>(0);
+  readonly audioMedia: AudioMedia;
+  constructor(
+    public readonly audioMediaManagerService: AudioMediaManagerService
+  ) {
+    super();
+    const mediaRef = audioMediaManagerService.ensureMediaEntity(this.props);
+    this.audioMedia = mediaRef.media;
+    this.disposables.push(() => mediaRef.release());
+  }
+
+  // rendering means the attachment is visible in the editor
+  rendering$ = this.refCount$.map(refCount => refCount > 0);
+  expanded$ = new LiveData<boolean>(true);
+  transcribing$ = new LiveData<boolean>(false);
+  transcriptionError$ = new LiveData<Error | null>(null);
+  transcribed$ = LiveData.computed(get => {
+    const transcriptionBlock = get(this.transcriptionBlock$);
+    if (!transcriptionBlock) {
+      return null;
+    }
+    const childMap = get(LiveData.fromSignal(transcriptionBlock.childMap));
+    return childMap.size > 0;
+  });
+
+  transcribe = effect(
+    switchMap(() =>
+      fromPromise(this.doTranscribe()).pipe(
+        mergeMap(result => {
+          // attach transcription result to the block
+          this.fillTranscriptionResult(result);
+          return EMPTY;
+        }),
+        catchErrorInto(this.transcriptionError$),
+        onStart(() => this.transcribing$.setValue(true)),
+        onComplete(() => this.transcribing$.setValue(false))
+      )
+    )
+  );
+
+  readonly transcriptionBlock$ = LiveData.fromSignal(
+    computed(() => {
+      // find the last transcription block
+      for (const key of [...this.props.childMap.value.keys()].reverse()) {
+        const block = this.props.doc.getBlock$(key);
+        if (block?.flavour === TranscriptionBlockFlavour) {
+          return block.model as unknown as TranscriptionBlockModel;
+        }
+      }
+      return null;
+    })
+  );
+
+  // TODO: use real implementation
+  private readonly doTranscribe = async (): Promise<TranscriptionResult> => {
+    try {
+      const buffer = await this.audioMedia.getBuffer();
+      if (!buffer) {
+        throw new Error('No audio buffer available');
+      }
+
+      // Send binary audio data directly
+      const blob = new Blob([buffer], { type: 'audio/wav' }); // adjust mime type if needed
+      const formData = new FormData();
+      formData.append('audio', blob);
+
+      const response = await fetch('http://localhost:6544/transcribe', {
+        method: 'POST',
+        body: formData,
+      });
+
+      if (!response.ok) {
+        throw new Error(`Transcription failed: ${response.statusText}`);
+      }
+
+      const result = await response.json();
+      return result.transcription;
+    } catch (error) {
+      console.error('Error transcribing audio:', error);
+      throw error;
+    }
+  };
+
+  private readonly fillTranscriptionResult = (result: TranscriptionResult) => {
+    this.props.props.caption = result.title;
+    // todo: add transcription block schema etc.
+    const transcriptionBlockId = this.props.doc.addBlock(
+      'affine:transcription',
+      {
+        transcription: result,
+      },
+      this.props.id
+    );
+
+    const calloutId = this.props.doc.addBlock(
+      'affine:callout',
+      {
+        emoji: '💬',
+      },
+      transcriptionBlockId
+    );
+
+    // todo: refactor
+    const spearkerToColors = new Map<string, string>();
+    for (const segment of result.segments) {
+      let color = spearkerToColors.get(segment.speaker);
+      const colorOptions = [
+        cssVarV2.text.highlight.fg.red,
+        cssVarV2.text.highlight.fg.green,
+        cssVarV2.text.highlight.fg.blue,
+        cssVarV2.text.highlight.fg.yellow,
+        cssVarV2.text.highlight.fg.purple,
+        cssVarV2.text.highlight.fg.orange,
+        cssVarV2.text.highlight.fg.teal,
+        cssVarV2.text.highlight.fg.grey,
+        cssVarV2.text.highlight.fg.magenta,
+      ];
+      if (!color) {
+        color = colorOptions[spearkerToColors.size % colorOptions.length];
+        spearkerToColors.set(segment.speaker, color);
+      }
+      const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [
+        {
+          insert: sanitizeText(segment.start_time + ' ' + segment.speaker),
+          attributes: {
+            color,
+            bold: true,
+          },
+        },
+        {
+          insert: ': ' + sanitizeText(segment.transcription),
+        },
+      ];
+      this.props.doc.addBlock(
+        'affine:paragraph',
+        {
+          text: new Text(deltaInserts),
+        },
+        calloutId
+      );
+    }
+  };
+
+  mount() {
+    this.refCount$.setValue(this.refCount$.value + 1);
+  }
+
+  unmount() {
+    this.refCount$.setValue(this.refCount$.value - 1);
+  }
+}
@@ -0,0 +1,443 @@
+import { DebugLogger } from '@affine/debug';
+import {
+  catchErrorInto,
+  effect,
+  Entity,
+  fromPromise,
+  LiveData,
+  type MediaStats,
+  onComplete,
+  onStart,
+} from '@toeverything/infra';
+import { clamp } from 'lodash-es';
+import { EMPTY, mergeMap, switchMap } from 'rxjs';
+
+import type { WorkspaceService } from '../../workspace';
+
+const logger = new DebugLogger('AttachmentBlockMedia');
+
+/**
+ * Interface for audio sources that can be played by AudioMedia
+ */
+export interface AudioSource {
+  /**
+   * The source ID (blob id) for the blob
+   */
+  blobId: string;
+
+  /**
+   * The metadata of the audio source Web Media Session API
+   */
+  metadata: MediaMetadata;
+}
+
+export type AudioMediaPlaybackState = 'idle' | 'playing' | 'paused' | 'stopped';
+
+export interface AudioMediaSyncState {
+  state: AudioMediaPlaybackState;
+  seekOffset: number;
+  updateTime: number; // the time when the playback state is updated
+}
+
+/**
+ * Audio media entity.
+ * Controls the playback of audio media.
+ */
+export class AudioMedia extends Entity<AudioSource> {
+  constructor(private readonly workspaceService: WorkspaceService) {
+    super();
+
+    // Create audio element
+    this.audioElement = new Audio();
+
+    // Set up event listeners for the audio element
+    const onPlay = () => {
+      this.updatePlaybackState(
+        'playing',
+        this.playbackState$.getValue().seekOffset,
+        Date.now()
+      );
+      this.updateMediaSessionPlaybackState('playing');
+    };
+
+    const onPause = () => {
+      this.pause();
+    };
+
+    const onEnded = () => {
+      this.pause();
+    };
+
+    // Add event listeners
+    this.audioElement.addEventListener('play', onPlay);
+    this.audioElement.addEventListener('pause', onPause);
+    this.audioElement.addEventListener('ended', onEnded);
+
+    this.revalidateBuffer();
+
+    this.disposables.push(() => {
+      // Clean up audio resources before calling super.dispose
+      try {
+        // Remove event listeners
+        this.audioElement.removeEventListener('play', onPlay);
+        this.audioElement.removeEventListener('pause', onPause);
+        this.audioElement.removeEventListener('ended', onEnded);
+
+        // Revoke blob URL if it exists
+        if (
+          this.audioElement.src &&
+          this.audioElement.src.startsWith('blob:')
+        ) {
+          URL.revokeObjectURL(this.audioElement.src);
+        }
+
+        this.audioElement.pause();
+        this.audioElement.src = '';
+        this.audioElement.load(); // Reset and release resources
+
+        // Clean up media session
+        this.cleanupMediaSession();
+      } catch (e) {
+        // Ignore errors during cleanup
+        logger.warn('Error cleaning up audio element during disposal', e);
+      }
+    });
+  }
+
+  loading$ = new LiveData(false);
+  loadError$ = new LiveData<Error | null>(null);
+  waveform$ = new LiveData<number[] | null>(null);
+  duration$ = new LiveData<number | null>(null);
+
+  /**
+   * LiveData that exposes the current playback state and data for global state synchronization
+   */
+  playbackState$ = new LiveData<AudioMediaSyncState>({
+    state: 'idle',
+    seekOffset: 0,
+    updateTime: 0,
+  });
+
+  stats$ = LiveData.computed(get => {
+    const waveform = get(this.waveform$) ?? [];
+    const duration = get(this.duration$) ?? 0;
+    return { waveform, duration };
+  });
+
+  private readonly audioElement: HTMLAudioElement;
+
+  private updatePlaybackState(
+    state: AudioMediaPlaybackState,
+    seekOffset: number,
+    updateTime = Date.now()
+  ) {
+    this.playbackState$.setValue({
+      state,
+      seekOffset,
+      updateTime,
+    });
+  }
+
+  async getBuffer() {
+    const blobId = this.props.blobId;
+    if (!blobId) {
+      throw new Error('Audio source ID not found');
+    }
+
+    const blobRecord =
+      await this.workspaceService.workspace.engine.blob.get(blobId);
+
+    if (!blobRecord) {
+      throw new Error('Audio blob not found');
+    }
+
+    return blobRecord.data;
+  }
+
+  private async loadAudioBuffer() {
+    const uint8Array = await this.getBuffer();
+
+    // Create a blob from the uint8Array
+    const blob = new Blob([uint8Array]);
+
+    const startTime = performance.now();
+    // calculating audio stats is expensive. Maybe persist the result in cache?
+    const stats = await this.calcuateStatsFromBuffer(blob);
+    logger.debug(
+      `Calculate audio stats time: ${performance.now() - startTime}ms`
+    );
+    return {
+      blob,
+      ...stats,
+    };
+  }
+
+  readonly revalidateBuffer = effect(
+    switchMap(() => {
+      return fromPromise(async () => {
+        return this.loadAudioBuffer();
+      }).pipe(
+        mergeMap(({ blob, duration, waveform }) => {
+          const url = URL.createObjectURL(blob);
+          this.duration$.setValue(duration);
+          // Set the audio element source
+          this.audioElement.src = url;
+          this.waveform$.setValue(waveform);
+          // If the media is playing, resume the playback
+          if (this.playbackState$.getValue().state === 'playing') {
+            this.play(true);
+          }
+          return EMPTY;
+        }),
+        onStart(() => this.loading$.setValue(true)),
+        onComplete(() => {
+          this.loading$.setValue(false);
+        }),
+        catchErrorInto(this.loadError$)
+      );
+    })
+  );
+
+  get waveform() {
+    return this.waveform$.getValue();
+  }
+
+  getStats(): Pick<MediaStats, 'duration' | 'waveform'> {
+    return this.stats$.getValue();
+  }
+
+  private setupMediaSession() {
+    if (!('mediaSession' in navigator)) {
+      return;
+    }
+
+    // Set up media session action handlers
+    navigator.mediaSession.setActionHandler('play', () => {
+      this.play();
+    });
+
+    navigator.mediaSession.setActionHandler('pause', () => {
+      this.pause();
+    });
+
+    navigator.mediaSession.setActionHandler('stop', () => {
+      this.stop();
+    });
+
+    navigator.mediaSession.setActionHandler('seekto', details => {
+      if (details.seekTime !== undefined) {
+        this.seekTo(details.seekTime);
+      }
+    });
+  }
+
+  private updateMediaSessionMetadata() {
+    if (!('mediaSession' in navigator) || !this.props.metadata) {
+      return;
+    }
+    navigator.mediaSession.metadata = this.props.metadata;
+  }
+
+  private updateMediaSessionPositionState(seekTime: number) {
+    if (!('mediaSession' in navigator)) {
+      return;
+    }
+
+    const duration = this.audioElement.duration || 0;
+    if (duration > 0) {
+      navigator.mediaSession.setPositionState({
+        duration,
+        position: seekTime,
+        playbackRate: 1.0,
+      });
+    }
+  }
+
+  private updateMediaSessionPlaybackState(state: AudioMediaPlaybackState) {
+    if (!('mediaSession' in navigator)) {
+      return;
+    }
+
+    navigator.mediaSession.playbackState =
+      state === 'playing' ? 'playing' : 'paused';
+    this.updateMediaSessionMetadata();
+  }
+
+  private cleanupMediaSession() {
+    if (!('mediaSession' in navigator)) {
+      return;
+    }
+    navigator.mediaSession.metadata = null;
+    // Reset all action handlers
+    navigator.mediaSession.setActionHandler('play', null);
+    navigator.mediaSession.setActionHandler('pause', null);
+    navigator.mediaSession.setActionHandler('stop', null);
+    navigator.mediaSession.setActionHandler('seekto', null);
+  }
+
+  play(skipUpdate?: boolean) {
+    if (!this.audioElement.src) {
+      return;
+    }
+    const duration = this.audioElement.duration || 0;
+    const currentSeek = this.getCurrentSeekPosition();
+    if (!skipUpdate || currentSeek >= duration) {
+      // If we're at the end of the track, reset the seek position to 0
+      if (currentSeek >= duration) {
+        this.audioElement.currentTime = 0;
+        this.updatePlaybackState('playing', 0);
+      } else {
+        this.updatePlaybackState(
+          'playing',
+          this.playbackState$.getValue().seekOffset
+        );
+      }
+    }
+
+    // Play the audio element
+    this.audioElement.play().catch(error => {
+      logger.error('Error playing audio:', error);
+      this.updatePlaybackState('paused', this.audioElement.currentTime);
+    });
+
+    // Set up media session when playback starts
+    this.setupMediaSession();
+    this.updateMediaSessionPositionState(this.audioElement.currentTime);
+    this.updateMediaSessionPlaybackState('playing');
+  }
+
+  pause(skipUpdate?: boolean) {
+    if (!this.audioElement.src) {
+      return;
+    }
+    if (!skipUpdate) {
+      // Update startSeekOffset before pausing
+      this.updatePlaybackState('paused', this.audioElement.currentTime);
+    }
+
+    // Pause the audio element
+    this.audioElement.pause();
+    this.updateMediaSessionPlaybackState('paused');
+  }
+
+  stop(skipUpdate?: boolean) {
+    if (!this.audioElement.src) {
+      return;
+    }
+    // Pause the audio element and reset position
+    this.audioElement.pause();
+    this.audioElement.currentTime = 0;
+
+    if (!skipUpdate) {
+      // Reset the seek position
+      this.updatePlaybackState('stopped', 0);
+    }
+
+    this.updateMediaSessionPlaybackState('stopped');
+
+    // Clean up media session when stopped
+    this.cleanupMediaSession();
+  }
+
+  // Add a seekTo method to handle seeking
+  seekTo(seekTime: number, skipUpdate?: boolean) {
+    if (!this.audioElement.src) {
+      return;
+    }
+
+    const duration = this.audioElement.duration;
+    // Clamp the time value between 0 and duration
+    const clampedTime = clamp(0, seekTime, duration || 0);
+
+    // Update the audio element's current time
+    this.audioElement.currentTime = clampedTime;
+
+    // Update startSeekOffset and startTime if playing
+    const currentState = this.playbackState$.getValue();
+    if (!skipUpdate) {
+      this.updatePlaybackState(currentState.state, clampedTime);
+    }
+    this.updateMediaSessionPositionState(clampedTime);
+  }
+
+  syncState(state: AudioMediaSyncState) {
+    const currentState = this.playbackState$.getValue();
+    if (state.updateTime <= currentState.updateTime) {
+      return;
+    }
+    this.updatePlaybackState(state.state, state.seekOffset, state.updateTime);
+    if (state.state !== currentState.state) {
+      if (state.state === 'playing') {
+        this.play(true);
+      } else if (state.state === 'paused') {
+        this.pause(true);
+      } else if (state.state === 'stopped') {
+        this.stop(true);
+      }
+    }
+    this.seekTo(state.seekOffset, true);
+  }
+
+  /**
+   * Get the current playback seek position
+   */
+  getCurrentSeekPosition(): number {
+    if (this.playbackState$.getValue().state === 'playing') {
+      // For playing state, use the actual current time from audio element
+      return this.audioElement.currentTime;
+    }
+    // For other states, return the stored offset
+    return this.playbackState$.getValue().seekOffset;
+  }
+
+  /**
+   * Get the playback state data
+   */
+  getPlaybackStateData() {
+    return this.playbackState$.getValue();
+  }
+
+  private async calcuateStatsFromBuffer(buffer: Blob) {
+    const audioContext = new AudioContext();
+    const audioBuffer = await audioContext.decodeAudioData(
+      await buffer.arrayBuffer()
+    );
+    const waveform = await this.calculateWaveform(audioBuffer);
+    return { waveform, duration: audioBuffer.duration };
+  }
+
+  /**
+   * Calculate the waveform of the audio buffer for visualization
+   */
+  private async calculateWaveform(audioBuffer: AudioBuffer) {
+    // Get the first channel's data
+    const channelData = audioBuffer.getChannelData(0);
+    const samples = 1000; // Number of points in the waveform
+    const blockSize = Math.floor(channelData.length / samples);
+    const waveform = [];
+
+    // First pass: calculate raw averages
+    for (let i = 0; i < samples; i++) {
+      const start = i * blockSize;
+      const end = start + blockSize;
+      let sum = 0;
+
+      for (let j = start; j < end; j++) {
+        sum += Math.abs(channelData[j]);
+      }
+
+      const average = sum / blockSize;
+      waveform.push(average);
+    }
+
+    // Second pass: normalize to make max value 1
+    const maxValue = Math.max(...waveform);
+    if (maxValue > 0) {
+      for (let i = 0; i < waveform.length; i++) {
+        waveform[i] = waveform[i] / maxValue;
+      }
+    }
+
+    return waveform;
+  }
+}
@@ -0,0 +1,46 @@
+import type { Framework } from '@toeverything/infra';
+
+import { DesktopApiService } from '../desktop-api';
+import { GlobalState } from '../storage';
+import { WorkbenchService } from '../workbench';
+import { WorkspaceScope, WorkspaceService } from '../workspace';
+import { AudioAttachmentBlock } from './entities/audio-attachment-block';
+import { AudioMedia } from './entities/audio-media';
+import {
+  ElectronGlobalMediaStateProvider,
+  GlobalMediaStateProvider,
+  WebGlobalMediaStateProvider,
+} from './providers/global-audio-state';
+import { AudioAttachmentService } from './services/audio-attachment';
+import { AudioMediaManagerService } from './services/audio-media-manager';
+
+export function configureMediaModule(framework: Framework) {
+  if (BUILD_CONFIG.isElectron) {
+    framework
+      .impl(GlobalMediaStateProvider, ElectronGlobalMediaStateProvider, [
+        GlobalState,
+      ])
+      .scope(WorkspaceScope)
+      .entity(AudioMedia, [WorkspaceService])
+      .entity(AudioAttachmentBlock, [AudioMediaManagerService])
+      .service(AudioMediaManagerService, [
+        GlobalMediaStateProvider,
+        WorkbenchService,
+        DesktopApiService,
+      ])
+      .service(AudioAttachmentService);
+  } else {
+    framework
+      .impl(GlobalMediaStateProvider, WebGlobalMediaStateProvider)
+      .scope(WorkspaceScope)
+      .entity(AudioMedia, [WorkspaceService])
+      .entity(AudioAttachmentBlock, [AudioMediaManagerService])
+      .service(AudioMediaManagerService, [
+        GlobalMediaStateProvider,
+        WorkbenchService,
+      ])
+      .service(AudioAttachmentService);
+  }
+}
+
+export { AudioMedia, AudioMediaManagerService };
@@ -0,0 +1,126 @@
+import {
+  createIdentifier,
+  LiveData,
+  type MediaStats,
+  type PlaybackState,
+} from '@toeverything/infra';
+
+import type { GlobalState } from '../../storage';
+
+const GLOBAL_MEDIA_PLAYBACK_STATE_KEY = 'media:playback-state';
+const GLOBAL_MEDIA_STATS_KEY = 'media:stats';
+
+export const GlobalMediaStateProvider =
+  createIdentifier<BaseGlobalMediaStateProvider>('GlobalMediaStateProvider');
+
+/**
+ * Base class for media state providers
+ */
+export abstract class BaseGlobalMediaStateProvider {
+  abstract readonly playbackState$: LiveData<PlaybackState | null | undefined>;
+  abstract readonly stats$: LiveData<MediaStats | null | undefined>;
+
+  /**
+   * Update the playback state
+   * @param state Full state object or partial state to update
+   */
+  abstract updatePlaybackState(state: Partial<PlaybackState> | null): void;
+
+  /**
+   * Update the media stats
+   * @param stats Full stats object or partial stats to update
+   */
+  abstract updateStats(stats: Partial<MediaStats> | null): void;
+}
+
+/**
+ * Provider for global media state in Electron environment
+ * This ensures only one media is playing at a time across all tabs
+ */
+export class ElectronGlobalMediaStateProvider extends BaseGlobalMediaStateProvider {
+  constructor(private readonly globalState: GlobalState) {
+    super();
+  }
+
+  readonly playbackState$ = LiveData.from<PlaybackState | null | undefined>(
+    this.globalState.watch(GLOBAL_MEDIA_PLAYBACK_STATE_KEY),
+    this.globalState.get(GLOBAL_MEDIA_PLAYBACK_STATE_KEY)
+  );
+  readonly stats$ = LiveData.from<MediaStats | null | undefined>(
+    this.globalState.watch(GLOBAL_MEDIA_STATS_KEY),
+    this.globalState.get(GLOBAL_MEDIA_STATS_KEY)
+  );
+
+  override updatePlaybackState(state: Partial<PlaybackState> | null): void {
+    if (state === null) {
+      this.globalState.set(GLOBAL_MEDIA_PLAYBACK_STATE_KEY, null);
+      return;
+    }
+
+    const currentState = this.playbackState$.value;
+    const newState = currentState
+      ? { ...currentState, ...state }
+      : (state as PlaybackState);
+
+    this.globalState.set(GLOBAL_MEDIA_PLAYBACK_STATE_KEY, newState);
+  }
+
+  override updateStats(stats: Partial<MediaStats> | null): void {
+    if (stats === null) {
+      this.globalState.set(GLOBAL_MEDIA_STATS_KEY, null);
+      return;
+    }
+
+    const currentStats = this.stats$.value;
+    const newStats = currentStats
+      ? { ...currentStats, ...stats }
+      : (stats as MediaStats);
+
+    this.globalState.set(GLOBAL_MEDIA_STATS_KEY, newStats);
+  }
+}
+
+/**
+ * Provider for global media state in Web environment
+ * This is a simplified version that only works within the current tab
+ */
+export class WebGlobalMediaStateProvider extends BaseGlobalMediaStateProvider {
+  readonly playbackState$ = new LiveData<PlaybackState | null | undefined>(
+    null
+  );
+  readonly stats$ = new LiveData<MediaStats | null | undefined>(null);
+
+  /**
+   * Update the playback state
+   */
+  override updatePlaybackState(state: Partial<PlaybackState> | null): void {
+    if (state === null) {
+      this.playbackState$.setValue(null);
+      return;
+    }
+
+    const currentState = this.playbackState$.value;
+    const newState = currentState
+      ? { ...currentState, ...state }
+      : (state as PlaybackState);
+
+    this.playbackState$.setValue(newState);
+  }
+
+  /**
+   * Update the media stats
+   */
+  override updateStats(stats: Partial<MediaStats> | null): void {
+    if (stats === null) {
+      this.stats$.setValue(null);
+      return;
+    }
+
+    const currentStats = this.stats$.value;
+    const newStats = currentStats
+      ? { ...currentStats, ...stats }
+      : (stats as MediaStats);
+
+    this.stats$.setValue(newStats);
+  }
+}
@@ -0,0 +1,41 @@
+import type { AttachmentBlockModel } from '@blocksuite/affine/model';
+import {
+  attachmentBlockAudioMediaKey,
+  type AudioMediaKey,
+  ObjectPool,
+  Service,
+} from '@toeverything/infra';
+
+import { AudioAttachmentBlock } from '../entities/audio-attachment-block';
+
+export class AudioAttachmentService extends Service {
+  private readonly pool = new ObjectPool<AudioMediaKey, AudioAttachmentBlock>({
+    onDelete: block => {
+      block.dispose();
+    },
+    onDangling: block => {
+      return !block.rendering$.value;
+    },
+  });
+
+  get(model: AttachmentBlockModel | AudioMediaKey) {
+    if (typeof model === 'string') {
+      return this.pool.get(model);
+    }
+    if (!model.props.sourceId) {
+      throw new Error('Source ID is required');
+    }
+    const key = attachmentBlockAudioMediaKey({
+      blobId: model.props.sourceId,
+      blockId: model.id,
+      docId: model.doc.id,
+      workspaceId: model.doc.rootDoc.guid,
+    });
+    let exists = this.pool.get(key);
+    if (!exists) {
+      const entity = this.framework.createEntity(AudioAttachmentBlock, model);
+      exists = this.pool.put(key, entity);
+    }
+    return exists;
+  }
+}
@@ -0,0 +1,357 @@
+import { generateUrl } from '@affine/core/components/hooks/affine/use-share-url';
+import { AttachmentBlockModel } from '@blocksuite/affine/model';
+import {
+  attachmentBlockAudioMediaKey,
+  type AudioMediaDescriptor,
+  type AudioMediaKey,
+  type MediaStats,
+  ObjectPool,
+  parseAudioMediaKey,
+  type PlaybackState,
+  Service,
+} from '@toeverything/infra';
+import { clamp } from 'lodash-es';
+import { distinctUntilChanged } from 'rxjs';
+
+import type { DesktopApiService } from '../../desktop-api';
+import type { WorkbenchService } from '../../workbench';
+import { AudioMedia } from '../entities/audio-media';
+import type { BaseGlobalMediaStateProvider } from '../providers/global-audio-state';
+
+// Media service is to control how media should be played for attachment block
+// At a time, only one media can be played.
+export class AudioMediaManagerService extends Service {
+  private readonly mediaPool = new ObjectPool<AudioMediaKey, AudioMedia>({
+    onDelete: media => {
+      media.dispose();
+      const disposables = this.mediaDisposables.get(media);
+      if (disposables) {
+        disposables.forEach(dispose => dispose());
+        this.mediaDisposables.delete(media);
+      }
+    },
+    onDangling: media => {
+      return media.playbackState$.getValue().state !== 'playing';
+    },
+  });
+
+  private readonly mediaDisposables = new WeakMap<AudioMedia, (() => void)[]>();
+
+  constructor(
+    private readonly globalMediaState: BaseGlobalMediaStateProvider,
+    private readonly workbench: WorkbenchService,
+    private readonly desktopApi?: DesktopApiService
+  ) {
+    super();
+
+    if (!BUILD_CONFIG.isElectron) {
+      this.desktopApi = undefined;
+    }
+
+    this.disposables.push(() => {
+      this.mediaPool.clear();
+    });
+
+    // Subscribe to global playback state changes to manage playback across tabs
+    this.disposables.push(
+      this.observeGlobalPlaybackState(state => {
+        if (!state) {
+          // If global state is cleared, stop all media
+          this.stopAllMedia();
+          return;
+        }
+
+        const activeStats = this.getGlobalMediaStats();
+
+        if (!activeStats) return;
+
+        if (
+          BUILD_CONFIG.isElectron &&
+          activeStats.tabId !== this.desktopApi?.appInfo.viewId
+        ) {
+          // other tab is playing, pause the current media
+          if (state.state === 'playing') {
+            this.pauseAllMedia();
+          }
+          return;
+        }
+
+        const mediaRef = this.ensureMediaEntity(activeStats);
+        const media = mediaRef.media;
+
+        this.ensureExclusivePlayback();
+        media.syncState(state);
+
+        // Return cleanup function
+        return () => {
+          mediaRef.release();
+        };
+      })
+    );
+
+    window.addEventListener('beforeunload', () => {
+      this.stopAllMedia();
+    });
+  }
+
+  // Helper method to observe global playback state changes
+  private observeGlobalPlaybackState(
+    callback: (state: PlaybackState | undefined) => (() => void) | undefined
+  ): () => void {
+    const unsubscribe = this.globalMediaState.playbackState$
+      .pipe(distinctUntilChanged((a, b) => a?.updateTime === b?.updateTime))
+      .subscribe(state => {
+        if (state) {
+          return callback(state);
+        }
+        return;
+      });
+    return () => {
+      unsubscribe.unsubscribe();
+    };
+  }
+
+  get playbackState$() {
+    return this.globalMediaState.playbackState$;
+  }
+
+  get playbackStats$() {
+    return this.globalMediaState.stats$;
+  }
+
+  ensureMediaEntity(input: AttachmentBlockModel | MediaStats) {
+    const descriptor = this.normalizeEntityDescriptor(input);
+
+    let rc = this.mediaPool.get(descriptor.key);
+    if (!rc) {
+      rc = this.mediaPool.put(
+        descriptor.key,
+        this.framework.createEntity(AudioMedia, {
+          blobId: descriptor.blobId,
+          metadata: new MediaMetadata({
+            title: descriptor.name,
+            artist: 'AFFiNE',
+            // todo: add artwork, like the app icon?
+          }),
+        })
+      );
+
+      const audioMedia = rc.obj;
+
+      // Set up playback state synchronization (broadcast to global state)
+      const playbackStateSubscription = audioMedia.playbackState$
+        .pipe(distinctUntilChanged((a, b) => a.updateTime === b.updateTime))
+        .subscribe(state => {
+          if (state.state === 'playing') {
+            this.globalMediaState.updateStats({
+              ...audioMedia.getStats(),
+              tabId: descriptor.tabId,
+              key: descriptor.key,
+              name: descriptor.name,
+              size: descriptor.size,
+            });
+            this.globalMediaState.updatePlaybackState({
+              tabId: descriptor.tabId,
+              key: descriptor.key,
+              ...audioMedia.getPlaybackStateData(),
+            });
+          } else if (
+            (state.state === 'paused' || state.state === 'stopped') &&
+            this.globalMediaState.stats$.value?.key === descriptor.key
+          ) {
+            // If this is the active media and it's paused/stopped, update global state
+            this.globalMediaState.updatePlaybackState({
+              tabId: descriptor.tabId,
+              key: descriptor.key,
+              ...audioMedia.getPlaybackStateData(),
+            });
+            if (state.state === 'stopped') {
+              this.globalMediaState.updateStats(null);
+              this.globalMediaState.updatePlaybackState(null);
+            }
+          }
+        });
+
+      this.mediaDisposables.set(audioMedia, [
+        () => playbackStateSubscription.unsubscribe(),
+        () => {
+          // if the audioMedia is the active media, remove it
+          if (this.getActiveMediaKey() === descriptor.key) {
+            this.globalMediaState.updatePlaybackState(null);
+            this.globalMediaState.updateStats(null);
+          }
+        },
+      ]);
+    }
+
+    return { media: rc.obj, release: rc.release };
+  }
+
+  play() {
+    const stats = this.getGlobalMediaStats();
+    const currentState = this.getGlobalPlaybackState();
+    if (!stats || !currentState) {
+      return;
+    }
+    const seekOffset =
+      currentState.seekOffset + (Date.now() - currentState.updateTime) / 1000;
+    this.globalMediaState.updatePlaybackState({
+      state: 'playing',
+      // rewind to the beginning if the seek offset is greater than the duration
+      seekOffset: seekOffset >= stats.duration ? 0 : seekOffset,
+      updateTime: Date.now(),
+    });
+  }
+
+  pause() {
+    const state = this.getGlobalPlaybackState();
+
+    if (!state) {
+      return;
+    }
+
+    this.globalMediaState.updatePlaybackState({
+      state: 'paused',
+      seekOffset: (Date.now() - state.updateTime) / 1000 + state.seekOffset,
+      updateTime: Date.now(),
+    });
+  }
+
+  stop() {
+    this.globalMediaState.updatePlaybackState({
+      state: 'stopped',
+      seekOffset: 0,
+      updateTime: Date.now(),
+    });
+  }
+
+  seekTo(time: number) {
+    const stats = this.getGlobalMediaStats();
+    if (!stats) {
+      return;
+    }
+    this.globalMediaState.updatePlaybackState({
+      seekOffset: clamp(0, time, stats.duration),
+      updateTime: Date.now(),
+    });
+  }
+
+  focusAudioMedia(key: AudioMediaKey, tabId: string | null) {
+    const mediaProps = parseAudioMediaKey(key);
+    if (tabId === this.currentTabId) {
+      this.workbench.workbench.openDoc({
+        docId: mediaProps.docId,
+        mode: 'page',
+        blockIds: [mediaProps.blockId],
+      });
+    } else if (BUILD_CONFIG.isElectron && tabId) {
+      const url = generateUrl({
+        baseUrl: window.location.origin,
+        workspaceId: mediaProps.workspaceId,
+        pageId: mediaProps.docId,
+        blockIds: [mediaProps.blockId],
+      });
+
+      this.desktopApi?.showTab(tabId, url).catch(console.error);
+    }
+  }
+
+  private getActiveMediaKey(): AudioMediaKey | null {
+    const stats = this.getGlobalMediaStats();
+    return stats?.key || null;
+  }
+
+  private getGlobalPlaybackState(): PlaybackState | null {
+    const provider = this.globalMediaState;
+    return provider.playbackState$.value || null;
+  }
+
+  private getGlobalMediaStats(): MediaStats | null {
+    const provider = this.globalMediaState;
+    return provider.stats$.value || null;
+  }
+
+  // Ensure only one media is playing at a time
+  private ensureExclusivePlayback() {
+    const activeKey = this.getActiveMediaKey();
+    if (activeKey) {
+      this.pauseAllMedia(activeKey);
+    }
+  }
+
+  get currentTabId() {
+    return this.desktopApi?.appInfo.viewId || 'web';
+  }
+
+  private normalizeEntityDescriptor(
+    input: AttachmentBlockModel | MediaStats
+  ): AudioMediaDescriptor {
+    if (input instanceof AttachmentBlockModel) {
+      if (!input.props.sourceId) {
+        throw new Error('Invalid media');
+      }
+      return {
+        key: attachmentBlockAudioMediaKey({
+          blobId: input.props.sourceId,
+          blockId: input.id,
+          docId: input.doc.id,
+          workspaceId: input.doc.rootDoc.guid,
+        }),
+        name: input.props.name,
+        size: input.props.size,
+        blobId: input.props.sourceId,
+        // when input is AttachmentBlockModel, it is always in the current tab
+        tabId: this.currentTabId,
+      };
+    } else {
+      const { blobId } = parseAudioMediaKey(input.key);
+      return {
+        key: input.key,
+        name: input.name,
+        size: input.size,
+        blobId,
+        tabId: input.tabId,
+      };
+    }
+  }
+
+  /**
+   * Pause all playing media except the one with the given ID
+   * IN THE CURRENT TAB
+   */
+  private pauseAllMedia(exceptId?: AudioMediaKey) {
+    // Iterate through all objects in the pool
+    for (const [id, ref] of this.mediaPool.objects) {
+      if (
+        id !== exceptId &&
+        ref.obj.playbackState$.getValue().state === 'playing'
+      ) {
+        ref.obj.pause();
+      }
+    }
+  }
+
+  private stopAllMedia(exceptId?: AudioMediaKey) {
+    // Iterate through all objects in the pool
+    for (const [id, ref] of this.mediaPool.objects) {
+      if (
+        id !== exceptId &&
+        ref.obj.playbackState$.getValue().state === 'playing'
+      ) {
+        ref.obj.stop();
+      }
+    }
+
+    // The media entity may not being created yet
+    // so we need to change the state
+    const globalState = this.getGlobalPlaybackState();
+    if (
+      globalState &&
+      globalState.key !== exceptId &&
+      globalState.tabId === this.currentTabId
+    ) {
+      this.globalMediaState.updatePlaybackState(null);
+      this.globalMediaState.updateStats(null);
+    }
+  }
+}
@@ -0,0 +1,70 @@
+import type { AttachmentBlockModel } from '@blocksuite/affine/model';
+
+export function getAttachmentType(model: AttachmentBlockModel) {
+  // Check MIME type first
+  if (model.props.type.startsWith('image/')) {
+    return 'image';
+  }
+
+  if (model.props.type.startsWith('audio/')) {
+    return 'audio';
+  }
+
+  if (model.props.type.startsWith('video/')) {
+    return 'video';
+  }
+
+  if (model.props.type === 'application/pdf') {
+    return 'pdf';
+  }
+
+  // If MIME type doesn't match, check file extension
+  const ext = model.props.name.split('.').pop()?.toLowerCase() || '';
+
+  if (
+    [
+      'jpg',
+      'jpeg',
+      'png',
+      'gif',
+      'webp',
+      'svg',
+      'avif',
+      'tiff',
+      'bmp',
+    ].includes(ext)
+  ) {
+    return 'image';
+  }
+
+  if (['mp3', 'wav', 'ogg', 'flac', 'm4a', 'aac', 'opus'].includes(ext)) {
+    return 'audio';
+  }
+
+  if (
+    ['mp4', 'webm', 'avi', 'mov', 'mkv', 'mpeg', 'ogv', '3gp'].includes(ext)
+  ) {
+    return 'video';
+  }
+
+  if (ext === 'pdf') {
+    return 'pdf';
+  }
+
+  return 'unknown';
+}
+
+export async function downloadBlobToBuffer(model: AttachmentBlockModel) {
+  const sourceId = model.props.sourceId;
+  if (!sourceId) {
+    throw new Error('Attachment not found');
+  }
+
+  const blob = await model.doc.blobSync.get(sourceId);
+  if (!blob) {
+    throw new Error('Attachment not found');
+  }
+
+  const arrayBuffer = await blob.arrayBuffer();
+  return arrayBuffer;
+}
@@ -0,0 +1,31 @@
+import type { AttachmentBlockModel } from '@blocksuite/affine/model';
+import { useService } from '@toeverything/infra';
+import { useEffect, useState } from 'react';
+
+import type { AudioAttachmentBlock } from '../entities/audio-attachment-block';
+import { AudioAttachmentService } from '../services/audio-attachment';
+
+export const useAttachmentMediaBlock = (model: AttachmentBlockModel) => {
+  const audioAttachmentService = useService(AudioAttachmentService);
+  const [audioAttachmentBlock, setAttachmentMedia] = useState<
+    AudioAttachmentBlock | undefined
+  >(undefined);
+
+  useEffect(() => {
+    if (!model.props.sourceId) {
+      return;
+    }
+    const entity = audioAttachmentService.get(model);
+    if (!entity) {
+      return;
+    }
+    const audioAttachmentBlock = entity.obj;
+    setAttachmentMedia(audioAttachmentBlock);
+    audioAttachmentBlock.mount();
+    return () => {
+      audioAttachmentBlock.unmount();
+      entity.release();
+    };
+  }, [audioAttachmentService, model]);
+  return audioAttachmentBlock;
+};
@@ -2,8 +2,9 @@ import type { AttachmentBlockModel } from '@blocksuite/affine/model';
 import { Entity, LiveData, ObjectPool } from '@toeverything/infra';
 import { catchError, from, map, of, startWith, switchMap } from 'rxjs';

+import { downloadBlobToBuffer } from '../../media/utils';
 import type { PDFMeta } from '../renderer';
-import { downloadBlobToBuffer, PDFRenderer } from '../renderer';
+import { PDFRenderer } from '../renderer';
 import { PDFPage } from './pdf-page';

 export enum PDFStatus {
@@ -1,3 +1,2 @@
 export { PDFRenderer } from './renderer';
 export type { PDFMeta, RenderedPage, RenderPageOpts } from './types';
-export { downloadBlobToBuffer } from './utils';
@@ -1,15 +0,0 @@
-import type { AttachmentBlockModel } from '@blocksuite/affine/model';
-
-export async function downloadBlobToBuffer(model: AttachmentBlockModel) {
-  const sourceId = model.props.sourceId;
-  if (!sourceId) {
-    throw new Error('Attachment not found');
-  }
-
-  const blob = await model.doc.blobSync.get(sourceId);
-  if (!blob) {
-    throw new Error('Attachment not found');
-  }
-
-  return await blob.arrayBuffer();
-}
@@ -1,7 +1,7 @@
+import { AttachmentViewer } from '@affine/core/blocksuite/attachment-viewer';
 import type { AttachmentBlockModel } from '@blocksuite/affine/model';
 import { useMemo } from 'react';

-import { AttachmentViewer } from '../../../../components/attachment-viewer';
 import { useEditor } from '../utils';

 export type AttachmentPreviewModalProps = {
@@ -111,7 +111,6 @@ export const PeekViewManagerModal = () => {
  }, []);

  const onAnimationEnd = useCallback(() => {
-    console.log('onAnimationEnd');
    setAnimating(false);
  }, []);

@@ -84,6 +84,12 @@ export class DesktopStateSynchronizer extends Service {
      }
    });

+    this.electronApi.events.ui.onTabGoToRequest(opts => {
+      if (opts.tabId === appInfo?.viewId) {
+        this.workbenchService.workbench.open(opts.to);
+      }
+    });
+
    // sync workbench state with main process
    // also fill tab view meta with title & moduleName
    LiveData.computed(get => {