diff --git a/packages/frontend/component/src/ui/audio-player/audio-player.tsx b/packages/frontend/component/src/ui/audio-player/audio-player.tsx
index 2d5f142bde..ea8b53dfab 100644
--- a/packages/frontend/component/src/ui/audio-player/audio-player.tsx
+++ b/packages/frontend/component/src/ui/audio-player/audio-player.tsx
@@ -111,7 +111,7 @@ export const AudioPlayer = ({
           waveform={waveform || []}
           progress={progressPercentage}
           onManualSeek={handleProgressClick}
-          loading={loading}
+          loading={!waveform || waveform.length === 0}
         />
         <div className={styles.timeDisplay}>{formatTime(duration)}</div>
       </div>
diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx
index a5e409fe29..5f31a4966c 100644
--- a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx
+++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx
@@ -27,7 +27,6 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {
   const audioMedia = block.audioMedia;
   const playbackState = useLiveData(audioMedia.playbackState$);
   const stats = useLiveData(audioMedia.stats$);
-  const loading = useLiveData(audioMedia.loading$);
   const expanded = useLiveData(block.expanded$);
   const [preflightChecking, setPreflightChecking] = useState(false);
   const transcribing =
@@ -184,7 +183,7 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {
     <AudioPlayer
       name={block.props.props.name}
       size={sizeEntry}
-      loading={loading}
+      loading={stats.duration === 0}
       playbackState={playbackState?.state || 'idle'}
       waveform={stats.waveform}
       seekTime={seekTime}
diff --git a/packages/frontend/core/src/modules/media/entities/audio-media.ts b/packages/frontend/core/src/modules/media/entities/audio-media.ts
index cff3570290..0e4b47c092 100644
--- a/packages/frontend/core/src/modules/media/entities/audio-media.ts
+++ b/packages/frontend/core/src/modules/media/entities/audio-media.ts
@@ -156,20 +156,7 @@ export class AudioMedia extends Entity<AudioSource> {
 
   private async loadAudioBuffer() {
     const uint8Array = await this.getBuffer();
-
-    // Create a blob from the uint8Array
-    const blob = new Blob([uint8Array]);
-
-    const startTime = performance.now();
-    // calculating audio stats is expensive. Maybe persist the result in cache?
-    const stats = await this.calculateStatsFromBuffer(blob);
-    logger.debug(
-      `Calculate audio stats time: ${performance.now() - startTime}ms`
-    );
-    return {
-      blob,
-      ...stats,
-    };
+    return new Blob([uint8Array]);
   }
 
   readonly revalidateBuffer = effect(
@@ -177,11 +164,10 @@ export class AudioMedia extends Entity<AudioSource> {
       return fromPromise(async () => {
         return this.loadAudioBuffer();
       }).pipe(
-        mergeMap(({ blob, waveform }) => {
+        mergeMap(async blob => {
           const url = URL.createObjectURL(blob);
           // Set the audio element source
           this.audioElement.src = url;
-          this.waveform$.setValue(waveform);
           // If the media is playing, resume the playback
           if (this.playbackState$.getValue().state === 'playing') {
             this.play(true);
@@ -189,8 +175,15 @@ export class AudioMedia extends Entity<AudioSource> {
           this.audioElement.onloadedmetadata = () => {
             this.duration$.setValue(this.audioElement.duration);
           };
-          return EMPTY;
+          const startTime = performance.now();
+          // calculating audio stats is expensive. Maybe persist the result in cache?
+          const stats = await this.calculateStatsFromBuffer(blob);
+          this.waveform$.setValue(stats.waveform);
+          logger.debug(
+            `Calculate audio stats time: ${performance.now() - startTime}ms`
+          );
         }),
+        mergeMap(() => EMPTY),
         onStart(() => this.loading$.setValue(true)),
         onComplete(() => {
           this.loading$.setValue(false);