feat(editor): audio block (#10947)

AudioMedia entity for loading & controlling a single audio media
AudioMediaManagerService: Global audio state synchronization across tabs
AudioAttachmentService + AudioAttachmentBlock for manipulating AttachmentBlock in affine - e.g., filling transcription (using mock endpoint for now)
Added AudioBlock + AudioPlayer for rendering audio block in affine (new transcription block whose renderer is provided in affine)

fix AF-2292
fix AF-2337
This commit is contained in:
pengx17
2025-03-20 12:46:14 +00:00
parent 8a5393ea50
commit fad49bb070
120 changed files with 5407 additions and 950 deletions
@@ -0,0 +1,49 @@
import {
BlockComponent,
BlockViewExtension,
} from '@blocksuite/affine/block-std';
import type { TranscriptionBlockModel } from '@blocksuite/affine/model';
import type { ExtensionType } from '@blocksuite/affine/store';
import { css, type PropertyValues } from 'lit';
import { property } from 'lit/decorators.js';
import { literal } from 'lit/static-html.js';
export class LitTranscriptionBlock extends BlockComponent<TranscriptionBlockModel> {
static override styles = [
css`
transcription-block {
outline: none;
}
`,
];
override render() {
return this.std.host.renderChildren(this.model);
}
@property({ type: String, attribute: 'data-block-id' })
override accessor blockId!: string;
constructor() {
super();
// questionable:
this.widgets = {};
// to allow text selection across paragraphs in the callout block
this.contentEditable = 'true';
}
override firstUpdated(changedProperties: PropertyValues) {
super.firstUpdated(changedProperties);
this.disposables.addFromEvent(this, 'click', this.onClick);
}
protected onClick(event: MouseEvent) {
event.stopPropagation();
}
}
export const AITranscriptionBlockSpec: ExtensionType[] = [
BlockViewExtension('affine:transcription', () => {
return literal`transcription-block`;
}),
];
@@ -2,6 +2,7 @@ import { SpecProvider } from '@blocksuite/affine/shared/utils';
import { AIChatBlockComponent } from './blocks/ai-chat-block/ai-chat-block';
import { EdgelessAIChatBlockComponent } from './blocks/ai-chat-block/ai-chat-edgeless-block';
import { LitTranscriptionBlock } from './blocks/ai-chat-block/ai-transcription-block';
import {
AIChatMessage,
AIChatMessages,
@@ -151,5 +152,7 @@ export function registerAIEffects() {
EdgelessCopilotToolbarEntry
);
customElements.define('transcription-block', LitTranscriptionBlock);
SpecProvider._.extendSpec('store', [AIChatBlockSchemaExtension]);
}
@@ -9,6 +9,7 @@ import type { SpecBuilder } from '@blocksuite/affine/shared/utils';
import type { FrameworkProvider } from '@toeverything/infra';
import { AIChatBlockSpec } from '../blocks';
import { AITranscriptionBlockSpec } from '../blocks/ai-chat-block/ai-transcription-block';
import { AICodeBlockSpec } from './ai-code';
import { createAIEdgelessRootBlockSpec } from './ai-edgeless-root';
import { AIImageBlockSpec } from './ai-image';
@@ -39,4 +40,5 @@ export function enableAIExtension(
}
specBuilder.extend(AIChatBlockSpec);
specBuilder.extend(AITranscriptionBlockSpec);
}
@@ -0,0 +1,25 @@
import { useMemo } from 'react';
import { AudioBlockEmbedded } from './audio/audio-block';
import { AttachmentPreviewErrorBoundary } from './error';
import { PDFViewerEmbedded } from './pdf/pdf-viewer-embedded';
import type { AttachmentViewerProps } from './types';
import { getAttachmentType } from './utils';
// In Embed view
export const AttachmentEmbedPreview = ({ model }: AttachmentViewerProps) => {
const attachmentType = getAttachmentType(model);
const element = useMemo(() => {
switch (attachmentType) {
case 'pdf':
return <PDFViewerEmbedded model={model} />;
case 'audio':
return <AudioBlockEmbedded model={model} />;
default:
return null;
}
}, [attachmentType, model]);
return (
<AttachmentPreviewErrorBoundary>{element}</AttachmentPreviewErrorBoundary>
);
};
@@ -0,0 +1,25 @@
import { cssVar } from '@toeverything/theme';
import { cssVarV2 } from '@toeverything/theme/v2';
import { style } from '@vanilla-extract/css';
export const root = style({
display: 'flex',
flexDirection: 'column',
gap: 12,
width: '100%',
height: 'auto',
});
export const notesButton = style({
padding: '4px 8px',
color: cssVarV2('icon/primary'),
fontSize: cssVar('fontXs'),
fontWeight: 500,
userSelect: 'none',
});
export const notesButtonIcon = style({
fontSize: 24,
width: '1em',
height: '1em',
});
@@ -0,0 +1,116 @@
import { Button } from '@affine/component';
import { AudioPlayer } from '@affine/core/components/audio-player';
import { useSeekTime } from '@affine/core/components/audio-player/use-seek-time';
import { useEnableAI } from '@affine/core/components/hooks/affine/use-enable-ai';
import type { AudioAttachmentBlock } from '@affine/core/modules/media/entities/audio-attachment-block';
import { useAttachmentMediaBlock } from '@affine/core/modules/media/views/use-attachment-media';
import { useI18n } from '@affine/i18n';
import { TranscriptWithAiIcon } from '@blocksuite/icons/rc';
import { useLiveData } from '@toeverything/infra';
import { useCallback, useMemo } from 'react';
import type { AttachmentViewerProps } from '../types';
import * as styles from './audio-block.css';
import { TranscriptionBlock } from './transcription-block';
const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {
const audioMedia = block.audioMedia;
const playbackState = useLiveData(audioMedia.playbackState$);
const stats = useLiveData(audioMedia.stats$);
const loading = useLiveData(audioMedia.loading$);
const expanded = useLiveData(block.expanded$);
const transcribing = useLiveData(block.transcribing$);
const transcribed = useLiveData(block.transcribed$);
const handleClick = useCallback((e: React.MouseEvent<HTMLDivElement>) => {
e.stopPropagation();
}, []);
const seekTime = useSeekTime(playbackState, stats.duration);
const handlePlay = useCallback(() => {
audioMedia?.play();
}, [audioMedia]);
const handlePause = useCallback(() => {
audioMedia?.pause();
}, [audioMedia]);
const handleStop = useCallback(() => {
audioMedia?.stop();
}, [audioMedia]);
const handleSeek = useCallback(
(time: number) => {
audioMedia?.seekTo(time);
},
[audioMedia]
);
const t = useI18n();
const enableAi = useEnableAI();
const notesEntry = useMemo(() => {
if (!enableAi) {
return null;
}
return (
<Button
variant="plain"
prefix={<TranscriptWithAiIcon />}
loading={transcribing}
disabled={transcribing}
size="large"
prefixClassName={styles.notesButtonIcon}
className={styles.notesButton}
onClick={() => {
if (transcribed) {
block.expanded$.setValue(!expanded);
} else {
block.transcribe();
}
}}
>
{transcribing
? t['com.affine.attachmentViewer.audio.transcribing']()
: t['com.affine.attachmentViewer.audio.notes']()}
</Button>
);
}, [enableAi, transcribing, t, transcribed, block, expanded]);
return (
<AudioPlayer
name={block.props.props.name}
size={block.props.props.size}
loading={loading}
playbackState={playbackState?.state || 'idle'}
waveform={stats.waveform}
seekTime={seekTime}
duration={stats.duration}
onClick={handleClick}
onPlay={handlePlay}
onPause={handlePause}
onStop={handleStop}
onSeek={handleSeek}
notesEntry={notesEntry}
/>
);
};
export const AudioBlockEmbedded = (props: AttachmentViewerProps) => {
const audioAttachmentBlock = useAttachmentMediaBlock(props.model);
const transcriptionBlock = useLiveData(
audioAttachmentBlock?.transcriptionBlock$
);
const expanded = useLiveData(audioAttachmentBlock?.expanded$);
return (
<div className={styles.root}>
{audioAttachmentBlock && (
<AttachmentAudioPlayer block={audioAttachmentBlock} />
)}
{transcriptionBlock && expanded && (
<TranscriptionBlock block={transcriptionBlock} />
)}
</div>
);
};
@@ -0,0 +1,5 @@
import { style } from '@vanilla-extract/css';
export const root = style({
borderRadius: 6,
});
@@ -0,0 +1,40 @@
import { createReactComponentFromLit } from '@affine/component';
import { LitTranscriptionBlock } from '@affine/core/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block';
import type { TranscriptionBlockModel } from '@blocksuite/affine/model';
import { LiveData, useLiveData } from '@toeverything/infra';
import React, { useCallback, useMemo } from 'react';
import * as styles from './transcription-block.css';
const AdaptedTranscriptionBlock = createReactComponentFromLit({
react: React,
elementClass: LitTranscriptionBlock,
});
export const TranscriptionBlock = ({
block,
}: {
block: TranscriptionBlockModel;
}) => {
const childMap$ = useMemo(
() => LiveData.fromSignal(block.childMap),
[block.childMap]
);
const childMap = useLiveData(childMap$);
const onDragStart = useCallback((e: React.DragEvent<HTMLDivElement>) => {
e.preventDefault();
}, []);
if (childMap.size === 0) {
return null;
}
return (
<div
className={styles.root}
// draggable + onDragStart to blacklist blocksuite's default drag behavior for attachment block
draggable="true"
onDragStart={onDragStart}
>
<AdaptedTranscriptionBlock blockId={block.id} />
</div>
);
};
@@ -0,0 +1,33 @@
import { cssVarV2 } from '@toeverything/theme/v2';
import { style } from '@vanilla-extract/css';
export { viewer } from './viewer.css';
export const error = style({
flexDirection: 'column',
alignItems: 'center',
justifyContent: 'center',
gap: '4px',
});
export const errorTitle = style({
fontSize: '15px',
fontWeight: 500,
lineHeight: '24px',
color: cssVarV2('text/primary'),
marginTop: '12px',
});
export const errorMessage = style({
fontSize: '12px',
fontWeight: 500,
lineHeight: '20px',
color: cssVarV2('text/tertiary'),
});
export const errorBtns = style({
display: 'flex',
flexDirection: 'column',
gap: '10px',
marginTop: '28px',
});
@@ -0,0 +1,131 @@
import { Button } from '@affine/component';
import { useI18n } from '@affine/i18n';
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
import { ArrowDownBigIcon } from '@blocksuite/icons/rc';
import clsx from 'clsx';
import type { PropsWithChildren, ReactElement } from 'react';
import { Suspense } from 'react';
import { ErrorBoundary, type FallbackProps } from 'react-error-boundary';
import * as styles from './error.css';
import { download } from './utils';
// https://github.com/toeverything/blocksuite/blob/master/packages/affine/components/src/icons/file-icons.ts
// TODO: should move file icons to icons repo
const FileIcon = () => (
<svg
width="96"
height="96"
viewBox="0 0 20 20"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M6.80781 1.875L10.8332 1.875C10.9989 1.875 11.1579 1.94085 11.2751 2.05806L16.2751 7.05806C16.3923 7.17527 16.4582 7.33424 16.4582 7.5V14.8587C16.4582 15.3038 16.4582 15.6754 16.4334 15.9789C16.4075 16.2955 16.3516 16.5927 16.2084 16.8737C15.9887 17.3049 15.6381 17.6555 15.2069 17.8752C14.9258 18.0184 14.6286 18.0744 14.3121 18.1002C14.0085 18.125 13.637 18.125 13.1919 18.125H6.80779C6.36267 18.125 5.99114 18.125 5.68761 18.1002C5.37104 18.0744 5.07383 18.0184 4.79278 17.8752C4.36157 17.6555 4.01099 17.3049 3.79128 16.8737C3.64808 16.5927 3.59215 16.2955 3.56629 15.9789C3.54149 15.6754 3.5415 15.3038 3.5415 14.8587V5.1413C3.5415 4.69618 3.54149 4.32464 3.56629 4.02111C3.59215 3.70454 3.64808 3.40732 3.79128 3.12627C4.01099 2.69507 4.36157 2.34449 4.79278 2.12478C5.07383 1.98157 5.37104 1.92565 5.68761 1.89978C5.99114 1.87498 6.36268 1.87499 6.80781 1.875ZM5.7894 3.14563C5.55013 3.16518 5.43573 3.20008 5.36026 3.23854C5.16426 3.3384 5.00491 3.49776 4.90504 3.69376C4.86659 3.76923 4.83168 3.88363 4.81214 4.1229C4.79199 4.36946 4.7915 4.68964 4.7915 5.16667V14.8333C4.7915 15.3104 4.79199 15.6305 4.81214 15.8771C4.83168 16.1164 4.86659 16.2308 4.90504 16.3062C5.00491 16.5022 5.16426 16.6616 5.36026 16.7615C5.43573 16.7999 5.55013 16.8348 5.7894 16.8544C6.03597 16.8745 6.35615 16.875 6.83317 16.875H13.1665C13.6435 16.875 13.9637 16.8745 14.2103 16.8544C14.4495 16.8348 14.5639 16.7999 14.6394 16.7615C14.8354 16.6616 14.9948 16.5022 15.0946 16.3062C15.1331 16.2308 15.168 16.1164 15.1875 15.8771C15.2077 15.6305 15.2082 15.3104 15.2082 14.8333V8.125H11.6665C10.8611 8.125 10.2082 7.47208 10.2082 6.66667V3.125H6.83317C6.35615 3.125 6.03597 3.12549 5.7894 3.14563ZM11.4582 4.00888L14.3243 6.875H11.6665C11.5514 6.875 11.4582 6.78173 11.4582 6.66667V4.00888Z"
fill="#77757D"
/>
</svg>
);
const PDFFileIcon = () => (
<svg
width="96"
height="96"
viewBox="0 0 40 40"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M7.75 4C7.75 2.20508 9.20508 0.75 11 0.75H27C27.1212 0.75 27.2375 0.798159 27.3232 0.883885L38.1161 11.6768C38.2018 11.7625 38.25 11.8788 38.25 12V36C38.25 37.7949 36.7949 39.25 35 39.25H11C9.20507 39.25 7.75 37.7949 7.75 36V4Z"
fill="white"
stroke="#D0D5DD"
strokeWidth="1.5"
/>
<path
d="M27 0.5V8C27 10.2091 28.7909 12 31 12H38.5"
stroke="#D0D5DD"
strokeWidth="1.5"
/>
<rect x="1" y="18" width="26" height="16" rx="2" fill="#D92D20" />
<path
d="M4.8323 30V22.7273H7.70162C8.25323 22.7273 8.72316 22.8326 9.11142 23.0433C9.49967 23.2517 9.7956 23.5417 9.9992 23.9134C10.2052 24.2827 10.3082 24.7088 10.3082 25.1918C10.3082 25.6747 10.204 26.1009 9.99565 26.4702C9.78732 26.8395 9.48547 27.1271 9.09011 27.3331C8.69712 27.5391 8.22127 27.642 7.66255 27.642H5.83372V26.4098H7.41397C7.7099 26.4098 7.95375 26.3589 8.14551 26.2571C8.33964 26.1529 8.48405 26.0097 8.57875 25.8274C8.67581 25.6428 8.72434 25.4309 8.72434 25.1918C8.72434 24.9503 8.67581 24.7396 8.57875 24.5597C8.48405 24.3774 8.33964 24.2365 8.14551 24.1371C7.95138 24.0353 7.70517 23.9844 7.40687 23.9844H6.36994V30H4.8323ZM13.885 30H11.3069V22.7273H13.9063C14.6379 22.7273 15.2676 22.8729 15.7955 23.1641C16.3235 23.4529 16.7295 23.8684 17.0136 24.4105C17.3 24.9527 17.4433 25.6013 17.4433 26.3565C17.4433 27.1141 17.3 27.7652 17.0136 28.3097C16.7295 28.8542 16.3211 29.272 15.7884 29.5632C15.2581 29.8544 14.6237 30 13.885 30ZM12.8445 28.6825H13.8211C14.2757 28.6825 14.658 28.602 14.9681 28.4411C15.2806 28.2777 15.515 28.0256 15.6713 27.6847C15.8299 27.3414 15.9092 26.8987 15.9092 26.3565C15.9092 25.8191 15.8299 25.38 15.6713 25.0391C15.515 24.6982 15.2818 24.4472 14.9717 24.2862C14.6615 24.1252 14.2792 24.0447 13.8247 24.0447H12.8445V28.6825ZM18.5823 30V22.7273H23.3976V23.995H20.1199V25.728H23.078V26.9957H20.1199V30H18.5823Z"
fill="white"
/>
</svg>
);
const FILE_ICONS: Record<string, () => ReactElement> = {
'application/pdf': PDFFileIcon,
};
interface ErrorBaseProps {
title: string;
subtitle?: string;
icon?: ReactElement;
buttons?: ReactElement[];
}
const ErrorBase = ({
title,
subtitle,
icon = <FileIcon />,
buttons = [],
}: ErrorBaseProps) => {
return (
<div className={clsx([styles.viewer, styles.error])}>
{icon}
<h3 className={styles.errorTitle}>{title}</h3>
<p className={styles.errorMessage}>{subtitle}</p>
<div className={styles.errorBtns}>{buttons}</div>
</div>
);
};
interface ErrorProps {
model: AttachmentBlockModel;
ext: string;
}
export const AttachmentFallback = ({ model, ext }: ErrorProps) => {
const t = useI18n();
const Icon = FILE_ICONS[model.props.type] ?? FileIcon;
const title = t['com.affine.attachment.preview.error.title']();
const subtitle = `.${ext} ${t['com.affine.attachment.preview.error.subtitle']()}`;
return (
<ErrorBase
icon={<Icon />}
title={title}
subtitle={subtitle}
buttons={[
<Button
key="download"
variant="primary"
prefix={<ArrowDownBigIcon />}
onClick={() => {
download(model).catch(console.error);
}}
>
Download
</Button>,
]}
/>
);
};
const ErrorBoundaryInner = (props: FallbackProps): ReactElement => {
const t = useI18n();
const title = t['com.affine.attachment.preview.error.title']();
const subtitle = `${props.error}`;
return <ErrorBase title={title} subtitle={subtitle} />;
};
export const AttachmentPreviewErrorBoundary = (props: PropsWithChildren) => {
return (
<ErrorBoundary FallbackComponent={ErrorBoundaryInner}>
<Suspense>{props.children}</Suspense>
</ErrorBoundary>
);
};
@@ -0,0 +1,46 @@
import { ViewBody, ViewHeader } from '@affine/core/modules/workbench';
import { AttachmentFallback, AttachmentPreviewErrorBoundary } from './error';
import { PDFViewer } from './pdf/pdf-viewer';
import type { AttachmentViewerBaseProps, AttachmentViewerProps } from './types';
import { buildAttachmentProps } from './utils';
import { Titlebar } from './viewer';
import * as styles from './viewer.css';
// In Peek view
export const AttachmentViewer = ({ model }: AttachmentViewerProps) => {
const props = buildAttachmentProps(model);
return (
<div className={styles.viewerContainer}>
<Titlebar {...props} />
<AttachmentViewerInner {...props} />
</div>
);
};
// In View container
export const AttachmentViewerView = ({ model }: AttachmentViewerProps) => {
const props = buildAttachmentProps(model);
return (
<>
<ViewHeader>
<Titlebar {...props} />
</ViewHeader>
<ViewBody>
<AttachmentViewerInner {...props} />
</ViewBody>
</>
);
};
const AttachmentViewerInner = (props: AttachmentViewerBaseProps) => {
return props.model.props.type.endsWith('pdf') ? (
<AttachmentPreviewErrorBoundary>
<PDFViewer {...props} />
</AttachmentPreviewErrorBoundary>
) : (
<AttachmentFallback {...props} />
);
};
@@ -0,0 +1,261 @@
import { IconButton, observeIntersection } from '@affine/component';
import {
type PDF,
type PDFPage,
PDFService,
PDFStatus,
} from '@affine/core/modules/pdf';
import type { PDFMeta } from '@affine/core/modules/pdf/renderer';
import type { PageSize } from '@affine/core/modules/pdf/renderer/types';
import { LoadingSvg, PDFPageCanvas } from '@affine/core/modules/pdf/views';
import { PeekViewService } from '@affine/core/modules/peek-view/services/peek-view';
import { stopPropagation } from '@affine/core/utils';
import {
ArrowDownSmallIcon,
ArrowUpSmallIcon,
AttachmentIcon,
CenterPeekIcon,
} from '@blocksuite/icons/rc';
import { LiveData, useLiveData, useService } from '@toeverything/infra';
import clsx from 'clsx';
import { debounce } from 'lodash-es';
import {
type MouseEvent,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import type { AttachmentViewerProps } from '../types';
import * as styles from './styles.css';
function defaultMeta() {
return {
pageCount: 0,
pageSizes: [],
maxSize: { width: 0, height: 0 },
};
}
export function PDFViewerEmbedded({ model }: AttachmentViewerProps) {
const scale = window.devicePixelRatio;
const peekView = useService(PeekViewService).peekView;
const pdfService = useService(PDFService);
const [pdfEntity, setPdfEntity] = useState<{
pdf: PDF;
release: () => void;
} | null>(null);
const [pageEntity, setPageEntity] = useState<{
page: PDFPage;
release: () => void;
} | null>(null);
const [pageSize, setPageSize] = useState<PageSize | null>(null);
const meta = useLiveData(
useMemo(() => {
return pdfEntity
? pdfEntity.pdf.state$.map(s => {
return s.status === PDFStatus.Opened ? s.meta : defaultMeta();
})
: new LiveData<PDFMeta>(defaultMeta());
}, [pdfEntity])
);
const img = useLiveData(
useMemo(() => (pageEntity ? pageEntity.page.bitmap$ : null), [pageEntity])
);
const [name, setName] = useState(model.props.name);
const [cursor, setCursor] = useState(0);
const [isLoading, setIsLoading] = useState(true);
const [visibility, setVisibility] = useState(false);
const viewerRef = useRef<HTMLDivElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const peek = useCallback(() => {
const target = model.doc.getBlock(model.id);
if (!target) return;
peekView.open({ element: target }).catch(console.error);
}, [peekView, model]);
const navigator = useMemo(() => {
const p = cursor - 1;
const n = cursor + 1;
return {
prev: {
disabled: p < 0,
onClick: (e: MouseEvent) => {
e.stopPropagation();
setCursor(p);
},
},
next: {
disabled: n >= meta.pageCount,
onClick: (e: MouseEvent) => {
e.stopPropagation();
setCursor(n);
},
},
peek: {
onClick: (e: MouseEvent) => {
e.stopPropagation();
peek();
},
},
};
}, [cursor, meta, peek]);
useEffect(() => model.props.name$.subscribe(val => setName(val)), [model]);
useEffect(() => {
const canvas = canvasRef.current;
if (!canvas) return;
if (!img) return;
const ctx = canvas.getContext('2d');
if (!ctx) return;
setIsLoading(false);
canvas.width = img.width;
canvas.height = img.height;
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
}, [img]);
useEffect(() => {
if (!visibility) return;
if (!pageEntity) return;
if (!pageSize) return;
const { width, height } = pageSize;
pageEntity.page.render({ width, height, scale });
return () => {
pageEntity.page.render.unsubscribe();
};
}, [visibility, pageEntity, pageSize, scale]);
useEffect(() => {
if (!visibility) return;
if (!pdfEntity) return;
const size = meta.pageSizes[cursor];
if (!size) return;
const { width, height } = size;
const pageEntity = pdfEntity.pdf.page(
cursor,
`${width}:${height}:${scale}`
);
setPageEntity(pageEntity);
setPageSize(size);
return () => {
pageEntity.release();
setPageSize(null);
setPageEntity(null);
};
}, [visibility, pdfEntity, cursor, meta, scale]);
useEffect(() => {
if (!visibility) return;
const pdfEntity = pdfService.get(model);
setPdfEntity(pdfEntity);
return () => {
pdfEntity.release();
setPdfEntity(null);
};
}, [model, pdfService, visibility]);
useEffect(() => {
const viewer = viewerRef.current;
if (!viewer) return;
return observeIntersection(
viewer,
debounce(
entry => {
setVisibility(entry.isIntersecting);
},
377,
{
trailing: true,
}
)
);
}, []);
return (
<div ref={viewerRef} className={styles.pdfContainer}>
<main className={styles.pdfViewer}>
<div
className={styles.pdfPage}
style={{
position: 'relative',
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
width: '100%',
minHeight: '253px',
}}
>
<PDFPageCanvas ref={canvasRef} />
<LoadingSvg
style={{
position: 'absolute',
visibility: isLoading ? 'visible' : 'hidden',
}}
/>
</div>
<div className={styles.pdfControls}>
<IconButton
size={16}
icon={<ArrowUpSmallIcon />}
className={styles.pdfControlButton}
onDoubleClick={stopPropagation}
aria-label="Prev"
{...navigator.prev}
/>
<IconButton
size={16}
icon={<ArrowDownSmallIcon />}
className={styles.pdfControlButton}
onDoubleClick={stopPropagation}
aria-label="Next"
{...navigator.next}
/>
<IconButton
size={16}
icon={<CenterPeekIcon />}
className={styles.pdfControlButton}
onDoubleClick={stopPropagation}
{...navigator.peek}
/>
</div>
</main>
<footer className={styles.pdfFooter}>
<div className={clsx([styles.pdfFooterItem, { truncate: true }])}>
<AttachmentIcon />
<span className={clsx([styles.pdfTitle, 'pdf-name'])}>{name}</span>
</div>
<div className={clsx([styles.pdfFooterItem, styles.pdfPageCount])}>
<span className="page-cursor">
{meta.pageCount > 0 ? cursor + 1 : '-'}
</span>
/
<span className="page-count">
{meta.pageCount > 0 ? meta.pageCount : '-'}
</span>
</div>
</footer>
</div>
);
}
@@ -0,0 +1,291 @@
import { IconButton, observeResize } from '@affine/component';
import type { PDF, PDFRendererState } from '@affine/core/modules/pdf';
import { PDFService, PDFStatus } from '@affine/core/modules/pdf';
import {
Item,
List,
ListPadding,
ListWithSmallGap,
LoadingSvg,
PDFPageRenderer,
type PDFVirtuosoContext,
type PDFVirtuosoProps,
Scroller,
ScrollSeekPlaceholder,
} from '@affine/core/modules/pdf/views';
import track from '@affine/track';
import { CollapseIcon, ExpandIcon } from '@blocksuite/icons/rc';
import { useLiveData, useService } from '@toeverything/infra';
import clsx from 'clsx';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import {
type ScrollSeekConfiguration,
Virtuoso,
type VirtuosoHandle,
} from 'react-virtuoso';
import type { AttachmentViewerProps } from '../types';
import * as styles from './styles.css';
import { fitToPage } from './utils';
const THUMBNAIL_WIDTH = 94;
function calculatePageNum(el: HTMLElement, pageCount: number) {
const { scrollTop, scrollHeight } = el;
const pageHeight = scrollHeight / pageCount;
const n = scrollTop / pageHeight;
const t = n / pageCount;
const index = Math.floor(n + t);
const cursor = Math.min(index, pageCount - 1);
return cursor;
}
export interface PDFViewerInnerProps {
pdf: PDF;
state: Extract<PDFRendererState, { status: PDFStatus.Opened }>;
}
export const PDFViewerInner = ({ pdf, state }: PDFViewerInnerProps) => {
const [cursor, setCursor] = useState(0);
const [collapsed, setCollapsed] = useState(true);
const [viewportInfo, setViewportInfo] = useState({ width: 0, height: 0 });
const viewerRef = useRef<HTMLDivElement>(null);
const pagesScrollerRef = useRef<HTMLElement | null>(null);
const pagesScrollerHandleRef = useRef<VirtuosoHandle>(null);
const thumbnailsScrollerHandleRef = useRef<VirtuosoHandle>(null);
const updateScrollerRef = useCallback(
(scroller: HTMLElement | Window | null) => {
pagesScrollerRef.current = scroller as HTMLElement;
},
[]
);
const onScroll = useCallback(() => {
const el = pagesScrollerRef.current;
if (!el) return;
const { pageCount } = state.meta;
if (!pageCount) return;
const cursor = calculatePageNum(el, pageCount);
setCursor(cursor);
}, [pagesScrollerRef, state]);
const onPageSelect = useCallback(
(index: number) => {
const scroller = pagesScrollerHandleRef.current;
if (!scroller) return;
scroller.scrollToIndex({
index,
align: 'center',
behavior: 'smooth',
});
},
[pagesScrollerHandleRef]
);
const pageContent = useCallback(
(
index: number,
_: unknown,
{
viewportInfo,
meta,
onPageSelect,
pageClassName,
resize,
isThumbnail,
}: PDFVirtuosoContext
) => {
return (
<PDFPageRenderer
key={`${pageClassName}-${index}`}
pdf={pdf}
pageNum={index}
className={pageClassName}
viewportInfo={viewportInfo}
actualSize={meta.pageSizes[index]}
maxSize={meta.maxSize}
onSelect={onPageSelect}
resize={resize}
isThumbnail={isThumbnail}
/>
);
},
[pdf]
);
const thumbnailsConfig = useMemo(() => {
const { height: vh } = viewportInfo;
const { pageCount, pageSizes, maxSize } = state.meta;
const t = Math.min(maxSize.width / maxSize.height, 1);
const pw = THUMBNAIL_WIDTH / t;
const newMaxSize = {
width: pw,
height: pw * (maxSize.height / maxSize.width),
};
const newPageSizes = pageSizes.map(({ width, height }) => {
const w = newMaxSize.width * (width / maxSize.width);
return {
width: w,
height: w * (height / width),
};
});
const height = Math.min(
vh - 60 - 24 - 24 - 2 - 8,
newPageSizes.reduce((h, { height }) => h + height * t, 0) +
(pageCount - 1) * 12
);
return {
context: {
onPageSelect,
viewportInfo: {
width: pw,
height,
},
meta: {
pageCount,
maxSize: newMaxSize,
pageSizes: newPageSizes,
},
resize: fitToPage,
isThumbnail: true,
pageClassName: styles.pdfThumbnail,
},
style: { height },
};
}, [state, viewportInfo, onPageSelect]);
// 1. works fine if they are the same size
// 2. uses the `observeIntersection` when targeting different sizes
const scrollSeekConfig = useMemo<ScrollSeekConfiguration>(() => {
return {
enter: velocity => Math.abs(velocity) > 1024,
exit: velocity => Math.abs(velocity) < 10,
};
}, []);
useEffect(() => {
const viewer = viewerRef.current;
if (!viewer) return;
return observeResize(viewer, ({ contentRect: { width, height } }) =>
setViewportInfo({ width, height })
);
}, []);
return (
<div
ref={viewerRef}
data-testid="pdf-viewer"
className={clsx([styles.viewer, { gridding: true, scrollable: true }])}
>
<Virtuoso<PDFVirtuosoProps>
key={pdf.id}
ref={pagesScrollerHandleRef}
scrollerRef={updateScrollerRef}
onScroll={onScroll}
className={styles.virtuoso}
totalCount={state.meta.pageCount}
itemContent={pageContent}
components={{
Item,
List,
Scroller,
Header: ListPadding,
Footer: ListPadding,
ScrollSeekPlaceholder,
}}
context={{
viewportInfo: {
width: viewportInfo.width - 40,
height: viewportInfo.height - 40,
},
meta: state.meta,
resize: fitToPage,
pageClassName: styles.pdfPage,
}}
scrollSeekConfiguration={scrollSeekConfig}
/>
<div className={clsx(['thumbnails', styles.pdfThumbnails])}>
<div className={clsx([styles.pdfThumbnailsList, { collapsed }])}>
<Virtuoso<PDFVirtuosoProps>
key={`${pdf.id}-thumbnail`}
ref={thumbnailsScrollerHandleRef}
className={styles.virtuoso}
totalCount={state.meta.pageCount}
itemContent={pageContent}
components={{
Item,
List: ListWithSmallGap,
Scroller,
ScrollSeekPlaceholder,
}}
style={thumbnailsConfig.style}
context={thumbnailsConfig.context}
scrollSeekConfiguration={scrollSeekConfig}
/>
</div>
<div className={clsx(['indicator', styles.pdfIndicator])}>
<div>
<span className="page-cursor">
{state.meta.pageCount > 0 ? cursor + 1 : 0}
</span>
/<span className="page-count">{state.meta.pageCount}</span>
</div>
<IconButton
icon={collapsed ? <CollapseIcon /> : <ExpandIcon />}
onClick={() => setCollapsed(!collapsed)}
/>
</div>
</div>
</div>
);
};
function PDFViewerStatus({
pdf,
...props
}: AttachmentViewerProps & { pdf: PDF }) {
const state = useLiveData(pdf.state$);
useEffect(() => {
if (state.status !== PDFStatus.Error) return;
track.$.attachment.$.openPDFRendererFail();
}, [state]);
if (state?.status !== PDFStatus.Opened) {
return <PDFLoading />;
}
return <PDFViewerInner {...props} pdf={pdf} state={state} />;
}
export function PDFViewer({ model, ...props }: AttachmentViewerProps) {
const pdfService = useService(PDFService);
const [pdf, setPdf] = useState<PDF | null>(null);
useEffect(() => {
const { pdf, release } = pdfService.get(model);
setPdf(pdf);
return () => {
release();
};
}, [model, pdfService, setPdf]);
if (!pdf) {
return <PDFLoading />;
}
return <PDFViewerStatus {...props} model={model} pdf={pdf} />;
}
const PDFLoading = () => (
<div style={{ margin: 'auto' }}>
<LoadingSvg />
</div>
);
@@ -0,0 +1,175 @@
import { cssVar } from '@toeverything/theme';
import { cssVarV2 } from '@toeverything/theme/v2';
import { style } from '@vanilla-extract/css';
export { viewer } from '../viewer.css';
export const virtuoso = style({
width: '100%',
});
export const pdfIndicator = style({
display: 'flex',
alignItems: 'center',
justifyContent: 'space-between',
padding: '0 12px',
});
export const pdfPage = style({
maxWidth: 'calc(100% - 40px)',
background: cssVarV2('layer/white'),
boxSizing: 'border-box',
borderWidth: '1px',
borderStyle: 'solid',
borderColor: cssVarV2('layer/insideBorder/border'),
boxShadow:
'0px 4px 20px 0px var(--transparent-black-200, rgba(0, 0, 0, 0.10))',
overflow: 'hidden',
maxHeight: 'max-content',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
});
export const pdfThumbnails = style({
display: 'flex',
flexDirection: 'column',
position: 'absolute',
boxSizing: 'border-box',
width: '120px',
padding: '12px 0',
right: '30px',
bottom: '30px',
maxHeight: 'calc(100% - 60px)',
borderRadius: '8px',
borderWidth: '1px',
borderStyle: 'solid',
borderColor: cssVarV2('layer/insideBorder/border'),
backgroundColor: cssVarV2('layer/background/primary'),
fontSize: '12px',
fontWeight: 500,
lineHeight: '20px',
color: cssVarV2('text/secondary'),
});
export const pdfThumbnailsList = style({
position: 'relative',
display: 'flex',
flexDirection: 'column',
maxHeight: '100%',
overflow: 'hidden',
selectors: {
'&.collapsed': {
display: 'none',
},
'&:not(.collapsed)': {
marginBottom: '8px',
},
},
});
export const pdfThumbnail = style({
display: 'flex',
overflow: 'hidden',
// width: '100%',
borderRadius: '4px',
borderWidth: '1px',
borderStyle: 'solid',
borderColor: cssVarV2('layer/insideBorder/border'),
selectors: {
'&.selected': {
borderColor: '#29A3FA',
},
},
});
export const pdfContainer = style({
width: '100%',
borderRadius: '8px',
borderWidth: '1px',
borderStyle: 'solid',
borderColor: cssVarV2('layer/insideBorder/border'),
background: cssVar('--affine-background-primary-color'),
userSelect: 'none',
contentVisibility: 'visible',
display: 'flex',
minHeight: 'fit-content',
height: '100%',
flexDirection: 'column',
justifyContent: 'space-between',
});
export const pdfViewer = style({
position: 'relative',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
padding: '12px',
overflow: 'hidden',
background: cssVarV2('layer/background/secondary'),
flex: 1,
});
export const pdfPlaceholder = style({
position: 'absolute',
maxWidth: 'calc(100% - 24px)',
overflow: 'hidden',
height: 'auto',
pointerEvents: 'none',
});
export const pdfControls = style({
position: 'absolute',
bottom: '16px',
right: '14px',
display: 'flex',
flexDirection: 'column',
gap: '10px',
});
export const pdfControlButton = style({
width: '36px',
height: '36px',
borderWidth: '1px',
borderStyle: 'solid',
borderColor: cssVar('--affine-border-color'),
background: cssVar('--affine-white'),
});
export const pdfFooter = style({
display: 'flex',
alignItems: 'center',
flexDirection: 'row',
justifyContent: 'space-between',
gap: '12px',
padding: '12px',
textWrap: 'nowrap',
});
export const pdfFooterItem = style({
display: 'flex',
alignItems: 'center',
selectors: {
'&.truncate': {
overflow: 'hidden',
},
},
});
export const pdfTitle = style({
marginLeft: '8px',
fontSize: '14px',
fontWeight: 600,
lineHeight: '22px',
color: cssVarV2('text/primary'),
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
});
export const pdfPageCount = style({
fontSize: '12px',
fontWeight: 400,
lineHeight: '20px',
color: cssVarV2('text/secondary'),
});
@@ -0,0 +1,27 @@
import type { PageSize } from '@affine/core/modules/pdf/renderer/types';
export function fitToPage(
viewportInfo: PageSize,
actualSize: PageSize,
maxSize: PageSize,
isThumbnail?: boolean
) {
const { width: vw, height: vh } = viewportInfo;
const { width: w, height: h } = actualSize;
const { width: mw, height: mh } = maxSize;
let width = 0;
let height = 0;
if (h / w > vh / vw) {
height = vh * (h / mh);
width = (w / h) * height;
} else {
const t = isThumbnail ? Math.min(w / h, 1) : w / mw;
width = vw * t;
height = (h / w) * width;
}
return {
width: Math.ceil(width),
height: Math.ceil(height),
aspectRatio: width / height,
};
}
@@ -0,0 +1,14 @@
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
export type AttachmentType = 'pdf' | 'image' | 'audio' | 'video' | 'unknown';
export type AttachmentViewerProps = {
model: AttachmentBlockModel;
};
export type AttachmentViewerBaseProps = {
model: AttachmentBlockModel;
name: string;
ext: string;
size: string;
};
@@ -0,0 +1,40 @@
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
import { filesize } from 'filesize';
import { downloadBlob } from '../../utils/resource';
import type { AttachmentViewerBaseProps } from './types';
export async function getAttachmentBlob(model: AttachmentBlockModel) {
const sourceId = model.props.sourceId;
if (!sourceId) {
return null;
}
const doc = model.doc;
let blob = await doc.blobSync.get(sourceId);
if (blob) {
blob = new Blob([blob], { type: model.props.type });
}
return blob;
}
export async function download(model: AttachmentBlockModel) {
const blob = await getAttachmentBlob(model);
if (!blob) return;
await downloadBlob(blob, model.props.name);
}
export function buildAttachmentProps(
model: AttachmentBlockModel
): AttachmentViewerBaseProps {
const pieces = model.props.name.split('.');
const ext = pieces.pop() || '';
const name = pieces.join('.');
const size = filesize(model.props.size);
return { model, name, ext, size };
}
export { getAttachmentType } from '@affine/core/modules/media/utils';
@@ -0,0 +1,78 @@
import { cssVarV2 } from '@toeverything/theme/v2';
import { style } from '@vanilla-extract/css';
export const viewer = style({
position: 'relative',
zIndex: 0,
display: 'flex',
flex: 1,
overflow: 'hidden',
resize: 'none',
selectors: {
'&:before': {
position: 'absolute',
content: '',
top: 0,
right: 0,
bottom: 0,
left: 0,
zIndex: -1,
},
'&:not(.gridding):before': {
backgroundColor: cssVarV2('layer/background/secondary'),
},
'&.gridding:before': {
opacity: 0.25,
backgroundSize: '20px 20px',
backgroundImage: `linear-gradient(${cssVarV2('button/grabber/default')} 1px, transparent 1px), linear-gradient(to right, ${cssVarV2('button/grabber/default')} 1px, transparent 1px)`,
},
},
});
export const viewerContainer = style({
position: 'relative',
display: 'flex',
flexDirection: 'column',
width: '100%',
height: '100%',
});
export const titlebar = style({
display: 'flex',
justifyContent: 'space-between',
height: '52px',
padding: '10px 8px',
background: cssVarV2('layer/background/primary'),
fontSize: '12px',
fontWeight: 400,
color: cssVarV2('text/secondary'),
borderTopWidth: '0.5px',
borderTopStyle: 'solid',
borderTopColor: cssVarV2('layer/insideBorder/border'),
textWrap: 'nowrap',
overflow: 'hidden',
});
export const titlebarChild = style({
overflow: 'hidden',
selectors: {
[`${titlebar} > &`]: {
display: 'flex',
gap: '12px',
alignItems: 'center',
paddingLeft: '12px',
paddingRight: '12px',
},
'&.zoom:not(.show)': {
display: 'none',
},
},
});
export const titlebarName = style({
display: 'flex',
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'pre',
wordWrap: 'break-word',
});
@@ -0,0 +1,105 @@
import { IconButton, Menu, MenuItem } from '@affine/component';
import type { AttachmentBlockModel } from '@blocksuite/affine/model';
import {
//EditIcon,
LocalDataIcon,
MoreHorizontalIcon,
ZoomDownIcon,
ZoomUpIcon,
} from '@blocksuite/icons/rc';
import clsx from 'clsx';
import { useState } from 'react';
import { download } from './utils';
import * as styles from './viewer.css';
const items = [
/*
{
name: 'Rename',
icon: <EditIcon />,
action(_model: AttachmentBlockModel) {},
},
*/
{
name: 'Download',
icon: <LocalDataIcon />,
action: download,
},
];
export const MenuItems = ({ model }: { model: AttachmentBlockModel }) =>
items.map(({ name, icon, action }) => (
<MenuItem
key={name}
onClick={() => {
action(model).catch(console.error);
}}
prefixIcon={icon}
>
{name}
</MenuItem>
));
export interface TitlebarProps {
model: AttachmentBlockModel;
name: string;
ext: string;
size: string;
zoom?: number;
}
export const Titlebar = ({
model,
name,
ext,
size,
zoom = 100,
}: TitlebarProps) => {
const [openMenu, setOpenMenu] = useState(false);
return (
<div className={styles.titlebar}>
<div className={styles.titlebarChild}>
<div className={styles.titlebarName}>
<div>{name}</div>
<span>.{ext}</span>
</div>
<div>{size}</div>
<IconButton
icon={<LocalDataIcon />}
onClick={() => {
download(model).catch(console.error);
}}
></IconButton>
<Menu
items={<MenuItems model={model} />}
rootOptions={{
open: openMenu,
onOpenChange: setOpenMenu,
}}
contentOptions={{
side: 'bottom',
align: 'center',
avoidCollisions: false,
}}
>
<IconButton icon={<MoreHorizontalIcon />}></IconButton>
</Menu>
</div>
<div
className={clsx([
styles.titlebarChild,
'zoom',
{
show: false,
},
])}
>
<IconButton icon={<ZoomDownIcon />}></IconButton>
<div>{zoom}%</div>
<IconButton icon={<ZoomUpIcon />}></IconButton>
</div>
</div>
);
};
@@ -50,7 +50,10 @@ import {
type DefaultOpenProperty,
DocPropertiesTable,
} from '../../components/doc-properties';
import { patchForAttachmentEmbedViews } from '../extensions/attachment-embed-view';
import {
patchForAudioEmbedView,
patchForPDFEmbedView,
} from '../extensions/attachment-embed-view';
import { patchDatabaseBlockConfigService } from '../extensions/database-block-config-service';
import { patchDocModeService } from '../extensions/doc-mode-service';
import { patchDocUrlExtensions } from '../extensions/doc-url';
@@ -148,6 +151,14 @@ const usePatchSpecs = (mode: DocMode) => {
featureFlagService.flags.enable_turbo_renderer.$
);
const enablePDFEmbedPreview = useLiveData(
featureFlagService.flags.enable_pdf_embed_preview.$
);
const enableAudioBlock = useLiveData(
featureFlagService.flags.enable_audio_block.$
);
const patchedSpecs = useMemo(() => {
const builder = enableEditorExtension(framework, mode, enableAI);
@@ -176,8 +187,11 @@ const usePatchSpecs = (mode: DocMode) => {
].flat()
);
if (featureFlagService.flags.enable_pdf_embed_preview.value) {
builder.extend([patchForAttachmentEmbedViews(reactToLit)]);
if (enablePDFEmbedPreview) {
builder.extend([patchForPDFEmbedView(reactToLit)]);
}
if (enableAudioBlock) {
builder.extend([patchForAudioEmbedView(reactToLit)]);
}
if (BUILD_CONFIG.isMobileEdition) {
enableMobileExtension(builder, framework);
@@ -202,7 +216,8 @@ const usePatchSpecs = (mode: DocMode) => {
memberSearchService,
publicUserService,
enableTurboRenderer,
featureFlagService.flags.enable_pdf_embed_preview.value,
enablePDFEmbedPreview,
enableAudioBlock,
]);
return [
@@ -1,11 +1,11 @@
import type { ElementOrFactory } from '@affine/component';
import { AttachmentEmbedPreview } from '@affine/core/components/attachment-viewer/pdf-viewer-embedded';
import { AttachmentEmbedPreview } from '@affine/core/blocksuite/attachment-viewer/attachment-embed-preview';
import { AttachmentEmbedConfigIdentifier } from '@blocksuite/affine/blocks/attachment';
import { Bound } from '@blocksuite/affine/global/gfx';
import type { ExtensionType } from '@blocksuite/affine/store';
import type { TemplateResult } from 'lit';
export function patchForAttachmentEmbedViews(
export function patchForPDFEmbedView(
reactToLit: (
element: ElementOrFactory,
rerendering?: boolean
@@ -34,3 +34,23 @@ export function patchForAttachmentEmbedViews(
},
};
}
export function patchForAudioEmbedView(
reactToLit: (
element: ElementOrFactory,
rerendering?: boolean
) => TemplateResult
): ExtensionType {
return {
setup: di => {
di.override(AttachmentEmbedConfigIdentifier('audio'), () => ({
name: 'audio',
check: (model, maxFileSize) =>
model.props.type.startsWith('audio/') &&
model.props.size <= maxFileSize,
template: (model, _blobUrl) =>
reactToLit(<AttachmentEmbedPreview model={model} />, false),
}));
},
};
}
@@ -21,6 +21,7 @@ import type { FrameworkProvider } from '@toeverything/infra';
import type { Observable } from 'rxjs';
import { AIChatBlockSpec } from '../../ai/blocks';
import { AITranscriptionBlockSpec } from '../../ai/blocks/ai-chat-block/ai-transcription-block';
import { buildDocDisplayMetaExtension } from '../display-meta';
import { getFontConfigExtension } from '../font-config';
import { patchPeekViewService } from '../peek-view-service';
@@ -104,6 +105,7 @@ export function enablePreviewExtension(framework: FrameworkProvider): void {
_previewExtensions = [
...AIChatBlockSpec,
...AITranscriptionBlockSpec,
fontConfig,
getThemeExtension(framework),
getPagePreviewThemeExtension(framework),