Merge pull request #10745 from toeverything/doodl/gfx-turbo-renderer

refactor(editor): add gfx turbo renderer package
This commit is contained in:
Yifeng Wang
2025-03-11 12:48:31 +08:00
committed by GitHub
21 changed files with 80 additions and 11 deletions

View File

@@ -1,2 +0,0 @@
export * from './types.js';
export * from './viewport-renderer.js';

View File

@@ -1,142 +0,0 @@
import { type ViewportLayout } from './types.js';
type WorkerMessagePaint = {
type: 'paintLayout';
data: {
layout: ViewportLayout;
width: number;
height: number;
dpr: number;
zoom: number;
version: number;
};
};
type WorkerMessage = WorkerMessagePaint;
const meta = {
emSize: 2048,
hHeadAscent: 1984,
hHeadDescent: -494,
};
const font = new FontFace(
'Inter',
`url(https://fonts.gstatic.com/s/inter/v18/UcCo3FwrK3iLTcviYwYZ8UA3.woff2)`
);
// @ts-expect-error worker env
self.fonts && self.fonts.add(font);
const debugSentenceBoarder = false;
function getBaseline() {
const fontSize = 15;
const lineHeight = 1.2 * fontSize;
const A = fontSize * (meta.hHeadAscent / meta.emSize); // ascent
const D = fontSize * (meta.hHeadDescent / meta.emSize); // descent
const AD = A + Math.abs(D); // ascent + descent
const L = lineHeight - AD; // leading
const y = A + L / 2;
return y;
}
/** Layout painter in worker */
class LayoutPainter {
private readonly canvas: OffscreenCanvas = new OffscreenCanvas(0, 0);
private ctx: OffscreenCanvasRenderingContext2D | null = null;
private zoom = 1;
setSize(layoutRectW: number, layoutRectH: number, dpr: number, zoom: number) {
const width = layoutRectW * dpr * zoom;
const height = layoutRectH * dpr * zoom;
this.canvas.width = width;
this.canvas.height = height;
this.ctx = this.canvas.getContext('2d')!;
this.ctx.scale(dpr, dpr);
this.zoom = zoom;
this.clearBackground();
}
private clearBackground() {
if (!this.canvas || !this.ctx) return;
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
}
paint(layout: ViewportLayout, version: number) {
const { canvas, ctx } = this;
if (!canvas || !ctx) return;
if (layout.rect.w === 0 || layout.rect.h === 0) {
console.warn('empty layout rect');
return;
}
this.clearBackground();
ctx.scale(this.zoom, this.zoom);
// Track rendered positions to avoid duplicate rendering across all paragraphs and sentences
const renderedPositions = new Set<string>();
layout.paragraphs.forEach(paragraph => {
const fontSize = 15;
ctx.font = `300 ${fontSize}px Inter`;
const baselineY = getBaseline();
paragraph.sentences.forEach(sentence => {
ctx.strokeStyle = 'yellow';
sentence.rects.forEach(textRect => {
const x = textRect.rect.x - layout.rect.x;
const y = textRect.rect.y - layout.rect.y;
const posKey = `${x},${y}`;
// Only render if we haven't rendered at this position before
if (renderedPositions.has(posKey)) return;
if (debugSentenceBoarder) {
ctx.strokeRect(x, y, textRect.rect.w, textRect.rect.h);
}
ctx.fillStyle = 'black';
ctx.fillText(textRect.text, x, y + baselineY);
renderedPositions.add(posKey);
});
});
});
const bitmap = canvas.transferToImageBitmap();
self.postMessage(
{ type: 'bitmapPainted', bitmap, version },
{ transfer: [bitmap] }
);
}
}
const painter = new LayoutPainter();
let fontLoaded = false;
font
.load()
.then(() => {
fontLoaded = true;
})
.catch(console.error);
self.onmessage = async (e: MessageEvent<WorkerMessage>) => {
const { type, data } = e.data;
if (!fontLoaded) {
await font.load();
fontLoaded = true;
}
switch (type) {
case 'paintLayout': {
const { layout, width, height, dpr, zoom, version } = data;
painter.setSize(width, height, dpr, zoom);
painter.paint(layout, version);
break;
}
}
};

View File

@@ -1,192 +0,0 @@
import type { EditorHost, GfxBlockComponent } from '@blocksuite/block-std';
import {
clientToModelCoord,
GfxBlockElementModel,
GfxControllerIdentifier,
type Viewport,
} from '@blocksuite/block-std/gfx';
import { Pane } from 'tweakpane';
import { getSentenceRects, segmentSentences } from './text-utils.js';
import type {
ParagraphLayout,
RenderingState,
ViewportLayout,
} from './types.js';
import type { ViewportTurboRendererExtension } from './viewport-renderer.js';
export function syncCanvasSize(canvas: HTMLCanvasElement, host: HTMLElement) {
const hostRect = host.getBoundingClientRect();
const dpr = window.devicePixelRatio;
canvas.style.position = 'absolute';
canvas.style.left = '0px';
canvas.style.top = '0px';
canvas.style.width = '100%';
canvas.style.height = '100%';
canvas.width = hostRect.width * dpr;
canvas.height = hostRect.height * dpr;
canvas.style.pointerEvents = 'none';
}
function getParagraphs(host: EditorHost) {
const gfx = host.std.get(GfxControllerIdentifier);
const models = gfx.gfxElements.filter(e => e instanceof GfxBlockElementModel);
const components = models
.map(model => gfx.view.get(model.id))
.filter(Boolean) as GfxBlockComponent[];
const paragraphs: ParagraphLayout[] = [];
const selector = '.affine-paragraph-rich-text-wrapper [data-v-text="true"]';
components.forEach(component => {
const paragraphNodes = component.querySelectorAll(selector);
const viewportRecord = component.gfx.viewport.deserializeRecord(
component.dataset.viewportState
);
if (!viewportRecord) return;
const { zoom, viewScale } = viewportRecord;
paragraphNodes.forEach(paragraphNode => {
const paragraph: ParagraphLayout = {
sentences: [],
};
const sentences = segmentSentences(paragraphNode.textContent || '');
paragraph.sentences = sentences.map(sentence => {
const sentenceRects = getSentenceRects(paragraphNode, sentence);
const rects = sentenceRects.map(({ text, rect }) => {
const [modelX, modelY] = clientToModelCoord(viewportRecord, [
rect.x,
rect.y,
]);
return {
text,
...rect,
rect: {
x: modelX,
y: modelY,
w: rect.w / zoom / viewScale,
h: rect.h / zoom / viewScale,
},
};
});
return {
text: sentence,
rects,
};
});
paragraphs.push(paragraph);
});
});
return paragraphs;
}
export function getViewportLayout(
host: EditorHost,
viewport: Viewport
): ViewportLayout {
const zoom = viewport.zoom;
let layoutMinX = Infinity;
let layoutMinY = Infinity;
let layoutMaxX = -Infinity;
let layoutMaxY = -Infinity;
const paragraphs = getParagraphs(host);
paragraphs.forEach(paragraph => {
paragraph.sentences.forEach(sentence => {
sentence.rects.forEach(r => {
layoutMinX = Math.min(layoutMinX, r.rect.x);
layoutMinY = Math.min(layoutMinY, r.rect.y);
layoutMaxX = Math.max(layoutMaxX, r.rect.x + r.rect.w);
layoutMaxY = Math.max(layoutMaxY, r.rect.y + r.rect.h);
});
});
});
const layoutModelCoord = [layoutMinX, layoutMinY];
const w = (layoutMaxX - layoutMinX) / zoom / viewport.viewScale;
const h = (layoutMaxY - layoutMinY) / zoom / viewport.viewScale;
const layout: ViewportLayout = {
paragraphs,
rect: {
x: layoutModelCoord[0],
y: layoutModelCoord[1],
w: Math.max(w, 0),
h: Math.max(h, 0),
},
};
return layout;
}
export function initTweakpane(
renderer: ViewportTurboRendererExtension,
viewportElement: HTMLElement
) {
const debugPane = new Pane({ container: viewportElement });
const paneElement = debugPane.element;
paneElement.style.position = 'absolute';
paneElement.style.top = '10px';
paneElement.style.right = '10px';
paneElement.style.width = '250px';
debugPane.title = 'Viewport Turbo Renderer';
debugPane.addButton({ title: 'Invalidate' }).on('click', () => {
renderer.invalidate();
});
}
export function debugLog(message: string, state: RenderingState) {
console.log(
`%c[ViewportTurboRenderer]%c ${message} | state=${state}`,
'color: #4285f4; font-weight: bold;',
'color: inherit;'
);
}
export function paintPlaceholder(
canvas: HTMLCanvasElement,
layout: ViewportLayout | null,
viewport: Viewport
) {
const ctx = canvas.getContext('2d');
if (!ctx) return;
if (!layout) return;
const dpr = window.devicePixelRatio;
const layoutViewCoord = viewport.toViewCoord(layout.rect.x, layout.rect.y);
const offsetX = layoutViewCoord[0];
const offsetY = layoutViewCoord[1];
const colors = [
'rgba(200, 200, 200, 0.7)',
'rgba(180, 180, 180, 0.7)',
'rgba(160, 160, 160, 0.7)',
];
layout.paragraphs.forEach((paragraph, paragraphIndex) => {
ctx.fillStyle = colors[paragraphIndex % colors.length];
const renderedPositions = new Set<string>();
paragraph.sentences.forEach(sentence => {
sentence.rects.forEach(textRect => {
const x =
((textRect.rect.x - layout.rect.x) * viewport.zoom + offsetX) * dpr;
const y =
((textRect.rect.y - layout.rect.y) * viewport.zoom + offsetY) * dpr;
dpr;
const width = textRect.rect.w * viewport.zoom * dpr;
const height = textRect.rect.h * viewport.zoom * dpr;
const posKey = `${x},${y}`;
if (renderedPositions.has(posKey)) return;
ctx.fillRect(x, y, width, height);
if (width > 10 && height > 5) {
ctx.strokeStyle = 'rgba(150, 150, 150, 0.3)';
ctx.strokeRect(x, y, width, height);
}
renderedPositions.add(posKey);
});
});
});
}

View File

@@ -1,145 +0,0 @@
import type { TextRect } from './types.js';
interface WordSegment {
text: string;
start: number;
end: number;
}
const CJK_REGEX = /[\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]/u;
const sentenceSegmenter = new Intl.Segmenter(undefined, {
granularity: 'sentence',
});
const wordSegmenter = new Intl.Segmenter(undefined, {
granularity: 'word',
});
const graphemeSegmenter = new Intl.Segmenter(undefined, {
granularity: 'grapheme',
});
function hasCJK(text: string): boolean {
return CJK_REGEX.test(text);
}
function getWordSegments(text: string): WordSegment[] {
const segmenter = hasCJK(text) ? graphemeSegmenter : wordSegmenter;
return Array.from(segmenter.segment(text)).map(({ segment, index }) => ({
text: segment,
start: index,
end: index + segment.length,
}));
}
function getRangeRects(range: Range, fullText: string): TextRect[] {
const rects = Array.from(range.getClientRects());
const textRects: TextRect[] = [];
if (rects.length === 0) return textRects;
// If there's only one rect, use the full text
if (rects.length === 1) {
const rect = rects[0];
textRects.push({
rect: {
x: rect.x,
y: rect.y,
w: rect.width,
h: rect.height,
},
text: fullText,
});
return textRects;
}
const segments = getWordSegments(fullText);
// Calculate the total width and average width per character
const totalWidth = Math.floor(
rects.reduce((sum, rect) => sum + rect.width, 0)
);
const charWidthEstimate = totalWidth / fullText.length;
let currentRect = 0;
let currentSegments: WordSegment[] = [];
let currentWidth = 0;
segments.forEach(segment => {
const segmentWidth = segment.text.length * charWidthEstimate;
if (
currentWidth + segmentWidth > rects[currentRect]?.width &&
currentSegments.length > 0
) {
const rect = rects[currentRect];
textRects.push({
rect: {
x: rect.x,
y: rect.y,
w: rect.width,
h: rect.height,
},
text: currentSegments.map(seg => seg.text).join(''),
});
currentRect++;
currentSegments = [segment];
currentWidth = segmentWidth;
} else {
currentSegments.push(segment);
currentWidth += segmentWidth;
}
});
// Handle remaining segments if any
if (currentSegments.length > 0) {
const rect = rects[Math.min(currentRect, rects.length - 1)];
textRects.push({
rect: {
x: rect.x,
y: rect.y,
w: rect.width,
h: rect.height,
},
text: currentSegments.map(seg => seg.text).join(''),
});
}
return textRects;
}
export function getSentenceRects(
element: Element,
sentence: string
): TextRect[] {
const textNode = Array.from(element.childNodes).find(
node => node.nodeType === Node.TEXT_NODE
);
if (!textNode) return [];
const text = textNode.textContent || '';
let rects: TextRect[] = [];
let startIndex = 0;
// Find all occurrences of the sentence and ensure we capture complete words
while ((startIndex = text.indexOf(sentence, startIndex)) !== -1) {
const range = document.createRange();
let endIndex = startIndex + sentence.length;
range.setStart(textNode, startIndex);
range.setEnd(textNode, endIndex);
rects = rects.concat(
getRangeRects(range, text.slice(startIndex, endIndex))
);
startIndex = endIndex;
}
return rects;
}
export function segmentSentences(text: string): string[] {
return Array.from(sentenceSegmenter.segment(text)).map(
({ segment }) => segment
);
}

View File

@@ -1,48 +0,0 @@
export interface Rect {
x: number;
y: number;
w: number;
h: number;
}
// We can't use viewport instance here because it can't be reused in worker
export interface ViewportState {
zoom: number;
viewScale: number;
viewportX: number;
viewportY: number;
}
export interface SentenceLayout {
text: string;
rects: TextRect[];
}
export interface ParagraphLayout {
sentences: SentenceLayout[];
}
export interface ViewportLayout {
paragraphs: ParagraphLayout[];
rect: Rect;
}
export interface TextRect {
rect: Rect;
text: string;
}
/**
* Represents the rendering state of the ViewportTurboRenderer
* - inactive: Renderer is not active
* - pending: Bitmap is invalid or not yet available, falling back to DOM rendering
* - zooming: Zooming in or out, will use fast canvas placeholder rendering
* - rendering: Currently rendering to a bitmap (async operation in progress)
* - ready: Bitmap is valid and rendered, DOM elements can be safely removed
*/
export type RenderingState =
| 'inactive'
| 'pending'
| 'zooming'
| 'rendering'
| 'ready';

View File

@@ -1,305 +0,0 @@
import {
LifeCycleWatcher,
LifeCycleWatcherIdentifier,
StdIdentifier,
} from '@blocksuite/block-std';
import {
GfxControllerIdentifier,
type GfxViewportElement,
} from '@blocksuite/block-std/gfx';
import type { Container, ServiceIdentifier } from '@blocksuite/global/di';
import { DisposableGroup } from '@blocksuite/global/slot';
import debounce from 'lodash-es/debounce';
import {
debugLog,
getViewportLayout,
initTweakpane,
paintPlaceholder,
syncCanvasSize,
} from './renderer-utils.js';
import type { RenderingState, ViewportLayout } from './types.js';
const debug = false; // Toggle for debug logs
const zoomThreshold = 1; // With high enough zoom, fallback to DOM rendering
const debounceTime = 1000; // During this period, fallback to DOM
const workerUrl = new URL('./painter.worker.ts', import.meta.url);
export class ViewportTurboRendererExtension extends LifeCycleWatcher {
public state: RenderingState = 'inactive';
public readonly canvas: HTMLCanvasElement = document.createElement('canvas');
private readonly worker: Worker = new Worker(workerUrl, { type: 'module' });
private readonly disposables = new DisposableGroup();
private layoutCacheData: ViewportLayout | null = null;
private layoutVersion = 0;
private bitmap: ImageBitmap | null = null;
private viewportElement: GfxViewportElement | null = null;
static override setup(di: Container) {
di.addImpl(ViewportTurboRendererIdentifier, this, [StdIdentifier]);
}
override mounted() {
const mountPoint = document.querySelector('.affine-edgeless-viewport');
if (mountPoint) {
mountPoint.append(this.canvas);
initTweakpane(this, mountPoint as HTMLElement);
}
this.viewport.elementReady.once(element => {
this.viewportElement = element;
syncCanvasSize(this.canvas, this.std.host);
this.setState('pending');
this.disposables.add(
this.viewport.sizeUpdated.on(() => this.handleResize())
);
this.disposables.add(
this.viewport.viewportUpdated.on(() => {
this.refresh().catch(console.error);
})
);
this.disposables.add({
dispose: this.viewport.zooming$.subscribe(isZooming => {
this.debugLog(`Zooming signal changed: ${isZooming}`);
if (isZooming) {
this.setState('zooming');
} else if (this.state === 'zooming') {
this.setState('pending');
this.refresh().catch(console.error);
}
}),
});
});
this.disposables.add(
this.selection.slots.updated.on(() => this.invalidate())
);
this.disposables.add(
this.std.store.slots.blockUpdated.on(() => this.invalidate())
);
}
override unmounted() {
this.debugLog('Unmounting renderer');
this.clearBitmap();
this.clearOptimizedBlocks();
this.worker.terminate();
this.canvas.remove();
this.disposables.dispose();
this.setState('inactive');
}
get gfx() {
return this.std.get(GfxControllerIdentifier);
}
get viewport() {
return this.gfx.viewport;
}
get selection() {
return this.gfx.selection;
}
get layoutCache() {
if (this.layoutCacheData) return this.layoutCacheData;
const layout = getViewportLayout(this.std.host, this.viewport);
this.debugLog('Layout cache updated');
return (this.layoutCacheData = layout);
}
async refresh() {
if (this.state === 'inactive') return;
this.clearCanvas();
// -> pending
if (this.viewport.zoom > zoomThreshold) {
this.debugLog('Zoom above threshold, falling back to DOM rendering');
this.setState('pending');
this.clearOptimizedBlocks();
}
// -> zooming
else if (this.isZooming()) {
this.debugLog('Currently zooming, using placeholder rendering');
this.setState('zooming');
this.paintPlaceholder();
this.updateOptimizedBlocks();
}
// -> ready
else if (this.canUseBitmapCache()) {
this.debugLog('Using cached bitmap');
this.setState('ready');
this.drawCachedBitmap();
this.updateOptimizedBlocks();
}
// -> rendering
else {
this.setState('rendering');
await this.paintLayout();
this.drawCachedBitmap();
this.updateOptimizedBlocks();
}
}
debouncedRefresh = debounce(() => {
this.refresh().catch(console.error);
}, debounceTime);
invalidate() {
this.layoutVersion++;
this.layoutCacheData = null;
this.clearBitmap();
this.clearCanvas();
this.clearOptimizedBlocks();
this.setState('pending');
this.debugLog(`Invalidated renderer (layoutVersion=${this.layoutVersion})`);
}
private debugLog(message: string) {
if (!debug) return;
debugLog(message, this.state);
}
private clearBitmap() {
if (!this.bitmap) return;
this.bitmap.close();
this.bitmap = null;
this.debugLog('Bitmap cleared');
}
private async paintLayout(): Promise<void> {
return new Promise(resolve => {
if (!this.worker) return;
const layout = this.layoutCache;
const dpr = window.devicePixelRatio;
const currentVersion = this.layoutVersion;
this.debugLog(`Requesting bitmap painting (version=${currentVersion})`);
this.worker.postMessage({
type: 'paintLayout',
data: {
layout,
width: layout.rect.w,
height: layout.rect.h,
dpr,
zoom: this.viewport.zoom,
version: currentVersion,
},
});
this.worker.onmessage = (e: MessageEvent) => {
if (e.data.type === 'bitmapPainted') {
if (e.data.version === this.layoutVersion) {
this.debugLog(
`Bitmap painted successfully (version=${e.data.version})`
);
this.clearBitmap();
this.bitmap = e.data.bitmap;
this.setState('ready');
resolve();
} else {
this.debugLog(
`Received outdated bitmap (got=${e.data.version}, current=${this.layoutVersion})`
);
e.data.bitmap.close();
this.setState('pending');
resolve();
}
}
};
});
}
private canUseBitmapCache(): boolean {
// Never use bitmap cache during zooming
if (this.isZooming()) return false;
return !!(this.layoutCache && this.bitmap);
}
private isZooming(): boolean {
return this.viewport.zooming$.value;
}
private clearCanvas() {
const ctx = this.canvas.getContext('2d');
if (!ctx) return;
ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.debugLog('Canvas cleared');
}
private drawCachedBitmap() {
if (!this.bitmap) {
this.debugLog('No cached bitmap available, requesting refresh');
this.debouncedRefresh();
return;
}
const layout = this.layoutCache;
const bitmap = this.bitmap;
const ctx = this.canvas.getContext('2d');
if (!ctx) return;
this.clearCanvas();
const layoutViewCoord = this.viewport.toViewCoord(
layout.rect.x,
layout.rect.y
);
ctx.drawImage(
bitmap,
layoutViewCoord[0] * window.devicePixelRatio,
layoutViewCoord[1] * window.devicePixelRatio,
layout.rect.w * window.devicePixelRatio * this.viewport.zoom,
layout.rect.h * window.devicePixelRatio * this.viewport.zoom
);
this.debugLog('Bitmap drawn to canvas');
}
setState(newState: RenderingState) {
if (this.state === newState) return;
this.state = newState;
this.debugLog(`State change: ${this.state} -> ${newState}`);
}
private canOptimize(): boolean {
const isBelowZoomThreshold = this.viewport.zoom <= zoomThreshold;
return (
(this.state === 'ready' || this.state === 'zooming') &&
isBelowZoomThreshold
);
}
private updateOptimizedBlocks() {
requestAnimationFrame(() => {
if (!this.viewportElement || !this.layoutCache) return;
if (!this.canOptimize()) return;
const blockElements = this.viewportElement.getModelsInViewport();
const blockIds = Array.from(blockElements).map(model => model.id);
this.debugLog(`Optimized ${blockIds.length} blocks`);
});
}
private clearOptimizedBlocks() {
this.debugLog('Cleared optimized blocks');
}
private handleResize() {
this.debugLog('Container resized, syncing canvas size');
syncCanvasSize(this.canvas, this.std.host);
this.invalidate();
this.debouncedRefresh();
}
private paintPlaceholder() {
paintPlaceholder(this.canvas, this.layoutCache, this.viewport);
}
}
export const ViewportTurboRendererIdentifier = LifeCycleWatcherIdentifier(
'ViewportTurboRenderer'
) as ServiceIdentifier<ViewportTurboRendererExtension>;