From c023b724d01d1b260aebc9776447716cdaab21ab Mon Sep 17 00:00:00 2001 From: doodlewind <7312949+doodlewind@users.noreply.github.com> Date: Thu, 13 Mar 2025 05:18:11 +0000 Subject: [PATCH] refactor(editor): generic layout type support for turbo renderer (#10766) This PR refactored the turbo renderer architecture to support multiple block layout types. - New base class `BlockLayoutPainter` and `BlockLayoutProvider` are introduced for writing extendable per-block layout querying and painting logic. - Paragraph-specific lines are all moved into dedicated classes (`ParagraphLayoutProvider` and `ParagraphLayoutPainter`) under the `/variants/paragraph` dir. - The `renderer-utils.ts` doesn't contain paragraph-specific logic now. - The `text-utils.ts` is also now scoped for paragraph only. - Worker messages are now strongly typed. Upcoming PR should further implement the block registration system using extension API. The `variants` dir could still exist, since there will be similar rendering logic that can be reused among block types (i.e., between paragraph block and list block). --- .../blocks/block-paragraph/package.json | 4 +- .../blocks/block-paragraph/src/index.ts | 2 + .../src/turbo/paragraph-layout-provider.ts | 99 ++++++++++++ .../src/turbo/paragraph-painter-config.ts | 16 ++ .../src/turbo/paragraph-painter.worker.ts | 109 ++++++++++++++ .../affine/gfx/turbo-renderer/package.json | 4 +- .../affine/gfx/turbo-renderer/src/index.ts | 6 +- .../src/layout/block-layout-provider.ts | 21 +++ .../gfx/turbo-renderer/src/painter.worker.ts | 142 ------------------ .../src/painter/painter.worker.ts | 91 +++++++++++ .../gfx/turbo-renderer/src/renderer-utils.ts | 139 +++++++---------- .../gfx/turbo-renderer/src/text-utils.ts | 2 +- .../gfx/turbo-renderer/src/turbo-renderer.ts | 80 ++++++++-- .../affine/gfx/turbo-renderer/src/types.ts | 70 ++++++++- .../src/__tests__/utils/renderer-entry.ts | 17 ++- .../block-suite-editor/lit-adaper.tsx | 4 +- .../blocksuite/extensions/turbo-renderer.ts | 22 +++ yarn.lock | 2 + 18 files changed, 578 insertions(+), 252 deletions(-) create mode 100644 blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-layout-provider.ts create mode 100644 blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter-config.ts create mode 100644 blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter.worker.ts create mode 100644 blocksuite/affine/gfx/turbo-renderer/src/layout/block-layout-provider.ts delete mode 100644 blocksuite/affine/gfx/turbo-renderer/src/painter.worker.ts create mode 100644 blocksuite/affine/gfx/turbo-renderer/src/painter/painter.worker.ts create mode 100644 packages/frontend/core/src/blocksuite/extensions/turbo-renderer.ts diff --git a/blocksuite/affine/blocks/block-paragraph/package.json b/blocksuite/affine/blocks/block-paragraph/package.json index 57b0bb56dc..2cea7699c2 100644 --- a/blocksuite/affine/blocks/block-paragraph/package.json +++ b/blocksuite/affine/blocks/block-paragraph/package.json @@ -14,6 +14,7 @@ "license": "MIT", "dependencies": { "@blocksuite/affine-components": "workspace:*", + "@blocksuite/affine-gfx-turbo-renderer": "workspace:*", "@blocksuite/affine-model": "workspace:*", "@blocksuite/affine-rich-text": "workspace:*", "@blocksuite/affine-shared": "workspace:*", @@ -33,7 +34,8 @@ }, "exports": { ".": "./src/index.ts", - "./effects": "./src/effects.ts" + "./effects": "./src/effects.ts", + "./turbo-painter": "./src/turbo/paragraph-painter.worker.ts" }, "files": [ "src", diff --git a/blocksuite/affine/blocks/block-paragraph/src/index.ts b/blocksuite/affine/blocks/block-paragraph/src/index.ts index de229b245b..86bba19416 100644 --- a/blocksuite/affine/blocks/block-paragraph/src/index.ts +++ b/blocksuite/affine/blocks/block-paragraph/src/index.ts @@ -3,3 +3,5 @@ export * from './commands'; export * from './paragraph-block.js'; export * from './paragraph-service.js'; export * from './paragraph-spec.js'; +export * from './turbo/paragraph-layout-provider.js'; +export * from './turbo/paragraph-painter-config.js'; diff --git a/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-layout-provider.ts b/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-layout-provider.ts new file mode 100644 index 0000000000..410ae14c8b --- /dev/null +++ b/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-layout-provider.ts @@ -0,0 +1,99 @@ +import type { Rect } from '@blocksuite/affine-gfx-turbo-renderer'; +import { + BlockLayoutHandlerExtension, + BlockLayoutHandlersIdentifier, + getSentenceRects, + segmentSentences, +} from '@blocksuite/affine-gfx-turbo-renderer'; +import type { GfxBlockComponent } from '@blocksuite/block-std'; +import { clientToModelCoord } from '@blocksuite/block-std/gfx'; +import type { Container } from '@blocksuite/global/di'; + +import type { ParagraphLayout } from './paragraph-painter.worker'; + +export class ParagraphLayoutHandlerExtension extends BlockLayoutHandlerExtension { + readonly blockType = 'affine:paragraph'; + + static override setup(di: Container) { + const layoutHandler = new ParagraphLayoutHandlerExtension(); + di.addImpl(BlockLayoutHandlersIdentifier, layoutHandler); + } + + queryLayout(component: GfxBlockComponent): ParagraphLayout | null { + const paragraphSelector = + '.affine-paragraph-rich-text-wrapper [data-v-text="true"]'; + const paragraphNodes = component.querySelectorAll(paragraphSelector); + + if (paragraphNodes.length === 0) return null; + + const viewportRecord = component.gfx.viewport.deserializeRecord( + component.dataset.viewportState + ); + + if (!viewportRecord) return null; + + const { zoom, viewScale } = viewportRecord; + const paragraph: ParagraphLayout = { + type: 'affine:paragraph', + sentences: [], + }; + + paragraphNodes.forEach(paragraphNode => { + const sentences = segmentSentences(paragraphNode.textContent || ''); + const sentenceLayouts = sentences.map(sentence => { + const sentenceRects = getSentenceRects(paragraphNode, sentence); + const rects = sentenceRects.map(({ text, rect }) => { + const [modelX, modelY] = clientToModelCoord(viewportRecord, [ + rect.x, + rect.y, + ]); + return { + text, + rect: { + x: modelX, + y: modelY, + w: rect.w / zoom / viewScale, + h: rect.h / zoom / viewScale, + }, + }; + }); + return { + text: sentence, + rects, + }; + }); + + paragraph.sentences.push(...sentenceLayouts); + }); + + return paragraph; + } + + calculateBound(layout: ParagraphLayout) { + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + + layout.sentences.forEach(sentence => { + sentence.rects.forEach(r => { + minX = Math.min(minX, r.rect.x); + minY = Math.min(minY, r.rect.y); + maxX = Math.max(maxX, r.rect.x + r.rect.w); + maxY = Math.max(maxY, r.rect.y + r.rect.h); + }); + }); + + const rect: Rect = { + x: minX, + y: minY, + w: maxX - minX, + h: maxY - minY, + }; + + return { + rect, + subRects: layout.sentences.flatMap(s => s.rects.map(r => r.rect)), + }; + } +} diff --git a/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter-config.ts b/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter-config.ts new file mode 100644 index 0000000000..9e2bd43c75 --- /dev/null +++ b/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter-config.ts @@ -0,0 +1,16 @@ +import { BlockPainterConfigIdentifier } from '@blocksuite/affine-gfx-turbo-renderer'; +import type { Container } from '@blocksuite/global/di'; +import { Extension } from '@blocksuite/store'; + +export class ParagraphPaintConfigExtension extends Extension { + static override setup(di: Container) { + const config = { + type: 'affine:paragraph', + path: new URL( + '@blocksuite/affine-block-paragraph/turbo-painter', + import.meta.url + ).href, + }; + di.addImpl(BlockPainterConfigIdentifier, config); + } +} diff --git a/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter.worker.ts b/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter.worker.ts new file mode 100644 index 0000000000..3256b92043 --- /dev/null +++ b/blocksuite/affine/blocks/block-paragraph/src/turbo/paragraph-painter.worker.ts @@ -0,0 +1,109 @@ +import type { + BlockLayout, + BlockLayoutPainter, + TextRect, + WorkerToHostMessage, +} from '@blocksuite/affine-gfx-turbo-renderer'; + +interface SentenceLayout { + text: string; + rects: TextRect[]; +} + +export interface ParagraphLayout extends BlockLayout { + type: 'affine:paragraph'; + sentences: SentenceLayout[]; +} + +const meta = { + emSize: 2048, + hHeadAscent: 1984, + hHeadDescent: -494, +}; + +const debugSentenceBorder = false; + +function getBaseline() { + const fontSize = 15; + const lineHeight = 1.2 * fontSize; + + const A = fontSize * (meta.hHeadAscent / meta.emSize); // ascent + const D = fontSize * (meta.hHeadDescent / meta.emSize); // descent + const AD = A + Math.abs(D); // ascent + descent + const L = lineHeight - AD; // leading + const y = A + L / 2; + return y; +} + +function isParagraphLayout(layout: BlockLayout): layout is ParagraphLayout { + return layout.type === 'affine:paragraph'; +} + +export default class ParagraphLayoutPainter implements BlockLayoutPainter { + static readonly font = new FontFace( + 'Inter', + `url(https://fonts.gstatic.com/s/inter/v18/UcCo3FwrK3iLTcviYwYZ8UA3.woff2)` + ); + + static fontLoaded = false; + + static { + if (typeof self !== 'undefined' && 'fonts' in self) { + // @ts-expect-error worker fonts API + self.fonts.add(ParagraphLayoutPainter.font); + + ParagraphLayoutPainter.font + .load() + .then(() => { + ParagraphLayoutPainter.fontLoaded = true; + }) + .catch(error => { + console.error('Failed to load Inter font:', error); + }); + } + } + + paint( + ctx: OffscreenCanvasRenderingContext2D, + layout: BlockLayout, + layoutBaseX: number, + layoutBaseY: number + ): void { + if (!ParagraphLayoutPainter.fontLoaded) { + const message: WorkerToHostMessage = { + type: 'paintError', + error: 'Font not loaded', + blockType: 'affine:paragraph', + }; + self.postMessage(message); + return; + } + + if (!isParagraphLayout(layout)) return; // cast to ParagraphLayout + + const fontSize = 15; + ctx.font = `300 ${fontSize}px Inter`; + const baselineY = getBaseline(); + const renderedPositions = new Set(); + + layout.sentences.forEach(sentence => { + ctx.strokeStyle = 'yellow'; + sentence.rects.forEach(textRect => { + const x = textRect.rect.x - layoutBaseX; + const y = textRect.rect.y - layoutBaseY; + + const posKey = `${x},${y}`; + // Only render if we haven't rendered at this position before + if (renderedPositions.has(posKey)) return; + + if (debugSentenceBorder) { + ctx.strokeRect(x, y, textRect.rect.w, textRect.rect.h); + } + ctx.fillStyle = 'black'; + ctx.fillText(textRect.text, x, y + baselineY); + + renderedPositions.add(posKey); + }); + }); + } +} diff --git a/blocksuite/affine/gfx/turbo-renderer/package.json b/blocksuite/affine/gfx/turbo-renderer/package.json index d57b42705d..a446d1b187 100644 --- a/blocksuite/affine/gfx/turbo-renderer/package.json +++ b/blocksuite/affine/gfx/turbo-renderer/package.json @@ -15,13 +15,15 @@ "dependencies": { "@blocksuite/block-std": "workspace:*", "@blocksuite/global": "workspace:*", + "@blocksuite/store": "workspace:*", "@types/lodash-es": "^4.17.12", "lodash-es": "^4.17.21", "rxjs": "^7.8.1", "tweakpane": "^4.0.5" }, "exports": { - ".": "./src/index.ts" + ".": "./src/index.ts", + "./painter": "./src/painter/painter.worker.ts" }, "files": [ "src", diff --git a/blocksuite/affine/gfx/turbo-renderer/src/index.ts b/blocksuite/affine/gfx/turbo-renderer/src/index.ts index 13662850f3..2d5533a045 100644 --- a/blocksuite/affine/gfx/turbo-renderer/src/index.ts +++ b/blocksuite/affine/gfx/turbo-renderer/src/index.ts @@ -1,2 +1,4 @@ -export * from './turbo-renderer.js'; -export * from './types.js'; +export * from './layout/block-layout-provider'; +export * from './text-utils'; +export * from './turbo-renderer'; +export * from './types'; diff --git a/blocksuite/affine/gfx/turbo-renderer/src/layout/block-layout-provider.ts b/blocksuite/affine/gfx/turbo-renderer/src/layout/block-layout-provider.ts new file mode 100644 index 0000000000..d494e11764 --- /dev/null +++ b/blocksuite/affine/gfx/turbo-renderer/src/layout/block-layout-provider.ts @@ -0,0 +1,21 @@ +import type { GfxBlockComponent } from '@blocksuite/block-std'; +import { createIdentifier } from '@blocksuite/global/di'; +import { Extension } from '@blocksuite/store'; + +import type { BlockLayout, Rect } from '../types'; + +export abstract class BlockLayoutHandlerExtension< + T extends BlockLayout = BlockLayout, +> extends Extension { + abstract readonly blockType: string; + abstract queryLayout(component: GfxBlockComponent): T | null; + abstract calculateBound(layout: T): { + rect: Rect; + subRects: Rect[]; + }; +} + +export const BlockLayoutHandlersIdentifier = + createIdentifier( + 'BlockLayoutHandlersIdentifier' + ); diff --git a/blocksuite/affine/gfx/turbo-renderer/src/painter.worker.ts b/blocksuite/affine/gfx/turbo-renderer/src/painter.worker.ts deleted file mode 100644 index 67c9aa5afc..0000000000 --- a/blocksuite/affine/gfx/turbo-renderer/src/painter.worker.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { type ViewportLayout } from './types.js'; - -type WorkerMessagePaint = { - type: 'paintLayout'; - data: { - layout: ViewportLayout; - width: number; - height: number; - dpr: number; - zoom: number; - version: number; - }; -}; - -type WorkerMessage = WorkerMessagePaint; - -const meta = { - emSize: 2048, - hHeadAscent: 1984, - hHeadDescent: -494, -}; - -const font = new FontFace( - 'Inter', - `url(https://fonts.gstatic.com/s/inter/v18/UcCo3FwrK3iLTcviYwYZ8UA3.woff2)` -); -// @ts-expect-error worker env -self.fonts && self.fonts.add(font); - -const debugSentenceBoarder = false; - -function getBaseline() { - const fontSize = 15; - const lineHeight = 1.2 * fontSize; - - const A = fontSize * (meta.hHeadAscent / meta.emSize); // ascent - const D = fontSize * (meta.hHeadDescent / meta.emSize); // descent - const AD = A + Math.abs(D); // ascent + descent - const L = lineHeight - AD; // leading - const y = A + L / 2; - return y; -} - -/** Layout painter in worker */ -class LayoutPainter { - private readonly canvas: OffscreenCanvas = new OffscreenCanvas(0, 0); - private ctx: OffscreenCanvasRenderingContext2D | null = null; - private zoom = 1; - - setSize(layoutRectW: number, layoutRectH: number, dpr: number, zoom: number) { - const width = layoutRectW * dpr * zoom; - const height = layoutRectH * dpr * zoom; - - this.canvas.width = width; - this.canvas.height = height; - this.ctx = this.canvas.getContext('2d')!; - this.ctx.scale(dpr, dpr); - this.zoom = zoom; - this.clearBackground(); - } - - private clearBackground() { - if (!this.canvas || !this.ctx) return; - this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); - } - - paint(layout: ViewportLayout, version: number) { - const { canvas, ctx } = this; - if (!canvas || !ctx) return; - if (layout.rect.w === 0 || layout.rect.h === 0) { - console.warn('empty layout rect'); - return; - } - - this.clearBackground(); - - ctx.scale(this.zoom, this.zoom); - - // Track rendered positions to avoid duplicate rendering across all paragraphs and sentences - const renderedPositions = new Set(); - - layout.paragraphs.forEach(paragraph => { - const fontSize = 15; - ctx.font = `300 ${fontSize}px Inter`; - const baselineY = getBaseline(); - - paragraph.sentences.forEach(sentence => { - ctx.strokeStyle = 'yellow'; - sentence.rects.forEach(textRect => { - const x = textRect.rect.x - layout.rect.x; - const y = textRect.rect.y - layout.rect.y; - - const posKey = `${x},${y}`; - // Only render if we haven't rendered at this position before - if (renderedPositions.has(posKey)) return; - - if (debugSentenceBoarder) { - ctx.strokeRect(x, y, textRect.rect.w, textRect.rect.h); - } - ctx.fillStyle = 'black'; - ctx.fillText(textRect.text, x, y + baselineY); - - renderedPositions.add(posKey); - }); - }); - }); - - const bitmap = canvas.transferToImageBitmap(); - self.postMessage( - { type: 'bitmapPainted', bitmap, version }, - { transfer: [bitmap] } - ); - } -} - -const painter = new LayoutPainter(); -let fontLoaded = false; - -font - .load() - .then(() => { - fontLoaded = true; - }) - .catch(console.error); - -self.onmessage = async (e: MessageEvent) => { - const { type, data } = e.data; - - if (!fontLoaded) { - await font.load(); - fontLoaded = true; - } - - switch (type) { - case 'paintLayout': { - const { layout, width, height, dpr, zoom, version } = data; - painter.setSize(width, height, dpr, zoom); - painter.paint(layout, version); - break; - } - } -}; diff --git a/blocksuite/affine/gfx/turbo-renderer/src/painter/painter.worker.ts b/blocksuite/affine/gfx/turbo-renderer/src/painter/painter.worker.ts new file mode 100644 index 0000000000..b26daaee21 --- /dev/null +++ b/blocksuite/affine/gfx/turbo-renderer/src/painter/painter.worker.ts @@ -0,0 +1,91 @@ +import type { + BlockLayoutPainter, + HostToWorkerMessage, + ViewportLayout, + WorkerToHostMessage, +} from '../types'; + +class BlockPainterRegistry { + private readonly painters = new Map(); + + register(type: string, painter: BlockLayoutPainter) { + this.painters.set(type, painter); + } + + getPainter(type: string): BlockLayoutPainter | undefined { + return this.painters.get(type); + } +} + +class ViewportLayoutPainter { + private readonly canvas: OffscreenCanvas = new OffscreenCanvas(0, 0); + private ctx: OffscreenCanvasRenderingContext2D | null = null; + private zoom = 1; + public readonly registry = new BlockPainterRegistry(); + + setSize(layoutRectW: number, layoutRectH: number, dpr: number, zoom: number) { + const width = layoutRectW * dpr * zoom; + const height = layoutRectH * dpr * zoom; + + this.canvas.width = width; + this.canvas.height = height; + this.ctx = this.canvas.getContext('2d')!; + this.ctx.scale(dpr, dpr); + this.zoom = zoom; + this.clearBackground(); + } + + private clearBackground() { + if (!this.canvas || !this.ctx) return; + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + } + + paint(layout: ViewportLayout, version: number) { + const { canvas, ctx } = this; + if (!canvas || !ctx) return; + if (layout.rect.w === 0 || layout.rect.h === 0) { + console.warn('empty layout rect'); + return; + } + + this.clearBackground(); + + ctx.scale(this.zoom, this.zoom); + + layout.blocks.forEach(blockLayout => { + const painter = this.registry.getPainter(blockLayout.type); + if (!painter) return; + painter.paint(ctx, blockLayout, layout.rect.x, layout.rect.y); + }); + + const bitmap = canvas.transferToImageBitmap(); + const message: WorkerToHostMessage = { + type: 'bitmapPainted', + bitmap, + version, + }; + self.postMessage(message, { transfer: [bitmap] }); + } +} + +const painter = new ViewportLayoutPainter(); + +self.onmessage = async (e: MessageEvent) => { + const { type, data } = e.data; + + switch (type) { + case 'paintLayout': { + const { layout, width, height, dpr, zoom, version } = data; + painter.setSize(width, height, dpr, zoom); + painter.paint(layout, version); + break; + } + case 'registerPainter': { + const { painterConfigs } = data; + painterConfigs.forEach(async ({ type, path }) => { + painter.registry.register(type, new (await import(path)).default()); + }); + break; + } + } +}; diff --git a/blocksuite/affine/gfx/turbo-renderer/src/renderer-utils.ts b/blocksuite/affine/gfx/turbo-renderer/src/renderer-utils.ts index 977a91fb06..6b4625fb1d 100644 --- a/blocksuite/affine/gfx/turbo-renderer/src/renderer-utils.ts +++ b/blocksuite/affine/gfx/turbo-renderer/src/renderer-utils.ts @@ -1,19 +1,14 @@ import type { EditorHost, GfxBlockComponent } from '@blocksuite/block-std'; import { - clientToModelCoord, GfxBlockElementModel, GfxControllerIdentifier, type Viewport, } from '@blocksuite/block-std/gfx'; import { Pane } from 'tweakpane'; -import { getSentenceRects, segmentSentences } from './text-utils.js'; -import type { ViewportTurboRendererExtension } from './turbo-renderer.js'; -import type { - ParagraphLayout, - RenderingState, - ViewportLayout, -} from './types.js'; +import { BlockLayoutHandlersIdentifier } from './layout/block-layout-provider'; +import type { ViewportTurboRendererExtension } from './turbo-renderer'; +import type { BlockLayout, RenderingState, ViewportLayout } from './types'; export function syncCanvasSize(canvas: HTMLCanvasElement, host: HTMLElement) { const hostRect = host.getBoundingClientRect(); @@ -28,58 +23,27 @@ export function syncCanvasSize(canvas: HTMLCanvasElement, host: HTMLElement) { canvas.style.pointerEvents = 'none'; } -function getParagraphs(host: EditorHost) { +function getBlockLayouts(host: EditorHost): BlockLayout[] { const gfx = host.std.get(GfxControllerIdentifier); const models = gfx.gfxElements.filter(e => e instanceof GfxBlockElementModel); const components = models .map(model => gfx.view.get(model.id)) .filter(Boolean) as GfxBlockComponent[]; - const paragraphs: ParagraphLayout[] = []; - const selector = '.affine-paragraph-rich-text-wrapper [data-v-text="true"]'; - + const layouts: BlockLayout[] = []; components.forEach(component => { - const paragraphNodes = component.querySelectorAll(selector); - const viewportRecord = component.gfx.viewport.deserializeRecord( - component.dataset.viewportState + const layoutHandlers = host.std.provider.getAll( + BlockLayoutHandlersIdentifier ); - if (!viewportRecord) return; - const { zoom, viewScale } = viewportRecord; - - paragraphNodes.forEach(paragraphNode => { - const paragraph: ParagraphLayout = { - sentences: [], - }; - const sentences = segmentSentences(paragraphNode.textContent || ''); - paragraph.sentences = sentences.map(sentence => { - const sentenceRects = getSentenceRects(paragraphNode, sentence); - const rects = sentenceRects.map(({ text, rect }) => { - const [modelX, modelY] = clientToModelCoord(viewportRecord, [ - rect.x, - rect.y, - ]); - return { - text, - ...rect, - rect: { - x: modelX, - y: modelY, - w: rect.w / zoom / viewScale, - h: rect.h / zoom / viewScale, - }, - }; - }); - return { - text: sentence, - rects, - }; - }); - - paragraphs.push(paragraph); - }); + const handlersArray = Array.from(layoutHandlers.values()); + for (const handler of handlersArray) { + const layout = handler.queryLayout(component); + if (layout) { + layouts.push(layout); + } + } }); - - return paragraphs; + return layouts; } export function getViewportLayout( @@ -93,23 +57,28 @@ export function getViewportLayout( let layoutMaxX = -Infinity; let layoutMaxY = -Infinity; - const paragraphs = getParagraphs(host); - paragraphs.forEach(paragraph => { - paragraph.sentences.forEach(sentence => { - sentence.rects.forEach(r => { - layoutMinX = Math.min(layoutMinX, r.rect.x); - layoutMinY = Math.min(layoutMinY, r.rect.y); - layoutMaxX = Math.max(layoutMaxX, r.rect.x + r.rect.w); - layoutMaxY = Math.max(layoutMaxY, r.rect.y + r.rect.h); - }); - }); + const blockLayouts = getBlockLayouts(host); + + const providers = host.std.provider.getAll(BlockLayoutHandlersIdentifier); + const providersArray = Array.from(providers.values()); + + blockLayouts.forEach(blockLayout => { + const provider = providersArray.find(p => p.blockType === blockLayout.type); + if (!provider) return; + + const { rect } = provider.calculateBound(blockLayout); + + layoutMinX = Math.min(layoutMinX, rect.x); + layoutMinY = Math.min(layoutMinY, rect.y); + layoutMaxX = Math.max(layoutMaxX, rect.x + rect.w); + layoutMaxY = Math.max(layoutMaxY, rect.y + rect.h); }); const layoutModelCoord = [layoutMinX, layoutMinY]; const w = (layoutMaxX - layoutMinX) / zoom / viewport.viewScale; const h = (layoutMaxY - layoutMinY) / zoom / viewport.viewScale; const layout: ViewportLayout = { - paragraphs, + blocks: blockLayouts, rect: { x: layoutModelCoord[0], y: layoutModelCoord[1], @@ -145,6 +114,7 @@ export function debugLog(message: string, state: RenderingState) { } export function paintPlaceholder( + host: EditorHost, canvas: HTMLCanvasElement, layout: ViewportLayout | null, viewport: Viewport @@ -163,30 +133,35 @@ export function paintPlaceholder( 'rgba(160, 160, 160, 0.7)', ]; - layout.paragraphs.forEach((paragraph, paragraphIndex) => { - ctx.fillStyle = colors[paragraphIndex % colors.length]; + const layoutHandlers = host.std.provider.getAll( + BlockLayoutHandlersIdentifier + ); + const handlersArray = Array.from(layoutHandlers.values()); + + layout.blocks.forEach((blockLayout, blockIndex) => { + ctx.fillStyle = colors[blockIndex % colors.length]; const renderedPositions = new Set(); - paragraph.sentences.forEach(sentence => { - sentence.rects.forEach(textRect => { - const x = - ((textRect.rect.x - layout.rect.x) * viewport.zoom + offsetX) * dpr; - const y = - ((textRect.rect.y - layout.rect.y) * viewport.zoom + offsetY) * dpr; - dpr; - const width = textRect.rect.w * viewport.zoom * dpr; - const height = textRect.rect.h * viewport.zoom * dpr; + const handler = handlersArray.find(h => h.blockType === blockLayout.type); + if (!handler) return; + const { subRects } = handler.calculateBound(blockLayout); - const posKey = `${x},${y}`; - if (renderedPositions.has(posKey)) return; - ctx.fillRect(x, y, width, height); - if (width > 10 && height > 5) { - ctx.strokeStyle = 'rgba(150, 150, 150, 0.3)'; - ctx.strokeRect(x, y, width, height); - } + subRects.forEach(rect => { + const x = ((rect.x - layout.rect.x) * viewport.zoom + offsetX) * dpr; + const y = ((rect.y - layout.rect.y) * viewport.zoom + offsetY) * dpr; - renderedPositions.add(posKey); - }); + const width = rect.w * viewport.zoom * dpr; + const height = rect.h * viewport.zoom * dpr; + + const posKey = `${x},${y}`; + if (renderedPositions.has(posKey)) return; + ctx.fillRect(x, y, width, height); + if (width > 10 && height > 5) { + ctx.strokeStyle = 'rgba(150, 150, 150, 0.3)'; + ctx.strokeRect(x, y, width, height); + } + + renderedPositions.add(posKey); }); }); } diff --git a/blocksuite/affine/gfx/turbo-renderer/src/text-utils.ts b/blocksuite/affine/gfx/turbo-renderer/src/text-utils.ts index 1ab2517e74..121e11b51e 100644 --- a/blocksuite/affine/gfx/turbo-renderer/src/text-utils.ts +++ b/blocksuite/affine/gfx/turbo-renderer/src/text-utils.ts @@ -1,4 +1,4 @@ -import type { TextRect } from './types.js'; +import type { TextRect } from './types'; interface WordSegment { text: string; diff --git a/blocksuite/affine/gfx/turbo-renderer/src/turbo-renderer.ts b/blocksuite/affine/gfx/turbo-renderer/src/turbo-renderer.ts index fdbe177d90..c28f7aa201 100644 --- a/blocksuite/affine/gfx/turbo-renderer/src/turbo-renderer.ts +++ b/blocksuite/affine/gfx/turbo-renderer/src/turbo-renderer.ts @@ -1,4 +1,5 @@ import { + ConfigExtensionFactory, LifeCycleWatcher, LifeCycleWatcherIdentifier, StdIdentifier, @@ -8,6 +9,7 @@ import { type GfxViewportElement, } from '@blocksuite/block-std/gfx'; import type { Container, ServiceIdentifier } from '@blocksuite/global/di'; +import { createIdentifier } from '@blocksuite/global/di'; import { DisposableGroup } from '@blocksuite/global/disposable'; import debounce from 'lodash-es/debounce'; @@ -17,13 +19,31 @@ import { initTweakpane, paintPlaceholder, syncCanvasSize, -} from './renderer-utils.js'; -import type { RenderingState, ViewportLayout } from './types.js'; +} from './renderer-utils'; +import type { + BlockPainterConfig, + MessagePaint, + MessageRegisterPainter, + RendererOptions, + RenderingState, + TurboRendererConfig, + ViewportLayout, + WorkerToHostMessage, +} from './types'; const debug = false; // Toggle for debug logs -const zoomThreshold = 1; // With high enough zoom, fallback to DOM rendering -const debounceTime = 1000; // During this period, fallback to DOM -const workerUrl = new URL('./painter.worker.ts', import.meta.url); +const workerUrl = new URL('./painter/painter.worker.ts', import.meta.url); + +const defaultOptions: RendererOptions = { + zoomThreshold: 1, // With high enough zoom, fallback to DOM rendering + debounceTime: 1000, // During this period, fallback to DOM +}; + +export const BlockPainterConfigIdentifier = + createIdentifier('block-painter-config'); + +export const TurboRendererConfigFactory = + ConfigExtensionFactory('viewport-turbo-renderer'); export class ViewportTurboRendererExtension extends LifeCycleWatcher { public state: RenderingState = 'inactive'; @@ -39,6 +59,22 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { di.addImpl(ViewportTurboRendererIdentifier, this, [StdIdentifier]); } + get options(): RendererOptions { + const id = TurboRendererConfigFactory.identifier; + const { options } = this.std.getOptional(id) || {}; + return { + ...defaultOptions, + ...options, + }; + } + + get painterConfigs() { + const painterConfigMap = this.std.provider.getAll( + BlockPainterConfigIdentifier + ); + return Array.from(painterConfigMap.values()); + } + override mounted() { const mountPoint = document.querySelector('.affine-edgeless-viewport'); if (mountPoint) { @@ -80,6 +116,13 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { this.disposables.add( this.std.store.slots.blockUpdated.subscribe(() => this.invalidate()) ); + + const painterConfigs = this.painterConfigs; + const message: MessageRegisterPainter = { + type: 'registerPainter', + data: { painterConfigs }, + }; + this.worker.postMessage(message); } override unmounted() { @@ -116,7 +159,7 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { this.clearCanvas(); // -> pending - if (this.viewport.zoom > zoomThreshold) { + if (this.viewport.zoom > this.options.zoomThreshold) { this.debugLog('Zoom above threshold, falling back to DOM rendering'); this.setState('pending'); this.clearOptimizedBlocks(); @@ -146,7 +189,7 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { debouncedRefresh = debounce(() => { this.refresh().catch(console.error); - }, debounceTime); + }, this.options.debounceTime); invalidate() { this.layoutVersion++; @@ -179,7 +222,7 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { const currentVersion = this.layoutVersion; this.debugLog(`Requesting bitmap painting (version=${currentVersion})`); - this.worker.postMessage({ + const message: MessagePaint = { type: 'paintLayout', data: { layout, @@ -189,9 +232,10 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { zoom: this.viewport.zoom, version: currentVersion, }, - }); + }; + this.worker.postMessage(message); - this.worker.onmessage = (e: MessageEvent) => { + this.worker.onmessage = (e: MessageEvent) => { if (e.data.type === 'bitmapPainted') { if (e.data.version === this.layoutVersion) { this.debugLog( @@ -209,6 +253,12 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { this.setState('pending'); resolve(); } + } else if (e.data.type === 'paintError') { + this.debugLog( + `Paint error: ${e.data.error} for blockType: ${e.data.blockType}` + ); + this.setState('pending'); + resolve(); } }; }); @@ -267,7 +317,8 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { } private canOptimize(): boolean { - const isBelowZoomThreshold = this.viewport.zoom <= zoomThreshold; + const isBelowZoomThreshold = + this.viewport.zoom <= this.options.zoomThreshold; return ( (this.state === 'ready' || this.state === 'zooming') && isBelowZoomThreshold @@ -297,7 +348,12 @@ export class ViewportTurboRendererExtension extends LifeCycleWatcher { } private paintPlaceholder() { - paintPlaceholder(this.canvas, this.layoutCache, this.viewport); + paintPlaceholder( + this.std.host, + this.canvas, + this.layoutCache, + this.viewport + ); } } diff --git a/blocksuite/affine/gfx/turbo-renderer/src/types.ts b/blocksuite/affine/gfx/turbo-renderer/src/types.ts index 964482cddf..15c873897a 100644 --- a/blocksuite/affine/gfx/turbo-renderer/src/types.ts +++ b/blocksuite/affine/gfx/turbo-renderer/src/types.ts @@ -13,17 +13,13 @@ export interface ViewportState { viewportY: number; } -export interface SentenceLayout { - text: string; - rects: TextRect[]; -} - -export interface ParagraphLayout { - sentences: SentenceLayout[]; +export interface BlockLayout extends Record { + type: string; + rect?: Rect; } export interface ViewportLayout { - paragraphs: ParagraphLayout[]; + blocks: BlockLayout[]; rect: Rect; } @@ -46,3 +42,61 @@ export type RenderingState = | 'zooming' | 'rendering' | 'ready'; + +export type MessageBitmapPainted = { + type: 'bitmapPainted'; + bitmap: ImageBitmap; + version: number; +}; + +export type MessagePaintError = { + type: 'paintError'; + error: string; + blockType: string; +}; + +export type WorkerToHostMessage = MessageBitmapPainted | MessagePaintError; + +export type MessagePaint = { + type: 'paintLayout'; + data: { + layout: ViewportLayout; + width: number; + height: number; + dpr: number; + zoom: number; + version: number; + }; +}; + +export interface BlockLayoutPainter { + paint( + ctx: OffscreenCanvasRenderingContext2D, + block: BlockLayout, + layoutBaseX: number, + layoutBaseY: number + ): void; +} + +export interface RendererOptions { + zoomThreshold: number; + debounceTime: number; +} + +export interface BlockPainterConfig { + type: string; + path: string; +} + +export interface TurboRendererConfig { + options?: Partial; +} + +export type MessageRegisterPainter = { + type: 'registerPainter'; + data: { + painterConfigs: BlockPainterConfig[]; + }; +}; + +export type HostToWorkerMessage = MessagePaint | MessageRegisterPainter; diff --git a/blocksuite/integration-test/src/__tests__/utils/renderer-entry.ts b/blocksuite/integration-test/src/__tests__/utils/renderer-entry.ts index ad722e42db..f0e182299b 100644 --- a/blocksuite/integration-test/src/__tests__/utils/renderer-entry.ts +++ b/blocksuite/integration-test/src/__tests__/utils/renderer-entry.ts @@ -1,4 +1,9 @@ import { + ParagraphLayoutHandlerExtension, + ParagraphPaintConfigExtension, +} from '@blocksuite/affine/blocks/paragraph'; +import { + TurboRendererConfigFactory, ViewportTurboRendererExtension, ViewportTurboRendererIdentifier, } from '@blocksuite/affine/gfx/turbo-renderer'; @@ -7,7 +12,17 @@ import { addSampleNotes } from './doc-generator.js'; import { setupEditor } from './setup.js'; async function init() { - setupEditor('edgeless', [ViewportTurboRendererExtension]); + setupEditor('edgeless', [ + ParagraphLayoutHandlerExtension, + ParagraphPaintConfigExtension, + TurboRendererConfigFactory({ + options: { + zoomThreshold: 1, + debounceTime: 1000, + }, + }), + ViewportTurboRendererExtension, + ]); addSampleNotes(doc, 100); doc.load(); diff --git a/packages/frontend/core/src/blocksuite/block-suite-editor/lit-adaper.tsx b/packages/frontend/core/src/blocksuite/block-suite-editor/lit-adaper.tsx index 1fc8004d31..c0abe61229 100644 --- a/packages/frontend/core/src/blocksuite/block-suite-editor/lit-adaper.tsx +++ b/packages/frontend/core/src/blocksuite/block-suite-editor/lit-adaper.tsx @@ -24,7 +24,6 @@ import { MemberSearchService } from '@affine/core/modules/permissions'; import { WorkspaceService } from '@affine/core/modules/workspace'; import track from '@affine/track'; import type { DocTitle } from '@blocksuite/affine/fragments/doc-title'; -import { ViewportTurboRendererExtension } from '@blocksuite/affine/gfx/turbo-renderer'; import type { DocMode } from '@blocksuite/affine/model'; import type { Store } from '@blocksuite/affine/store'; import { @@ -69,6 +68,7 @@ import { type ReferenceReactRenderer, } from '../extensions/reference-renderer'; import { patchSideBarService } from '../extensions/side-bar-service'; +import { patchTurboRendererExtension } from '../extensions/turbo-renderer'; import { patchUserExtensions } from '../extensions/user'; import { patchUserListExtensions } from '../extensions/user-list'; import { BiDirectionalLinkPanel } from './bi-directional-link-panel'; @@ -169,7 +169,7 @@ const usePatchSpecs = (mode: DocMode) => { ] : [], mode === 'edgeless' && enableTurboRenderer - ? [ViewportTurboRendererExtension] + ? patchTurboRendererExtension() : [], ].flat() ); diff --git a/packages/frontend/core/src/blocksuite/extensions/turbo-renderer.ts b/packages/frontend/core/src/blocksuite/extensions/turbo-renderer.ts new file mode 100644 index 0000000000..82dc789204 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/extensions/turbo-renderer.ts @@ -0,0 +1,22 @@ +import { + ParagraphLayoutHandlerExtension, + ParagraphPaintConfigExtension, +} from '@blocksuite/affine/blocks/paragraph'; +import { + TurboRendererConfigFactory, + ViewportTurboRendererExtension, +} from '@blocksuite/affine/gfx/turbo-renderer'; + +export function patchTurboRendererExtension() { + return [ + ParagraphLayoutHandlerExtension, + ParagraphPaintConfigExtension, + TurboRendererConfigFactory({ + options: { + zoomThreshold: 1, + debounceTime: 1000, + }, + }), + ViewportTurboRendererExtension, + ]; +} diff --git a/yarn.lock b/yarn.lock index c0c0de3e62..a5929d2f1b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2593,6 +2593,7 @@ __metadata: resolution: "@blocksuite/affine-block-paragraph@workspace:blocksuite/affine/blocks/block-paragraph" dependencies: "@blocksuite/affine-components": "workspace:*" + "@blocksuite/affine-gfx-turbo-renderer": "workspace:*" "@blocksuite/affine-model": "workspace:*" "@blocksuite/affine-rich-text": "workspace:*" "@blocksuite/affine-shared": "workspace:*" @@ -2900,6 +2901,7 @@ __metadata: dependencies: "@blocksuite/block-std": "workspace:*" "@blocksuite/global": "workspace:*" + "@blocksuite/store": "workspace:*" "@types/lodash-es": "npm:^4.17.12" lodash-es: "npm:^4.17.21" rxjs: "npm:^7.8.1"