fix: update toggle switch active class name

Close [BS-3029](https://linear.app/affine-design/issue/BS-3029/frame-里面的-shape-没办法进入文本编辑模式)
Close [BS-3082](https://linear.app/affine-design/issue/BS-3082/按s切换至shape工具，在白板上点击会创建两个shape)
Close [BS-3091](https://linear.app/affine-design/issue/BS-3082/按s切换至shape工具，在白板上点击会创建两个shape)

## Fix Shape Tool Issues

This PR addresses several issues with the shape and mindmap tools functionality in the editor:

1. **Fix text editing after mode switching**: Resolves an issue where users couldn't edit text in shapes after switching editor modes. The fix ensures the edgeless block is properly retrieved when double-clicking on a shape.

2. **Improve tool switching behavior**: Fixes issues with tool overlays not showing or being repeated when switching between tools. This includes:
   - Properly handling tool overlay visibility
   - Ensuring only one tool is active at a time when using keyboard shortcuts
   - Adding proper cleanup when switching tools

3. **Add comprehensive tests**: Adds new test cases to verify:
   - Shape creation with keyboard shortcuts
   - Shape text editing after mode switching
   - Tool switching behavior with keyboard shortcuts

.devcontainer/Dockerfile

@@ -1,10 +0,0 @@
-FROM mcr.microsoft.com/devcontainers/base:bookworm
-
-USER vscode
-# Install Homebrew For Linux
-RUN /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" && \
-    eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" && \
-    echo "eval \"\$($(brew --prefix)/bin/brew shellenv)\"" >> /home/vscode/.zshrc && \
-    echo "eval \"\$($(brew --prefix)/bin/brew shellenv)\"" >> /home/vscode/.bashrc && \
-    # Install Graphite
-    brew install withgraphite/tap/graphite && gt --version

.devcontainer/build.sh

@@ -1,10 +1,6 @@
 #!/bin/bash
 # This is a script used by the devcontainer to build the project
 
-#Enable yarn
-corepack enable
-corepack prepare yarn@stable --activate
-
 # install dependencies
 yarn install
 

.devcontainer/devcontainer.json

@@ -1,12 +1,16 @@
 // For format details, see https://aka.ms/devcontainer.json.
 {
-  "name": "Debian",
+  "name": "AFFiNE Dev Container",
   "dockerComposeFile": "docker-compose.yml",
   "service": "app",
   "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+  "containerEnv": {
+    "COREPACK_ENABLE_DOWNLOAD_PROMPT": "0"
+  },
   "features": {
     "ghcr.io/devcontainers/features/node:1": {
-      "version": "22"
+      "version": "lts",
+      "installYarnUsingApt": false
     },
     "ghcr.io/devcontainers/features/rust:1": {}
   },
@@ -16,7 +20,7 @@
       "extensions": [
         "ms-playwright.playwright",
         "esbenp.prettier-vscode",
-        "streetsidesoftware.code-spell-checker"
+        "dbaeumer.vscode-eslint"
       ]
     }
   },

.devcontainer/docker-compose.yml

@@ -2,9 +2,7 @@ version: '3.8'
 
 services:
   app:
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: mcr.microsoft.com/devcontainers/base:bookworm
     volumes:
       - ../..:/workspaces:cached
     command: sleep infinity
@@ -24,8 +22,6 @@ services:
       POSTGRES_DB: affine
   redis:
     image: redis
-    ports:
-      - 6379:6379
 
 volumes:
   postgres-data:

.devcontainer/setup-user.sh

@@ -1,9 +1,9 @@
 set -e
 
+npm install -g @withgraphite/graphite-cli@stable
+
 if [ -v GRAPHITE_TOKEN ];then
     gt auth --token $GRAPHITE_TOKEN
 fi
 
-git fetch origin canary:canary --depth=1
-git branch canary -t origin/canary
 gt init --trunk canary

.github/workflows/build-test.yml

@@ -752,8 +752,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3]
-        shardTotal: [3]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
+        shardTotal: [8]
     needs:
       - build-server-native
     services:
@@ -791,6 +791,8 @@ jobs:
         with:
           filters: |
             changed:
+              - 'packages/backend/server/src/plugins/copilot/**'
+              - 'packages/backend/server/tests/copilot.*'
               - 'packages/frontend/core/src/blocksuite/ai/**'
               - 'tests/affine-cloud-copilot/**'
 

.github/workflows/copilot-test.yml

@@ -108,8 +108,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3]
-        shardTotal: [3]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
+        shardTotal: [8]
     needs:
       - build-server-native
     services:

.yarnrc.yml

@@ -12,4 +12,4 @@ npmPublishAccess: public
 
 npmPublishRegistry: "https://registry.npmjs.org"
 
-yarnPath: .yarn/releases/yarn-4.8.1.cjs
+yarnPath: .yarn/releases/yarn-4.9.0.cjs

Cargo.lock

@@ -601,9 +601,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.17"
+version = "1.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a"
+checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c"
 dependencies = [
  "shlex",
 ]
@@ -1997,9 +1997,9 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
 
 [[package]]
 name = "libmimalloc-sys"
-version = "0.1.41"
+version = "0.1.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b20daca3a4ac14dbdc753c5e90fc7b490a48a9131daed3c9a9ced7b2defd37b"
+checksum = "ec9d6fac27761dabcd4ee73571cdb06b7022dc99089acbe5435691edffaac0f4"
 dependencies = [
  "cc",
  "libc",
@@ -2153,9 +2153,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
 [[package]]
 name = "mimalloc"
-version = "0.1.45"
+version = "0.1.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03cb1f88093fe50061ca1195d336ffec131347c7b833db31f9ab62a2d1b7925f"
+checksum = "995942f432bbb4822a7e9c3faa87a695185b0d09273ba85f097b54f4e458f2af"
 dependencies = [
  "libmimalloc-sys",
 ]

blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts

@@ -3769,6 +3769,48 @@ bbb
       });
       expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot);
     });
+
+    test('escapes dollar signs followed by a digit or space and digit', async () => {
+      const markdown =
+        'The price of the T-shirt is $9.15 and the price of the hat is $ 8\n';
+      const blockSnapshot: BlockSnapshot = {
+        type: 'block',
+        id: 'matchesReplaceMap[0]',
+        flavour: 'affine:note',
+        props: {
+          xywh: '[0,0,800,95]',
+          background: DefaultTheme.noteBackgrounColor,
+          index: 'a0',
+          hidden: false,
+          displayMode: NoteDisplayMode.DocAndEdgeless,
+        },
+        children: [
+          {
+            type: 'block',
+            id: 'matchesReplaceMap[1]',
+            flavour: 'affine:paragraph',
+            props: {
+              type: 'text',
+              text: {
+                '$blocksuite:internal:text$': true,
+                delta: [
+                  {
+                    insert:
+                      'The price of the T-shirt is $9.15 and the price of the hat is $ 8',
+                  },
+                ],
+              },
+            },
+            children: [],
+          },
+        ],
+      };
+      const mdAdapter = new MarkdownAdapter(createJob(), provider);
+      const rawBlockSnapshot = await mdAdapter.toBlockSnapshot({
+        file: markdown,
+      });
+      expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot);
+    });
   });
 
   test('reference', async () => {

blocksuite/affine/all/src/adapters/extension.ts

@@ -24,36 +24,46 @@ import { defaultMarkdownPreprocessors } from './markdown/preprocessor';
 import { defaultBlockNotionHtmlAdapterMatchers } from './notion-html/block-matcher';
 import { defaultBlockPlainTextAdapterMatchers } from './plain-text/block-matcher';
 
-export const AdapterFactoryExtensions: ExtensionType[] = [
-  AttachmentAdapterFactoryExtension,
-  ImageAdapterFactoryExtension,
-  MarkdownAdapterFactoryExtension,
-  PlainTextAdapterFactoryExtension,
-  HtmlAdapterFactoryExtension,
-  NotionTextAdapterFactoryExtension,
-  NotionHtmlAdapterFactoryExtension,
-  MixTextAdapterFactoryExtension,
-];
+export function getAdapterFactoryExtensions(): ExtensionType[] {
+  return [
+    AttachmentAdapterFactoryExtension,
+    ImageAdapterFactoryExtension,
+    MarkdownAdapterFactoryExtension,
+    PlainTextAdapterFactoryExtension,
+    HtmlAdapterFactoryExtension,
+    NotionTextAdapterFactoryExtension,
+    NotionHtmlAdapterFactoryExtension,
+    MixTextAdapterFactoryExtension,
+  ];
+}
 
-export const HtmlAdapterExtension: ExtensionType[] = [
-  ...HtmlInlineToDeltaAdapterExtensions,
-  ...defaultBlockHtmlAdapterMatchers,
-  ...InlineDeltaToHtmlAdapterExtensions,
-];
+export function getHtmlAdapterExtensions(): ExtensionType[] {
+  return [
+    ...HtmlInlineToDeltaAdapterExtensions,
+    ...defaultBlockHtmlAdapterMatchers,
+    ...InlineDeltaToHtmlAdapterExtensions,
+  ];
+}
 
-export const MarkdownAdapterExtension: ExtensionType[] = [
-  ...MarkdownInlineToDeltaAdapterExtensions,
-  ...defaultBlockMarkdownAdapterMatchers,
-  ...InlineDeltaToMarkdownAdapterExtensions,
-  ...defaultMarkdownPreprocessors,
-];
+export function getMarkdownAdapterExtensions(): ExtensionType[] {
+  return [
+    ...MarkdownInlineToDeltaAdapterExtensions,
+    ...defaultBlockMarkdownAdapterMatchers,
+    ...InlineDeltaToMarkdownAdapterExtensions,
+    ...defaultMarkdownPreprocessors,
+  ];
+}
 
-export const NotionHtmlAdapterExtension: ExtensionType[] = [
-  ...NotionHtmlInlineToDeltaAdapterExtensions,
-  ...defaultBlockNotionHtmlAdapterMatchers,
-];
+export function getNotionHtmlAdapterExtensions(): ExtensionType[] {
+  return [
+    ...NotionHtmlInlineToDeltaAdapterExtensions,
+    ...defaultBlockNotionHtmlAdapterMatchers,
+  ];
+}
 
-export const PlainTextAdapterExtension: ExtensionType[] = [
-  ...defaultBlockPlainTextAdapterMatchers,
-  ...InlineDeltaToPlainTextAdapterExtensions,
-];
+export function getPlainTextAdapterExtensions(): ExtensionType[] {
+  return [
+    ...defaultBlockPlainTextAdapterMatchers,
+    ...InlineDeltaToPlainTextAdapterExtensions,
+  ];
+}

blocksuite/affine/all/src/extensions/store.ts

@@ -51,11 +51,11 @@ import {
 import type { ExtensionType } from '@blocksuite/store';
 
 import {
-  AdapterFactoryExtensions,
-  HtmlAdapterExtension,
-  MarkdownAdapterExtension,
-  NotionHtmlAdapterExtension,
-  PlainTextAdapterExtension,
+  getAdapterFactoryExtensions,
+  getHtmlAdapterExtensions,
+  getMarkdownAdapterExtensions,
+  getNotionHtmlAdapterExtensions,
+  getPlainTextAdapterExtensions,
 } from '../adapters/extension.js';
 
 export const StoreExtensions: ExtensionType[] = [
@@ -96,11 +96,11 @@ export const StoreExtensions: ExtensionType[] = [
   DatabaseSelectionExtension,
   TableSelectionExtension,
 
-  HtmlAdapterExtension,
-  MarkdownAdapterExtension,
-  NotionHtmlAdapterExtension,
-  PlainTextAdapterExtension,
-  AdapterFactoryExtensions,
+  getHtmlAdapterExtensions(),
+  getMarkdownAdapterExtensions(),
+  getNotionHtmlAdapterExtensions(),
+  getPlainTextAdapterExtensions(),
+  getAdapterFactoryExtensions(),
 
   FeatureFlagService,
   LinkPreviewerService,

blocksuite/affine/blocks/latex/src/adapters/markdown/preprocessor.ts

@@ -3,21 +3,84 @@ import {
   MarkdownPreprocessorExtension,
 } from '@blocksuite/affine-shared/adapters';
 
+function escapeBrackets(text: string) {
+  const pattern =
+    /(```[\S\s]*?```|`.*?`)|\\\[([\S\s]*?[^\\])\\]|\\\((.*?)\\\)/g;
+  return text.replaceAll(
+    pattern,
+    (match, codeBlock, squareBracket, roundBracket) => {
+      if (codeBlock) {
+        return codeBlock;
+      } else if (squareBracket) {
+        return `$$${squareBracket}$$`;
+      } else if (roundBracket) {
+        return `$${roundBracket}$`;
+      }
+      return match;
+    }
+  );
+}
+
+function escapeMhchem(text: string) {
+  return text.replaceAll('$\\ce{', '$\\\\ce{').replaceAll('$\\pu{', '$\\\\pu{');
+}
+
+/**
+ * Preprocess the content to protect code blocks and LaTeX expressions
+ * reference issue: https://github.com/remarkjs/react-markdown/issues/785
+ * reference comment: https://github.com/remarkjs/react-markdown/issues/785#issuecomment-2307567823
+ * @param content - The content to preprocess
+ * @returns The preprocessed content
+ */
+function preprocessLatex(content: string) {
+  // Protect code blocks
+  const codeBlocks: string[] = [];
+  let preprocessedContent = content;
+  preprocessedContent = preprocessedContent.replace(
+    /(```[\s\S]*?```|`[^`\n]+`)/g,
+    (_, code) => {
+      codeBlocks.push(code);
+      return `<<CODE_BLOCK_${codeBlocks.length - 1}>>`;
+    }
+  );
+
+  // Protect existing LaTeX expressions
+  const latexExpressions: string[] = [];
+  preprocessedContent = preprocessedContent.replace(
+    /(\$\$[\s\S]*?\$\$|\\\[[\s\S]*?\\\]|\\\(.*?\\\))/g,
+    match => {
+      latexExpressions.push(match);
+      return `<<LATEX_${latexExpressions.length - 1}>>`;
+    }
+  );
+
+  // Escape dollar signs that are likely currency indicators
+  preprocessedContent = preprocessedContent.replace(/\$(?=\d)/g, '\\$');
+
+  // Restore LaTeX expressions
+  preprocessedContent = preprocessedContent.replace(
+    /<<LATEX_(\d+)>>/g,
+    (_, index) => latexExpressions[parseInt(index)]
+  );
+
+  // Restore code blocks
+  preprocessedContent = preprocessedContent.replace(
+    /<<CODE_BLOCK_(\d+)>>/g,
+    (_, index) => codeBlocks[parseInt(index)]
+  );
+
+  // Apply additional escaping functions
+  preprocessedContent = escapeBrackets(preprocessedContent);
+  preprocessedContent = escapeMhchem(preprocessedContent);
+
+  return preprocessedContent;
+}
+
 const latexPreprocessor: MarkdownAdapterPreprocessor = {
   name: 'latex',
   levels: ['block', 'slice', 'doc'],
   preprocess: content => {
-    // Replace block-level LaTeX delimiters \[ \] with $$ $$
-    const blockProcessedContent = content.replace(
-      /\\\[(.*?)\\\]/gs,
-      (_, equation) => `$$${equation}$$`
-    );
-    // Replace inline LaTeX delimiters \( \) with $ $
-    const inlineProcessedContent = blockProcessedContent.replace(
-      /\\\((.*?)\\\)/gs,
-      (_, equation) => `$${equation}$`
-    );
-    return inlineProcessedContent;
+    return preprocessLatex(content);
   },
 };
 

blocksuite/affine/blocks/list/src/turbo/list-layout-handler.ts

@@ -6,8 +6,9 @@ import {
   segmentSentences,
 } from '@blocksuite/affine-gfx-turbo-renderer';
 import type { Container } from '@blocksuite/global/di';
-import type { GfxBlockComponent } from '@blocksuite/std';
-import { clientToModelCoord } from '@blocksuite/std/gfx';
+import type { EditorHost, GfxBlockComponent } from '@blocksuite/std';
+import { clientToModelCoord, type ViewportRecord } from '@blocksuite/std/gfx';
+import type { BlockModel } from '@blocksuite/store';
 
 import type { ListLayout } from './list-painter.worker';
 
@@ -21,24 +22,27 @@ export class ListLayoutHandlerExtension extends BlockLayoutHandlerExtension<List
     );
   }
 
-  queryLayout(component: GfxBlockComponent): ListLayout | null {
-    // Select all list items within this list block
+  override queryLayout(
+    model: BlockModel,
+    host: EditorHost,
+    viewportRecord: ViewportRecord
+  ): ListLayout | null {
+    const component = host.std.view.getBlock(model.id) as GfxBlockComponent;
+    if (!component) return null;
+
+    // Find the list items within this specific list component
     const listItemSelector =
       '.affine-list-block-container .affine-list-rich-text-wrapper [data-v-text="true"]';
     const listItemNodes = component.querySelectorAll(listItemSelector);
 
     if (listItemNodes.length === 0) return null;
 
-    const viewportRecord = component.gfx.viewport.deserializeRecord(
-      component.dataset.viewportState
-    );
-
-    if (!viewportRecord) return null;
-
     const { zoom, viewScale } = viewportRecord;
     const list: ListLayout = {
       type: 'affine:list',
       items: [],
+      blockId: model.id,
+      rect: { x: 0, y: 0, w: 0, h: 0 },
     };
 
     listItemNodes.forEach(listItemNode => {

blocksuite/affine/blocks/list/src/turbo/list-painter.worker.ts

@@ -77,10 +77,15 @@ class ListLayoutPainter implements BlockLayoutPainter {
       return;
     }
 
-    if (!isListLayout(layout)) return;
+    if (!isListLayout(layout)) {
+      console.warn(
+        'Expected list layout but received different format:',
+        layout
+      );
+      return;
+    }
 
     const renderedPositions = new Set<string>();
-
     layout.items.forEach(item => {
       const fontSize = item.fontSize;
       const baselineY = getBaseline(fontSize);

blocksuite/affine/blocks/note/package.json

@@ -14,6 +14,7 @@
     "@blocksuite/affine-block-surface": "workspace:*",
     "@blocksuite/affine-components": "workspace:*",
     "@blocksuite/affine-fragment-doc-title": "workspace:*",
+    "@blocksuite/affine-gfx-turbo-renderer": "workspace:*",
     "@blocksuite/affine-inline-preset": "workspace:*",
     "@blocksuite/affine-model": "workspace:*",
     "@blocksuite/affine-rich-text": "workspace:*",
@@ -37,7 +38,8 @@
   },
   "exports": {
     ".": "./src/index.ts",
-    "./effects": "./src/effects.ts"
+    "./effects": "./src/effects.ts",
+    "./turbo-painter": "./src/turbo/note-painter.worker.ts"
   },
   "files": [
     "src",

blocksuite/affine/blocks/note/src/index.ts

@@ -6,3 +6,5 @@ export * from './edgeless-clipboard-config';
 export * from './note-block';
 export * from './note-edgeless-block';
 export * from './note-spec';
+export * from './turbo/note-layout-handler';
+export * from './turbo/note-painter.worker';

blocksuite/affine/blocks/note/src/turbo/note-layout-handler.ts

@@ -0,0 +1,81 @@
+import type { Rect } from '@blocksuite/affine-gfx-turbo-renderer';
+import {
+  BlockLayoutHandlerExtension,
+  BlockLayoutHandlersIdentifier,
+} from '@blocksuite/affine-gfx-turbo-renderer';
+import {
+  ColorScheme,
+  type NoteBlockModel,
+  resolveColor,
+} from '@blocksuite/affine-model';
+import type { Container } from '@blocksuite/global/di';
+import type { EditorHost, GfxBlockComponent } from '@blocksuite/std';
+import { clientToModelCoord, type ViewportRecord } from '@blocksuite/std/gfx';
+import type { BlockModel } from '@blocksuite/store';
+
+import type { NoteLayout } from './note-painter.worker';
+
+export class NoteLayoutHandlerExtension extends BlockLayoutHandlerExtension<NoteLayout> {
+  readonly blockType = 'affine:note';
+
+  static override setup(di: Container) {
+    di.addImpl(
+      BlockLayoutHandlersIdentifier('note'),
+      NoteLayoutHandlerExtension
+    );
+  }
+
+  override queryLayout(
+    model: BlockModel,
+    host: EditorHost,
+    viewportRecord: ViewportRecord
+  ): NoteLayout | null {
+    const component = host.std.view.getBlock(model.id) as GfxBlockComponent;
+    if (!component) return null;
+
+    // Get the note container element
+    const noteContainer = component.querySelector('.affine-note-mask');
+    if (!noteContainer) return null;
+
+    // Get the bounding client rect of the note container
+    const clientRect = noteContainer.getBoundingClientRect();
+
+    // Convert client coordinates to model coordinates
+    const [modelX, modelY] = clientToModelCoord(viewportRecord, [
+      clientRect.x,
+      clientRect.y,
+    ]);
+
+    const { zoom, viewScale } = viewportRecord;
+
+    // Cast model to NoteBlockModel to access background property from props
+    const noteModel = model as NoteBlockModel;
+    const background = noteModel.props.background;
+    // Resolve the color to a string
+    const backgroundString = resolveColor(background, ColorScheme.Light);
+
+    // Create the note layout object
+    const noteLayout: NoteLayout = {
+      type: 'affine:note',
+      blockId: model.id,
+      rect: {
+        x: modelX,
+        y: modelY,
+        w: clientRect.width / zoom / viewScale,
+        h: clientRect.height / zoom / viewScale,
+      },
+      background: backgroundString,
+    };
+
+    return noteLayout;
+  }
+
+  calculateBound(layout: NoteLayout) {
+    const rect: Rect = layout.rect;
+
+    return {
+      rect,
+      subRects: [rect], // The note is represented by a single rectangle
+    };
+  }
+}

blocksuite/affine/blocks/note/src/turbo/note-painter.worker.ts

@@ -0,0 +1,49 @@
+import type {
+  BlockLayout,
+  BlockLayoutPainter,
+  WorkerToHostMessage,
+} from '@blocksuite/affine-gfx-turbo-renderer';
+import { BlockLayoutPainterExtension } from '@blocksuite/affine-gfx-turbo-renderer/painter';
+
+export interface NoteLayout extends BlockLayout {
+  type: 'affine:note';
+  background?: string;
+}
+
+function isNoteLayout(layout: BlockLayout): layout is NoteLayout {
+  return layout.type === 'affine:note';
+}
+
+class NoteLayoutPainter implements BlockLayoutPainter {
+  paint(
+    ctx: OffscreenCanvasRenderingContext2D,
+    layout: BlockLayout,
+    layoutBaseX: number,
+    layoutBaseY: number
+  ): void {
+    if (!isNoteLayout(layout)) {
+      const message: WorkerToHostMessage = {
+        type: 'paintError',
+        error: 'Invalid layout format',
+        blockType: 'affine:note',
+      };
+      self.postMessage(message);
+      return;
+    }
+
+    // Get the layout rectangle
+    const x = layout.rect.x - layoutBaseX;
+    const y = layout.rect.y - layoutBaseY;
+    const width = layout.rect.w;
+    const height = layout.rect.h;
+
+    ctx.fillStyle = layout.background || 'rgb(255, 255, 255)';
+    ctx.fillRect(x, y, width, height);
+    ctx.strokeRect(x, y, width, height);
+  }
+}
+
+export const NoteLayoutPainterExtension = BlockLayoutPainterExtension(
+  'affine:note',
+  NoteLayoutPainter
+);

blocksuite/affine/blocks/note/tsconfig.json

@@ -11,6 +11,7 @@
     { "path": "../surface" },
     { "path": "../../components" },
     { "path": "../../fragments/doc-title" },
+    { "path": "../../gfx/turbo-renderer" },
     { "path": "../../inlines/preset" },
     { "path": "../../model" },
     { "path": "../../rich-text" },

blocksuite/affine/blocks/paragraph/src/adapters/markdown.ts

@@ -71,6 +71,7 @@ export const paragraphBlockMarkdownAdapterMatcher: BlockMarkdownAdapterMatcher =
                 'children'
               )
               .closeNode();
+            walkerContext.skipAllChildren();
             break;
           }
           case 'heading': {

blocksuite/affine/blocks/paragraph/src/turbo/paragraph-layout-handler.ts

@@ -6,8 +6,9 @@ import {
   segmentSentences,
 } from '@blocksuite/affine-gfx-turbo-renderer';
 import type { Container } from '@blocksuite/global/di';
-import type { GfxBlockComponent } from '@blocksuite/std';
-import { clientToModelCoord } from '@blocksuite/std/gfx';
+import type { EditorHost, GfxBlockComponent } from '@blocksuite/std';
+import { clientToModelCoord, type ViewportRecord } from '@blocksuite/std/gfx';
+import type { BlockModel } from '@blocksuite/store';
 
 import type { ParagraphLayout } from './paragraph-painter.worker';
 
@@ -21,58 +22,54 @@ export class ParagraphLayoutHandlerExtension extends BlockLayoutHandlerExtension
     );
   }
 
-  queryLayout(component: GfxBlockComponent): ParagraphLayout | null {
+  override queryLayout(
+    model: BlockModel,
+    host: EditorHost,
+    viewportRecord: ViewportRecord
+  ): ParagraphLayout | null {
+    const component = host.std.view.getBlock(model.id) as GfxBlockComponent;
     const paragraphSelector =
       '.affine-paragraph-rich-text-wrapper [data-v-text="true"]';
-    const paragraphNodes = component.querySelectorAll(paragraphSelector);
-
-    if (paragraphNodes.length === 0) return null;
-
-    const viewportRecord = component.gfx.viewport.deserializeRecord(
-      component.dataset.viewportState
-    );
-
-    if (!viewportRecord) return null;
+    const paragraphNode = component.querySelector(paragraphSelector);
+    if (!paragraphNode) return null;
 
     const { zoom, viewScale } = viewportRecord;
     const paragraph: ParagraphLayout = {
       type: 'affine:paragraph',
       sentences: [],
+      blockId: model.id,
+      rect: { x: 0, y: 0, w: 0, h: 0 },
     };
 
-    paragraphNodes.forEach(paragraphNode => {
-      const computedStyle = window.getComputedStyle(paragraphNode);
-      const fontSizeStr = computedStyle.fontSize;
-      const fontSize = parseInt(fontSizeStr);
+    const computedStyle = window.getComputedStyle(paragraphNode);
+    const fontSizeStr = computedStyle.fontSize;
+    const fontSize = parseInt(fontSizeStr);
 
-      const sentences = segmentSentences(paragraphNode.textContent || '');
-      const sentenceLayouts = sentences.map(sentence => {
-        const sentenceRects = getSentenceRects(paragraphNode, sentence);
-        const rects = sentenceRects.map(({ text, rect }) => {
-          const [modelX, modelY] = clientToModelCoord(viewportRecord, [
-            rect.x,
-            rect.y,
-          ]);
-          return {
-            text,
-            rect: {
-              x: modelX,
-              y: modelY,
-              w: rect.w / zoom / viewScale,
-              h: rect.h / zoom / viewScale,
-            },
-          };
-        });
+    const sentences = segmentSentences(paragraphNode.textContent || '');
+    const sentenceLayouts = sentences.map(sentence => {
+      const sentenceRects = getSentenceRects(paragraphNode, sentence);
+      const rects = sentenceRects.map(({ text, rect }) => {
+        const [modelX, modelY] = clientToModelCoord(viewportRecord, [
+          rect.x,
+          rect.y,
+        ]);
         return {
-          text: sentence,
-          rects,
-          fontSize,
+          text,
+          rect: {
+            x: modelX,
+            y: modelY,
+            w: rect.w / zoom / viewScale,
+            h: rect.h / zoom / viewScale,
+          },
         };
       });
-
-      paragraph.sentences.push(...sentenceLayouts);
+      return {
+        text: sentence,
+        rects,
+        fontSize,
+      };
     });
-
+    paragraph.sentences.push(...sentenceLayouts);
     return paragraph;
   }
 

blocksuite/affine/blocks/paragraph/src/turbo/paragraph-painter.worker.ts

@@ -73,10 +73,15 @@ class ParagraphLayoutPainter implements BlockLayoutPainter {
       return;
     }
 
-    if (!isParagraphLayout(layout)) return; // cast to ParagraphLayout
+    if (!isParagraphLayout(layout)) {
+      console.warn(
+        'Expected paragraph layout but received different format:',
+        layout
+      );
+      return;
+    }
 
     const renderedPositions = new Set<string>();
-
     layout.sentences.forEach(sentence => {
       const fontSize = sentence.fontSize;
       const baselineY = getBaseline(fontSize);

blocksuite/affine/blocks/root/src/clipboard/readonly-clipboard.ts

@@ -34,12 +34,6 @@ const NotionClipboardConfig = ClipboardAdapterConfigExtension({
   priority: 95,
 });
 
-const HtmlClipboardConfig = ClipboardAdapterConfigExtension({
-  mimeType: 'text/html',
-  adapter: HtmlAdapter,
-  priority: 90,
-});
-
 const imageClipboardConfigs = [
   'image/apng',
   'image/avif',
@@ -52,14 +46,20 @@ const imageClipboardConfigs = [
   return ClipboardAdapterConfigExtension({
     mimeType,
     adapter: ImageAdapter,
-    priority: 80,
+    priority: 85,
   });
 });
 
 const PlainTextClipboardConfig = ClipboardAdapterConfigExtension({
   mimeType: 'text/plain',
   adapter: MixTextAdapter,
-  priority: 70,
+  priority: 80,
+});
+
+const HtmlClipboardConfig = ClipboardAdapterConfigExtension({
+  mimeType: 'text/html',
+  adapter: HtmlAdapter,
+  priority: 75,
 });
 
 const AttachmentClipboardConfig = ClipboardAdapterConfigExtension({

blocksuite/affine/blocks/root/src/common-specs/index.ts

@@ -23,6 +23,7 @@ import {
 } from '@blocksuite/affine-gfx-text';
 import { NoteBlockSchema } from '@blocksuite/affine-model';
 import {
+  AutoClearSelectionService,
   DNDAPIExtension,
   DocModeService,
   EmbedOptionService,
@@ -43,12 +44,7 @@ import { RootBlockAdapterExtensions } from '../adapters/extension';
 import { clipboardConfigs } from '../clipboard';
 import { builtinToolbarConfig } from '../configs/toolbar';
 import { fallbackKeymap } from '../keyboard/keymap';
-import {
-  innerModalWidget,
-  linkedDocWidget,
-  modalWidget,
-  viewportOverlayWidget,
-} from './widgets';
+import { linkedDocWidget, modalWidget, viewportOverlayWidget } from './widgets';
 
 /**
  * Why do we add these extensions into CommonSpecs?
@@ -82,12 +78,12 @@ export const CommonSpecs: ExtensionType[] = [
   DNDAPIExtension,
   FileDropExtension,
   ToolbarRegistryExtension,
+  AutoClearSelectionService,
   ...RootBlockAdapterExtensions,
   ...clipboardConfigs,
   ...EdgelessElementViews,
   ...EdgelessElementRendererExtension,
   modalWidget,
-  innerModalWidget,
   SlashMenuExtension,
   linkedDocWidget,
   dragHandleWidget,

blocksuite/affine/blocks/root/src/common-specs/widgets.ts

@@ -1,7 +1,6 @@
 import { WidgetViewExtension } from '@blocksuite/std';
 import { literal, unsafeStatic } from 'lit/static-html.js';
 
-import { AFFINE_INNER_MODAL_WIDGET } from '../widgets/inner-modal/inner-modal.js';
 import { AFFINE_LINKED_DOC_WIDGET } from '../widgets/linked-doc/config.js';
 import { AFFINE_MODAL_WIDGET } from '../widgets/modal/modal.js';
 import { AFFINE_VIEWPORT_OVERLAY_WIDGET } from '../widgets/viewport-overlay/viewport-overlay.js';
@@ -11,11 +10,6 @@ export const modalWidget = WidgetViewExtension(
   AFFINE_MODAL_WIDGET,
   literal`${unsafeStatic(AFFINE_MODAL_WIDGET)}`
 );
-export const innerModalWidget = WidgetViewExtension(
-  'affine:page',
-  AFFINE_INNER_MODAL_WIDGET,
-  literal`${unsafeStatic(AFFINE_INNER_MODAL_WIDGET)}`
-);
 export const linkedDocWidget = WidgetViewExtension(
   'affine:page',
   AFFINE_LINKED_DOC_WIDGET,

blocksuite/affine/blocks/root/src/effects.ts

@@ -43,10 +43,6 @@ import {
 } from './widgets/edgeless-zoom-toolbar/index.js';
 import { ZoomBarToggleButton } from './widgets/edgeless-zoom-toolbar/zoom-bar-toggle-button.js';
 import { EdgelessZoomToolbar } from './widgets/edgeless-zoom-toolbar/zoom-toolbar.js';
-import {
-  AFFINE_INNER_MODAL_WIDGET,
-  AffineInnerModalWidget,
-} from './widgets/inner-modal/inner-modal.js';
 import { effects as widgetMobileToolbarEffects } from './widgets/keyboard-toolbar/effects.js';
 import { effects as widgetLinkedDocEffects } from './widgets/linked-doc/effects.js';
 import { Loader } from './widgets/linked-doc/import-doc/loader.js';
@@ -97,7 +93,6 @@ function registerGfxEffects() {
 }
 
 function registerWidgets() {
-  customElements.define(AFFINE_INNER_MODAL_WIDGET, AffineInnerModalWidget);
   customElements.define(AFFINE_MODAL_WIDGET, AffineModalWidget);
   customElements.define(
     AFFINE_PAGE_DRAGGING_AREA_WIDGET,
@@ -185,6 +180,5 @@ declare global {
     'edgeless-zoom-toolbar': EdgelessZoomToolbar;
 
     [AFFINE_EDGELESS_ZOOM_TOOLBAR_WIDGET]: AffineEdgelessZoomToolbarWidget;
-    [AFFINE_INNER_MODAL_WIDGET]: AffineInnerModalWidget;
   }
 }

blocksuite/affine/blocks/root/src/types.ts

@@ -10,7 +10,6 @@ import type { EdgelessRootBlockComponent } from './edgeless/edgeless-root-block.
 import type { PageRootBlockComponent } from './page/page-root-block.js';
 import type { AFFINE_EDGELESS_ZOOM_TOOLBAR_WIDGET } from './widgets/edgeless-zoom-toolbar/index.js';
 import type { AFFINE_KEYBOARD_TOOLBAR_WIDGET } from './widgets/index.js';
-import type { AFFINE_INNER_MODAL_WIDGET } from './widgets/inner-modal/inner-modal.js';
 import type { AFFINE_LINKED_DOC_WIDGET } from './widgets/linked-doc/config.js';
 import type { AFFINE_MODAL_WIDGET } from './widgets/modal/modal.js';
 import type { AFFINE_PAGE_DRAGGING_AREA_WIDGET } from './widgets/page-dragging-area/page-dragging-area.js';
@@ -19,7 +18,6 @@ import type { AFFINE_VIEWPORT_OVERLAY_WIDGET } from './widgets/viewport-overlay/
 export type PageRootBlockWidgetName =
   | typeof AFFINE_KEYBOARD_TOOLBAR_WIDGET
   | typeof AFFINE_MODAL_WIDGET
-  | typeof AFFINE_INNER_MODAL_WIDGET
   | typeof AFFINE_SLASH_MENU_WIDGET
   | typeof AFFINE_LINKED_DOC_WIDGET
   | typeof AFFINE_PAGE_DRAGGING_AREA_WIDGET
@@ -29,7 +27,6 @@ export type PageRootBlockWidgetName =
 
 export type EdgelessRootBlockWidgetName =
   | typeof AFFINE_MODAL_WIDGET
-  | typeof AFFINE_INNER_MODAL_WIDGET
   | typeof AFFINE_SLASH_MENU_WIDGET
   | typeof AFFINE_LINKED_DOC_WIDGET
   | typeof AFFINE_DRAG_HANDLE_WIDGET

blocksuite/affine/blocks/root/src/widgets/index.ts

@@ -1,5 +1,4 @@
 export { AffineEdgelessZoomToolbarWidget } from './edgeless-zoom-toolbar/index.js';
-export { AffineInnerModalWidget } from './inner-modal/inner-modal.js';
 export * from './keyboard-toolbar/index.js';
 export {
   type LinkedMenuAction,

blocksuite/affine/blocks/root/src/widgets/inner-modal/inner-modal.ts

@@ -1,58 +0,0 @@
-import { WidgetComponent } from '@blocksuite/std';
-import {
-  autoUpdate,
-  computePosition,
-  type FloatingElement,
-  type ReferenceElement,
-  size,
-} from '@floating-ui/dom';
-import { nothing } from 'lit';
-
-export const AFFINE_INNER_MODAL_WIDGET = 'affine-inner-modal-widget';
-
-export class AffineInnerModalWidget extends WidgetComponent {
-  private _getTarget?: () => ReferenceElement;
-
-  get target(): ReferenceElement {
-    if (this._getTarget) {
-      return this._getTarget();
-    }
-    return document.body;
-  }
-
-  open(
-    modal: FloatingElement,
-    ops: { onClose?: () => void }
-  ): { close(): void } {
-    const cancel = autoUpdate(this.target, modal, () => {
-      computePosition(this.target, modal, {
-        middleware: [
-          size({
-            apply: ({ rects }) => {
-              Object.assign(modal.style, {
-                left: `${rects.reference.x}px`,
-                top: `${rects.reference.y}px`,
-                width: `${rects.reference.width}px`,
-                height: `${rects.reference.height}px`,
-              });
-            },
-          }),
-        ],
-      }).catch(console.error);
-    });
-    const close = () => {
-      modal.remove();
-      ops.onClose?.();
-      cancel();
-    };
-    return { close };
-  }
-
-  override render() {
-    return nothing;
-  }
-
-  setTarget(fn: () => ReferenceElement) {
-    this._getTarget = fn;
-  }
-}

blocksuite/affine/blocks/surface-ref/src/surface-ref-block.ts

@@ -49,7 +49,7 @@ import {
 import type { BaseSelection, Store } from '@blocksuite/store';
 import { effect, signal } from '@preact/signals-core';
 import { css, html, nothing } from 'lit';
-import { query, state } from 'lit/decorators.js';
+import { query } from 'lit/decorators.js';
 import { classMap } from 'lit/directives/class-map.js';
 import { guard } from 'lit/directives/guard.js';
 import { styleMap } from 'lit/directives/style-map.js';
@@ -103,17 +103,12 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
       margin: 0 auto;
       position: relative;
       overflow: hidden;
-      pointer-events: none;
       user-select: none;
     }
 
     .ref-viewport-event-mask {
       position: absolute;
-      top: 0;
-      left: 0;
-      width: 100%;
-      height: 100%;
-      pointer-events: auto;
+      inset: 0;
     }
   `;
 
@@ -139,11 +134,11 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
     return this._referencedModel;
   }
 
-  private _focusBlock() {
+  private readonly _handleClick = () => {
     this.selection.update(() => {
       return [this.selection.create(BlockSelection, { blockId: this.blockId })];
     });
-  }
+  };
 
   private _initHotkey() {
     const selection = this.host.selection;
@@ -178,7 +173,7 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
 
     this.bindHotKey({
       Enter: () => {
-        if (!this._focused) return;
+        if (!this.selected$.value) return;
         addParagraph();
         return true;
       },
@@ -260,17 +255,6 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
     }
   }
 
-  private _initSelection() {
-    const selection = this.host.selection;
-    this._disposables.add(
-      selection.slots.changed.subscribe(selList => {
-        this._focused = selList.some(
-          sel => sel.blockId === this.blockId && sel.is(BlockSelection)
-        );
-      })
-    );
-  }
-
   private _initViewport() {
     const refreshViewport = () => {
       if (!this._referenceXYWH$.value) return;
@@ -436,7 +420,6 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
     this._initHotkey();
     this._initViewport();
     this._initReferencedModel();
-    this._initSelection();
   }
 
   override firstUpdated() {
@@ -462,10 +445,10 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
       <div
         class=${classMap({
           'affine-surface-ref': true,
-          focused: this._focused,
+          focused: this.selected$.value,
         })}
         data-theme=${edgelessTheme}
-        @click=${this._focusBlock}
+        @click=${this._handleClick}
       >
         ${content}
       </div>
@@ -488,9 +471,6 @@ export class SurfaceRefBlockComponent extends BlockComponent<SurfaceRefBlockMode
     this.std.get(DocModeProvider).setEditorMode('edgeless');
   }
 
-  @state()
-  private accessor _focused: boolean = false;
-
   @query('.affine-surface-ref')
   accessor hoverableContainer!: HTMLDivElement;
 

blocksuite/affine/blocks/surface/src/consts.ts

@@ -17,8 +17,12 @@ export interface IModelCoord {
   y: number;
 }
 
+// TODO(@L-Sun): we should remove this list when refactor the pointerOut event to pointerLeave,
+// since the previous will be triggered when the pointer move to the area of the its children elements
+// see: https://developer.mozilla.org/en-US/docs/Web/API/Element/pointerout_event
 export const EXCLUDING_MOUSE_OUT_CLASS_LIST = [
   'affine-note-mask',
   'edgeless-block-portal-note',
   'affine-block-children-container',
+  'edgeless-container',
 ];

blocksuite/affine/blocks/surface/src/managers/connector-manager.ts

@@ -12,7 +12,6 @@ import type { IBound, IVec, IVec3 } from '@blocksuite/global/gfx';
 import {
   almostEqual,
   Bound,
-  clamp,
   getBezierCurveBoundingBox,
   getBezierParameters,
   getBoundFromPoints,
@@ -85,7 +84,7 @@ export function calculateNearestLocation(
 ) {
   const { x, y, w, h } = bounds;
   return locations
-    .map(offset => [x + offset[0] * w, y + offset[1] * h] as IVec)
+    .map<IVec>(offset => [x + offset[0] * w, y + offset[1] * h])
     .map(point => getPointFromBoundsWithRotation(bounds, point))
     .reduce(
       (prev, curr, index) => {
@@ -99,7 +98,7 @@ export function calculateNearestLocation(
         return prev;
       },
       [...locations[0]]
-    ) as IVec;
+    );
 }
 
 function rBound(ele: GfxModel, anti = false): IBound {
@@ -139,21 +138,19 @@ export function getAnchors(ele: GfxModel) {
   const anchors: { point: PointLocation; coord: IVec }[] = [];
   const rotate = ele.rotate;
 
-  [
-    [bound.center[0], bound.y - offset],
-    [bound.center[0], bound.maxY + offset],
-    [bound.x - offset, bound.center[1]],
-    [bound.maxX + offset, bound.center[1]],
-  ]
-    .map(vec =>
-      getPointFromBoundsWithRotation({ ...bound, rotate }, vec as IVec)
-    )
+  (
+    [
+      [bound.center[0], bound.y - offset],
+      [bound.center[0], bound.maxY + offset],
+      [bound.x - offset, bound.center[1]],
+      [bound.maxX + offset, bound.center[1]],
+    ] satisfies IVec[]
+  )
+    .map(vec => getPointFromBoundsWithRotation({ ...bound, rotate }, vec))
     .forEach(vec => {
-      const rst = ele.getLineIntersections(bound.center as IVec, vec as IVec);
-      if (!rst) {
-        console.error(`Failed to get line intersections for ${ele.id}`);
-        return;
-      }
+      const rst = ele.getLineIntersections(bound.center, vec);
+      if (!rst) return;
+
       const originPoint = getPointFromBoundsWithRotation(
         { ...bound, rotate: -rotate },
         rst[0]
@@ -164,7 +161,7 @@ export function getAnchors(ele: GfxModel) {
 }
 
 function getConnectableRelativePosition(connectable: GfxModel, position: IVec) {
-  const location = connectable.getRelativePointLocation(position as IVec);
+  const location = connectable.getRelativePointLocation(position);
   if (isVecZero(Vec.sub(position, [0, 0.5])))
     location.tangent = Vec.rot([0, -1], toRadian(connectable.rotate));
   else if (isVecZero(Vec.sub(position, [1, 0.5])))
@@ -184,7 +181,11 @@ export function getNearestConnectableAnchor(ele: Connectable, point: IVec) {
   );
 }
 
-function closestPoint(points: PointLocation[], point: IVec) {
+function closestPoint(
+  points: PointLocation[],
+  point: IVec
+): PointLocation | null {
+  if (points.length === 0) return null;
   const rst = points.map(p => ({ p, d: Vec.dist(p, point) }));
   rst.sort((a, b) => a.d - b.d);
   return rst[0].p;
@@ -245,7 +246,7 @@ function filterConnectablePoints<T extends IVec3 | IVec>(
 ): T[] {
   return points.filter(point => {
     if (!bound) return true;
-    return !bound.isPointInBound(point as IVec);
+    return !bound.isPointInBound([point[0], point[1]]);
   });
 }
 
@@ -368,15 +369,17 @@ function pushGapMidPoint(
       bound.lowerLine,
       bound2.upperLine,
       bound2.lowerLine,
-    ].map(line => {
-      return lineIntersects(
-        point as unknown as IVec,
-        [point[0], point[1] + 1],
-        line[0],
-        line[1],
-        true
-      ) as IVec;
-    });
+    ]
+      .map(line => {
+        return lineIntersects(
+          [point[0], point[1]],
+          [point[0], point[1] + 1],
+          line[0],
+          line[1],
+          true
+        );
+      })
+      .filter(p => p !== null);
     rst.sort((a, b) => a[1] - b[1]);
     const midPoint = Vec.lrp(rst[1], rst[2], 0.5);
     pushWithPriority(points, [midPoint], 6);
@@ -399,15 +402,17 @@ function pushGapMidPoint(
       bound.rightLine,
       bound2.leftLine,
       bound2.rightLine,
-    ].map(line => {
-      return lineIntersects(
-        point as unknown as IVec,
-        [point[0] + 1, point[1]],
-        line[0],
-        line[1],
-        true
-      ) as IVec;
-    });
+    ]
+      .map(line => {
+        return lineIntersects(
+          [point[0], point[1]],
+          [point[0] + 1, point[1]],
+          line[0],
+          line[1],
+          true
+        );
+      })
+      .filter(p => p !== null);
     rst.sort((a, b) => a[0] - b[0]);
     const midPoint = Vec.lrp(rst[1], rst[2], 0.5);
     pushWithPriority(points, [midPoint], 6);
@@ -480,14 +485,14 @@ function getConnectablePoints(
   expandEndBound: Bound | null
 ) {
   const lineBound = Bound.fromPoints([
-    startPoint,
-    endPoint,
-  ] as unknown[] as IVec[]);
+    [startPoint[0], startPoint[1]],
+    [endPoint[0], endPoint[1]],
+  ]);
   const outerBound =
     expandStartBound &&
     expandEndBound &&
     expandStartBound.unite(expandEndBound);
-  let points = [nextStartPoint, lastEndPoint] as IVec3[];
+  let points = [nextStartPoint, lastEndPoint];
   pushWithPriority(points, lineBound.getVerticesAndMidpoints());
 
   if (!startBound || !endBound) {
@@ -534,7 +539,7 @@ function getConnectablePoints(
     pushWithPriority(points, expandStartBound.getVerticesAndMidpoints());
     pushWithPriority(
       points,
-      expandStartBound.include(lastEndPoint as unknown as IVec).points
+      expandStartBound.include([lastEndPoint[0], lastEndPoint[1]]).points
     );
   }
 
@@ -542,7 +547,7 @@ function getConnectablePoints(
     pushWithPriority(points, expandEndBound.getVerticesAndMidpoints());
     pushWithPriority(
       points,
-      expandEndBound.include(nextStartPoint as unknown as IVec).points
+      expandEndBound.include([nextStartPoint[0], nextStartPoint[1]]).points
     );
   }
 
@@ -561,7 +566,7 @@ function getConnectablePoints(
         almostEqual(item[0], point[0], 0.02) &&
         almostEqual(item[1], point[1], 0.02)
     );
-  }) as IVec3[];
+  });
   if (!startEnds[0] || !startEnds[1]) {
     throw new BlockSuiteError(
       BlockSuiteError.ErrorCode.ValueNotExists,
@@ -603,7 +608,9 @@ function mergePath(points: IVec[] | IVec3[]) {
       continue;
     result.push([cur[0], cur[1]]);
   }
-  result.push(last(points as IVec[]) as IVec);
+  if (points.length !== 0) {
+    result.push([points[points.length - 1][0], points[points.length - 1][1]]);
+  }
   for (let i = 0; i < result.length - 1; i++) {
     const cur = result[i];
     const next = result[i + 1];
@@ -687,7 +694,7 @@ function getNextPoint(
   offsetW = 10,
   offsetH = 10
 ) {
-  const result: IVec = Array.from(point) as IVec;
+  const result: IVec = [point[0], point[1]];
   if (almostEqual(bound.x, result[0])) result[0] -= offsetX;
   else if (almostEqual(bound.y, result[1])) result[1] -= offsetY;
   else if (almostEqual(bound.maxX, result[0])) result[0] += offsetW;
@@ -993,7 +1000,7 @@ export class ConnectionOverlay extends Overlay {
           this.highlightPoint = anchor.point;
           result = {
             id: connectable.id,
-            position: anchor.coord as IVec,
+            position: anchor.coord,
           };
         }
       }
@@ -1001,7 +1008,7 @@ export class ConnectionOverlay extends Overlay {
       if (shortestDistance < 8 && result) break;
 
       // if not, check if closes to bound
-      const nearestPoint = connectable.getNearestPoint(point as IVec) as IVec;
+      const nearestPoint = connectable.getNearestPoint(point);
 
       if (Vec.dist(nearestPoint, point) < 8) {
         this.highlightPoint = nearestPoint;
@@ -1013,9 +1020,7 @@ export class ConnectionOverlay extends Overlay {
         target.push(connectable);
         result = {
           id: connectable.id,
-          position: bound
-            .toRelative(originPoint)
-            .map(n => clamp(n, 0, 1)) as IVec,
+          position: Vec.clampV(bound.toRelative(originPoint), 0, 1),
         };
       }
 
@@ -1048,7 +1053,7 @@ export class ConnectionOverlay extends Overlay {
     // at last, if not, just return the point
     if (!result) {
       result = {
-        position: point as IVec,
+        position: point,
       };
     }
 
@@ -1383,7 +1388,7 @@ export class ConnectorPathGenerator extends PathGenerator {
       const eb = Bound.deserialize(end.xywh);
       const startPoint = getNearestConnectableAnchor(start, eb.center);
       const endPoint = getNearestConnectableAnchor(end, sb.center);
-      return [startPoint, endPoint];
+      return (startPoint && endPoint && [startPoint, endPoint]) ?? [];
     } else {
       const endPoint = this._getConnectionPoint(connector, 'target');
       const startPoint = this._getConnectionPoint(connector, 'source');

blocksuite/affine/blocks/surface/src/renderer/tool-overlay.ts

@@ -1,6 +1,7 @@
 import { DisposableGroup } from '@blocksuite/global/disposable';
 import { noop } from '@blocksuite/global/utils';
 import type { GfxController } from '@blocksuite/std/gfx';
+import { startWith } from 'rxjs';
 
 import type { RoughCanvas } from '../utils/rough/canvas';
 import { Overlay } from './overlay';
@@ -18,10 +19,11 @@ export class ToolOverlay extends Overlay {
     super(gfx);
     this.x = 0;
     this.y = 0;
-    this.globalAlpha = 0;
+    this.globalAlpha = 1;
     this.gfx = gfx;
+
     this.disposables.add(
-      this.gfx.viewport.viewportUpdated.subscribe(() => {
+      this.gfx.viewport.viewportUpdated.pipe(startWith(null)).subscribe(() => {
         // when viewport is updated, we should keep the overlay in the same position
         // to get last mouse position and convert it to model coordinates
         const pos = this.gfx.tool.lastMousePos$.value;

blocksuite/affine/components/src/toggle-switch/index.ts

@@ -38,11 +38,11 @@ const styles = css`
     transition: 0.1s;
   }
 
-  label.subscribe {
+  label.on {
     background: var(--affine-primary-color);
   }
 
-  label.subscribe:after {
+  label.on:after {
     left: calc(100% - 1px);
     transform: translateX(-100%);
   }

blocksuite/affine/gfx/mindmap/src/toolbar/mindmap-tool-button.ts

@@ -326,6 +326,16 @@ export class EdgelessMindmapToolButton extends EdgelessToolbarToolMixin(
       },
       { global: true }
     );
+
+    // since there is not a tool called mindmap, we need to cancel the drag when the tool is changed
+    this.disposables.add(
+      this.gfx.tool.currentToolName$.subscribe(toolName => {
+        // FIXME: remove the assertion after gfx tool refactor
+        if ((toolName as string) !== 'empty' && this.readyToDrop) {
+          this.draggableController.cancel();
+        }
+      })
+    );
   }
 
   override render() {

blocksuite/affine/gfx/note/src/overlay/overlay.ts

@@ -26,7 +26,6 @@ export class NoteOverlay extends ToolOverlay {
 
   constructor(gfx: GfxController, background: Color) {
     super(gfx);
-    this.globalAlpha = 0;
     this.backgroundColor = gfx.std
       .get(ThemeProvider)
       .getColorValue(background, DefaultTheme.noteBackgrounColor, true);

blocksuite/affine/gfx/shape/src/draggable/shape-draggable.ts

@@ -236,30 +236,39 @@ export class EdgelessToolbarShapeDraggable extends EdgelessToolbarToolMixin(
           const locked = this.gfx.viewport.locked;
           const selection = this.gfx.selection;
           if (locked || selection.editing) return;
-
-          if (this.readyToDrop) {
-            const activeIndex = shapes.findIndex(
-              s => s.name === this.draggingShape
-            );
-            const nextIndex = (activeIndex + 1) % shapes.length;
-            const next = shapes[nextIndex];
-            this.draggingShape = next.name;
-
-            this.draggableController.cancelWithoutAnimation();
-          }
-
-          const el = this.shapeContainer.querySelector(
-            `.shape.${this.draggingShape}`
-          ) as HTMLElement;
-          if (!el) {
-            console.error('Edgeless toolbar Shape element not found');
+          if (
+            this.gfx.tool.dragging$.peek() &&
+            this.gfx.tool.currentToolName$.peek() === 'shape'
+          ) {
             return;
           }
-          const { x, y } = this.gfx.tool.lastMousePos$.peek();
-          const { viewport } = this.edgeless.std.get(ViewportElementProvider);
-          const { left, top } = viewport;
-          const clientPos = { x: x + left, y: y + top };
-          this.draggableController.dragAndMoveTo(el, clientPos);
+
+          const activeIndex = shapes.findIndex(
+            s => s.name === this.draggingShape
+          );
+          const nextIndex = (activeIndex + 1) % shapes.length;
+          const next = shapes[nextIndex];
+          this.draggingShape = next.name;
+
+          if (this.readyToDrop) {
+            this.draggableController.cancelWithoutAnimation();
+            const el = this.shapeContainer.querySelector(
+              `.shape.${this.draggingShape}`
+            ) as HTMLElement;
+            if (!el) {
+              console.error('Edgeless toolbar Shape element not found');
+              return;
+            }
+            const { x, y } = this.gfx.tool.lastMousePos$.peek();
+            const { viewport } = this.edgeless.std.get(ViewportElementProvider);
+            const { left, top } = viewport;
+            const clientPos = { x: x + left, y: y + top };
+            this.draggableController.dragAndMoveTo(el, clientPos);
+          } else {
+            this.setEdgelessTool('shape', {
+              shapeName: this.draggingShape,
+            });
+          }
         },
       },
       { global: true }

blocksuite/affine/gfx/shape/src/shape-tool.ts

@@ -89,7 +89,6 @@ export class ShapeTool extends BaseTool<ShapeToolOption> {
 
   private _hideOverlay() {
     if (!this._shapeOverlay) return;
-
     this._shapeOverlay.globalAlpha = 0;
     (this.gfx.surfaceComponent as SurfaceBlockComponent)?.refresh();
   }

blocksuite/affine/gfx/shape/src/view.ts

@@ -13,9 +13,9 @@ export class ShapeElementView extends GfxElementModelView<ShapeElementModel> {
   }
 
   private _initDblClickToEdit(): void {
-    const edgeless = this.std.view.getBlock(this.std.store.root!.id);
-
     this.on('dblclick', () => {
+      const edgeless = this.std.view.getBlock(this.std.store.root!.id);
+
       if (edgeless && !this.model.isLocked()) {
         mountShapeTextEditor(this.model, edgeless);
       }

blocksuite/affine/gfx/turbo-renderer/src/layout/block-layout-provider.ts

@@ -1,5 +1,7 @@
 import { createIdentifier } from '@blocksuite/global/di';
-import type { GfxBlockComponent } from '@blocksuite/std';
+import type { EditorHost } from '@blocksuite/std';
+import type { ViewportRecord } from '@blocksuite/std/gfx';
+import type { BlockModel } from '@blocksuite/store';
 import { Extension } from '@blocksuite/store';
 
 import type { BlockLayout, Rect } from '../types';
@@ -8,7 +10,13 @@ export abstract class BlockLayoutHandlerExtension<
   T extends BlockLayout = BlockLayout,
 > extends Extension {
   abstract readonly blockType: string;
-  abstract queryLayout(component: GfxBlockComponent): T | null;
+
+  abstract queryLayout(
+    model: BlockModel,
+    host: EditorHost,
+    viewportRecord: ViewportRecord
+  ): T | null;
+
   abstract calculateBound(layout: T): {
     rect: Rect;
     subRects: Rect[];

blocksuite/affine/gfx/turbo-renderer/src/painter/painter.worker.ts

@@ -7,8 +7,9 @@ import type { ExtensionType } from '@blocksuite/store';
 
 import type {
   BlockLayoutPainter,
+  BlockLayoutTreeNode,
   HostToWorkerMessage,
-  ViewportLayout,
+  ViewportLayoutTree,
   WorkerToHostMessage,
 } from '../types';
 
@@ -33,8 +34,8 @@ export class ViewportLayoutPainter {
   private zoom = 1;
   public provider: ServiceProvider;
 
-  getPainter(type: string): BlockLayoutPainter | undefined {
-    return this.provider.get(BlockPainterProvider(type));
+  getPainter(type: string): BlockLayoutPainter | null {
+    return this.provider.getOptional(BlockPainterProvider(type));
   }
 
   constructor(extensions: ExtensionType[]) {
@@ -66,24 +67,28 @@ export class ViewportLayoutPainter {
     this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
   }
 
-  paint(layout: ViewportLayout, version: number) {
+  paint(layout: ViewportLayoutTree, version: number) {
     const { canvas, ctx } = this;
     if (!canvas || !ctx) return;
-    if (layout.rect.w === 0 || layout.rect.h === 0) {
-      console.warn('empty layout rect');
-      return;
-    }
+
+    this.paintTree(layout, version);
+  }
+
+  paintTree(layout: ViewportLayoutTree, version: number) {
+    const { canvas, ctx } = this;
+    const { overallRect } = layout;
+    if (!canvas || !ctx) return;
 
     this.clearBackground();
-
     ctx.scale(this.zoom, this.zoom);
 
-    layout.blocks.forEach(blockLayout => {
-      const painter = this.getPainter(blockLayout.type);
-      if (!painter) return;
-      painter.paint(ctx, blockLayout, layout.rect.x, layout.rect.y);
-    });
+    const paintNode = (node: BlockLayoutTreeNode) => {
+      const painter = this.getPainter(node.type);
+      painter?.paint(ctx, node.layout, overallRect.x, overallRect.y);
+      node.children.forEach(paintNode);
+    };
 
+    layout.roots.forEach(root => paintNode(root));
     const bitmap = canvas.transferToImageBitmap();
     const message: WorkerToHostMessage = {
       type: 'bitmapPainted',

blocksuite/affine/gfx/turbo-renderer/src/renderer-utils.ts

@@ -1,12 +1,14 @@
 import type { EditorHost, GfxBlockComponent } from '@blocksuite/std';
-import {
-  GfxBlockElementModel,
-  GfxControllerIdentifier,
-  type Viewport,
-} from '@blocksuite/std/gfx';
+import { type Viewport } from '@blocksuite/std/gfx';
+import type { BlockModel } from '@blocksuite/store';
 
 import { BlockLayoutHandlersIdentifier } from './layout/block-layout-provider';
-import type { BlockLayout, RenderingState, ViewportLayout } from './types';
+import type {
+  BlockLayout,
+  BlockLayoutTreeNode,
+  RenderingState,
+  ViewportLayoutTree,
+} from './types';
 
 export function syncCanvasSize(canvas: HTMLCanvasElement, host: HTMLElement) {
   const hostRect = host.getBoundingClientRect();
@@ -21,33 +23,10 @@ export function syncCanvasSize(canvas: HTMLCanvasElement, host: HTMLElement) {
   canvas.style.pointerEvents = 'none';
 }
 
-function getBlockLayouts(host: EditorHost): BlockLayout[] {
-  const gfx = host.std.get(GfxControllerIdentifier);
-  const models = gfx.gfxElements.filter(e => e instanceof GfxBlockElementModel);
-  const components = models
-    .map(model => gfx.view.get(model.id))
-    .filter(Boolean) as GfxBlockComponent[];
-
-  const layouts: BlockLayout[] = [];
-  components.forEach(component => {
-    const layoutHandlers = host.std.provider.getAll(
-      BlockLayoutHandlersIdentifier
-    );
-    const handlersArray = Array.from(layoutHandlers.values());
-    for (const handler of handlersArray) {
-      const layout = handler.queryLayout(component);
-      if (layout) {
-        layouts.push(layout);
-      }
-    }
-  });
-  return layouts;
-}
-
-export function getViewportLayout(
+export function getViewportLayoutTree(
   host: EditorHost,
   viewport: Viewport
-): ViewportLayout {
+): ViewportLayoutTree {
   const zoom = viewport.zoom;
 
   let layoutMinX = Infinity;
@@ -55,36 +34,106 @@ export function getViewportLayout(
   let layoutMaxX = -Infinity;
   let layoutMaxY = -Infinity;
 
-  const blockLayouts = getBlockLayouts(host);
+  const store = host.std.store;
+  const rootModel = store.root;
+
+  if (!rootModel) {
+    return { roots: [], overallRect: { x: 0, y: 0, w: 0, h: 0 } };
+  }
 
   const providers = host.std.provider.getAll(BlockLayoutHandlersIdentifier);
   const providersArray = Array.from(providers.values());
 
-  blockLayouts.forEach(blockLayout => {
-    const provider = providersArray.find(p => p.blockType === blockLayout.type);
-    if (!provider) return;
+  // Recursive function to build the tree structure
+  const buildLayoutTreeNode = (
+    model: BlockModel,
+    ancestorViewportState?: string | null
+  ): BlockLayoutTreeNode | null => {
+    const baseLayout: BlockLayout = {
+      blockId: model.id,
+      type: model.flavour,
+      rect: { x: 0, y: 0, w: 0, h: 0 },
+    };
 
-    const { rect } = provider.calculateBound(blockLayout);
+    const handler = providersArray.find(p => p.blockType === model.flavour);
 
-    layoutMinX = Math.min(layoutMinX, rect.x);
-    layoutMinY = Math.min(layoutMinY, rect.y);
-    layoutMaxX = Math.max(layoutMaxX, rect.x + rect.w);
-    layoutMaxY = Math.max(layoutMaxY, rect.y + rect.h);
-  });
+    // Determine the correct viewport state to use
+    const component = host.std.view.getBlock(model.id) as GfxBlockComponent;
+    const currentViewportState = component?.dataset.viewportState;
+    const effectiveViewportState =
+      currentViewportState ?? ancestorViewportState;
+    const defaultViewportState = {
+      left: 0,
+      top: 0,
+      viewportX: 0,
+      viewportY: 0,
+      zoom: 1,
+      viewScale: 1,
+    };
 
-  const layoutModelCoord = [layoutMinX, layoutMinY];
+    const viewportRecord = effectiveViewportState
+      ? viewport.deserializeRecord(effectiveViewportState) ||
+        defaultViewportState
+      : defaultViewportState;
+
+    const layoutData = handler?.queryLayout(model, host, viewportRecord);
+
+    if (handler && layoutData) {
+      const { rect } = handler.calculateBound(layoutData);
+      baseLayout.rect = rect;
+      layoutMinX = Math.min(layoutMinX, rect.x);
+      layoutMinY = Math.min(layoutMinY, rect.y);
+      layoutMaxX = Math.max(layoutMaxX, rect.x + rect.w);
+      layoutMaxY = Math.max(layoutMaxY, rect.y + rect.h);
+    }
+
+    const children: BlockLayoutTreeNode[] = [];
+    for (const childModel of model.children) {
+      const childNode = buildLayoutTreeNode(childModel, effectiveViewportState);
+      if (childNode) {
+        children.push(childNode);
+      }
+    }
+
+    // Create node for this block - ALWAYS return a node
+    // Return the node structure including the layout (either real or fallback)
+    return {
+      blockId: model.id,
+      type: model.flavour,
+      layout: layoutData ? { ...baseLayout, ...layoutData } : baseLayout,
+      children,
+    };
+  };
+
+  const roots: BlockLayoutTreeNode[] = [];
+  const rootNode = buildLayoutTreeNode(rootModel);
+  if (rootNode) {
+    roots.push(rootNode);
+  }
+
+  // If no valid layouts were found, use default values
+  if (layoutMinX === Infinity) {
+    layoutMinX = 0;
+    layoutMinY = 0;
+    layoutMaxX = 0;
+    layoutMaxY = 0;
+  }
+
+  // Calculate overall rectangle
   const w = (layoutMaxX - layoutMinX) / zoom / viewport.viewScale;
   const h = (layoutMaxY - layoutMinY) / zoom / viewport.viewScale;
-  const layout: ViewportLayout = {
-    blocks: blockLayouts,
-    rect: {
-      x: layoutModelCoord[0],
-      y: layoutModelCoord[1],
+
+  const result = {
+    roots,
+    overallRect: {
+      x: layoutMinX,
+      y: layoutMinY,
       w: Math.max(w, 0),
       h: Math.max(h, 0),
     },
   };
-  return layout;
+
+  return result;
 }
 
 export function debugLog(message: string, state: RenderingState) {
@@ -98,14 +147,15 @@ export function debugLog(message: string, state: RenderingState) {
 export function paintPlaceholder(
   host: EditorHost,
   canvas: HTMLCanvasElement,
-  layout: ViewportLayout | null,
+  layout: ViewportLayoutTree | null,
   viewport: Viewport
 ) {
   const ctx = canvas.getContext('2d');
-  if (!ctx) return;
-  if (!layout) return;
+  if (!ctx || !layout) return;
+
   const dpr = window.devicePixelRatio;
-  const layoutViewCoord = viewport.toViewCoord(layout.rect.x, layout.rect.y);
+  const { overallRect } = layout;
+  const layoutViewCoord = viewport.toViewCoord(overallRect.x, overallRect.y);
 
   const offsetX = layoutViewCoord[0];
   const offsetY = layoutViewCoord[1];
@@ -120,30 +170,28 @@ export function paintPlaceholder(
   );
   const handlersArray = Array.from(layoutHandlers.values());
 
-  layout.blocks.forEach((blockLayout, blockIndex) => {
-    ctx.fillStyle = colors[blockIndex % colors.length];
-    const renderedPositions = new Set<string>();
-
-    const handler = handlersArray.find(h => h.blockType === blockLayout.type);
-    if (!handler) return;
-    const { subRects } = handler.calculateBound(blockLayout);
-
-    subRects.forEach(rect => {
-      const x = ((rect.x - layout.rect.x) * viewport.zoom + offsetX) * dpr;
-      const y = ((rect.y - layout.rect.y) * viewport.zoom + offsetY) * dpr;
-
+  const paintNode = (node: BlockLayoutTreeNode, depth: number = 0) => {
+    const { layout: nodeLayout, type } = node;
+    const handler = handlersArray.find(h => h.blockType === type);
+    if (handler) {
+      ctx.fillStyle = colors[depth % colors.length];
+      const rect = nodeLayout.rect;
+      const x = ((rect.x - overallRect.x) * viewport.zoom + offsetX) * dpr;
+      const y = ((rect.y - overallRect.y) * viewport.zoom + offsetY) * dpr;
       const width = rect.w * viewport.zoom * dpr;
       const height = rect.h * viewport.zoom * dpr;
 
-      const posKey = `${x},${y}`;
-      if (renderedPositions.has(posKey)) return;
       ctx.fillRect(x, y, width, height);
       if (width > 10 && height > 5) {
         ctx.strokeStyle = 'rgba(150, 150, 150, 0.3)';
         ctx.strokeRect(x, y, width, height);
       }
+    }
 
-      renderedPositions.add(posKey);
-    });
-  });
+    if (node.children.length > 0) {
+      node.children.forEach(childNode => paintNode(childNode, depth + 1));
+    }
+  };
+
+  layout.roots.forEach(rootNode => paintNode(rootNode));
 }

blocksuite/affine/gfx/turbo-renderer/src/turbo-renderer.ts

@@ -19,7 +19,7 @@ import { debounceTime } from 'rxjs/operators';
 
 import {
   debugLog,
-  getViewportLayout,
+  getViewportLayoutTree,
   paintPlaceholder,
   syncCanvasSize,
 } from './renderer-utils';
@@ -28,7 +28,7 @@ import type {
   RendererOptions,
   RenderingState,
   TurboRendererConfig,
-  ViewportLayout,
+  ViewportLayoutTree,
   WorkerToHostMessage,
 } from './types';
 
@@ -42,6 +42,34 @@ const defaultOptions: RendererOptions = {
 export const TurboRendererConfigFactory =
   ConfigExtensionFactory<TurboRendererConfig>('viewport-turbo-renderer');
 
+/**
+ * Manages the Turbo Rendering process for the viewport, coordinating between the main thread and a painter worker.
+ * Turbo Rendering optimizes performance by rendering block content onto a canvas bitmap,
+ * falling back to standard DOM rendering during interactions.
+ *
+ * To add Turbo Rendering support for a new block type (e.g., 'affine:my-block'):
+ *
+ * 1.  **In the block's package (e.g., `blocksuite/affine/blocks/my-block`):**
+ *   a.  Add `@blocksuite/affine/gfx/turbo-renderer` as a dependency in `package.json` and create a `src/turbo` directory.
+ *   b.  Implement the Layout Handler (e.g., `MyBlockLayoutHandlerExtension`) and Painter Worker (e.g., `MyBlockLayoutPainterExtension`). Refer to `ParagraphLayoutHandlerExtension` and `ParagraphLayoutPainterExtension` in `blocksuite/affine/blocks/block-paragraph` for implementation examples.
+ *   c.  Export the Layout Handler and Painter Worker extensions from the block package's main `src/index.ts` by adding these two explicit export statements:
+ *       ```typescript
+ *       export * from './turbo/my-block-layout-handler';
+ *       export * from './turbo/my-block-painter.worker';
+ *       ```
+ *   d.  Add an export mapping for the painter worker in `package.json` under the `exports` field (e.g., `"./turbo-painter": "./src/turbo/my-block-painter.worker.ts"`).
+ *   e.  Add a TypeScript project reference to `blocksuite/affine/gfx/turbo-renderer` in `tsconfig.json`.
+ *
+ * 2.  **In the application integration point (e.g., `packages/frontend/core/src/blocksuite/extensions` and `blocksuite/integration-test/src/__tests__/utils/renderer-entry.ts`):**
+ *   a.  In `turbo-renderer.ts` (or the file setting up `TurboRendererConfigFactory`):
+ *     - Import and add the new Layout Handler extension to the `patchTurboRendererExtension` array (or equivalent DI setup). See how `ParagraphLayoutHandlerExtension` is added as a reference.
+ *   b.  In `turbo-painter.worker.ts` (the painter worker entry point):
+ *     - Import and add the new Painter Worker extension to the `ViewportLayoutPainter` constructor's extension array. See how `ParagraphLayoutPainterExtension` is added as a reference.
+ *
+ * 3.  **Run `yarn affine init`** from the workspace root to update generated configuration files (`workspace.gen.ts`) and the lockfile (`yarn.lock`).
+ *
+ * **Note:** Always ensure the directory structure and export patterns match the `paragraph` block (`blocksuite/affine/blocks/block-paragraph`) for consistency.
+ */
 export class ViewportTurboRendererExtension extends GfxExtension {
   static override key = 'viewportTurboRenderer';
 
@@ -49,7 +77,7 @@ export class ViewportTurboRendererExtension extends GfxExtension {
   public readonly canvas: HTMLCanvasElement = document.createElement('canvas');
   private readonly worker: Worker;
   private readonly disposables = new DisposableGroup();
-  private layoutCacheData: ViewportLayout | null = null;
+  private layoutCacheData: ViewportLayoutTree | null = null;
   private layoutVersion = 0;
   private bitmap: ImageBitmap | null = null;
   private viewportElement: GfxViewportElement | null = null;
@@ -172,9 +200,9 @@ export class ViewportTurboRendererExtension extends GfxExtension {
 
   get layoutCache() {
     if (this.layoutCacheData) return this.layoutCacheData;
-    const layout = getViewportLayout(this.std.host, this.viewport);
+    const layoutTree = getViewportLayoutTree(this.std.host, this.viewport);
     this.debugLog('Layout cache updated');
-    return (this.layoutCacheData = layout);
+    return (this.layoutCacheData = layoutTree);
   }
 
   async refresh() {
@@ -248,8 +276,8 @@ export class ViewportTurboRendererExtension extends GfxExtension {
         type: 'paintLayout',
         data: {
           layout,
-          width: layout.rect.w,
-          height: layout.rect.h,
+          width: layout.overallRect.w,
+          height: layout.overallRect.h,
           dpr,
           zoom: this.viewport.zoom,
           version: currentVersion,
@@ -316,17 +344,18 @@ export class ViewportTurboRendererExtension extends GfxExtension {
     if (!ctx) return;
 
     this.clearCanvas();
+
     const layoutViewCoord = this.viewport.toViewCoord(
-      layout.rect.x,
-      layout.rect.y
+      layout.overallRect.x,
+      layout.overallRect.y
     );
 
     ctx.drawImage(
       bitmap,
       layoutViewCoord[0] * window.devicePixelRatio,
       layoutViewCoord[1] * window.devicePixelRatio,
-      layout.rect.w * window.devicePixelRatio * this.viewport.zoom,
-      layout.rect.h * window.devicePixelRatio * this.viewport.zoom
+      layout.overallRect.w * window.devicePixelRatio * this.viewport.zoom,
+      layout.overallRect.h * window.devicePixelRatio * this.viewport.zoom
     );
 
     this.debugLog('Bitmap drawn to canvas');

blocksuite/affine/gfx/turbo-renderer/src/types.ts

@@ -14,13 +14,14 @@ export interface ViewportState {
 }
 
 export interface BlockLayout extends Record<string, unknown> {
+  blockId: string;
   type: string;
-  rect?: Rect;
-}
-
-export interface ViewportLayout {
-  blocks: BlockLayout[];
-  rect: Rect;
+  rect: {
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+  };
 }
 
 export interface TextRect {
@@ -60,7 +61,7 @@ export type WorkerToHostMessage = MessageBitmapPainted | MessagePaintError;
 export type MessagePaint = {
   type: 'paintLayout';
   data: {
-    layout: ViewportLayout;
+    layout: ViewportLayoutTree;
     width: number;
     height: number;
     dpr: number;
@@ -89,3 +90,15 @@ export interface TurboRendererConfig {
 }
 
 export type HostToWorkerMessage = MessagePaint;
+
+export interface BlockLayoutTreeNode {
+  blockId: string;
+  type: string;
+  layout: BlockLayout;
+  children: BlockLayoutTreeNode[];
+}
+
+export interface ViewportLayoutTree {
+  roots: BlockLayoutTreeNode[];
+  overallRect: BlockLayout['rect'];
+}

blocksuite/affine/inlines/preset/src/adapters/markdown/markdown-inline.ts

@@ -79,6 +79,17 @@ export const markdownListToDeltaMatcher = MarkdownASTToDeltaExtension({
   toDelta: () => [],
 });
 
+export const markdownHtmlToDeltaMatcher = MarkdownASTToDeltaExtension({
+  name: 'html',
+  match: ast => ast.type === 'html',
+  toDelta: ast => {
+    if (!('value' in ast)) {
+      return [];
+    }
+    return [{ insert: ast.value }];
+  },
+});
+
 export const MarkdownInlineToDeltaAdapterExtensions = [
   markdownTextToDeltaMatcher,
   markdownInlineCodeToDeltaMatcher,
@@ -89,4 +100,5 @@ export const MarkdownInlineToDeltaAdapterExtensions = [
   markdownInlineMathToDeltaMatcher,
   markdownListToDeltaMatcher,
   markdownFootnoteReferenceToDeltaMatcher,
+  markdownHtmlToDeltaMatcher,
 ];

blocksuite/affine/model/src/elements/mindmap/mindmap.ts

@@ -743,7 +743,7 @@ export class MindmapElementModel extends GfxGroupLikeElementModel<MindmapElement
     const targetPos =
       typeof targetXYWH === 'string' ? deserializeXYWH(targetXYWH) : targetXYWH;
     const offsetX = targetPos[0] - x;
-    const offsetY = targetPos[1] - y + targetPos[3];
+    const offsetY = targetPos[1] - y;
 
     this.surface.doc.transact(() => {
       this.childElements.forEach(el => {

blocksuite/affine/shared/src/services/auto-clear-selection-service.ts

@@ -0,0 +1,12 @@
+import { LifeCycleWatcher } from '@blocksuite/std';
+
+// Auto Clear selection when switching doc mode.
+export class AutoClearSelectionService extends LifeCycleWatcher {
+  static override readonly key = 'auto-clear-selection-service';
+
+  override unmounted() {
+    if (this.std.store.readonly) return;
+
+    this.std.selection.clear();
+  }
+}

blocksuite/affine/shared/src/services/index.ts

@@ -1,3 +1,4 @@
+export * from './auto-clear-selection-service';
 export * from './block-meta-service';
 export * from './doc-display-meta-service';
 export * from './doc-mode-service';

blocksuite/docs/api/@blocksuite/std/gfx/functions/generateKeyBetween.md

@@ -6,7 +6,7 @@
 
 # Function: generateKeyBetween()
 
-> **generateKeyBetween**(`a`, `b`, `digits`?): `string`
+> **generateKeyBetween**(`a`, `b`, `digits?`): `string`
 
 ## Parameters
 

blocksuite/docs/api/@blocksuite/std/gfx/functions/generateNKeysBetween.md

@@ -6,7 +6,7 @@
 
 # Function: generateNKeysBetween()
 
-> **generateNKeysBetween**(`a`, `b`, `n`, `digits`?): `string`[]
+> **generateNKeysBetween**(`a`, `b`, `n`, `digits?`): `string`[]
 
 same preconditions as generateKeysBetween.
 n >= 0.

blocksuite/docs/api/@blocksuite/std/index/functions/KeymapExtension.md

@@ -6,7 +6,7 @@
 
 # Function: KeymapExtension()
 
-> **KeymapExtension**(`keymapFactory`, `options`?): `ExtensionType`
+> **KeymapExtension**(`keymapFactory`, `options?`): `ExtensionType`
 
 Create a keymap extension.
 

blocksuite/docs/api/@blocksuite/store/classes/Store.md

@@ -118,7 +118,7 @@ Get the root block of the store.
 
 ### addBlock()
 
-> **addBlock**(`flavour`, `blockProps`, `parent`?, `parentIndex`?): `string`
+> **addBlock**(`flavour`, `blockProps`, `parent?`, `parentIndex?`): `string`
 
 Creates and adds a new block to the store
 
@@ -162,7 +162,7 @@ When store is in readonly mode
 
 ### addBlocks()
 
-> **addBlocks**(`blocks`, `parent`?, `parentIndex`?): `string`[]
+> **addBlocks**(`blocks`, `parent?`, `parentIndex?`): `string`[]
 
 Add multiple blocks to the store
 
@@ -598,7 +598,7 @@ When the block is not found or schema validation fails
 
 #### Get Signature
 
-> **get** **get**(): \<`T`\>(`identifier`, `options`?) => `T`
+> **get** **get**(): \<`T`\>(`identifier`, `options?`) => `T`
 
 Get an extension instance from the store
 
@@ -612,7 +612,7 @@ const extension = store.get(SomeExtension);
 
 The extension instance
 
-> \<`T`\>(`identifier`, `options`?): `T`
+> \<`T`\>(`identifier`, `options?`): `T`
 
 ###### Type Parameters
 
@@ -640,7 +640,7 @@ The extension instance
 
 #### Get Signature
 
-> **get** **getOptional**(): \<`T`\>(`identifier`, `options`?) => `null` \| `T`
+> **get** **getOptional**(): \<`T`\>(`identifier`, `options?`) => `null` \| `T`
 
 Optional get an extension instance from the store.
 The major difference between `get` and `getOptional` is that `getOptional` will not throw an error if the extension is not found.
@@ -655,7 +655,7 @@ const extension = store.getOptional(SomeExtension);
 
 The extension instance
 
-> \<`T`\>(`identifier`, `options`?): `null` \| `T`
+> \<`T`\>(`identifier`, `options?`): `null` \| `T`
 
 ###### Type Parameters
 
@@ -805,7 +805,7 @@ Reset the history of the store.
 
 #### Get Signature
 
-> **get** **transact**(): (`fn`, `shouldTransact`?) => `void`
+> **get** **transact**(): (`fn`, `shouldTransact?`) => `void`
 
 Execute a transaction.
 
@@ -820,7 +820,7 @@ store.transact(() => {
 
 ##### Returns
 
-> (`fn`, `shouldTransact`?): `void`
+> (`fn`, `shouldTransact?`): `void`
 
 ###### Parameters
 
@@ -971,7 +971,7 @@ Disposes the store and releases all resources
 
 ### load()
 
-> **load**(`initFn`?): `Store`
+> **load**(`initFn?`): `Store`
 
 Initializes and loads the store
 

blocksuite/docs/api/@blocksuite/store/classes/Text.md

@@ -26,7 +26,7 @@ Text [delta](https://docs.yjs.dev/api/delta-format) is a format from Y.js.
 
 ### Constructor
 
-> **new Text**(`input`?): `Text`
+> **new Text**(`input?`): `Text`
 
 #### Parameters
 
@@ -176,7 +176,7 @@ text.format(7, 1, { bold: true });
 
 ### insert()
 
-> **insert**(`content`, `index`, `attributes`?): `void`
+> **insert**(`content`, `index`, `attributes?`): `void`
 
 Insert content at the specified index.
 
@@ -241,7 +241,7 @@ text.join(other);
 
 ### replace()
 
-> **replace**(`index`, `length`, `content`, `attributes`?): `void`
+> **replace**(`index`, `length`, `content`, `attributes?`): `void`
 
 Replace the text content with a new content.
 
@@ -286,7 +286,7 @@ text.replace(7, 1, ' blocksuite');
 
 ### sliceToDelta()
 
-> **sliceToDelta**(`begin`, `end`?): `DeltaOperation`[]
+> **sliceToDelta**(`begin`, `end?`): `DeltaOperation`[]
 
 Slice the text to a delta.
 

blocksuite/framework/global/src/gfx/math.ts

@@ -11,6 +11,24 @@ export function randomSeed(): number {
   return Math.floor(Math.random() * 2 ** 31);
 }
 
+/**
+ * Calculates the intersection point of two line segments.
+ *
+ * @param sp - Start point of the first line segment [x, y]
+ * @param ep - End point of the first line segment [x, y]
+ * @param sp2 - Start point of the second line segment [x, y]
+ * @param ep2 - End point of the second line segment [x, y]
+ * @param infinite - If true, treats the lines as infinite lines rather than line segments
+ * @returns The intersection point [x, y] if the lines intersect, null if they are parallel or coincident
+ *
+ * @example
+ * const intersection = lineIntersects([0, 0], [2, 2], [0, 2], [2, 0]);
+ * // Returns [1, 1] - the intersection point of the two line segments
+ *
+ * @example
+ * const parallel = lineIntersects([0, 0], [2, 2], [0, 1], [2, 3], true);
+ * // Returns null - the lines are parallel
+ */
 export function lineIntersects(
   sp: IVec,
   ep: IVec,
@@ -45,10 +63,23 @@ export function lineIntersects(
   return null;
 }
 
+/**
+ * Finds the nearest point on a polygon to a given point.
+ *
+ * @param points - Array of points defining the polygon vertices [x, y][]
+ * @param point - The point to find the nearest point to [x, y]
+ * @returns The nearest point on the polygon to the given point
+ * @throws Error if points array is empty or has less than 2 points
+ */
 export function polygonNearestPoint(points: IVec[], point: IVec) {
   const len = points.length;
-  let rst: IVec;
-  let dis = Infinity;
+  if (len < 2) {
+    throw new Error('Polygon must have at least 2 points');
+  }
+
+  let rst: IVec = points[0]; // Initialize with first point as fallback
+  let dis = Vec.dist(points[0], point);
+
   for (let i = 0; i < len; i++) {
     const p = points[i];
     const p2 = points[(i + 1) % len];
@@ -59,7 +90,7 @@ export function polygonNearestPoint(points: IVec[], point: IVec) {
       rst = temp;
     }
   }
-  return rst!;
+  return rst;
 }
 
 export function polygonPointDistance(points: IVec[], point: IVec) {

blocksuite/framework/global/src/gfx/model/bound.ts

@@ -341,8 +341,15 @@ export class Bound implements IBound {
     return serializeXYWH(this.x, this.y, this.w, this.h);
   }
 
+  /**
+   * Convert a point to relative coordinates.
+   * @param point - The point to convert.
+   * @returns The normalized relative coordinates of the point.
+   */
   toRelative([x, y]: IVec): IVec {
-    return [(x - this.x) / this.w, (y - this.y) / this.h];
+    const normalizedX = this.w === 0 ? 0 : (x - this.x) / this.w;
+    const normalizedY = this.h === 0 ? 0 : (y - this.y) / this.h;
+    return [normalizedX, normalizedY];
   }
 
   toXYWH(): XYWH {

blocksuite/framework/global/src/gfx/model/vec.ts

@@ -565,6 +565,8 @@ export class Vec {
    * @param n
    * @param min
    */
+  static clampV(A: IVec, min: number, max?: number): IVec;
+
   static clampV(A: number[], min: number): number[];
 
   // eslint-disable-next-line @typescript-eslint/unified-signatures

blocksuite/framework/std/src/gfx/viewport.ts

@@ -214,7 +214,12 @@ export class Viewport {
    * This property is used to calculate the scale of the editor.
    */
   get viewScale() {
-    if (!this._shell || this._cachedOffsetWidth === null) return 1;
+    if (
+      !this._shell ||
+      this._cachedOffsetWidth === null ||
+      this._cachedOffsetWidth === 0
+    )
+      return 1;
     return this.boundingClientRect.width / this._cachedOffsetWidth;
   }
 

blocksuite/framework/std/src/inline/consts.ts

@@ -1,7 +1,5 @@
-import { IS_SAFARI } from '@blocksuite/global/env';
-
-export const ZERO_WIDTH_SPACE = IS_SAFARI ? '\u200C' : '\u200B';
+export const ZERO_WIDTH_SPACE = '\u200C';
 // see https://en.wikipedia.org/wiki/Zero-width_non-joiner
-export const ZERO_WIDTH_NON_JOINER = '\u200C';
+export const ZERO_WIDTH_NON_JOINER = '\u200B';
 
 export const INLINE_ROOT_ATTR = 'data-v-root';

blocksuite/integration-test/src/__tests__/utils/renderer-entry.ts

@@ -1,3 +1,4 @@
+import { ListLayoutHandlerExtension } from '@blocksuite/affine/blocks/list';
 import { ParagraphLayoutHandlerExtension } from '@blocksuite/affine/blocks/paragraph';
 import {
   TurboRendererConfigFactory,
@@ -11,6 +12,7 @@ import { createPainterWorker, setupEditor } from './setup.js';
 async function init() {
   setupEditor('edgeless', [
     ParagraphLayoutHandlerExtension,
+    ListLayoutHandlerExtension,
     TurboRendererConfigFactory({
       painterWorkerEntry: createPainterWorker,
     }),

blocksuite/integration-test/src/__tests__/utils/setup.ts

@@ -103,7 +103,7 @@ async function createEditor(
 
 export function createPainterWorker() {
   const worker = new Worker(
-    new URL('./turbo-painter-entry.worker.ts', import.meta.url),
+    new URL('./turbo-painter.worker.ts', import.meta.url),
     {
       type: 'module',
     }

blocksuite/integration-test/src/__tests__/utils/turbo-painter-entry.worker.ts

@@ -1,4 +0,0 @@
-import { ParagraphLayoutPainterExtension } from '@blocksuite/affine-block-paragraph/turbo-painter';
-import { ViewportLayoutPainter } from '@blocksuite/affine-gfx-turbo-renderer/painter';
-
-new ViewportLayoutPainter([ParagraphLayoutPainterExtension]);

blocksuite/integration-test/src/__tests__/utils/turbo-painter.worker.ts

@@ -0,0 +1,10 @@
+import { ListLayoutPainterExtension } from '@blocksuite/affine-block-list/turbo-painter';
+import { NoteLayoutPainterExtension } from '@blocksuite/affine-block-note/turbo-painter';
+import { ParagraphLayoutPainterExtension } from '@blocksuite/affine-block-paragraph/turbo-painter';
+import { ViewportLayoutPainter } from '@blocksuite/affine-gfx-turbo-renderer/painter';
+
+new ViewportLayoutPainter([
+  ParagraphLayoutPainterExtension,
+  ListLayoutPainterExtension,
+  NoteLayoutPainterExtension,
+]);

package.json

@@ -82,7 +82,7 @@
     "husky": "^9.1.7",
     "lint-staged": "^15.2.11",
     "msw": "^2.6.8",
-    "oxlint": "0.16.4",
+    "oxlint": "0.16.5",
     "prettier": "^3.4.2",
     "semver": "^7.6.3",
     "serve": "^14.2.4",
@@ -92,7 +92,7 @@
     "vite": "^6.0.3",
     "vitest": "3.1.1"
   },
-  "packageManager": "yarn@4.8.1",
+  "packageManager": "yarn@4.9.0",
   "resolutions": {
     "array-buffer-byte-length": "npm:@nolyfill/array-buffer-byte-length@^1",
     "array-includes": "npm:@nolyfill/array-includes@^1",
@@ -126,6 +126,8 @@
     "is-symbol": "npm:@nolyfill/is-symbol@^1",
     "is-weakref": "npm:@nolyfill/is-weakref@^1",
     "iterator.prototype": "npm:@nolyfill/iterator.prototype@^1",
+    "json-stable-stringify": "npm:@nolyfill/json-stable-stringify@^1",
+    "jsonify": "npm:@nolyfill/jsonify@^1",
     "object-is": "npm:@nolyfill/object-is@^1",
     "object-keys": "npm:@nolyfill/object-keys@^1",
     "object.assign": "npm:@nolyfill/object.assign@^1",

packages/backend/native/Cargo.toml

@@ -16,7 +16,7 @@ rand          = { workspace = true }
 sha3          = { workspace = true }
 tiktoken-rs   = { workspace = true }
 v_htmlescape  = { workspace = true }
-y-octo        = { workspace = true }
+y-octo        = { workspace = true, features = ["large_refs"] }
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
 mimalloc = { workspace = true }

packages/backend/server/package.json

@@ -25,7 +25,9 @@
     "postinstall": "prisma generate"
   },
   "dependencies": {
-    "@ai-sdk/google": "^1.1.19",
+    "@ai-sdk/google": "^1.2.10",
+    "@ai-sdk/openai": "^1.3.9",
+    "@ai-sdk/perplexity": "^1.1.6",
     "@apollo/server": "^4.11.3",
     "@aws-sdk/client-s3": "^3.779.0",
     "@aws-sdk/s3-request-presigner": "^3.779.0",
@@ -65,7 +67,7 @@
     "@opentelemetry/semantic-conventions": "^1.28.0",
     "@prisma/client": "^5.22.0",
     "@prisma/instrumentation": "^5.22.0",
-    "@react-email/components": "0.0.35",
+    "@react-email/components": "0.0.36",
     "@socket.io/redis-adapter": "^8.3.0",
     "ai": "^4.1.51",
     "bullmq": "^5.40.2",
@@ -73,7 +75,6 @@
     "date-fns": "^4.0.0",
     "dotenv": "^16.4.7",
     "eventemitter2": "^6.4.9",
-    "eventsource-parser": "^3.0.0",
     "express": "^5.0.1",
     "fast-xml-parser": "^5.0.0",
     "get-stream": "^9.0.1",
@@ -95,7 +96,6 @@
     "nestjs-cls": "^5.0.0",
     "nodemailer": "^6.9.16",
     "on-headers": "^1.0.2",
-    "openai": "^4.83.0",
     "piscina": "^5.0.0-alpha.0",
     "prisma": "^5.22.0",
     "react": "19.1.0",
@@ -140,7 +140,7 @@
     "c8": "^10.1.3",
     "cross-env": "^7.0.3",
     "nodemon": "^3.1.7",
-    "react-email": "4.0.3",
+    "react-email": "4.0.7",
     "sinon": "^20.0.0",
     "supertest": "^7.0.0",
     "why-is-node-running": "^3.2.2"

packages/backend/server/src/__tests__/__snapshots__/copilot.e2e.ts.md

@@ -54,3 +54,81 @@ Generated by [AVA](https://avajs.dev).
         id: 'docId1',
       },
     ]
+
+## should be able to transcript
+
+> should submit audio transcription job
+
+    [
+      {
+        status: 'running',
+      },
+    ]
+
+> should claim audio transcription job
+
+    [
+      {
+        status: 'claimed',
+        summary: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]',
+        title: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]',
+        transcription: [
+          {
+            end: '00:00:45',
+            speaker: 'A',
+            start: '00:00:30',
+            transcription: 'Hello, everyone.',
+          },
+          {
+            end: '00:01:10',
+            speaker: 'B',
+            start: '00:00:46',
+            transcription: 'Hi, thank you for joining the meeting today.',
+          },
+        ],
+      },
+    ]
+
+> should submit audio transcription job
+
+    [
+      {
+        status: 'running',
+      },
+    ]
+
+> should claim audio transcription job
+
+    [
+      {
+        status: 'claimed',
+        summary: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]',
+        title: '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]',
+        transcription: [
+          {
+            end: '00:00:45',
+            speaker: 'A',
+            start: '00:00:30',
+            transcription: 'Hello, everyone.',
+          },
+          {
+            end: '00:01:10',
+            speaker: 'B',
+            start: '00:00:46',
+            transcription: 'Hi, thank you for joining the meeting today.',
+          },
+          {
+            end: '00:10:45',
+            speaker: 'A',
+            start: '00:10:30',
+            transcription: 'Hello, everyone.',
+          },
+          {
+            end: '00:11:10',
+            speaker: 'B',
+            start: '00:10:46',
+            transcription: 'Hi, thank you for joining the meeting today.',
+          },
+        ],
+      },
+    ]

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -476,6 +476,19 @@ const actions = [
     },
     type: 'image' as const,
   },
+  {
+    promptName: ['debug:action:dalle3'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: 'Panda',
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, link: string) => {
+      t.truthy(checkUrl(link), 'should be a valid url');
+    },
+    type: 'image' as const,
+  },
 ];
 
 for (const { name, promptName, messages, verifier, type } of actions) {

packages/backend/server/src/__tests__/copilot.e2e.ts

@@ -19,6 +19,7 @@ import { MockEmbeddingClient } from '../plugins/copilot/context/embedding';
 import { prompts, PromptService } from '../plugins/copilot/prompt';
 import {
   CopilotProviderFactory,
+  GeminiProvider,
   OpenAIProvider,
 } from '../plugins/copilot/providers';
 import { CopilotStorage } from '../plugins/copilot/storage';
@@ -35,10 +36,12 @@ import {
   addContextDoc,
   addContextFile,
   array2sse,
+  audioTranscription,
   chatWithImages,
   chatWithText,
   chatWithTextStream,
   chatWithWorkflow,
+  claimAudioTranscription,
   cleanObject,
   createCopilotContext,
   createCopilotMessage,
@@ -50,6 +53,7 @@ import {
   matchFiles,
   matchWorkspaceDocs,
   sse2array,
+  submitAudioTranscription,
   textToEventStream,
   unsplashSearch,
   updateCopilotSession,
@@ -96,6 +100,7 @@ test.before(async t => {
         },
       });
       m.overrideProvider(OpenAIProvider).useClass(MockCopilotProvider);
+      m.overrideProvider(GeminiProvider).useClass(MockCopilotProvider);
     },
   });
 
@@ -868,3 +873,77 @@ test('should be able to manage context', async t => {
     t.is(result[0].docId, docId, 'should match doc id');
   }
 });
+
+test('should be able to transcript', async t => {
+  const { app } = t.context;
+
+  const { id: workspaceId } = await createWorkspace(app);
+
+  Sinon.stub(app.get(GeminiProvider), 'generateText').resolves(
+    '[{"a":"A","s":30,"e":45,"t":"Hello, everyone."},{"a":"B","s":46,"e":70,"t":"Hi, thank you for joining the meeting today."}]'
+  );
+
+  {
+    const job = await submitAudioTranscription(app, workspaceId, '1', '1.mp3', [
+      Buffer.from([1, 1]),
+    ]);
+    t.snapshot(
+      cleanObject([job], ['id']),
+      'should submit audio transcription job'
+    );
+    t.truthy(job.id, 'should have job id');
+
+    // wait for processing
+    {
+      let { status } =
+        (await audioTranscription(app, workspaceId, job.id)) || {};
+
+      while (status !== 'finished') {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        ({ status } =
+          (await audioTranscription(app, workspaceId, job.id)) || {});
+      }
+    }
+
+    {
+      const result = await claimAudioTranscription(app, job.id);
+      t.snapshot(
+        cleanObject([result], ['id']),
+        'should claim audio transcription job'
+      );
+    }
+  }
+
+  {
+    // sliced audio
+    const job = await submitAudioTranscription(app, workspaceId, '2', '2.mp3', [
+      Buffer.from([1, 1]),
+      Buffer.from([1, 2]),
+    ]);
+    t.snapshot(
+      cleanObject([job], ['id']),
+      'should submit audio transcription job'
+    );
+    t.truthy(job.id, 'should have job id');
+
+    // wait for processing
+    {
+      let { status } =
+        (await audioTranscription(app, workspaceId, job.id)) || {};
+
+      while (status !== 'finished') {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        ({ status } =
+          (await audioTranscription(app, workspaceId, job.id)) || {});
+      }
+    }
+
+    {
+      const result = await claimAudioTranscription(app, job.id);
+      t.snapshot(
+        cleanObject([result], ['id']),
+        'should claim audio transcription job'
+      );
+    }
+  }
+});

packages/backend/server/src/__tests__/copilot.spec.ts

@@ -24,9 +24,10 @@ import {
   CopilotProviderType,
   OpenAIProvider,
 } from '../plugins/copilot/providers';
-import { CitationParser } from '../plugins/copilot/providers/perplexity';
+import { CitationParser } from '../plugins/copilot/providers/utils';
 import { ChatSessionService } from '../plugins/copilot/session';
 import { CopilotStorage } from '../plugins/copilot/storage';
+import { CopilotTranscriptionService } from '../plugins/copilot/transcript';
 import {
   CopilotChatTextExecutor,
   CopilotWorkflowService,
@@ -57,6 +58,7 @@ const test = ava as TestFn<{
   event: EventBus;
   context: CopilotContextService;
   prompt: PromptService;
+  transcript: CopilotTranscriptionService;
   factory: CopilotProviderFactory;
   session: ChatSessionService;
   jobs: CopilotContextDocJob;
@@ -100,25 +102,30 @@ test.before(async t => {
   const auth = module.get(AuthService);
   const db = module.get(PrismaClient);
   const event = module.get(EventBus);
-  const context = module.get(CopilotContextService);
   const prompt = module.get(PromptService);
   const factory = module.get(CopilotProviderFactory);
+
   const session = module.get(ChatSessionService);
   const workflow = module.get(CopilotWorkflowService);
-  const jobs = module.get(CopilotContextDocJob);
   const storage = module.get(CopilotStorage);
 
+  const context = module.get(CopilotContextService);
+  const jobs = module.get(CopilotContextDocJob);
+  const transcript = module.get(CopilotTranscriptionService);
+
   t.context.module = module;
   t.context.auth = auth;
   t.context.db = db;
   t.context.event = event;
-  t.context.context = context;
   t.context.prompt = prompt;
   t.context.factory = factory;
   t.context.session = session;
   t.context.workflow = workflow;
-  t.context.jobs = jobs;
   t.context.storage = storage;
+  t.context.context = context;
+  t.context.jobs = jobs;
+  t.context.transcript = transcript;
+
   t.context.executors = {
     image: module.get(CopilotChatImageExecutor),
     text: module.get(CopilotChatTextExecutor),

packages/backend/server/src/__tests__/mocks/copilot.mock.ts

@@ -21,6 +21,7 @@ export class MockCopilotProvider extends OpenAIProvider {
     'lcm-sd15-i2i',
     'clarity-upscaler',
     'imageutils/rembg',
+    'gemini-2.5-pro-preview-03-25',
   ];
 
   override readonly capabilities = [

packages/backend/server/src/__tests__/utils/copilot.ts

@@ -330,6 +330,161 @@ export async function listContextDocAndFiles(
   return { docs, files };
 }
 
+export async function submitAudioTranscription(
+  app: TestingApp,
+  workspaceId: string,
+  blobId: string,
+  fileName: string,
+  content: Buffer[]
+): Promise<{ id: string; status: string }> {
+  let resp = app
+    .POST('/graphql')
+    .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
+    .field(
+      'operations',
+      JSON.stringify({
+        query: `
+          mutation submitAudioTranscription($blob: Upload, $blobs: [Upload!], $blobId: String!, $workspaceId: String!) {
+            submitAudioTranscription(blob: $blob, blobs: $blobs, blobId: $blobId, workspaceId: $workspaceId) {
+              id
+              status
+            }
+          }
+        `,
+        variables: {
+          blob: null,
+          blobs: [],
+          blobId,
+          workspaceId,
+        },
+      })
+    )
+    .field(
+      'map',
+      JSON.stringify(
+        Array.from<any>({ length: content.length }).reduce((acc, _, idx) => {
+          acc[idx.toString()] = [`variables.blobs.${idx}`];
+          return acc;
+        }, {})
+      )
+    );
+  for (const [idx, buffer] of content.entries()) {
+    resp = resp.attach(idx.toString(), buffer, {
+      filename: fileName,
+      contentType: 'application/octet-stream',
+    });
+  }
+
+  const res = await resp.expect(200);
+
+  return res.body.data.submitAudioTranscription;
+}
+
+export async function retryAudioTranscription(
+  app: TestingApp,
+  workspaceId: string,
+  jobId: string
+): Promise<{ id: string; status: string }> {
+  const res = await app.gql(
+    `
+      mutation retryAudioTranscription($workspaceId: String!, $jobId: String!) {
+        retryAudioTranscription(workspaceId: $workspaceId, jobId: $jobId) {
+          id
+          status
+        }
+      }
+    `,
+    { workspaceId, jobId }
+  );
+
+  return res.retryAudioTranscription;
+}
+
+export async function claimAudioTranscription(
+  app: TestingApp,
+  jobId: string
+): Promise<{
+  id: string;
+  status: string;
+  title: string | null;
+  summary: string | null;
+  transcription:
+    | {
+        speaker: string;
+        start: number;
+        end: number;
+        transcription: string;
+      }[]
+    | null;
+}> {
+  const res = await app.gql(
+    `
+      mutation claimAudioTranscription($jobId: String!) {
+        claimAudioTranscription(jobId: $jobId) {
+          id
+          status
+          title
+          summary
+          transcription {
+            speaker
+            start
+            end
+            transcription
+          }
+        }
+      }
+    `,
+    { jobId }
+  );
+
+  return res.claimAudioTranscription;
+}
+
+export async function audioTranscription(
+  app: TestingApp,
+  workspaceId: string,
+  jobId: string
+): Promise<{
+  id: string;
+  status: string;
+  title: string | null;
+  summary: string | null;
+  transcription:
+    | {
+        speaker: string;
+        start: number;
+        end: number;
+        transcription: string;
+      }[]
+    | null;
+}> {
+  const res = await app.gql(
+    `
+      query audioTranscription($workspaceId: String!, $jobId: String!) {
+        currentUser {
+          copilot(workspaceId: $workspaceId) {
+            audioTranscription(jobId: $jobId) {
+              id
+              status
+              title
+              summary
+              transcription {
+                speaker
+                start
+                end
+                transcription
+              }
+            }
+          }
+        }
+      }
+    `,
+    { workspaceId, jobId }
+  );
+
+  return res.currentUser?.copilot?.audioTranscription;
+}
+
 export async function createCopilotMessage(
   app: TestingApp,
   sessionId: string,

packages/backend/server/src/base/error/def.ts

@@ -709,6 +709,10 @@ export const USER_FRIENDLY_ERRORS = {
     type: 'bad_request',
     message: () => `Transcription job not found.`,
   },
+  copilot_transcription_audio_not_provided: {
+    type: 'bad_request',
+    message: () => `Audio not provided.`,
+  },
 
   // Quota & Limit errors
   blob_quota_exceeded: {

packages/backend/server/src/base/error/errors.gen.ts

@@ -771,6 +771,12 @@ export class CopilotTranscriptionJobNotFound extends UserFriendlyError {
   }
 }
 
+export class CopilotTranscriptionAudioNotProvided extends UserFriendlyError {
+  constructor(message?: string) {
+    super('bad_request', 'copilot_transcription_audio_not_provided', message);
+  }
+}
+
 export class BlobQuotaExceeded extends UserFriendlyError {
   constructor(message?: string) {
     super('quota_exceeded', 'blob_quota_exceeded', message);
@@ -1027,6 +1033,7 @@ export enum ErrorNames {
   COPILOT_EMBEDDING_UNAVAILABLE,
   COPILOT_TRANSCRIPTION_JOB_EXISTS,
   COPILOT_TRANSCRIPTION_JOB_NOT_FOUND,
+  COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED,
   BLOB_QUOTA_EXCEEDED,
   STORAGE_QUOTA_EXCEEDED,
   MEMBER_QUOTA_EXCEEDED,

packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts

@@ -2,7 +2,7 @@ import { getQueueToken } from '@nestjs/bullmq';
 import { Injectable } from '@nestjs/common';
 import { TestingModule } from '@nestjs/testing';
 import test from 'ava';
-import { Queue as Bullmq } from 'bullmq';
+import { Queue as Bullmq, Worker } from 'bullmq';
 import Sinon from 'sinon';
 
 import { createTestingModule } from '../../../../__tests__/utils';
@@ -15,6 +15,7 @@ import { JobHandlerScanner } from '../scanner';
 let module: TestingModule;
 let queue: JobQueue;
 let executor: JobExecutor;
+let worker: Worker;
 let bullmq: Bullmq;
 
 declare global {
@@ -69,6 +70,9 @@ test.before(async () => {
   queue = module.get(JobQueue);
   executor = module.get(JobExecutor);
   bullmq = module.get(getQueueToken('nightly'), { strict: false });
+  // @ts-expect-error private api
+  worker = executor.workers.get('nightly')!;
+  await worker.pause();
 });
 
 test.beforeEach(async () => {
@@ -116,13 +120,6 @@ test('should remove job from queue', async t => {
 // #endregion
 
 // #region executor
-test('should start workers', async t => {
-  // @ts-expect-error private api
-  const worker = executor.workers.get('nightly')!;
-
-  t.truthy(worker);
-});
-
 test('should dispatch job handler', async t => {
   const handlers = module.get(JobHandlers);
   const spy = Sinon.spy(handlers, 'handleJob');

packages/backend/server/src/core/workspaces/resolvers/team.ts

@@ -162,7 +162,7 @@ export class TeamWorkspaceResolver {
       if (Number.isSafeInteger(expireTime)) {
         return {
           link: this.url.link(`/invite/${id.inviteId}`),
-          expireTime: new Date(Date.now() + expireTime),
+          expireTime: new Date(Date.now() + expireTime * 1000), // Convert seconds to milliseconds
         };
       }
     }
@@ -188,7 +188,7 @@ export class TeamWorkspaceResolver {
       if (Number.isSafeInteger(expireTime)) {
         return {
           link: this.url.link(`/invite/${invite.inviteId}`),
-          expireTime: new Date(Date.now() + expireTime),
+          expireTime: new Date(Date.now() + expireTime * 1000), // Convert seconds to milliseconds
         };
       }
     }

packages/backend/server/src/env.ts

@@ -80,7 +80,7 @@ export class Env implements AppEnv {
   );
   DEPLOYMENT_TYPE = readEnv(
     'DEPLOYMENT_TYPE',
-    DeploymentType.Selfhosted,
+    this.dev ? DeploymentType.Affine : DeploymentType.Selfhosted,
     Object.values(DeploymentType)
   );
   FLAVOR = readEnv('SERVER_FLAVOR', Flavor.AllInOne, Object.values(Flavor));

packages/backend/server/src/plugins/copilot/context/embedding.ts

@@ -1,27 +1,39 @@
-import OpenAI from 'openai';
+import {
+  createOpenAI,
+  type OpenAIProvider as VercelOpenAIProvider,
+} from '@ai-sdk/openai';
+import { embedMany } from 'ai';
 
 import { Embedding } from '../../../models';
+import { OpenAIConfig } from '../providers/openai';
 import { EmbeddingClient } from './types';
 
 export class OpenAIEmbeddingClient extends EmbeddingClient {
-  constructor(private readonly client: OpenAI) {
+  readonly #instance: VercelOpenAIProvider;
+
+  constructor(config: OpenAIConfig) {
     super();
+    this.#instance = createOpenAI({
+      apiKey: config.apiKey,
+      baseURL: config.baseUrl,
+    });
   }
 
-  async getEmbeddings(
-    input: string[],
-    signal?: AbortSignal
-  ): Promise<Embedding[]> {
-    const resp = await this.client.embeddings.create(
-      {
-        input,
-        model: 'text-embedding-3-large',
-        dimensions: 1024,
-        encoding_format: 'float',
-      },
-      { signal }
-    );
-    return resp.data.map(e => ({ ...e, content: input[e.index] }));
+  async getEmbeddings(input: string[]): Promise<Embedding[]> {
+    const modelInstance = this.#instance.embedding('text-embedding-3-large', {
+      dimensions: 1024,
+    });
+
+    const { embeddings } = await embedMany({
+      model: modelInstance,
+      values: input,
+    });
+
+    return Array.from(embeddings.entries()).map(([index, embedding]) => ({
+      index,
+      embedding,
+      content: input[index],
+    }));
   }
 }
 

packages/backend/server/src/plugins/copilot/context/job.ts

@@ -1,5 +1,4 @@
 import { Injectable } from '@nestjs/common';
-import OpenAI from 'openai';
 
 import {
   AFFiNELogger,
@@ -49,7 +48,7 @@ export class CopilotContextDocJob {
     this.supportEmbedding =
       await this.models.copilotContext.checkEmbeddingAvailable();
     this.client = new OpenAIEmbeddingClient(
-      new OpenAI(this.config.copilot.providers.openai)
+      this.config.copilot.providers.openai
     );
   }
 

packages/backend/server/src/plugins/copilot/context/service.ts

@@ -1,5 +1,4 @@
 import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
-import OpenAI from 'openai';
 
 import {
   Cache,
@@ -46,7 +45,7 @@ export class CopilotContextService implements OnApplicationBootstrap {
   private setup() {
     const configure = this.config.copilot.providers.openai;
     if (configure.apiKey) {
-      this.client = new OpenAIEmbeddingClient(new OpenAI(configure));
+      this.client = new OpenAIEmbeddingClient(configure);
     }
   }
 

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -320,7 +320,7 @@ const actions: Prompt[] = [
   {
     name: 'Transcript audio',
     action: 'Transcript audio',
-    model: 'gemini-2.5-pro-exp-03-25',
+    model: 'gemini-2.5-pro-preview-03-25',
     messages: [
       {
         role: 'system',

packages/backend/server/src/plugins/copilot/providers/gemini.ts

@@ -4,14 +4,10 @@ import {
 } from '@ai-sdk/google';
 import {
   AISDKError,
-  type CoreAssistantMessage,
-  type CoreUserMessage,
-  FilePart,
   generateObject,
   generateText,
   JSONParseError,
   streamText,
-  TextPart,
 } from 'ai';
 
 import {
@@ -29,35 +25,15 @@ import {
   CopilotTextToTextProvider,
   PromptMessage,
 } from './types';
+import { chatToGPTMessage } from './utils';
 
 export const DEFAULT_DIMENSIONS = 256;
 
-const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
-const FORMAT_INFER_MAP: Record<string, string> = {
-  pdf: 'application/pdf',
-  mp3: 'audio/mpeg',
-  wav: 'audio/wav',
-  png: 'image/png',
-  jpeg: 'image/jpeg',
-  jpg: 'image/jpeg',
-  webp: 'image/webp',
-  txt: 'text/plain',
-  md: 'text/plain',
-  mov: 'video/mov',
-  mpeg: 'video/mpeg',
-  mp4: 'video/mp4',
-  avi: 'video/avi',
-  wmv: 'video/wmv',
-  flv: 'video/flv',
-};
-
 export type GeminiConfig = {
   apiKey: string;
   baseUrl?: string;
 };
 
-type ChatMessage = CoreUserMessage | CoreAssistantMessage;
-
 export class GeminiProvider
   extends CopilotProvider<GeminiConfig>
   implements CopilotTextToTextProvider
@@ -67,7 +43,7 @@ export class GeminiProvider
   override readonly models = [
     // text to text
     'gemini-2.0-flash-001',
-    'gemini-2.5-pro-exp-03-25',
+    'gemini-2.5-pro-preview-03-25',
     // embeddings
     'text-embedding-004',
   ];
@@ -86,67 +62,6 @@ export class GeminiProvider
     });
   }
 
-  private inferMimeType(url: string) {
-    if (url.startsWith('data:')) {
-      return url.split(';')[0].split(':')[1];
-    }
-    const extension = url.split('.').pop();
-    if (extension) {
-      return FORMAT_INFER_MAP[extension];
-    }
-    return undefined;
-  }
-
-  protected async chatToGPTMessage(
-    messages: PromptMessage[]
-  ): Promise<[string | undefined, ChatMessage[], any]> {
-    const system =
-      messages[0]?.role === 'system' ? messages.shift() : undefined;
-    const schema = system?.params?.schema;
-
-    // filter redundant fields
-    const msgs: ChatMessage[] = [];
-    for (let { role, content, attachments, params } of messages.filter(
-      m => m.role !== 'system'
-    )) {
-      content = content.trim();
-      role = role as 'user' | 'assistant';
-      const mimetype = params?.mimetype;
-      if (Array.isArray(attachments)) {
-        const contents: (TextPart | FilePart)[] = [];
-        if (content.length) {
-          contents.push({
-            type: 'text',
-            text: content,
-          });
-        }
-
-        for (const url of attachments) {
-          if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
-            const mimeType =
-              typeof mimetype === 'string' ? mimetype : this.inferMimeType(url);
-            if (mimeType) {
-              const data = url.startsWith('data:')
-                ? await fetch(url).then(r => r.arrayBuffer())
-                : new URL(url);
-              contents.push({
-                type: 'file' as const,
-                data,
-                mimeType,
-              });
-            }
-          }
-        }
-
-        msgs.push({ role, content: contents } as ChatMessage);
-      } else {
-        msgs.push({ role, content });
-      }
-    }
-
-    return [system?.content, msgs, schema];
-  }
-
   protected async checkParams({
     messages,
     embeddings,
@@ -223,7 +138,7 @@ export class GeminiProvider
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
 
-      const [system, msgs, schema] = await this.chatToGPTMessage(messages);
+      const [system, msgs, schema] = await chatToGPTMessage(messages);
 
       const modelInstance = this.#instance(model, {
         structuredOutputs: Boolean(options.jsonMode),
@@ -237,7 +152,8 @@ export class GeminiProvider
             abortSignal: options.signal,
             experimental_repairText: async ({ text, error }) => {
               if (error instanceof JSONParseError) {
-                const ret = text.trim();
+                // strange fixed response, temporarily replace it
+                const ret = text.replaceAll(/^ny\n/g, ' ').trim();
                 if (ret.startsWith('```') || ret.endsWith('```')) {
                   return ret
                     .replace(/```[\w\s]+\n/g, '')
@@ -273,7 +189,7 @@ export class GeminiProvider
 
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model });
-      const [system, msgs] = await this.chatToGPTMessage(messages);
+      const [system, msgs] = await chatToGPTMessage(messages);
 
       const { textStream } = streamText({
         model: this.#instance(model),

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -1,4 +1,15 @@
-import { APIError, BadRequestError, ClientOptions, OpenAI } from 'openai';
+import {
+  createOpenAI,
+  type OpenAIProvider as VercelOpenAIProvider,
+} from '@ai-sdk/openai';
+import {
+  AISDKError,
+  embedMany,
+  experimental_generateImage as generateImage,
+  generateObject,
+  generateText,
+  streamText,
+} from 'ai';
 
 import {
   CopilotPromptInvalid,
@@ -20,12 +31,14 @@ import {
   CopilotTextToTextProvider,
   PromptMessage,
 } from './types';
+import { chatToGPTMessage } from './utils';
 
 export const DEFAULT_DIMENSIONS = 256;
 
-const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
-
-export type OpenAIConfig = ClientOptions;
+export type OpenAIConfig = {
+  apiKey: string;
+  baseUrl?: string;
+};
 
 export class OpenAIProvider
   extends CopilotProvider<OpenAIConfig>
@@ -62,8 +75,7 @@ export class OpenAIProvider
     'dall-e-3',
   ];
 
-  #existsModels: string[] = [];
-  #instance!: OpenAI;
+  #instance!: VercelOpenAIProvider;
 
   override configured(): boolean {
     return !!this.config.apiKey;
@@ -71,55 +83,9 @@ export class OpenAIProvider
 
   protected override setup() {
     super.setup();
-    this.#instance = new OpenAI(this.config);
-  }
-
-  override async isModelAvailable(model: string): Promise<boolean> {
-    const knownModels = this.models.includes(model);
-    if (knownModels) return true;
-
-    if (!this.#existsModels) {
-      try {
-        this.#existsModels = await this.#instance.models
-          .list()
-          .then(({ data }) => data.map(m => m.id));
-      } catch (e: any) {
-        this.logger.error('Failed to fetch online model list', e.stack);
-      }
-    }
-    return !!this.#existsModels?.includes(model);
-  }
-
-  protected chatToGPTMessage(
-    messages: PromptMessage[]
-  ): OpenAI.Chat.Completions.ChatCompletionMessageParam[] {
-    // filter redundant fields
-    return messages.map(({ role, content, attachments }) => {
-      content = content.trim();
-      if (Array.isArray(attachments) && attachments.length) {
-        const contents: OpenAI.Chat.Completions.ChatCompletionContentPart[] =
-          [];
-        if (content.length) {
-          contents.push({
-            type: 'text',
-            text: content,
-          });
-        }
-        contents.push(
-          ...(attachments
-            .filter(url => SIMPLE_IMAGE_URL_REGEX.test(url))
-            .map(url => ({
-              type: 'image_url',
-              image_url: { url, detail: 'high' },
-            })) as OpenAI.Chat.Completions.ChatCompletionContentPartImage[])
-        );
-        return {
-          role,
-          content: contents,
-        } as OpenAI.Chat.Completions.ChatCompletionMessageParam;
-      } else {
-        return { role, content };
-      }
+    this.#instance = createOpenAI({
+      apiKey: this.config.apiKey,
+      baseURL: this.config.baseUrl,
     });
   }
 
@@ -186,11 +152,8 @@ export class OpenAIProvider
   ) {
     if (e instanceof UserFriendlyError) {
       return e;
-    } else if (e instanceof APIError) {
-      if (
-        e instanceof BadRequestError &&
-        (e.message.includes('safety') || e.message.includes('risk'))
-      ) {
+    } else if (e instanceof AISDKError) {
+      if (e.message.includes('safety') || e.message.includes('risk')) {
         metrics.ai
           .counter('chat_text_risk_errors')
           .add(1, { model, user: options.user || undefined });
@@ -198,7 +161,7 @@ export class OpenAIProvider
 
       return new CopilotProviderSideError({
         provider: this.type,
-        kind: e.type || 'unknown',
+        kind: e.name || 'unknown',
         message: e.message,
       });
     } else {
@@ -217,26 +180,42 @@ export class OpenAIProvider
     options: CopilotChatOptions = {}
   ): Promise<string> {
     await this.checkParams({ messages, model, options });
+    console.log('messages', messages);
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
-      const result = await this.#instance.chat.completions.create(
-        {
-          messages: this.chatToGPTMessage(messages),
-          model: model,
-          temperature: options.temperature || 0,
-          max_completion_tokens: options.maxTokens || 4096,
-          response_format: {
-            type: options.jsonMode ? 'json_object' : 'text',
-          },
-          user: options.user,
-        },
-        { signal: options.signal }
-      );
-      const { content } = result.choices[0].message;
-      if (!content) throw new Error('Failed to generate text');
-      return content.trim();
+
+      const [system, msgs, schema] = await chatToGPTMessage(messages);
+
+      const modelInstance = this.#instance(model, {
+        structuredOutputs: Boolean(options.jsonMode),
+        user: options.user,
+      });
+
+      const commonParams = {
+        model: modelInstance,
+        system,
+        messages: msgs,
+        temperature: options.temperature || 0,
+        maxTokens: options.maxTokens || 4096,
+        abortSignal: options.signal,
+      };
+
+      const { text } = schema
+        ? await generateObject({
+            ...commonParams,
+            schema,
+          }).then(r => ({ text: JSON.stringify(r.object) }))
+        : await generateText({
+            ...commonParams,
+            providerOptions: {
+              openai: options.user ? { user: options.user } : {},
+            },
+          });
+
+      return text.trim();
     } catch (e: any) {
+      console.log('error', e);
       metrics.ai.counter('chat_text_errors').add(1, { model });
       throw this.handleError(e, model, options);
     }
@@ -251,34 +230,30 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model });
-      const result = await this.#instance.chat.completions.create(
-        {
-          stream: true,
-          messages: this.chatToGPTMessage(messages),
-          model: model,
-          frequency_penalty: options.frequencyPenalty || 0,
-          presence_penalty: options.presencePenalty || 0,
-          temperature: options.temperature || 0.5,
-          max_completion_tokens: options.maxTokens || 4096,
-          response_format: {
-            type: options.jsonMode ? 'json_object' : 'text',
-          },
-          user: options.user,
-        },
-        {
-          signal: options.signal,
-        }
-      );
 
-      for await (const message of result) {
-        if (!Array.isArray(message.choices) || !message.choices.length) {
-          continue;
-        }
-        const content = message.choices[0].delta.content;
-        if (content) {
-          yield content;
+      const [system, msgs] = await chatToGPTMessage(messages);
+
+      const modelInstance = this.#instance(model, {
+        structuredOutputs: Boolean(options.jsonMode),
+        user: options.user,
+      });
+
+      const { textStream } = streamText({
+        model: modelInstance,
+        system,
+        messages: msgs,
+        frequencyPenalty: options.frequencyPenalty || 0,
+        presencePenalty: options.presencePenalty || 0,
+        temperature: options.temperature || 0,
+        maxTokens: options.maxTokens || 4096,
+        abortSignal: options.signal,
+      });
+
+      for await (const message of textStream) {
+        if (message) {
+          yield message;
           if (options.signal?.aborted) {
-            result.controller.abort();
+            await textStream.cancel();
             break;
           }
         }
@@ -301,15 +276,18 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('generate_embedding_calls').add(1, { model });
-      const result = await this.#instance.embeddings.create({
-        model: model,
-        input: messages,
+
+      const modelInstance = this.#instance.embedding(model, {
         dimensions: options.dimensions || DEFAULT_DIMENSIONS,
         user: options.user,
       });
-      return result.data
-        .map(e => e?.embedding)
-        .filter(v => v && Array.isArray(v));
+
+      const { embeddings } = await embedMany({
+        model: modelInstance,
+        values: messages,
+      });
+
+      return embeddings.filter(v => v && Array.isArray(v));
     } catch (e: any) {
       metrics.ai.counter('generate_embedding_errors').add(1, { model });
       throw this.handleError(e, model, options);
@@ -327,19 +305,17 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('generate_images_calls').add(1, { model });
-      const result = await this.#instance.images.generate(
-        {
-          prompt,
-          model,
-          response_format: 'url',
-          user: options.user,
-        },
-        { signal: options.signal }
-      );
 
-      return result.data
-        .map(image => image.url)
-        .filter((v): v is string => !!v);
+      const modelInstance = this.#instance.image(model);
+
+      const result = await generateImage({
+        model: modelInstance,
+        prompt,
+      });
+
+      return result.images.map(
+        image => `data:image/png;base64,${image.base64}`
+      );
     } catch (e: any) {
       metrics.ai.counter('generate_images_errors').add(1, { model });
       throw this.handleError(e, model, options);

packages/backend/server/src/plugins/copilot/providers/perplexity.ts

@@ -1,4 +1,8 @@
-import { EventSourceParserStream } from 'eventsource-parser/stream';
+import {
+  createPerplexity,
+  type PerplexityProvider as VercelPerplexityProvider,
+} from '@ai-sdk/perplexity';
+import { generateText, streamText } from 'ai';
 import { z } from 'zod';
 
 import {
@@ -14,6 +18,7 @@ import {
   CopilotTextToTextProvider,
   PromptMessage,
 } from './types';
+import { chatToGPTMessage, CitationParser } from './utils';
 
 export type PerplexityConfig = {
   apiKey: string;
@@ -39,130 +44,8 @@ const PerplexityErrorSchema = z.union([
   }),
 ]);
 
-const PerplexityDataSchema = z.object({
-  citations: z.array(z.string()),
-  choices: z.array(
-    z.object({
-      message: z.object({
-        content: z.string(),
-        role: z.literal('assistant'),
-      }),
-      delta: z.object({
-        content: z.string(),
-        role: z.literal('assistant'),
-      }),
-      finish_reason: z.union([z.literal('stop'), z.literal(null)]),
-    })
-  ),
-});
-
-const PerplexitySchema = z.union([PerplexityDataSchema, PerplexityErrorSchema]);
-
 type PerplexityError = z.infer<typeof PerplexityErrorSchema>;
 
-export class CitationParser {
-  private readonly SQUARE_BRACKET_OPEN = '[';
-
-  private readonly SQUARE_BRACKET_CLOSE = ']';
-
-  private readonly PARENTHESES_OPEN = '(';
-
-  private startToken: string[] = [];
-
-  private endToken: string[] = [];
-
-  private numberToken: string[] = [];
-
-  private citations: string[] = [];
-
-  public parse(content: string, citations: string[]) {
-    this.citations = citations;
-    let result = '';
-    const contentArray = content.split('');
-    for (const [index, char] of contentArray.entries()) {
-      if (char === this.SQUARE_BRACKET_OPEN) {
-        if (this.numberToken.length === 0) {
-          this.startToken.push(char);
-        } else {
-          result += this.flush() + char;
-        }
-        continue;
-      }
-
-      if (char === this.SQUARE_BRACKET_CLOSE) {
-        this.endToken.push(char);
-        if (this.startToken.length === this.endToken.length) {
-          const cIndex = Number(this.numberToken.join('').trim());
-          if (
-            cIndex > 0 &&
-            cIndex <= citations.length &&
-            contentArray[index + 1] !== this.PARENTHESES_OPEN
-          ) {
-            const content = `[^${cIndex}]`;
-            result += content;
-            this.resetToken();
-          } else {
-            result += this.flush();
-          }
-        } else if (this.startToken.length < this.endToken.length) {
-          result += this.flush();
-        }
-        continue;
-      }
-
-      if (this.isNumeric(char)) {
-        if (this.startToken.length > 0) {
-          this.numberToken.push(char);
-        } else {
-          result += this.flush() + char;
-        }
-        continue;
-      }
-
-      if (this.startToken.length > 0) {
-        result += this.flush() + char;
-      } else {
-        result += char;
-      }
-    }
-
-    return result;
-  }
-
-  public end() {
-    return this.flush() + '\n' + this.getFootnotes();
-  }
-
-  private flush() {
-    const content = this.getTokenContent();
-    this.resetToken();
-    return content;
-  }
-
-  private getFootnotes() {
-    const footnotes = this.citations.map((citation, index) => {
-      return `[^${index + 1}]: {"type":"url","url":"${encodeURIComponent(
-        citation
-      )}"}`;
-    });
-    return footnotes.join('\n');
-  }
-
-  private getTokenContent() {
-    return this.startToken.concat(this.numberToken, this.endToken).join('');
-  }
-
-  private resetToken() {
-    this.startToken = [];
-    this.endToken = [];
-    this.numberToken = [];
-  }
-
-  private isNumeric(str: string) {
-    return !isNaN(Number(str)) && str.trim() !== '';
-  }
-}
-
 export class PerplexityProvider
   extends CopilotProvider<PerplexityConfig>
   implements CopilotTextToTextProvider
@@ -176,10 +59,20 @@ export class PerplexityProvider
     'sonar-reasoning-pro',
   ];
 
+  #instance!: VercelPerplexityProvider;
+
   override configured(): boolean {
     return !!this.config.apiKey;
   }
 
+  protected override setup() {
+    super.setup();
+    this.#instance = createPerplexity({
+      apiKey: this.config.apiKey,
+      baseURL: this.config.endpoint,
+    });
+  }
+
   async generateText(
     messages: PromptMessage[],
     model: string = 'sonar',
@@ -188,38 +81,26 @@ export class PerplexityProvider
     await this.checkParams({ messages, model, options });
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
-      const sMessages = messages
-        .map(({ content, role }) => ({ content, role }))
-        .filter(({ content }) => typeof content === 'string');
 
-      const params = {
-        method: 'POST',
-        headers: {
-          Authorization: `Bearer ${this.config.apiKey}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model,
-          messages: sMessages,
-          max_tokens: options.maxTokens || 4096,
-        }),
-      };
-      const response = await fetch(
-        this.config.endpoint || 'https://api.perplexity.ai/chat/completions',
-        params
-      );
-      const data = PerplexitySchema.parse(await response.json());
-      if ('detail' in data || 'error' in data) {
-        throw this.convertError(data);
-      } else {
-        const citationParser = new CitationParser();
-        const { content } = data.choices[0].message;
-        const { citations } = data;
-        let result = content.replaceAll(/<\/?think>\n/g, '\n---\n');
-        result = citationParser.parse(result, citations);
-        result += citationParser.end();
-        return result;
-      }
+      const [system, msgs] = await chatToGPTMessage(messages);
+
+      const modelInstance = this.#instance(model);
+
+      const { text, sources } = await generateText({
+        model: modelInstance,
+        system,
+        messages: msgs,
+        temperature: options.temperature || 0,
+        maxTokens: options.maxTokens || 4096,
+        abortSignal: options.signal,
+      });
+
+      const citationParser = new CitationParser();
+      const citations = sources.map(s => s.url);
+      let result = text.replaceAll(/<\/?think>\n/g, '\n---\n');
+      result = citationParser.parse(result, citations);
+      result += citationParser.end();
+      return result;
     } catch (e: any) {
       metrics.ai.counter('chat_text_errors').add(1, { model });
       throw this.handleError(e);
@@ -234,69 +115,54 @@ export class PerplexityProvider
     await this.checkParams({ messages, model, options });
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model });
-      const sMessages = messages
-        .map(({ content, role }) => ({ content, role }))
-        .filter(({ content }) => typeof content === 'string');
 
-      const params = {
-        method: 'POST',
-        headers: {
-          Authorization: `Bearer ${this.config.apiKey}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          model,
-          messages: sMessages,
-          max_tokens: options.maxTokens || 4096,
-          stream: true,
-        }),
-      };
-      const response = await fetch(
-        this.config.endpoint || 'https://api.perplexity.ai/chat/completions',
-        params
-      );
-      const errorHandler = this.convertError;
-      if (response.ok && response.body) {
-        const citationParser = new CitationParser();
-        const eventStream = response.body
-          .pipeThrough(new TextDecoderStream())
-          .pipeThrough(new EventSourceParserStream())
-          .pipeThrough(
-            new TransformStream({
-              transform(chunk, controller) {
-                if (options.signal?.aborted) {
-                  controller.enqueue(null);
-                  return;
-                }
-                const json = JSON.parse(chunk.data);
-                if (json) {
-                  const data = PerplexitySchema.parse(json);
-                  if ('detail' in data || 'error' in data) {
-                    throw errorHandler(data);
-                  }
-                  const { content } = data.choices[0].delta;
-                  const { citations } = data;
-                  let result = content.replaceAll(/<\/?think>\n?/g, '\n---\n');
-                  result = citationParser.parse(result, citations);
-                  controller.enqueue(result);
-                }
-              },
-              flush(controller) {
-                controller.enqueue(citationParser.end());
-                controller.enqueue(null);
-              },
-            })
-          );
+      const [system, msgs] = await chatToGPTMessage(messages);
 
-        const reader = eventStream.getReader();
-        while (true) {
-          const { done, value } = await reader.read();
-          if (done) break;
-          yield value;
+      const modelInstance = this.#instance(model);
+
+      const stream = streamText({
+        model: modelInstance,
+        system,
+        messages: msgs,
+        temperature: options.temperature || 0,
+        maxTokens: options.maxTokens || 4096,
+        abortSignal: options.signal,
+      });
+
+      const citationParser = new CitationParser();
+      const citations = [];
+      for await (const chunk of stream.fullStream) {
+        switch (chunk.type) {
+          case 'source': {
+            citations.push(chunk.source.url);
+            break;
+          }
+          case 'text-delta': {
+            const result = citationParser.parse(
+              chunk.textDelta.replaceAll(/<\/?think>\n?/g, '\n---\n'),
+              citations
+            );
+            yield result;
+            break;
+          }
+          case 'step-finish': {
+            const result = citationParser.end();
+            yield result;
+            break;
+          }
+          case 'error': {
+            const json =
+              typeof chunk.error === 'string'
+                ? JSON.parse(chunk.error)
+                : chunk.error;
+            if (json && typeof json === 'object') {
+              const data = PerplexityErrorSchema.parse(json);
+              if ('detail' in data || 'error' in data) {
+                throw this.convertError(data);
+              }
+            }
+          }
         }
-      } else {
-        const result = await this.generateText(messages, model, options);
-        yield result;
       }
     } catch (e) {
       metrics.ai.counter('chat_text_stream_errors').add(1, { model });

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -0,0 +1,201 @@
+import {
+  CoreAssistantMessage,
+  CoreUserMessage,
+  FilePart,
+  ImagePart,
+  TextPart,
+} from 'ai';
+
+import { PromptMessage } from './types';
+
+type ChatMessage = CoreUserMessage | CoreAssistantMessage;
+
+const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
+const FORMAT_INFER_MAP: Record<string, string> = {
+  pdf: 'application/pdf',
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  png: 'image/png',
+  jpeg: 'image/jpeg',
+  jpg: 'image/jpeg',
+  webp: 'image/webp',
+  txt: 'text/plain',
+  md: 'text/plain',
+  mov: 'video/mov',
+  mpeg: 'video/mpeg',
+  mp4: 'video/mp4',
+  avi: 'video/avi',
+  wmv: 'video/wmv',
+  flv: 'video/flv',
+};
+
+function inferMimeType(url: string) {
+  if (url.startsWith('data:')) {
+    return url.split(';')[0].split(':')[1];
+  }
+  const extension = url.split('.').pop();
+  if (extension) {
+    return FORMAT_INFER_MAP[extension];
+  }
+  return undefined;
+}
+
+export async function chatToGPTMessage(
+  messages: PromptMessage[]
+): Promise<[string | undefined, ChatMessage[], any]> {
+  const system = messages[0]?.role === 'system' ? messages.shift() : undefined;
+  const schema = system?.params?.schema;
+
+  // filter redundant fields
+  const msgs: ChatMessage[] = [];
+  for (let { role, content, attachments, params } of messages.filter(
+    m => m.role !== 'system'
+  )) {
+    content = content.trim();
+    role = role as 'user' | 'assistant';
+    const mimetype = params?.mimetype;
+    if (Array.isArray(attachments)) {
+      const contents: (TextPart | ImagePart | FilePart)[] = [];
+      if (content.length) {
+        contents.push({
+          type: 'text',
+          text: content,
+        });
+      }
+
+      for (const url of attachments) {
+        if (SIMPLE_IMAGE_URL_REGEX.test(url)) {
+          const mimeType =
+            typeof mimetype === 'string' ? mimetype : inferMimeType(url);
+          if (mimeType) {
+            if (mimeType.startsWith('image/')) {
+              contents.push({
+                type: 'image',
+                image: url,
+                mimeType,
+              });
+            } else {
+              const data = url.startsWith('data:')
+                ? await fetch(url).then(r => r.arrayBuffer())
+                : new URL(url);
+              contents.push({
+                type: 'file' as const,
+                data,
+                mimeType,
+              });
+            }
+          }
+        }
+      }
+
+      msgs.push({ role, content: contents } as ChatMessage);
+    } else {
+      msgs.push({ role, content });
+    }
+  }
+
+  return [system?.content, msgs, schema];
+}
+
+export class CitationParser {
+  private readonly SQUARE_BRACKET_OPEN = '[';
+
+  private readonly SQUARE_BRACKET_CLOSE = ']';
+
+  private readonly PARENTHESES_OPEN = '(';
+
+  private startToken: string[] = [];
+
+  private endToken: string[] = [];
+
+  private numberToken: string[] = [];
+
+  private citations: string[] = [];
+
+  public parse(content: string, citations: string[]) {
+    this.citations = citations;
+    let result = '';
+    const contentArray = content.split('');
+    for (const [index, char] of contentArray.entries()) {
+      if (char === this.SQUARE_BRACKET_OPEN) {
+        if (this.numberToken.length === 0) {
+          this.startToken.push(char);
+        } else {
+          result += this.flush() + char;
+        }
+        continue;
+      }
+
+      if (char === this.SQUARE_BRACKET_CLOSE) {
+        this.endToken.push(char);
+        if (this.startToken.length === this.endToken.length) {
+          const cIndex = Number(this.numberToken.join('').trim());
+          if (
+            cIndex > 0 &&
+            cIndex <= citations.length &&
+            contentArray[index + 1] !== this.PARENTHESES_OPEN
+          ) {
+            const content = `[^${cIndex}]`;
+            result += content;
+            this.resetToken();
+          } else {
+            result += this.flush();
+          }
+        } else if (this.startToken.length < this.endToken.length) {
+          result += this.flush();
+        }
+        continue;
+      }
+
+      if (this.isNumeric(char)) {
+        if (this.startToken.length > 0) {
+          this.numberToken.push(char);
+        } else {
+          result += this.flush() + char;
+        }
+        continue;
+      }
+
+      if (this.startToken.length > 0) {
+        result += this.flush() + char;
+      } else {
+        result += char;
+      }
+    }
+
+    return result;
+  }
+
+  public end() {
+    return this.flush() + '\n' + this.getFootnotes();
+  }
+
+  private flush() {
+    const content = this.getTokenContent();
+    this.resetToken();
+    return content;
+  }
+
+  private getFootnotes() {
+    const footnotes = this.citations.map((citation, index) => {
+      return `[^${index + 1}]: {"type":"url","url":"${encodeURIComponent(
+        citation
+      )}"}`;
+    });
+    return footnotes.join('\n');
+  }
+
+  private getTokenContent() {
+    return this.startToken.concat(this.numberToken, this.endToken).join('');
+  }
+
+  private resetToken() {
+    this.startToken = [];
+    this.endToken = [];
+    this.numberToken = [];
+  }
+
+  private isNumeric(str: string) {
+    return !isNaN(Number(str)) && str.trim() !== '';
+  }
+}

packages/backend/server/src/plugins/copilot/transcript/resolver.ts

@@ -14,6 +14,7 @@ import { AiJobStatus } from '@prisma/client';
 import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';
 
 import {
+  CopilotTranscriptionAudioNotProvided,
   CopilotTranscriptionJobNotFound,
   type FileUpload,
 } from '../../../base';
@@ -100,20 +101,27 @@ export class CopilotTranscriptionResolver {
     @CurrentUser() user: CurrentUser,
     @Args('workspaceId') workspaceId: string,
     @Args('blobId') blobId: string,
-    @Args({ name: 'blob', type: () => GraphQLUpload })
-    blob: FileUpload
+    @Args({ name: 'blob', type: () => GraphQLUpload, nullable: true })
+    blob: FileUpload | null,
+    @Args({ name: 'blobs', type: () => [GraphQLUpload], nullable: true })
+    blobs: FileUpload[] | null
   ): Promise<TranscriptionResultType | null> {
     await this.ac
       .user(user.id)
       .workspace(workspaceId)
       .allowLocal()
       .assert('Workspace.Copilot');
+    // merge blobs
+    const allBlobs = blob ? [blob, ...(blobs || [])].filter(v => !!v) : blobs;
+    if (!allBlobs || allBlobs.length === 0) {
+      throw new CopilotTranscriptionAudioNotProvided();
+    }
 
     const jobResult = await this.service.submitTranscriptionJob(
       user.id,
       workspaceId,
       blobId,
-      blob
+      await Promise.all(allBlobs)
     );
 
     return this.handleJobResult(jobResult);
@@ -136,14 +144,13 @@ export class CopilotTranscriptionResolver {
       workspaceId,
       jobId
     );
-    if (!job || !job.url || !job.mimeType) {
+    if (!job || !job.infos) {
       throw new CopilotTranscriptionJobNotFound();
     }
 
     const jobResult = await this.service.executeTranscriptionJob(
       job.id,
-      job.url,
-      job.mimeType
+      job.infos
     );
 
     return this.handleJobResult(jobResult);

packages/backend/server/src/plugins/copilot/transcript/service.ts

@@ -23,6 +23,7 @@ import {
 } from '../providers';
 import { CopilotStorage } from '../storage';
 import {
+  AudioBlobInfos,
   TranscriptionPayload,
   TranscriptionResponseSchema,
   TranscriptPayloadSchema,
@@ -32,8 +33,7 @@ import { readStream } from './utils';
 export type TranscriptionJob = {
   id: string;
   status: AiJobStatus;
-  url?: string;
-  mimeType?: string;
+  infos?: AudioBlobInfos;
   transcription?: TranscriptionPayload;
 };
 
@@ -52,7 +52,7 @@ export class CopilotTranscriptionService {
     userId: string,
     workspaceId: string,
     blobId: string,
-    blob: FileUpload
+    blobs: FileUpload[]
   ): Promise<TranscriptionJob> {
     if (await this.models.copilotJob.has(userId, workspaceId, blobId)) {
       throw new CopilotTranscriptionJobExists();
@@ -65,21 +65,24 @@ export class CopilotTranscriptionService {
       type: AiJobType.transcription,
     });
 
-    const buffer = await readStream(blob.createReadStream());
-    const url = await this.storage.put(userId, workspaceId, blobId, buffer);
+    const infos: AudioBlobInfos = [];
+    for (const blob of blobs) {
+      const buffer = await readStream(blob.createReadStream());
+      const url = await this.storage.put(userId, workspaceId, blobId, buffer);
+      infos.push({ url, mimeType: blob.mimetype });
+    }
 
-    return await this.executeTranscriptionJob(jobId, url, blob.mimetype);
+    return await this.executeTranscriptionJob(jobId, infos);
   }
 
   async executeTranscriptionJob(
     jobId: string,
-    url: string,
-    mimeType: string
+    infos: AudioBlobInfos
   ): Promise<TranscriptionJob> {
     const status = AiJobStatus.running;
     const success = await this.models.copilotJob.update(jobId, {
       status,
-      payload: { url, mimeType },
+      payload: { infos },
     });
 
     if (!success) {
@@ -88,8 +91,7 @@ export class CopilotTranscriptionService {
 
     await this.job.add('copilot.transcript.submit', {
       jobId,
-      url,
-      mimeType,
+      infos,
     });
 
     return { id: jobId, status };
@@ -132,8 +134,13 @@ export class CopilotTranscriptionService {
 
     const payload = TranscriptPayloadSchema.safeParse(job.payload);
     if (payload.success) {
-      ret.url = payload.data.url || undefined;
-      ret.mimeType = payload.data.mimeType || undefined;
+      let { url, mimeType, infos } = payload.data;
+      infos = infos || [];
+      if (url && mimeType) {
+        infos.push({ url, mimeType });
+      }
+
+      ret.infos = this.mergeInfos(infos, url, mimeType);
       if (job.status === AiJobStatus.claimed) {
         ret.transcription = payload.data;
       }
@@ -173,7 +180,24 @@ export class CopilotTranscriptionService {
     );
   }
 
-  private convertTime(time: number) {
+  // TODO(@darkskygit): remove after old server down
+  private mergeInfos(
+    infos?: AudioBlobInfos | null,
+    url?: string | null,
+    mimeType?: string | null
+  ) {
+    if (url && mimeType) {
+      if (infos) {
+        infos.push({ url, mimeType });
+      } else {
+        infos = [{ url, mimeType }];
+      }
+    }
+    return infos || [];
+  }
+
+  private convertTime(time: number, offset = 0) {
+    time = time + offset;
     const minutes = Math.floor(time / 60);
     const seconds = Math.floor(time % 60);
     const hours = Math.floor(minutes / 60);
@@ -186,29 +210,38 @@ export class CopilotTranscriptionService {
   @OnJob('copilot.transcript.submit')
   async transcriptAudio({
     jobId,
+    infos,
+    // @deprecated
     url,
     mimeType,
   }: Jobs['copilot.transcript.submit']) {
     try {
-      const result = await this.chatWithPrompt(
-        'Transcript audio',
-        {
-          attachments: [url],
-          params: { mimetype: mimeType },
-        },
-        TranscriptionResponseSchema
-      );
+      const blobInfos = this.mergeInfos(infos, url, mimeType);
+      const transcriptions = [];
+      for (const [idx, { url, mimeType }] of blobInfos.entries()) {
+        const result = await this.chatWithPrompt(
+          'Transcript audio',
+          {
+            attachments: [url],
+            params: { mimetype: mimeType },
+          },
+          TranscriptionResponseSchema
+        );
+
+        const offset = idx * 10 * 60;
+        const transcription = TranscriptionResponseSchema.parse(
+          JSON.parse(result)
+        ).map(t => ({
+          speaker: t.a,
+          start: this.convertTime(t.s, offset),
+          end: this.convertTime(t.e, offset),
+          transcription: t.t,
+        }));
+        transcriptions.push(transcription);
+      }
 
-      const transcription = TranscriptionResponseSchema.parse(
-        JSON.parse(result)
-      ).map(t => ({
-        speaker: t.a,
-        start: this.convertTime(t.s),
-        end: this.convertTime(t.e),
-        transcription: t.t,
-      }));
       await this.models.copilotJob.update(jobId, {
-        payload: { transcription },
+        payload: { transcription: transcriptions.flat() },
       });
 
       await this.job.add('copilot.transcript.summary.submit', {

packages/backend/server/src/plugins/copilot/transcript/types.ts

@@ -20,9 +20,17 @@ const TranscriptionItemSchema = z.object({
 
 export const TranscriptionSchema = z.array(TranscriptionItemSchema);
 
+export const AudioBlobInfosSchema = z
+  .object({
+    url: z.string(),
+    mimeType: z.string(),
+  })
+  .array();
+
 export const TranscriptPayloadSchema = z.object({
   url: z.string().nullable().optional(),
   mimeType: z.string().nullable().optional(),
+  infos: AudioBlobInfosSchema.nullable().optional(),
   title: z.string().nullable().optional(),
   summary: z.string().nullable().optional(),
   transcription: TranscriptionSchema.nullable().optional(),
@@ -32,6 +40,8 @@ export type TranscriptionItem = z.infer<typeof TranscriptionItemSchema>;
 export type Transcription = z.infer<typeof TranscriptionSchema>;
 export type TranscriptionPayload = z.infer<typeof TranscriptPayloadSchema>;
 
+export type AudioBlobInfos = z.infer<typeof AudioBlobInfosSchema>;
+
 declare global {
   interface Events {
     'workspace.file.transcript.finished': {
@@ -44,8 +54,11 @@ declare global {
   interface Jobs {
     'copilot.transcript.submit': {
       jobId: string;
-      url: string;
-      mimeType: string;
+      infos?: AudioBlobInfos;
+      /// @deprecated use `infos` instead
+      url?: string;
+      /// @deprecated use `infos` instead
+      mimeType?: string;
     };
     'copilot.transcript.summary.submit': {
       jobId: string;

packages/backend/server/src/schema.gql

@@ -435,6 +435,7 @@ enum ErrorNames {
   COPILOT_QUOTA_EXCEEDED
   COPILOT_SESSION_DELETED
   COPILOT_SESSION_NOT_FOUND
+  COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED
   COPILOT_TRANSCRIPTION_JOB_EXISTS
   COPILOT_TRANSCRIPTION_JOB_NOT_FOUND
   CUSTOMER_PORTAL_CREATE_FAILED
@@ -1004,7 +1005,7 @@ type Mutation {
   sendVerifyChangeEmail(callbackUrl: String!, email: String!, token: String!): Boolean!
   sendVerifyEmail(callbackUrl: String!): Boolean!
   setBlob(blob: Upload!, workspaceId: String!): String!
-  submitAudioTranscription(blob: Upload!, blobId: String!, workspaceId: String!): TranscriptionResultType
+  submitAudioTranscription(blob: Upload, blobId: String!, blobs: [Upload!], workspaceId: String!): TranscriptionResultType
 
   """update app configuration"""
   updateAppConfig(updates: [UpdateAppConfigInput!]!): JSONObject!

packages/common/env/src/worker.ts

@@ -1,7 +1,5 @@
 export function getWorkerUrl(name: string) {
-  if (BUILD_CONFIG.debug && !name.endsWith('.worker.js')) {
-    throw new Error(`worker should be named with '.worker.js', get ${name}`);
-  }
-
-  return environment.workerPath + name + '?v=' + BUILD_CONFIG.appVersion;
+  return (
+    environment.workerPath + `${name}-${BUILD_CONFIG.appVersion}.worker.js`
+  );
 }

packages/common/graphql/export-gql-plugin.cjs

@@ -163,7 +163,10 @@ module.exports = {
 
             // parse 'file' fields
             const containsFile = node.variableDefinitions.some(def => {
-              const varType = def?.type?.type?.name?.value;
+              const varType =
+                def.type.kind === 'NamedType'
+                  ? def.type.name.value
+                  : def?.type?.type?.name?.value;
               const checkContainFile = type => {
                 if (schema.getType(type)?.name === 'Upload') return true;
                 const typeDef = schema.getType(type);

packages/common/graphql/src/graphql/copilot-jobs-transcription-add.gql

@@ -1,5 +1,5 @@
-mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload!) {
-  submitAudioTranscription(blob: $blob, blobId: $blobId, workspaceId: $workspaceId) {
+mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload, $blobs: [Upload!]) {
+  submitAudioTranscription(blob: $blob, blobs: $blobs, blobId: $blobId, workspaceId: $workspaceId) {
     id
     status
   }

packages/common/graphql/src/graphql/index.ts

@@ -601,9 +601,10 @@ export const getCopilotHistoriesQuery = {
 export const submitAudioTranscriptionMutation = {
   id: 'submitAudioTranscriptionMutation' as const,
   op: 'submitAudioTranscription',
-  query: `mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload!) {
+  query: `mutation submitAudioTranscription($workspaceId: String!, $blobId: String!, $blob: Upload, $blobs: [Upload!]) {
   submitAudioTranscription(
     blob: $blob
+    blobs: $blobs
     blobId: $blobId
     workspaceId: $workspaceId
   ) {

packages/common/graphql/src/schema.ts

@@ -580,6 +580,7 @@ export enum ErrorNames {
   COPILOT_QUOTA_EXCEEDED = 'COPILOT_QUOTA_EXCEEDED',
   COPILOT_SESSION_DELETED = 'COPILOT_SESSION_DELETED',
   COPILOT_SESSION_NOT_FOUND = 'COPILOT_SESSION_NOT_FOUND',
+  COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED = 'COPILOT_TRANSCRIPTION_AUDIO_NOT_PROVIDED',
   COPILOT_TRANSCRIPTION_JOB_EXISTS = 'COPILOT_TRANSCRIPTION_JOB_EXISTS',
   COPILOT_TRANSCRIPTION_JOB_NOT_FOUND = 'COPILOT_TRANSCRIPTION_JOB_NOT_FOUND',
   CUSTOMER_PORTAL_CREATE_FAILED = 'CUSTOMER_PORTAL_CREATE_FAILED',
@@ -1432,8 +1433,9 @@ export interface MutationSetBlobArgs {
 }
 
 export interface MutationSubmitAudioTranscriptionArgs {
-  blob: Scalars['Upload']['input'];
+  blob?: InputMaybe<Scalars['Upload']['input']>;
   blobId: Scalars['String']['input'];
+  blobs?: InputMaybe<Array<Scalars['Upload']['input']>>;
   workspaceId: Scalars['String']['input'];
 }
 
@@ -2991,7 +2993,10 @@ export type GetCopilotHistoriesQuery = {
 export type SubmitAudioTranscriptionMutationVariables = Exact<{
   workspaceId: Scalars['String']['input'];
   blobId: Scalars['String']['input'];
-  blob: Scalars['Upload']['input'];
+  blob?: InputMaybe<Scalars['Upload']['input']>;
+  blobs?: InputMaybe<
+    Array<Scalars['Upload']['input']> | Scalars['Upload']['input']
+  >;
 }>;
 
 export type SubmitAudioTranscriptionMutation = {

packages/common/infra/package.json

@@ -16,7 +16,6 @@
     "@affine/env": "workspace:*",
     "@affine/error": "workspace:*",
     "@affine/templates": "workspace:*",
-    "@datastructures-js/binary-search-tree": "^5.3.2",
     "@preact/signals-core": "^1.8.0",
     "eventemitter2": "^6.4.9",
     "foxact": "^0.2.43",

packages/common/infra/src/livedata/livedata.ts

@@ -577,8 +577,8 @@ export type LiveDataOperation = 'set' | 'get' | 'watch' | 'unwatch';
 export type Unwrap<T> =
   T extends LiveData<infer Z>
     ? Unwrap<Z>
-    : T extends LiveData<infer A>[]
-      ? Unwrap<A>[]
+    : T extends readonly [...infer Elements]
+      ? { [K in keyof Elements]: Unwrap<Elements[K]> }
       : T;
 
 export type Flat<T> = T extends LiveData<infer P> ? LiveData<Unwrap<P>> : T;