diff --git a/blocksuite/affine/all/package.json b/blocksuite/affine/all/package.json
index 8143ace925..25e6ecf658 100644
--- a/blocksuite/affine/all/package.json
+++ b/blocksuite/affine/all/package.json
@@ -174,7 +174,9 @@
     "./fragments/frame-panel": "./src/fragments/frame-panel.ts",
     "./fragments/outline": "./src/fragments/outline.ts",
     "./gfx/text": "./src/gfx/text.ts",
-    "./gfx/brush": "./src/gfx/brush.ts",
+    "./gfx/brush": "./src/gfx/brush/index.ts",
+    "./gfx/brush/store": "./src/gfx/brush/store.ts",
+    "./gfx/brush/view": "./src/gfx/brush/view.ts",
     "./gfx/shape": "./src/gfx/shape.ts",
     "./gfx/note": "./src/gfx/note.ts",
     "./gfx/mindmap": "./src/gfx/mindmap.ts",
diff --git a/blocksuite/affine/all/src/extensions/migrating.ts b/blocksuite/affine/all/src/extensions/migrating.ts
index cb8f1e3ac4..bd444b1c80 100644
--- a/blocksuite/affine/all/src/extensions/migrating.ts
+++ b/blocksuite/affine/all/src/extensions/migrating.ts
@@ -11,10 +11,6 @@ import {
   PageSurfaceBlockSpec,
   SurfaceBlockAdapterExtensions,
 } from '@blocksuite/affine-block-surface';
-import {
-  brushToMarkdownAdapterMatcher,
-  brushToPlainTextAdapterMatcher,
-} from '@blocksuite/affine-gfx-brush';
 import {
   connectorToMarkdownAdapterMatcher,
   connectorToPlainTextAdapterMatcher,
@@ -48,7 +44,6 @@ const elementToPlainTextAdapterMatchers = [
   groupToPlainTextAdapterMatcher,
   shapeToPlainTextAdapterMatcher,
   connectorToPlainTextAdapterMatcher,
-  brushToPlainTextAdapterMatcher,
   textToPlainTextAdapterMatcher,
   mindmapToPlainTextAdapterMatcher,
 ];
@@ -57,7 +52,6 @@ const elementToMarkdownAdapterMatchers = [
   groupToMarkdownAdapterMatcher,
   shapeToMarkdownAdapterMatcher,
   connectorToMarkdownAdapterMatcher,
-  brushToMarkdownAdapterMatcher,
   textToMarkdownAdapterMatcher,
   mindmapToMarkdownAdapterMatcher,
 ];
diff --git a/blocksuite/affine/all/src/extensions/store.ts b/blocksuite/affine/all/src/extensions/store.ts
index 4515a95f40..0ffe078d76 100644
--- a/blocksuite/affine/all/src/extensions/store.ts
+++ b/blocksuite/affine/all/src/extensions/store.ts
@@ -15,6 +15,7 @@ import { NoteStoreExtension } from '@blocksuite/affine-block-note/store';
 import { ParagraphStoreExtension } from '@blocksuite/affine-block-paragraph/store';
 import { SurfaceRefStoreExtension } from '@blocksuite/affine-block-surface-ref/store';
 import { TableStoreExtension } from '@blocksuite/affine-block-table/store';
+import { BrushStoreExtension } from '@blocksuite/affine-gfx-brush/store';
 import { FootnoteStoreExtension } from '@blocksuite/affine-inline-footnote/store';
 import { LatexStoreExtension as InlineLatexStoreExtension } from '@blocksuite/affine-inline-latex/store';
 import { LinkStoreExtension } from '@blocksuite/affine-inline-link/store';
@@ -49,6 +50,8 @@ export function getInternalStoreExtensions() {
     InlineLatexStoreExtension,
     InlinePresetStoreExtension,
 
+    BrushStoreExtension,
+
     MigratingStoreExtension,
   ];
 }
diff --git a/blocksuite/affine/all/src/extensions/view.ts b/blocksuite/affine/all/src/extensions/view.ts
index 79b211b27c..ca39773f7a 100644
--- a/blocksuite/affine/all/src/extensions/view.ts
+++ b/blocksuite/affine/all/src/extensions/view.ts
@@ -15,6 +15,8 @@ import { NoteViewExtension } from '@blocksuite/affine-block-note/view';
 import { ParagraphViewExtension } from '@blocksuite/affine-block-paragraph/view';
 import { SurfaceRefViewExtension } from '@blocksuite/affine-block-surface-ref/view';
 import { TableViewExtension } from '@blocksuite/affine-block-table/view';
+import { BrushViewExtension } from '@blocksuite/affine-gfx-brush/view';
+import { NoteViewExtension as GfxNoteViewExtension } from '@blocksuite/affine-gfx-note/view';
 import { FootnoteViewExtension } from '@blocksuite/affine-inline-footnote/view';
 import { LatexViewExtension as InlineLatexViewExtension } from '@blocksuite/affine-inline-latex/view';
 import { LinkViewExtension } from '@blocksuite/affine-inline-link/view';
@@ -26,6 +28,12 @@ import { MigratingViewExtension } from './migrating-view';
 
 export function getInternalViewExtensions() {
   return [
+    GfxNoteViewExtension,
+    BrushViewExtension,
+    // shape
+    // mind
+    // template
+
     MigratingViewExtension,
 
     // Block
diff --git a/blocksuite/affine/all/src/gfx/brush.ts b/blocksuite/affine/all/src/gfx/brush/index.ts
similarity index 100%
rename from blocksuite/affine/all/src/gfx/brush.ts
rename to blocksuite/affine/all/src/gfx/brush/index.ts
diff --git a/blocksuite/affine/all/src/gfx/brush/store.ts b/blocksuite/affine/all/src/gfx/brush/store.ts
new file mode 100644
index 0000000000..de2f133d9d
--- /dev/null
+++ b/blocksuite/affine/all/src/gfx/brush/store.ts
@@ -0,0 +1 @@
+export * from '@blocksuite/affine-gfx-brush/store';
diff --git a/blocksuite/affine/all/src/gfx/brush/view.ts b/blocksuite/affine/all/src/gfx/brush/view.ts
new file mode 100644
index 0000000000..06fb434a2a
--- /dev/null
+++ b/blocksuite/affine/all/src/gfx/brush/view.ts
@@ -0,0 +1 @@
+export * from '@blocksuite/affine-gfx-brush/view';
diff --git a/blocksuite/affine/all/src/gfx/note.ts b/blocksuite/affine/all/src/gfx/note/index.ts
similarity index 100%
rename from blocksuite/affine/all/src/gfx/note.ts
rename to blocksuite/affine/all/src/gfx/note/index.ts
diff --git a/blocksuite/affine/all/src/gfx/note/view.ts b/blocksuite/affine/all/src/gfx/note/view.ts
new file mode 100644
index 0000000000..a271a4842c
--- /dev/null
+++ b/blocksuite/affine/all/src/gfx/note/view.ts
@@ -0,0 +1 @@
+export * from '@blocksuite/affine-gfx-note/view';
diff --git a/blocksuite/affine/blocks/root/src/common-specs/index.ts b/blocksuite/affine/blocks/root/src/common-specs/index.ts
index 84558fa364..55031c5d47 100644
--- a/blocksuite/affine/blocks/root/src/common-specs/index.ts
+++ b/blocksuite/affine/blocks/root/src/common-specs/index.ts
@@ -1,5 +1,4 @@
 import { FileDropExtension } from '@blocksuite/affine-components/drop-indicator';
-import { BrushElementRendererExtension } from '@blocksuite/affine-gfx-brush';
 import {
   ConnectorElementRendererExtension,
   ConnectorElementView,
@@ -61,7 +60,6 @@ const EdgelessElementViews = [
 ];
 
 export const EdgelessElementRendererExtension: ExtensionType[] = [
-  BrushElementRendererExtension,
   HighlighterElementRendererExtension,
   ShapeElementRendererExtension,
   TextElementRendererExtension,
diff --git a/blocksuite/affine/blocks/root/src/edgeless/components/toolbar/tools.ts b/blocksuite/affine/blocks/root/src/edgeless/components/toolbar/tools.ts
index 9642bc4668..83c8d97f12 100644
--- a/blocksuite/affine/blocks/root/src/edgeless/components/toolbar/tools.ts
+++ b/blocksuite/affine/blocks/root/src/edgeless/components/toolbar/tools.ts
@@ -1,8 +1,6 @@
 import { frameQuickTool } from '@blocksuite/affine-block-frame';
-import { penSeniorTool } from '@blocksuite/affine-gfx-brush';
 import { connectorQuickTool } from '@blocksuite/affine-gfx-connector';
 import { mindMapSeniorTool } from '@blocksuite/affine-gfx-mindmap';
-import { noteSeniorTool } from '@blocksuite/affine-gfx-note';
 import { shapeSeniorTool } from '@blocksuite/affine-gfx-shape';
 import { templateSeniorTool } from '@blocksuite/affine-gfx-template';
 import { QuickToolExtension } from '@blocksuite/affine-widget-edgeless-toolbar';
@@ -36,8 +34,6 @@ export const quickTools = [
 ];
 
 export const seniorTools = [
-  noteSeniorTool,
-  penSeniorTool,
   shapeSeniorTool,
   mindMapSeniorTool,
   templateSeniorTool,
diff --git a/blocksuite/affine/blocks/root/src/edgeless/configs/toolbar/index.ts b/blocksuite/affine/blocks/root/src/edgeless/configs/toolbar/index.ts
index 70490d5f08..0cefe12c50 100644
--- a/blocksuite/affine/blocks/root/src/edgeless/configs/toolbar/index.ts
+++ b/blocksuite/affine/blocks/root/src/edgeless/configs/toolbar/index.ts
@@ -1,9 +1,5 @@
 import { edgelessTextToolbarExtension } from '@blocksuite/affine-block-edgeless-text';
 import { frameToolbarExtension } from '@blocksuite/affine-block-frame';
-import {
-  brushToolbarExtension,
-  highlighterToolbarExtension,
-} from '@blocksuite/affine-gfx-brush';
 import { connectorToolbarExtension } from '@blocksuite/affine-gfx-connector';
 import { groupToolbarExtension } from '@blocksuite/affine-gfx-group';
 import {
@@ -23,10 +19,6 @@ export const EdgelessElementToolbarExtension: ExtensionType[] = [
 
   groupToolbarExtension,
 
-  brushToolbarExtension,
-
-  highlighterToolbarExtension,
-
   connectorToolbarExtension,
 
   shapeToolbarExtension,
diff --git a/blocksuite/affine/blocks/root/src/edgeless/edgeless-builtin-spec.ts b/blocksuite/affine/blocks/root/src/edgeless/edgeless-builtin-spec.ts
index 1603272272..54e32ef030 100644
--- a/blocksuite/affine/blocks/root/src/edgeless/edgeless-builtin-spec.ts
+++ b/blocksuite/affine/blocks/root/src/edgeless/edgeless-builtin-spec.ts
@@ -4,11 +4,7 @@ import {
   PresentTool,
 } from '@blocksuite/affine-block-frame';
 import { ConnectionOverlay } from '@blocksuite/affine-block-surface';
-import {
-  BrushTool,
-  EraserTool,
-  HighlighterTool,
-} from '@blocksuite/affine-gfx-brush';
+import type * as BrushEffect from '@blocksuite/affine-gfx-brush';
 import {
   ConnectorFilter,
   ConnectorTool,
@@ -17,7 +13,7 @@ import {
   MindMapDragExtension,
   MindMapIndicatorOverlay,
 } from '@blocksuite/affine-gfx-mindmap';
-import { NoteTool } from '@blocksuite/affine-gfx-note';
+import type * as NoteEffect from '@blocksuite/affine-gfx-note';
 import { ShapeTool } from '@blocksuite/affine-gfx-shape';
 import { TemplateTool } from '@blocksuite/affine-gfx-template';
 import { TextTool } from '@blocksuite/affine-gfx-text';
@@ -35,20 +31,18 @@ import { SnapExtension } from './interact-extensions/snap-manager.js';
 import { EditPropsMiddlewareBuilder } from './middlewares/base.js';
 import { SnapOverlay } from './utils/snap-manager.js';
 
+declare type _GLOBAL_ = typeof NoteEffect | typeof BrushEffect;
+
 export const EdgelessToolExtension: ExtensionType[] = [
   DefaultTool,
   PanTool,
-  EraserTool,
   TextTool,
   ShapeTool,
-  NoteTool,
-  BrushTool,
   ConnectorTool,
   TemplateTool,
   EmptyTool,
   FrameTool,
   PresentTool,
-  HighlighterTool,
 ];
 
 export const EdgelessEditExtensions: ExtensionType[] = [
diff --git a/blocksuite/affine/blocks/root/src/effects.ts b/blocksuite/affine/blocks/root/src/effects.ts
index 6efa601f1b..57a151518e 100644
--- a/blocksuite/affine/blocks/root/src/effects.ts
+++ b/blocksuite/affine/blocks/root/src/effects.ts
@@ -1,8 +1,6 @@
-import { effects as gfxBrushEffects } from '@blocksuite/affine-gfx-brush/effects';
 import { effects as gfxConnectorEffects } from '@blocksuite/affine-gfx-connector/effects';
 import { effects as gfxGroupEffects } from '@blocksuite/affine-gfx-group/effects';
 import { effects as gfxMindmapEffects } from '@blocksuite/affine-gfx-mindmap/effects';
-import { effects as gfxNoteEffects } from '@blocksuite/affine-gfx-note/effects';
 import { effects as gfxShapeEffects } from '@blocksuite/affine-gfx-shape/effects';
 import { effects as gfxTemplateEffects } from '@blocksuite/affine-gfx-template/effects';
 import { effects as gfxCanvasTextEffects } from '@blocksuite/affine-gfx-text/effects';
@@ -80,11 +78,9 @@ function registerRootComponents() {
 function registerGfxEffects() {
   gfxCanvasTextEffects();
   gfxShapeEffects();
-  gfxNoteEffects();
   gfxConnectorEffects();
   gfxMindmapEffects();
   gfxGroupEffects();
-  gfxBrushEffects();
   gfxTemplateEffects();
 }
 
diff --git a/blocksuite/affine/gfx/brush/package.json b/blocksuite/affine/gfx/brush/package.json
index 2d8b935471..187b768c79 100644
--- a/blocksuite/affine/gfx/brush/package.json
+++ b/blocksuite/affine/gfx/brush/package.json
@@ -12,6 +12,7 @@
   "dependencies": {
     "@blocksuite/affine-block-surface": "workspace:*",
     "@blocksuite/affine-components": "workspace:*",
+    "@blocksuite/affine-ext-loader": "workspace:*",
     "@blocksuite/affine-model": "workspace:*",
     "@blocksuite/affine-rich-text": "workspace:*",
     "@blocksuite/affine-shared": "workspace:*",
@@ -33,7 +34,9 @@
   },
   "exports": {
     ".": "./src/index.ts",
-    "./effects": "./src/effects.ts"
+    "./effects": "./src/effects.ts",
+    "./store": "./src/store.ts",
+    "./view": "./src/view.ts"
   },
   "files": [
     "src",
diff --git a/blocksuite/affine/gfx/brush/src/store.ts b/blocksuite/affine/gfx/brush/src/store.ts
new file mode 100644
index 0000000000..41bb69aa05
--- /dev/null
+++ b/blocksuite/affine/gfx/brush/src/store.ts
@@ -0,0 +1,19 @@
+import {
+  type StoreExtensionContext,
+  StoreExtensionProvider,
+} from '@blocksuite/affine-ext-loader';
+
+import {
+  brushToMarkdownAdapterMatcher,
+  brushToPlainTextAdapterMatcher,
+} from './adapter';
+
+export class BrushStoreExtension extends StoreExtensionProvider {
+  override name = 'affine-brush-gfx';
+
+  override setup(context: StoreExtensionContext) {
+    super.setup(context);
+    context.register(brushToMarkdownAdapterMatcher);
+    context.register(brushToPlainTextAdapterMatcher);
+  }
+}
diff --git a/blocksuite/affine/gfx/brush/src/view.ts b/blocksuite/affine/gfx/brush/src/view.ts
new file mode 100644
index 0000000000..e0c61c5ee4
--- /dev/null
+++ b/blocksuite/affine/gfx/brush/src/view.ts
@@ -0,0 +1,39 @@
+import {
+  type ViewExtensionContext,
+  ViewExtensionProvider,
+} from '@blocksuite/affine-ext-loader';
+
+import { BrushTool } from './brush-tool';
+import { effects } from './effects';
+import { BrushElementRendererExtension } from './element-renderer';
+import { EraserTool } from './eraser-tool';
+import { HighlighterTool } from './highlighter-tool';
+import {
+  brushToolbarExtension,
+  highlighterToolbarExtension,
+} from './toolbar/configs';
+import { penSeniorTool } from './toolbar/senior-tool';
+
+export class BrushViewExtension extends ViewExtensionProvider {
+  override name = 'affine-brush-gfx';
+
+  override effect(): void {
+    super.effect();
+    effects();
+  }
+
+  override setup(context: ViewExtensionContext) {
+    super.setup(context);
+
+    context.register(BrushTool);
+    context.register(EraserTool);
+    context.register(HighlighterTool);
+
+    context.register(BrushElementRendererExtension);
+
+    context.register(brushToolbarExtension);
+    context.register(highlighterToolbarExtension);
+
+    context.register(penSeniorTool);
+  }
+}
diff --git a/blocksuite/affine/gfx/brush/tsconfig.json b/blocksuite/affine/gfx/brush/tsconfig.json
index 4af984eb19..05c8f632c6 100644
--- a/blocksuite/affine/gfx/brush/tsconfig.json
+++ b/blocksuite/affine/gfx/brush/tsconfig.json
@@ -9,6 +9,7 @@
   "references": [
     { "path": "../../blocks/surface" },
     { "path": "../../components" },
+    { "path": "../../ext-loader" },
     { "path": "../../model" },
     { "path": "../../rich-text" },
     { "path": "../../shared" },
diff --git a/blocksuite/affine/gfx/note/package.json b/blocksuite/affine/gfx/note/package.json
index ed292a60d1..8692b98e04 100644
--- a/blocksuite/affine/gfx/note/package.json
+++ b/blocksuite/affine/gfx/note/package.json
@@ -15,6 +15,7 @@
     "@blocksuite/affine-block-image": "workspace:*",
     "@blocksuite/affine-block-surface": "workspace:*",
     "@blocksuite/affine-components": "workspace:*",
+    "@blocksuite/affine-ext-loader": "workspace:*",
     "@blocksuite/affine-model": "workspace:*",
     "@blocksuite/affine-rich-text": "workspace:*",
     "@blocksuite/affine-shared": "workspace:*",
@@ -36,7 +37,8 @@
   },
   "exports": {
     ".": "./src/index.ts",
-    "./effects": "./src/effects.ts"
+    "./effects": "./src/effects.ts",
+    "./view": "./src/view.ts"
   },
   "files": [
     "src",
diff --git a/blocksuite/affine/gfx/note/src/view.ts b/blocksuite/affine/gfx/note/src/view.ts
new file mode 100644
index 0000000000..7fe8b48fbf
--- /dev/null
+++ b/blocksuite/affine/gfx/note/src/view.ts
@@ -0,0 +1,23 @@
+import {
+  type ViewExtensionContext,
+  ViewExtensionProvider,
+} from '@blocksuite/affine-ext-loader';
+
+import { effects } from './effects';
+import { NoteTool } from './note-tool';
+import { noteSeniorTool } from './toolbar/senior-tool';
+
+export class NoteViewExtension extends ViewExtensionProvider {
+  override name = 'affine-note-gfx';
+
+  override effect(): void {
+    super.effect();
+    effects();
+  }
+
+  override setup(context: ViewExtensionContext) {
+    super.setup(context);
+    context.register(NoteTool);
+    context.register(noteSeniorTool);
+  }
+}
diff --git a/blocksuite/affine/gfx/note/tsconfig.json b/blocksuite/affine/gfx/note/tsconfig.json
index a5aa6e1e81..84abda762a 100644
--- a/blocksuite/affine/gfx/note/tsconfig.json
+++ b/blocksuite/affine/gfx/note/tsconfig.json
@@ -12,6 +12,7 @@
     { "path": "../../blocks/image" },
     { "path": "../../blocks/surface" },
     { "path": "../../components" },
+    { "path": "../../ext-loader" },
     { "path": "../../model" },
     { "path": "../../rich-text" },
     { "path": "../../shared" },
diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts
index b39a285b3c..c8ffec5888 100644
--- a/tools/utils/src/workspace.gen.ts
+++ b/tools/utils/src/workspace.gen.ts
@@ -497,6 +497,7 @@ export const PackageList = [
     workspaceDependencies: [
       'blocksuite/affine/blocks/surface',
       'blocksuite/affine/components',
+      'blocksuite/affine/ext-loader',
       'blocksuite/affine/model',
       'blocksuite/affine/rich-text',
       'blocksuite/affine/shared',
@@ -568,6 +569,7 @@ export const PackageList = [
       'blocksuite/affine/blocks/image',
       'blocksuite/affine/blocks/surface',
       'blocksuite/affine/components',
+      'blocksuite/affine/ext-loader',
       'blocksuite/affine/model',
       'blocksuite/affine/rich-text',
       'blocksuite/affine/shared',
diff --git a/yarn.lock b/yarn.lock
index 6d4cb4dacd..d1069986e3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3102,6 +3102,7 @@ __metadata:
   dependencies:
     "@blocksuite/affine-block-surface": "workspace:*"
     "@blocksuite/affine-components": "workspace:*"
+    "@blocksuite/affine-ext-loader": "workspace:*"
     "@blocksuite/affine-model": "workspace:*"
     "@blocksuite/affine-rich-text": "workspace:*"
     "@blocksuite/affine-shared": "workspace:*"
@@ -3222,6 +3223,7 @@ __metadata:
     "@blocksuite/affine-block-image": "workspace:*"
     "@blocksuite/affine-block-surface": "workspace:*"
     "@blocksuite/affine-components": "workspace:*"
+    "@blocksuite/affine-ext-loader": "workspace:*"
     "@blocksuite/affine-model": "workspace:*"
     "@blocksuite/affine-rich-text": "workspace:*"
     "@blocksuite/affine-shared": "workspace:*"