diff --git a/blocksuite/affine/widgets/toolbar/src/toolbar.ts b/blocksuite/affine/widgets/toolbar/src/toolbar.ts
index 92ee9d99a5..8ce6201de3 100644
--- a/blocksuite/affine/widgets/toolbar/src/toolbar.ts
+++ b/blocksuite/affine/widgets/toolbar/src/toolbar.ts
@@ -162,10 +162,11 @@ export class AffineToolbarWidget extends WidgetComponent {
   }
 
   setReferenceElementWithElements(gfx: GfxController, elements: GfxModel[]) {
+    const surfaceBounds = getCommonBoundWithRotation(elements);
+
     const getBoundingClientRect = () => {
-      const bounds = getCommonBoundWithRotation(elements);
       const { x: offsetX, y: offsetY } = this.getBoundingClientRect();
-      const [x, y, w, h] = gfx.viewport.toViewBound(bounds).toXYWH();
+      const [x, y, w, h] = gfx.viewport.toViewBound(surfaceBounds).toXYWH();
       const rect = new DOMRect(x + offsetX, y + offsetY, w, h);
       return rect;
     };
diff --git a/blocksuite/framework/std/src/gfx/model/surface/element-model.ts b/blocksuite/framework/std/src/gfx/model/surface/element-model.ts
index d0d6653088..bd107c8278 100644
--- a/blocksuite/framework/std/src/gfx/model/surface/element-model.ts
+++ b/blocksuite/framework/std/src/gfx/model/surface/element-model.ts
@@ -103,8 +103,9 @@ export abstract class GfxPrimitiveElementModel<
   }
 
   get deserializedXYWH() {
-    if (!this._lastXYWH || this.xywh !== this._lastXYWH) {
-      const xywh = this.xywh;
+    const xywh = this.xywh;
+
+    if (!this._lastXYWH || xywh !== this._lastXYWH) {
       this._local.set('deserializedXYWH', deserializeXYWH(xywh));
       this._lastXYWH = xywh;
     }
@@ -386,6 +387,8 @@ export abstract class GfxGroupLikeElementModel<
 {
   private _childIds: string[] = [];
 
+  private _xywhDirty = true;
+
   private readonly _mutex = createMutex();
 
   abstract children: Y.Map<any>;
@@ -420,24 +423,9 @@ export abstract class GfxGroupLikeElementModel<
 
   get xywh() {
     this._mutex(() => {
-      const curXYWH =
-        (this._local.get('xywh') as SerializedXYWH) ?? '[0,0,0,0]';
-      const newXYWH = this._getXYWH().serialize();
-
-      if (curXYWH !== newXYWH || !this._local.has('xywh')) {
-        this._local.set('xywh', newXYWH);
-
-        if (curXYWH !== newXYWH) {
-          this._onChange({
-            props: {
-              xywh: newXYWH,
-            },
-            oldValues: {
-              xywh: curXYWH,
-            },
-            local: true,
-          });
-        }
+      if (this._xywhDirty || !this._local.has('xywh')) {
+        this._local.set('xywh', this._getXYWH().serialize());
+        this._xywhDirty = false;
       }
     });
 
@@ -457,15 +445,41 @@ export abstract class GfxGroupLikeElementModel<
       bound = bound ? bound.unite(child.elementBound) : child.elementBound;
     });
 
-    if (bound) {
-      this._local.set('xywh', bound.serialize());
-    } else {
-      this._local.delete('xywh');
-    }
-
     return bound ?? new Bound(0, 0, 0, 0);
   }
 
+  invalidateXYWH() {
+    this._xywhDirty = true;
+    this._local.delete('deserializedXYWH');
+  }
+
+  refreshXYWH(local: boolean) {
+    this._mutex(() => {
+      const oldXYWH =
+        (this._local.get('xywh') as SerializedXYWH) ?? '[0,0,0,0]';
+      const nextXYWH = this._getXYWH().serialize();
+
+      this._xywhDirty = false;
+
+      if (oldXYWH === nextXYWH && this._local.has('xywh')) {
+        return;
+      }
+
+      this._local.set('xywh', nextXYWH);
+      this._local.delete('deserializedXYWH');
+
+      this._onChange({
+        props: {
+          xywh: nextXYWH,
+        },
+        oldValues: {
+          xywh: oldXYWH,
+        },
+        local,
+      });
+    });
+  }
+
   abstract addChild(element: GfxModel): void;
 
   /**
@@ -496,6 +510,7 @@ export abstract class GfxGroupLikeElementModel<
   setChildIds(value: string[], fromLocal: boolean) {
     const oldChildIds = this.childIds;
     this._childIds = value;
+    this.invalidateXYWH();
 
     this._onChange({
       props: {
diff --git a/blocksuite/framework/std/src/gfx/model/surface/surface-model.ts b/blocksuite/framework/std/src/gfx/model/surface/surface-model.ts
index 5cc92fbc86..dae831be1d 100644
--- a/blocksuite/framework/std/src/gfx/model/surface/surface-model.ts
+++ b/blocksuite/framework/std/src/gfx/model/surface/surface-model.ts
@@ -52,6 +52,12 @@ export type MiddlewareCtx = {
 export type SurfaceMiddleware = (ctx: MiddlewareCtx) => void;
 
 export class SurfaceBlockModel extends BlockModel<SurfaceBlockProps> {
+  private static readonly _groupBoundImpactKeys = new Set([
+    'xywh',
+    'rotate',
+    'hidden',
+  ]);
+
   protected _decoratorState = createDecoratorState();
 
   protected _elementCtorMap: Record<
@@ -308,6 +314,42 @@ export class SurfaceBlockModel extends BlockModel<SurfaceBlockProps> {
     Object.keys(payload.props).forEach(key => {
       model.propsUpdated.next({ key });
     });
+
+    this._refreshParentGroupBoundsForElement(model, payload);
+  }
+
+  private _refreshParentGroupBounds(id: string, local: boolean) {
+    const group = this.getGroup(id);
+
+    if (group instanceof GfxGroupLikeElementModel) {
+      group.refreshXYWH(local);
+    }
+  }
+
+  private _refreshParentGroupBoundsForElement(
+    model: GfxPrimitiveElementModel,
+    payload: ElementUpdatedData
+  ) {
+    if (
+      model instanceof GfxGroupLikeElementModel &&
+      ('childIds' in payload.props || 'childIds' in payload.oldValues)
+    ) {
+      model.refreshXYWH(payload.local);
+      return;
+    }
+
+    const affectedKeys = new Set([
+      ...Object.keys(payload.props),
+      ...Object.keys(payload.oldValues),
+    ]);
+
+    if (
+      Array.from(affectedKeys).some(key =>
+        SurfaceBlockModel._groupBoundImpactKeys.has(key)
+      )
+    ) {
+      this._refreshParentGroupBounds(model.id, payload.local);
+    }
   }
 
   private _initElementModels() {
@@ -458,6 +500,10 @@ export class SurfaceBlockModel extends BlockModel<SurfaceBlockProps> {
             );
           }
 
+          if (payload.model instanceof BlockModel) {
+            this._refreshParentGroupBounds(payload.id, payload.isLocal);
+          }
+
           break;
         case 'delete':
           if (isGfxGroupCompatibleModel(payload.model)) {
@@ -482,6 +528,13 @@ export class SurfaceBlockModel extends BlockModel<SurfaceBlockProps> {
             }
           }
 
+          if (
+            payload.props.key &&
+            SurfaceBlockModel._groupBoundImpactKeys.has(payload.props.key)
+          ) {
+            this._refreshParentGroupBounds(payload.id, payload.isLocal);
+          }
+
           break;
       }
     });
diff --git a/blocksuite/integration-test/src/__tests__/edgeless/surface-model.spec.ts b/blocksuite/integration-test/src/__tests__/edgeless/surface-model.spec.ts
index 6ae5d6d2c5..61b6ce1e4b 100644
--- a/blocksuite/integration-test/src/__tests__/edgeless/surface-model.spec.ts
+++ b/blocksuite/integration-test/src/__tests__/edgeless/surface-model.spec.ts
@@ -4,6 +4,7 @@ import type {
   ConnectorElementModel,
   GroupElementModel,
 } from '@blocksuite/affine/model';
+import { serializeXYWH } from '@blocksuite/global/gfx';
 import { beforeEach, describe, expect, test } from 'vitest';
 
 import { wait } from '../utils/common.js';
@@ -138,6 +139,29 @@ describe('group', () => {
 
     expect(group.childIds).toEqual([id]);
   });
+
+  test('group xywh should update when child xywh changes', () => {
+    const shapeId = model.addElement({
+      type: 'shape',
+      xywh: serializeXYWH(0, 0, 100, 100),
+    });
+    const groupId = model.addElement({
+      type: 'group',
+      children: {
+        [shapeId]: true,
+      },
+    });
+
+    const group = model.getElementById(groupId) as GroupElementModel;
+
+    expect(group.xywh).toBe(serializeXYWH(0, 0, 100, 100));
+
+    model.updateElement(shapeId, {
+      xywh: serializeXYWH(50, 60, 100, 100),
+    });
+
+    expect(group.xywh).toBe(serializeXYWH(50, 60, 100, 100));
+  });
 });
 
 describe('connector', () => {