diff --git a/blocksuite/affine/gfx/connector/src/element-transform/connector-filter.ts b/blocksuite/affine/gfx/connector/src/element-transform/connector-filter.ts
index 0614c8d07f..21e8a79053 100644
--- a/blocksuite/affine/gfx/connector/src/element-transform/connector-filter.ts
+++ b/blocksuite/affine/gfx/connector/src/element-transform/connector-filter.ts
@@ -36,6 +36,8 @@ export class ConnectorFilter extends InteractivityExtension {
         elements.sort((a, _) => (a instanceof ConnectorElementModel ? -1 : 1));
       }
 
+      context.elements = elements;
+
       return {};
     });
   }
diff --git a/blocksuite/affine/gfx/connector/src/text/edgeless-connector-label-editor.ts b/blocksuite/affine/gfx/connector/src/text/edgeless-connector-label-editor.ts
index b9b657a2eb..b436e2f64d 100644
--- a/blocksuite/affine/gfx/connector/src/text/edgeless-connector-label-editor.ts
+++ b/blocksuite/affine/gfx/connector/src/text/edgeless-connector-label-editor.ts
@@ -188,6 +188,8 @@ export class EdgelessConnectorLabelEditor extends WithDisposable(
     });
     this._resizeObserver.observe(this.richText);
 
+    this.connector.stash('labelXYWH');
+
     this.updateComplete
       .then(() => {
         if (!this.inlineEditor) return;
@@ -257,7 +259,8 @@ export class EdgelessConnectorLabelEditor extends WithDisposable(
             }
           }
 
-          connector.lableEditing = false;
+          connector.labelEditing = false;
+          connector.pop('labelXYWH');
 
           selection.set({
             elements: [],
@@ -293,7 +296,7 @@ export class EdgelessConnectorLabelEditor extends WithDisposable(
           }
         );
 
-        connector.lableEditing = true;
+        connector.labelEditing = true;
       })
       .catch(console.error);
   }
diff --git a/blocksuite/affine/model/src/elements/connector/connector.ts b/blocksuite/affine/model/src/elements/connector/connector.ts
index c37426f708..ccc86d65d3 100644
--- a/blocksuite/affine/model/src/elements/connector/connector.ts
+++ b/blocksuite/affine/model/src/elements/connector/connector.ts
@@ -278,7 +278,13 @@ export class ConnectorElementModel extends GfxPrimitiveElementModel<ConnectorEle
   }
 
   hasLabel() {
-    return Boolean(!this.lableEditing && this.labelDisplay && this.labelXYWH);
+    return Boolean(
+      !this.labelEditing &&
+        this.labelDisplay &&
+        this.labelXYWH &&
+        this.text &&
+        this.text.length
+    );
   }
 
   override includesPoint(
@@ -450,7 +456,7 @@ export class ConnectorElementModel extends GfxPrimitiveElementModel<ConnectorEle
    * Local control display and hide, mainly used in editing scenarios.
    */
   @local()
-  accessor lableEditing: boolean = false;
+  accessor labelEditing: boolean = false;
 
   @field()
   accessor mode: ConnectorMode = DEFAULT_CONNECTOR_MODE;
diff --git a/blocksuite/affine/widgets/edgeless-selected-rect/src/edgeless-auto-complete.ts b/blocksuite/affine/widgets/edgeless-selected-rect/src/edgeless-auto-complete.ts
index 9f4e06d723..81210d817c 100644
--- a/blocksuite/affine/widgets/edgeless-selected-rect/src/edgeless-auto-complete.ts
+++ b/blocksuite/affine/widgets/edgeless-selected-rect/src/edgeless-auto-complete.ts
@@ -744,7 +744,7 @@ export class EdgelessAutoComplete extends WithDisposable(LitElement) {
 
     if (
       this._isMoving ||
-      (this._isHover && !isShape && this._canAutoComplete())
+      (this._isHover && !isShape && !this._canAutoComplete())
     ) {
       this.removeOverlay();
       return nothing;
diff --git a/tests/blocksuite/e2e/edgeless/connector/connector.spec.ts b/tests/blocksuite/e2e/edgeless/connector/connector.spec.ts
index c7e58ddac2..f36f030ccb 100644
--- a/tests/blocksuite/e2e/edgeless/connector/connector.spec.ts
+++ b/tests/blocksuite/e2e/edgeless/connector/connector.spec.ts
@@ -26,6 +26,7 @@ import {
   assertConnectorPath,
   assertEdgelessNonSelectedRect,
   assertEdgelessSelectedRect,
+  getSelectedRect,
 } from '../../utils/asserts.js';
 import { test } from '../../utils/playwright.js';
 
@@ -373,4 +374,66 @@ test.describe('quick connect', () => {
       });
     }
   });
+
+  test('connector can not be moved directly if the source or target is not selected', async ({
+    page,
+  }) => {
+    await commonSetup(page);
+
+    const normalConnectorId = await createConnectorElement(
+      page,
+      [0, 0],
+      [100, 100]
+    );
+    await selectElementInEdgeless(page, [normalConnectorId]);
+
+    const normalRect1 = await getSelectedRect(page);
+
+    // connector with no source and target can be moved
+    await dragBetweenViewCoords(page, [50, 50], [100, 100]);
+    const normalRect2 = await getSelectedRect(page);
+    expect(normalRect2).toEqual({
+      x: normalRect1.x + 50,
+      y: normalRect1.y + 50,
+      width: normalRect1.width,
+      height: normalRect1.height,
+    });
+
+    const shape1 = await createShapeElement(
+      page,
+      [150, 150],
+      [200, 200],
+      Shape.Square
+    );
+    const shape2 = await createShapeElement(
+      page,
+      [250, 250],
+      [300, 300],
+      Shape.Square
+    );
+    const connectorWithShapes = await createConnectorElement(
+      page,
+      [190, 175],
+      [260, 275]
+    );
+    await selectElementInEdgeless(page, [connectorWithShapes]);
+
+    // cannot be moved because the source and target are not selected
+    const initialShapeConnectorRect = await getSelectedRect(page);
+    await dragBetweenViewCoords(page, [225, 200], [275, 250]);
+    const shapeConnectorRect1 = await getSelectedRect(page);
+    expect(shapeConnectorRect1).toEqual(initialShapeConnectorRect);
+
+    // can be moved because the source and target are selected
+    await selectElementInEdgeless(page, [shape1, shape2, connectorWithShapes]);
+    await dragBetweenViewCoords(page, [225, 200], [275, 250]);
+    await selectElementInEdgeless(page, [connectorWithShapes]);
+    const shapeConnectorRect2 = await getSelectedRect(page);
+    expect(shapeConnectorRect2).toEqual({
+      x: initialShapeConnectorRect.x + 50,
+      y: initialShapeConnectorRect.y + 50,
+      width: initialShapeConnectorRect.width,
+      height: initialShapeConnectorRect.height,
+    });
+  });
 });
diff --git a/tests/blocksuite/e2e/utils/actions/edgeless.ts b/tests/blocksuite/e2e/utils/actions/edgeless.ts
index d8899a616e..7aa505c47f 100644
--- a/tests/blocksuite/e2e/utils/actions/edgeless.ts
+++ b/tests/blocksuite/e2e/utils/actions/edgeless.ts
@@ -1886,6 +1886,7 @@ export async function createConnectorElement(
     { x: start[0], y: start[1] },
     { x: end[0], y: end[1] }
   );
+  return (await getSelectedIds(page))[0];
 }
 
 export async function createFrameElement(