feat(core): support copy as image in electron app (#8939)

Close issue [AF-1785](https://linear.app/affine-design/issue/AF-1785).

### What changed?
- Support copy as image in electron app：
  -  Select the whole mindmap if any of the mindmap nodes is selected.
  -  Hide unselected overlap elements before taking a snapshot.
  -  Fit the selected elements to the screen.
  -  Add CSS style to hide irrelevant dom nodes, like widgets, whiteboard background and so on.
     - Due to the usage of Shadow Dom in our code, not all node styles can be controlled. Thus this PR use `-2px` padding for `affine:frame` snapshots.
  - Using electron `capturePage` API to take a snapshot of selected elements.

<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/c1b7b772-ddf8-4a85-b670-e96af7bd5cc0.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/c1b7b772-ddf8-4a85-b670-e96af7bd5cc0.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/c1b7b772-ddf8-4a85-b670-e96af7bd5cc0.mov">录屏2024-11-27 16.11.03.mov</video>

packages/frontend/apps/android/package.json

@@ -14,7 +14,7 @@
     "@affine/core": "workspace:*",
     "@affine/i18n": "workspace:*",
     "@blocksuite/affine": "0.18.1",
-    "@blocksuite/icons": "^2.1.70",
+    "@blocksuite/icons": "2.1.71",
     "@capacitor/android": "^6.1.2",
     "@capacitor/core": "^6.1.2",
     "@sentry/react": "^8.0.0",

packages/frontend/apps/electron/src/main/ui/handlers.ts

@@ -1,4 +1,4 @@
-import { app, nativeTheme, shell } from 'electron';
+import { app, clipboard, nativeImage, nativeTheme, shell } from 'electron';
 import { getLinkPreview } from 'link-preview-js';
 
 import { isMacOS } from '../../shared/utils';
@@ -232,4 +232,23 @@ export const uiHandlers = {
       e.sender.session.setSpellCheckerLanguages([language, 'en-US']);
     }
   },
+  captureArea: async (e, { x, y, width, height }: Electron.Rectangle) => {
+    const image = await e.sender.capturePage({
+      x: Math.floor(x),
+      y: Math.floor(y),
+      width: Math.floor(width),
+      height: Math.floor(height),
+    });
+
+    if (image.isEmpty()) {
+      throw new Error('Image is empty or invalid');
+    }
+
+    const buffer = image.toPNG();
+    if (!buffer || !buffer.length) {
+      throw new Error('Failed to generate PNG buffer from image');
+    }
+
+    clipboard.writeImage(nativeImage.createFromBuffer(buffer));
+  },
 } satisfies NamespaceHandlers;

packages/frontend/apps/ios/package.json

@@ -16,7 +16,7 @@
     "@affine/core": "workspace:*",
     "@affine/i18n": "workspace:*",
     "@blocksuite/affine": "0.18.1",
-    "@blocksuite/icons": "^2.1.70",
+    "@blocksuite/icons": "2.1.71",
     "@capacitor/app": "^6.0.1",
     "@capacitor/browser": "^6.0.3",
     "@capacitor/core": "^6.1.2",

packages/frontend/apps/mobile/package.json

@@ -14,7 +14,7 @@
     "@affine/core": "workspace:*",
     "@affine/i18n": "workspace:*",
     "@blocksuite/affine": "0.18.1",
-    "@blocksuite/icons": "^2.1.70",
+    "@blocksuite/icons": "2.1.71",
     "@sentry/react": "^8.0.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",

packages/frontend/component/package.json

@@ -14,7 +14,7 @@
   },
   "peerDependencies": {
     "@blocksuite/affine": "*",
-    "@blocksuite/icons": "2.1.68"
+    "@blocksuite/icons": "2.1.71"
   },
   "dependencies": {
     "@affine/cli": "workspace:*",

packages/frontend/core/src/components/blocksuite/block-suite-editor/specs/custom/widgets/copy-as-image.ts

@@ -0,0 +1,234 @@
+import { notify } from '@affine/component';
+import {
+  isMindmapChild,
+  isMindMapRoot,
+} from '@affine/core/blocksuite/presets/ai/utils/edgeless';
+import { EditorService } from '@affine/core/modules/editor';
+import { apis } from '@affine/electron-api';
+import { I18n } from '@affine/i18n';
+import type { BlockStdScope } from '@blocksuite/affine/block-std';
+import {
+  type GfxBlockElementModel,
+  type GfxModel,
+  GfxPrimitiveElementModel,
+  isGfxGroupCompatibleModel,
+} from '@blocksuite/affine/block-std/gfx';
+import type {
+  EdgelessRootService,
+  MenuContext,
+} from '@blocksuite/affine/blocks';
+import { Bound, getCommonBound } from '@blocksuite/affine/global/utils';
+import { CopyAsImgaeIcon } from '@blocksuite/icons/lit';
+import type { FrameworkProvider } from '@toeverything/infra';
+
+const snapshotStyle = `
+  affine-edgeless-root .widgets-container,
+  .copy-as-image-transparent {
+    opacity: 0;
+  }
+  .edgeless-background {
+    background-image: none;
+  }
+`;
+
+function getSelectedRect() {
+  const selected = document
+    .querySelector('edgeless-selected-rect')
+    ?.shadowRoot?.querySelector('.affine-edgeless-selected-rect');
+  if (!selected) {
+    throw new Error('Missing edgeless selected rect');
+  }
+  return selected.getBoundingClientRect();
+}
+
+function expandBound(bound: Bound, margin: number) {
+  const x = bound.x - margin;
+  const y = bound.y - margin;
+  const w = bound.w + margin * 2;
+  const h = bound.h + margin * 2;
+  return new Bound(x, y, w, h);
+}
+
+function isOverlap(target: Bound, source: Bound) {
+  const { x, y, w, h } = source;
+  const left = target.x;
+  const top = target.y;
+  const right = target.x + target.w;
+  const bottom = target.y + target.h;
+
+  return x < right && y < bottom && x + w > left && y + h > top;
+}
+
+function isInside(target: Bound, source: Bound) {
+  const { x, y, w, h } = source;
+  const left = target.x;
+  const top = target.y;
+  const right = target.x + target.w;
+  const bottom = target.y + target.h;
+
+  return x >= left && y >= top && x + w <= right && y + h <= bottom;
+}
+
+function hideEdgelessElements(elements: GfxModel[], std: BlockStdScope) {
+  elements.forEach(ele => {
+    if (ele instanceof GfxPrimitiveElementModel) {
+      (ele as any).lastOpacity = ele.opacity;
+      ele.opacity = 0;
+    } else {
+      const block = std.view.getBlock(ele.id);
+      if (!block) return;
+      block.classList.add('copy-as-image-transparent');
+    }
+  });
+}
+
+function showEdgelessElements(elements: GfxModel[], std: BlockStdScope) {
+  elements.forEach(ele => {
+    if (ele instanceof GfxPrimitiveElementModel) {
+      ele.opacity = (ele as any).lastOpacity;
+      delete (ele as any).lastOpacity;
+    } else {
+      const block = std.view.getBlock(ele.id);
+      if (!block) return;
+      block.classList.remove('copy-as-image-transparent');
+    }
+  });
+}
+
+function withDescendantElements(elements: GfxModel[]) {
+  const set = new Set<GfxModel>();
+  elements.forEach(element => {
+    if (set.has(element)) return;
+    set.add(element);
+    if (isGfxGroupCompatibleModel(element)) {
+      element.descendantElements.map(descendant => set.add(descendant));
+    }
+  });
+  return [...set];
+}
+
+const MARGIN = 20;
+
+export function createCopyAsPngMenuItem(framework: FrameworkProvider) {
+  return {
+    icon: CopyAsImgaeIcon({ width: '20', height: '20' }),
+    label: 'Copy as Image',
+    type: 'copy-as-image',
+    when: (ctx: MenuContext) => {
+      if (ctx.isEmpty()) return false;
+      const { editor } = framework.get(EditorService);
+      const mode = editor.mode$.value;
+      return mode === 'edgeless';
+    },
+    action: async (ctx: MenuContext) => {
+      if (!apis) {
+        notify.error({
+          title: I18n.t('com.affine.copy.asImage.notAvailable.title'),
+          message: I18n.t('com.affine.copy.asImage.notAvailable.message'),
+          action: {
+            label: I18n.t('com.affine.copy.asImage.notAvailable.action'),
+            onClick: () => {
+              window.open('https://affine.pro/download');
+            },
+          },
+        });
+        return;
+      }
+
+      const service =
+        ctx.host.std.getService<EdgelessRootService>('affine:page');
+      if (!service) return;
+
+      let selected = service.selection.selectedElements;
+      // select mindmap if root node selected
+      const maybeMindmap = selected[0];
+      const mindmapId = maybeMindmap.group?.id;
+      if (
+        selected.length === 1 &&
+        mindmapId &&
+        (isMindMapRoot(maybeMindmap) || isMindmapChild(maybeMindmap))
+      ) {
+        service.gfx.selection.set({ elements: [mindmapId] });
+      }
+
+      // select bound
+      selected = service.selection.selectedElements;
+      const elements = withDescendantElements(selected);
+      const bounds = elements.map(element => Bound.deserialize(element.xywh));
+      const bound = getCommonBound(bounds);
+      if (!bound) return;
+      const { zoom } = service.viewport;
+      const exBound = expandBound(bound, MARGIN * zoom);
+
+      // fit to screen
+      if (
+        !isInside(service.viewport.viewportBounds, exBound) ||
+        service.viewport.zoom < 1
+      ) {
+        service.viewport.setViewportByBound(bound, [20, 20, 20, 20], false);
+        if (service.viewport.zoom > 1) {
+          service.viewport.setZoom(1);
+        }
+      }
+
+      // hide unselected overlap elements
+      const overlapElements = service.gfx.gfxElements.filter(ele => {
+        const eleBound = Bound.deserialize(ele.xywh);
+        const exEleBound = expandBound(eleBound, MARGIN * zoom);
+        const isSelected = elements.includes(ele);
+        return !isSelected && isOverlap(exBound, exEleBound);
+      });
+      hideEdgelessElements(overlapElements, ctx.host.std);
+
+      // add css style
+      const styleEle = document.createElement('style');
+      styleEle.innerHTML = snapshotStyle;
+      document.head.append(styleEle);
+
+      // capture image
+      setTimeout(() => {
+        if (!apis) return;
+        try {
+          const domRect = getSelectedRect();
+          const { zoom } = service.viewport;
+          const isFrameSelected =
+            selected.length === 1 &&
+            (selected[0] as GfxBlockElementModel).flavour === 'affine:frame';
+          const margin = isFrameSelected ? -2 : MARGIN * zoom;
+
+          service.selection.clear();
+          // eslint-disable-next-line @typescript-eslint/no-floating-promises
+          apis.ui
+            .captureArea({
+              x: domRect.left - margin,
+              y: domRect.top - margin,
+              width: domRect.width + margin * 2,
+              height: domRect.height + margin * 2,
+            })
+            .then(() => {
+              notify.success({
+                title: I18n.t('com.affine.copy.asImage.success'),
+              });
+            })
+            .catch(e => {
+              notify.error({
+                title: I18n.t('com.affine.copy.asImage.failed'),
+                message: String(e),
+              });
+            })
+            .finally(() => {
+              styleEle.remove();
+              showEdgelessElements(overlapElements, ctx.host.std);
+            });
+        } catch (e) {
+          styleEle.remove();
+          showEdgelessElements(overlapElements, ctx.host.std);
+          notify.error({
+            title: I18n.t('com.affine.copy.asImage.failed'),
+            message: String(e),
+          });
+        }
+      }, 100);
+    },
+  };
+}

packages/frontend/core/src/components/blocksuite/block-suite-editor/specs/custom/widgets/toolbar.ts

@@ -12,11 +12,13 @@ import type {
   GfxBlockElementModel,
   GfxPrimitiveElementModel,
 } from '@blocksuite/affine/block-std/gfx';
-import type { MenuContext } from '@blocksuite/affine/blocks';
+import { type MenuContext } from '@blocksuite/affine/blocks';
 import type { MenuItemGroup } from '@blocksuite/affine-components/toolbar';
 import { LinkIcon } from '@blocksuite/icons/lit';
 import type { FrameworkProvider } from '@toeverything/infra';
 
+import { createCopyAsPngMenuItem } from './copy-as-image';
+
 export function createToolbarMoreMenuConfig(framework: FrameworkProvider) {
   return {
     configure: <T extends MenuContext>(groups: MenuItemGroup<T>[]) => {
@@ -41,6 +43,12 @@ export function createToolbarMoreMenuConfig(framework: FrameworkProvider) {
           0,
           createCopyLinkToBlockMenuItem(framework)
         );
+
+        clipboardGroup.items.splice(
+          copyIndex + 1,
+          0,
+          createCopyAsPngMenuItem(framework)
+        );
       }
 
       return groups;

packages/frontend/i18n/src/i18n-completenesses.json

@@ -5,7 +5,7 @@
   "de": 28,
   "el-GR": 0,
   "en": 100,
-  "es-AR": 14,
+  "es-AR": 13,
   "es-CL": 15,
   "es": 13,
   "fr": 66,
@@ -15,10 +15,10 @@
   "ja": 99,
   "ko": 79,
   "pl": 0,
-  "pt-BR": 86,
+  "pt-BR": 85,
   "ru": 73,
   "sv-SE": 4,
   "ur": 3,
   "zh-Hans": 100,
-  "zh-Hant": 100
+  "zh-Hant": 99
 }

packages/frontend/i18n/src/resources/en.json

@@ -397,6 +397,11 @@
   "com.affine.collections.empty.message": "No collections",
   "com.affine.collections.empty.new-collection-button": "New collection",
   "com.affine.collections.header": "Collections",
+  "com.affine.copy.asImage.notAvailable.title": "Couldn't copy image",
+  "com.affine.copy.asImage.notAvailable.message": "The 'Copy as image' feature is only available on our desktop app. Please download and install the client to access this feature.",
+  "com.affine.copy.asImage.notAvailable.action": "Download Client",
+  "com.affine.copy.asImage.success": "Image copied",
+  "com.affine.copy.asImage.failed": "Image copy failed",
   "com.affine.confirmModal.button.cancel": "Cancel",
   "com.affine.currentYear": "Current year",
   "com.affine.delete-tags.confirm.description": "Deleting <1>{{tag}}</1> cannot be undone, please proceed with caution.",