feat(core): support ai insert image, mindmap, slides and make it real in page mode (#9164)

Support issue [BS-2085](https://linear.app/affine-design/issue/BS-2085).

### What changed?
- Refactor the `actionToAnswerRenderer` function to support reuse in both page mode and edgeless mode.
- Add a new `page-response.ts` module to handle AI-generated answers in page mode.
    - Remove the redundant `edgelessHandler` function from `_common/config.ts`.
- Introduce the `AIContext` class along with the `ctx` TypeScript type to standardize context management.
- Implement the `createTemplateJob` function to enable AI slide insertion in both page mode and edgeless mode.

Insert mindmap on page mode:
<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/30630d3e-ebd9-416b-9bb9-5f27086e48a3.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/30630d3e-ebd9-416b-9bb9-5f27086e48a3.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/30630d3e-ebd9-416b-9bb9-5f27086e48a3.mov">mindmap.mov</video>

Insert image on edgeless note
<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/b850ee5a-a06b-4ae7-8b68-ed5929a6e81a.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/b850ee5a-a06b-4ae7-8b68-ed5929a6e81a.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/b850ee5a-a06b-4ae7-8b68-ed5929a6e81a.mov">image3.mov</video>

Insert image on page mode:
<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/c4f98e2d-0b15-4310-b3e0-0725e330302b.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/c4f98e2d-0b15-4310-b3e0-0725e330302b.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/c4f98e2d-0b15-4310-b3e0-0725e330302b.mov">image.mov</video>

Generate image from image:
<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/2776a55f-cbb7-47ce-8e7d-7cae243fa3e9.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/2776a55f-cbb7-47ce-8e7d-7cae243fa3e9.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/2776a55f-cbb7-47ce-8e7d-7cae243fa3e9.mov">image2.mov</video>

Insert presentation on page mode:
<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/4e228fa5-88f4-478c-8b79-647612d5515c.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/4e228fa5-88f4-478c-8b79-647612d5515c.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/4e228fa5-88f4-478c-8b79-647612d5515c.mov">ppt.mov</video>

Insert make it real on page mode:

<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/c71139b2-fb55-4d89-84e2-d52eeb905b57.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/c71139b2-fb55-4d89-84e2-d52eeb905b57.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/c71139b2-fb55-4d89-84e2-d52eeb905b57.mov">make it real.mov</video>
This commit is contained in:
akumatus
2024-12-16 10:04:15 +00:00
parent e6bf4ca6e5
commit 2f79104bdb
38 changed files with 1066 additions and 797 deletions

View File

@@ -9,8 +9,8 @@ import { flip, offset } from '@floating-ui/dom';
import { css, html, LitElement } from 'lit';
import { property } from 'lit/decorators.js';
import { getAIPanel } from '../../ai-panel';
import { AIProvider } from '../../provider';
import { getAIPanelWidget } from '../../utils/ai-widgets';
import { extractContext } from '../../utils/extract';
export class AskAIToolbarButton extends WithDisposable(LitElement) {
@@ -33,7 +33,7 @@ export class AskAIToolbarButton extends WithDisposable(LitElement) {
accessor actionGroups!: AIItemGroupConfig[];
private readonly _onItemClick = () => {
const aiPanel = getAIPanel(this.host);
const aiPanel = getAIPanelWidget(this.host);
aiPanel.restoreSelection();
this._clearAbortController();
};
@@ -45,7 +45,7 @@ export class AskAIToolbarButton extends WithDisposable(LitElement) {
private readonly _openAIPanel = () => {
this._clearAbortController();
const aiPanel = getAIPanel(this.host);
const aiPanel = getAIPanelWidget(this.host);
this._abortController = new AbortController();
this._panelRoot = createLitPortal({
template: html`
@@ -69,7 +69,7 @@ export class AskAIToolbarButton extends WithDisposable(LitElement) {
private readonly _generateAnswer: AffineAIPanelWidgetConfig['generateAnswer'] =
({ finish, input }) => {
finish('success');
const aiPanel = getAIPanel(this.host);
const aiPanel = getAIPanelWidget(this.host);
aiPanel.discard();
AIProvider.slots.requestOpenWithChat.emit({ host: this.host });
extractContext(this.host)
@@ -80,7 +80,7 @@ export class AskAIToolbarButton extends WithDisposable(LitElement) {
};
private readonly _onClick = () => {
const aiPanel = getAIPanel(this.host);
const aiPanel = getAIPanelWidget(this.host);
if (!aiPanel.config) return;
aiPanel.config.generateAnswer = this._generateAnswer;
aiPanel.config.inputCallback = text => {

View File

@@ -1,32 +1,19 @@
import type {
Chain,
EditorHost,
InitCommandCtx,
} from '@blocksuite/affine/block-std';
import type { Chain, InitCommandCtx } from '@blocksuite/affine/block-std';
import {
type AIItemGroupConfig,
type AISubItemConfig,
EDGELESS_ELEMENT_TOOLBAR_WIDGET,
type EdgelessElementToolbarWidget,
matchFlavours,
} from '@blocksuite/affine/blocks';
import type { TemplateResult } from 'lit';
import { actionToHandler } from '../actions/doc-handler';
import { actionToHandler as edgelessActionToHandler } from '../actions/edgeless-handler';
import {
imageFilterStyles,
imageProcessingTypes,
textTones,
translateLangs,
} from '../actions/types';
import { getAIPanel } from '../ai-panel';
import { AIProvider } from '../provider';
import {
getSelectedImagesAsBlobs,
getSelectedTextContent,
getSelections,
} from '../utils/selection-utils';
import { getAIPanelWidget } from '../utils/ai-widgets';
import {
AIDoneIcon,
AIImageIcon,
@@ -71,7 +58,7 @@ export function createImageFilterSubItem(
return imageFilterStyles.map(style => {
return {
type: style,
handler: edgelessHandler(
handler: actionToHandler(
'filterImage',
AIImageIconWithAnimation,
{
@@ -89,7 +76,7 @@ export function createImageProcessingSubItem(
return imageProcessingTypes.map(type => {
return {
type,
handler: edgelessHandler(
handler: actionToHandler(
'processImage',
AIImageIconWithAnimation,
{
@@ -224,59 +211,6 @@ const DraftAIGroup: AIItemGroupConfig = {
],
};
// actions that initiated from a note in edgeless mode
// 1. when running in doc mode, call requestRunInEdgeless (let affine to show toast)
// 2. when running in edgeless mode
// a. get selected in the note and let the edgeless action to handle it
// b. insert the result using the note shape
function edgelessHandler<T extends keyof BlockSuitePresets.AIActions>(
id: T,
generatingIcon: TemplateResult<1>,
variants?: Omit<
Parameters<BlockSuitePresets.AIActions[T]>[0],
keyof BlockSuitePresets.AITextActionOptions
>,
trackerOptions?: BlockSuitePresets.TrackerOptions
) {
return (host: EditorHost) => {
if (host.doc.root?.id === undefined) return;
const edgeless = (
host.view.getWidget(
EDGELESS_ELEMENT_TOOLBAR_WIDGET,
host.doc.root.id
) as EdgelessElementToolbarWidget
)?.edgeless;
if (!edgeless) {
AIProvider.slots.requestRunInEdgeless.emit({ host });
} else {
const selectedElements = edgeless.service.selection.selectedElements;
if (!selectedElements.length) return;
return edgelessActionToHandler(
id,
generatingIcon,
variants,
async () => {
const selections = getSelections(host);
const [markdown, attachments] = await Promise.all([
getSelectedTextContent(host),
getSelectedImagesAsBlobs(host),
]);
// for now if there are more than one selected blocks, we will not omit the attachments
const sendAttachments =
selections?.selectedBlocks?.length === 1 && attachments.length > 0;
return {
attachments: sendAttachments ? attachments : undefined,
input: sendAttachments ? '' : markdown,
};
},
trackerOptions
)(host);
}
};
}
const ReviewWIthAIGroup: AIItemGroupConfig = {
name: 'review with ai',
items: [
@@ -353,7 +287,7 @@ const GenerateWithAIGroup: AIItemGroupConfig = {
name: 'Generate an image',
icon: AIImageIcon,
showWhen: textBlockShowWhen,
handler: edgelessHandler('createImage', AIImageIconWithAnimation),
handler: actionToHandler('createImage', AIImageIconWithAnimation),
},
{
name: 'Generate outline',
@@ -365,13 +299,13 @@ const GenerateWithAIGroup: AIItemGroupConfig = {
name: 'Brainstorm ideas with mind map',
icon: AIMindMapIcon,
showWhen: textBlockShowWhen,
handler: edgelessHandler('brainstormMindmap', AIPenIconWithAnimation),
handler: actionToHandler('brainstormMindmap', AIPenIconWithAnimation),
},
{
name: 'Generate presentation',
icon: AIPresentationIcon,
showWhen: textBlockShowWhen,
handler: edgelessHandler('createSlides', AIPresentationIconWithAnimation),
handler: actionToHandler('createSlides', AIPresentationIconWithAnimation),
beta: true,
},
{
@@ -379,7 +313,7 @@ const GenerateWithAIGroup: AIItemGroupConfig = {
icon: MakeItRealIcon,
beta: true,
showWhen: textBlockShowWhen,
handler: edgelessHandler('makeItReal', MakeItRealIconWithAnimation),
handler: actionToHandler('makeItReal', MakeItRealIconWithAnimation),
},
{
name: 'Find actions',
@@ -398,7 +332,7 @@ const OthersAIGroup: AIItemGroupConfig = {
name: 'Continue with AI',
icon: CommentIcon,
handler: host => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
AIProvider.slots.requestOpenWithChat.emit({
host,
autoSelect: true,
@@ -411,7 +345,7 @@ const OthersAIGroup: AIItemGroupConfig = {
name: 'Open AI Chat',
icon: ChatWithAIIcon,
handler: host => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
AIProvider.slots.requestOpenWithChat.emit({
host,
appendCard: true,
@@ -422,7 +356,7 @@ const OthersAIGroup: AIItemGroupConfig = {
],
};
export const AIItemGroups: AIItemGroupConfig[] = [
export const pageAIGroups: AIItemGroupConfig[] = [
ReviewWIthAIGroup,
EditAIGroup,
GenerateWithAIGroup,
@@ -455,7 +389,7 @@ export function buildAIImageItemGroups(): AIItemGroupConfig[] {
name: 'Generate an image',
icon: AIImageIcon,
showWhen: () => true,
handler: edgelessHandler(
handler: actionToHandler(
'createImage',
AIImageIconWithAnimation,
undefined,

View File

@@ -0,0 +1,58 @@
import type { EditorHost } from '@blocksuite/affine/block-std';
import type {
AffineAIPanelWidget,
MindmapElementModel,
} from '@blocksuite/affine/blocks';
import { createTextRenderer } from '../../_common';
import {
createMindmapExecuteRenderer,
createMindmapRenderer,
} from '../messages/mindmap';
import { createSlidesRenderer } from '../messages/slides-renderer';
import { createIframeRenderer, createImageRenderer } from '../messages/wrapper';
import type { AIContext } from '../utils/context';
import { isMindmapChild, isMindMapRoot } from '../utils/edgeless';
import { IMAGE_ACTIONS } from './consts';
import { responseToExpandMindmap } from './edgeless-response';
type AnswerRenderer = NonNullable<
AffineAIPanelWidget['config']
>['answerRenderer'];
export function actionToAnswerRenderer<
T extends keyof BlockSuitePresets.AIActions,
>(id: T, host: EditorHost, ctx: AIContext): AnswerRenderer {
if (id === 'brainstormMindmap') {
const selectedElements = ctx.get().selectedElements;
if (
selectedElements &&
(isMindMapRoot(selectedElements[0]) ||
isMindmapChild(selectedElements[0]))
) {
const mindmap = selectedElements[0].group as MindmapElementModel;
return createMindmapRenderer(host, ctx, mindmap.style);
}
return createMindmapRenderer(host, ctx);
}
if (id === 'expandMindmap') {
return createMindmapExecuteRenderer(host, ctx, responseToExpandMindmap);
}
if (id === 'createSlides') {
return createSlidesRenderer(host, ctx);
}
if (id === 'makeItReal') {
return createIframeRenderer(host, { height: 300 });
}
if (IMAGE_ACTIONS.includes(id)) {
return createImageRenderer(host, { height: 300 });
}
return createTextRenderer(host, { maxHeight: 320 });
}

View File

@@ -9,6 +9,16 @@ export const EXCLUDING_COPY_ACTIONS = [
'processImage',
];
export const EXCLUDING_REPLACE_ACTIONS = [
'brainstormMindmap',
'expandMindmap',
'makeItReal',
'createSlides',
'createImage',
'filterImage',
'processImage',
];
export const EXCLUDING_INSERT_ACTIONS = ['generateCaption'];
export const IMAGE_ACTIONS = ['createImage', 'processImage', 'filterImage'];

View File

@@ -7,22 +7,23 @@ import type {
import { assertExists } from '@blocksuite/affine/global/utils';
import type { TemplateResult } from 'lit';
import { createTextRenderer } from '../../_common';
import {
buildCopyConfig,
buildErrorConfig,
buildFinishConfig,
buildGeneratingConfig,
getAIPanel,
} from '../ai-panel';
import { AIProvider } from '../provider';
import { reportResponse } from '../utils/action-reporter';
import { getAIPanelWidget } from '../utils/ai-widgets';
import { AIContext } from '../utils/context';
import {
getSelectedImagesAsBlobs,
getSelectedTextContent,
getSelections,
selectAboveBlocks,
} from '../utils/selection-utils';
import { actionToAnswerRenderer } from './answer-renderer';
export function bindTextStream(
stream: BlockSuitePresets.TextStream,
@@ -174,8 +175,10 @@ function updateAIPanelConfig<T extends keyof BlockSuitePresets.AIActions>(
variants,
trackerOptions
)(host);
config.answerRenderer = createTextRenderer(host, { maxHeight: 320 });
config.finishStateConfig = buildFinishConfig(aiPanel, id);
const ctx = new AIContext();
config.answerRenderer = actionToAnswerRenderer(id, host, ctx);
config.finishStateConfig = buildFinishConfig(aiPanel, id, ctx);
config.generatingStateConfig = buildGeneratingConfig(generatingIcon);
config.errorStateConfig = buildErrorConfig(aiPanel);
config.copy = buildCopyConfig(aiPanel);
@@ -194,7 +197,7 @@ export function actionToHandler<T extends keyof BlockSuitePresets.AIActions>(
trackerOptions?: BlockSuitePresets.TrackerOptions
) {
return (host: EditorHost) => {
const aiPanel = getAIPanel(host);
const aiPanel = getAIPanelWidget(host);
updateAIPanelConfig(aiPanel, id, generatingIcon, variants, trackerOptions);
const { selectedBlocks: blocks } = getSelections(aiPanel.host);
if (!blocks || blocks.length === 0) return;
@@ -205,7 +208,7 @@ export function actionToHandler<T extends keyof BlockSuitePresets.AIActions>(
}
export function handleInlineAskAIAction(host: EditorHost) {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const selection = host.selection.find('text');
const lastBlockPath = selection
? (selection.to?.blockId ?? selection.blockId)

View File

@@ -3,7 +3,6 @@ import type {
AffineAIPanelWidget,
AIError,
EdgelessCopilotWidget,
MindmapElementModel,
} from '@blocksuite/affine/blocks';
import {
BlocksUtils,
@@ -20,16 +19,11 @@ import { GfxControllerIdentifier } from '@blocksuite/block-std/gfx';
import { AIChatBlockModel } from '@toeverything/infra';
import type { TemplateResult } from 'lit';
import { createTextRenderer, getContentFromSlice } from '../../_common';
import { getAIPanel } from '../ai-panel';
import {
createMindmapExecuteRenderer,
createMindmapRenderer,
} from '../messages/mindmap';
import { createSlidesRenderer } from '../messages/slides-renderer';
import { createIframeRenderer, createImageRenderer } from '../messages/wrapper';
import { getContentFromSlice } from '../../_common';
import { AIProvider } from '../provider';
import { reportResponse } from '../utils/action-reporter';
import { getAIPanelWidget } from '../utils/ai-widgets';
import { AIContext } from '../utils/context';
import {
getEdgelessCopilotWidget,
isMindmapChild,
@@ -41,63 +35,15 @@ import {
getSelectedNoteAnchor,
getSelections,
} from '../utils/selection-utils';
import { EXCLUDING_COPY_ACTIONS, IMAGE_ACTIONS } from './consts';
import { actionToAnswerRenderer } from './answer-renderer';
import { EXCLUDING_COPY_ACTIONS } from './consts';
import { bindTextStream } from './doc-handler';
import {
actionToErrorResponse,
actionToGenerating,
actionToResponse,
getElementToolbar,
responses,
} from './edgeless-response';
import type { CtxRecord } from './types';
type AnswerRenderer = NonNullable<
AffineAIPanelWidget['config']
>['answerRenderer'];
function actionToRenderer<T extends keyof BlockSuitePresets.AIActions>(
id: T,
host: EditorHost,
ctx: CtxRecord
): AnswerRenderer {
if (id === 'brainstormMindmap') {
const selectedElements = ctx.get()[
'selectedElements'
] as BlockSuite.EdgelessModel[];
if (
isMindMapRoot(selectedElements[0]) ||
isMindmapChild(selectedElements[0])
) {
const mindmap = selectedElements[0].group as MindmapElementModel;
return createMindmapRenderer(host, ctx, mindmap.style);
}
return createMindmapRenderer(host, ctx);
}
if (id === 'expandMindmap') {
return createMindmapExecuteRenderer(host, ctx, ctx => {
responses['expandMindmap']?.(host, ctx);
});
}
if (id === 'createSlides') {
return createSlidesRenderer(host, ctx);
}
if (id === 'makeItReal') {
return createIframeRenderer(host, { height: 300 });
}
if (IMAGE_ACTIONS.includes(id)) {
return createImageRenderer(host, { height: 300 });
}
return createTextRenderer(host, { maxHeight: 320 });
}
async function getContentFromEmbedSyncedDocModel(
host: EditorHost,
@@ -212,7 +158,7 @@ function actionToStream<T extends keyof BlockSuitePresets.AIActions>(
>,
extract?: (
host: EditorHost,
ctx: CtxRecord
ctx: AIContext
) => Promise<{
content?: string;
attachments?: (string | Blob)[];
@@ -226,7 +172,7 @@ function actionToStream<T extends keyof BlockSuitePresets.AIActions>(
if (!action || typeof action !== 'function') return;
if (extract && typeof extract === 'function') {
return (host: EditorHost, ctx: CtxRecord): BlockSuitePresets.TextStream => {
return (host: EditorHost, ctx: AIContext): BlockSuitePresets.TextStream => {
let stream: BlockSuitePresets.TextStream | undefined;
const control = trackerOptions?.control || 'format-bar';
const where = trackerOptions?.where || 'ai-panel';
@@ -270,7 +216,7 @@ function actionToStream<T extends keyof BlockSuitePresets.AIActions>(
let stream: BlockSuitePresets.TextStream | undefined;
return {
async *[Symbol.asyncIterator]() {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const models = getCopilotSelectedElems(host);
const markdown = await getTextFromSelected(panel.host);
@@ -304,7 +250,7 @@ function actionToGeneration<T extends keyof BlockSuitePresets.AIActions>(
>,
extract?: (
host: EditorHost,
ctx: CtxRecord
ctx: AIContext
) => Promise<{
content?: string;
attachments?: (string | Blob)[];
@@ -312,7 +258,7 @@ function actionToGeneration<T extends keyof BlockSuitePresets.AIActions>(
} | void>,
trackerOptions?: BlockSuitePresets.TrackerOptions
) {
return (host: EditorHost, ctx: CtxRecord) => {
return (host: EditorHost, ctx: AIContext) => {
return ({
input,
signal,
@@ -352,14 +298,14 @@ function updateEdgelessAIPanelConfig<
edgelessCopilot: EdgelessCopilotWidget,
id: T,
generatingIcon: TemplateResult<1>,
ctx: CtxRecord,
ctx: AIContext,
variants?: Omit<
Parameters<BlockSuitePresets.AIActions[T]>[0],
keyof BlockSuitePresets.AITextActionOptions
>,
customInput?: (
host: EditorHost,
ctx: CtxRecord
ctx: AIContext
) => Promise<{
input?: string;
content?: string;
@@ -371,7 +317,7 @@ function updateEdgelessAIPanelConfig<
const host = aiPanel.host;
const { config } = aiPanel;
assertExists(config);
config.answerRenderer = actionToRenderer(id, host, ctx);
config.answerRenderer = actionToAnswerRenderer(id, host, ctx);
config.generateAnswer = actionToGeneration(
id,
variants,
@@ -420,7 +366,7 @@ export function actionToHandler<T extends keyof BlockSuitePresets.AIActions>(
>,
customInput?: (
host: EditorHost,
ctx: CtxRecord
ctx: AIContext
) => Promise<{
input?: string;
content?: string;
@@ -430,22 +376,11 @@ export function actionToHandler<T extends keyof BlockSuitePresets.AIActions>(
trackerOptions?: BlockSuitePresets.TrackerOptions
) {
return (host: EditorHost) => {
const aiPanel = getAIPanel(host);
const aiPanel = getAIPanelWidget(host);
const edgelessCopilot = getEdgelessCopilotWidget(host);
let internal: Record<string, unknown> = {};
const selectedElements = getCopilotSelectedElems(host);
const { selectedBlocks } = getSelections(host);
const ctx = {
get() {
return {
...internal,
selectedElements,
};
},
set(data: Record<string, unknown>) {
internal = data;
},
};
const ctx = new AIContext({ selectedElements });
edgelessCopilot.hideCopilotPanel();
edgelessCopilot.lockToolbar(true);

View File

@@ -31,9 +31,10 @@ import { styleMap } from 'lit/directives/style-map.js';
import { insertFromMarkdown } from '../../_common';
import { AIPenIcon, ChatWithAIIcon } from '../_common/icons';
import { getAIPanel } from '../ai-panel';
import { AIProvider } from '../provider';
import { reportResponse } from '../utils/action-reporter';
import { getAIPanelWidget } from '../utils/ai-widgets';
import type { AIContext } from '../utils/context';
import {
getEdgelessCopilotWidget,
getService,
@@ -47,8 +48,8 @@ import {
getEdgelessService,
getSurfaceElementFromEditor,
} from '../utils/selection-utils';
import { createTemplateJob } from '../utils/template-job';
import { EXCLUDING_INSERT_ACTIONS, generatingStages } from './consts';
import type { CtxRecord } from './types';
type FinishConfig = Exclude<
AffineAIPanelWidget['config'],
@@ -106,14 +107,17 @@ export function retry(panel: AffineAIPanelWidget): AIItemConfig {
const extraConditions: Record<string, (data: any) => boolean> = {
createSlides: data => !!data.contents,
};
export function createInsertResp<T extends keyof BlockSuitePresets.AIActions>(
export function createInsertItems<T extends keyof BlockSuitePresets.AIActions>(
id: T,
handler: (host: EditorHost, ctx: CtxRecord) => void,
host: EditorHost,
ctx: CtxRecord,
buttonText: string = 'Insert below'
ctx: AIContext,
variants?: Omit<
Parameters<BlockSuitePresets.AIActions[T]>[0],
keyof BlockSuitePresets.AITextActionOptions
>
): AIItemConfig[] {
const extraCondition = extraConditions[id] || ((_: any) => true);
const buttonText = getButtonText[id]?.(variants) ?? 'Insert below';
return [
{
name: `${buttonText} - Loading...`,
@@ -121,7 +125,7 @@ export function createInsertResp<T extends keyof BlockSuitePresets.AIActions>(
${LightLoadingIcon}
</div>`,
showWhen: () => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const data = ctx.get();
return (
!EXCLUDING_INSERT_ACTIONS.includes(id) &&
@@ -135,7 +139,7 @@ export function createInsertResp<T extends keyof BlockSuitePresets.AIActions>(
name: buttonText,
icon: InsertBelowIcon,
showWhen: () => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const data = ctx.get();
return (
!EXCLUDING_INSERT_ACTIONS.includes(id) &&
@@ -146,14 +150,41 @@ export function createInsertResp<T extends keyof BlockSuitePresets.AIActions>(
},
handler: () => {
reportResponse('result:insert');
handler(host, ctx);
const panel = getAIPanel(host);
edgelessResponseHandler(id, host, ctx).catch(console.error);
const panel = getAIPanelWidget(host);
panel.hide();
},
},
];
}
async function edgelessResponseHandler<
T extends keyof BlockSuitePresets.AIActions,
>(id: T, host: EditorHost, ctx: AIContext) {
switch (id) {
case 'expandMindmap':
responseToExpandMindmap(host, ctx);
break;
case 'brainstormMindmap':
responseToBrainstormMindmap(host, ctx);
break;
case 'makeItReal':
responseToMakeItReal(host, ctx);
break;
case 'createSlides':
await responseToCreateSlides(host, ctx);
break;
case 'createImage':
case 'filterImage':
case 'processImage':
responseToCreateImage(host);
break;
default:
defaultHandler(host);
break;
}
}
export function asCaption<T extends keyof BlockSuitePresets.AIActions>(
id: T,
host: EditorHost
@@ -162,12 +193,12 @@ export function asCaption<T extends keyof BlockSuitePresets.AIActions>(
name: 'Use as caption',
icon: AIPenIcon,
showWhen: () => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
return id === 'generateCaption' && !!panel.answer;
},
handler: () => {
reportResponse('result:use-as-caption');
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const caption = panel.answer;
if (!caption) return;
@@ -183,11 +214,6 @@ export function asCaption<T extends keyof BlockSuitePresets.AIActions>(
};
}
type MindMapNode = {
text: string;
children: MindMapNode[];
};
function insertBelow(
host: EditorHost,
markdown: string,
@@ -256,7 +282,7 @@ function createBlockAndInsert(
* @param host EditorHost
*/
const defaultHandler = (host: EditorHost) => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const selectedElements = getCopilotSelectedElems(host);
assertExists(panel.answer);
@@ -282,8 +308,8 @@ const defaultHandler = (host: EditorHost) => {
* Should make the inserting image size same with the input image if there is an input image.
* @param host
*/
const imageHandler = (host: EditorHost) => {
const aiPanel = getAIPanel(host);
function responseToCreateImage(host: EditorHost) {
const aiPanel = getAIPanelWidget(host);
// `DataURL` or `URL`
const data = aiPanel.answer;
if (!data) return;
@@ -334,206 +360,181 @@ const imageHandler = (host: EditorHost) => {
});
})
.catch(console.error);
};
}
export const responses: {
[key in keyof Partial<BlockSuitePresets.AIActions>]: (
host: EditorHost,
ctx: CtxRecord
) => void;
} = {
expandMindmap: (host, ctx) => {
const [surface] = host.doc.getBlockByFlavour(
'affine:surface'
) as SurfaceBlockModel[];
export function responseToExpandMindmap(host: EditorHost, ctx: AIContext) {
const [surface] = host.doc.getBlockByFlavour(
'affine:surface'
) as SurfaceBlockModel[];
const elements = ctx.get()[
'selectedElements'
] as BlockSuite.EdgelessModel[];
const data = ctx.get() as {
node: MindMapNode;
};
const elements = ctx.get().selectedElements;
const mindmapNode = ctx.get().node;
queueMicrotask(() => {
getAIPanel(host).hide();
queueMicrotask(() => {
getAIPanelWidget(host).hide();
});
if (!mindmapNode || !elements) return;
const mindmap = elements[0].group as MindmapElementModel;
if (mindmapNode.children) {
mindmapNode.children.forEach(childTree => {
MindmapUtils.addTree(mindmap, elements[0].id, childTree);
});
const mindmap = elements[0].group as MindmapElementModel;
const subtree = mindmap.getNode(elements[0].id);
if (!data?.node) return;
if (!subtree) return;
if (data.node.children) {
data.node.children.forEach(childTree => {
MindmapUtils.addTree(mindmap, elements[0].id, childTree);
});
surface.doc.transact(() => {
const updateNodeSize = (node: typeof subtree) => {
fitContent(node.element as ShapeElementModel);
const subtree = mindmap.getNode(elements[0].id);
if (!subtree) return;
surface.doc.transact(() => {
const updateNodeSize = (node: typeof subtree) => {
fitContent(node.element as ShapeElementModel);
node.children.forEach(child => {
updateNodeSize(child);
});
};
updateNodeSize(subtree);
});
setTimeout(() => {
const edgelessService = getEdgelessService(host);
edgelessService.selection.set({
elements: [subtree.element.id],
editing: false,
node.children.forEach(child => {
updateNodeSize(child);
});
});
}
},
brainstormMindmap: (host, ctx) => {
const aiPanel = getAIPanel(host);
const edgelessService = getEdgelessService(host);
const edgelessCopilot = getEdgelessCopilotWidget(host);
const selectionRect = edgelessCopilot.selectionModelRect;
const [surface] = host.doc.getBlockByFlavour(
'affine:surface'
) as SurfaceBlockModel[];
const elements = ctx.get()[
'selectedElements'
] as BlockSuite.EdgelessModel[];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const data = ctx.get() as any;
let newGenerated = true;
};
// This means regenerate
if (isMindMapRoot(elements[0])) {
const mindmap = elements[0].group as MindmapElementModel;
const xywh = mindmap.tree.element.xywh;
surface.deleteElement(mindmap.id);
if (data.node) {
data.node.xywh = xywh;
newGenerated = false;
}
}
edgelessCopilot.hideCopilotPanel();
aiPanel.hide();
const mindmapId = surface.addElement({
type: 'mindmap',
children: data.node,
style: data.style,
});
const mindmap = surface.getElementById(mindmapId) as MindmapElementModel;
host.doc.transact(() => {
mindmap.childElements.forEach(shape => {
fitContent(shape as ShapeElementModel);
});
updateNodeSize(subtree);
});
const telemetryService = host.std.getOptional(TelemetryProvider);
telemetryService?.track('CanvasElementAdded', {
control: 'ai',
page: 'whiteboard editor',
module: 'toolbar',
segment: 'toolbar',
type: 'mindmap',
});
queueMicrotask(() => {
if (newGenerated && selectionRect) {
mindmap.moveTo([
selectionRect.x,
selectionRect.y,
selectionRect.width,
selectionRect.height,
]);
}
});
// This is a workaround to make sure mindmap and other microtask are done
setTimeout(() => {
edgelessService.viewport.setViewportByBound(
mindmap.elementBound,
[20, 20, 20, 20],
true
);
const edgelessService = getEdgelessService(host);
edgelessService.selection.set({
elements: [mindmap.tree.element.id],
elements: [subtree.element.id],
editing: false,
});
});
},
makeItReal: (host, ctx) => {
const aiPanel = getAIPanel(host);
let html = aiPanel.answer;
if (!html) return;
html = preprocessHtml(html);
}
}
const edgelessCopilot = getEdgelessCopilotWidget(host);
const [surface] = host.doc.getBlockByFlavour(
'affine:surface'
) as SurfaceBlockModel[];
function responseToBrainstormMindmap(host: EditorHost, ctx: AIContext) {
const aiPanel = getAIPanelWidget(host);
const edgelessService = getEdgelessService(host);
const edgelessCopilot = getEdgelessCopilotWidget(host);
const selectionRect = edgelessCopilot.selectionModelRect;
const [surface] = host.doc.getBlockByFlavour(
'affine:surface'
) as SurfaceBlockModel[];
const data = ctx.get();
const bounds = edgelessCopilot.determineInsertionBounds(
(data['width'] as number) || 800,
(data['height'] as number) || 600
const { node, style, selectedElements } = ctx.get();
if (!node) return;
const elements = selectedElements;
// This means regenerate
if (elements && isMindMapRoot(elements[0])) {
const mindmap = elements[0].group as MindmapElementModel;
const xywh = mindmap.tree.element.xywh;
surface.deleteElement(mindmap.id);
node.xywh = xywh;
} else {
node.xywh = `[${selectionRect.x + selectionRect.width + 100},${selectionRect.y},0,0]`;
}
edgelessCopilot.hideCopilotPanel();
aiPanel.hide();
const mindmapId = surface.addElement({
type: 'mindmap',
children: node,
style: style,
});
const mindmap = surface.getElementById(mindmapId) as MindmapElementModel;
host.doc.transact(() => {
mindmap.childElements.forEach(shape => {
fitContent(shape as ShapeElementModel);
});
});
const telemetryService = host.std.getOptional(TelemetryProvider);
telemetryService?.track('CanvasElementAdded', {
control: 'ai',
page: 'whiteboard editor',
module: 'toolbar',
segment: 'toolbar',
type: 'mindmap',
});
// This is a workaround to make sure mindmap and other microtask are done
setTimeout(() => {
edgelessService.viewport.setViewportByBound(
mindmap.elementBound,
[20, 20, 20, 20],
true
);
edgelessCopilot.hideCopilotPanel();
aiPanel.hide();
const edgelessRoot = getEdgelessRootFromEditor(host);
host.doc.transact(() => {
edgelessRoot.doc.addBlock(
'affine:embed-html',
{
html,
design: 'ai:makeItReal', // as tag
xywh: bounds.serialize(),
},
surface.id
);
edgelessService.selection.set({
elements: [mindmap.tree.element.id],
editing: false,
});
},
createSlides: (host, ctx) => {
const data = ctx.get();
const contents = data.contents as unknown[];
if (!contents) return;
const images = data.images as { url: string; id: string }[][];
const service = host.std.getService<EdgelessRootService>('affine:page');
if (!service) return;
});
}
(async function () {
for (let i = 0; i < contents.length - 1; i++) {
const image = images[i];
const content = contents[i];
const job = service.createTemplateJob('template');
await Promise.all(
image.map(({ id, url }) =>
fetch(url)
.then(res => res.blob())
.then(blob => job.job.assets.set(id, blob))
)
);
await job.insertTemplate(content);
getSurfaceElementFromEditor(host).refresh();
}
})().catch(console.error);
},
createImage: imageHandler,
processImage: imageHandler,
filterImage: imageHandler,
};
function responseToMakeItReal(host: EditorHost, ctx: AIContext) {
const aiPanel = getAIPanelWidget(host);
let html = aiPanel.answer;
if (!html) return;
html = preprocessHtml(html);
const edgelessCopilot = getEdgelessCopilotWidget(host);
const [surface] = host.doc.getBlockByFlavour(
'affine:surface'
) as SurfaceBlockModel[];
const data = ctx.get();
const bounds = edgelessCopilot.determineInsertionBounds(
data.width || 800,
data.height || 600
);
edgelessCopilot.hideCopilotPanel();
aiPanel.hide();
const edgelessRoot = getEdgelessRootFromEditor(host);
host.doc.transact(() => {
edgelessRoot.doc.addBlock(
'affine:embed-html',
{
html,
design: 'ai:makeItReal', // as tag
xywh: bounds.serialize(),
},
surface.id
);
});
}
async function responseToCreateSlides(host: EditorHost, ctx: AIContext) {
const data = ctx.get();
const { contents = [], images = [] } = data;
if (contents.length === 0) return;
const service = host.std.getService<EdgelessRootService>('affine:page');
if (!service) return;
try {
for (let i = 0; i < contents.length; i++) {
const image = images[i] || [];
const content = contents[i];
const job = createTemplateJob(host);
const imagePromises = image.map(async ({ id, url }) => {
const response = await fetch(url);
const blob = await response.blob();
job.job.assets.set(id, blob);
});
await Promise.all(imagePromises);
await job.insertTemplate(content);
}
getSurfaceElementFromEditor(host).refresh();
} catch (error) {
console.error('Error creating slides:', error);
}
}
const getButtonText: {
[key in keyof Partial<BlockSuitePresets.AIActions>]: (
@@ -548,27 +549,10 @@ const getButtonText: {
},
};
export function getInsertAndReplaceHandler<
T extends keyof BlockSuitePresets.AIActions,
>(
id: T,
host: EditorHost,
ctx: CtxRecord,
variants?: Omit<
Parameters<BlockSuitePresets.AIActions[T]>[0],
keyof BlockSuitePresets.AITextActionOptions
>
) {
const handler = responses[id] ?? defaultHandler;
const buttonText = getButtonText[id]?.(variants) ?? undefined;
return createInsertResp(id, handler, host, ctx, buttonText);
}
export function actionToResponse<T extends keyof BlockSuitePresets.AIActions>(
id: T,
host: EditorHost,
ctx: CtxRecord,
ctx: AIContext,
variants?: Omit<
Parameters<BlockSuitePresets.AIActions[T]>[0],
keyof BlockSuitePresets.AITextActionOptions
@@ -584,7 +568,7 @@ export function actionToResponse<T extends keyof BlockSuitePresets.AIActions>(
icon: ChatWithAIIcon,
handler: () => {
reportResponse('result:continue-in-chat');
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
AIProvider.slots.requestOpenWithChat.emit({
host,
appendCard: true,
@@ -592,10 +576,10 @@ export function actionToResponse<T extends keyof BlockSuitePresets.AIActions>(
panel.hide();
},
},
...getInsertAndReplaceHandler(id, host, ctx, variants),
...createInsertItems(id, host, ctx, variants),
asCaption(id, host),
retry(getAIPanel(host)),
discard(getAIPanel(host), getEdgelessCopilotWidget(host)),
retry(getAIPanelWidget(host)),
discard(getAIPanelWidget(host), getEdgelessCopilotWidget(host)),
],
},
],
@@ -619,7 +603,7 @@ export function actionToErrorResponse<
panel: AffineAIPanelWidget,
id: T,
host: EditorHost,
ctx: CtxRecord,
ctx: AIContext,
variants?: Omit<
Parameters<BlockSuitePresets.AIActions[T]>[0],
keyof BlockSuitePresets.AITextActionOptions
@@ -640,13 +624,13 @@ export function actionToErrorResponse<
responses: [
{
name: 'Response',
items: getInsertAndReplaceHandler(id, host, ctx, variants),
items: createInsertItems(id, host, ctx, variants),
},
{
name: '',
items: [
retry(getAIPanel(host)),
discard(getAIPanel(host), getEdgelessCopilotWidget(host)),
retry(getAIPanelWidget(host)),
discard(getAIPanelWidget(host), getEdgelessCopilotWidget(host)),
],
},
],

View File

@@ -0,0 +1,309 @@
import type { EditorHost } from '@blocksuite/affine/block-std';
import {
GfxBlockElementModel,
type GfxModel,
LayerManager,
} from '@blocksuite/affine/block-std/gfx';
import type {
MindmapElementModel,
ShapeElementModel,
} from '@blocksuite/affine/blocks';
import {
fitContent,
getSurfaceBlock,
SurfaceBlockModel,
TelemetryProvider,
uploadBlobForImage,
} from '@blocksuite/affine/blocks';
import { Bound, getCommonBound } from '@blocksuite/affine/global/utils';
import { type BlockProps, DocCollection, Text } from '@blocksuite/affine/store';
import { getAIPanelWidget } from '../utils/ai-widgets';
import type { AffineNode, AIContext } from '../utils/context';
import { insertAbove, insertBelow, replace } from '../utils/editor-actions';
import { preprocessHtml } from '../utils/html';
import { fetchImageToFile } from '../utils/image';
import { getSelections } from '../utils/selection-utils';
import { createTemplateJob } from '../utils/template-job';
const PADDING = 100;
type Place = 'after' | 'before';
export async function pageResponseHandler<
T extends keyof BlockSuitePresets.AIActions,
>(id: T, host: EditorHost, ctx: AIContext, place: Place = 'after') {
switch (id) {
case 'brainstormMindmap':
responseToBrainstormMindmap(host, ctx, place);
break;
case 'makeItReal':
responseToMakeItReal(host, ctx, place);
break;
case 'createSlides':
await responseToCreateSlides(host, ctx, place);
break;
case 'createImage':
case 'filterImage':
case 'processImage':
responseToCreateImage(host, place);
break;
default:
await (place === 'after'
? insertMarkdownBelow(host)
: insertMarkdownAbove(host));
break;
}
}
function responseToBrainstormMindmap(
host: EditorHost,
ctx: AIContext,
place: Place
) {
const surface = getSurfaceBlock(host.doc);
if (!surface) return;
host.doc.transact(() => {
const { node, style } = ctx.get();
if (!node) return;
const bound = getEdgelessContentBound(host);
if (bound) {
node.xywh = `[${bound.x + bound.w + PADDING * 2},${bound.y},0,0]`;
}
const mindmapId = surface.addElement({
type: 'mindmap',
children: node,
style: style,
});
const mindmap = surface.getElementById(mindmapId) as MindmapElementModel;
mindmap.childElements.forEach(shape => {
fitContent(shape as ShapeElementModel);
});
// wait for mindmap xywh update
setTimeout(() => {
const frameBound = expandBound(mindmap.elementBound, PADDING);
addSurfaceRefBlock(host, frameBound, place);
}, 0);
});
const telemetryService = host.std.getOptional(TelemetryProvider);
telemetryService?.track('CanvasElementAdded', {
control: 'ai',
page: 'doc editor',
module: 'toolbar',
segment: 'toolbar',
type: 'mindmap',
});
}
function responseToMakeItReal(host: EditorHost, ctx: AIContext, place: Place) {
const surface = getSurfaceBlock(host.doc);
const aiPanel = getAIPanelWidget(host);
if (!aiPanel.answer || !surface) return;
const { width, height } = ctx.get();
const bound = getEdgelessContentBound(host);
const x = bound ? bound.x + bound.w + PADDING * 2 : 0;
const y = bound ? bound.y : 0;
const htmlBound = new Bound(x, y, width || 800, height || 600);
const html = preprocessHtml(aiPanel.answer);
host.doc.transact(() => {
host.doc.addBlock(
'affine:embed-html',
{
html,
design: 'ai:makeItReal', // as tag
xywh: htmlBound.serialize(),
},
surface.id
);
const frameBound = expandBound(htmlBound, PADDING);
addSurfaceRefBlock(host, frameBound, place);
});
}
async function responseToCreateSlides(
host: EditorHost,
ctx: AIContext,
place: Place
) {
const { contents, images = [] } = ctx.get();
const surface = getSurfaceBlock(host.doc);
if (!contents || !surface) return;
try {
const frameIds: string[] = [];
for (let i = 0; i < contents.length; i++) {
const image = images[i];
const content = contents[i];
const job = createTemplateJob(host);
await Promise.all(
image.map(({ id, url }) =>
fetch(url)
.then(res => res.blob())
.then(blob => job.job.assets.set(id, blob))
)
);
await job.insertTemplate(content);
const frame = findFrameObject(content.blocks);
frame && frameIds.push(frame.id);
}
const props = frameIds.map(id => ({
flavour: 'affine:surface-ref',
refFlavour: 'affine:frame',
reference: id,
}));
addSiblingBlocks(host, props, place);
} catch (error) {
console.error('Error creating slides:', error);
}
}
export function responseToCreateImage(host: EditorHost, place: Place) {
const aiPanel = getAIPanelWidget(host);
const { answer } = aiPanel;
if (!answer) return;
const filename = 'image';
const imageProxy = host.std.clipboard.configs.get('imageProxy');
fetchImageToFile(answer, filename, imageProxy)
.then(file => {
if (!file) return;
host.doc.transact(() => {
const props = {
flavour: 'affine:image',
size: file.size,
};
const blockId = addSiblingBlocks(host, [props], place)?.[0];
blockId && uploadBlobForImage(host, blockId, file).catch(console.error);
});
})
.catch(console.error);
}
export async function replaceWithMarkdown(host: EditorHost) {
const aiPanel = getAIPanelWidget(host);
const { answer } = aiPanel;
const selection = getSelection(host);
if (!answer || !selection) return;
const { textSelection, firstBlock, selectedModels } = selection;
await replace(host, answer, firstBlock, selectedModels, textSelection);
}
async function insertMarkdownBelow(host: EditorHost) {
const aiPanel = getAIPanelWidget(host);
const { answer } = aiPanel;
const selection = getSelection(host);
if (!answer || !selection) return;
const { lastBlock } = selection;
await insertBelow(host, answer, lastBlock);
}
async function insertMarkdownAbove(host: EditorHost) {
const aiPanel = getAIPanelWidget(host);
const { answer } = aiPanel;
const selection = getSelection(host);
if (!answer || !selection) return;
const { firstBlock } = selection;
await insertAbove(host, answer, firstBlock);
}
function getSelection(host: EditorHost) {
const textSelection = host.selection.find('text');
const mode = textSelection ? 'flat' : 'highest';
const { selectedBlocks } = getSelections(host, mode);
if (!selectedBlocks) return;
const length = selectedBlocks.length;
const firstBlock = selectedBlocks[0];
const lastBlock = selectedBlocks[length - 1];
const selectedModels = selectedBlocks.map(block => block.model);
return {
textSelection,
selectedModels,
firstBlock,
lastBlock,
};
}
function getEdgelessContentBound(host: EditorHost) {
const surface = getSurfaceBlock(host.doc);
if (!surface) return;
const elements = (
host.doc
.getBlocks()
.filter(
model =>
model instanceof GfxBlockElementModel &&
(model.parent instanceof SurfaceBlockModel ||
model.parent?.role === 'root')
) as GfxModel[]
).concat(surface.elementModels ?? []);
const bounds = elements.map(element => Bound.deserialize(element.xywh));
const bound = getCommonBound(bounds);
return bound;
}
function expandBound(bound: Bound, margin: number) {
const x = bound.x - margin;
const y = bound.y - margin;
const w = bound.w + margin * 2;
const h = bound.h + margin * 2;
return new Bound(x, y, w, h);
}
function addSurfaceRefBlock(host: EditorHost, bound: Bound, place: Place) {
const surface = getSurfaceBlock(host.doc);
if (!surface) return;
const frame = host.doc.addBlock(
'affine:frame',
{
title: new Text(new DocCollection.Y.Text('Frame')),
xywh: bound.serialize(),
index: LayerManager.INITIAL_INDEX,
},
surface
);
const props = {
flavour: 'affine:surface-ref',
refFlavour: 'affine:frame',
reference: frame,
};
return addSiblingBlocks(host, [props], place);
}
function addSiblingBlocks(
host: EditorHost,
props: Array<Partial<BlockProps>>,
place: Place
) {
const { selectedModels } = getSelection(host) || {};
if (!selectedModels) return;
const targetModel =
place === 'before'
? selectedModels[0]
: selectedModels[selectedModels.length - 1];
return host.doc.addSiblingBlocks(targetModel, props, place);
}
function findFrameObject(obj: AffineNode): AffineNode | null {
if (obj.flavour === 'affine:frame') {
return obj;
}
if (obj.children) {
for (const child of obj.children) {
const result = findFrameObject(child);
if (result) {
return result;
}
}
}
return null;
}

View File

@@ -36,11 +36,6 @@ export const imageProcessingTypes = [
'Convert to sticker',
] as const;
export type CtxRecord = {
get(): Record<string, unknown>;
set(data: Record<string, unknown>): void;
};
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace BlockSuitePresets {

View File

@@ -1,7 +1,6 @@
import type { EditorHost } from '@blocksuite/affine/block-std';
import {
AFFINE_AI_PANEL_WIDGET,
AffineAIPanelWidget,
type AffineAIPanelWidget,
type AffineAIPanelWidgetConfig,
type AIItemConfig,
ImageBlockModel,
@@ -24,35 +23,22 @@ import {
ReplaceIcon,
RetryIcon,
} from './_common/icons';
import { INSERT_ABOVE_ACTIONS } from './actions/consts';
import {
EXCLUDING_REPLACE_ACTIONS,
INSERT_ABOVE_ACTIONS,
} from './actions/consts';
import {
pageResponseHandler,
replaceWithMarkdown,
} from './actions/page-response';
import { AIProvider } from './provider';
import { reportResponse } from './utils/action-reporter';
import { getAIPanelWidget } from './utils/ai-widgets';
import { AIContext } from './utils/context';
import { findNoteBlockModel, getService } from './utils/edgeless';
import {
copyTextAnswer,
insertAbove,
insertBelow,
replace,
} from './utils/editor-actions';
import { copyTextAnswer } from './utils/editor-actions';
import { getSelections } from './utils/selection-utils';
function getSelection(host: EditorHost) {
const textSelection = host.selection.find('text');
const mode = textSelection ? 'flat' : 'highest';
const { selectedBlocks } = getSelections(host, mode);
assertExists(selectedBlocks);
const length = selectedBlocks.length;
const firstBlock = selectedBlocks[0];
const lastBlock = selectedBlocks[length - 1];
const selectedModels = selectedBlocks.map(block => block.model);
return {
textSelection,
selectedModels,
firstBlock,
lastBlock,
};
}
function asCaption<T extends keyof BlockSuitePresets.AIActions>(
host: EditorHost,
id?: T
@@ -61,12 +47,12 @@ function asCaption<T extends keyof BlockSuitePresets.AIActions>(
name: 'Use as caption',
icon: AIPenIcon,
showWhen: () => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
return id === 'generateCaption' && !!panel.answer;
},
handler: () => {
reportResponse('result:use-as-caption');
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const caption = panel.answer;
if (!caption) return;
@@ -87,7 +73,7 @@ function createNewNote(host: EditorHost): AIItemConfig {
name: 'Create new note',
icon: CreateIcon,
showWhen: () => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
return !!panel.answer && isInsideEdgelessEditor(host);
},
handler: () => {
@@ -103,7 +89,7 @@ function createNewNote(host: EditorHost): AIItemConfig {
const bound = Bound.deserialize(noteModel.xywh);
const newBound = new Bound(bound.x - bound.w - 20, bound.y, bound.w, 72);
const doc = host.doc;
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
const service = getService(host);
doc.transact(() => {
assertExists(doc.root);
@@ -147,43 +133,11 @@ function createNewNote(host: EditorHost): AIItemConfig {
};
}
async function replaceWithAnswer(panel: AffineAIPanelWidget) {
const { host } = panel;
const selection = getSelection(host);
if (!selection || !panel.answer) return;
const { textSelection, firstBlock, selectedModels } = selection;
await replace(host, panel.answer, firstBlock, selectedModels, textSelection);
panel.hide();
}
async function insertAnswerBelow(panel: AffineAIPanelWidget) {
const { host } = panel;
const selection = getSelection(host);
if (!selection || !panel.answer) {
return;
}
const { lastBlock } = selection;
await insertBelow(host, panel.answer, lastBlock);
panel.hide();
}
async function insertAnswerAbove(panel: AffineAIPanelWidget) {
const { host } = panel;
const selection = getSelection(host);
if (!selection || !panel.answer) return;
const { firstBlock } = selection;
await insertAbove(host, panel.answer, firstBlock);
panel.hide();
}
export function buildTextResponseConfig<
T extends keyof BlockSuitePresets.AIActions,
>(panel: AffineAIPanelWidget, id?: T) {
function buildPageResponseConfig<T extends keyof BlockSuitePresets.AIActions>(
panel: AffineAIPanelWidget,
id: T,
ctx: AIContext
) {
const host = panel.host;
return [
@@ -197,7 +151,8 @@ export function buildTextResponseConfig<
!!panel.answer && (!id || !INSERT_ABOVE_ACTIONS.includes(id)),
handler: () => {
reportResponse('result:insert');
insertAnswerBelow(panel).catch(console.error);
pageResponseHandler(id, host, ctx, 'after').catch(console.error);
panel.hide();
},
},
{
@@ -207,17 +162,20 @@ export function buildTextResponseConfig<
!!panel.answer && !!id && INSERT_ABOVE_ACTIONS.includes(id),
handler: () => {
reportResponse('result:insert');
insertAnswerAbove(panel).catch(console.error);
pageResponseHandler(id, host, ctx, 'before').catch(console.error);
panel.hide();
},
},
asCaption(host, id),
{
name: 'Replace selection',
icon: ReplaceIcon,
showWhen: () => !!panel.answer,
showWhen: () =>
!!panel.answer && !EXCLUDING_REPLACE_ACTIONS.includes(id),
handler: () => {
reportResponse('result:replace');
replaceWithAnswer(panel).catch(console.error);
replaceWithMarkdown(host).catch(console.error);
panel.hide();
},
},
createNewNote(host),
@@ -258,46 +216,8 @@ export function buildTextResponseConfig<
];
}
export function buildErrorResponseConfig<
T extends keyof BlockSuitePresets.AIActions,
>(panel: AffineAIPanelWidget, id?: T) {
const host = panel.host;
export function buildErrorResponseConfig(panel: AffineAIPanelWidget) {
return [
{
name: 'Response',
items: [
{
name: 'Replace selection',
icon: ReplaceIcon,
showWhen: () => !!panel.answer,
handler: () => {
replaceWithAnswer(panel).catch(console.error);
},
},
{
name: 'Insert below',
icon: InsertBelowIcon,
showWhen: () =>
!!panel.answer && (!id || !INSERT_ABOVE_ACTIONS.includes(id)),
handler: () => {
insertAnswerBelow(panel).catch(console.error);
},
},
{
name: 'Insert above',
icon: InsertTopIcon,
showWhen: () =>
!!panel.answer && !!id && INSERT_ABOVE_ACTIONS.includes(id),
handler: () => {
reportResponse('result:insert');
insertAnswerAbove(panel).catch(console.error);
},
},
asCaption(host, id),
createNewNote(host),
],
},
{
name: '',
items: [
@@ -325,18 +245,16 @@ export function buildErrorResponseConfig<
export function buildFinishConfig<T extends keyof BlockSuitePresets.AIActions>(
panel: AffineAIPanelWidget,
id?: T
id: T,
ctx: AIContext
) {
return {
responses: buildTextResponseConfig(panel, id),
responses: buildPageResponseConfig(panel, id, ctx),
actions: [],
};
}
export function buildErrorConfig<T extends keyof BlockSuitePresets.AIActions>(
panel: AffineAIPanelWidget,
id?: T
) {
export function buildErrorConfig(panel: AffineAIPanelWidget) {
return {
upgrade: () => {
AIProvider.slots.requestUpgradePlan.emit({ host: panel.host });
@@ -349,7 +267,7 @@ export function buildErrorConfig<T extends keyof BlockSuitePresets.AIActions>(
cancel: () => {
panel.hide();
},
responses: buildErrorResponseConfig(panel, id),
responses: buildErrorResponseConfig(panel),
};
}
@@ -371,22 +289,12 @@ export function buildCopyConfig(panel: AffineAIPanelWidget) {
export function buildAIPanelConfig(
panel: AffineAIPanelWidget
): AffineAIPanelWidgetConfig {
const ctx = new AIContext();
return {
answerRenderer: createTextRenderer(panel.host, { maxHeight: 320 }),
finishStateConfig: buildFinishConfig(panel),
finishStateConfig: buildFinishConfig(panel, 'chat', ctx),
generatingStateConfig: buildGeneratingConfig(),
errorStateConfig: buildErrorConfig(panel),
copy: buildCopyConfig(panel),
};
}
export const getAIPanel = (host: EditorHost): AffineAIPanelWidget => {
const rootBlockId = host.doc.root?.id;
assertExists(rootBlockId);
const aiPanel = host.view.getWidget(AFFINE_AI_PANEL_WIDGET, rootBlockId);
assertExists(aiPanel);
if (!(aiPanel instanceof AffineAIPanelWidget)) {
throw new Error('AI panel not found');
}
return aiPanel;
};

View File

@@ -57,6 +57,11 @@ const cardsStyles = css`
font-size: 14px;
font-weight: 400;
color: var(--affine-text-secondary-color);
display: -webkit-box;
-webkit-box-orient: vertical;
-webkit-line-clamp: 3;
overflow: hidden;
text-overflow: ellipsis;
}
}
`;

View File

@@ -398,15 +398,13 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
}
scrollToEnd() {
this.updateComplete
.then(() => {
if (!this.messagesContainer) return;
this.messagesContainer.scrollTo({
top: this.messagesContainer.scrollHeight,
behavior: 'smooth',
});
})
.catch(console.error);
requestAnimationFrame(() => {
if (!this.messagesContainer) return;
this.messagesContainer.scrollTo({
top: this.messagesContainer.scrollHeight,
behavior: 'smooth',
});
});
}
retry = async () => {

View File

@@ -159,7 +159,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
};
private readonly _scrollToEnd = () => {
requestAnimationFrame(() => this._chatMessages.value?.scrollToEnd());
this._chatMessages.value?.scrollToEnd();
};
private readonly _cleanupHistories = async () => {
@@ -200,7 +200,15 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
_changedProperties.has('chatContextValue') &&
this.chatContextValue.status !== 'idle'
) {
setTimeout(this._scrollToEnd, 500);
if (this.chatContextValue.status === 'transmitting') {
this._scrollToEnd();
} else if (
this.chatContextValue.status === 'loading' ||
this.chatContextValue.status === 'error' ||
this.chatContextValue.status === 'success'
) {
setTimeout(this._scrollToEnd, 500);
}
}
}

View File

@@ -45,8 +45,8 @@ import {
textTones,
translateLangs,
} from '../../actions/types';
import { getAIPanel } from '../../ai-panel';
import { AIProvider } from '../../provider';
import { getAIPanelWidget } from '../../utils/ai-widgets';
import { mindMapToMarkdown } from '../../utils/edgeless';
import { canvasToBlob, randomSeed } from '../../utils/image';
import {
@@ -105,7 +105,7 @@ const othersGroup: AIItemGroupConfig = {
icon: CommentIcon,
showWhen: () => true,
handler: host => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
AIProvider.slots.requestOpenWithChat.emit({
host,
mode: 'edgeless',
@@ -120,7 +120,7 @@ const othersGroup: AIItemGroupConfig = {
icon: ChatWithAIIcon,
showWhen: () => true,
handler: host => {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
AIProvider.slots.requestOpenWithChat.emit({
host,
mode: 'edgeless',
@@ -286,7 +286,7 @@ const generateGroup: AIItemGroupConfig = {
const selectedElements = getCopilotSelectedElems(host);
const len = selectedElements.length;
const aiPanel = getAIPanel(host);
const aiPanel = getAIPanelWidget(host);
// text to image
// from user input
if (len === 0) {
@@ -297,7 +297,7 @@ const generateGroup: AIItemGroupConfig = {
};
}
let content = (ctx.get()['content'] as string) || '';
let content = ctx.get().content || '';
// from user input
if (content.length === 0) {
@@ -419,7 +419,7 @@ const generateGroup: AIItemGroupConfig = {
// from user input
if (selectedElements.length === 0) {
const aiPanel = getAIPanel(host);
const aiPanel = getAIPanelWidget(host);
const content = aiPanel.inputText?.trim();
if (!content) return;
return {
@@ -438,7 +438,7 @@ const generateGroup: AIItemGroupConfig = {
if (f + i + n + s + e === 0) {
return;
}
let content = (ctx.get()['content'] as string) || '';
let content = ctx.get().content || '';
// single note, text
if (
@@ -455,7 +455,7 @@ const generateGroup: AIItemGroupConfig = {
// from user input
if (content.length === 0) {
const aiPanel = getAIPanel(host);
const aiPanel = getAIPanelWidget(host);
content = aiPanel.inputText?.trim() || '';
}
@@ -521,7 +521,7 @@ const generateGroup: AIItemGroupConfig = {
],
};
export const edgelessActionGroups = [
export const edgelessAIGroups: AIItemGroupConfig[] = [
reviewGroup,
editGroup,
generateGroup,

View File

@@ -9,16 +9,16 @@ import { EdgelessCopilotToolbarEntry } from '@blocksuite/affine/blocks';
import { noop } from '@blocksuite/affine/global/utils';
import { html } from 'lit';
import { getAIPanel } from '../../ai-panel';
import { AIProvider } from '../../provider';
import { getAIPanelWidget } from '../../utils/ai-widgets';
import { getEdgelessCopilotWidget } from '../../utils/edgeless';
import { extractContext } from '../../utils/extract';
import { edgelessActionGroups } from './actions-config';
import { edgelessAIGroups } from './actions-config';
noop(EdgelessCopilotToolbarEntry);
export function setupEdgelessCopilot(widget: EdgelessCopilotWidget) {
widget.groups = edgelessActionGroups;
widget.groups = edgelessAIGroups;
}
export function setupEdgelessElementToolbarAIEntry(
@@ -30,7 +30,7 @@ export function setupEdgelessElementToolbarAIEntry(
},
render: (edgeless: EdgelessRootBlockComponent) => {
const chain = edgeless.service.std.command.chain();
const filteredGroups = edgelessActionGroups.reduce((pre, group) => {
const filteredGroups = edgelessAIGroups.reduce((pre, group) => {
const filtered = group.items.filter(item =>
item.showWhen?.(chain, 'edgeless' as DocMode, edgeless.host)
);
@@ -43,7 +43,7 @@ export function setupEdgelessElementToolbarAIEntry(
if (filteredGroups.every(group => group.items.length === 0)) return null;
const handler = () => {
const aiPanel = getAIPanel(edgeless.host);
const aiPanel = getAIPanelWidget(edgeless.host);
if (aiPanel.config) {
aiPanel.config.generateAnswer = ({ finish, input }) => {
finish('success');
@@ -70,7 +70,7 @@ export function setupEdgelessElementToolbarAIEntry(
return html`<edgeless-copilot-toolbar-entry
.edgeless=${edgeless}
.host=${edgeless.host}
.groups=${edgelessActionGroups}
.groups=${edgelessAIGroups}
.onClick=${handler}
></edgeless-copilot-toolbar-entry>`;
},

View File

@@ -6,7 +6,7 @@ import {
} from '@blocksuite/affine/blocks';
import { html, type TemplateResult } from 'lit';
import { AIItemGroups } from '../../_common/config';
import { pageAIGroups } from '../../_common/config';
export function setupFormatBarAIEntry(formatBar: AffineFormatBarWidget) {
toolbarDefaultConfig(formatBar);
@@ -18,7 +18,7 @@ export function setupFormatBarAIEntry(formatBar: AffineFormatBarWidget) {
return html`
<ask-ai-toolbar-button
.host=${formatBar.host}
.actionGroups=${AIItemGroups}
.actionGroups=${pageAIGroups}
></ask-ai-toolbar-button>
`;
},

View File

@@ -14,12 +14,12 @@ import {
import { assertExists } from '@blocksuite/affine/global/utils';
import { html } from 'lit';
import { AIItemGroups } from '../../_common/config';
import { pageAIGroups } from '../../_common/config';
import { handleInlineAskAIAction } from '../../actions/doc-handler';
import { AIProvider } from '../../provider';
export function setupSlashMenuAIEntry(slashMenu: AffineSlashMenuWidget) {
const AIItems = AIItemGroups.map(group => group.items).flat();
const AIItems = pageAIGroups.map(group => group.items).flat();
const iconWrapper = (icon: AIItemConfig['icon']) => {
return html`<div style="color: var(--affine-primary-color)">

View File

@@ -10,7 +10,8 @@ import {
import { noop } from '@blocksuite/affine/global/utils';
import { html, nothing } from 'lit';
import { getAIPanel } from '../ai-panel';
import { getAIPanelWidget } from '../utils/ai-widgets';
import type { AIContext } from '../utils/context';
noop(MiniMindmapPreview);
@@ -19,15 +20,12 @@ export const createMindmapRenderer: (
/**
* Used to store data for later use during rendering.
*/
ctx: {
get: () => Record<string, unknown>;
set: (data: Record<string, unknown>) => void;
},
ctx: AIContext,
style?: MindmapStyle
) => AffineAIPanelWidgetConfig['answerRenderer'] = (host, ctx, style) => {
return (answer, state) => {
if (state === 'generating') {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
panel.generatingElement?.updateLoadingProgress(2);
}
@@ -55,18 +53,12 @@ export const createMindmapExecuteRenderer: (
/**
* Used to store data for later use during rendering.
*/
ctx: {
get: () => Record<string, unknown>;
set: (data: Record<string, unknown>) => void;
},
handler: (ctx: {
get: () => Record<string, unknown>;
set: (data: Record<string, unknown>) => void;
}) => void
ctx: AIContext,
handler: (host: EditorHost, ctx: AIContext) => void
) => AffineAIPanelWidgetConfig['answerRenderer'] = (host, ctx, handler) => {
return (answer, state) => {
if (state !== 'finished') {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
panel.generatingElement?.updateLoadingProgress(2);
return nothing;
}
@@ -75,7 +67,7 @@ export const createMindmapExecuteRenderer: (
node: markdownToMindmap(answer, host.doc),
});
handler(ctx);
handler(host, ctx);
return nothing;
};

View File

@@ -11,19 +11,17 @@ import { css, html, LitElement, nothing } from 'lit';
import { property, query } from 'lit/decorators.js';
import { createRef, type Ref, ref } from 'lit/directives/ref.js';
import { getAIPanel } from '../ai-panel';
import { PPTBuilder } from '../slides/index';
import { getAIPanelWidget } from '../utils/ai-widgets';
import type { AIContext } from '../utils/context';
export const createSlidesRenderer: (
host: EditorHost,
ctx: {
get: () => Record<string, unknown>;
set: (data: Record<string, unknown>) => void;
}
ctx: AIContext
) => AffineAIPanelWidgetConfig['answerRenderer'] = (host, ctx) => {
return (answer, state) => {
if (state === 'generating') {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
panel.generatingElement?.updateLoadingProgress(2);
return nothing;
}
@@ -77,7 +75,7 @@ export class AISlidesRenderer extends WithDisposable(LitElement) {
requestAnimationFrame(() => {
if (!this._editorHost) return;
PPTBuilder(this._editorHost)
.process(this.text)
?.process(this.text)
.then(res => {
if (res && this.ctx) {
this.ctx.set({
@@ -85,7 +83,7 @@ export class AISlidesRenderer extends WithDisposable(LitElement) {
images: res.images,
});
// refresh loading menu item
getAIPanel(this.host)
getAIPanelWidget(this.host)
.shadowRoot?.querySelector('ai-panel-answer')
?.requestUpdate();
}

View File

@@ -3,7 +3,7 @@ import type { AffineAIPanelWidgetConfig } from '@blocksuite/affine/blocks';
import { css, html, LitElement, nothing } from 'lit';
import { property } from 'lit/decorators.js';
import { getAIPanel } from '../ai-panel';
import { getAIPanelWidget } from '../utils/ai-widgets';
import { preprocessHtml } from '../utils/html';
type AIAnswerWrapperOptions = {
@@ -62,7 +62,7 @@ export const createIframeRenderer: (
) => AffineAIPanelWidgetConfig['answerRenderer'] = (host, options) => {
return (answer, state) => {
if (state === 'generating') {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
panel.generatingElement?.updateLoadingProgress(2);
return nothing;
}
@@ -91,7 +91,7 @@ export const createImageRenderer: (
) => AffineAIPanelWidgetConfig['answerRenderer'] = (host, options) => {
return (answer, state) => {
if (state === 'generating') {
const panel = getAIPanel(host);
const panel = getAIPanelWidget(host);
panel.generatingElement?.updateLoadingProgress(2);
return nothing;
}

View File

@@ -119,8 +119,6 @@ export class AIProvider {
}>(),
requestLogin: new Slot<{ host: EditorHost }>(),
requestUpgradePlan: new Slot<{ host: EditorHost }>(),
// when an action is requested to run in edgeless mode (show a toast in affine)
requestRunInEdgeless: new Slot<{ host: EditorHost }>(),
// stream of AI actions triggered by users
actions: new Slot<{
action: keyof BlockSuitePresets.AIActions;
@@ -311,7 +309,6 @@ export class AIProvider {
options: BlockSuitePresets.AIForkChatSessionOptions
) => string | Promise<string>;
} else {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
AIProvider.instance.provideAction(id as any, action as any);
}
}

View File

@@ -4,6 +4,7 @@ import type { BlockSnapshot } from '@blocksuite/affine/store';
import { markdownToSnapshot } from '../../_common';
import { getSurfaceElementFromEditor } from '../utils/selection-utils';
import { createTemplateJob } from '../utils/template-job';
import {
basicTheme,
type PPTDoc,
@@ -16,6 +17,7 @@ export const PPTBuilder = (host: EditorHost) => {
const docs: PPTDoc[] = [];
const contents: unknown[] = [];
const allImages: TemplateImage[][] = [];
if (!service) return;
const addDoc = async (block: BlockSnapshot) => {
const sections = block.children.map(v => {
@@ -35,8 +37,7 @@ export const PPTBuilder = (host: EditorHost) => {
};
docs.push(doc);
if (doc.isCover || !service) return;
const job = service.createTemplateJob('template');
const job = createTemplateJob(host);
const { images, content } = await basicTheme(doc);
contents.push(content);
allImages.push(images);
@@ -56,7 +57,6 @@ export const PPTBuilder = (host: EditorHost) => {
return {
process: async (text: string) => {
try {
if (!service) return;
const snapshot = await markdownToSnapshot(text, host);
const block = snapshot.snapshot.content[0];

View File

@@ -3,7 +3,6 @@ import { nanoid } from '@blocksuite/affine/store';
import { AIProvider } from '../provider';
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const replaceText = (text: Record<string, string>, template: any) => {
if (template != null && typeof template === 'object') {
if (Array.isArray(template)) {
@@ -55,7 +54,7 @@ const getImageUrlByKeyword =
const getImages = async (
images: Record<string, (w: number, h: number) => Promise<string> | string>,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
template: any
): Promise<TemplateImage[]> => {
const imgs: Record<
@@ -66,7 +65,7 @@ const getImages = async (
height: number;
}
> = {};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const run = (data: any) => {
if (data != null && typeof data === 'object') {
if (Array.isArray(data)) {
@@ -93,19 +92,21 @@ const getImages = async (
Object.entries(imgs).map(async ([name, data]) => {
const getImage = images[name];
if (!getImage) {
return;
return null;
}
try {
const url = await getImage(data.width, data.height);
return {
id: data.id,
url,
} as TemplateImage;
} catch (error) {
console.error('Error getting image:', error);
return null;
}
const url = await getImage(data.width, data.height);
return {
id: data.id,
url,
} satisfies TemplateImage;
})
);
const notNull = (v?: TemplateImage): v is TemplateImage => {
return v != null;
};
return list.filter(notNull);
return list.filter(v => !!v);
};
export type PPTSection = {

View File

@@ -0,0 +1,17 @@
import type { EditorHost } from '@blocksuite/affine/block-std';
import {
AFFINE_AI_PANEL_WIDGET,
AffineAIPanelWidget,
} from '@blocksuite/affine/blocks';
import { assertExists } from '@blocksuite/affine/global/utils';
export const getAIPanelWidget = (host: EditorHost): AffineAIPanelWidget => {
const rootBlockId = host.doc.root?.id;
assertExists(rootBlockId);
const aiPanel = host.view.getWidget(AFFINE_AI_PANEL_WIDGET, rootBlockId);
assertExists(aiPanel);
if (!(aiPanel instanceof AffineAIPanelWidget)) {
throw new Error('AI panel not found');
}
return aiPanel;
};

View File

@@ -0,0 +1,50 @@
import type { MindmapStyle } from '@blocksuite/affine/blocks';
import type { SerializedXYWH } from '@blocksuite/affine/global/utils';
import type { TemplateImage } from '../slides/template';
export interface ContextValue {
selectedElements?: BlockSuite.EdgelessModel[];
content?: string;
// make it real
width?: number;
height?: number;
// mindmap
node?: MindMapNode | null;
style?: MindmapStyle;
centerPosition?: SerializedXYWH;
// slides
contents?: Array<{ blocks: AffineNode }>;
images?: TemplateImage[][];
}
export interface AffineNode {
id: string;
flavour: string;
children: AffineNode[];
}
type MindMapNode = {
xywh?: SerializedXYWH;
text: string;
children: MindMapNode[];
};
export class AIContext {
private _value: ContextValue;
constructor(initData: ContextValue = {}) {
this._value = initData;
}
get = () => {
return this._value;
};
set = (data: ContextValue) => {
this._value = {
...this._value,
...data,
};
};
}

View File

@@ -0,0 +1,45 @@
import type { EditorHost } from '@blocksuite/affine/block-std';
import { LayerManager } from '@blocksuite/affine/block-std/gfx';
import {
getSurfaceBlock,
TemplateJob,
TemplateMiddlewares,
} from '@blocksuite/affine/blocks';
import {
assertExists,
Bound,
getCommonBound,
} from '@blocksuite/affine/global/utils';
export function createTemplateJob(host: EditorHost) {
const surface = getSurfaceBlock(host.doc);
assertExists(surface);
const middlewares: ((job: TemplateJob) => void)[] = [];
const layer = new LayerManager(host.doc, surface, {
watch: false,
});
const bounds = [...layer.blocks, ...layer.canvasElements].map(i =>
Bound.deserialize(i.xywh)
);
const currentContentBound = getCommonBound(bounds);
if (currentContentBound) {
currentContentBound.x += currentContentBound.w + 100;
middlewares.push(
TemplateMiddlewares.createInsertPlaceMiddleware(currentContentBound)
);
}
const idxGenerator = layer.createIndexGenerator();
middlewares.push(
TemplateMiddlewares.createRegenerateIndexMiddleware(() => idxGenerator())
);
middlewares.push(TemplateMiddlewares.replaceIdMiddleware);
return TemplateJob.create({
model: surface,
type: 'template',
middlewares,
});
}

View File

@@ -1,4 +1,4 @@
import { notify, Scrollable } from '@affine/component';
import { Scrollable } from '@affine/component';
import { PageDetailSkeleton } from '@affine/component/page-detail-skeleton';
import type { ChatPanel } from '@affine/core/blocksuite/presets/ai';
import { AIProvider } from '@affine/core/blocksuite/presets/ai';
@@ -10,7 +10,6 @@ import { useDocMetaHelper } from '@affine/core/components/hooks/use-block-suite-
import { EditorService } from '@affine/core/modules/editor';
import { RecentDocsService } from '@affine/core/modules/quicksearch';
import { ViewService } from '@affine/core/modules/workbench/services/view';
import { useI18n } from '@affine/i18n';
import { RefNodeSlotsProvider } from '@blocksuite/affine/blocks';
import { DisposableGroup } from '@blocksuite/affine/global/utils';
import { type AffineEditorContainer } from '@blocksuite/affine/presets';
@@ -102,7 +101,6 @@ const DetailPageImpl = memo(function DetailPageImpl() {
// TODO(@eyhn): remove jotai here
const [_, setActiveBlockSuiteEditor] = useActiveBlocksuiteEditor();
const t = useI18n();
const enableAI = featureFlagService.flags.enable_ai.value;
useEffect(() => {
@@ -203,22 +201,6 @@ const DetailPageImpl = memo(function DetailPageImpl() {
}
}
disposable.add(
AIProvider.slots.requestRunInEdgeless.on(({ host }) => {
if (host === editorHost) {
notify.warning({
title: t['com.affine.ai.action.edgeless-only.dialog-title'](),
action: {
label: t['Switch'](),
onClick: () => {
editor.setMode('edgeless');
},
},
});
}
})
);
const unbind = editor.bindEditorContainer(
editorContainer,
(editorContainer as any).docTitle, // set from proxy
@@ -230,7 +212,7 @@ const DetailPageImpl = memo(function DetailPageImpl() {
disposable.dispose();
};
},
[editor, openPage, docCollection.id, jumpToPageBlock, t]
[editor, openPage, docCollection.id, jumpToPageBlock]
);
const [hasScrollTop, setHasScrollTop] = useState(false);