Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-17 18:16:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: migrate fal workflow to server (#7581)

Browse Source
This commit is contained in:
darkskygit
2024-07-26 04:04:38 +00:00
parent cb0d91facd
commit 470262d400
24 changed files with 741 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/src/plugins/copilot/workflow/executor/chat-image.ts
+14 -11
View File
@@ -63,28 +63,31 @@ export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor {
params: Record<string, string>,
options?: CopilotChatOptions
): AsyncIterable<NodeExecuteResult> {
const [{ paramKey, id }, prompt, provider] = await this.initExecutor(data);
const [{ paramKey, paramToucher, id }, prompt, provider] =
await this.initExecutor(data);
const finalMessage = prompt.finish(params);
const config = { ...prompt.config, ...options };
if (paramKey) {
// update params with custom key
const result = {
[paramKey]: await provider.generateImages(
finalMessage,
prompt.model,
config
),
};
yield {
type: NodeExecuteState.Params,
params: {
[paramKey]: await provider.generateImages(
finalMessage,
prompt.model,
options
),
},
params: paramToucher?.(result) ?? result,
};
} else {
for await (const content of provider.generateImagesStream(
for await (const attachment of provider.generateImagesStream(
finalMessage,
prompt.model,
options
config
)) {
yield { type: NodeExecuteState.Content, nodeId: id, content };
yield { type: NodeExecuteState.Attachment, nodeId: id, attachment };
}
}
}
packages/backend/server/src/plugins/copilot/workflow/executor/chat-text.ts
+12 -9
View File
@@ -63,26 +63,29 @@ export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor {
params: Record<string, string>,
options?: CopilotChatOptions
): AsyncIterable<NodeExecuteResult> {
const [{ paramKey, id }, prompt, provider] = await this.initExecutor(data);
const [{ paramKey, paramToucher, id }, prompt, provider] =
await this.initExecutor(data);
const finalMessage = prompt.finish(params);
const config = { ...prompt.config, ...options };
if (paramKey) {
// update params with custom key
const result = {
[paramKey]: await provider.generateText(
finalMessage,
prompt.model,
config
),
};
yield {
type: NodeExecuteState.Params,
params: {
[paramKey]: await provider.generateText(
finalMessage,
prompt.model,
options
),
},
params: paramToucher?.(result) ?? result,
};
} else {
for await (const content of provider.generateTextStream(
finalMessage,
prompt.model,
options
config
)) {
yield { type: NodeExecuteState.Content, nodeId: id, content };
}
packages/backend/server/src/plugins/copilot/workflow/executor/check-html.ts
+1 -1
View File
@@ -26,7 +26,7 @@ export class CopilotCheckHtmlExecutor extends AutoRegisteredWorkflowExecutor {
}
private async checkHtml(
content?: string | string[],
content?: string | string[] | Record<string, any>,
strict?: boolean
): Promise<boolean> {
try {
packages/backend/server/src/plugins/copilot/workflow/executor/check-json.ts
+3 -1
View File
@@ -25,7 +25,9 @@ export class CopilotCheckJsonExecutor extends AutoRegisteredWorkflowExecutor {
return NodeExecutorType.CheckJson;
}
private checkJson(content?: string | string[]): boolean {
private checkJson(
content?: string | string[] | Record<string, any>
): boolean {
try {
if (content && typeof content === 'string') {
JSON.parse(content);
packages/backend/server/src/plugins/copilot/workflow/executor/types.ts
+3 -1
View File
@@ -14,13 +14,15 @@ export enum NodeExecuteState {
EndRun,
Params,
Content,
Attachment,
}
export type NodeExecuteResult =
| { type: NodeExecuteState.StartRun; nodeId: string }
| { type: NodeExecuteState.EndRun; nextNode?: WorkflowNode }
| { type: NodeExecuteState.Params; params: WorkflowParams }
| { type: NodeExecuteState.Content; nodeId: string; content: string };
| { type: NodeExecuteState.Content; nodeId: string; content: string }
| { type: NodeExecuteState.Attachment; nodeId: string; attachment: string };
export abstract class NodeExecutor {
abstract get type(): NodeExecutorType;
packages/backend/server/src/plugins/copilot/workflow/graph.ts
-87
View File
@@ -1,87 +0,0 @@
import { NodeExecutorType } from './executor';
import type { WorkflowGraphs, WorkflowNodeState } from './types';
import { WorkflowNodeType } from './types';
export const WorkflowGraphList: WorkflowGraphs = [
{
name: 'presentation',
graph: [
{
id: 'start',
name: 'Start: check language',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:presentation:step1',
paramKey: 'language',
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate presentation',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:presentation:step2',
edges: ['step3'],
},
{
id: 'step3',
name: 'Step 3: format presentation if needed',
nodeType: WorkflowNodeType.Decision,
condition: (nodeIds: string[], params: WorkflowNodeState) => {
const lines = params.content?.split('\n') || [];
return nodeIds[
Number(
!lines.some(line => {
try {
if (line.trim()) {
JSON.parse(line);
}
return false;
} catch {
return true;
}
})
)
];
},
edges: ['step4', 'step5'],
},
{
id: 'step4',
name: 'Step 4: format presentation',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:presentation:step4',
edges: ['step5'],
},
{
id: 'step5',
name: 'Step 5: finish',
nodeType: WorkflowNodeType.Nope,
edges: [],
},
],
},
{
name: 'brainstorm',
graph: [
{
id: 'start',
name: 'Start: check language',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:brainstorm:step1',
paramKey: 'language',
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate brainstorm mind map',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:brainstorm:step2',
edges: [],
},
],
},
];
packages/backend/server/src/plugins/copilot/workflow/graph/brainstorm.ts
+25
View File
@@ -0,0 +1,25 @@
import { NodeExecutorType } from '../executor';
import { type WorkflowGraph, WorkflowNodeType } from '../types';
export const brainstorm: WorkflowGraph = {
name: 'brainstorm',
graph: [
{
id: 'start',
name: 'Start: check language',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:brainstorm:step1',
paramKey: 'language',
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate brainstorm mind map',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:brainstorm:step2',
edges: [],
},
],
};
packages/backend/server/src/plugins/copilot/workflow/graph/image-filter.ts
+183
View File
@@ -0,0 +1,183 @@
import { NodeExecutorType } from '../executor';
import type { WorkflowGraph, WorkflowParams } from '../types';
import { WorkflowNodeType } from '../types';
export const sketch: WorkflowGraph = {
name: 'image-sketch',
graph: [
{
id: 'start',
name: 'Start: extract edge',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'debug:action:fal-teed',
paramKey: 'controlnets',
paramToucher: params => {
if (Array.isArray(params.controlnets)) {
const controlnets = params.controlnets.map(image_url => ({
path: 'diffusers/controlnet-canny-sdxl-1.0',
image_url,
start_percentage: 0.1,
end_percentage: 0.6,
}));
return { controlnets } as WorkflowParams;
} else {
return {};
}
},
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate tags',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:image-sketch:step2',
paramKey: 'tags',
edges: ['step3'],
},
{
id: 'step3',
name: 'Step3: generate image',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'workflow:image-sketch:step3',
edges: [],
},
],
};
export const clay: WorkflowGraph = {
name: 'image-clay',
graph: [
{
id: 'start',
name: 'Start: extract edge',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'debug:action:fal-teed',
paramKey: 'controlnets',
paramToucher: params => {
if (Array.isArray(params.controlnets)) {
const controlnets = params.controlnets.map(image_url => ({
path: 'diffusers/controlnet-canny-sdxl-1.0',
image_url,
start_percentage: 0.1,
end_percentage: 0.6,
}));
return { controlnets } as WorkflowParams;
} else {
return {};
}
},
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate tags',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:image-clay:step2',
paramKey: 'tags',
edges: ['step3'],
},
{
id: 'step3',
name: 'Step3: generate image',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'workflow:image-clay:step3',
edges: [],
},
],
};
export const anime: WorkflowGraph = {
name: 'image-anime',
graph: [
{
id: 'start',
name: 'Start: extract edge',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'debug:action:fal-teed',
paramKey: 'controlnets',
paramToucher: params => {
if (Array.isArray(params.controlnets)) {
const controlnets = params.controlnets.map(image_url => ({
path: 'diffusers/controlnet-canny-sdxl-1.0',
image_url,
start_percentage: 0.1,
end_percentage: 0.6,
}));
return { controlnets } as WorkflowParams;
} else {
return {};
}
},
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate tags',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:image-anime:step2',
paramKey: 'tags',
edges: ['step3'],
},
{
id: 'step3',
name: 'Step3: generate image',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'workflow:image-anime:step3',
edges: [],
},
],
};
export const pixel: WorkflowGraph = {
name: 'image-pixel',
graph: [
{
id: 'start',
name: 'Start: extract edge',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'debug:action:fal-teed',
paramKey: 'controlnets',
paramToucher: params => {
if (Array.isArray(params.controlnets)) {
const controlnets = params.controlnets.map(image_url => ({
path: 'diffusers/controlnet-canny-sdxl-1.0',
image_url,
start_percentage: 0.1,
end_percentage: 0.6,
}));
return { controlnets } as WorkflowParams;
} else {
return {};
}
},
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate tags',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:image-pixel:step2',
paramKey: 'tags',
edges: ['step3'],
},
{
id: 'step3',
name: 'Step3: generate image',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatImage,
promptName: 'workflow:image-pixel:step3',
edges: [],
},
],
};
packages/backend/server/src/plugins/copilot/workflow/graph/index.ts
+13
View File
@@ -0,0 +1,13 @@
import type { WorkflowGraphs } from '../types';
import { brainstorm } from './brainstorm';
import { anime, clay, pixel, sketch } from './image-filter';
import { presentation } from './presentation';
export const WorkflowGraphList: WorkflowGraphs = [
brainstorm,
presentation,
sketch,
clay,
anime,
pixel,
];
packages/backend/server/src/plugins/copilot/workflow/graph/presentation.ts
+63
View File
@@ -0,0 +1,63 @@
import { NodeExecutorType } from '../executor';
import type { WorkflowGraph, WorkflowNodeState } from '../types';
import { WorkflowNodeType } from '../types';
export const presentation: WorkflowGraph = {
name: 'presentation',
graph: [
{
id: 'start',
name: 'Start: check language',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:presentation:step1',
paramKey: 'language',
edges: ['step2'],
},
{
id: 'step2',
name: 'Step 2: generate presentation',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:presentation:step2',
edges: ['step3'],
},
{
id: 'step3',
name: 'Step 3: format presentation if needed',
nodeType: WorkflowNodeType.Decision,
condition: (nodeIds: string[], params: WorkflowNodeState) => {
const lines = params.content?.split('\n') || [];
return nodeIds[
Number(
!lines.some(line => {
try {
if (line.trim()) {
JSON.parse(line);
}
return false;
} catch {
return true;
}
})
)
];
},
edges: ['step4', 'step5'],
},
{
id: 'step4',
name: 'Step 4: format presentation',
nodeType: WorkflowNodeType.Basic,
type: NodeExecutorType.ChatText,
promptName: 'workflow:presentation:step4',
edges: ['step5'],
},
{
id: 'step5',
name: 'Step 5: finish',
nodeType: WorkflowNodeType.Nope,
edges: [],
},
],
};
packages/backend/server/src/plugins/copilot/workflow/types.ts
+5 -1
View File
@@ -16,6 +16,7 @@ export type WorkflowNodeData = { id: string; name: string } & (
promptName?: string;
// update the prompt params by output with the custom key
paramKey?: string;
paramToucher?: (params: WorkflowParams) => WorkflowParams;
}
| {
nodeType: WorkflowNodeType.Decision;
@@ -44,5 +45,8 @@ export type WorkflowGraphs = Array<WorkflowGraph>;
// ===================== executor =====================
export type WorkflowParams = Record<string, string | string[]>;
export type WorkflowParams = Record<
string,
string | string[] | Record<string, any>
>;
export type WorkflowNodeState = Record<string, string>;
packages/backend/server/src/plugins/copilot/workflow/workflow.ts
+11
View File
@@ -9,12 +9,14 @@ import { WorkflowNodeType } from './types';
export enum GraphExecutorState {
EnterNode = 'EnterNode',
EmitContent = 'EmitContent',
EmitAttachment = 'EmitAttachment',
ExitNode = 'ExitNode',
}
export type GraphExecutorStatus = { status: GraphExecutorState } & (
| { status: GraphExecutorState.EnterNode; node: WorkflowNode }
| { status: GraphExecutorState.EmitContent; content: string }
| { status: GraphExecutorState.EmitAttachment; attachment: string }
| { status: GraphExecutorState.ExitNode; node: WorkflowNode }
);
@@ -66,6 +68,15 @@ export class WorkflowGraphExecutor {
} else {
result += ret.content;
}
} else if (
ret.type === NodeExecuteState.Attachment &&
!currentNode.hasEdges
) {
// pass through content as a stream response if node is end node
yield {
status: GraphExecutorState.EmitAttachment,
attachment: ret.attachment,
};
}
}