mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-16 17:46:18 +08:00
feat: online ci
This commit is contained in:
DarkSky
@@ -0,0 +1,214 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { createRandomAIUser } from '@affine-test/kit/utils/cloud';
|
||||
import type { ExecutionContext, TestFn } from 'ava';
|
||||
import ava from 'ava';
|
||||
|
||||
import { createWorkspace } from './utils';
|
||||
import {
|
||||
chatWithImages,
|
||||
chatWithText,
|
||||
chatWithWorkflow,
|
||||
createCopilotMessage,
|
||||
createCopilotSession,
|
||||
ProviderActionTestCase,
|
||||
ProviderWorkflowTestCase,
|
||||
sse2array,
|
||||
} from './utils/copilot';
|
||||
|
||||
type Tester = {
|
||||
app: any;
|
||||
userEmail: string;
|
||||
userToken: string;
|
||||
workspaceId: string;
|
||||
};
|
||||
const test = ava as TestFn<Tester>;
|
||||
|
||||
const e2eConfig = {
|
||||
endpoint: process.env.COPILOT_E2E_ENDPOINT || 'http://localhost:3010',
|
||||
};
|
||||
|
||||
const isCopilotConfigured =
|
||||
!!process.env.COPILOT_OPENAI_API_KEY &&
|
||||
!!process.env.COPILOT_FAL_API_KEY &&
|
||||
process.env.COPILOT_OPENAI_API_KEY !== '1' &&
|
||||
process.env.COPILOT_FAL_API_KEY !== '1';
|
||||
const runIfCopilotConfigured = test.macro(
|
||||
async (
|
||||
t,
|
||||
callback: (t: ExecutionContext<Tester>) => Promise<void> | void
|
||||
) => {
|
||||
if (isCopilotConfigured) {
|
||||
await callback(t);
|
||||
} else {
|
||||
t.log('Skip test because copilot is not configured');
|
||||
t.pass();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export const runPrisma = async <T>(
|
||||
cb: (
|
||||
prisma: InstanceType<
|
||||
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
|
||||
typeof import('../../../../packages/backend/server/node_modules/@prisma/client').PrismaClient
|
||||
>
|
||||
) => Promise<T>
|
||||
): Promise<T> => {
|
||||
const {
|
||||
PrismaClient,
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
} = await import(
|
||||
'../../../../packages/backend/server/node_modules/@prisma/client'
|
||||
);
|
||||
const client = new PrismaClient();
|
||||
await client.$connect();
|
||||
try {
|
||||
return await cb(client);
|
||||
} finally {
|
||||
await client.$disconnect();
|
||||
}
|
||||
};
|
||||
|
||||
test.before(async t => {
|
||||
if (!isCopilotConfigured) return;
|
||||
const { endpoint } = e2eConfig;
|
||||
|
||||
const { email, sessionId: token } = await createRandomAIUser(
|
||||
'affine.fail',
|
||||
runPrisma
|
||||
);
|
||||
const app = { getHttpServer: () => endpoint } as any;
|
||||
const { id } = await createWorkspace(app, token);
|
||||
|
||||
t.context.app = app;
|
||||
t.context.userEmail = email;
|
||||
t.context.userToken = token;
|
||||
t.context.workspaceId = id;
|
||||
});
|
||||
|
||||
test.after(async t => {
|
||||
if (!isCopilotConfigured) return;
|
||||
await runPrisma(async client => {
|
||||
await client.user.delete({
|
||||
where: {
|
||||
email: t.context.userEmail,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const retry = async (
|
||||
action: string,
|
||||
t: ExecutionContext<Tester>,
|
||||
callback: (t: ExecutionContext<Tester>) => void
|
||||
) => {
|
||||
let i = 3;
|
||||
while (i--) {
|
||||
const ret = await t.try(callback);
|
||||
if (ret.passed) {
|
||||
return ret.commit();
|
||||
} else {
|
||||
ret.discard();
|
||||
t.log(ret.errors.map(e => e.message).join('\n'));
|
||||
t.log(`retrying ${action} ${3 - i}/3 ...`);
|
||||
}
|
||||
}
|
||||
t.fail(`failed to run ${action}`);
|
||||
};
|
||||
|
||||
const makeCopilotChat = async (
|
||||
t: ExecutionContext<Tester>,
|
||||
promptName: string,
|
||||
{ content, attachments, params }: any
|
||||
) => {
|
||||
const { app, userToken, workspaceId } = t.context;
|
||||
const sessionId = await createCopilotSession(
|
||||
app,
|
||||
userToken,
|
||||
workspaceId,
|
||||
randomUUID(),
|
||||
promptName
|
||||
);
|
||||
const messageId = await createCopilotMessage(
|
||||
app,
|
||||
userToken,
|
||||
sessionId,
|
||||
content,
|
||||
attachments,
|
||||
undefined,
|
||||
params
|
||||
);
|
||||
return { sessionId, messageId };
|
||||
};
|
||||
|
||||
// ==================== action ====================
|
||||
|
||||
for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
|
||||
const prompts = Array.isArray(promptName) ? promptName : [promptName];
|
||||
for (const promptName of prompts) {
|
||||
test(
|
||||
`should be able to run action: ${promptName}`,
|
||||
runIfCopilotConfigured,
|
||||
async t => {
|
||||
await retry(`action: ${promptName}`, t, async t => {
|
||||
const { app, userToken } = t.context;
|
||||
const { sessionId, messageId } = await makeCopilotChat(
|
||||
t,
|
||||
promptName,
|
||||
messages[0]
|
||||
);
|
||||
|
||||
if (type === 'text') {
|
||||
const result = await chatWithText(
|
||||
app,
|
||||
userToken,
|
||||
sessionId,
|
||||
messageId
|
||||
);
|
||||
t.truthy(result, 'should return result');
|
||||
verifier?.(t, result);
|
||||
} else if (type === 'image') {
|
||||
const result = sse2array(
|
||||
await chatWithImages(app, userToken, sessionId, messageId)
|
||||
)
|
||||
.filter(e => e.event !== 'event')
|
||||
.map(e => e.data)
|
||||
.filter(Boolean);
|
||||
t.truthy(result.length, 'should return result');
|
||||
for (const r of result) {
|
||||
verifier?.(t, r);
|
||||
}
|
||||
} else {
|
||||
t.fail('unsupported provider type');
|
||||
}
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== workflow ====================
|
||||
|
||||
for (const { name, content, verifier } of ProviderWorkflowTestCase) {
|
||||
test(
|
||||
`should be able to run workflow: ${name}`,
|
||||
runIfCopilotConfigured,
|
||||
async t => {
|
||||
await retry(`workflow: ${name}`, t, async t => {
|
||||
const { app, userToken } = t.context;
|
||||
const { sessionId, messageId } = await makeCopilotChat(
|
||||
t,
|
||||
`workflow:${name}`,
|
||||
{ content }
|
||||
);
|
||||
const r = await chatWithWorkflow(app, userToken, sessionId, messageId);
|
||||
const result = sse2array(r)
|
||||
.filter(e => e.event !== 'event' && e.data)
|
||||
.reduce((p, c) => p + c.data, '');
|
||||
t.truthy(result, 'should return result');
|
||||
verifier?.(t, result);
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -28,7 +28,11 @@ import {
|
||||
CopilotCheckJsonExecutor,
|
||||
} from '../plugins/copilot/workflow/executor';
|
||||
import { createTestingModule } from './utils';
|
||||
import { TestAssets } from './utils/copilot';
|
||||
import {
|
||||
checkMDList,
|
||||
ProviderActionTestCase,
|
||||
ProviderWorkflowTestCase,
|
||||
} from './utils/copilot';
|
||||
|
||||
type Tester = {
|
||||
auth: AuthService;
|
||||
@@ -138,58 +142,6 @@ test.after(async t => {
|
||||
await t.context.module.close();
|
||||
});
|
||||
|
||||
const assertNotWrappedInCodeBlock = (
|
||||
t: ExecutionContext<Tester>,
|
||||
result: string
|
||||
) => {
|
||||
t.assert(
|
||||
!result.replaceAll('\n', '').trim().startsWith('```') &&
|
||||
!result.replaceAll('\n', '').trim().endsWith('```'),
|
||||
'should not wrap in code block'
|
||||
);
|
||||
};
|
||||
|
||||
const checkMDList = (text: string) => {
|
||||
const lines = text.split('\n');
|
||||
const listItemRegex = /^( {2})*(-|\u2010-\u2015|\*|\+)? .+$/;
|
||||
let prevIndent = null;
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim() === '') continue;
|
||||
if (!listItemRegex.test(line)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentIndent = line.match(/^( *)/)?.[0].length!;
|
||||
if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (prevIndent !== null && currentIndent > 0) {
|
||||
const indentDiff = currentIndent - prevIndent;
|
||||
// allow 1 level of indentation difference
|
||||
if (indentDiff > 2) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (line.trim().startsWith('-')) {
|
||||
prevIndent = currentIndent;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const checkUrl = (url: string) => {
|
||||
try {
|
||||
new URL(url);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const retry = async (
|
||||
action: string,
|
||||
t: ExecutionContext<Tester>,
|
||||
@@ -271,140 +223,7 @@ test('should validate markdown list', t => {
|
||||
|
||||
// ==================== action ====================
|
||||
|
||||
const actions = [
|
||||
{
|
||||
promptName: [
|
||||
'Summary',
|
||||
'Explain this',
|
||||
'Write an article about this',
|
||||
'Write a twitter about this',
|
||||
'Write a poem about this',
|
||||
'Write a blog post about this',
|
||||
'Write outline',
|
||||
'Change tone to',
|
||||
'Improve writing for it',
|
||||
'Improve grammar for it',
|
||||
'Fix spelling for it',
|
||||
'Create headings',
|
||||
'Make it longer',
|
||||
'Make it shorter',
|
||||
'Continue writing',
|
||||
'Chat With AFFiNE AI',
|
||||
'Search With AFFiNE AI',
|
||||
],
|
||||
messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
t.assert(
|
||||
result.toLowerCase().includes('single source of truth'),
|
||||
'should include original keyword'
|
||||
);
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: ['Brainstorm ideas about this', 'Brainstorm mindmap'],
|
||||
messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
t.assert(checkMDList(result), 'should be a markdown list');
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: 'Expand mind map',
|
||||
messages: [{ role: 'user' as const, content: '- Single source of truth' }],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
t.assert(checkMDList(result), 'should be a markdown list');
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: 'Find action items from it',
|
||||
messages: [{ role: 'user' as const, content: TestAssets.TODO }],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
t.assert(checkMDList(result), 'should be a markdown list');
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: ['Explain this code', 'Check code error'],
|
||||
messages: [{ role: 'user' as const, content: TestAssets.Code }],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
t.assert(
|
||||
result.toLowerCase().includes('distance'),
|
||||
'explain code result should include keyword'
|
||||
);
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: 'Translate to',
|
||||
messages: [
|
||||
{
|
||||
role: 'user' as const,
|
||||
content: TestAssets.SSOT,
|
||||
params: { language: 'Simplified Chinese' },
|
||||
},
|
||||
],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
t.assert(
|
||||
result.toLowerCase().includes('单一事实来源'),
|
||||
'explain code result should include keyword'
|
||||
);
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: ['Generate a caption', 'Explain this image'],
|
||||
messages: [
|
||||
{
|
||||
role: 'user' as const,
|
||||
content: '',
|
||||
attachments: [
|
||||
'https://cdn.affine.pro/copilot-test/Qgqy9qZT3VGIEuMIotJYoCCH.jpg',
|
||||
],
|
||||
},
|
||||
],
|
||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||
assertNotWrappedInCodeBlock(t, result);
|
||||
const content = result.toLowerCase();
|
||||
t.assert(
|
||||
content.includes('classroom') ||
|
||||
content.includes('school') ||
|
||||
content.includes('sky'),
|
||||
'explain code result should include keyword'
|
||||
);
|
||||
},
|
||||
type: 'text' as const,
|
||||
},
|
||||
{
|
||||
promptName: [
|
||||
'debug:action:fal-face-to-sticker',
|
||||
'debug:action:fal-remove-bg',
|
||||
'debug:action:fal-sd15',
|
||||
'debug:action:fal-upscaler',
|
||||
],
|
||||
messages: [
|
||||
{
|
||||
role: 'user' as const,
|
||||
content: '',
|
||||
attachments: [
|
||||
'https://cdn.affine.pro/copilot-test/Zkas098lkjdf-908231.jpg',
|
||||
],
|
||||
},
|
||||
],
|
||||
verifier: (t: ExecutionContext<Tester>, link: string) => {
|
||||
t.truthy(checkUrl(link), 'should be a valid url');
|
||||
},
|
||||
type: 'image' as const,
|
||||
},
|
||||
];
|
||||
for (const { promptName, messages, verifier, type } of actions) {
|
||||
for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
|
||||
const prompts = Array.isArray(promptName) ? promptName : [promptName];
|
||||
for (const promptName of prompts) {
|
||||
test(
|
||||
@@ -464,28 +283,7 @@ for (const { promptName, messages, verifier, type } of actions) {
|
||||
|
||||
// ==================== workflow ====================
|
||||
|
||||
const workflows = [
|
||||
{
|
||||
name: 'brainstorm',
|
||||
content: 'apple company',
|
||||
verifier: (t: ExecutionContext, result: string) => {
|
||||
t.assert(checkMDList(result), 'should be a markdown list');
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'presentation',
|
||||
content: 'apple company',
|
||||
verifier: (t: ExecutionContext, result: string) => {
|
||||
for (const l of result.split('\n')) {
|
||||
t.notThrows(() => {
|
||||
JSON.parse(l.trim());
|
||||
}, 'should be valid json');
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const { name, content, verifier } of workflows) {
|
||||
for (const { name, content, verifier } of ProviderWorkflowTestCase) {
|
||||
test(
|
||||
`should be able to run workflow: ${name}`,
|
||||
runIfCopilotConfigured,
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user