feat(server): online copilot api test (#8807)

packages/backend/server/package.json

@@ -12,9 +12,11 @@
     "dev": "nodemon ./src/index.ts",
     "dev:mail": "email dev -d src/mails",
     "test": "ava --concurrency 1 --serial",
-    "test:copilot": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e": "ava \"src/__tests__/**/copilot-*.e2e.ts\"",
+    "test:copilot:spec": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
     "test:coverage": "c8 ava --concurrency 1 --serial",
-    "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.e2e.ts\"",
+    "test:copilot:spec:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
     "data-migration": "cross-env NODE_ENV=development r ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run",
     "postinstall": "prisma generate"

packages/backend/server/src/__tests__/copilot-provider.e2e.ts

@@ -0,0 +1,196 @@
+import { randomUUID } from 'node:crypto';
+
+import { createRandomAIUser } from '@affine-test/kit/utils/cloud';
+import type { ExecutionContext, TestFn } from 'ava';
+import ava from 'ava';
+
+import { createWorkspace } from './utils';
+import {
+  chatWithImages,
+  chatWithText,
+  chatWithWorkflow,
+  createCopilotMessage,
+  createCopilotSession,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+  sse2array,
+} from './utils/copilot';
+
+type Tester = {
+  app: any;
+  userEmail: string;
+  workspaceId: string;
+};
+const test = ava as TestFn<Tester>;
+
+const e2eConfig = {
+  endpoint: process.env.COPILOT_E2E_ENDPOINT || 'http://localhost:3010',
+};
+
+const isCopilotConfigured =
+  !!process.env.COPILOT_OPENAI_API_KEY &&
+  !!process.env.COPILOT_FAL_API_KEY &&
+  process.env.COPILOT_OPENAI_API_KEY !== '1' &&
+  process.env.COPILOT_FAL_API_KEY !== '1';
+const runIfCopilotConfigured = test.macro(
+  async (
+    t,
+    callback: (t: ExecutionContext<Tester>) => Promise<void> | void
+  ) => {
+    if (isCopilotConfigured) {
+      await callback(t);
+    } else {
+      t.log('Skip test because copilot is not configured');
+      t.pass();
+    }
+  }
+);
+
+export const runPrisma = async <T>(
+  cb: (
+    prisma: InstanceType<
+      typeof import('../../node_modules/@prisma/client').PrismaClient
+    >
+  ) => Promise<T>
+): Promise<T> => {
+  const { PrismaClient } = await import('../../node_modules/@prisma/client');
+  const client = new PrismaClient();
+  await client.$connect();
+  try {
+    return await cb(client);
+  } finally {
+    await client.$disconnect();
+  }
+};
+
+test.before(async t => {
+  if (!isCopilotConfigured) return;
+  const { endpoint } = e2eConfig;
+
+  const { email } = await createRandomAIUser('affine.fail', runPrisma);
+  const app = { getHttpServer: () => endpoint } as any;
+  const { id } = await createWorkspace(app);
+
+  t.context.app = app;
+  t.context.userEmail = email;
+  t.context.workspaceId = id;
+});
+
+test.after(async t => {
+  if (!isCopilotConfigured) return;
+  await runPrisma(async client => {
+    await client.user.delete({
+      where: {
+        email: t.context.userEmail,
+      },
+    });
+  });
+});
+
+const retry = async (
+  action: string,
+  t: ExecutionContext<Tester>,
+  callback: (t: ExecutionContext<Tester>) => void
+) => {
+  let i = 3;
+  while (i--) {
+    const ret = await t.try(callback);
+    if (ret.passed) {
+      return ret.commit();
+    } else {
+      ret.discard();
+      t.log(ret.errors.map(e => e.message).join('\n'));
+      t.log(`retrying ${action} ${3 - i}/3 ...`);
+    }
+  }
+  t.fail(`failed to run ${action}`);
+};
+
+const makeCopilotChat = async (
+  t: ExecutionContext<Tester>,
+  promptName: string,
+  { content, attachments, params }: any
+) => {
+  const { app, workspaceId } = t.context;
+  const sessionId = await createCopilotSession(
+    app,
+    workspaceId,
+    randomUUID(),
+    promptName
+  );
+  const messageId = await createCopilotMessage(
+    app,
+    sessionId,
+    content,
+    attachments,
+    undefined,
+    params
+  );
+  return { sessionId, messageId };
+};
+
+// ==================== action ====================
+
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
+  const prompts = Array.isArray(promptName) ? promptName : [promptName];
+  for (const promptName of prompts) {
+    test(
+      `should be able to run action: ${promptName}`,
+      runIfCopilotConfigured,
+      async t => {
+        await retry(`action: ${promptName}`, t, async t => {
+          const { app } = t.context;
+          const { sessionId, messageId } = await makeCopilotChat(
+            t,
+            promptName,
+            messages[0]
+          );
+
+          if (type === 'text') {
+            const result = await chatWithText(app, sessionId, messageId);
+            t.truthy(result, 'should return result');
+            verifier?.(t, result);
+          } else if (type === 'image') {
+            const result = sse2array(
+              await chatWithImages(app, sessionId, messageId)
+            )
+              .filter(e => e.event !== 'event')
+              .map(e => e.data)
+              .filter(Boolean);
+            t.truthy(result.length, 'should return result');
+            for (const r of result) {
+              verifier?.(t, r);
+            }
+          } else {
+            t.fail('unsupported provider type');
+          }
+        });
+      }
+    );
+  }
+}
+
+// ==================== workflow ====================
+
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
+  test(
+    `should be able to run workflow: ${name}`,
+    runIfCopilotConfigured,
+    async t => {
+      await retry(`workflow: ${name}`, t, async t => {
+        const { app } = t.context;
+        const { sessionId, messageId } = await makeCopilotChat(
+          t,
+          `workflow:${name}`,
+          { content }
+        );
+        const r = await chatWithWorkflow(app, sessionId, messageId);
+        const result = sse2array(r)
+          .filter(e => e.event !== 'event' && e.data)
+          .reduce((p, c) => p + c.data, '');
+        t.truthy(result, 'should return result');
+        verifier?.(t, result);
+      });
+    }
+  );
+}

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -25,7 +25,11 @@ import {
   CopilotCheckJsonExecutor,
 } from '../plugins/copilot/workflow/executor';
 import { createTestingModule, TestingModule } from './utils';
-import { TestAssets } from './utils/copilot';
+import {
+  checkMDList,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+} from './utils/copilot';
 
 type Tester = {
   auth: AuthService;
@@ -135,58 +139,6 @@ test.after(async t => {
   await t.context.module.close();
 });
 
-const assertNotWrappedInCodeBlock = (
-  t: ExecutionContext<Tester>,
-  result: string
-) => {
-  t.assert(
-    !result.replaceAll('\n', '').trim().startsWith('```') &&
-      !result.replaceAll('\n', '').trim().endsWith('```'),
-    'should not wrap in code block'
-  );
-};
-
-const checkMDList = (text: string) => {
-  const lines = text.split('\n');
-  const listItemRegex = /^( {2})*(-|\u2010-\u2015|\*|\+)? .+$/;
-  let prevIndent = null;
-
-  for (const line of lines) {
-    if (line.trim() === '') continue;
-    if (!listItemRegex.test(line)) {
-      return false;
-    }
-
-    const currentIndent = line.match(/^( *)/)?.[0].length!;
-    if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) {
-      return false;
-    }
-
-    if (prevIndent !== null && currentIndent > 0) {
-      const indentDiff = currentIndent - prevIndent;
-      // allow 1 level of indentation difference
-      if (indentDiff > 2) {
-        return false;
-      }
-    }
-
-    if (line.trim().startsWith('-')) {
-      prevIndent = currentIndent;
-    }
-  }
-
-  return true;
-};
-
-const checkUrl = (url: string) => {
-  try {
-    new URL(url);
-    return true;
-  } catch {
-    return false;
-  }
-};
-
 const retry = async (
   action: string,
   t: ExecutionContext<Tester>,
@@ -268,140 +220,7 @@ test('should validate markdown list', t => {
 
 // ==================== action ====================
 
-const actions = [
-  {
-    promptName: [
-      'Summary',
-      'Explain this',
-      'Write an article about this',
-      'Write a twitter about this',
-      'Write a poem about this',
-      'Write a blog post about this',
-      'Write outline',
-      'Change tone to',
-      'Improve writing for it',
-      'Improve grammar for it',
-      'Fix spelling for it',
-      'Create headings',
-      'Make it longer',
-      'Make it shorter',
-      'Continue writing',
-      'Chat With AFFiNE AI',
-      'Search With AFFiNE AI',
-    ],
-    messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('single source of truth'),
-        'should include original keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Brainstorm ideas about this', 'Brainstorm mindmap'],
-    messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Expand mind map',
-    messages: [{ role: 'user' as const, content: '- Single source of truth' }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Find action items from it',
-    messages: [{ role: 'user' as const, content: TestAssets.TODO }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Explain this code', 'Check code error'],
-    messages: [{ role: 'user' as const, content: TestAssets.Code }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('distance'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Translate to',
-    messages: [
-      {
-        role: 'user' as const,
-        content: TestAssets.SSOT,
-        params: { language: 'Simplified Chinese' },
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('单一事实来源'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Generate a caption', 'Explain this image'],
-    messages: [
-      {
-        role: 'user' as const,
-        content: '',
-        attachments: [
-          'https://cdn.affine.pro/copilot-test/Qgqy9qZT3VGIEuMIotJYoCCH.jpg',
-        ],
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      const content = result.toLowerCase();
-      t.assert(
-        content.includes('classroom') ||
-          content.includes('school') ||
-          content.includes('sky'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: [
-      'debug:action:fal-face-to-sticker',
-      'debug:action:fal-remove-bg',
-      'debug:action:fal-sd15',
-      'debug:action:fal-upscaler',
-    ],
-    messages: [
-      {
-        role: 'user' as const,
-        content: '',
-        attachments: [
-          'https://cdn.affine.pro/copilot-test/Zkas098lkjdf-908231.jpg',
-        ],
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, link: string) => {
-      t.truthy(checkUrl(link), 'should be a valid url');
-    },
-    type: 'image' as const,
-  },
-];
-for (const { promptName, messages, verifier, type } of actions) {
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
   const prompts = Array.isArray(promptName) ? promptName : [promptName];
   for (const promptName of prompts) {
     test(
@@ -461,28 +280,7 @@ for (const { promptName, messages, verifier, type } of actions) {
 
 // ==================== workflow ====================
 
-const workflows = [
-  {
-    name: 'brainstorm',
-    content: 'apple company',
-    verifier: (t: ExecutionContext, result: string) => {
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-  },
-  {
-    name: 'presentation',
-    content: 'apple company',
-    verifier: (t: ExecutionContext, result: string) => {
-      for (const l of result.split('\n')) {
-        t.notThrows(() => {
-          JSON.parse(l.trim());
-        }, 'should be valid json');
-      }
-    },
-  },
-];
-
-for (const { name, content, verifier } of workflows) {
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
   test(
     `should be able to run workflow: ${name}`,
     runIfCopilotConfigured,