feat: online ci

.github/actions/copilot-test/action.yml

@@ -3,7 +3,7 @@ description: 'Run Copilot E2E Test'
 inputs:
   script:
     description: 'Script to run'
-    default: 'yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only'
+    default: 'yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only'
     required: false
   openai-key:
     description: 'OpenAI secret key'

.github/workflows/build-test.yml

@@ -505,6 +505,7 @@ jobs:
             changed:
               - 'packages/backend/server/src/plugins/copilot/**'
               - 'packages/backend/server/tests/copilot.*'
+              - 'tests/affine-cloud-copilot/**'
 
       - name: Setup Node.js
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
@@ -526,7 +527,7 @@ jobs:
 
       - name: Run server tests
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
-        run: yarn affine @affine/server test:copilot:coverage --forbid-only
+        run: yarn affine @affine/server test:copilot:spec:coverage --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
@@ -617,7 +618,7 @@ jobs:
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.e2efilter.outputs.changed == 'true' }}
         uses: ./.github/actions/copilot-test
         with:
-          script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          script: yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
           perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}

.github/workflows/copilot-test.yml

@@ -38,6 +38,16 @@ jobs:
       DISTRIBUTION: web
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       REDIS_SERVER_HOST: localhost
+    strategy:
+      fail-fast: false
+      matrix:
+        spec:
+          - {
+              name: e2e,
+              package: '@affine-test/affine-cloud-copilot',
+              type: e2e,
+            }
+          - { name: spec, package: '@affine/server', type: copilot:spec }
     services:
       postgres:
         image: postgres
@@ -78,13 +88,14 @@ jobs:
       - name: Prepare Server Test Environment
         uses: ./.github/actions/server-test-env
 
-      - name: Run server tests
-        run: yarn affine @affine/server test:copilot:coverage --forbid-only
+      - name: Run copilot api ${{ matrix.spec.name }} tests
+        run: yarn affine ${{ matrix.spec.package }} test:${{ matrix.spec.type }}:coverage --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
           COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
+          COPILOT_E2E_ENDPOINT: ${{ secrets.COPILOT_E2E_ENDPOINT }}
 
       - name: Upload server test coverage results
         uses: codecov/codecov-action@v5
@@ -145,7 +156,7 @@ jobs:
       - name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
         uses: ./.github/actions/copilot-test
         with:
-          script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+          script: yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
           perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}

package.json

@@ -12,7 +12,8 @@
     "tools/*",
     "docs/reference",
     "tools/@types/*",
-    "tests/*"
+    "tests/*",
+    "tests/affine-cloud/*"
   ],
   "engines": {
     "node": "<23.0.0"

packages/backend/server/package.json

@@ -12,9 +12,11 @@
     "dev": "nodemon ./src/index.ts",
     "dev:mail": "email dev -d src/mails",
     "test": "ava --concurrency 1 --serial",
-    "test:copilot": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e": "ava \"e2e/copilot.e2e.ts\"",
+    "test:copilot:spec": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
     "test:coverage": "c8 ava --concurrency 1 --serial",
-    "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"e2e/copilot.e2e.ts\"",
+    "test:copilot:spec:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
     "data-migration": "cross-env NODE_ENV=script r ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && NODE_ENV=script node --import ./scripts/register.js ./dist/data/index.js run",
     "postinstall": "prisma generate"

packages/backend/server/src/__tests__/copilot-provider.e2e.ts

@@ -0,0 +1,214 @@
+import { randomUUID } from 'node:crypto';
+
+import { createRandomAIUser } from '@affine-test/kit/utils/cloud';
+import type { ExecutionContext, TestFn } from 'ava';
+import ava from 'ava';
+
+import { createWorkspace } from './utils';
+import {
+  chatWithImages,
+  chatWithText,
+  chatWithWorkflow,
+  createCopilotMessage,
+  createCopilotSession,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+  sse2array,
+} from './utils/copilot';
+
+type Tester = {
+  app: any;
+  userEmail: string;
+  userToken: string;
+  workspaceId: string;
+};
+const test = ava as TestFn<Tester>;
+
+const e2eConfig = {
+  endpoint: process.env.COPILOT_E2E_ENDPOINT || 'http://localhost:3010',
+};
+
+const isCopilotConfigured =
+  !!process.env.COPILOT_OPENAI_API_KEY &&
+  !!process.env.COPILOT_FAL_API_KEY &&
+  process.env.COPILOT_OPENAI_API_KEY !== '1' &&
+  process.env.COPILOT_FAL_API_KEY !== '1';
+const runIfCopilotConfigured = test.macro(
+  async (
+    t,
+    callback: (t: ExecutionContext<Tester>) => Promise<void> | void
+  ) => {
+    if (isCopilotConfigured) {
+      await callback(t);
+    } else {
+      t.log('Skip test because copilot is not configured');
+      t.pass();
+    }
+  }
+);
+
+export const runPrisma = async <T>(
+  cb: (
+    prisma: InstanceType<
+      // eslint-disable-next-line @typescript-eslint/consistent-type-imports
+      typeof import('../../../../packages/backend/server/node_modules/@prisma/client').PrismaClient
+    >
+  ) => Promise<T>
+): Promise<T> => {
+  const {
+    PrismaClient,
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+  } = await import(
+    '../../../../packages/backend/server/node_modules/@prisma/client'
+  );
+  const client = new PrismaClient();
+  await client.$connect();
+  try {
+    return await cb(client);
+  } finally {
+    await client.$disconnect();
+  }
+};
+
+test.before(async t => {
+  if (!isCopilotConfigured) return;
+  const { endpoint } = e2eConfig;
+
+  const { email, sessionId: token } = await createRandomAIUser(
+    'affine.fail',
+    runPrisma
+  );
+  const app = { getHttpServer: () => endpoint } as any;
+  const { id } = await createWorkspace(app, token);
+
+  t.context.app = app;
+  t.context.userEmail = email;
+  t.context.userToken = token;
+  t.context.workspaceId = id;
+});
+
+test.after(async t => {
+  if (!isCopilotConfigured) return;
+  await runPrisma(async client => {
+    await client.user.delete({
+      where: {
+        email: t.context.userEmail,
+      },
+    });
+  });
+});
+
+const retry = async (
+  action: string,
+  t: ExecutionContext<Tester>,
+  callback: (t: ExecutionContext<Tester>) => void
+) => {
+  let i = 3;
+  while (i--) {
+    const ret = await t.try(callback);
+    if (ret.passed) {
+      return ret.commit();
+    } else {
+      ret.discard();
+      t.log(ret.errors.map(e => e.message).join('\n'));
+      t.log(`retrying ${action} ${3 - i}/3 ...`);
+    }
+  }
+  t.fail(`failed to run ${action}`);
+};
+
+const makeCopilotChat = async (
+  t: ExecutionContext<Tester>,
+  promptName: string,
+  { content, attachments, params }: any
+) => {
+  const { app, userToken, workspaceId } = t.context;
+  const sessionId = await createCopilotSession(
+    app,
+    userToken,
+    workspaceId,
+    randomUUID(),
+    promptName
+  );
+  const messageId = await createCopilotMessage(
+    app,
+    userToken,
+    sessionId,
+    content,
+    attachments,
+    undefined,
+    params
+  );
+  return { sessionId, messageId };
+};
+
+// ==================== action ====================
+
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
+  const prompts = Array.isArray(promptName) ? promptName : [promptName];
+  for (const promptName of prompts) {
+    test(
+      `should be able to run action: ${promptName}`,
+      runIfCopilotConfigured,
+      async t => {
+        await retry(`action: ${promptName}`, t, async t => {
+          const { app, userToken } = t.context;
+          const { sessionId, messageId } = await makeCopilotChat(
+            t,
+            promptName,
+            messages[0]
+          );
+
+          if (type === 'text') {
+            const result = await chatWithText(
+              app,
+              userToken,
+              sessionId,
+              messageId
+            );
+            t.truthy(result, 'should return result');
+            verifier?.(t, result);
+          } else if (type === 'image') {
+            const result = sse2array(
+              await chatWithImages(app, userToken, sessionId, messageId)
+            )
+              .filter(e => e.event !== 'event')
+              .map(e => e.data)
+              .filter(Boolean);
+            t.truthy(result.length, 'should return result');
+            for (const r of result) {
+              verifier?.(t, r);
+            }
+          } else {
+            t.fail('unsupported provider type');
+          }
+        });
+      }
+    );
+  }
+}
+
+// ==================== workflow ====================
+
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
+  test(
+    `should be able to run workflow: ${name}`,
+    runIfCopilotConfigured,
+    async t => {
+      await retry(`workflow: ${name}`, t, async t => {
+        const { app, userToken } = t.context;
+        const { sessionId, messageId } = await makeCopilotChat(
+          t,
+          `workflow:${name}`,
+          { content }
+        );
+        const r = await chatWithWorkflow(app, userToken, sessionId, messageId);
+        const result = sse2array(r)
+          .filter(e => e.event !== 'event' && e.data)
+          .reduce((p, c) => p + c.data, '');
+        t.truthy(result, 'should return result');
+        verifier?.(t, result);
+      });
+    }
+  );
+}

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -28,7 +28,11 @@ import {
   CopilotCheckJsonExecutor,
 } from '../plugins/copilot/workflow/executor';
 import { createTestingModule } from './utils';
-import { TestAssets } from './utils/copilot';
+import {
+  checkMDList,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+} from './utils/copilot';
 
 type Tester = {
   auth: AuthService;
@@ -138,58 +142,6 @@ test.after(async t => {
   await t.context.module.close();
 });
 
-const assertNotWrappedInCodeBlock = (
-  t: ExecutionContext<Tester>,
-  result: string
-) => {
-  t.assert(
-    !result.replaceAll('\n', '').trim().startsWith('```') &&
-      !result.replaceAll('\n', '').trim().endsWith('```'),
-    'should not wrap in code block'
-  );
-};
-
-const checkMDList = (text: string) => {
-  const lines = text.split('\n');
-  const listItemRegex = /^( {2})*(-|\u2010-\u2015|\*|\+)? .+$/;
-  let prevIndent = null;
-
-  for (const line of lines) {
-    if (line.trim() === '') continue;
-    if (!listItemRegex.test(line)) {
-      return false;
-    }
-
-    const currentIndent = line.match(/^( *)/)?.[0].length!;
-    if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) {
-      return false;
-    }
-
-    if (prevIndent !== null && currentIndent > 0) {
-      const indentDiff = currentIndent - prevIndent;
-      // allow 1 level of indentation difference
-      if (indentDiff > 2) {
-        return false;
-      }
-    }
-
-    if (line.trim().startsWith('-')) {
-      prevIndent = currentIndent;
-    }
-  }
-
-  return true;
-};
-
-const checkUrl = (url: string) => {
-  try {
-    new URL(url);
-    return true;
-  } catch {
-    return false;
-  }
-};
-
 const retry = async (
   action: string,
   t: ExecutionContext<Tester>,
@@ -271,140 +223,7 @@ test('should validate markdown list', t => {
 
 // ==================== action ====================
 
-const actions = [
-  {
-    promptName: [
-      'Summary',
-      'Explain this',
-      'Write an article about this',
-      'Write a twitter about this',
-      'Write a poem about this',
-      'Write a blog post about this',
-      'Write outline',
-      'Change tone to',
-      'Improve writing for it',
-      'Improve grammar for it',
-      'Fix spelling for it',
-      'Create headings',
-      'Make it longer',
-      'Make it shorter',
-      'Continue writing',
-      'Chat With AFFiNE AI',
-      'Search With AFFiNE AI',
-    ],
-    messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('single source of truth'),
-        'should include original keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Brainstorm ideas about this', 'Brainstorm mindmap'],
-    messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Expand mind map',
-    messages: [{ role: 'user' as const, content: '- Single source of truth' }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Find action items from it',
-    messages: [{ role: 'user' as const, content: TestAssets.TODO }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Explain this code', 'Check code error'],
-    messages: [{ role: 'user' as const, content: TestAssets.Code }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('distance'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Translate to',
-    messages: [
-      {
-        role: 'user' as const,
-        content: TestAssets.SSOT,
-        params: { language: 'Simplified Chinese' },
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('单一事实来源'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Generate a caption', 'Explain this image'],
-    messages: [
-      {
-        role: 'user' as const,
-        content: '',
-        attachments: [
-          'https://cdn.affine.pro/copilot-test/Qgqy9qZT3VGIEuMIotJYoCCH.jpg',
-        ],
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      const content = result.toLowerCase();
-      t.assert(
-        content.includes('classroom') ||
-          content.includes('school') ||
-          content.includes('sky'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: [
-      'debug:action:fal-face-to-sticker',
-      'debug:action:fal-remove-bg',
-      'debug:action:fal-sd15',
-      'debug:action:fal-upscaler',
-    ],
-    messages: [
-      {
-        role: 'user' as const,
-        content: '',
-        attachments: [
-          'https://cdn.affine.pro/copilot-test/Zkas098lkjdf-908231.jpg',
-        ],
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, link: string) => {
-      t.truthy(checkUrl(link), 'should be a valid url');
-    },
-    type: 'image' as const,
-  },
-];
-for (const { promptName, messages, verifier, type } of actions) {
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
   const prompts = Array.isArray(promptName) ? promptName : [promptName];
   for (const promptName of prompts) {
     test(
@@ -464,28 +283,7 @@ for (const { promptName, messages, verifier, type } of actions) {
 
 // ==================== workflow ====================
 
-const workflows = [
-  {
-    name: 'brainstorm',
-    content: 'apple company',
-    verifier: (t: ExecutionContext, result: string) => {
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-  },
-  {
-    name: 'presentation',
-    content: 'apple company',
-    verifier: (t: ExecutionContext, result: string) => {
-      for (const l of result.split('\n')) {
-        t.notThrows(() => {
-          JSON.parse(l.trim());
-        }, 'should be valid json');
-      }
-    },
-  },
-];
-
-for (const { name, content, verifier } of workflows) {
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
   test(
     `should be able to run workflow: ${name}`,
     runIfCopilotConfigured,

tests/affine-cloud/copilot/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "e2e": "yarn playwright test"
+    "test:e2e": "yarn playwright test"
   },
   "devDependencies": {
     "@affine-test/kit": "workspace:*",

tests/affine-cloud/copilot/playwright.config.ts

@@ -6,6 +6,7 @@ import type {
 
 const config: PlaywrightTestConfig = {
   testDir: './e2e',
+  testMatch: '**/*.spec.ts',
   fullyParallel: !process.env.CI,
   timeout: 120_000,
   outputDir: testResultDir,

tests/kit/src/utils/cloud.ts

@@ -155,18 +155,22 @@ export async function createRandomUser(): Promise<{
   } as any;
 }
 
-export async function createRandomAIUser(): Promise<{
+export async function createRandomAIUser(
+  provider?: string,
+  connector: typeof runPrisma = runPrisma
+): Promise<{
   name: string;
   email: string;
   password: string;
   id: string;
+  sessionId: string;
 }> {
   const user = {
     name: faker.internet.username(),
-    email: faker.internet.email().toLowerCase(),
+    email: faker.internet.email({ provider }).toLowerCase(),
     password: '123456',
   };
-  const result = await runPrisma(async client => {
+  const result = await connector(async client => {
     const freeFeatureId = await client.feature
       .findFirst({
         where: { feature: 'free_plan_v1' },
@@ -182,7 +186,7 @@ export async function createRandomAIUser(): Promise<{
       })
       .then(f => f!.id);
 
-    await client.user.create({
+    const { id: userId } = await client.user.create({
       data: {
         ...user,
         emailVerifiedAt: new Date(),
@@ -204,11 +208,23 @@ export async function createRandomAIUser(): Promise<{
       },
     });
 
-    return await client.user.findUnique({
-      where: {
-        email: user.email,
+    const { id: sessionId } = await client.session.create({ data: {} });
+    await client.userSession.create({
+      data: {
+        sessionId,
+        userId,
+        // half an hour
+        expiresAt: new Date(Date.now() + 60 * 30 * 1000),
       },
     });
+
+    return await client.user
+      .findUnique({
+        where: {
+          email: user.email,
+        },
+      })
+      .then(r => ({ ...r, sessionId }));
   });
   cloudUserSchema.parse(result);
   return {

yarn.lock

@@ -19,18 +19,18 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud-copilot":
+"@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud/copilot":
   version: 0.0.0-use.local
-  resolution: "@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud-copilot"
+  resolution: "@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud/copilot"
   dependencies:
     "@affine-test/kit": "workspace:*"
     "@playwright/test": "npm:=1.49.1"
   languageName: unknown
   linkType: soft
 
-"@affine-test/affine-cloud@workspace:tests/affine-cloud":
+"@affine-test/affine-cloud@workspace:tests/affine-cloud/cloud":
   version: 0.0.0-use.local
-  resolution: "@affine-test/affine-cloud@workspace:tests/affine-cloud"
+  resolution: "@affine-test/affine-cloud@workspace:tests/affine-cloud/cloud"
   dependencies:
     "@affine-test/kit": "workspace:*"
     "@playwright/test": "npm:=1.49.1"