fix: ava ts loader

.github/actions/deploy/deploy.mjs

@@ -19,6 +19,8 @@ const {
   COPILOT_FAL_API_KEY,
   COPILOT_PERPLEXITY_API_KEY,
   COPILOT_UNSPLASH_API_KEY,
+  SLACK_BOT_TOKEN,
+  RELEASE_SLACK_CHANNEL_ID,
   MAILER_SENDER,
   MAILER_USER,
   MAILER_PASSWORD,
@@ -150,6 +152,8 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
     `--set-string graphql.app.copilot.perplexity.key="${COPILOT_PERPLEXITY_API_KEY}"`,
     `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
+    `--set-string graphql.app.copilot.slack.botToken="${SLACK_BOT_TOKEN}"`,
+    `--set-string graphql.app.copilot.slack.channelId="${RELEASE_SLACK_CHANNEL_ID}"`,
     `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
     `--set-string graphql.app.mailer.user="${MAILER_USER}"`,
     `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`,

.github/helm/affine/charts/graphql/templates/copilot-secret.yaml

@@ -9,4 +9,6 @@ data:
   falSecret: {{ .Values.app.copilot.fal.key | b64enc }}
   perplexitySecret: {{ .Values.app.copilot.perplexity.key | b64enc }}
   unsplashSecret: {{ .Values.app.copilot.unsplash.key | b64enc }}
+  slackBotToken: {{ .Values.app.copilot.slack.botToken | b64enc }}
+  slackChannelId: {{ .Values.app.copilot.slack.channelId | b64enc }}
 {{- end }}

.github/helm/affine/charts/graphql/templates/copilot-test.yaml

@@ -0,0 +1,66 @@
+{{ if .Values.app.copilot.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "graphql.fullname" . }}-copilot-test
+  labels:
+    {{- include "graphql.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  schedule: "0 8 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ include "graphql.serviceAccountName" . }}
+          containers:
+          - name: {{ .Chart.Name }}
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+            command: ["yarn", "test:copilot:e2e:cron"]
+            env:
+              - name: AFFINE_ENV
+                value: "{{ .Release.Namespace }}"
+              - name: SLACK_BOT_TOKEN
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: slackBotToken
+              - name: CHANNEL_ID
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: slackChannelId
+              - name: COPILOT_E2E_ENDPOINT
+                value: "http://{{ include "graphql.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:3000"
+              - name: DATABASE_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: pg-postgresql
+                    key: postgres-password
+              - name: DATABASE_URL
+                value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
+              - name: COPILOT_OPENAI_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: openaiSecret
+              - name: COPILOT_FAL_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: falSecret
+              - name: COPILOT_UNSPLASH_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: unsplashSecret
+            resources:
+              requests:
+                cpu: '100m'
+                memory: '200Mi'
+          restartPolicy: Never
+      backoffLimit: 1
+{{ end }}

.github/workflows/build-test.yml

@@ -504,7 +504,8 @@ jobs:
           filters: |
             changed:
               - 'packages/backend/server/src/plugins/copilot/**'
-              - 'packages/backend/server/tests/copilot.*'
+              - 'packages/backend/server/tests/copilot*'
+              - 'tests/affine-cloud-copilot/**'
 
       - name: Setup Node.js
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
@@ -526,7 +527,7 @@ jobs:
 
       - name: Run server tests
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
-        run: yarn affine @affine/server test:copilot:coverage --forbid-only
+        run: yarn affine @affine/server test:copilot:spec:coverage --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}

.github/workflows/copilot-test.yml

@@ -38,6 +38,16 @@ jobs:
       DISTRIBUTION: web
       DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
       REDIS_SERVER_HOST: localhost
+    strategy:
+      fail-fast: false
+      matrix:
+        spec:
+          - {
+              name: e2e,
+              package: '@affine-test/affine-cloud-copilot',
+              type: e2e,
+            }
+          - { name: spec, package: '@affine/server', type: copilot:spec }
     services:
       postgres:
         image: postgres
@@ -78,13 +88,14 @@ jobs:
       - name: Prepare Server Test Environment
         uses: ./.github/actions/server-test-env
 
-      - name: Run server tests
-        run: yarn affine @affine/server test:copilot:coverage --forbid-only
+      - name: Run copilot api ${{ matrix.spec.name }} tests
+        run: yarn affine ${{ matrix.spec.package }} test:${{ matrix.spec.type }}:coverage --forbid-only
         env:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
           COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
+          COPILOT_E2E_ENDPOINT: ${{ secrets.COPILOT_E2E_ENDPOINT }}
 
       - name: Upload server test coverage results
         uses: codecov/codecov-action@v5
@@ -170,7 +181,7 @@ jobs:
         if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
         run: node ./tools/copilot-result/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           BRANCH_SHA: ${{ github.sha }}
           BRANCH_NAME: ${{ github.ref }}
@@ -180,7 +191,7 @@ jobs:
         if: ${{ always() && contains(needs.*.result, 'failure') }}
         run: node ./tools/copilot-result/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           BRANCH_SHA: ${{ github.sha }}
           BRANCH_NAME: ${{ github.ref }}
@@ -190,7 +201,7 @@ jobs:
         if: ${{ always() && contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
         run: node ./tools/copilot-result/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           BRANCH_SHA: ${{ github.sha }}
           BRANCH_NAME: ${{ github.ref }}

.github/workflows/deploy.yml

@@ -100,6 +100,9 @@ jobs:
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
           COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
           COPILOT_UNSPLASH_API_KEY: ${{ secrets.COPILOT_UNSPLASH_API_KEY }}
+          # used for slack notifications
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RELEASE_SLACK_CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           METRICS_CUSTOMER_IO_TOKEN: ${{ secrets.METRICS_CUSTOMER_IO_TOKEN }}
           MAILER_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}
           MAILER_USER: ${{ secrets.OAUTH_EMAIL_LOGIN }}
@@ -162,7 +165,7 @@ jobs:
         if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
         run: node ./tools/changelog/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           DEPLOYED_URL: ${{ steps.set_info.outputs.deployed_url }}
           PREV_VERSION: ${{ needs.output-prev-version.outputs.prev }}
@@ -178,7 +181,7 @@ jobs:
           method: chat.postMessage
           token: ${{ secrets.SLACK_BOT_TOKEN }}
           payload: |
-            channel: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+            channel: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
             text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Backend deploy failed `${{ github.event.inputs.flavor }}`>"
             blocks:
               - type: section
@@ -193,7 +196,7 @@ jobs:
           token: ${{ secrets.SLACK_BOT_TOKEN }}
           method: chat.postMessage
           payload: |
-            channel: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+            channel: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
             text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Backend deploy cancelled `${{ github.event.inputs.flavor }}`>"
             blocks:
               - type: section

oxlint.json

@@ -29,6 +29,7 @@
     "**/*.d.ts",
     "tools/cli/src/webpack/error-handler.js",
     "packages/backend/native/index.d.ts",
+    "packages/backend/server/src/__tests__/__snapshots__",
     "packages/frontend/native/index.d.ts",
     "packages/frontend/native/index.js",
     "packages/frontend/graphql/src/graphql/index.ts",

package.json

@@ -12,7 +12,8 @@
     "tools/*",
     "docs/reference",
     "tools/@types/*",
-    "tests/*"
+    "tests/*",
+    "tests/affine-cloud/*"
   ],
   "engines": {
     "node": "<23.0.0"

packages/backend/server/package.json

@@ -12,9 +12,12 @@
     "dev": "nodemon ./src/index.ts",
     "dev:mail": "email dev -d src/mails",
     "test": "ava --concurrency 1 --serial",
-    "test:copilot": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e": "ava \"src/__tests__/**/copilot-*.e2e.ts\"",
+    "test:copilot:spec": "ava \"src/__tests__/**/copilot-*.spec.ts\"",
     "test:coverage": "c8 ava --concurrency 1 --serial",
-    "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.e2e.ts\"",
+    "test:copilot:spec:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:cron": "node ./scripts/copilot-cron-test.js",
     "data-migration": "cross-env NODE_ENV=script r ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && NODE_ENV=script node --import ./scripts/register.js ./dist/data/index.js run",
     "postinstall": "prisma generate"
@@ -59,9 +62,11 @@
     "@opentelemetry/semantic-conventions": "^1.28.0",
     "@prisma/client": "^5.22.0",
     "@prisma/instrumentation": "^5.22.0",
-    "@react-email/components": "0.0.31",
+    "@react-email/components": "0.0.32",
     "@react-email/render": "1.0.3",
+    "@slack/web-api": "^7.3.4",
     "@socket.io/redis-adapter": "^8.3.0",
+    "ava": "^6.1.2",
     "cookie-parser": "^1.4.7",
     "dotenv": "^16.4.7",
     "eventsource-parser": "^3.0.0",
@@ -74,8 +79,10 @@
     "html-validate": "^9.0.0",
     "ioredis": "^5.4.1",
     "is-mobile": "^5.0.0",
+    "jsx-slack": "^6.1.1",
     "keyv": "^5.2.2",
     "lodash-es": "^4.17.21",
+    "marked": "^15.0.0",
     "mixpanel": "^0.18.0",
     "mustache": "^4.2.0",
     "nanoid": "^5.0.9",
@@ -93,6 +100,8 @@
     "ses": "^1.10.0",
     "socket.io": "^4.8.1",
     "stripe": "^17.4.0",
+    "supertest": "^7.0.0",
+    "tap-parser": "^18.0.0",
     "ts-node": "^10.9.2",
     "typescript": "^5.7.2",
     "winston": "^3.17.0",
@@ -118,7 +127,6 @@
     "@types/react-dom": "^19.0.2",
     "@types/sinon": "^17.0.3",
     "@types/supertest": "^6.0.2",
-    "ava": "^6.2.0",
     "c8": "^10.1.3",
     "cross-env": "^7.0.3",
     "nodemon": "^3.1.7",

packages/backend/server/scripts/copilot-cron-test.js

@@ -0,0 +1,95 @@
+// start process
+
+import { spawn } from 'node:child_process';
+
+import { WebClient } from '@slack/web-api';
+import { jsxslack } from 'jsx-slack';
+import { marked, Renderer } from 'marked';
+import { Parser } from 'tap-parser';
+
+async function runTest() {
+  const tester = new Promise(resolve => {
+    const test = spawn(
+      'npx',
+      [
+        'ava',
+        '--config',
+        'tests/ava.docker.config.js',
+        'tests/**/copilot-*.e2e.ts',
+        '--tap',
+      ],
+      { env: { ...process.env, NODE_NO_WARNINGS: 1 } }
+    );
+
+    const parser = new Parser();
+    test.stdout.on('data', data => {
+      console.log(data.toString());
+      parser.write(data);
+    });
+
+    test.on('close', _ => {
+      const failures = parser?.failures.filter(f => !!f.fullname);
+      const timeouts = parser?.failures.filter(f => !f.fullname);
+      const result = [
+        `${parser.results.pass} passed`,
+        `${parser.results.fail - timeouts.length} failed`,
+        `${timeouts.length} timeouts`,
+        `${parser.results.skip} skipped`,
+      ];
+      const report = [
+        `Test finished with ${result.join(', ')}.`,
+        failures?.length > 0
+          ? `Failed tests: \n\n${failures.map(failure => `- ${failure.fullname}`).join('\n')}`
+          : '',
+      ];
+      resolve(report.join('\n\n'));
+    });
+  });
+
+  try {
+    return await tester;
+  } catch (e) {
+    return e.message;
+  }
+}
+
+function render(markdown) {
+  const rendered = marked(markdown, {
+    renderer: new (class CustomRenderer extends Renderer {
+      heading({ tokens }) {
+        return `
+            <Fragment>
+              <Section><b>${tokens[0].text}</b></Section>
+              <Divider />
+            </Fragment>`;
+      }
+
+      paragraph({ tokens }) {
+        return `<Section><p>${tokens[0].text}</p></Section>`;
+      }
+
+      list(token) {
+        return `<Section>${super.list(token)}</Section>`;
+      }
+
+      hr() {
+        return `<Divider />`;
+      }
+    })(),
+  });
+  return jsxslack([`<Blocks>${rendered}</Blocks>`]);
+}
+
+const { CHANNEL_ID, SLACK_BOT_TOKEN, AFFINE_ENV } = process.env;
+
+const report = await runTest();
+const blocks = render(
+  [`# AFFiNE Copilot Test ${AFFINE_ENV} Env Test Result`, report].join('\n\n')
+);
+const { ok } = await new WebClient(SLACK_BOT_TOKEN).chat.postMessage({
+  channel: CHANNEL_ID,
+  text: `AFFiNE Copilot Test ${AFFINE_ENV} Env Test Result`,
+  blocks,
+});
+
+console.assert(ok, 'Failed to send a message to Slack');

packages/backend/server/src/__tests__/ava.docker.config.js

@@ -0,0 +1,15 @@
+import packageJson from '../package.json' with { type: 'json' };
+
+export default {
+  ...packageJson.ava,
+  nodeArguments: [
+    '--trace-sigint',
+    '--loader',
+    'ts-node/esm/transpile-only.mjs',
+    '--es-module-specifier-resolution=node',
+  ],
+  environmentVariables: {
+    ...packageJson.ava.environmentVariables,
+    TS_NODE_PROJECT: './tests/tsconfig.docker.json',
+  },
+};

packages/backend/server/src/__tests__/copilot-provider.e2e.ts

@@ -0,0 +1,294 @@
+import { randomInt, randomUUID } from 'node:crypto';
+
+import { hash } from '@node-rs/argon2';
+import type { ExecutionContext, TestFn } from 'ava';
+import ava from 'ava';
+import { z } from 'zod';
+
+import {
+  chatWithImages,
+  chatWithText,
+  chatWithWorkflow,
+  createCopilotMessage,
+  createCopilotSession,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+  sse2array,
+} from './utils/copilot';
+import { createWorkspace } from './utils/workspace';
+
+type Tester = {
+  app: any;
+  userEmail: string;
+  userToken: string;
+  workspaceId: string;
+};
+const test = ava as TestFn<Tester>;
+
+const e2eConfig = {
+  endpoint: process.env.COPILOT_E2E_ENDPOINT || 'http://localhost:3010',
+};
+
+const isCopilotConfigured =
+  !!process.env.COPILOT_OPENAI_API_KEY &&
+  !!process.env.COPILOT_FAL_API_KEY &&
+  process.env.COPILOT_OPENAI_API_KEY !== '1' &&
+  process.env.COPILOT_FAL_API_KEY !== '1';
+const runIfCopilotConfigured = test.macro(
+  async (
+    t,
+    callback: (t: ExecutionContext<Tester>) => Promise<void> | void
+  ) => {
+    if (isCopilotConfigured) {
+      await callback(t);
+    } else {
+      t.log('Skip test because copilot is not configured');
+      t.pass();
+    }
+  }
+);
+
+const runPrisma = async <T>(
+  cb: (
+    prisma: InstanceType<
+      typeof import('../../node_modules/@prisma/client').PrismaClient
+    >
+  ) => Promise<T>
+): Promise<T> => {
+  const { PrismaClient } = await import('../../node_modules/@prisma/client');
+  const client = new PrismaClient();
+  await client.$connect();
+  try {
+    return await cb(client);
+  } finally {
+    await client.$disconnect();
+  }
+};
+
+const cloudUserSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  email: z.string().email(),
+  password: z.string(),
+});
+
+function randomName() {
+  return Array.from({ length: 10 }, () =>
+    String.fromCharCode(randomInt(65, 90))
+  )
+    .join('')
+    .toLowerCase();
+}
+
+async function createRandomAIUser(): Promise<{
+  name: string;
+  email: string;
+  password: string;
+  id: string;
+  sessionId: string;
+}> {
+  const name = randomName();
+  const email = `${name}@affine.fail`;
+  const user = { name, email, password: '123456' };
+  const result = await runPrisma(async client => {
+    const freeFeatureId = await client.feature
+      .findFirst({
+        where: { feature: 'free_plan_v1' },
+        select: { id: true },
+        orderBy: { version: 'desc' },
+      })
+      .then(f => f!.id);
+    const aiFeatureId = await client.feature
+      .findFirst({
+        where: { feature: 'unlimited_copilot' },
+        select: { id: true },
+        orderBy: { version: 'desc' },
+      })
+      .then(f => f!.id);
+
+    const { id: userId } = await client.user.create({
+      data: {
+        ...user,
+        emailVerifiedAt: new Date(),
+        password: await hash(user.password),
+        features: {
+          create: [
+            {
+              reason: 'created by test case',
+              activated: true,
+              featureId: freeFeatureId,
+            },
+            {
+              reason: 'created by test case',
+              activated: true,
+              featureId: aiFeatureId,
+            },
+          ],
+        },
+      },
+    });
+
+    const { id: sessionId } = await client.session.create({ data: {} });
+    await client.userSession.create({
+      data: {
+        sessionId,
+        userId,
+        // half an hour
+        expiresAt: new Date(Date.now() + 60 * 30 * 1000),
+      },
+    });
+
+    return await client.user
+      .findUnique({
+        where: {
+          email: user.email,
+        },
+      })
+      .then(r => ({ ...r, sessionId }));
+  });
+  cloudUserSchema.parse(result);
+  return {
+    ...result,
+    password: user.password,
+  } as any;
+}
+
+test.before(async t => {
+  if (!isCopilotConfigured) return;
+  const { endpoint } = e2eConfig;
+
+  const { email, sessionId: token } = await createRandomAIUser();
+  const app = { getHttpServer: () => endpoint } as any;
+  const { id } = await createWorkspace(app, token);
+
+  t.context.app = app;
+  t.context.userEmail = email;
+  t.context.userToken = token;
+  t.context.workspaceId = id;
+});
+
+test.after(async t => {
+  if (!isCopilotConfigured) return;
+  await runPrisma(async client => {
+    await client.user.delete({
+      where: {
+        email: t.context.userEmail,
+      },
+    });
+  });
+});
+
+const retry = async (
+  action: string,
+  t: ExecutionContext<Tester>,
+  callback: (t: ExecutionContext<Tester>) => void
+) => {
+  let i = 3;
+  while (i--) {
+    const ret = await t.try(callback);
+    if (ret.passed) {
+      return ret.commit();
+    } else {
+      ret.discard();
+      t.log(ret.errors.map(e => e.message).join('\n'));
+      t.log(`retrying ${action} ${3 - i}/3 ...`);
+    }
+  }
+  t.fail(`failed to run ${action}`);
+};
+
+const makeCopilotChat = async (
+  t: ExecutionContext<Tester>,
+  promptName: string,
+  { content, attachments, params }: any
+) => {
+  const { app, userToken, workspaceId } = t.context;
+  const sessionId = await createCopilotSession(
+    app,
+    userToken,
+    workspaceId,
+    randomUUID(),
+    promptName
+  );
+  const messageId = await createCopilotMessage(
+    app,
+    userToken,
+    sessionId,
+    content,
+    attachments,
+    undefined,
+    params
+  );
+  return { sessionId, messageId };
+};
+
+// ==================== action ====================
+
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
+  const prompts = Array.isArray(promptName) ? promptName : [promptName];
+  for (const promptName of prompts) {
+    test(
+      `should be able to run action: ${promptName}`,
+      runIfCopilotConfigured,
+      async t => {
+        await retry(`action: ${promptName}`, t, async t => {
+          const { app, userToken } = t.context;
+          const { sessionId, messageId } = await makeCopilotChat(
+            t,
+            promptName,
+            messages[0]
+          );
+
+          if (type === 'text') {
+            const result = await chatWithText(
+              app,
+              userToken,
+              sessionId,
+              messageId
+            );
+            t.truthy(result, 'should return result');
+            verifier?.(t, result);
+          } else if (type === 'image') {
+            const result = sse2array(
+              await chatWithImages(app, userToken, sessionId, messageId)
+            )
+              .filter(e => e.event !== 'event')
+              .map(e => e.data)
+              .filter(Boolean);
+            t.truthy(result.length, 'should return result');
+            for (const r of result) {
+              verifier?.(t, r);
+            }
+          } else {
+            t.fail('unsupported provider type');
+          }
+        });
+      }
+    );
+  }
+}
+
+// ==================== workflow ====================
+
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
+  test(
+    `should be able to run workflow: ${name}`,
+    runIfCopilotConfigured,
+    async t => {
+      await retry(`workflow: ${name}`, t, async t => {
+        const { app, userToken } = t.context;
+        const { sessionId, messageId } = await makeCopilotChat(
+          t,
+          `workflow:${name}`,
+          { content }
+        );
+        const r = await chatWithWorkflow(app, userToken, sessionId, messageId);
+        const result = sse2array(r)
+          .filter(e => e.event !== 'event' && e.data)
+          .reduce((p, c) => p + c.data, '');
+        t.truthy(result, 'should return result');
+        verifier?.(t, result);
+      });
+    }
+  );
+}

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -28,7 +28,11 @@ import {
   CopilotCheckJsonExecutor,
 } from '../plugins/copilot/workflow/executor';
 import { createTestingModule } from './utils';
-import { TestAssets } from './utils/copilot';
+import {
+  checkMDList,
+  ProviderActionTestCase,
+  ProviderWorkflowTestCase,
+} from './utils/copilot';
 
 type Tester = {
   auth: AuthService;
@@ -138,58 +142,6 @@ test.after(async t => {
   await t.context.module.close();
 });
 
-const assertNotWrappedInCodeBlock = (
-  t: ExecutionContext<Tester>,
-  result: string
-) => {
-  t.assert(
-    !result.replaceAll('\n', '').trim().startsWith('```') &&
-      !result.replaceAll('\n', '').trim().endsWith('```'),
-    'should not wrap in code block'
-  );
-};
-
-const checkMDList = (text: string) => {
-  const lines = text.split('\n');
-  const listItemRegex = /^( {2})*(-|\u2010-\u2015|\*|\+)? .+$/;
-  let prevIndent = null;
-
-  for (const line of lines) {
-    if (line.trim() === '') continue;
-    if (!listItemRegex.test(line)) {
-      return false;
-    }
-
-    const currentIndent = line.match(/^( *)/)?.[0].length!;
-    if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) {
-      return false;
-    }
-
-    if (prevIndent !== null && currentIndent > 0) {
-      const indentDiff = currentIndent - prevIndent;
-      // allow 1 level of indentation difference
-      if (indentDiff > 2) {
-        return false;
-      }
-    }
-
-    if (line.trim().startsWith('-')) {
-      prevIndent = currentIndent;
-    }
-  }
-
-  return true;
-};
-
-const checkUrl = (url: string) => {
-  try {
-    new URL(url);
-    return true;
-  } catch {
-    return false;
-  }
-};
-
 const retry = async (
   action: string,
   t: ExecutionContext<Tester>,
@@ -271,140 +223,7 @@ test('should validate markdown list', t => {
 
 // ==================== action ====================
 
-const actions = [
-  {
-    promptName: [
-      'Summary',
-      'Explain this',
-      'Write an article about this',
-      'Write a twitter about this',
-      'Write a poem about this',
-      'Write a blog post about this',
-      'Write outline',
-      'Change tone to',
-      'Improve writing for it',
-      'Improve grammar for it',
-      'Fix spelling for it',
-      'Create headings',
-      'Make it longer',
-      'Make it shorter',
-      'Continue writing',
-      'Chat With AFFiNE AI',
-      'Search With AFFiNE AI',
-    ],
-    messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('single source of truth'),
-        'should include original keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Brainstorm ideas about this', 'Brainstorm mindmap'],
-    messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Expand mind map',
-    messages: [{ role: 'user' as const, content: '- Single source of truth' }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Find action items from it',
-    messages: [{ role: 'user' as const, content: TestAssets.TODO }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Explain this code', 'Check code error'],
-    messages: [{ role: 'user' as const, content: TestAssets.Code }],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('distance'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: 'Translate to',
-    messages: [
-      {
-        role: 'user' as const,
-        content: TestAssets.SSOT,
-        params: { language: 'Simplified Chinese' },
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      t.assert(
-        result.toLowerCase().includes('单一事实来源'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: ['Generate a caption', 'Explain this image'],
-    messages: [
-      {
-        role: 'user' as const,
-        content: '',
-        attachments: [
-          'https://cdn.affine.pro/copilot-test/Qgqy9qZT3VGIEuMIotJYoCCH.jpg',
-        ],
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, result: string) => {
-      assertNotWrappedInCodeBlock(t, result);
-      const content = result.toLowerCase();
-      t.assert(
-        content.includes('classroom') ||
-          content.includes('school') ||
-          content.includes('sky'),
-        'explain code result should include keyword'
-      );
-    },
-    type: 'text' as const,
-  },
-  {
-    promptName: [
-      'debug:action:fal-face-to-sticker',
-      'debug:action:fal-remove-bg',
-      'debug:action:fal-sd15',
-      'debug:action:fal-upscaler',
-    ],
-    messages: [
-      {
-        role: 'user' as const,
-        content: '',
-        attachments: [
-          'https://cdn.affine.pro/copilot-test/Zkas098lkjdf-908231.jpg',
-        ],
-      },
-    ],
-    verifier: (t: ExecutionContext<Tester>, link: string) => {
-      t.truthy(checkUrl(link), 'should be a valid url');
-    },
-    type: 'image' as const,
-  },
-];
-for (const { promptName, messages, verifier, type } of actions) {
+for (const { promptName, messages, verifier, type } of ProviderActionTestCase) {
   const prompts = Array.isArray(promptName) ? promptName : [promptName];
   for (const promptName of prompts) {
     test(
@@ -464,28 +283,7 @@ for (const { promptName, messages, verifier, type } of actions) {
 
 // ==================== workflow ====================
 
-const workflows = [
-  {
-    name: 'brainstorm',
-    content: 'apple company',
-    verifier: (t: ExecutionContext, result: string) => {
-      t.assert(checkMDList(result), 'should be a markdown list');
-    },
-  },
-  {
-    name: 'presentation',
-    content: 'apple company',
-    verifier: (t: ExecutionContext, result: string) => {
-      for (const l of result.split('\n')) {
-        t.notThrows(() => {
-          JSON.parse(l.trim());
-        }, 'should be valid json');
-      }
-    },
-  },
-];
-
-for (const { name, content, verifier } of workflows) {
+for (const { name, content, verifier } of ProviderWorkflowTestCase) {
   test(
     `should be able to run workflow: ${name}`,
     runIfCopilotConfigured,

packages/backend/server/src/__tests__/tsconfig.docker.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2024",
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "rootDir": ".",
+    "outDir": "../../dist/tests",
+    "verbatimModuleSyntax": false,
+    "tsBuildInfoFile": "../../dist/tests/.tsbuildinfo"
+  },
+  "include": [".", "utils"],
+  "exclude": [],
+  "ts-node": {
+    "esm": true,
+    "compilerOptions": {
+      "module": "ESNext",
+      "moduleResolution": "Node"
+    }
+  }
+}

packages/backend/server/src/__tests__/utils/blobs.ts

@@ -1,7 +1,7 @@
 import type { INestApplication } from '@nestjs/common';
 import request from 'supertest';
 
-import { gql } from './common';
+import { gqlEndpoint } from './common';
 
 export async function listBlobs(
   app: INestApplication,
@@ -9,7 +9,7 @@ export async function listBlobs(
   workspaceId: string
 ): Promise<string[]> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -29,7 +29,7 @@ export async function getWorkspaceBlobsSize(
   workspaceId: string
 ): Promise<number> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .send({
       query: `
@@ -49,7 +49,7 @@ export async function collectAllBlobSizes(
   token: string
 ): Promise<number> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .send({
       query: `
@@ -73,7 +73,7 @@ export async function setBlob(
   buffer: Buffer
 ): Promise<string> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .field(

packages/backend/server/src/__tests__/utils/common.ts

@@ -1 +1,37 @@
-export const gql = '/graphql';
+import { INestApplication } from '@nestjs/common';
+import type { Response } from 'supertest';
+import supertest from 'supertest';
+
+export function handleGraphQLError(resp: Response) {
+  const { errors } = resp.body;
+  if (errors) {
+    const cause = errors[0];
+    const stacktrace = cause.extensions?.stacktrace;
+    throw new Error(
+      stacktrace
+        ? Array.isArray(stacktrace)
+          ? stacktrace.join('\n')
+          : String(stacktrace)
+        : cause.message,
+      cause
+    );
+  }
+}
+
+export const gqlEndpoint = '/graphql';
+
+export function gql(app: INestApplication, query?: string) {
+  const req = supertest(app.getHttpServer())
+    .post(gqlEndpoint)
+    .set({ 'x-request-id': 'test', 'x-operation-name': 'test' });
+
+  if (query) {
+    return req.send({ query });
+  }
+
+  return req;
+}
+
+export async function sleep(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

packages/backend/server/src/__tests__/utils/index.ts

@@ -1,5 +1,7 @@
 export * from './blobs';
+export * from './common';
 export * from './invite';
 export * from './user';
 export * from './utils';
 export * from './workspace';
+export * from './common';

packages/backend/server/src/__tests__/utils/invite.ts

@@ -2,7 +2,7 @@ import type { INestApplication } from '@nestjs/common';
 import request from 'supertest';
 
 import type { InvitationType } from '../../core/workspaces';
-import { gql } from './common';
+import { gqlEndpoint } from './common';
 
 export async function inviteUser(
   app: INestApplication,
@@ -12,7 +12,7 @@ export async function inviteUser(
   sendInviteMail = false
 ): Promise<string> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -37,7 +37,7 @@ export async function inviteUsers(
   sendInviteMail = false
 ): Promise<Array<{ email: string; inviteId?: string; sentSuccess?: boolean }>> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -69,7 +69,7 @@ export async function getInviteLink(
   workspaceId: string
 ): Promise<{ link: string; expireTime: string }> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -98,7 +98,7 @@ export async function createInviteLink(
   expireTime: 'OneDay' | 'ThreeDays' | 'OneWeek' | 'OneMonth'
 ): Promise<{ link: string; expireTime: string }> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -124,7 +124,7 @@ export async function revokeInviteLink(
   workspaceId: string
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -149,7 +149,7 @@ export async function acceptInviteById(
   token: string = ''
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .auth(token, { type: 'bearer' })
     .send({
@@ -175,7 +175,7 @@ export async function approveMember(
   userId: string
 ): Promise<string> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .auth(token, { type: 'bearer' })
     .send({
@@ -201,7 +201,7 @@ export async function leaveWorkspace(
   sendLeaveMail = false
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -225,7 +225,7 @@ export async function revokeUser(
   userId: string
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -250,7 +250,7 @@ export async function getInviteInfo(
   inviteId: string
 ): Promise<InvitationType> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({

packages/backend/server/src/__tests__/utils/user.ts

@@ -9,7 +9,7 @@ import {
 import { sessionUser } from '../../core/auth/service';
 import { UserType } from '../../core/user';
 import { Models } from '../../models';
-import { gql } from './common';
+import { gqlEndpoint } from './common';
 
 export type UserAuthedType = UserType & { token: ClientTokenType };
 
@@ -69,7 +69,7 @@ export async function signUp(
 
 export async function currentUser(app: INestApplication, token: string) {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -93,7 +93,7 @@ export async function sendChangeEmail(
   callbackUrl: string
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(userToken, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -115,7 +115,7 @@ export async function sendSetPasswordEmail(
   callbackUrl: string
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(userToken, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -137,7 +137,7 @@ export async function changePassword(
   password: string
 ): Promise<string> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
       query: `
@@ -159,7 +159,7 @@ export async function sendVerifyChangeEmail(
   callbackUrl: string
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(userToken, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -181,7 +181,7 @@ export async function changeEmail(
   email: string
 ): Promise<UserAuthedType> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(userToken, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({

packages/backend/server/src/__tests__/utils/utils.ts

@@ -5,8 +5,6 @@ import { Test, TestingModuleBuilder } from '@nestjs/testing';
 import { PrismaClient } from '@prisma/client';
 import cookieParser from 'cookie-parser';
 import graphqlUploadExpress from 'graphql-upload/graphqlUploadExpress.mjs';
-import type { Response } from 'supertest';
-import supertest from 'supertest';
 
 import { AppModule, FunctionalityModules } from '../../app.module';
 import { GlobalExceptionFilter, Runtime } from '../../base';
@@ -156,35 +154,3 @@ export async function createTestingApp(moduleDef: TestingModuleMeatdata = {}) {
     app,
   };
 }
-
-export function handleGraphQLError(resp: Response) {
-  const { errors } = resp.body;
-  if (errors) {
-    const cause = errors[0];
-    const stacktrace = cause.extensions?.stacktrace;
-    throw new Error(
-      stacktrace
-        ? Array.isArray(stacktrace)
-          ? stacktrace.join('\n')
-          : String(stacktrace)
-        : cause.message,
-      cause
-    );
-  }
-}
-
-export function gql(app: INestApplication, query?: string) {
-  const req = supertest(app.getHttpServer())
-    .post('/graphql')
-    .set({ 'x-request-id': 'test', 'x-operation-name': 'test' });
-
-  if (query) {
-    return req.send({ query });
-  }
-
-  return req;
-}
-
-export async function sleep(ms: number) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}

packages/backend/server/src/__tests__/utils/workspace.ts

@@ -2,7 +2,7 @@ import type { INestApplication } from '@nestjs/common';
 import request from 'supertest';
 
 import type { WorkspaceType } from '../../core/workspaces';
-import { gql } from './common';
+import { gqlEndpoint } from './common';
 import { PermissionEnum } from './utils';
 
 export async function createWorkspace(
@@ -10,7 +10,7 @@ export async function createWorkspace(
   token: string
 ): Promise<WorkspaceType> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .field(
@@ -37,7 +37,7 @@ export async function getWorkspacePublicPages(
   workspaceId: string
 ) {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -64,7 +64,7 @@ export async function getWorkspace(
   take = 8
 ): Promise<WorkspaceType> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -87,7 +87,7 @@ export async function updateWorkspace(
   isPublic: boolean
 ): Promise<boolean> {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -110,7 +110,7 @@ export async function publishPage(
   pageId: string
 ) {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -134,7 +134,7 @@ export async function revokePublicPage(
   pageId: string
 ) {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({
@@ -160,7 +160,7 @@ export async function grantMember(
   permission: PermissionEnum
 ) {
   const res = await request(app.getHttpServer())
-    .post(gql)
+    .post(gqlEndpoint)
     .auth(token, { type: 'bearer' })
     .set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
     .send({

tests/affine-cloud-copilot/playwright.config.ts

@@ -6,6 +6,7 @@ import type {
 
 const config: PlaywrightTestConfig = {
   testDir: './e2e',
+  testMatch: '**/*.spec.ts',
   fullyParallel: !process.env.CI,
   timeout: 120_000,
   outputDir: testResultDir,

tests/kit/src/utils/cloud.ts

@@ -155,18 +155,22 @@ export async function createRandomUser(): Promise<{
   } as any;
 }
 
-export async function createRandomAIUser(): Promise<{
+export async function createRandomAIUser(
+  provider?: string,
+  connector: typeof runPrisma = runPrisma
+): Promise<{
   name: string;
   email: string;
   password: string;
   id: string;
+  sessionId: string;
 }> {
   const user = {
     name: faker.internet.username(),
-    email: faker.internet.email().toLowerCase(),
+    email: faker.internet.email({ provider }).toLowerCase(),
     password: '123456',
   };
-  const result = await runPrisma(async client => {
+  const result = await connector(async client => {
     const freeFeatureId = await client.feature
       .findFirst({
         where: { feature: 'free_plan_v1' },
@@ -182,7 +186,7 @@ export async function createRandomAIUser(): Promise<{
       })
       .then(f => f!.id);
 
-    await client.user.create({
+    const { id: userId } = await client.user.create({
       data: {
         ...user,
         emailVerifiedAt: new Date(),
@@ -204,11 +208,23 @@ export async function createRandomAIUser(): Promise<{
       },
     });
 
-    return await client.user.findUnique({
-      where: {
-        email: user.email,
+    const { id: sessionId } = await client.session.create({ data: {} });
+    await client.userSession.create({
+      data: {
+        sessionId,
+        userId,
+        // half an hour
+        expiresAt: new Date(Date.now() + 60 * 30 * 1000),
       },
     });
+
+    return await client.user
+      .findUnique({
+        where: {
+          email: user.email,
+        },
+      })
+      .then(r => ({ ...r, sessionId }));
   });
   cloudUserSchema.parse(result);
   return {

tools/copilot-result/index.js

@@ -2,8 +2,16 @@ import { WebClient } from '@slack/web-api';
 
 import { render } from './markdown.js';
 
-const { CHANNEL_ID, SLACK_BOT_TOKEN, COPILOT_RESULT, BRANCH_SHA, BRANCH_NAME } =
-  process.env;
+const {
+  CHANNEL_ID,
+  SLACK_BOT_TOKEN,
+  COPILOT_RESULT,
+  BRANCH_SHA,
+  BRANCH_NAME,
+  GITHUB_SERVER_URL,
+  GITHUB_REPOSITORY,
+  GITHUB_RUN_ID,
+} = process.env;
 
 const { ok } = await new WebClient(SLACK_BOT_TOKEN).chat.postMessage({
   channel: CHANNEL_ID,
@@ -11,7 +19,8 @@ const { ok } = await new WebClient(SLACK_BOT_TOKEN).chat.postMessage({
   blocks: render(
     `# AFFiNE Copilot Test ${COPILOT_RESULT}
 
-- [${BRANCH_NAME?.replace('refs/heads/', '') || BRANCH_SHA}](https://github.com/toeverything/AFFiNE/commit/${BRANCH_SHA})
+- Branch: [${BRANCH_NAME?.replace('refs/heads/', '') || BRANCH_SHA}](https://github.com/toeverything/AFFiNE/commit/${BRANCH_SHA})
+- Job: [${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})
 `
   ),
 });

yarn.lock

@@ -805,8 +805,9 @@ __metadata:
     "@opentelemetry/semantic-conventions": "npm:^1.28.0"
     "@prisma/client": "npm:^5.22.0"
     "@prisma/instrumentation": "npm:^5.22.0"
-    "@react-email/components": "npm:0.0.31"
+    "@react-email/components": "npm:0.0.32"
     "@react-email/render": "npm:1.0.3"
+    "@slack/web-api": "npm:^7.3.4"
     "@socket.io/redis-adapter": "npm:^8.3.0"
     "@types/cookie-parser": "npm:^1.4.8"
     "@types/express": "npm:^4.17.21"
@@ -821,7 +822,7 @@ __metadata:
     "@types/react-dom": "npm:^19.0.2"
     "@types/sinon": "npm:^17.0.3"
     "@types/supertest": "npm:^6.0.2"
-    ava: "npm:^6.2.0"
+    ava: "npm:^6.1.2"
     c8: "npm:^10.1.3"
     cookie-parser: "npm:^1.4.7"
     cross-env: "npm:^7.0.3"
@@ -836,8 +837,10 @@ __metadata:
     html-validate: "npm:^9.0.0"
     ioredis: "npm:^5.4.1"
     is-mobile: "npm:^5.0.0"
+    jsx-slack: "npm:^6.1.1"
     keyv: "npm:^5.2.2"
     lodash-es: "npm:^4.17.21"
+    marked: "npm:^15.0.0"
     mixpanel: "npm:^0.18.0"
     mustache: "npm:^4.2.0"
     nanoid: "npm:^5.0.9"
@@ -860,6 +863,7 @@ __metadata:
     socket.io: "npm:^4.8.1"
     stripe: "npm:^17.4.0"
     supertest: "npm:^7.0.0"
+    tap-parser: "npm:^18.0.0"
     ts-node: "npm:^10.9.2"
     typescript: "npm:^5.7.2"
     winston: "npm:^3.17.0"
@@ -12245,9 +12249,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@react-email/components@npm:0.0.31":
-  version: 0.0.31
-  resolution: "@react-email/components@npm:0.0.31"
+"@react-email/components@npm:0.0.32":
+  version: 0.0.32
+  resolution: "@react-email/components@npm:0.0.32"
   dependencies:
     "@react-email/body": "npm:0.0.11"
     "@react-email/button": "npm:0.0.19"
@@ -12264,14 +12268,14 @@ __metadata:
     "@react-email/link": "npm:0.0.12"
     "@react-email/markdown": "npm:0.0.14"
     "@react-email/preview": "npm:0.0.12"
-    "@react-email/render": "npm:1.0.3"
+    "@react-email/render": "npm:1.0.4"
     "@react-email/row": "npm:0.0.12"
     "@react-email/section": "npm:0.0.16"
     "@react-email/tailwind": "npm:1.0.4"
     "@react-email/text": "npm:0.0.11"
   peerDependencies:
     react: ^18.0 || ^19.0 || ^19.0.0-rc
-  checksum: 10/2850943e9e326ea3e9d6a2e55665c5ab6a00f962ff5f01541cca61abcf45770a4cc6a59de503a60eb4cff257805406470ad8a3969b2b5994754bfe1872830d14
+  checksum: 10/0a57d122a71581e36f352479165487299d45585656faea5bc66db27637a152ff0bc1edd97da3d78a131d799a3cba188ad7ec85b54987c53c7b24747fdca48bfe
   languageName: node
   linkType: hard
 
@@ -12381,6 +12385,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@react-email/render@npm:1.0.4":
+  version: 1.0.4
+  resolution: "@react-email/render@npm:1.0.4"
+  dependencies:
+    html-to-text: "npm:9.0.5"
+    prettier: "npm:3.4.2"
+    react-promise-suspense: "npm:0.3.4"
+  peerDependencies:
+    react: ^18.0 || ^19.0 || ^19.0.0-rc
+    react-dom: ^18.0 || ^19.0 || ^19.0.0-rc
+  checksum: 10/468da62f5fefb75eef813406be1b7f5c73bb60c135c8afb1d0adf511d5b31f807d19821396c2df75a1884998a7ea71b10d99439a2c9b4872adfa273ae70b3af7
+  languageName: node
+  linkType: hard
+
 "@react-email/row@npm:0.0.12":
   version: 0.0.12
   resolution: "@react-email/row@npm:0.0.12"
@@ -13176,7 +13194,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@slack/web-api@npm:^7.8.0":
+"@slack/web-api@npm:^7.3.4, @slack/web-api@npm:^7.8.0":
   version: 7.8.0
   resolution: "@slack/web-api@npm:7.8.0"
   dependencies:
@@ -17226,7 +17244,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ava@npm:^6.2.0":
+"ava@npm:^6.1.2, ava@npm:^6.2.0":
   version: 6.2.0
   resolution: "ava@npm:6.2.0"
   dependencies:
@@ -21794,6 +21812,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"events-to-array@npm:^2.0.3":
+  version: 2.0.3
+  resolution: "events-to-array@npm:2.0.3"
+  checksum: 10/d392eb0013013c3dfa66710a017902760edb2a588f6b1a3f1c92219563ba1c24bcb99c48e3754423a3538ebfd70318c3536d30bfd80c00e7fec77fdd088540d0
+  languageName: node
+  linkType: hard
+
 "events@npm:^3.2.0":
   version: 3.3.0
   resolution: "events@npm:3.3.0"
@@ -25498,7 +25523,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"jsx-slack@npm:^6.1.2":
+"jsx-slack@npm:^6.1.1, jsx-slack@npm:^6.1.2":
   version: 6.1.2
   resolution: "jsx-slack@npm:6.1.2"
   dependencies:
@@ -26511,7 +26536,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"marked@npm:^15.0.3":
+"marked@npm:^15.0.0, marked@npm:^15.0.3":
   version: 15.0.6
   resolution: "marked@npm:15.0.6"
   bin:
@@ -30047,7 +30072,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prettier@npm:^3.2.5, prettier@npm:^3.4.2":
+"prettier@npm:3.4.2, prettier@npm:^3.2.5, prettier@npm:^3.4.2":
   version: 3.4.2
   resolution: "prettier@npm:3.4.2"
   bin:
@@ -33537,6 +33562,28 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tap-parser@npm:^18.0.0":
+  version: 18.0.0
+  resolution: "tap-parser@npm:18.0.0"
+  dependencies:
+    events-to-array: "npm:^2.0.3"
+    tap-yaml: "npm:4.0.0"
+  bin:
+    tap-parser: bin/cmd.cjs
+  checksum: 10/6e13dc475bfdc880307cc935b9917d43255f7e2b0902d171b7f51b41b029aae9dce5b8683aa48170bb720819b3551d793663c0a11f0660ef78d4a0fe87228ad0
+  languageName: node
+  linkType: hard
+
+"tap-yaml@npm:4.0.0":
+  version: 4.0.0
+  resolution: "tap-yaml@npm:4.0.0"
+  dependencies:
+    yaml: "npm:^2.4.1"
+    yaml-types: "npm:^0.4.0"
+  checksum: 10/21d3a27328aa419bb90249357689446488351d350a4ba56c5b1afce9dfe33f60db49014170ea7d16eb90381d3f9f31b59d9637bda632e55167f33c834eb02171
+  languageName: node
+  linkType: hard
+
 "tapable@npm:^2.0.0, tapable@npm:^2.1.1, tapable@npm:^2.2.0, tapable@npm:^2.2.1":
   version: 2.2.1
   resolution: "tapable@npm:2.2.1"
@@ -35971,6 +36018,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"yaml-types@npm:^0.4.0":
+  version: 0.4.0
+  resolution: "yaml-types@npm:0.4.0"
+  peerDependencies:
+    yaml: ^2.3.0
+  checksum: 10/8a3cd3a0420d5d09981e3e1add46d7482336531e3bdc02192d26caa915c7d0795ad28dd8766e357234d6bfa3a2bd986687f967079e47aecfd4b191250f041cec
+  languageName: node
+  linkType: hard
+
 "yaml@npm:^1.10.0":
   version: 1.10.2
   resolution: "yaml@npm:1.10.2"
@@ -35978,7 +36034,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"yaml@npm:^2.3.1, yaml@npm:^2.3.4, yaml@npm:^2.5.1, yaml@npm:^2.6.1":
+"yaml@npm:^2.3.1, yaml@npm:^2.3.4, yaml@npm:^2.4.1, yaml@npm:^2.5.1, yaml@npm:^2.6.1":
   version: 2.7.0
   resolution: "yaml@npm:2.7.0"
   bin: