feat(server): add real-world prompt test for copilot apis (#8629)

fix AF-1432, PD-1176

tests/affine-cloud-copilot/e2e/copilot.spec.ts

@@ -0,0 +1,591 @@
+import { test } from '@affine-test/kit/playwright';
+import {
+  createRandomAIUser,
+  loginUser,
+  loginUserDirectly,
+} from '@affine-test/kit/utils/cloud';
+import { openHomePage, setCoreUrl } from '@affine-test/kit/utils/load-page';
+import {
+  clickNewPageButton,
+  getBlockSuiteEditorTitle,
+  waitForEditorLoad,
+} from '@affine-test/kit/utils/page-logic';
+import { clickSideBarAllPageButton } from '@affine-test/kit/utils/sidebar';
+import { createLocalWorkspace } from '@affine-test/kit/utils/workspace';
+import { expect, type Page } from '@playwright/test';
+
+test.describe.configure({ mode: 'parallel' });
+
+const ONE_SECOND = 1000;
+const TEN_SECONDS = 10 * ONE_SECOND;
+const ONE_MINUTE = 60 * ONE_SECOND;
+
+let isProduction = process.env.NODE_ENV === 'production';
+if (
+  process.env.PLAYWRIGHT_USER_AGENT &&
+  process.env.PLAYWRIGHT_EMAIL &&
+  !process.env.PLAYWRIGHT_PASSWORD
+) {
+  test.use({
+    userAgent: process.env.PLAYWRIGHT_USER_AGENT || 'affine-tester',
+  });
+  setCoreUrl(process.env.PLAYWRIGHT_CORE_URL || 'http://localhost:8080');
+  isProduction = true;
+}
+
+function getUser() {
+  if (
+    !isProduction ||
+    !process.env.PLAYWRIGHT_EMAIL ||
+    !process.env.PLAYWRIGHT_PASSWORD
+  ) {
+    return createRandomAIUser();
+  }
+
+  return {
+    email: process.env.PLAYWRIGHT_EMAIL,
+    password: process.env.PLAYWRIGHT_PASSWORD,
+  };
+}
+
+test.skip(
+  () =>
+    !process.env.COPILOT_OPENAI_API_KEY ||
+    !process.env.COPILOT_FAL_API_KEY ||
+    process.env.COPILOT_OPENAI_API_KEY === '1' ||
+    process.env.COPILOT_FAL_API_KEY === '1',
+  'skip test if no copilot api key'
+);
+
+test('can open chat side panel', async ({ page }) => {
+  await openHomePage(page);
+  await waitForEditorLoad(page);
+  await clickNewPageButton(page);
+  await page.getByTestId('right-sidebar-toggle').click({
+    delay: 200,
+  });
+  await page.getByTestId('sidebar-tab-chat').click();
+  await expect(page.getByTestId('sidebar-tab-content-chat')).toBeVisible();
+});
+
+const makeChat = async (page: Page, content: string) => {
+  if (await page.getByTestId('sidebar-tab-chat').isHidden()) {
+    await page.getByTestId('right-sidebar-toggle').click({
+      delay: 200,
+    });
+  }
+  await page.getByTestId('sidebar-tab-chat').click();
+  await page.getByTestId('chat-panel-input').focus();
+  await page.keyboard.type(content);
+  await page.keyboard.press('Enter');
+};
+
+const clearChat = async (page: Page) => {
+  await page.getByTestId('chat-panel-clear').click();
+  await page.getByTestId('confirm-modal-confirm').click();
+};
+
+const collectChat = async (page: Page) => {
+  const chatPanel = await page.waitForSelector('.chat-panel-messages');
+  if (await chatPanel.$('.chat-panel-messages-placeholder')) {
+    return [];
+  }
+  // wait ai response
+  await page.waitForSelector('.chat-panel-messages .message chat-copy-more');
+  const lastMessage = await chatPanel.$$('.message').then(m => m[m.length - 1]);
+  await lastMessage.waitForSelector('chat-copy-more');
+  await page.waitForTimeout(ONE_SECOND);
+  return Promise.all(
+    Array.from(await chatPanel.$$('.message')).map(async m => ({
+      name: await m.$('.user-info').then(i => i?.innerText()),
+      content: await m
+        .$('chat-text')
+        .then(t => t?.$('editor-host'))
+        .then(e => e?.innerText()),
+    }))
+  );
+};
+
+const focusToEditor = async (page: Page) => {
+  const title = getBlockSuiteEditorTitle(page);
+  await title.focus();
+  await page.keyboard.press('Enter');
+};
+
+const getEditorContent = async (page: Page) => {
+  const lines = await page.$$('page-editor .inline-editor');
+  const contents = await Promise.all(lines.map(el => el.innerText()));
+  return (
+    contents
+      // cleanup zero width space
+      .map(c => c.replace(/\u200B/g, '').trim())
+      .filter(c => !!c)
+      .join('\n')
+  );
+};
+
+const switchToEdgelessMode = async (page: Page) => {
+  const editor = await page.waitForSelector('page-editor');
+  await page.getByTestId('switch-edgeless-mode-button').click();
+  // wait for new editor
+  editor.waitForElementState('hidden');
+  await page.waitForSelector('edgeless-editor');
+};
+
+test('can trigger login at chat side panel', async ({ page }) => {
+  await openHomePage(page);
+  await waitForEditorLoad(page);
+  await clickNewPageButton(page);
+  await makeChat(page, 'hello');
+  const loginTips = await page.waitForSelector('ai-error-wrapper');
+  expect(await loginTips.innerText()).toBe('Login');
+});
+
+test('can chat after login at chat side panel', async ({ page }) => {
+  await openHomePage(page);
+  await waitForEditorLoad(page);
+  await clickNewPageButton(page);
+  await makeChat(page, 'hello');
+  const loginTips = await page.waitForSelector('ai-error-wrapper');
+  (await loginTips.$('div'))!.click();
+  // login
+  const user = await getUser();
+  await loginUserDirectly(page, user);
+  // after login
+  await makeChat(page, 'hello');
+  const history = await collectChat(page);
+  expect(history[0]).toEqual({ name: 'You', content: 'hello' });
+  expect(history[1].name).toBe('AFFiNE AI');
+});
+
+test.describe('chat panel', () => {
+  let user: {
+    email: string;
+    password: string;
+  };
+
+  test.beforeEach(async ({ page }) => {
+    user = await getUser();
+    await loginUser(page, user);
+  });
+
+  test('basic chat', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const history = await collectChat(page);
+    expect(history[0]).toEqual({ name: 'You', content: 'hello' });
+    expect(history[1].name).toBe('AFFiNE AI');
+    await clearChat(page);
+    expect((await collectChat(page)).length).toBe(0);
+  });
+
+  test('chat actions', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    await page.getByTestId('action-copy-button').click();
+    await page.waitForTimeout(500);
+    expect(await page.evaluate(() => navigator.clipboard.readText())).toBe(
+      content
+    );
+    await page.getByTestId('action-retry-button').click();
+    expect((await collectChat(page))[1].content).not.toBe(content);
+  });
+
+  test('can be insert below', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    await focusToEditor(page);
+    // insert below
+    await page.getByTestId('action-insert-below').click();
+    await page.waitForSelector('affine-format-bar-widget editor-toolbar');
+    const editorContent = await getEditorContent(page);
+    expect(editorContent).toBe(content);
+  });
+
+  test('can be add to edgeless as node', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    await switchToEdgelessMode(page);
+    // delete default note
+    await (await page.waitForSelector('affine-edgeless-note')).click();
+    page.keyboard.press('Delete');
+    // insert note
+    await page.getByTestId('action-add-to-edgeless-as-note').click();
+    const edgelessNode = await page.waitForSelector('affine-edgeless-note');
+    expect(await edgelessNode.innerText()).toBe(content);
+  });
+
+  test('can be create as a doc', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    const editor = await page.waitForSelector('page-editor');
+    await page.getByTestId('action-create-as-a-doc').click();
+    // wait for new editor
+    editor.waitForElementState('hidden');
+    await page.waitForSelector('page-editor');
+    const editorContent = await getEditorContent(page);
+    expect(editorContent).toBe(content);
+  });
+
+  // feature not launched yet
+  test.skip('can be save chat to block', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const contents = (await collectChat(page)).map(m => m.content);
+    await switchToEdgelessMode(page);
+    await page.getByTestId('action-save-chat-to-block').click();
+    const chatBlock = await page.waitForSelector('affine-edgeless-ai-chat');
+    expect(
+      await Promise.all(
+        (await chatBlock.$$('.ai-chat-user-message')).map(m => m.innerText())
+      )
+    ).toBe(contents);
+  });
+
+  test('can be chat and insert below in page mode', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await focusToEditor(page);
+    await page.keyboard.type('/');
+    await page.getByTestId('sub-menu-0').getByText('Ask AI').click();
+    const input = await page.waitForSelector('ai-panel-input textarea');
+    await input.fill('hello');
+    await input.press('Enter');
+    const resp = await page.waitForSelector(
+      'ai-panel-answer .response-list-container'
+    ); // wait response
+    const content = await (
+      await page.waitForSelector('ai-panel-answer editor-host')
+    ).innerText();
+    await (await resp.waitForSelector('.ai-item-insert-below')).click();
+    const editorContent = await getEditorContent(page);
+    expect(editorContent).toBe(content);
+  });
+
+  test('can be retry or discard chat in page mode', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await focusToEditor(page);
+    await page.keyboard.type('/');
+    await page.getByTestId('sub-menu-0').getByText('Ask AI').click();
+    const input = await page.waitForSelector('ai-panel-input textarea');
+    await input.fill('hello');
+    await input.press('Enter');
+    // regenerate
+    {
+      const resp = await page.waitForSelector(
+        'ai-panel-answer .response-list-container:last-child'
+      );
+      const answerEditor = await page.waitForSelector(
+        'ai-panel-answer editor-host'
+      );
+      const content = await answerEditor.innerText();
+      await (await resp.waitForSelector('.ai-item-regenerate')).click();
+      await page.waitForSelector('ai-panel-answer .response-list-container'); // wait response
+      expect(
+        await (
+          await (
+            await page.waitForSelector('ai-panel-answer')
+          ).waitForSelector('editor-host')
+        ).innerText()
+      ).not.toBe(content);
+    }
+
+    // discard
+    {
+      const resp = await page.waitForSelector(
+        'ai-panel-answer .response-list-container:last-child'
+      );
+      await (await resp.waitForSelector('.ai-item-discard')).click();
+      await page.getByTestId('confirm-modal-confirm').click();
+      const editorContent = await getEditorContent(page);
+      expect(editorContent).toBe('');
+    }
+  });
+});
+
+test.describe('chat with block', () => {
+  let user: {
+    email: string;
+    password: string;
+  };
+
+  test.beforeEach(async ({ page }) => {
+    user = await getUser();
+    await loginUser(page, user);
+  });
+
+  const collectTextAnswer = async (page: Page) => {
+    // wait ai response
+    await page.waitForSelector(
+      'affine-ai-panel-widget .response-list-container',
+      { timeout: ONE_MINUTE }
+    );
+    const answer = await page.waitForSelector(
+      'affine-ai-panel-widget ai-panel-answer editor-host'
+    );
+    return answer.innerText();
+  };
+
+  const collectImageAnswer = async (page: Page, timeout = TEN_SECONDS) => {
+    // wait ai response
+    await page.waitForSelector(
+      'affine-ai-panel-widget .response-list-container',
+      { timeout }
+    );
+    const answer = await page.waitForSelector(
+      'affine-ai-panel-widget .ai-answer-image img'
+    );
+    return answer.getAttribute('src');
+  };
+
+  const disableEditorBlank = async (page: Page) => {
+    // hide blank element, this may block the click
+    const blank = page.getByTestId('page-editor-blank');
+    if (await blank.isVisible()) {
+      await blank.evaluate(node => (node.style.pointerEvents = 'none'));
+    } else {
+      console.warn('blank element not found');
+    }
+  };
+
+  test.describe('chat with text', () => {
+    const pasteTextToPageEditor = async (page: Page, content: string) => {
+      await focusToEditor(page);
+      await page.keyboard.type(content);
+    };
+
+    test.beforeEach(async ({ page }) => {
+      await page.reload();
+      await clickSideBarAllPageButton(page);
+      await page.waitForTimeout(200);
+      await createLocalWorkspace({ name: 'test' }, page);
+      await clickNewPageButton(page);
+      await pasteTextToPageEditor(page, 'hello');
+    });
+
+    test.beforeEach(async ({ page }) => {
+      await page.waitForSelector('affine-paragraph').then(i => i.click());
+      await page.keyboard.press('ControlOrMeta+A');
+      await page
+        .waitForSelector('page-editor editor-toolbar ask-ai-button')
+        .then(b => b.click());
+    });
+
+    const options = [
+      // review with ai
+      'Fix spelling',
+      'Fix Grammar',
+      // edit with ai
+      'Explain selection',
+      'Improve writing',
+      'Make it longer',
+      'Make it shorter',
+      'Continue writing',
+      // generate with ai
+      'Summarize',
+      'Generate headings',
+      'Generate outline',
+      'Find actions',
+      // draft with ai
+      'Write an article about this',
+      'Write a tweet about this',
+      'Write a poem about this',
+      'Write a blog post about this',
+      'Brainstorm ideas about this',
+    ];
+    for (const option of options) {
+      test(option, async ({ page }) => {
+        await disableEditorBlank(page);
+        await page
+          .waitForSelector(
+            `.ai-item-${option.replaceAll(' ', '-').toLowerCase()}`
+          )
+          .then(i => i.click());
+        expect(await collectTextAnswer(page)).toBeTruthy();
+      });
+    }
+  });
+
+  test.describe('chat with image block', () => {
+    const pasteImageToPageEditor = async (page: Page) => {
+      await page.evaluate(async () => {
+        const canvas = document.createElement('canvas');
+        canvas.width = 100;
+        canvas.height = 100;
+        const blob = await new Promise<Blob>(resolve =>
+          canvas.toBlob(blob => resolve(blob!), 'image/png')
+        );
+        await navigator.clipboard.write([
+          new ClipboardItem({ [blob.type]: blob }),
+        ]);
+      });
+      await focusToEditor(page);
+      await page.keyboard.press('ControlOrMeta+V');
+    };
+
+    test.beforeEach(async ({ page }) => {
+      await page.reload();
+      await clickSideBarAllPageButton(page);
+      await page.waitForTimeout(200);
+      await createLocalWorkspace({ name: 'test' }, page);
+      await clickNewPageButton(page);
+      await pasteImageToPageEditor(page);
+    });
+
+    test.describe('page mode', () => {
+      test.beforeEach(async ({ page }) => {
+        await disableEditorBlank(page);
+        await page.waitForSelector('affine-image').then(i => i.click());
+        await page
+          .waitForSelector('affine-image editor-toolbar ask-ai-button')
+          .then(b => b.click());
+      });
+
+      test('explain this image', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-explain-this-image')
+          .then(i => i.click());
+        expect(await collectTextAnswer(page)).toBeTruthy();
+      });
+
+      test('generate a caption', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-generate-a-caption')
+          .then(i => i.click());
+        expect(await collectTextAnswer(page)).toBeTruthy();
+      });
+
+      test('continue with ai', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-continue-with-ai')
+          .then(i => i.click());
+        await page
+          .waitForSelector('chat-panel-input .chat-panel-images')
+          .then(el => el.waitForElementState('visible'));
+      });
+
+      test('open ai chat', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-open-ai-chat')
+          .then(i => i.click());
+        const cards = await page.waitForSelector('chat-panel chat-cards');
+        await cards.waitForElementState('visible');
+        const cardTitles = await Promise.all(
+          await cards
+            .$$('.card-wrapper .card-title')
+            .then(els => els.map(async el => await el.innerText()))
+        );
+        expect(cardTitles).toContain('Start with this Image');
+      });
+    });
+
+    // TODO(@darkskygit): block by BS-1709, enable this after bug fix
+    test.describe.skip('edgeless mode', () => {
+      test.beforeEach(async ({ page }) => {
+        await switchToEdgelessMode(page);
+        const note = await page.waitForSelector('affine-edgeless-note');
+
+        {
+          // move note to avoid menu overlap
+          const box = (await note.boundingBox())!;
+          page.mouse.move(box.x + box.width / 2, box.y + box.height / 2);
+          page.mouse.down();
+          // sleep to avoid flicker
+          await page.waitForTimeout(500);
+          page.mouse.move(box.x + box.width / 2, box.y + box.height / 2 - 200);
+          await page.waitForTimeout(500);
+          page.mouse.up();
+          note.click();
+        }
+
+        await disableEditorBlank(page);
+        await page.waitForSelector('affine-image').then(i => i.click());
+        await page
+          .waitForSelector('affine-image editor-toolbar ask-ai-button')
+          .then(b => b.click());
+      });
+
+      // skip by default, dalle is very slow
+      test.skip('generate an image', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-generate-an-image')
+          .then(i => i.click());
+        await page.keyboard.type('a cat');
+        await page.keyboard.press('Enter');
+        expect(await collectImageAnswer(page)).toBeTruthy();
+      });
+
+      const processes = [
+        'Clearer',
+        'Remove background',
+        // skip by default, need a face in image
+        // 'Convert to sticker',
+      ];
+      for (const process of processes) {
+        test(`image processing ${process}`, async ({ page }) => {
+          await page
+            .waitForSelector('.ai-item-image-processing')
+            .then(i => i.hover());
+          await page.getByText(process).click();
+          {
+            // to be remove
+            await page.keyboard.type(',');
+            await page.keyboard.press('Enter');
+          }
+          expect(await collectImageAnswer(page, ONE_MINUTE * 2)).toBeTruthy();
+        });
+      }
+
+      const filters = ['Clay', 'Sketch', 'Anime', 'Pixel'];
+      for (const filter of filters) {
+        test(`ai image ${filter.toLowerCase()} filter`, async ({ page }) => {
+          await page
+            .waitForSelector('.ai-item-ai-image-filter')
+            .then(i => i.hover());
+          await page.getByText(`${filter} style`).click();
+          {
+            // to be remove
+            await page.keyboard.type(',');
+            await page.keyboard.press('Enter');
+          }
+          expect(await collectImageAnswer(page, ONE_MINUTE * 2)).toBeTruthy();
+        });
+      }
+    });
+  });
+});

tests/affine-cloud-copilot/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@affine-test/affine-cloud-copilot",
+  "private": true,
+  "scripts": {
+    "e2e": "yarn playwright test"
+  },
+  "devDependencies": {
+    "@affine-test/kit": "workspace:*",
+    "@playwright/test": "=1.47.2"
+  },
+  "version": "0.17.0"
+}

tests/affine-cloud-copilot/playwright.config.ts

@@ -0,0 +1,79 @@
+import { testResultDir } from '@affine-test/kit/playwright';
+import type {
+  PlaywrightTestConfig,
+  PlaywrightWorkerOptions,
+} from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  testDir: './e2e',
+  fullyParallel: !process.env.CI,
+  timeout: 120_000,
+  outputDir: testResultDir,
+  use: {
+    baseURL: 'http://localhost:8080/',
+    browserName:
+      (process.env.BROWSER as PlaywrightWorkerOptions['browserName']) ??
+      'chromium',
+    permissions: ['clipboard-read', 'clipboard-write'],
+    viewport: { width: 1440, height: 800 },
+    actionTimeout: 10 * 1000,
+    locale: 'en-US',
+    trace: 'on',
+    video: 'on',
+  },
+  forbidOnly: !!process.env.CI,
+  workers: 4,
+  retries: 3,
+  reporter: process.env.CI ? 'github' : 'list',
+  webServer: [
+    {
+      command: 'yarn run serve:test-static',
+      port: 8081,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+      env: {
+        COVERAGE: process.env.COVERAGE || 'false',
+        ENABLE_DEBUG_PAGE: '1',
+      },
+    },
+    // Intentionally not building the web, reminds you to run it by yourself.
+    {
+      command: 'yarn -T run start:web-static',
+      port: 8080,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+      env: {
+        COVERAGE: process.env.COVERAGE || 'false',
+      },
+    },
+    {
+      command: 'yarn workspace @affine/server start',
+      port: 3010,
+      timeout: 120 * 1000,
+      reuseExistingServer: !process.env.CI,
+      stdout: 'pipe',
+      stderr: 'pipe',
+      env: {
+        DATABASE_URL:
+          process.env.DATABASE_URL ??
+          'postgresql://affine:affine@localhost:5432/affine',
+        NODE_ENV: 'development',
+        AFFINE_ENV: process.env.AFFINE_ENV ?? 'dev',
+        DEBUG: 'affine:*',
+        FORCE_COLOR: 'true',
+        DEBUG_COLORS: 'true',
+        MAILER_HOST: '0.0.0.0',
+        MAILER_PORT: '1025',
+        MAILER_SENDER: 'noreply@toeverything.info',
+        MAILER_USER: 'noreply@toeverything.info',
+        MAILER_PASSWORD: 'affine',
+      },
+    },
+  ],
+};
+
+if (process.env.CI) {
+  config.retries = 3;
+}
+
+export default config;

tests/affine-cloud-copilot/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "outDir": "lib"
+  },
+  "include": ["e2e"],
+  "references": [
+    {
+      "path": "../../tests/kit"
+    },
+    {
+      "path": "../../tests/fixtures"
+    }
+  ]
+}

tests/affine-cloud/playwright.config.ts

@@ -22,8 +22,8 @@ const config: PlaywrightTestConfig = {
     video: 'on',
   },
   forbidOnly: !!process.env.CI,
-  workers: process.env.CI ? 1 : 4,
-  retries: 1,
+  workers: process.env.CI && !process.env.COPILOT ? 1 : 4,
+  retries: process.env.COPILOT ? 1 : 3,
   reporter: process.env.CI ? 'github' : 'list',
   webServer: [
     {

tests/affine-local/e2e/all-page.spec.ts

@@ -16,32 +16,14 @@ import { openHomePage } from '@affine-test/kit/utils/load-page';
 import {
   clickNewPageButton,
   clickPageMoreActions,
+  getAllPage,
   getBlockSuiteEditorTitle,
   waitForAllPagesLoad,
   waitForEditorLoad,
 } from '@affine-test/kit/utils/page-logic';
 import { clickSideBarAllPageButton } from '@affine-test/kit/utils/sidebar';
-import type { Page } from '@playwright/test';
 import { expect } from '@playwright/test';
 
-function getAllPage(page: Page) {
-  const newPageButton = page.getByTestId('new-page-button-trigger');
-  const newPageDropdown = newPageButton.locator('svg');
-  const edgelessBlockCard = page.getByTestId('new-edgeless-button-in-all-page');
-
-  async function clickNewPageButton() {
-    const newPageButton = page.getByTestId('new-page-button-trigger');
-    return await newPageButton.click();
-  }
-
-  async function clickNewEdgelessDropdown() {
-    await newPageDropdown.click();
-    await edgelessBlockCard.click();
-  }
-
-  return { clickNewPageButton, clickNewEdgelessDropdown };
-}
-
 test('all page', async ({ page }) => {
   await openHomePage(page);
   await waitForEditorLoad(page);

tests/kit/utils/cloud.ts

@@ -150,6 +150,68 @@ export async function createRandomUser(): Promise<{
   } as any;
 }
 
+export async function createRandomAIUser(): Promise<{
+  name: string;
+  email: string;
+  password: string;
+  id: string;
+}> {
+  const user = {
+    name: faker.internet.userName(),
+    email: faker.internet.email().toLowerCase(),
+    password: '123456',
+  };
+  const result = await runPrisma(async client => {
+    const freeFeatureId = await client.feature
+      .findFirst({
+        where: { feature: 'free_plan_v1' },
+        select: { id: true },
+        orderBy: { version: 'desc' },
+      })
+      .then(f => f!.id);
+    const aiFeatureId = await client.feature
+      .findFirst({
+        where: { feature: 'unlimited_copilot' },
+        select: { id: true },
+        orderBy: { version: 'desc' },
+      })
+      .then(f => f!.id);
+
+    await client.user.create({
+      data: {
+        ...user,
+        emailVerifiedAt: new Date(),
+        password: await hash(user.password),
+        features: {
+          create: [
+            {
+              reason: 'created by test case',
+              activated: true,
+              featureId: freeFeatureId,
+            },
+            {
+              reason: 'created by test case',
+              activated: true,
+              featureId: aiFeatureId,
+            },
+          ],
+        },
+      },
+    });
+
+    return await client.user.findUnique({
+      where: {
+        email: user.email,
+      },
+    });
+  });
+  cloudUserSchema.parse(result);
+  return {
+    ...result,
+    password: user.password,
+  } as any;
+}
+
 export async function deleteUser(email: string) {
   await runPrisma(async client => {
     await client.user.delete({
@@ -178,7 +240,24 @@ export async function loginUser(
   }
 
   await clickSideBarCurrentWorkspaceBanner(page);
-  await page.getByTestId('cloud-signin-button').click();
+  await page.getByTestId('cloud-signin-button').click({
+    delay: 200,
+  });
+  await loginUserDirectly(page, user, config);
+}
+
+export async function loginUserDirectly(
+  page: Page,
+  user: {
+    email: string;
+    password: string;
+  },
+  config?: {
+    isElectron?: boolean;
+    beforeLogin?: () => Promise<void>;
+    afterLogin?: () => Promise<void>;
+  }
+) {
   await page.getByPlaceholder('Enter your email address').fill(user.email);
   await page.getByTestId('continue-login-button').click({
     delay: 200,
@@ -188,8 +267,10 @@ export async function loginUser(
     await config.beforeLogin();
   }
   await page.waitForTimeout(200);
-  await page.getByTestId('sign-in-button').click();
-  await page.waitForTimeout(500);
+  const signIn = page.getByTestId('sign-in-button');
+  await signIn.click();
+  await signIn.waitFor({ state: 'detached' });
+  await page.waitForTimeout(200);
   if (config?.afterLogin) {
     await config.afterLogin();
   }

tests/kit/utils/load-page.ts

@@ -1,7 +1,11 @@
 import type { Page } from '@playwright/test';
 import { expect } from '@playwright/test';
 
-export const coreUrl = 'http://localhost:8080';
+export let coreUrl = 'http://localhost:8080';
+
+export function setCoreUrl(url: string) {
+  coreUrl = url;
+}
 
 export async function openHomePage(page: Page) {
   await page.goto(coreUrl);

tests/kit/utils/page-logic.ts

@@ -1,6 +1,24 @@
 import type { Locator, Page } from '@playwright/test';
 import { expect } from '@playwright/test';
 
+export function getAllPage(page: Page) {
+  const newPageButton = page.getByTestId('new-page-button-trigger');
+  const newPageDropdown = newPageButton.locator('svg');
+  const edgelessBlockCard = page.getByTestId('new-edgeless-button-in-all-page');
+
+  async function clickNewPageButton() {
+    const newPageButton = page.getByTestId('new-page-button-trigger');
+    return await newPageButton.click();
+  }
+
+  async function clickNewEdgelessDropdown() {
+    await newPageDropdown.click();
+    await edgelessBlockCard.click();
+  }
+
+  return { clickNewPageButton, clickNewEdgelessDropdown };
+}
+
 export async function waitForEditorLoad(page: Page) {
   await page.waitForSelector('v-line', {
     timeout: 20000,