diff --git a/.github/actions/copilot-test/action.yml b/.github/actions/copilot-test/action.yml index cb75daafec..abd0101691 100644 --- a/.github/actions/copilot-test/action.yml +++ b/.github/actions/copilot-test/action.yml @@ -3,7 +3,7 @@ description: 'Run Copilot E2E Test' inputs: script: description: 'Script to run' - default: 'yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only' + default: 'yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only' required: false openai-key: description: 'OpenAI secret key' diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 4ee194c913..2f8bae6131 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -505,6 +505,7 @@ jobs: changed: - 'packages/backend/server/src/plugins/copilot/**' - 'packages/backend/server/tests/copilot.*' + - 'tests/affine-cloud-copilot/**' - name: Setup Node.js if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }} @@ -526,7 +527,7 @@ jobs: - name: Run server tests if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }} - run: yarn affine @affine/server test:copilot:coverage --forbid-only + run: yarn affine @affine/server test:copilot:spec:coverage --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} @@ -617,7 +618,7 @@ jobs: if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.e2efilter.outputs.changed == 'true' }} uses: ./.github/actions/copilot-test with: - script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + script: yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }} fal-key: ${{ secrets.COPILOT_FAL_API_KEY }} perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} diff --git a/.github/workflows/copilot-test.yml b/.github/workflows/copilot-test.yml index 5ce2300f33..be9d3ab70e 100644 --- a/.github/workflows/copilot-test.yml +++ b/.github/workflows/copilot-test.yml @@ -38,6 +38,16 @@ jobs: DISTRIBUTION: web DATABASE_URL: postgresql://affine:affine@localhost:5432/affine REDIS_SERVER_HOST: localhost + strategy: + fail-fast: false + matrix: + spec: + - { + name: e2e, + package: '@affine-test/affine-cloud-copilot', + type: e2e, + } + - { name: spec, package: '@affine/server', type: copilot:spec } services: postgres: image: postgres @@ -78,13 +88,14 @@ jobs: - name: Prepare Server Test Environment uses: ./.github/actions/server-test-env - - name: Run server tests - run: yarn affine @affine/server test:copilot:coverage --forbid-only + - name: Run copilot api ${{ matrix.spec.name }} tests + run: yarn affine ${{ matrix.spec.package }} test:${{ matrix.spec.type }}:coverage --forbid-only env: CARGO_TARGET_DIR: '${{ github.workspace }}/target' COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }} COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }} COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} + COPILOT_E2E_ENDPOINT: ${{ secrets.COPILOT_E2E_ENDPOINT }} - name: Upload server test coverage results uses: codecov/codecov-action@v5 @@ -145,7 +156,7 @@ jobs: - name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} uses: ./.github/actions/copilot-test with: - script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + script: yarn affine @affine-test/affine-cloud-copilot test:e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }} fal-key: ${{ secrets.COPILOT_FAL_API_KEY }} perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }} diff --git a/package.json b/package.json index f6b08e5846..a0594979e1 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "tools/*", "docs/reference", "tools/@types/*", - "tests/*" + "tests/*", + "tests/affine-cloud/*" ], "engines": { "node": "<23.0.0" diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 238d10a5c3..89974eb412 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -12,9 +12,11 @@ "dev": "nodemon ./src/index.ts", "dev:mail": "email dev -d src/mails", "test": "ava --concurrency 1 --serial", - "test:copilot": "ava \"src/__tests__/**/copilot-*.spec.ts\"", + "test:copilot:e2e": "ava \"e2e/copilot.e2e.ts\"", + "test:copilot:spec": "ava \"src/__tests__/**/copilot-*.spec.ts\"", "test:coverage": "c8 ava --concurrency 1 --serial", - "test:copilot:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"", + "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"e2e/copilot.e2e.ts\"", + "test:copilot:spec:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"", "data-migration": "cross-env NODE_ENV=script r ./src/data/index.ts", "predeploy": "yarn prisma migrate deploy && NODE_ENV=script node --import ./scripts/register.js ./dist/data/index.js run", "postinstall": "prisma generate" diff --git a/packages/backend/server/src/__tests__/copilot-provider.e2e.ts b/packages/backend/server/src/__tests__/copilot-provider.e2e.ts new file mode 100644 index 0000000000..530db68d3e --- /dev/null +++ b/packages/backend/server/src/__tests__/copilot-provider.e2e.ts @@ -0,0 +1,214 @@ +import { randomUUID } from 'node:crypto'; + +import { createRandomAIUser } from '@affine-test/kit/utils/cloud'; +import type { ExecutionContext, TestFn } from 'ava'; +import ava from 'ava'; + +import { createWorkspace } from './utils'; +import { + chatWithImages, + chatWithText, + chatWithWorkflow, + createCopilotMessage, + createCopilotSession, + ProviderActionTestCase, + ProviderWorkflowTestCase, + sse2array, +} from './utils/copilot'; + +type Tester = { + app: any; + userEmail: string; + userToken: string; + workspaceId: string; +}; +const test = ava as TestFn; + +const e2eConfig = { + endpoint: process.env.COPILOT_E2E_ENDPOINT || 'http://localhost:3010', +}; + +const isCopilotConfigured = + !!process.env.COPILOT_OPENAI_API_KEY && + !!process.env.COPILOT_FAL_API_KEY && + process.env.COPILOT_OPENAI_API_KEY !== '1' && + process.env.COPILOT_FAL_API_KEY !== '1'; +const runIfCopilotConfigured = test.macro( + async ( + t, + callback: (t: ExecutionContext) => Promise | void + ) => { + if (isCopilotConfigured) { + await callback(t); + } else { + t.log('Skip test because copilot is not configured'); + t.pass(); + } + } +); + +export const runPrisma = async ( + cb: ( + prisma: InstanceType< + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + typeof import('../../../../packages/backend/server/node_modules/@prisma/client').PrismaClient + > + ) => Promise +): Promise => { + const { + PrismaClient, + // eslint-disable-next-line @typescript-eslint/no-var-requires + } = await import( + '../../../../packages/backend/server/node_modules/@prisma/client' + ); + const client = new PrismaClient(); + await client.$connect(); + try { + return await cb(client); + } finally { + await client.$disconnect(); + } +}; + +test.before(async t => { + if (!isCopilotConfigured) return; + const { endpoint } = e2eConfig; + + const { email, sessionId: token } = await createRandomAIUser( + 'affine.fail', + runPrisma + ); + const app = { getHttpServer: () => endpoint } as any; + const { id } = await createWorkspace(app, token); + + t.context.app = app; + t.context.userEmail = email; + t.context.userToken = token; + t.context.workspaceId = id; +}); + +test.after(async t => { + if (!isCopilotConfigured) return; + await runPrisma(async client => { + await client.user.delete({ + where: { + email: t.context.userEmail, + }, + }); + }); +}); + +const retry = async ( + action: string, + t: ExecutionContext, + callback: (t: ExecutionContext) => void +) => { + let i = 3; + while (i--) { + const ret = await t.try(callback); + if (ret.passed) { + return ret.commit(); + } else { + ret.discard(); + t.log(ret.errors.map(e => e.message).join('\n')); + t.log(`retrying ${action} ${3 - i}/3 ...`); + } + } + t.fail(`failed to run ${action}`); +}; + +const makeCopilotChat = async ( + t: ExecutionContext, + promptName: string, + { content, attachments, params }: any +) => { + const { app, userToken, workspaceId } = t.context; + const sessionId = await createCopilotSession( + app, + userToken, + workspaceId, + randomUUID(), + promptName + ); + const messageId = await createCopilotMessage( + app, + userToken, + sessionId, + content, + attachments, + undefined, + params + ); + return { sessionId, messageId }; +}; + +// ==================== action ==================== + +for (const { promptName, messages, verifier, type } of ProviderActionTestCase) { + const prompts = Array.isArray(promptName) ? promptName : [promptName]; + for (const promptName of prompts) { + test( + `should be able to run action: ${promptName}`, + runIfCopilotConfigured, + async t => { + await retry(`action: ${promptName}`, t, async t => { + const { app, userToken } = t.context; + const { sessionId, messageId } = await makeCopilotChat( + t, + promptName, + messages[0] + ); + + if (type === 'text') { + const result = await chatWithText( + app, + userToken, + sessionId, + messageId + ); + t.truthy(result, 'should return result'); + verifier?.(t, result); + } else if (type === 'image') { + const result = sse2array( + await chatWithImages(app, userToken, sessionId, messageId) + ) + .filter(e => e.event !== 'event') + .map(e => e.data) + .filter(Boolean); + t.truthy(result.length, 'should return result'); + for (const r of result) { + verifier?.(t, r); + } + } else { + t.fail('unsupported provider type'); + } + }); + } + ); + } +} + +// ==================== workflow ==================== + +for (const { name, content, verifier } of ProviderWorkflowTestCase) { + test( + `should be able to run workflow: ${name}`, + runIfCopilotConfigured, + async t => { + await retry(`workflow: ${name}`, t, async t => { + const { app, userToken } = t.context; + const { sessionId, messageId } = await makeCopilotChat( + t, + `workflow:${name}`, + { content } + ); + const r = await chatWithWorkflow(app, userToken, sessionId, messageId); + const result = sse2array(r) + .filter(e => e.event !== 'event' && e.data) + .reduce((p, c) => p + c.data, ''); + t.truthy(result, 'should return result'); + verifier?.(t, result); + }); + } + ); +} diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index 70344ac26a..ca95287afa 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -28,7 +28,11 @@ import { CopilotCheckJsonExecutor, } from '../plugins/copilot/workflow/executor'; import { createTestingModule } from './utils'; -import { TestAssets } from './utils/copilot'; +import { + checkMDList, + ProviderActionTestCase, + ProviderWorkflowTestCase, +} from './utils/copilot'; type Tester = { auth: AuthService; @@ -138,58 +142,6 @@ test.after(async t => { await t.context.module.close(); }); -const assertNotWrappedInCodeBlock = ( - t: ExecutionContext, - result: string -) => { - t.assert( - !result.replaceAll('\n', '').trim().startsWith('```') && - !result.replaceAll('\n', '').trim().endsWith('```'), - 'should not wrap in code block' - ); -}; - -const checkMDList = (text: string) => { - const lines = text.split('\n'); - const listItemRegex = /^( {2})*(-|\u2010-\u2015|\*|\+)? .+$/; - let prevIndent = null; - - for (const line of lines) { - if (line.trim() === '') continue; - if (!listItemRegex.test(line)) { - return false; - } - - const currentIndent = line.match(/^( *)/)?.[0].length!; - if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) { - return false; - } - - if (prevIndent !== null && currentIndent > 0) { - const indentDiff = currentIndent - prevIndent; - // allow 1 level of indentation difference - if (indentDiff > 2) { - return false; - } - } - - if (line.trim().startsWith('-')) { - prevIndent = currentIndent; - } - } - - return true; -}; - -const checkUrl = (url: string) => { - try { - new URL(url); - return true; - } catch { - return false; - } -}; - const retry = async ( action: string, t: ExecutionContext, @@ -271,140 +223,7 @@ test('should validate markdown list', t => { // ==================== action ==================== -const actions = [ - { - promptName: [ - 'Summary', - 'Explain this', - 'Write an article about this', - 'Write a twitter about this', - 'Write a poem about this', - 'Write a blog post about this', - 'Write outline', - 'Change tone to', - 'Improve writing for it', - 'Improve grammar for it', - 'Fix spelling for it', - 'Create headings', - 'Make it longer', - 'Make it shorter', - 'Continue writing', - 'Chat With AFFiNE AI', - 'Search With AFFiNE AI', - ], - messages: [{ role: 'user' as const, content: TestAssets.SSOT }], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - t.assert( - result.toLowerCase().includes('single source of truth'), - 'should include original keyword' - ); - }, - type: 'text' as const, - }, - { - promptName: ['Brainstorm ideas about this', 'Brainstorm mindmap'], - messages: [{ role: 'user' as const, content: TestAssets.SSOT }], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - t.assert(checkMDList(result), 'should be a markdown list'); - }, - type: 'text' as const, - }, - { - promptName: 'Expand mind map', - messages: [{ role: 'user' as const, content: '- Single source of truth' }], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - t.assert(checkMDList(result), 'should be a markdown list'); - }, - type: 'text' as const, - }, - { - promptName: 'Find action items from it', - messages: [{ role: 'user' as const, content: TestAssets.TODO }], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - t.assert(checkMDList(result), 'should be a markdown list'); - }, - type: 'text' as const, - }, - { - promptName: ['Explain this code', 'Check code error'], - messages: [{ role: 'user' as const, content: TestAssets.Code }], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - t.assert( - result.toLowerCase().includes('distance'), - 'explain code result should include keyword' - ); - }, - type: 'text' as const, - }, - { - promptName: 'Translate to', - messages: [ - { - role: 'user' as const, - content: TestAssets.SSOT, - params: { language: 'Simplified Chinese' }, - }, - ], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - t.assert( - result.toLowerCase().includes('单一事实来源'), - 'explain code result should include keyword' - ); - }, - type: 'text' as const, - }, - { - promptName: ['Generate a caption', 'Explain this image'], - messages: [ - { - role: 'user' as const, - content: '', - attachments: [ - 'https://cdn.affine.pro/copilot-test/Qgqy9qZT3VGIEuMIotJYoCCH.jpg', - ], - }, - ], - verifier: (t: ExecutionContext, result: string) => { - assertNotWrappedInCodeBlock(t, result); - const content = result.toLowerCase(); - t.assert( - content.includes('classroom') || - content.includes('school') || - content.includes('sky'), - 'explain code result should include keyword' - ); - }, - type: 'text' as const, - }, - { - promptName: [ - 'debug:action:fal-face-to-sticker', - 'debug:action:fal-remove-bg', - 'debug:action:fal-sd15', - 'debug:action:fal-upscaler', - ], - messages: [ - { - role: 'user' as const, - content: '', - attachments: [ - 'https://cdn.affine.pro/copilot-test/Zkas098lkjdf-908231.jpg', - ], - }, - ], - verifier: (t: ExecutionContext, link: string) => { - t.truthy(checkUrl(link), 'should be a valid url'); - }, - type: 'image' as const, - }, -]; -for (const { promptName, messages, verifier, type } of actions) { +for (const { promptName, messages, verifier, type } of ProviderActionTestCase) { const prompts = Array.isArray(promptName) ? promptName : [promptName]; for (const promptName of prompts) { test( @@ -464,28 +283,7 @@ for (const { promptName, messages, verifier, type } of actions) { // ==================== workflow ==================== -const workflows = [ - { - name: 'brainstorm', - content: 'apple company', - verifier: (t: ExecutionContext, result: string) => { - t.assert(checkMDList(result), 'should be a markdown list'); - }, - }, - { - name: 'presentation', - content: 'apple company', - verifier: (t: ExecutionContext, result: string) => { - for (const l of result.split('\n')) { - t.notThrows(() => { - JSON.parse(l.trim()); - }, 'should be valid json'); - } - }, - }, -]; - -for (const { name, content, verifier } of workflows) { +for (const { name, content, verifier } of ProviderWorkflowTestCase) { test( `should be able to run workflow: ${name}`, runIfCopilotConfigured, diff --git a/packages/backend/server/src/__tests__/utils/copilot.ts b/packages/backend/server/src/__tests__/utils/copilot.ts index b78b256425..d4cda549cb 100644 --- a/packages/backend/server/src/__tests__/utils/copilot.ts +++ b/packages/backend/server/src/__tests__/utils/copilot.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'node:crypto'; import { INestApplication } from '@nestjs/common'; +import { ExecutionContext } from 'ava'; import request from 'supertest'; import { @@ -329,7 +330,14 @@ export function sse2array(eventSource: string) { block.split('\n').reduce( (prev, curr) => { const [key, ...values] = curr.split(': '); - return Object.assign(prev, { [key]: values.join(': ') }); + const value = values.join(': '); + const finalValue = value ? value : '\n'; + if (prev[key]) { + const prevValue = finalValue === '\n' ? prev[key] : prev[key] + '\n'; + return Object.assign(prev, { [key]: prevValue + finalValue }); + } else { + return Object.assign(prev, { [key]: finalValue }); + } }, {} as Record ) @@ -538,8 +546,214 @@ export const WorkflowTestCases: WorkflowTestCase[] = [ }, ]; -export const TestAssets = { +export const checkMDList = (text: string) => { + const lines = text.split('\n'); + const listItemRegex = /^( {2})*(-|\u2010-\u2015|\*|\+)? .+$/; + let prevIndent = null; + + for (const line of lines) { + if (line.trim() === '') continue; + if (!listItemRegex.test(line)) { + return false; + } + + // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain + const currentIndent = line.match(/^( *)/)?.[0].length!; + if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) { + return false; + } + + if (prevIndent !== null && currentIndent > 0) { + const indentDiff = currentIndent - prevIndent; + // allow 1 level of indentation difference + if (indentDiff > 2) { + return false; + } + } + + if (line.trim().startsWith('-')) { + prevIndent = currentIndent; + } + } + + return true; +}; + +const checkUrl = (url: string) => { + try { + new URL(url); + return true; + } catch { + return false; + } +}; + +const TestAssets = { SSOT: `In [information science](https://en.wikipedia.org/wiki/Information_science) and [information technology](https://en.wikipedia.org/wiki/Information_technology), **single source of truth** (**SSOT**) architecture, or **single point of truth** (**SPOT**) architecture, for [information systems](https://en.wikipedia.org/wiki/Information_system) is the practice of structuring [information models](https://en.wikipedia.org/wiki/Information_model) and associated [data schemas](https://en.wikipedia.org/wiki/Database_schema) such that every [data element](https://en.wikipedia.org/wiki/Data_element) is [mastered](https://en.wikipedia.org/wiki/Golden_record_(informatics)) (or edited) in only one place, providing [data normalization to a canonical form](https://en.wikipedia.org/wiki/Canonical_form#Computing) (for example, in [database normalization](https://en.wikipedia.org/wiki/Database_normalization) or content [transclusion](https://en.wikipedia.org/wiki/Transclusion)).\n\nThere are several scenarios with respect to copies and updates:\n\n* The master data is never copied and instead only references to it are made; this means that all reads and updates go directly to the SSOT.\n* The master data is copied but the copies are only read and only the master data is updated; if requests to read data are only made on copies, this is an instance of [CQRS](https://en.wikipedia.org/wiki/CQRS).\n* The master data is copied and the copies are updated; this needs a reconciliation mechanism when there are concurrent updates.\n * Updates on copies can be thrown out whenever a concurrent update is made on the master, so they are not considered fully committed until propagated to the master. (many blockchains work that way.)\n * Concurrent updates are merged. (if an automatic merge fails, it could fall back on another strategy, which could be the previous strategy or something else like manual intervention, which most source version control systems do.)\n\nThe advantages of SSOT architectures include easier prevention of mistaken inconsistencies (such as a duplicate value/copy somewhere being forgotten), and greatly simplified [version control](https://en.wikipedia.org/wiki/Version_control). Without a SSOT, dealing with inconsistencies implies either complex and error-prone consensus algorithms, or using a simpler architecture that's liable to lose data in the face of inconsistency (the latter may seem unacceptable but it is sometimes a very good choice; it is how most blockchains operate: a transaction is actually final only if it was included in the next block that is mined).\n\nIdeally, SSOT systems provide data that are authentic (and [authenticatable](https://en.wikipedia.org/wiki/Authentication)), relevant, and [referable](https://en.wikipedia.org/wiki/Reference_(computer_science)).[[1]](https://en.wikipedia.org/wiki/Single_source_of_truth#cite_note-1)\n\nDeployment of an SSOT architecture is becoming increasingly important in enterprise settings where incorrectly linked duplicate or de-normalized data elements (a direct consequence of intentional or unintentional [denormalization](https://en.wikipedia.org/wiki/Denormalization) of any explicit data model) pose a risk for retrieval of outdated, and therefore incorrect, information. Common examples (i.e., example classes of implementation) are as follows:\n\n* In [electronic health records](https://en.wikipedia.org/wiki/Electronic_health_record) (EHRs), it is imperative to accurately validate patient identity against a single referential repository, which serves as the SSOT. Duplicate representations of data within the enterprise would be implemented by the use of [pointers](https://en.wikipedia.org/wiki/Pointer_(computer_programming)) rather than duplicate database tables, rows, or cells. This ensures that data updates to elements in the authoritative location are comprehensively distributed to all [federated database](https://en.wikipedia.org/wiki/Federated_database) constituencies in the larger overall [enterprise architecture](https://en.wikipedia.org/wiki/Enterprise_architecture). EHRs are an excellent class for exemplifying how SSOT architecture is both poignantly necessary and challenging to achieve: it is challenging because inter-organization [health information exchange](https://en.wikipedia.org/wiki/Health_information_exchange) is inherently a [cybersecurity](https://en.wikipedia.org/wiki/Computer_security) competence hurdle, and nonetheless it is necessary, to prevent [medical errors](https://en.wikipedia.org/wiki/Medical_error), to prevent the wasted costs of inefficiency (such as duplicated work or rework), and to make the [primary care](https://en.wikipedia.org/wiki/Primary_care) and [medical home](https://en.wikipedia.org/wiki/Medical_home) concepts feasible (to achieve competent [care transitions](https://en.wikipedia.org/wiki/Transitional_care)).\n* [Single-source publishing](https://en.wikipedia.org/wiki/Single-source_publishing) as a general principle or ideal in [content management](https://en.wikipedia.org/wiki/Content_management) relies on having SSOTs, via [transclusion](https://en.wikipedia.org/wiki/Transclusion) or (otherwise, at least) substitution. Substitution happens via [libraries of objects](https://en.wikipedia.org/wiki/Library_(computing)#Object_libraries) that can be propagated as static copies which are later refreshed when necessary (that is, when refreshing of the [copy-paste](https://en.wikipedia.org/wiki/Cut,_copy,_and_paste) or [import](https://en.wikipedia.org/wiki/Import_and_export_of_data) is triggered by a larger updating event). [Component content management systems](https://en.wikipedia.org/wiki/Component_content_management_system) are a class of [content management systems](https://en.wikipedia.org/wiki/Content_management_system) that aim to provide competence on this level.`, Code: `fn euclidean_distance(a: &Vec, b: &Vec) -> f64 {\na.iter().zip(b.iter()).map(|(x, y)| (*x - *y).powi(2)).sum::().sqrt()\n}`, TODO: 'The PDF exporting feature in edgeless is flawed, which is not supposed to support rendering content with infinite logical size. We should remove this feature entry to user, but the current "export blob in surface ref" feature should be migrated and kept (which is base on the edgelessToCanvas API, which makes sense for exporting a partial viewport area for the page)', }; + +const assertNotWrappedInCodeBlock = (t: ExecutionContext, result: string) => { + t.assert( + !result.replaceAll('\n', '').trim().startsWith('```') && + !result.replaceAll('\n', '').trim().endsWith('```'), + 'should not wrap in code block' + ); +}; + +export const ProviderActionTestCase = [ + { + promptName: [ + 'Summary', + 'Explain this', + 'Write an article about this', + 'Write a twitter about this', + 'Write a poem about this', + 'Write a blog post about this', + 'Write outline', + 'Change tone to', + 'Improve writing for it', + 'Improve grammar for it', + 'Fix spelling for it', + 'Create headings', + 'Make it longer', + 'Make it shorter', + 'Continue writing', + 'Chat With AFFiNE AI', + 'Search With AFFiNE AI', + ], + messages: [{ role: 'user' as const, content: TestAssets.SSOT }], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + t.assert( + result.toLowerCase().includes('single source of truth') || + result.toLowerCase().includes('ssot'), + `should include original keyword: ${result}` + ); + }, + type: 'text' as const, + }, + { + promptName: ['Brainstorm ideas about this', 'Brainstorm mindmap'], + messages: [{ role: 'user' as const, content: TestAssets.SSOT }], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + t.assert(checkMDList(result), 'should be a markdown list'); + }, + type: 'text' as const, + }, + { + promptName: 'Expand mind map', + messages: [{ role: 'user' as const, content: '- Single source of truth' }], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + t.assert(checkMDList(result), 'should be a markdown list'); + }, + type: 'text' as const, + }, + { + promptName: 'Find action items from it', + messages: [{ role: 'user' as const, content: TestAssets.TODO }], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + t.assert(checkMDList(result), 'should be a markdown list'); + }, + type: 'text' as const, + }, + { + promptName: ['Explain this code', 'Check code error'], + messages: [{ role: 'user' as const, content: TestAssets.Code }], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + t.assert( + result.toLowerCase().includes('distance'), + 'explain code result should include keyword' + ); + }, + type: 'text' as const, + }, + { + promptName: 'Translate to', + messages: [ + { + role: 'user' as const, + content: TestAssets.SSOT, + params: { language: 'Simplified Chinese' }, + }, + ], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + t.assert( + result.toLowerCase().includes('单一事实来源'), + 'explain code result should include keyword' + ); + }, + type: 'text' as const, + }, + { + promptName: ['Generate a caption', 'Explain this image'], + messages: [ + { + role: 'user' as const, + content: '', + attachments: [ + 'https://cdn.affine.pro/copilot-test/Qgqy9qZT3VGIEuMIotJYoCCH.jpg', + ], + }, + ], + verifier: (t: ExecutionContext, result: string) => { + assertNotWrappedInCodeBlock(t, result); + const content = result.toLowerCase(); + t.assert( + content.includes('classroom') || + content.includes('school') || + content.includes('sky'), + 'explain code result should include keyword' + ); + }, + type: 'text' as const, + }, + { + promptName: [ + 'debug:action:fal-face-to-sticker', + 'debug:action:fal-remove-bg', + 'debug:action:fal-sd15', + 'debug:action:fal-upscaler', + ], + messages: [ + { + role: 'user' as const, + content: '', + attachments: [ + 'https://cdn.affine.pro/copilot-test/Zkas098lkjdf-908231.jpg', + ], + }, + ], + verifier: (t: ExecutionContext, link: string) => { + t.truthy(checkUrl(link), 'should be a valid url'); + }, + type: 'image' as const, + }, +]; + +export const ProviderWorkflowTestCase = [ + { + name: 'brainstorm', + content: 'apple company', + verifier: (t: ExecutionContext, result: string) => { + t.assert(checkMDList(result), 'should be a markdown list'); + }, + }, + { + name: 'presentation', + content: 'apple company', + verifier: (t: ExecutionContext, result: string) => { + for (const l of result.split('\n')) { + t.notThrows(() => { + JSON.parse(l.trim()); + }, 'should be valid json'); + } + }, + }, +]; diff --git a/tests/affine-cloud/e2e/collaboration.spec.ts b/tests/affine-cloud/cloud/e2e/collaboration.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/collaboration.spec.ts rename to tests/affine-cloud/cloud/e2e/collaboration.spec.ts diff --git a/tests/affine-cloud/e2e/fixtures/0.9.0-canary.9-snapshots.sql b/tests/affine-cloud/cloud/e2e/fixtures/0.9.0-canary.9-snapshots.sql similarity index 100% rename from tests/affine-cloud/e2e/fixtures/0.9.0-canary.9-snapshots.sql rename to tests/affine-cloud/cloud/e2e/fixtures/0.9.0-canary.9-snapshots.sql diff --git a/tests/affine-cloud/e2e/login.spec.ts b/tests/affine-cloud/cloud/e2e/login.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/login.spec.ts rename to tests/affine-cloud/cloud/e2e/login.spec.ts diff --git a/tests/affine-cloud/e2e/logo.svg b/tests/affine-cloud/cloud/e2e/logo.svg similarity index 100% rename from tests/affine-cloud/e2e/logo.svg rename to tests/affine-cloud/cloud/e2e/logo.svg diff --git a/tests/affine-cloud/e2e/migration.spec.ts b/tests/affine-cloud/cloud/e2e/migration.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/migration.spec.ts rename to tests/affine-cloud/cloud/e2e/migration.spec.ts diff --git a/tests/affine-cloud/e2e/open-in-app.spec.ts b/tests/affine-cloud/cloud/e2e/open-in-app.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/open-in-app.spec.ts rename to tests/affine-cloud/cloud/e2e/open-in-app.spec.ts diff --git a/tests/affine-cloud/e2e/page-history.spec.ts b/tests/affine-cloud/cloud/e2e/page-history.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/page-history.spec.ts rename to tests/affine-cloud/cloud/e2e/page-history.spec.ts diff --git a/tests/affine-cloud/e2e/share-page.spec.ts b/tests/affine-cloud/cloud/e2e/share-page.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/share-page.spec.ts rename to tests/affine-cloud/cloud/e2e/share-page.spec.ts diff --git a/tests/affine-cloud/e2e/template.spec.ts b/tests/affine-cloud/cloud/e2e/template.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/template.spec.ts rename to tests/affine-cloud/cloud/e2e/template.spec.ts diff --git a/tests/affine-cloud/e2e/workspace.spec.ts b/tests/affine-cloud/cloud/e2e/workspace.spec.ts similarity index 100% rename from tests/affine-cloud/e2e/workspace.spec.ts rename to tests/affine-cloud/cloud/e2e/workspace.spec.ts diff --git a/tests/affine-cloud/package.json b/tests/affine-cloud/cloud/package.json similarity index 100% rename from tests/affine-cloud/package.json rename to tests/affine-cloud/cloud/package.json diff --git a/tests/affine-cloud/playwright.config.ts b/tests/affine-cloud/cloud/playwright.config.ts similarity index 100% rename from tests/affine-cloud/playwright.config.ts rename to tests/affine-cloud/cloud/playwright.config.ts diff --git a/tests/affine-cloud-copilot/tsconfig.json b/tests/affine-cloud/cloud/tsconfig.json similarity index 100% rename from tests/affine-cloud-copilot/tsconfig.json rename to tests/affine-cloud/cloud/tsconfig.json diff --git a/tests/affine-cloud-copilot/e2e/copilot.spec.ts b/tests/affine-cloud/copilot/e2e/copilot.spec.ts similarity index 100% rename from tests/affine-cloud-copilot/e2e/copilot.spec.ts rename to tests/affine-cloud/copilot/e2e/copilot.spec.ts diff --git a/tests/affine-cloud-copilot/package.json b/tests/affine-cloud/copilot/package.json similarity index 85% rename from tests/affine-cloud-copilot/package.json rename to tests/affine-cloud/copilot/package.json index 5f92f83865..6c5b85176b 100644 --- a/tests/affine-cloud-copilot/package.json +++ b/tests/affine-cloud/copilot/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "scripts": { - "e2e": "yarn playwright test" + "test:e2e": "yarn playwright test" }, "devDependencies": { "@affine-test/kit": "workspace:*", diff --git a/tests/affine-cloud-copilot/playwright.config.ts b/tests/affine-cloud/copilot/playwright.config.ts similarity index 98% rename from tests/affine-cloud-copilot/playwright.config.ts rename to tests/affine-cloud/copilot/playwright.config.ts index faff4e14eb..5d4c0e3899 100644 --- a/tests/affine-cloud-copilot/playwright.config.ts +++ b/tests/affine-cloud/copilot/playwright.config.ts @@ -6,6 +6,7 @@ import type { const config: PlaywrightTestConfig = { testDir: './e2e', + testMatch: '**/*.spec.ts', fullyParallel: !process.env.CI, timeout: 120_000, outputDir: testResultDir, diff --git a/tests/affine-cloud/tsconfig.json b/tests/affine-cloud/copilot/tsconfig.json similarity index 100% rename from tests/affine-cloud/tsconfig.json rename to tests/affine-cloud/copilot/tsconfig.json diff --git a/tests/kit/src/utils/cloud.ts b/tests/kit/src/utils/cloud.ts index fc4bcbbcd9..e23fd7fc49 100644 --- a/tests/kit/src/utils/cloud.ts +++ b/tests/kit/src/utils/cloud.ts @@ -155,18 +155,22 @@ export async function createRandomUser(): Promise<{ } as any; } -export async function createRandomAIUser(): Promise<{ +export async function createRandomAIUser( + provider?: string, + connector: typeof runPrisma = runPrisma +): Promise<{ name: string; email: string; password: string; id: string; + sessionId: string; }> { const user = { name: faker.internet.username(), - email: faker.internet.email().toLowerCase(), + email: faker.internet.email({ provider }).toLowerCase(), password: '123456', }; - const result = await runPrisma(async client => { + const result = await connector(async client => { const freeFeatureId = await client.feature .findFirst({ where: { feature: 'free_plan_v1' }, @@ -182,7 +186,7 @@ export async function createRandomAIUser(): Promise<{ }) .then(f => f!.id); - await client.user.create({ + const { id: userId } = await client.user.create({ data: { ...user, emailVerifiedAt: new Date(), @@ -204,11 +208,23 @@ export async function createRandomAIUser(): Promise<{ }, }); - return await client.user.findUnique({ - where: { - email: user.email, + const { id: sessionId } = await client.session.create({ data: {} }); + await client.userSession.create({ + data: { + sessionId, + userId, + // half an hour + expiresAt: new Date(Date.now() + 60 * 30 * 1000), }, }); + + return await client.user + .findUnique({ + where: { + email: user.email, + }, + }) + .then(r => ({ ...r, sessionId })); }); cloudUserSchema.parse(result); return { diff --git a/yarn.lock b/yarn.lock index 7ec9924902..6e14a35285 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,18 +19,18 @@ __metadata: languageName: node linkType: hard -"@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud-copilot": +"@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud/copilot": version: 0.0.0-use.local - resolution: "@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud-copilot" + resolution: "@affine-test/affine-cloud-copilot@workspace:tests/affine-cloud/copilot" dependencies: "@affine-test/kit": "workspace:*" "@playwright/test": "npm:=1.49.1" languageName: unknown linkType: soft -"@affine-test/affine-cloud@workspace:tests/affine-cloud": +"@affine-test/affine-cloud@workspace:tests/affine-cloud/cloud": version: 0.0.0-use.local - resolution: "@affine-test/affine-cloud@workspace:tests/affine-cloud" + resolution: "@affine-test/affine-cloud@workspace:tests/affine-cloud/cloud" dependencies: "@affine-test/kit": "workspace:*" "@playwright/test": "npm:=1.49.1"