mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-11 11:58:41 +00:00
feat(server): add real-world prompt test for copilot apis (#8629)
fix AF-1432, PD-1176
This commit is contained in:
darkskygit
162
.github/workflows/build-test.yml
vendored
162
.github/workflows/build-test.yml
vendored
@@ -328,6 +328,7 @@ jobs:
|
||||
env:
|
||||
NODE_ENV: test
|
||||
DISTRIBUTION: web
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
@@ -360,27 +361,13 @@ jobs:
|
||||
name: server-native.node
|
||||
path: ./packages/backend/server
|
||||
|
||||
- name: Initialize database
|
||||
run: |
|
||||
psql -h localhost -U postgres -c "CREATE DATABASE affine;"
|
||||
psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
|
||||
psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
|
||||
env:
|
||||
PGPASSWORD: affine
|
||||
|
||||
- name: Run init-db script
|
||||
run: |
|
||||
yarn workspace @affine/server exec prisma generate
|
||||
yarn workspace @affine/server exec prisma db push
|
||||
yarn workspace @affine/server data-migration run
|
||||
env:
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
- name: Prepare Server Test Environment
|
||||
uses: ./.github/actions/server-test-env
|
||||
|
||||
- name: Run server tests
|
||||
run: yarn workspace @affine/server test:coverage
|
||||
env:
|
||||
CARGO_TARGET_DIR: '${{ github.workspace }}/target'
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
COPILOT_OPENAI_API_KEY: 'use_fake_openai_api_key'
|
||||
|
||||
- name: Upload server test coverage results
|
||||
@@ -392,6 +379,133 @@ jobs:
|
||||
name: affine
|
||||
fail_ci_if_error: false
|
||||
|
||||
copilot-api-test:
|
||||
name: Server Copilot Api Test
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-server-native
|
||||
env:
|
||||
NODE_ENV: test
|
||||
DISTRIBUTION: web
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_PASSWORD: affine
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
mailer:
|
||||
image: mailhog/mailhog
|
||||
ports:
|
||||
- 1025:1025
|
||||
- 8025:8025
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: ./.github/actions/setup-node
|
||||
with:
|
||||
electron-install: false
|
||||
full-cache: true
|
||||
|
||||
- name: Download server-native.node
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: server-native.node
|
||||
path: ./packages/backend/server
|
||||
|
||||
- name: Prepare Server Test Environment
|
||||
uses: ./.github/actions/server-test-env
|
||||
|
||||
- name: Run server tests
|
||||
run: yarn workspace @affine/server test:copilot:coverage --forbid-only
|
||||
env:
|
||||
CARGO_TARGET_DIR: '${{ github.workspace }}/target'
|
||||
COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
|
||||
COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
|
||||
|
||||
- name: Upload server test coverage results
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./packages/backend/server/.coverage/lcov.info
|
||||
flags: server-test
|
||||
name: affine
|
||||
fail_ci_if_error: false
|
||||
|
||||
copilot-e2e-test:
|
||||
name: Server Copilot E2E Test
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DISTRIBUTION: web
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
IN_CI_TEST: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2, 3]
|
||||
shardTotal: [3]
|
||||
needs:
|
||||
- build-server-native
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_PASSWORD: affine
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Check blocksuite update
|
||||
id: check-blocksuite-update
|
||||
env:
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
run: |
|
||||
if [ -n "$BASE_REF" ]; then
|
||||
if node ./scripts/detect-blocksuite-update.mjs "$BASE_REF"; then
|
||||
echo "skip=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "skip=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
else
|
||||
echo "skip=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Setup Node.js
|
||||
if: ${{ steps.check-blocksuite-update.outputs.skip == 'true' }}
|
||||
uses: ./.github/actions/setup-node
|
||||
with:
|
||||
playwright-install: true
|
||||
electron-install: false
|
||||
hard-link-nm: false
|
||||
|
||||
- name: Download server-native.node
|
||||
if: ${{ steps.check-blocksuite-update.outputs.skip == 'true' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: server-native.node
|
||||
path: ./packages/backend/server
|
||||
|
||||
- name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
if: ${{ steps.check-blocksuite-update.outputs.skip == 'true' }}
|
||||
uses: ./.github/actions/copilot-test
|
||||
with:
|
||||
script: yarn workspace @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
|
||||
fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
|
||||
|
||||
server-e2e-test:
|
||||
name: ${{ matrix.tests.name }}
|
||||
runs-on: ubuntu-latest
|
||||
@@ -458,19 +572,8 @@ jobs:
|
||||
name: affine.linux-x64-gnu.node
|
||||
path: ./packages/frontend/native
|
||||
|
||||
- name: Initialize database
|
||||
run: |
|
||||
psql -h localhost -U postgres -c "CREATE DATABASE affine;"
|
||||
psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
|
||||
psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
|
||||
env:
|
||||
PGPASSWORD: affine
|
||||
|
||||
- name: Run init-db script
|
||||
run: |
|
||||
yarn workspace @affine/server exec prisma generate
|
||||
yarn workspace @affine/server exec prisma db push
|
||||
yarn workspace @affine/server data-migration run
|
||||
- name: Prepare Server Test Environment
|
||||
uses: ./.github/actions/server-test-env
|
||||
|
||||
- name: ${{ matrix.tests.name }}
|
||||
run: |
|
||||
@@ -633,6 +736,7 @@ jobs:
|
||||
- build-server-native
|
||||
- build-electron-renderer
|
||||
- server-test
|
||||
- copilot-e2e-test
|
||||
- server-e2e-test
|
||||
- desktop-test
|
||||
- test-build-mobile-app
|
||||
|
||||
29
.github/workflows/copilot-test-automatically.yml
vendored
Normal file
29
.github/workflows/copilot-test-automatically.yml
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
name: Copilot Test Automatically
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]+.[0-9]+.[0-9]+-canary.[0-9]+'
|
||||
schedule:
|
||||
- cron: '0 8 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
|
||||
jobs:
|
||||
dispatch-test:
|
||||
runs-on: ubuntu-latest
|
||||
name: Setup Test
|
||||
steps:
|
||||
- name: dispatch test by tag
|
||||
if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
|
||||
uses: benc-uk/workflow-dispatch@v1
|
||||
with:
|
||||
workflow: copilot-test.yml
|
||||
- name: dispatch test by schedule
|
||||
if: ${{ github.event_name == 'schedule' }}
|
||||
uses: benc-uk/workflow-dispatch@v1
|
||||
with:
|
||||
workflow: copilot-test.yml
|
||||
ref: canary
|
||||
190
.github/workflows/copilot-test.yml
vendored
Normal file
190
.github/workflows/copilot-test.yml
vendored
Normal file
@@ -0,0 +1,190 @@
|
||||
name: Copilot Cron Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
|
||||
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/node_modules/.cache/ms-playwright
|
||||
|
||||
jobs:
|
||||
build-server-native:
|
||||
name: Build Server native
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CARGO_PROFILE_RELEASE_DEBUG: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Node.js
|
||||
uses: ./.github/actions/setup-node
|
||||
with:
|
||||
extra-flags: workspaces focus @affine/server-native
|
||||
electron-install: false
|
||||
- name: Build Rust
|
||||
uses: ./.github/actions/build-rust
|
||||
with:
|
||||
target: 'x86_64-unknown-linux-gnu'
|
||||
package: '@affine/server-native'
|
||||
nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
|
||||
- name: Upload server-native.node
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: server-native.node
|
||||
path: ./packages/backend/native/server-native.node
|
||||
if-no-files-found: error
|
||||
|
||||
copilot-api-test:
|
||||
name: Server Copilot Api Test
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-server-native
|
||||
env:
|
||||
NODE_ENV: test
|
||||
DISTRIBUTION: web
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_PASSWORD: affine
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
mailer:
|
||||
image: mailhog/mailhog
|
||||
ports:
|
||||
- 1025:1025
|
||||
- 8025:8025
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: ./.github/actions/setup-node
|
||||
with:
|
||||
playwright-install: true
|
||||
electron-install: false
|
||||
full-cache: true
|
||||
|
||||
- name: Download server-native.node
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: server-native.node
|
||||
path: ./packages/backend/server
|
||||
|
||||
- name: Prepare Server Test Environment
|
||||
uses: ./.github/actions/server-test-env
|
||||
|
||||
- name: Run server tests
|
||||
run: yarn workspace @affine/server test:copilot:coverage --forbid-only
|
||||
env:
|
||||
CARGO_TARGET_DIR: '${{ github.workspace }}/target'
|
||||
COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
|
||||
COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
|
||||
|
||||
- name: Upload server test coverage results
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./packages/backend/server/.coverage/lcov.info
|
||||
flags: server-test
|
||||
name: affine
|
||||
fail_ci_if_error: false
|
||||
|
||||
copilot-e2e-test:
|
||||
name: Server Copilot E2E Test
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DISTRIBUTION: web
|
||||
DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
|
||||
IN_CI_TEST: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2, 3]
|
||||
shardTotal: [3]
|
||||
needs:
|
||||
- build-server-native
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_PASSWORD: affine
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: ./.github/actions/setup-node
|
||||
with:
|
||||
playwright-install: true
|
||||
electron-install: false
|
||||
hard-link-nm: false
|
||||
|
||||
- name: Download server-native.node
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: server-native.node
|
||||
path: ./packages/backend/server
|
||||
|
||||
- name: Run Copilot E2E Test ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
uses: ./.github/actions/copilot-test
|
||||
with:
|
||||
script: yarn workspace @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
|
||||
fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
|
||||
|
||||
test-done:
|
||||
needs:
|
||||
- copilot-api-test
|
||||
- copilot-e2e-test
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
name: Post test result message
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Setup Node.js
|
||||
uses: ./.github/actions/setup-node
|
||||
with:
|
||||
extra-flags: 'workspaces focus @affine/copilot-result'
|
||||
electron-install: false
|
||||
- name: Post Success event to a Slack channel
|
||||
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
|
||||
run: node ./tools/copilot-result/index.js
|
||||
env:
|
||||
CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
|
||||
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
BRANCH_SHA: ${{ github.sha }}
|
||||
BRANCH_NAME: ${{ github.ref }}
|
||||
COPILOT_RESULT: success
|
||||
- name: Post Failed event to a Slack channel
|
||||
id: failed-slack
|
||||
if: ${{ always() && contains(needs.*.result, 'failure') }}
|
||||
run: node ./tools/copilot-result/index.js
|
||||
env:
|
||||
CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
|
||||
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
BRANCH_SHA: ${{ github.sha }}
|
||||
BRANCH_NAME: ${{ github.ref }}
|
||||
COPILOT_RESULT: failed
|
||||
- name: Post Cancel event to a Slack channel
|
||||
id: cancel-slack
|
||||
if: ${{ always() && contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
|
||||
run: node ./tools/copilot-result/index.js
|
||||
env:
|
||||
CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
|
||||
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
BRANCH_SHA: ${{ github.sha }}
|
||||
BRANCH_NAME: ${{ github.ref }}
|
||||
COPILOT_RESULT: canceled
|
||||
Reference in New Issue
Block a user