From 554a8d9fb865c1c41e78a063d0a1d65556661774 Mon Sep 17 00:00:00 2001 From: Brooooooklyn Date: Thu, 27 Mar 2025 15:56:24 +0000 Subject: [PATCH] ci: split jobs to speed (#11249) --- .github/workflows/build-test.yml | 199 ++++++++++++++++++++------ .github/workflows/release-desktop.yml | 4 +- .github/workflows/release-mobile.yml | 8 +- 3 files changed, 163 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9573b32532..b8099813dc 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -42,6 +42,8 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest + env: + NODE_OPTIONS: --max-old-space-size=14384 needs: optimize_ci if: needs.optimize_ci.outputs.skip == 'false' permissions: @@ -53,44 +55,30 @@ jobs: fail-fast: false matrix: language: ['javascript', 'typescript'] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + project: ['affine', 'blocksuite'] steps: - name: Checkout repository uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + source-root: ${{ matrix.project == 'affine' && '.' || 'blocksuite' }} - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + - name: Delete blocksuite before codeql analysis + if: ${{ matrix.project == 'affine' }} + run: rm -rf blocksuite - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 lint: name: Lint - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm needs: optimize_ci if: needs.optimize_ci.outputs.skip == 'false' @@ -115,6 +103,23 @@ jobs: yarn lint:prettier - name: Yarn Dedupe run: yarn dedupe --check + + typecheck: + name: Typecheck + runs-on: ubuntu-24.04-arm + needs: optimize_ci + if: needs.optimize_ci.outputs.skip == 'false' + env: + NODE_OPTIONS: --max-old-space-size=14384 + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + electron-install: false + full-cache: true + - name: Run i18n codegen + run: yarn affine @affine/i18n build - name: Run Type Check run: yarn typecheck - name: Run BS Docs Build @@ -219,12 +224,13 @@ jobs: e2e-test: name: E2E Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm needs: optimize_ci if: needs.optimize_ci.outputs.skip == 'false' env: DISTRIBUTION: web IN_CI_TEST: true + NODE_OPTIONS: --max-old-space-size=14384 strategy: fail-fast: false matrix: @@ -334,21 +340,11 @@ jobs: matrix: spec: - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu } - - { os: windows-latest, target: x86_64-pc-windows-msvc } - { os: macos-latest, target: x86_64-apple-darwin } - { os: macos-latest, target: aarch64-apple-darwin } steps: - uses: actions/checkout@v4 - - uses: samypr100/setup-dev-drive@v3 - if: ${{ matrix.spec.os == 'windows-latest' }} - with: - workspace-copy: true - drive-size: 8GB - drive-format: NTFS - env-mapping: | - CARGO_HOME,{{ DEV_DRIVE }}/.cargo - RUSTUP_HOME,{{ DEV_DRIVE }}/.rustup - name: Setup Node.js uses: ./.github/actions/setup-node with: @@ -356,7 +352,7 @@ jobs: electron-install: false - name: Setup filename id: filename - working-directory: ${{ env.DEV_DRIVE_WORKSPACE || github.workspace }} + working-directory: ${{ github.workspace }} shell: bash run: | export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)") @@ -371,7 +367,58 @@ jobs: if: always() with: name: ${{ steps.filename.outputs.filename }} - path: ${{ env.DEV_DRIVE_WORKSPACE || github.workspace }}/packages/frontend/native/${{ steps.filename.outputs.filename }} + path: ${{ github.workspace }}/packages/frontend/native/${{ steps.filename.outputs.filename }} + if-no-files-found: error + + # Split Windows build because it's too slow + # and other ci jobs required linux native + build-windows-native: + name: Build AFFiNE native (${{ matrix.spec.target }}) + runs-on: ${{ matrix.spec.os }} + needs: optimize_ci + if: needs.optimize_ci.outputs.skip == 'false' + env: + CARGO_PROFILE_RELEASE_DEBUG: '1' + strategy: + fail-fast: false + matrix: + spec: + - { os: windows-latest, target: x86_64-pc-windows-msvc } + - { os: windows-latest, target: aarch64-pc-windows-msvc } + + steps: + - uses: actions/checkout@v4 + - uses: samypr100/setup-dev-drive@v3 + with: + workspace-copy: true + drive-size: 8GB + drive-format: NTFS + env-mapping: | + CARGO_HOME,{{ DEV_DRIVE }}/.cargo + RUSTUP_HOME,{{ DEV_DRIVE }}/.rustup + - name: Setup Node.js + uses: ./.github/actions/setup-node + with: + extra-flags: workspaces focus @affine/native + electron-install: false + - name: Setup filename + id: filename + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} + shell: bash + run: | + export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)") + echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT" + - name: Build AFFiNE native + uses: ./.github/actions/build-rust + with: + target: ${{ matrix.spec.target }} + package: '@affine/native' + - name: Upload ${{ steps.filename.outputs.filename }} + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ steps.filename.outputs.filename }} + path: ${{ env.DEV_DRIVE_WORKSPACE }}/packages/frontend/native/${{ steps.filename.outputs.filename }} if-no-files-found: error build-server-native: @@ -972,6 +1019,79 @@ jobs: if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-latest' }} run: yarn affine @affine-test/affine-desktop e2e + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-e2e-${{ matrix.spec.os }}-${{ matrix.spec.arch }} + path: ./test-results + if-no-files-found: ignore + + desktop-bundle-check: + name: Desktop bundle check (${{ matrix.spec.os }}, ${{ matrix.spec.platform }}, ${{ matrix.spec.arch }}, ${{ matrix.spec.target }}, ${{ matrix.spec.test }}) + runs-on: ${{ matrix.spec.os }} + needs: + - optimize_ci + - build-electron-renderer + - build-native + if: needs.optimize_ci.outputs.skip == 'false' + strategy: + fail-fast: false + matrix: + spec: + - { + os: macos-latest, + platform: macos, + arch: x64, + target: x86_64-apple-darwin, + test: false, + } + - { + os: macos-latest, + platform: macos, + arch: arm64, + target: aarch64-apple-darwin, + test: true, + } + - { + os: ubuntu-latest, + platform: linux, + arch: x64, + target: x86_64-unknown-linux-gnu, + test: true, + } + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: ./.github/actions/setup-node + timeout-minutes: 10 + with: + extra-flags: workspaces focus @affine/electron @affine/monorepo @affine-test/affine-desktop @affine/nbstore @toeverything/infra + playwright-install: true + hard-link-nm: false + enableScripts: false + + - name: Setup filename + id: filename + shell: bash + run: | + export PLATFORM_ARCH_ABI=$(node -e "console.log(require('@napi-rs/cli').parseTriple('${{ matrix.spec.target }}').platformArchABI)") + echo "filename=affine.$PLATFORM_ARCH_ABI.node" >> "$GITHUB_OUTPUT" + + - name: Download ${{ steps.filename.outputs.filename }} + uses: actions/download-artifact@v4 + with: + name: ${{ steps.filename.outputs.filename }} + path: ./packages/frontend/native + + - name: Download web artifact + uses: ./.github/actions/download-web + with: + path: packages/frontend/apps/electron/resources/web-static + + - name: Build Desktop Layers + run: yarn affine @affine/electron build + - name: Make bundle (macOS) if: ${{ matrix.spec.target == 'aarch64-apple-darwin' }} env: @@ -995,18 +1115,10 @@ jobs: HOIST_NODE_MODULES: 1 - name: Output check - if: ${{ matrix.spec.os == 'macos-14' && matrix.spec.arch == 'arm64' }} + if: ${{ matrix.spec.os == 'macos-latest' && matrix.spec.arch == 'arm64' }} run: | yarn affine @affine/electron node ./scripts/macos-arm64-output-check.ts - - name: Upload test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-e2e-${{ matrix.spec.os }}-${{ matrix.spec.arch }} - path: ./test-results - if-no-files-found: ignore - test-build-mobile-app: uses: ./.github/workflows/release-mobile.yml needs: optimize_ci @@ -1022,6 +1134,7 @@ jobs: needs: - analyze - lint + - typecheck - lint-rust - check-git-status - check-yarn-binary @@ -1030,6 +1143,7 @@ jobs: - e2e-mobile-test - unit-test - build-native + - build-windows-native - build-server-native - build-electron-renderer - native-unit-test @@ -1039,6 +1153,7 @@ jobs: - copilot-api-test - copilot-e2e-test - desktop-test + - desktop-bundle-check - cloud-e2e-test - test-build-mobile-app if: always() diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 1a7c22feaf..22ee0deb17 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -73,11 +73,11 @@ jobs: fail-fast: false matrix: spec: - - runner: macos-14 + - runner: macos-latest platform: darwin arch: x64 target: x86_64-apple-darwin - - runner: macos-14 + - runner: macos-latest platform: darwin arch: arm64 target: aarch64-apple-darwin diff --git a/.github/workflows/release-mobile.yml b/.github/workflows/release-mobile.yml index 79f5634336..52849df424 100644 --- a/.github/workflows/release-mobile.yml +++ b/.github/workflows/release-mobile.yml @@ -54,7 +54,7 @@ jobs: build-ios-web: needs: - output-env - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm environment: ${{ needs.output-env.outputs.ENVIRONMENT }} outputs: RELEASE_VERSION: ${{ steps.version.outputs.APP_VERSION }} @@ -85,7 +85,7 @@ jobs: path: packages/frontend/apps/ios/dist build-android-web: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm needs: - output-env environment: ${{ needs.output-env.outputs.ENVIRONMENT }} @@ -118,7 +118,7 @@ jobs: path: packages/frontend/apps/android/dist ios: - runs-on: macos-latest + runs-on: namespace-profile-macos needs: - build-ios-web steps: @@ -139,7 +139,7 @@ jobs: enableScripts: false - uses: maxim-lobanov/setup-xcode@v1 with: - xcode-version: 16.1 + xcode-version: 16.2 - name: Install Swiftformat run: brew install swiftformat - name: Cap sync