Testing benchmark CI improvements #40
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark PR vs main | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - '**.swift' | |
| - '**.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-benchmark | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark-delta: | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 15 | |
| continue-on-error: true | |
| permissions: | |
| issues: write | |
| pull-requests: write | |
| contents: read | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| image: ["swift:6.1"] | |
| container: | |
| image: ${{ matrix.image }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install jemalloc | |
| run: | | |
| apt-get -q update | |
| apt-get install -y libjemalloc-dev | |
| # https://github.com/actions/checkout/issues/766 | |
| - name: Mark the workspace as safe | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Miscellaneous | |
| run: | | |
| [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV | |
| echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH | |
| - name: Run benchmarks for PR branch | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request | |
| - name: Checkout main | |
| run: | | |
| git checkout main | |
| - name: Run benchmarks for branch 'main' | |
| if: ${{ env.hasBenchmark == '1' }} | |
| run: | | |
| swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main | |
| - name: Compare PR and main | |
| if: ${{ env.hasBenchmark == '1' }} | |
| id: benchmark | |
| run: | | |
| echo '## Summary' >> $GITHUB_STEP_SUMMARY | |
| echo $(date) >> $GITHUB_STEP_SUMMARY | |
| set +e | |
| swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY | |
| echo "exit-status=$?" >> $GITHUB_OUTPUT | |
| set -e | |
| - name: Pull request comment text | |
| id: benchmark-comment | |
| run: | | |
| echo '---' >> $GITHUB_STEP_SUMMARY | |
| swift package benchmark baseline compare main pull_request --no-progress --quiet --format markdown >> $GITHUB_STEP_SUMMARY | |
| echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/adam-fowler/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> comment.md | |
| EXIT_CODE='${{steps.benchmark.outputs.exit-status}}' | |
| echo "Exit code $EXIT_CODE" | |
| case "${EXIT_CODE}" in | |
| 0) | |
| echo "_Pull request no significant performance differences ✅_" >> comment.md | |
| echo "exitStatus=0" >> $GITHUB_ENV | |
| ;; | |
| 1) | |
| echo "_Pull request has significant performance differences ❌_" >> comment.md | |
| echo "exitStatus=1" >> $GITHUB_ENV | |
| ;; | |
| 2) | |
| echo "_Pull request has performance regressions ❌_" >> comment.md | |
| echo "exitStatus=1" >> $GITHUB_ENV | |
| ;; | |
| 4) | |
| echo "_Pull request has performance improvements ✅_" >> comment.md | |
| echo "exitStatus=0" >> $GITHUB_ENV | |
| ;; | |
| *) | |
| echo "_Benchmark comparison failed to complete with exit code $EXIT_CODE ❌_" >> comment.md | |
| echo "exitStatus=1" >> $GITHUB_ENV | |
| ;; | |
| esac | |
| - name: Comment PR | |
| if: ${{ env.hasBenchmark == '1' }} | |
| uses: thollander/actions-comment-pull-request@v3 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| file-path: comment.md | |
| comment-tag: benchmark | |
| - name: Exit with correct status | |
| run: | | |
| exit ${{ env.exitStatus }} |