Add benchmarks to CI

Author: adam-fowler

.github/workflows/benchmark.yml

@@ -0,0 +1,95 @@
+name: Benchmark PR vs main
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+    - '**.swift'
+    - '**.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-benchmark
+  cancel-in-progress: true
+    
+jobs:
+  benchmark-delta:
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    continue-on-error: true
+    permissions:
+      issues: write
+      pull-requests: write
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Ubuntu deps
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo apt-get install -y libjemalloc-dev
+
+      - name: Git URL token override and misc
+        run: |
+          #git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com"
+          #/usr/bin/ordo-performance
+          [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV
+          echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
+      - name: Run benchmarks for PR branch
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request
+      - name: Switch to branch 'main'
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          git stash
+          git checkout main
+      - name: Run benchmarks for branch 'main'
+        if: ${{ env.hasBenchmark == '1' }}
+        run: |
+          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main
+      - name: Compare PR and main
+        if: ${{ env.hasBenchmark == '1' }}
+        id: benchmark
+        run: |
+          echo '## Summary' >> $GITHUB_STEP_SUMMARY
+          echo $(date) >> $GITHUB_STEP_SUMMARY
+          echo "exitStatus=1" >> $GITHUB_ENV
+          swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY
+          echo '---' >> $GITHUB_STEP_SUMMARY
+          swift package benchmark baseline compare main pull_request --no-progress --quiet --format markdown >> $GITHUB_STEP_SUMMARY
+          echo "exitStatus=0" >> $GITHUB_ENV
+        continue-on-error: true
+      - if: ${{ env.exitStatus == '0' }}
+        name: Pull request comment text success
+        id: prtestsuccess
+        run: |
+          echo 'PRTEST<<EOF' >> $GITHUB_ENV
+          echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/hummingbird-project/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - if: ${{ env.exitStatus == '1' }}
+        name: Pull request comment text failure
+        id: prtestfailure
+        run: |
+          echo 'PRTEST<<EOF' >> $GITHUB_ENV
+          echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/hummingbird-project/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV
+          echo "_Pull request had performance regressions_" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Comment PR
+        if: ${{ env.hasBenchmark == '1' }}
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          message: ${{ env.PRTEST }}
+          comment-tag: benchmark
+      - name: Exit with correct status
+        run: |
+          exit ${{ env.exitStatus }}

.gitignore

@@ -7,4 +7,5 @@ DerivedData/
 .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
 .netrc
 .vscode
-Package.resolved
+Package.resolved
+.benchmarkBaselines/

Benchmarks/ValkeyBenchmarks/ValkeyBenchmarks.swift

@@ -13,19 +13,27 @@
 //===----------------------------------------------------------------------===//
 
 import Benchmark
+import Foundation
 import Logging
 import NIOCore
 import NIOPosix
 import Valkey
 
-let benchmarks : @Sendable () -> Void = {
-    let defaultMetrics: [BenchmarkMetric] = [
-        .wallClock,
-        .cpuTotal,
-        .mallocCountTotal,
-        .throughput,
-        .instructions,
-    ]
+let benchmarks: @Sendable () -> Void = {
+    let defaultMetrics: [BenchmarkMetric] =
+        // There is no point comparing wallClock, cpuTotal or throughput on CI as they are too inconsistent
+        ProcessInfo.processInfo.environment["CI"] != nil
+        ? [
+            .mallocCountTotal,
+            .instructions,
+        ]
+        : [
+            .wallClock,
+            .cpuTotal,
+            .mallocCountTotal,
+            .throughput,
+            .instructions,
+        ]
 
     var server: Channel?
     Benchmark("GET benchmark", configuration: .init(metrics: defaultMetrics, scalingFactor: .kilo)) { benchmark in