Merge pull request #450 from tgross35/benchmark-soft-float

Set `force-soft-floats` for benchmarks

Author: tgross35

.github/workflows/main.yaml

@@ -170,29 +170,8 @@ jobs:
     - name: Run icount benchmarks
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        set -eux
-        iai_home="iai-home"
-        # Download the baseline from master
-        ./ci/ci-util.py locate-baseline --download --extract
-
-        # Run iai-callgrind benchmarks
-        cargo bench --no-default-features \
-          --features unstable,unstable-float,icount \
-          --bench icount \
-          -- \
-          --save-baseline=default \
-          --home "$(pwd)/$iai_home" \
-          --regression='ir=5.0' \
-          --save-summary
-        # NB: iai-callgrind should exit on error but does not, so we inspect the sumary
-        # for errors. See  https://github.com/iai-callgrind/iai-callgrind/issues/337
-        ./ci/ci-util.py check-regressions "$iai_home"
-
-        # Name and tar the new baseline
-        name="baseline-icount-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
-        echo "BASELINE_NAME=$name" >> "$GITHUB_ENV"
-        tar cJf "$name.tar.xz" "$iai_home"
+        PR_NUMBER: ${{ github.event.pull_request.number }}
+      run: ./ci/bench-icount.sh
 
     - name: Upload the benchmark baseline
       uses: actions/upload-artifact@v4
@@ -205,7 +184,7 @@ jobs:
         # Always use the same seed for benchmarks. Ideally we should switch to a
         # non-random generator.
         export LIBM_SEED=benchesbenchesbenchesbencheswoo!
-        cargo bench --all --features libm-test/short-benchmarks,libm-test/build-musl
+        cargo bench --all --features short-benchmarks,build-musl,force-soft-floats
 
     - name: Print test logs if available
       if: always()

.gitignore

@@ -6,3 +6,7 @@ target
 Cargo.lock
 musl/
 **.tar.gz
+
+# Benchmark cache
+iai-home
+baseline-*

ci/bench-icount.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -eux
+
+iai_home="iai-home"
+
+# Download the baseline from master
+./ci/ci-util.py locate-baseline --download --extract
+
+# Run benchmarks once 
+function run_icount_benchmarks() {
+    cargo_args=(
+        "--bench" "icount"
+        "--no-default-features"
+        "--features" "unstable,unstable-float,icount"
+    )
+
+    iai_args=(
+        "--home" "$(pwd)/$iai_home"
+        "--regression=ir=5.0"
+        "--save-summary"
+    )
+
+    # Parse `cargo_arg0 cargo_arg1 -- iai_arg0 iai_arg1` syntax
+    parsing_iai_args=0
+    while [ "$#" -gt 0 ]; do
+        if [ "$parsing_iai_args" == "1" ]; then
+            iai_args+=("$1")
+        elif [ "$1" == "--" ]; then
+            parsing_iai_args=1
+        else
+            cargo_args+=("$1")
+        fi
+
+        shift
+    done
+
+    # Run iai-callgrind benchmarks
+    cargo bench "${cargo_args[@]}" -- "${iai_args[@]}"
+
+    # NB: iai-callgrind should exit on error but does not, so we inspect the sumary
+    # for errors. See  https://github.com/iai-callgrind/iai-callgrind/issues/337
+    if [ -n "${PR_NUMBER:-}" ]; then
+        # If this is for a pull request, ignore regressions if specified.
+        ./ci/ci-util.py check-regressions --home "$iai_home" --allow-pr-override "$PR_NUMBER"
+    else
+        ./ci/ci-util.py check-regressions --home "$iai_home" || true
+    fi
+}
+
+# Run once with softfloats, once with arch instructions enabled
+run_icount_benchmarks --features force-soft-floats -- --save-baseline=softfloat
+run_icount_benchmarks -- --save-baseline=hardfloat
+
+# Name and tar the new baseline
+name="baseline-icount-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
+echo "BASELINE_NAME=$name" >>"$GITHUB_ENV"
+tar cJf "$name.tar.xz" "$iai_home"

ci/ci-util.py

@@ -33,11 +33,14 @@
 
             Note that `--extract` will overwrite files in `iai-home`.
 
-        check-regressions [iai-home]
+        check-regressions [--home iai-home] [--allow-pr-override pr_number]
             Check `iai-home` (or `iai-home` if unspecified) for `summary.json`
             files and see if there are any regressions. This is used as a workaround
             for `iai-callgrind` not exiting with error status; see
             <https://github.com/iai-callgrind/iai-callgrind/issues/337>.
+
+            If `--allow-pr-override` is specified, the regression check will not exit
+            with failure if any line in the PR starts with `allow-regressions`.
     """
 )
 
@@ -46,6 +49,8 @@
 DEFAULT_BRANCH = "master"
 WORKFLOW_NAME = "CI"  # Workflow that generates the benchmark artifacts
 ARTIFACT_GLOB = "baseline-icount*"
+# Place this in a PR body to skip regression checks (must be at the start of a line).
+REGRESSION_DIRECTIVE = "ci: allow-regressions"
 
 # Don't run exhaustive tests if these files change, even if they contaiin a function
 # definition.
@@ -256,12 +261,26 @@ def locate_baseline(flags: list[str]) -> None:
     eprint("baseline extracted successfully")
 
 
-def check_iai_regressions(iai_home: str | None | Path):
+def check_iai_regressions(args: list[str]):
     """Find regressions in iai summary.json files, exit with failure if any are
     found.
     """
-    if iai_home is None:
-        iai_home = "iai-home"
+
+    iai_home = "iai-home"
+    pr_number = False
+
+    while len(args) > 0:
+        match args:
+            case ["--home", home, *rest]:
+                iai_home = home
+                args = rest
+            case ["--allow-pr-override", pr_num, *rest]:
+                pr_number = pr_num
+                args = rest
+            case _:
+                eprint(USAGE)
+                exit(1)
+
     iai_home = Path(iai_home)
 
     found_summaries = False
@@ -286,9 +305,33 @@ def check_iai_regressions(iai_home: str | None | Path):
         eprint(f"did not find any summary.json files within {iai_home}")
         exit(1)
 
-    if len(regressions) > 0:
-        eprint("Found regressions:", json.dumps(regressions, indent=4))
-        exit(1)
+    if len(regressions) == 0:
+        eprint("No regressions found")
+        return
+
+    eprint("Found regressions:", json.dumps(regressions, indent=4))
+
+    if pr_number is not None:
+        pr_info = sp.check_output(
+            [
+                "gh",
+                "pr",
+                "view",
+                str(pr_number),
+                "--json=number,commits,body,createdAt",
+                "--jq=.commits |= map(.oid)",
+            ],
+            text=True,
+        )
+        pr = json.loads(pr_info)
+        eprint("PR info:", json.dumps(pr, indent=4))
+
+        lines = pr["body"].splitlines()
+        if any(line.startswith(REGRESSION_DIRECTIVE) for line in lines):
+            eprint("PR allows regressions, returning")
+            return
+
+    exit(1)
 
 
 def main():
@@ -299,10 +342,8 @@ def main():
             print(f"matrix={output}")
         case ["locate-baseline", *flags]:
             locate_baseline(flags)
-        case ["check-regressions"]:
-            check_iai_regressions(None)
-        case ["check-regressions", iai_home]:
-            check_iai_regressions(iai_home)
+        case ["check-regressions", *args]:
+            check_iai_regressions(args)
         case ["--help" | "-h"]:
             print(USAGE)
             exit()