diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1420e4e9..2c0f8f97 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -55,7 +55,7 @@ jobs: with: containerfiles: | ./benchmarks/Containerfile - image: ghcr.io/sigstore/model-transparency-benchmarks + image: ghcr.io/spencerschrock/model-transparency-benchmarks tags: "latest ${{ steps.config.outputs.head }}" archs: amd64 oci: false @@ -75,28 +75,26 @@ jobs: image: ${{ steps.build_image.outputs.image }} tags: ${{ steps.build_image.outputs.tags }} registry: ghcr.io - submit-cloud-batch: - needs: publish-benchmark-container - runs-on: ubuntu-latest - permissions: - id-token: 'write' - env: - MODEL: deepseek-ai/DeepSeek-R1-Distill-Qwen-14B - TAG: ${{needs.publish-benchmark-container.outputs.head}} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{needs.publish-benchmark-container.outputs.head}} - - uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 - with: - workload_identity_provider: projects/306323169285/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider - service_account: 'model-transparency-gha@sigstore-infra-playground.iam.gserviceaccount.com' - - run: | - export OUTPUT_FILE=$(date --utc +%Y%m%d%H%M%S)_$TAG.json - gcloud batch jobs submit \ - --job-prefix=bench \ - --project sigstore-infra-playground \ - --location us-central1 \ - --config - < argparse.ArgumentParser: return parser -def _human_size(size: int) -> str: - if size >= GB: - return str(size / GB) + " GB" - elif size >= MB: - return str(size / MB) + " MB" - elif size >= KB: - return str(size / KB) + " KB" - return str(size) + " B" - - def _get_hasher(hash_algorithm: str) -> hashing.StreamingHashEngine: # TODO: Once Python 3.9 support is deprecated revert to using `match` if hash_algorithm == "sha256": diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 00000000..97d26c81 --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euxo pipefail + +MODEL_DIR=$1 +OUTPUT_DIR=$2 +REVISION=$3 +FILENAME_BASE=$OUTPUT_DIR/$(date --utc +%Y%m%d%H%M%S)_$REVISION + +for SIZE in 32 256; do + for FILES in 64 512; do + MODEL=${SIZE}gb_${FILES}files + MODEL_PATH=$MODEL_DIR/$MODEL + mkdir -p "$MODEL_PATH" + SIZE_BYTES=$((SIZE * 1024 * 1024 * 1024)) + hatch run bench.py3.11:generate dir --root "$MODEL_PATH" -n "$FILES" "$SIZE_BYTES" + hatch run bench.py3.11:python benchmarks/time_serialize.py "$MODEL_PATH" \ + --output="${FILENAME_BASE}_${MODEL}.json" + rm -r "${MODEL_PATH}" + done +done + +