wip

Signed-off-by: Spencer Schrock <[email protected]>

Author: spencerschrock

benchmarks/Containerfile

@@ -14,17 +14,12 @@
 
 FROM python:3.13-slim
 
-RUN apt-get update && apt-get install --no-install-recommends --yes \
-    clang \
-    g++ \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN python -m pip install --no-cache-dir hatch
+RUN python -m pip install --no-cache-dir hatch huggingface_hub[cli]
 
 COPY pyproject.toml LICENSE README.md ./
 COPY src ./src
 COPY benchmarks ./benchmarks
 
-RUN hatch env create bench
+RUN hatch env create bench.py3.11
 
 ENTRYPOINT [ "bash" ]

benchmarks/cloud_batch.json

@@ -5,25 +5,25 @@
                 "runnables": [
                     {
                         "container": {
-                            "imageUri": "ghcr.io/spencerschrock/model-transparency-benchmarks:latest",
+                            "imageUri": "ghcr.io/spencerschrock/model-transparency-benchmarks:${TAG}",
                             "entrypoint": "/bin/sh",
                             "commands": [
                                 "-c",
-                                "benchmarks/run.sh | tee /mnt/disks/share/$(date --utc +%Y_%m_%d_%H%M%S).json"
+                                "benchmarks/run.sh ${MODEL} /mnt/disks/models /mnt/disks/gcs/${OUTPUT_FILE}"
                             ]
                         }
                     }
                 ],
                 "computeResource": {
                     "cpuMilli": 16000,
-                    "memoryMib": 61440
+                    "memoryMib": 65536
                 },
                 "volumes": [
                     {
                     "gcs": {
-                        "remotePath": "gosst-scare-sandbox-batch-test"
+                        "remotePath": "${BUCKET}"
                     },
-                    "mountPath": "/mnt/disks/share"
+                    "mountPath": "/mnt/disks/gcs"
                     },
                     {
                         "deviceName": "models",
@@ -42,7 +42,7 @@
         "instances": [
             {
                 "policy": { 
-                    "machineType": "n1-standard-16",
+                    "machineType": "c2d-standard-16",
                     "disks": [
                         {
                             "newDisk": {

benchmarks/run.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 set -euxo pipefail
 
-MODEL=deepseek-ai/DeepSeek-R1-Distill-Llama-70B
-MODEL_PATH=/mnt/disks/models/$(echo $MODEL | cut --delimiter='/' --fields=2-)
+MODEL=$1
+MODEL_DIR=$2
+MODEL_PATH=$MODEL_DIR/$(echo $MODEL | cut --delimiter='/' --fields=2-)
+OUTPUT_FILE=$3
 
-pip install huggingface_hub[cli]
 huggingface-cli download $MODEL --local-dir "$MODEL_PATH"
-hatch run bench.py3.11:python benchmarks/time_serialize.py "$MODEL_PATH"
+hatch run bench.py3.11:python benchmarks/time_serialize.py "$MODEL_PATH" \
+    --output=$OUTPUT_FILE

benchmarks/time_serialize.py

@@ -65,7 +65,7 @@ def build_parser() -> argparse.ArgumentParser:
     results["cpu"] = cpuinfo.get_cpu_info()
 
     if args.output:
-        with open("data.json", "w", encoding="utf-8") as f:
+        with open(args.output, "w", encoding="utf-8") as f:
             json.dump(results, f, ensure_ascii=False, indent=4)
     else:
             json.dump(results, sys.stdout, ensure_ascii=False, indent=4)