[Benchmark] Drop memory bound kernels and replace them with gemms (#887)

Author: oulgen

.github/workflows/benchmark.yml

@@ -120,7 +120,7 @@ jobs:
 
           source .venv/bin/activate
 
-          KERNELS=("softmax" "geglu" "swiglu" "jsd" "welford" "kl_div" "int4_gemm" "layer_norm" "layer_norm-bwd" "rms_norm" "rms_norm-bwd" "cross_entropy" "flash_attention")
+          KERNELS=("softmax" "jsd" "welford" "kl_div" "layer_norm" "layer_norm-bwd" "rms_norm" "rms_norm-bwd" "cross_entropy" "flash_attention" "gemm" "grouped_gemm")
           NUMSHARDS=${{ inputs.num-shards }}
           SHARD=${{ inputs.shard }}
 

.github/workflows/benchmark_dispatch.yml

@@ -26,7 +26,7 @@ jobs:
     if: ${{ github.event.inputs.run_h100 == 'true' || github.event_name == 'schedule' }}
     uses: ./.github/workflows/compute-benchmark-matrix.yml
     with:
-      max-runners: 13
+      max-runners: 12
 
   run-h100:
     needs: gen-matrix-h100
@@ -51,7 +51,7 @@ jobs:
     uses: ./.github/workflows/compute-benchmark-matrix.yml
     if: ${{ github.event.inputs.run_b200 == 'true' || github.event_name == 'schedule' }}
     with:
-      max-runners: 13
+      max-runners: 12
 
   run-b200:
     needs: gen-matrix-b200