Skip to content

Commit d8cdcce

Browse files
ggerganovggerganov
authored
ci : add self-hosted workflows (#3437)
* ci : add self-hosted workflows * cont : fail workflow if there is an error
1 parent b4909a6 commit d8cdcce

File tree

3 files changed

+276
-23
lines changed

3 files changed

+276
-23
lines changed

‎.github/workflows/build.yml‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ on:
2525
'**/*.metal',
2626
'**/*.comp',
2727
'**/*.java']
28+
2829
pull_request:
2930
types: [opened, synchronize, reopened]
3031
workflow_dispatch:
@@ -1369,3 +1370,211 @@ jobs:
13691370
shell: bash
13701371
run: |
13711372
ctest -R ^test-vad$ --test-dir build --output-on-failure -VV
1373+
1374+
# TODO: simplify the following workflows using a matrix
1375+
ggml-ci-x64-cpu-low-perf:
1376+
runs-on: ubuntu-22.04
1377+
1378+
steps:
1379+
- name: Clone
1380+
id: checkout
1381+
uses: actions/checkout@v4
1382+
1383+
- name: ccache
1384+
uses: ggml-org/[email protected]
1385+
with:
1386+
key: ggml-ci-x64-cpu-low-perf
1387+
evict-old-files: 1d
1388+
1389+
- name: Dependencies
1390+
id: depends
1391+
run: |
1392+
sudo apt-get update
1393+
sudo apt-get install build-essential libcurl4-openssl-dev
1394+
1395+
- name: Test
1396+
id: ggml-ci
1397+
run: |
1398+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1399+
1400+
ggml-ci-arm64-cpu-low-perf:
1401+
runs-on: ubuntu-22.04-arm
1402+
1403+
steps:
1404+
- name: Clone
1405+
id: checkout
1406+
uses: actions/checkout@v4
1407+
1408+
- name: ccache
1409+
uses: ggml-org/[email protected]
1410+
with:
1411+
key: ggml-ci-arm64-cpu-low-perf
1412+
evict-old-files: 1d
1413+
1414+
- name: Dependencies
1415+
id: depends
1416+
run: |
1417+
sudo apt-get update
1418+
sudo apt-get install build-essential libcurl4-openssl-dev
1419+
1420+
- name: Test
1421+
id: ggml-ci
1422+
run: |
1423+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1424+
1425+
ggml-ci-x64-cpu-high-perf:
1426+
runs-on: ubuntu-22.04
1427+
1428+
steps:
1429+
- name: Clone
1430+
id: checkout
1431+
uses: actions/checkout@v4
1432+
1433+
- name: ccache
1434+
uses: ggml-org/[email protected]
1435+
with:
1436+
key: ggml-ci-x64-cpu-high-perf
1437+
evict-old-files: 1d
1438+
1439+
- name: Dependencies
1440+
id: depends
1441+
run: |
1442+
sudo apt-get update
1443+
sudo apt-get install build-essential libcurl4-openssl-dev
1444+
1445+
- name: Test
1446+
id: ggml-ci
1447+
run: |
1448+
LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt
1449+
1450+
ggml-ci-arm64-cpu-high-perf:
1451+
runs-on: ubuntu-22.04-arm
1452+
1453+
steps:
1454+
- name: Clone
1455+
id: checkout
1456+
uses: actions/checkout@v4
1457+
1458+
- name: ccache
1459+
uses: ggml-org/[email protected]
1460+
with:
1461+
key: ggml-ci-arm64-cpu-high-perf
1462+
evict-old-files: 1d
1463+
1464+
- name: Dependencies
1465+
id: depends
1466+
run: |
1467+
sudo apt-get update
1468+
sudo apt-get install build-essential libcurl4-openssl-dev
1469+
1470+
- name: Test
1471+
id: ggml-ci
1472+
run: |
1473+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1474+
1475+
ggml-ci-arm64-cpu-high-perf-sve:
1476+
runs-on: ubuntu-22.04-arm
1477+
1478+
steps:
1479+
- name: Clone
1480+
id: checkout
1481+
uses: actions/checkout@v4
1482+
1483+
- name: ccache
1484+
uses: ggml-org/[email protected]
1485+
with:
1486+
key: ggml-ci-arm64-cpu-high-perf-sve
1487+
evict-old-files: 1d
1488+
1489+
- name: Dependencies
1490+
id: depends
1491+
run: |
1492+
sudo apt-get update
1493+
sudo apt-get install build-essential libcurl4-openssl-dev
1494+
1495+
- name: Test
1496+
id: ggml-ci
1497+
run: |
1498+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1499+
1500+
ggml-ci-x64-nvidia-cuda:
1501+
runs-on: [self-hosted, Linux, X64, NVIDIA]
1502+
1503+
steps:
1504+
- name: Clone
1505+
id: checkout
1506+
uses: actions/checkout@v4
1507+
1508+
- name: Test
1509+
id: ggml-ci
1510+
run: |
1511+
nvidia-smi
1512+
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp
1513+
1514+
ggml-ci-x64-nvidia-vulkan-cm:
1515+
runs-on: [self-hosted, Linux, X64, NVIDIA]
1516+
1517+
steps:
1518+
- name: Clone
1519+
id: checkout
1520+
uses: actions/checkout@v4
1521+
1522+
- name: Test
1523+
id: ggml-ci
1524+
run: |
1525+
vulkaninfo --summary
1526+
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp
1527+
1528+
ggml-ci-x64-nvidia-vulkan-cm2:
1529+
runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
1530+
1531+
steps:
1532+
- name: Clone
1533+
id: checkout
1534+
uses: actions/checkout@v4
1535+
1536+
- name: Test
1537+
id: ggml-ci
1538+
run: |
1539+
vulkaninfo --summary
1540+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp
1541+
1542+
ggml-ci-x64-cpu-amx:
1543+
runs-on: [self-hosted, Linux, X64, CPU, AMX]
1544+
1545+
steps:
1546+
- name: Clone
1547+
id: checkout
1548+
uses: actions/checkout@v4
1549+
1550+
- name: Test
1551+
id: ggml-ci
1552+
run: |
1553+
bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp
1554+
1555+
ggml-ci-mac-metal:
1556+
runs-on: [self-hosted, macOS, ARM64]
1557+
1558+
steps:
1559+
- name: Clone
1560+
id: checkout
1561+
uses: actions/checkout@v4
1562+
1563+
- name: Test
1564+
id: ggml-ci
1565+
run: |
1566+
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp
1567+
1568+
ggml-ci-mac-vulkan:
1569+
runs-on: [self-hosted, macOS, ARM64]
1570+
1571+
steps:
1572+
- name: Clone
1573+
id: checkout
1574+
uses: actions/checkout@v4
1575+
1576+
- name: Test
1577+
id: ggml-ci
1578+
run: |
1579+
vulkaninfo --summary
1580+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
build/
1616
build-*/
1717
build_*/
18+
tmp/
1819

1920
# SPM
2021
.build/
@@ -62,4 +63,4 @@ cmake-build-debug/
6263
.gradle/
6364
local.properties
6465
.log
65-
.exe
66+
.exe

‎ci/run.sh‎

Lines changed: 65 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ mkdir -p "$2"
2424
OUT=$(realpath "$1")
2525
MNT=$(realpath "$2")
2626

27-
rm -f "$OUT/*.log"
28-
rm -f "$OUT/*.exit"
29-
rm -f "$OUT/*.md"
27+
rm -vf $OUT/*.log
28+
rm -vf $OUT/*.exit
29+
rm -vf $OUT/*.md
3030

3131
sd=`dirname $0`
3232
cd $sd/../
@@ -50,8 +50,35 @@ fi
5050

5151
CMAKE_EXTRA="-DWHISPER_FATAL_WARNINGS=ON"
5252

53+
if [ ! -z ${GG_BUILD_METAL} ]; then
54+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
55+
fi
56+
5357
if [ ! -z ${GG_BUILD_CUDA} ]; then
54-
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native"
58+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON"
59+
60+
if command -v nvidia-smi >/dev/null 2>&1; then
61+
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
62+
if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
63+
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
64+
else
65+
echo "Warning: Using fallback CUDA architectures"
66+
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
67+
fi
68+
else
69+
echo "Error: nvidia-smi not found, cannot build with CUDA"
70+
exit 1
71+
fi
72+
fi
73+
74+
if [ ! -z ${GG_BUILD_ROCM} ]; then
75+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON"
76+
if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
77+
echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
78+
exit 1
79+
fi
80+
81+
CMAKE_EXTRA="${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
5582
fi
5683

5784
if [ ! -z ${GG_BUILD_SYCL} ]; then
@@ -60,28 +87,38 @@ if [ ! -z ${GG_BUILD_SYCL} ]; then
6087
echo "source /opt/intel/oneapi/setvars.sh"
6188
exit 1
6289
fi
63-
64-
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
90+
# Use only main GPU
91+
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
92+
# Enable sysman for correct memory reporting
93+
export ZES_ENABLE_SYSMAN=1
94+
# to circumvent precision issues on CPY operations
95+
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
96+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
6597
fi
6698

67-
if [ ! -z ${GG_BUILD_OPENVINO} ]; then
68-
CMAKE_EXTRA="${CMAKE_EXTRA} -DWHISPER_OPENVINO=ON"
69-
fi
99+
if [ ! -z ${GG_BUILD_VULKAN} ]; then
100+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
101+
102+
# if on Mac, disable METAL
103+
if [[ "$OSTYPE" == "darwin"* ]]; then
104+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF"
105+
fi
70106

71-
if [ ! -z ${GG_BUILD_METAL} ]; then
72-
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
73107
fi
74108

75-
if [ ! -z ${GG_BUILD_VULKAN} ]; then
76-
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=ON"
109+
if [ ! -z ${GG_BUILD_WEBGPU} ]; then
110+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
77111
fi
78112

79-
if [ ! -z ${GG_BUILD_BLAS} ]; then
80-
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=ON"
113+
if [ ! -z ${GG_BUILD_MUSA} ]; then
114+
# Use qy1 by default (MTT S80)
115+
MUSA_ARCH=${MUSA_ARCH:-21}
116+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
81117
fi
82118

83-
if [ ! -z ${GG_BUILD_COREML} ]; then
84-
CMAKE_EXTRA="${CMAKE_EXTRA} -DWHISPER_COREML=ON"
119+
if [ ! -z ${GG_BUILD_NO_SVE} ]; then
120+
# arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
121+
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
85122
fi
86123

87124
## helpers
@@ -178,7 +215,7 @@ function gg_run_ctest {
178215
mode=$2
179216

180217
cd ${SRC}
181-
218+
182219
rm -rf build-ci-${mode} && mkdir build-ci-${mode} && cd build-ci-${mode}
183220

184221
set -e
@@ -219,7 +256,7 @@ function gg_run_bench {
219256
echo "Running memcpy benchmark"
220257
(time ./build-ci-release/bin/whisper-bench -w 1 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-memcpy.log
221258
gg_check_last_command_status "$OUT/${ci}-memcpy.exit" "memcpy benchmark"
222-
259+
223260
echo "Running ggml_mul_mat benchmark with $BENCH_N_THREADS threads"
224261
(time ./build-ci-release/bin/whisper-bench -w 2 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-mul_mat.log
225262
gg_check_last_command_status "$OUT/${ci}-mul_mat.exit" "ggml_mul_mat benchmark"
@@ -233,6 +270,8 @@ function gg_run_bench {
233270
printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" "---" "---" "---" "---" "---" "---" "---" "---" "---"
234271
} | tee -a $OUT/${ci}-models-table.log
235272

273+
res=0
274+
236275
# run benchmark for each model
237276
for model in "${MODELS[@]}"; do
238277
echo "Benchmarking model: $model"
@@ -283,8 +322,11 @@ function gg_run_bench {
283322
| tee -a $OUT/${ci}-models-table.log
284323
else
285324
echo "Benchmark failed for model: $model" | tee -a $OUT/${ci}-bench-errors.log
325+
res=1
286326
fi
287327
done
328+
329+
return $res
288330
}
289331

290332
function gg_sum_bench {
@@ -326,11 +368,12 @@ ret=0
326368
for model in "${MODELS[@]}"; do
327369
test $ret -eq 0 && gg_download_model ${model}
328370
done
329-
if [ -z ${GG_BUILD_SYCL}]; then
330-
test $ret -eq 0 && gg_run ctest debug
331-
fi
371+
372+
test $ret -eq 0 && gg_run ctest debug
332373
test $ret -eq 0 && gg_run ctest release
333374

334375
test $ret -eq 0 && gg_run bench
335376

377+
cat $OUT/README.md
378+
336379
exit $ret

0 commit comments

Comments
 (0)