Add checkbox-gfx Vulkan tests (New)

contrib/checkbox-gfx/.github/workflows/build-vulkan-on-pr-arm.yaml

@@ -0,0 +1,31 @@
+name: "Build Vulkan - ARM"
+on:
+  push:
+    branches:
+      - test_gh_runner
+  pull_request:
+    branches:
+      - main
+    paths:
+      - bin/install-vulkan
+
+jobs:
+  run-install-vulkan:
+    name: "Run install-vulkan to generate test binaries"
+    runs-on: [ self-hosted, linux, "arm64", "noble" ]
+
+    steps:
+      - name: checkout the main repo
+        uses: actions/checkout@v4
+
+      - name: Run install-vulkan
+        shell: bash
+        run: |
+          bin/install-vulkan
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: output
+          path: |
+            /usr/local/checkbox-gfx/*

contrib/checkbox-gfx/.github/workflows/build-vulkan-on-pr.yaml

@@ -0,0 +1,31 @@
+name: "Build Vulkan - amd64"
+on:
+  push:
+    branches:
+      - test_gh_runner
+  pull_request:
+    branches:
+      - main
+    paths:
+      - bin/install-vulkan
+
+jobs:
+  run-install-vulkan:
+    name: "Run install-vulkan to generate test binaries"
+    runs-on: [ self-hosted, linux, "amd64", "noble" ]
+
+    steps:
+      - name: checkout the main repo
+        uses: actions/checkout@v4
+
+      - name: Run install-vulkan
+        shell: bash
+        run: |
+          bin/install-vulkan
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: output
+          path: |
+            /usr/local/checkbox-gfx/*

contrib/checkbox-gfx/.github/workflows/build-vulkan.yaml

@@ -0,0 +1,43 @@
+name: "Build Vulkan CTS"
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Ubuntu version of the runner'
+        required: true
+        default: 'noble'
+        type: choice
+        options:
+          - jammy
+          - noble
+
+      arch:
+        description: 'Architecture of the runner'
+        required: true
+        default: 'amd64'
+        type: choice
+        options:
+          - amd64
+          - ARM64
+
+jobs:
+  run-install-vulkan:
+    name: "Run install-vulkan to generate CTS binaries"
+    runs-on: [ self-hosted, linux, "${{ inputs.arch }}", "${{ inputs.version}}" ]
+
+    steps:
+      - name: checkout the main repo
+        uses: actions/checkout@v4
+
+      - name: Run install-vulkan
+        shell: bash
+        run: |
+          bin/install-vulkan
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: output
+          path: |
+            /usr/local/checkbox-gfx/*

contrib/checkbox-gfx/.github/workflows/check-pull-request.yaml

@@ -0,0 +1,31 @@
+name: Do a snap build
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**.md' # Don't trigger if doc updated
+
+jobs:
+  build:
+
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch all of history so DM can determine its own version from git.
+          fetch-depth: 0
+
+      - uses: snapcore/action-build@v1
+        name: Build checkbox-gfx snap
+        id: build
+        with:
+          snapcraft-channel: latest/stable
+
+      - name: Verify checkbox-gfx snap
+        run: |
+          # Make sure it is installable.
+          sudo snap install checkbox24
+          sudo snap install --dangerous --classic ${{ steps.build.outputs.snap }}
+

contrib/checkbox-gfx/.gitignore

@@ -0,0 +1 @@
+*.snap

contrib/checkbox-gfx/README.md

@@ -0,0 +1,87 @@
+# Welcome to the Checkbox GFX project!
+
+This repository contains the Checkbox GFX Provider (GFX-specific test cases and test plans for [Checkbox]) as well as everything that is required to build the [checkbox-gfx] snap in the snapstore.
+
+# Checkbox GFX Provider
+
+Located in the `checkbox-provider-gfx` directory, it contains:
+
+- the test cases (also called "jobs" in the Checkbox jargon) and test plans to be run by Checkbox (in the `units` directory)
+
+# Requirements
+
+- Ubuntu Noble (24.04)
+- Supported hardware platforms:
+  - Intel platforms with recent GPU (>= Broadwell)
+
+# Installation
+
+Install the Checkbox runtime and build/install the gfx provider snaps:
+
+```shell
+sudo snap install --classic snapcraft
+sudo snap install checkbox24
+lxd init --auto
+git clone https://github.com/canonical/checkbox-gfx
+cd checkbox-gfx
+snapcraft
+sudo snap install --dangerous --classic ./checkbox-gfx_1.0_<arch>.snap
+```
+
+Make sure that the provider service is running and active:
+
+```shell
+systemctl status snap.checkbox-gfx.remote-slave.service
+```
+
+# Install dependencies
+
+Most tests need dependencies, and a helper script is available to install each category of tests:
+
+```shell
+checkbox-gfx.install-lvl-zero
+checkbox-gfx.install-lvl-zero-rt
+checkbox-gfx.install-crucible
+checkbox-gfx.install-vulkan
+checkbox-gfx.install-opengl
+checkbox-gfx.install-opencl
+```
+
+# Automated Run
+
+Each category of tests is run separately:
+
+```shell
+checkbox-gfx.test-lvl-zero
+checkbox-gfx.test-lvl-zero-rt
+checkbox-gfx.test-opencl
+```
+
+Due to some tests causing dropped SSH connections, running the tests remotely should be done like this:
+
+1. Install checkbox-gfx on both the remote machine and the local machine
+2. From the checkbox-gfx directory, run the following command
+
+```shell
+checkbox-gfx.checkbox-cli control <REMOTE IP> bin/<test bin>
+```
+
+# Develop the Checkbox GFX provider
+
+Since snaps are immutable, it is not possible to modify the content of the scripts or the test cases. Fortunately, Checkbox provides a functionality to side-load a provider on the DUT.
+
+Therefore, if you want to edit a job definition, a script or a test plan, run the following commands on the DUT:
+
+```shell
+cd $HOME
+git clone https://github.com/canonical/checkbox-gfx
+mkdir /var/tmp/checkbox-providers
+cp -r $HOME/checkbox-gfx/checkbox-provider-gfx /var/tmp/checkbox-providers/
+```
+
+You can then modify the content of the provider in `/var/tmp/checkbox-providers/checkbox-provider-gfx/`, and it's this version that will be used when you run the tests.
+
+Please refer to the [Checkbox documentation] on side-loading providers for more information.
+
+[Checkbox]: https://checkbox.readthedocs.io/
+[Checkbox documentation]: https://checkbox.readthedocs.io/en/latest/side-loading.html

contrib/checkbox-gfx/bin/checkbox-cli-wrapper

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# wrapper around the checkbox-cli
+# use absolute path in order to not use system checkbox-cli (from deb packages)
+exec /snap/checkbox24/current/bin/checkbox-cli "$@"

contrib/checkbox-gfx/bin/configure

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# Copyright 2018-2022 Canonical Ltd.
+# All rights reserved.
+#
+# Written by:
+#   Maciej Kisielewski <[email protected]>
+#   Sylvain Pineau <[email protected]>
+import os
+import re
+import sys
+
+sys.path.append(os.path.expandvars("$SNAP/usr/lib/python3/dist-packages"))
+sitepkgpath = "$SNAP/lib/python3.12/site-packages"
+sys.path.append(os.path.expandvars(sitepkgpath))
+
+sys.path.append(os.path.expandvars(
+    "/snap/checkbox24/current/usr/lib/python3/dist-packages"))
+runtimepath = "/snap/checkbox24/current/lib/python3.12/site-packages"
+sys.path.append(os.path.expandvars(runtimepath))
+
+try:
+    from checkbox_support.snap_utils.config import update_configuration
+    from checkbox_support.snap_utils.config import print_checkbox_conf
+except ImportError:
+    msg = """
+checkbox-support not found!
+You need to install the checkbox24 snap:
+
+snap install checkbox24
+    """
+    print(os.path.expandvars(msg), file=sys.stderr)
+    sys.exit(1)
+
+
+def main():
+    # we need run as root to be able to write to /var/snap/...
+    if os.geteuid() != 0:
+        print('You have to run this command with sudo')
+        return
+
+    if len(sys.argv) > 1 and sys.argv[1] == '-l':
+        print_checkbox_conf()
+        return
+
+    key_re = re.compile(r"^(?:[A-Z0-9]+_?)*[A-Z](?:_?[A-Z0-9])*$")
+    vars_to_set = dict()
+    for pair in sys.argv[1:]:
+        k, _, v = pair.partition('=')
+        if not key_re.match(k) or not v:
+            raise SystemExit("'%s' is not a valid configuration entry. "
+                             "Should be KEY=val" % pair)
+        k = k.replace('_', '-').lower()
+        vars_to_set[k] = v
+    update_configuration(vars_to_set)
+
+
+if __name__ == '__main__':
+    main()

contrib/checkbox-gfx/bin/install-vulkan

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+source $(dirname "$0")/setup-env.sh
+
+sudo apt update -y
+sudo apt install -y build-essential cmake python3-pip meson pkg-config cmake libpng-dev libvulkan-dev mesa-common-dev 
+pip3 install lxml --break-system-packages
+
+echo "Installing VK-GL-CTS for Vulkan"
+cd $WORKING_DIR
+git clone https://github.com/KhronosGroup/VK-GL-CTS
+cd VK-GL-CTS
+git checkout vulkan-cts-1.4.1.3
+python3 external/fetch_sources.py
+mkdir build && cd build
+
+# Different archs need different buildflags
+if [ "$ARCH" == "x86_64" ]; then
+    CMAKE_CLANG_FLAGS=-m64
+elif [ "$ARCH" == "aarch64" ]; then
+    CMAKE_CLANG_FLAGS=-march=armv8-a
+fi
+
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=$CMAKE_CLANG_FLAGS -DCMAKE_CXX_FLAGS=$CMAKE_CLANG_FLAGS
+# 64GB RAM wasn't enough for -j. Add nproc
+make -j`nproc`
+sudo mv $WORKING_DIR/VK-GL-CTS $INSTALL_DIR

contrib/checkbox-gfx/bin/remove

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source $(dirname "$0")/setup-env.sh
+sudo rm -rf $INSTALL_DIR
+rm -rf $WORKING_DIR

contrib/checkbox-gfx/bin/setup-env.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+export ARCH=$(uname -p)
+export INSTALL_DIR=/usr/local/checkbox-gfx
+export WORKING_DIR=$HOME/.checkbox-gfx-working-dir
+
+sudo mkdir -p $INSTALL_DIR
+sudo mkdir -p $WORKING_DIR
+sudo chown -R $USER $WORKING_DIR
+
+# Get vendor
+if [[ $(lscpu | grep "GenuineIntel") ]]; then
+    export VENDOR=Intel
+elif [[ $(lscpu | grep "AuthenticAMD") ]]; then
+    export VENDOR=AMD
+elif [[ $(lscpu | grep "Qualcomm") ]]; then
+    export VENDOR=Qualcomm
+fi
+

contrib/checkbox-gfx/bin/shell-wrapper

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "$SNAP_NAME runtime shell, type 'exit' to quit the session"
+exec bash

contrib/checkbox-gfx/bin/test-vulkan

@@ -0,0 +1,18 @@
+#!/usr/bin/env -S checkbox-cli-wrapper control 127.0.0.1
+[launcher]
+app_id = com.canonical.qa.gfx:checkbox
+launcher_version = 1
+stock_reports = text, submission_files
+
+[test plan]
+unit = com.canonical.certification::vulkan-test-plan
+forced = yes
+
+[test selection]
+forced = yes
+
+[ui]
+type = silent
+auto_retry = yes
+max_attempts = 3
+delay_before_retry = 15