Add checkbox-gfx Vulkan tests (New)

contrib/checkbox-gfx/.github/workflows/build-installers.yaml

@@ -0,0 +1,78 @@
+name: "Build Installers"
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Ubuntu version of the runner'
+        required: true
+        default: 'noble'
+        type: choice
+        options:
+          - jammy
+          - noble
+
+      arch:
+        description: 'Architecture of the runner'
+        required: true
+        default: 'amd64'
+        type: choice
+        options:
+          - amd64
+          - arm64
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - bin/install-vulkan
+
+jobs:
+  manual-build:
+    if: github.event_name == 'workflow_dispatch'
+    strategy:
+      matrix:
+        test_name: [vulkan]
+    name: "Manual Build - ${{ matrix.test_name }}"
+    runs-on: [self-hosted, linux, "${{ inputs.arch }}", "${{ inputs.version }}"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run install-${{ matrix.test_name }}
+        shell: bash
+        run: bin/install-${{ matrix.test_name }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: "output-${{ matrix.test_name }}-\
+          ${{ inputs.arch }}-${{ inputs.version }}"
+          path: /usr/local/checkbox-gfx/*
+
+  pr-build:
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+        test_name: [vulkan]
+    name: "Installer testing"
+    runs-on: [self-hosted, linux, "${{ matrix.arch }}", noble]
+
+    steps:
+      - name: "Installers - ${{ matrix.test_name }} - ${{ matrix.arch }}"
+        run: echo "Running ${{ matrix.test_name }} ${{ matrix.arch }} install"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run install-${{ matrix.test_name }}
+        shell: bash
+        run: bin/install-${{ matrix.test_name }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: output-${{ matrix.test_name }}-${{ matrix.arch }}
+          path: /usr/local/checkbox-gfx/*

contrib/checkbox-gfx/.github/workflows/check-pull-request.yaml

@@ -0,0 +1,31 @@
+name: Do a snap build
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**.md' # Don't trigger if doc updated
+
+jobs:
+  build:
+
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch all of history so DM can determine its own version from git.
+          fetch-depth: 0
+
+      - uses: snapcore/action-build@v1
+        name: Build checkbox-gfx snap
+        id: build
+        with:
+          snapcraft-channel: latest/stable
+
+      - name: Verify checkbox-gfx snap
+        run: |
+          # Make sure it is installable.
+          sudo snap install checkbox24
+          sudo snap install --dangerous --classic ${{ steps.build.outputs.snap }}
+

contrib/checkbox-gfx/.gitignore

@@ -0,0 +1 @@
+*.snap

contrib/checkbox-gfx/README.md

@@ -0,0 +1,87 @@
+# Welcome to the Checkbox GFX project!
+
+This repository contains the Checkbox GFX Provider (GFX-specific test cases and test plans for [Checkbox]) as well as everything that is required to build the [checkbox-gfx] snap in the snapstore.
+
+# Checkbox GFX Provider
+
+Located in the `checkbox-provider-gfx` directory, it contains:
+
+- the test cases (also called "jobs" in the Checkbox jargon) and test plans to be run by Checkbox (in the `units` directory)
+
+# Requirements
+
+- Ubuntu Noble (24.04)
+- Supported hardware platforms:
+  - Intel platforms with recent GPU (>= Broadwell)
+
+# Installation
+
+Install the Checkbox runtime and build/install the gfx provider snaps:
+
+```shell
+sudo snap install --classic snapcraft
+sudo snap install checkbox24
+lxd init --auto
+git clone https://github.com/canonical/checkbox-gfx
+cd checkbox-gfx
+snapcraft
+sudo snap install --dangerous --classic ./checkbox-gfx_1.0_<arch>.snap
+```
+
+Make sure that the provider service is running and active:
+
+```shell
+systemctl status snap.checkbox-gfx.run-agent.service
+```
+
+# Install dependencies
+
+Most tests need dependencies, and a helper script is available to install each category of tests:
+
+```shell
+checkbox-gfx.install-lvl-zero
+checkbox-gfx.install-lvl-zero-rt
+checkbox-gfx.install-crucible
+checkbox-gfx.install-vulkan
+checkbox-gfx.install-opengl
+checkbox-gfx.install-opencl
+```
+
+# Automated Run
+
+Each category of tests is run separately:
+
+```shell
+checkbox-gfx.test-lvl-zero
+checkbox-gfx.test-lvl-zero-rt
+checkbox-gfx.test-opencl
+```
+
+Due to some tests causing dropped SSH connections, running the tests remotely should be done like this:
+
+1. Install checkbox-gfx on both the remote machine and the local machine
+2. From the checkbox-gfx directory, run the following command
+
+```shell
+checkbox-gfx.checkbox-cli control <REMOTE IP> bin/<test bin>
+```
+
+# Develop the Checkbox GFX provider
+
+Since snaps are immutable, it is not possible to modify the content of the scripts or the test cases. Fortunately, Checkbox provides a functionality to side-load a provider on the DUT.
+
+Therefore, if you want to edit a job definition, a script or a test plan, run the following commands on the DUT:
+
+```shell
+cd $HOME
+git clone https://github.com/canonical/checkbox-gfx
+mkdir /var/tmp/checkbox-providers
+cp -r $HOME/checkbox-gfx/checkbox-provider-gfx /var/tmp/checkbox-providers/
+```
+
+You can then modify the content of the provider in `/var/tmp/checkbox-providers/checkbox-provider-gfx/`, and it's this version that will be used when you run the tests.
+
+Please refer to the [Checkbox documentation] on side-loading providers for more information.
+
+[Checkbox]: https://checkbox.readthedocs.io/
+[Checkbox documentation]: https://checkbox.readthedocs.io/en/latest/side-loading.html

contrib/checkbox-gfx/bin/checkbox-cli-wrapper

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# wrapper around the checkbox-cli
+# use absolute path in order to not use system checkbox-cli (from deb packages)
+exec /snap/checkbox24/current/bin/checkbox-cli "$@"

contrib/checkbox-gfx/bin/configure

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# Copyright 2018-2022 Canonical Ltd.
+# All rights reserved.
+#
+# Written by:
+#   Maciej Kisielewski <[email protected]>
+#   Sylvain Pineau <[email protected]>
+import os
+import re
+import sys
+
+sys.path.append(os.path.expandvars("$SNAP/usr/lib/python3/dist-packages"))
+sitepkgpath = "$SNAP/lib/python3.12/site-packages"
+sys.path.append(os.path.expandvars(sitepkgpath))
+
+sys.path.append(os.path.expandvars(
+    "/snap/checkbox24/current/usr/lib/python3/dist-packages"))
+runtimepath = "/snap/checkbox24/current/lib/python3.12/site-packages"
+sys.path.append(os.path.expandvars(runtimepath))
+
+try:
+    from checkbox_support.snap_utils.config import update_configuration
+    from checkbox_support.snap_utils.config import print_checkbox_conf
+except ImportError:
+    msg = """
+checkbox-support not found!
+You need to install the checkbox24 snap:
+
+snap install checkbox24
+    """
+    print(os.path.expandvars(msg), file=sys.stderr)
+    sys.exit(1)
+
+
+def main():
+    # we need run as root to be able to write to /var/snap/...
+    if os.geteuid() != 0:
+        print('You have to run this command with sudo')
+        return
+
+    if len(sys.argv) > 1 and sys.argv[1] == '-l':
+        print_checkbox_conf()
+        return
+
+    key_re = re.compile(r"^(?:[A-Z0-9]+_?)*[A-Z](?:_?[A-Z0-9])*$")
+    vars_to_set = dict()
+    for pair in sys.argv[1:]:
+        k, _, v = pair.partition('=')
+        if not key_re.match(k) or not v:
+            raise SystemExit("'%s' is not a valid configuration entry. "
+                             "Should be KEY=val" % pair)
+        k = k.replace('_', '-').lower()
+        vars_to_set[k] = v
+    update_configuration(vars_to_set)
+
+
+if __name__ == '__main__':
+    main()

contrib/checkbox-gfx/bin/install-vulkan

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+source $(dirname "$0")/setup-env.sh
+
+sudo apt update -y
+sudo apt install -y build-essential cmake python3-pip meson pkg-config cmake libpng-dev libvulkan-dev mesa-common-dev 
+pip3 install lxml --break-system-packages
+
+echo "Installing VK-GL-CTS for Vulkan"
+cd $WORKING_DIR
+git clone https://github.com/KhronosGroup/VK-GL-CTS
+cd VK-GL-CTS
+git checkout vulkan-cts-1.4.1.3
+python3 external/fetch_sources.py
+mkdir build && cd build
+
+# Different archs need different buildflags
+if [ "$ARCH" == "x86_64" ]; then
+    CMAKE_CLANG_FLAGS=-m64
+elif [ "$ARCH" == "aarch64" ]; then
+    CMAKE_CLANG_FLAGS=-march=armv8-a
+fi
+
+cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=$CMAKE_CLANG_FLAGS -DCMAKE_CXX_FLAGS=$CMAKE_CLANG_FLAGS
+# 64GB RAM wasn't enough for -j. Add nproc
+make -j`nproc`
+sudo mv $WORKING_DIR/VK-GL-CTS $INSTALL_DIR

contrib/checkbox-gfx/bin/remove

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source $(dirname "$0")/setup-env.sh
+sudo rm -rf $INSTALL_DIR
+rm -rf $WORKING_DIR

contrib/checkbox-gfx/bin/setup-env.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+export ARCH=$(uname -p)
+export INSTALL_DIR=/usr/local/checkbox-gfx
+export WORKING_DIR=$HOME/.checkbox-gfx-working-dir
+
+sudo mkdir -p $INSTALL_DIR
+sudo mkdir -p $WORKING_DIR
+sudo chown -R $USER $WORKING_DIR
+
+# Get vendor
+if [[ $(lscpu | grep "GenuineIntel") ]]; then
+    export VENDOR=Intel
+elif [[ $(lscpu | grep "AuthenticAMD") ]]; then
+    export VENDOR=AMD
+elif [[ $(lscpu | grep "Qualcomm") ]]; then
+    export VENDOR=Qualcomm
+fi
+

contrib/checkbox-gfx/bin/shell-wrapper

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "$SNAP_NAME runtime shell, type 'exit' to quit the session"
+exec bash

contrib/checkbox-gfx/bin/test-vulkan

@@ -0,0 +1,18 @@
+#!/usr/bin/env checkbox-cli-wrapper
+[launcher]
+app_id = com.canonical.qa.gfx:checkbox
+launcher_version = 1
+stock_reports = text, submission_files
+
+[test plan]
+unit = com.canonical.certification::vulkan-test-plan
+forced = yes
+
+[test selection]
+forced = yes
+
+[ui]
+type = silent
+auto_retry = yes
+max_attempts = 3
+delay_before_retry = 15

contrib/checkbox-gfx/bin/wrapper_local

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+case "$SNAP_ARCH" in
+    "amd64") ARCH='x86_64-linux-gnu'
+    ;;
+    "i386") ARCH='i386-linux-gnu'
+    ;;
+    "arm64") ARCH='aarch64-linux-gnu'
+    ;;
+    "armhf") ARCH='arm-linux-gnueabihf'
+    ;;
+    *)
+        echo "Unsupported architecture: $SNAP_ARCH"
+    ;;
+esac
+
+################################################
+# Launcher common exports for any checkbox app #
+################################################
+
+RUNTIME=/snap/checkbox24/current
+if [ ! -d "$RUNTIME" ]; then
+    echo "You need to install the checkbox24 snap."
+    echo ""
+    echo "You can do this with this command:"
+    echo "snap install checkbox24"
+    exit 1
+fi
+
+export LC_ALL=C.UTF-8
+PERL_VERSION=$(perl -e '$^V=~/^v(\d+\.\d+)/;print $1')
+export PERL5LIB="$PERL5LIB:$SNAP/usr/lib/$ARCH/perl/$PERL_VERSION:$SNAP/usr/lib/$ARCH/perl5/$PERL_VERSION:$SNAP/usr/share/perl/$PERL_VERSION:$SNAP/usr/share/perl5"
+export GI_TYPELIB_PATH=$SNAP/usr/lib/girepository-1.0:$SNAP/usr/lib/$ARCH/girepository-1.0
+export PATH="$SNAP/usr/sbin:$SNAP/sbin:$SNAP/usr/bin:$SNAP/bin:/snap/bin:$PATH"
+export ALSA_CONFIG_PATH=$SNAP/usr/share/alsa/alsa.conf:$SNAP/usr/share/alsa/pcm/default.conf
+export PYTHONPATH="$SNAP/usr/lib/python3/dist-packages:$PYTHONPATH"
+
+if [ -e $RUNTIME/wrapper_common_classic ]; then
+  . $RUNTIME/wrapper_common_classic
+else
+  echo "ERROR: no $RUNTIME/wrapper_common_classic found"
+  # This is to fix an issue where tests are run and built with different deps
+  # due to checkbox confinement
+  unset LD_LIBRARY_PATH
+  exit 0
+fi
+
+exec "$@"