Add tests for DSS on NVIDIA GPUs and only CPUs (New)

.github/workflows/testflinger-contrib-dss-regression.yaml

@@ -21,36 +21,31 @@ jobs:
       run:
         working-directory: contrib/checkbox-dss-validation
     strategy:
+      fail-fast: false
       matrix:
         dss_channel:
           - latest/stable
           - latest/edge
+        microk8s_channel:
+          - 1.28/stable
+          - 1.31/stable
         queue:
-          - dell-precision-3470-c30322 #ADL iGPU + NVIDIA GPU
-          - dell-precision-5680-c31665 #RPL iGPU + Arc Pro A60M dGPU
+          - name: dell-precision-3470-c30322 #ADL iGPU + NVIDIA GPU
+            provision_data: "distro: jammy"
+          - name: dell-precision-5680-c31665 #RPL iGPU + Arc Pro A60M dGPU
+            provision_data: "url: http://10.102.196.9/somerville/Platforms/jellyfish-muk/X96_A00/dell-bto-jammy-jellyfish-muk-X96-20230419-19_A00.iso"
+          - name: nvidia-dgx-station-c25989  # NO iGPU + NVIDIA GPU
+            provision_data: "distro: jammy"
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Build job file from template with maas2 provisioning
-        if: ${{ matrix.queue == 'dell-precision-3470-c30322' }}
-        env:
-          PROVISION_DATA: "distro: jammy"
+      - name: Build job file from template
         run: |
           sed -e "s|REPLACE_BRANCH|${BRANCH}|" \
-          -e "s|REPLACE_QUEUE|${{ matrix.queue }}|" \
-          -e "s|REPLACE_PROVISION_DATA|${PROVISION_DATA}|" \
-          -e "s|REPLACE_DSS_CHANNEL|${{ matrix.dss_channel }}|" \
-          ${GITHUB_WORKSPACE}/contrib/checkbox-dss-validation/testflinger/job-def.yaml > \
-          ${GITHUB_WORKSPACE}/job.yaml
-      - name: Build job file from template with oemscript provisioning
-        if: ${{ matrix.queue == 'dell-precision-5680-c31665' }}
-        env:
-          PROVISION_DATA: "url: http://10.102.196.9/somerville/Platforms/jellyfish-muk/X96_A00/dell-bto-jammy-jellyfish-muk-X96-20230419-19_A00.iso"
-        run: |
-          sed -e "s|REPLACE_BRANCH|${BRANCH}|" \
-          -e "s|REPLACE_QUEUE|${{ matrix.queue }}|" \
-          -e "s|REPLACE_PROVISION_DATA|${PROVISION_DATA}|" \
+          -e "s|REPLACE_QUEUE|${{ matrix.queue.name }}|" \
+          -e "s|REPLACE_PROVISION_DATA|${{ matrix.queue.provision_data }}|" \
           -e "s|REPLACE_DSS_CHANNEL|${{ matrix.dss_channel }}|" \
+          -e "s|REPLACE_MICROK8S_CHANNEL|${{ matrix.microk8s_channel }}|" \
           ${GITHUB_WORKSPACE}/contrib/checkbox-dss-validation/testflinger/job-def.yaml > \
           ${GITHUB_WORKSPACE}/job.yaml
       - name: Submit testflinger job

contrib/checkbox-dss-validation/README.md

@@ -1,12 +1,14 @@
 # Welcome to the Checkbox DSS project!
 
-This repository contains the Checkbox DSS Provider (test cases and test plans for validating Intel GPU support in the [Data Science Stack](https://documentation.ubuntu.com/data-science-stack/en/latest/)) as well as everything that is required to build the `checkbox-dss` snap.
+This repository contains the Checkbox DSS Provider (test cases and test plans for validating Intel and NVIDIA GPU support in the [Data Science Stack](https://documentation.ubuntu.com/data-science-stack/en/latest/)) as well as everything that is required to build the `checkbox-dss` snap.
 
 # Requirements
 
 - Ubuntu Jammy (22.04)
 - Supported hardware platforms:
+  - No GPUs
   - Intel platforms with recent GPU (>= Broadwell)
+  - Recent NVIDIA GPU
 
 # Installation
 
@@ -19,7 +21,7 @@ lxd init --auto
 git clone https://github.com/canonical/checkbox
 cd checkbox/contrib/checkbox-dss-validation
 snapcraft
-sudo snap install --dangerous --classic ./checkbox-dss_2.0_amd64.snap
+sudo snap install --dangerous --classic ./checkbox-dss_3.0_amd64.snap
 ```
 
 Make sure that the provider service is running and active:
@@ -40,15 +42,27 @@ By default this will install the `data-science-stack` snap from the `latest/stab
 channel. To instead install from `latest/edge` use:
 
 ```shell
-checkbox-dss.install-deps --dss-snap-channel=latest/edge
+checkbox-dss.install-deps --dss-snap-channel latest/edge
 ```
 
+Furthermore, the default `microk8s` snap channel is `1.28/stable` in classic mode,
+but this can be customized as
+(please note that this snap must to be `--classic` to enable GPU support):
+
+```shell
+checkbox-dss.install-deps --microk8s-snap-channel 1.31/stable
+```
+
+These validations also need the `kubectl` snap installed, and the default channel
+used for that is `1.29/stable`, but can be customized as shown previously by passing
+the appropriate channel name for `--kubectl-snap-channel`.
+
 # Automated Run
 
 To run the test plans:
 
 ```shell
-checkbox-dss.validate-intel-gpu
+checkbox-dss.validate-with-gpu
 ```
 
 # Cleanup

contrib/checkbox-dss-validation/bin/install-deps

@@ -1,56 +1,89 @@
 #!/bin/bash
 set -e
 
-echo -e "\nStep 1/5: Installing microk8s snap"
-sudo snap install microk8s --channel 1.28/stable --classic
-
-USER=$(id -nu ${SNAP_UID})
-HOME=${SNAP_REAL_HOME}
-
-# microk8s commands run from tests are run without sudo
-sudo usermod -a -G microk8s $USER
-# Directory needed for sharing microk8s config with kubectl snap
-mkdir -p $HOME/.kube
-
-echo -e "\nStep 2/5: Configuring microk8s addons"
-sudo microk8s status --wait-ready
-# Give microk8s another minute to stabilize
-# to avoid intermittent failures when
-# enabling hostpath-storage
-echo "Giving microk8s a minute to stabilize..."
-sleep 60
-sudo microk8s enable hostpath-storage
-sudo microk8s enable dns
-sudo microk8s enable rbac
-
-echo "Waiting for microk8s addons to become ready..."
-sudo microk8s.kubectl wait \
-  --for=condition=available \
-  --timeout 1800s \
-  -n kube-system \
-  deployment/coredns \
-  deployment/hostpath-provisioner
-sudo microk8s.kubectl -n kube-system rollout status ds/calico-node
-
-# This is needed to overcome the following bug within microk8s:
-# https://github.com/canonical/microk8s/issues/4453
-echo -e "\nStep 3/5: Installing kubectl snap"
-sudo snap install kubectl --classic --channel=1.29/stable
-# hack as redirecting stdout anywhere but /dev/null throws a permission denied error
-# see: https://forum.snapcraft.io/t/eksctl-cannot-write-to-stdout/17254/4
-sudo microk8s.kubectl config view --raw | tee $HOME/.kube/config > /dev/null
-
-# intel_gpu_top command used for host-level GPU check
-# jq used for cases where jsonpath is insufficient for parsing json results
-echo -e "\nStep 4/5: Installing intel-gpu-tools"
-DEBIAN_FRONTEND=noninteractive sudo apt install -y intel-gpu-tools jq
-
-echo -e "\nStep 5/5: Installing data-science-stack snap"
-optional_arg=$1
-if [ "${optional_arg}" = "--dss-snap-channel=latest/edge" ]; then
-  echo "Installing from edge"
-  sudo snap install data-science-stack --channel latest/edge
-else
-  echo "Installing from stable"
-  sudo snap install data-science-stack --channel latest/stable
-fi
+dss_snap_channel="latest/stable"
+microk8s_snap_channel="1.28/stable"
+kubectl_snap_channel="1.29/stable"
+
+setup_microk8s_snap() {
+    echo -e "\nInstalling microk8s snap from channel $1"
+    sudo snap install microk8s --channel "$1" --classic
+
+    SNAP_USER=$(id -nu "${SNAP_UID}")
+
+    # microk8s commands run from tests are run without sudo
+    sudo usermod -a -G microk8s "$SNAP_USER"
+    # Directory needed for sharing microk8s config with kubectl snap
+    mkdir -p "${SNAP_REAL_HOME}/.kube"
+
+    echo -e "\nConfiguring microk8s addons"
+    sudo microk8s status --wait-ready
+    # Give microk8s another minute to stabilize
+    # to avoid intermittent failures when
+    # enabling hostpath-storage
+    echo "Giving microk8s a minute to stabilize..."
+    sleep 60
+    sudo microk8s enable hostpath-storage
+    sudo microk8s enable dns
+    sudo microk8s enable rbac
+
+    echo "Waiting for microk8s addons to become ready..."
+    sudo microk8s.kubectl wait \
+        --for=condition=available \
+        --timeout 1800s \
+        -n kube-system \
+        deployment/coredns \
+        deployment/hostpath-provisioner
+    sudo microk8s.kubectl -n kube-system rollout status ds/calico-node
+}
+
+setup_kubectl_snap() {
+    # This is needed to overcome the following bug within microk8s:
+    # https://github.com/canonical/microk8s/issues/4453
+    echo -e "\nInstalling kubectl snap from channel $1"
+    sudo snap install kubectl --classic --channel="$1"
+    # hack as redirecting stdout anywhere but /dev/null throws a permission denied error
+    # see: https://forum.snapcraft.io/t/eksctl-cannot-write-to-stdout/17254/4
+    sudo microk8s.kubectl config view --raw | tee "${SNAP_REAL_HOME}/.kube/config" >/dev/null
+}
+
+help_function() {
+    echo "This script is used install all dependencies for checkbox-dss to run; defaults for optional arguments are shown in usage"
+    echo "Usage: checkbox-dss.install-deps [--dss-snap-channel $dss_snap_channel] [--microk8s-snap-channel $microk8s_snap_channel] [--kubectl-snap-channel $kubectl_snap_channel]"
+}
+
+main() {
+    while [ $# -ne 0 ]; do
+        case $1 in
+        --dss-snap-channel)
+            dss_snap_channel="$2"
+            shift 2
+            ;;
+        --microk8s-snap-channel)
+            microk8s_snap_channel="$2"
+            shift 2
+            ;;
+        --kubectl-snap-channel)
+            kubectl_snap_channel="$2"
+            shift 2
+            ;;
+        *) help_function ;;
+        esac
+    done
+
+    echo -e "\n Step 1/4: Setting up microk8s"
+    setup_microk8s_snap "$microk8s_snap_channel"
+
+    echo -e "\n Step 2/4: Setting up kubectl"
+    setup_kubectl_snap "$kubectl_snap_channel"
+
+    # intel_gpu_top command used for host-level GPU check
+    # jq used for cases where jsonpath is insufficient for parsing json results
+    echo -e "\nStep 3/4: Installing intel-gpu-tools"
+    DEBIAN_FRONTEND=noninteractive sudo apt install -y intel-gpu-tools jq
+
+    echo -e "\nStep 4/4: Installing data-science-stack snap from channel $dss_snap_channel"
+    sudo snap install data-science-stack --channel "$dss_snap_channel"
+}
+
+main "$@"

contrib/checkbox-dss-validation/bin/validate-intel-gpu → contrib/checkbox-dss-validation/bin/validate-with-gpu

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S checkbox-cli-wrapper remote 127.0.0.1
+#!/usr/bin/env -S checkbox-cli-wrapper control 127.0.0.1
 [launcher]
 app_id = com.canonical.contrib.dss-validation:checkbox
 launcher_version = 1
@@ -14,5 +14,5 @@ forced = yes
 [ui]
 type = silent
 auto_retry = yes
-max_attempts = 10
+max_attempts = 2
 delay_before_retry = 10

contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_cuda.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+check_nvidia_gpu_addon_can_be_enabled() {
+    # TODO: enable changing GPU_OPERATOR_VERSION
+    GPU_OPERATOR_VERSION=24.6.2
+    echo "[INFO]: enabling the NVIDIA GPU addon"
+    sudo microk8s enable gpu --driver=operator --version="$GPU_OPERATOR_VERSION"
+    SLEEP_SECS=10
+    echo "[INFO]: sleeping for ${SLEEP_SECS} seconds before checking GPU feature discovery has rolled out."
+    sleep ${SLEEP_SECS}
+    microk8s.kubectl -n gpu-operator-resources rollout status ds/gpu-operator-node-feature-discovery-worker
+    echo "[INFO]: sleeping for ${SLEEP_SECS} seconds before checking if daemonsets have rolled out."
+    sleep ${SLEEP_SECS}
+    microk8s.kubectl -n gpu-operator-resources rollout status ds/nvidia-device-plugin-daemonset
+    echo "[INFO]: sleeping for ${SLEEP_SECS} seconds before checking GPU validations have rolled out."
+    sleep ${SLEEP_SECS}
+    echo "[INFO]: Waiting for the GPU validations to rollout"
+    microk8s.kubectl -n gpu-operator-resources rollout status ds/nvidia-operator-validator
+    echo "Test success: NVIDIA GPU addon enabled."
+}
+
+check_nvidia_gpu_validations_succeed() {
+    SLEEP_SECS=5
+    echo "[INFO]: sleeping for ${SLEEP_SECS} seconds before checking if GPU validations were successful."
+    sleep ${SLEEP_SECS}
+    result=$(microk8s.kubectl logs -n gpu-operator-resources -lapp=nvidia-operator-validator -c nvidia-operator-validator)
+    if [ "${result}" = "all validations are successful" ]; then
+        echo "Test success: NVIDIA GPU validations were successful!"
+    else
+        >&2 echo "Test failure: NVIDIA GPU validations were not successful, got ${result}"
+        exit 1
+    fi
+}
+
+help_function() {
+    echo "This script is used for tests related to CUDA"
+    echo "Usage: check_dss.sh <test_case>"
+    echo
+    echo "Test cases currently implemented:"
+    echo -e "\t<gpu_addon_can_be_enabled>: check_nvidia_gpu_addon_can_be_enabled"
+    echo -e "\t<gpu_validations_succeed>: check_nvidia_gpu_validations_succeed"
+}
+
+main() {
+    case ${1} in
+    gpu_addon_can_be_enabled) check_nvidia_gpu_addon_can_be_enabled ;;
+    gpu_validations_succeed) check_nvidia_gpu_validations_succeed ;;
+    *) help_function ;;
+    esac
+}
+
+main "$@"