Add documentation for the Checkbox Edge validation process (Infra)

docs/.sphinx/_static/checkbox-doc.css

@@ -0,0 +1,44 @@
+.mermaid {
+    display: flex;
+    justify-content: center;
+    padding-bottom: 0.5rem;
+    font-family: Ubuntu, sans-serif;
+}
+
+.mermaid .node rect,
+.mermaid .node circle,
+.mermaid .node ellipse,
+.mermaid .node polygon,
+.mermaid .node path,
+.mermaid rect.actor,
+.mermaid .labelBox polygon
+{
+    fill: var(--color-background-item) !important;
+    stroke: var(--color-background-border) !important;
+}
+
+
+.mermaid .label text,
+.mermaid span,
+.mermaid p
+{
+    color: var(--color-content-foreground) !important;
+}
+
+.mermaid .actor tspan,
+.mermaid text.messageText
+{
+    fill: var(--color-content-foreground) !important;
+}
+
+.mermaid .flowchart-link,
+.mermaid line
+{
+    stroke: var(--color-content-foreground) !important;
+}
+
+.mermaid .marker
+{
+    fill: var(--color-content-foreground) !important;
+    stroke: var(--color-content-foreground) !important;
+}

docs/.sphinx/requirements.txt

@@ -1,17 +1,17 @@
+furo
+lxd-sphinx-extensions
+myst-parser
 psutil
-urwid
-tqdm
+pyspelling
+setuptools-scm
 sphinx
 sphinx-autobuild
+sphinx-copybutton
 sphinx-design
-furo
-sphinx-tabs
+sphinx-jsonschema
 sphinx-reredirects
-pyspelling
+sphinx-tabs
+sphinxcontrib-mermaid
 sphinxext-opengraph
-lxd-sphinx-extensions
-sphinx-copybutton
-myst-parser
-setuptools-scm
-sphinx-jsonschema
-sphinxcontrib-mermaid
+tqdm
+urwid

docs/.sphinx/spellingcheck.yaml

@@ -24,4 +24,4 @@ matrix:
       - div.mermaid
       - pre
       - spellexception
-      - title
+      - title

docs/.sphinx/wordslist.txt

@@ -1,3 +1,5 @@
+AMD
+amd
 APIs
 autoconf
 backend
@@ -94,6 +96,7 @@ sys
 systemd
 Systemd
 templating
+Testflinger
 testplan
 tmp
 TODO

docs/conf.py

@@ -48,18 +48,23 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
-    "sphinxcontrib.mermaid",
-    "sphinx_design",
-    "sphinx_tabs.tabs",
-    "sphinx_reredirects",
-    "youtube-links",
-    "related-links",
     "custom-rst-roles",
-    "terminal-output",
-    "sphinx_copybutton",
-    "sphinxext.opengraph",
     "myst_parser",
+    "related-links",
+    "sphinx_copybutton",
+    "sphinx_design",
+    "sphinx_reredirects",
+    "sphinx_tabs.tabs",
     "sphinx-jsonschema",
+    "sphinxcontrib.mermaid",
+    "sphinxext.opengraph",
+    "terminal-output",
+    "youtube-links",
+]
+
+myst_enable_extensions = [
+    "substitution",
+    "deflist"
 ]
 
 myst_enable_extensions = ["substitution", "deflist"]
@@ -76,7 +81,10 @@
 }
 
 # Links to ignore when checking links
-linkcheck_ignore = ["http://127.0.0.1:8000"]
+linkcheck_ignore = [
+    'http://127.0.0.1:8000',
+    'https://github.com/canonical/hwcert-jenkins-jobs',
+]
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
@@ -152,8 +160,9 @@
 
 html_static_path = [".sphinx/_static"]
 html_css_files = [
-    "custom.css",
-    "github_issue_links.css",
+    'checkbox-doc.css'
+    'custom.css',
+    'github_issue_links.css',
 ]
 html_js_files = [
     "github_issue_links.js",

docs/explanation/index.rst

@@ -7,3 +7,4 @@ Explanation
    understanding
    remote
    configs
+   release_process/canary

docs/explanation/release_process/canary.rst

@@ -0,0 +1,96 @@
+Canary Testing for Checkbox Edge Version: In-depth Process
+==========================================================
+
+Introduction
+------------
+
+Canary testing applies on Checkbox snaps that are released through the `edge channel <https://snapcraft.io/docs/channels>`_ in the Snap Store. New versions are built daily if changes are made in the code repository.
+
+The following sections provide a detailed walk-through of the Canary Testing process for the Checkbox Edge version, starting from snap building to the outcomes of the testing.
+
+Snap Build and Release
+-----------------------
+
+Trigger Conditions
+^^^^^^^^^^^^^^^^^^
+The GitHub action responsible for building the snap runs every day. However, it only triggers when the following condition is met:
+
+At least one pull request (PR) has been merged since the last edge build.
+
+Build Workflow
+^^^^^^^^^^^^^^^
+
+To monitor the build process, or to review the configuration and logs, see `the GitHub workflow <https://github.com/canonical/checkbox/actions/workflows/checkbox-snap-daily-builds.yml>`_.
+
+Post-Build Actions
+^^^^^^^^^^^^^^^^^^
+
+Once the build is successful, the snap packages are automatically pushed to the Snap Store in the edge channel. Testers and early adopters can access the latest version through edge releases.
+
+Jenkins Monitoring and Validation
+---------------------------------
+
+Snap Monitoring
+^^^^^^^^^^^^^^^
+
+Once the snap is published to the edge channel in the Snap Store, our Jenkins job titled ``checkbox-edge-validation-detect-new-build`` gets into action.
+
+Monitoring job URL
+^^^^^^^^^^^^^^^^^^
+
+This Jenkins job monitors the Snap Store for the presence of the new snap using the following URL:
+
+``https://api.snapcraft.io/v2/snaps/find?q=checkbox22&channel=edge&fields=revision&architecture=amd64``
+
+The specific JSON path that's being monitored for changes is:
+
+``$.results[0].revision.revision``
+
+The job checks this path every minute for updates.
+
+Validation and Wait Logic
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Upon detecting a new snap:
+
+1. The job verifies the presence of all other related checkbox snaps.
+2. If any snap is missing, the job waits for an hour, periodically checking for its availability.
+3. If the snaps are available within the waiting period, the next stage of testing is initiated.
+
+checkbox-edge-canary-validation Pipeline
+----------------------------------------
+
+Upon successful snap validation, the ``checkbox-edge-canary-validation`` pipeline begins its operation.
+
+The :doc:`canary_pipeline` contains the groovy script implementing the pipeline.
+
+Testing Platforms and Specifications
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+The pipeline concurrently runs 3 distinct jobs on different platforms:
+
+1. **amd64 ubuntu core 22**: Utilizing machines that consume Testflinger tasks from the ``dearest team`` queue.
+2. **amd64 ubuntu core 16**: Also using machines from the ``dearest team`` queue, which corresponds to generic x86_64 machines.
+3. **arm64 ubuntu core 22**: Targeting machines in the ``cert-rpi4b4g`` queue, which corresponds to the Raspberry Pi4 4GB model.
+
+The :doc:`validation_job_example` contains the Jenkins job definition for the amd64 ubuntu core 22 validation.
+
+For a detailed look on how the job execution is carried out by all the entities in the chain,
+refer to the :doc:`validation_pipeline_execution`.
+
+Canary Test Plan Criteria
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The canary test plan outlines specific tests that are imperative for the new snap's validation. The pipeline's successful conclusion is contingent upon all these tests passing on each of the mentioned platforms.
+
+Outcome and Information propagation
+-----------------------------------
+
+Should the validation complete without errors, the `beta` reference in the Checkbox repository is set to point at the revision that got validated.
+The `beta` branch head is then updated on GitHub.
+
+Conclusion
+----------
+
+This Canary Testing process, complete from snap building to testing, ensures that every release of Checkbox in the edge channel is thoroughly vetted and stable. 

docs/explanation/release_process/canary_pipeline.rst

@@ -0,0 +1,44 @@
+.. _canary_pipeline:
+
+Canary Pipeline
+^^^^^^^^^^^^^^^
+This is an example on how the whole pipeline can be implemented in Jenkins.
+
+.. code-block:: groovy
+
+    pipeline {
+        agent any
+
+        stages {
+            stage('Run validation sub jobs') {
+                steps {
+                    script {
+                        parallel (
+                            "Checkbox series-22 for amd64": {
+                                echo 'Running Canary on core22 amd64'
+                                build job: 'checkbox-edge-validation-core22-amd64', wait: true, propagate: true
+                            },
+                            "Checkbox series-22 for arm64": {
+                                echo 'Running Canary on core22 arm64'
+                                build job: 'checkbox-edge-validation-core22-arm64', wait: true, propagate: true
+                            },
+                            "Checkbox series-16 for amd64": {
+                                echo 'Running Canary on core16 amd64'
+                                build job: 'checkbox-edge-validation-core16-amd64', wait: true, propagate: true
+                            }
+
+                        )
+                    }
+                }
+            }
+        }
+        post {
+            always {
+                script {
+                    def resultParam = currentBuild.resultIsBetterOrEqualTo('SUCCESS') ? 'edge-validation-succeeded' : 'edge-validation-failed'
+                    // Trigger the job shifting the beta reference
+                    build job: 'checkbox-edge-validation-move-beta', parameters: [string(name: 'RESULT', value: resultParam)]
+                }
+            }
+        }
+    }

docs/explanation/release_process/validation_job_example.rst

@@ -0,0 +1,66 @@
+.. _validation_job_example:
+
+Example of a validation job
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Below is the shell code that can be used as a build step in Jenkins in a job
+responsible for validating edge snap of Checkbox.
+
+.. code-block:: shell
+
+  #!/bin/bash
+  set -e
+  set -x
+
+  cat > job.yaml <<EOF
+      job_queue: dearest-team
+      global_timeout: 3600
+      output_timeout: 1800
+      provision_data:
+        distro: core22-latest
+      test_data:
+        test_cmds: |
+              #!/bin/bash
+
+              # the machine running this script is the test controller
+              # it runs on any device that consumes the jobs on given queue name, for instance "dearest-team"
+              # the controller has a 1:1 relationship with the DUT (device under test)
+              # to run anything on the DUT, the controller ssh's into the DUT and runs the commands there
+              # and then in the end runs checkbox to run the actual testing session
+              # the checkbox run is a typical remote session where the machine running this script is the
+              # Checkbox Controller and the DUT is the Checkbox Agent
+
+              set -x
+              set -e
+
+              # prepare Controller Machine
+              sudo add-apt-repository ppa:checkbox-dev/ppa -y
+              sudo apt-get -qq update
+              sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y python-cheetah git checkbox-ng
+
+              # get the tools necessary to prepare the target device for testing
+              git clone -b snap-update-tools https://github.com/kissiel/hwcert-jenkins-tools.git
+              export PATH=$PATH:hwcert-jenkins-tools/scriptlets
+
+              # install checkbox
+              _run_retry sudo snap install checkbox22 --no-wait --channel=latest/edge 
+              wait_for_snap_complete
+              # Let's list all the installed snaps for future debugging ease
+              _run snap list
+
+              _run_retry sudo snap install --devmode --channel=uc22/edge checkbox
+
+              # run the canary test plan
+              PYTHONUNBUFFERED=1 checkbox-cli remote \$DEVICE_IP hwcert-jenkins-tools/snap-update-tools/canary.launcher
+              EXITCODE=\$?
+  EOF
+
+  JOB_ID=$(testflinger submit -q job.yaml)
+  echo "JOB_ID: $JOB_ID"
+  echo "$JOB_ID" > JOB_ID
+  testflinger poll $JOB_ID
+
+  TEST_STATUS=$(testflinger results $JOB_ID |jq -r .test_status)
+
+  echo "Test exit status: $TEST_STATUS"
+  exit $EXITCODE