diff --git a/docs/.sphinx/_static/checkbox-doc.css b/docs/.sphinx/_static/checkbox-doc.css new file mode 100644 index 0000000000..3e6e4edc8a --- /dev/null +++ b/docs/.sphinx/_static/checkbox-doc.css @@ -0,0 +1,44 @@ +.mermaid { + display: flex; + justify-content: center; + padding-bottom: 0.5rem; + font-family: Ubuntu, sans-serif; +} + +.mermaid .node rect, +.mermaid .node circle, +.mermaid .node ellipse, +.mermaid .node polygon, +.mermaid .node path, +.mermaid rect.actor, +.mermaid .labelBox polygon +{ + fill: var(--color-background-item) !important; + stroke: var(--color-background-border) !important; +} + + +.mermaid .label text, +.mermaid span, +.mermaid p +{ + color: var(--color-content-foreground) !important; +} + +.mermaid .actor tspan, +.mermaid text.messageText +{ + fill: var(--color-content-foreground) !important; +} + +.mermaid .flowchart-link, +.mermaid line +{ + stroke: var(--color-content-foreground) !important; +} + +.mermaid .marker +{ + fill: var(--color-content-foreground) !important; + stroke: var(--color-content-foreground) !important; +} \ No newline at end of file diff --git a/docs/.sphinx/requirements.txt b/docs/.sphinx/requirements.txt index 9c91300d44..ef274a8bd8 100644 --- a/docs/.sphinx/requirements.txt +++ b/docs/.sphinx/requirements.txt @@ -1,17 +1,17 @@ +furo +lxd-sphinx-extensions +myst-parser psutil -urwid -tqdm +pyspelling +setuptools-scm sphinx sphinx-autobuild +sphinx-copybutton sphinx-design -furo -sphinx-tabs +sphinx-jsonschema sphinx-reredirects -pyspelling +sphinx-tabs +sphinxcontrib-mermaid sphinxext-opengraph -lxd-sphinx-extensions -sphinx-copybutton -myst-parser -setuptools-scm -sphinx-jsonschema -sphinxcontrib-mermaid \ No newline at end of file +tqdm +urwid \ No newline at end of file diff --git a/docs/.sphinx/spellingcheck.yaml b/docs/.sphinx/spellingcheck.yaml index 8c4a3c4d00..229625addd 100644 --- a/docs/.sphinx/spellingcheck.yaml +++ b/docs/.sphinx/spellingcheck.yaml @@ -24,4 +24,4 @@ matrix: - div.mermaid - pre - spellexception - - title + - title \ No newline at end of file diff --git a/docs/.sphinx/wordslist.txt b/docs/.sphinx/wordslist.txt index 5aa7e1a654..a7c4083379 100644 --- a/docs/.sphinx/wordslist.txt +++ b/docs/.sphinx/wordslist.txt @@ -1,3 +1,5 @@ +AMD +amd APIs autoconf backend @@ -94,6 +96,7 @@ sys systemd Systemd templating +Testflinger testplan tmp TODO diff --git a/docs/conf.py b/docs/conf.py index a19371bdef..6184542ef6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,18 +48,23 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "sphinxcontrib.mermaid", - "sphinx_design", - "sphinx_tabs.tabs", - "sphinx_reredirects", - "youtube-links", - "related-links", "custom-rst-roles", - "terminal-output", - "sphinx_copybutton", - "sphinxext.opengraph", "myst_parser", + "related-links", + "sphinx_copybutton", + "sphinx_design", + "sphinx_reredirects", + "sphinx_tabs.tabs", "sphinx-jsonschema", + "sphinxcontrib.mermaid", + "sphinxext.opengraph", + "terminal-output", + "youtube-links", +] + +myst_enable_extensions = [ + "substitution", + "deflist" ] myst_enable_extensions = ["substitution", "deflist"] @@ -76,7 +81,10 @@ } # Links to ignore when checking links -linkcheck_ignore = ["http://127.0.0.1:8000"] +linkcheck_ignore = [ + 'http://127.0.0.1:8000', + 'https://github.com/canonical/hwcert-jenkins-jobs', +] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -152,8 +160,9 @@ html_static_path = [".sphinx/_static"] html_css_files = [ - "custom.css", - "github_issue_links.css", + 'checkbox-doc.css' + 'custom.css', + 'github_issue_links.css', ] html_js_files = [ "github_issue_links.js", diff --git a/docs/explanation/index.rst b/docs/explanation/index.rst index 52fd29bb69..4bb8552d0f 100644 --- a/docs/explanation/index.rst +++ b/docs/explanation/index.rst @@ -7,3 +7,4 @@ Explanation understanding remote configs + release_process/canary diff --git a/docs/explanation/release_process/canary.rst b/docs/explanation/release_process/canary.rst new file mode 100644 index 0000000000..314db29cd8 --- /dev/null +++ b/docs/explanation/release_process/canary.rst @@ -0,0 +1,96 @@ +Canary Testing for Checkbox Edge Version: In-depth Process +========================================================== + +Introduction +------------ + +Canary testing applies on Checkbox snaps that are released through the `edge channel `_ in the Snap Store. New versions are built daily if changes are made in the code repository. + +The following sections provide a detailed walk-through of the Canary Testing process for the Checkbox Edge version, starting from snap building to the outcomes of the testing. + +Snap Build and Release +----------------------- + +Trigger Conditions +^^^^^^^^^^^^^^^^^^ +The GitHub action responsible for building the snap runs every day. However, it only triggers when the following condition is met: + +At least one pull request (PR) has been merged since the last edge build. + +Build Workflow +^^^^^^^^^^^^^^^ + +To monitor the build process, or to review the configuration and logs, see `the GitHub workflow `_. + +Post-Build Actions +^^^^^^^^^^^^^^^^^^ + +Once the build is successful, the snap packages are automatically pushed to the Snap Store in the edge channel. Testers and early adopters can access the latest version through edge releases. + +Jenkins Monitoring and Validation +--------------------------------- + +Snap Monitoring +^^^^^^^^^^^^^^^ + +Once the snap is published to the edge channel in the Snap Store, our Jenkins job titled ``checkbox-edge-validation-detect-new-build`` gets into action. + +Monitoring job URL +^^^^^^^^^^^^^^^^^^ + +This Jenkins job monitors the Snap Store for the presence of the new snap using the following URL: + +``https://api.snapcraft.io/v2/snaps/find?q=checkbox22&channel=edge&fields=revision&architecture=amd64`` + +The specific JSON path that's being monitored for changes is: + +``$.results[0].revision.revision`` + +The job checks this path every minute for updates. + +Validation and Wait Logic +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Upon detecting a new snap: + +1. The job verifies the presence of all other related checkbox snaps. +2. If any snap is missing, the job waits for an hour, periodically checking for its availability. +3. If the snaps are available within the waiting period, the next stage of testing is initiated. + +checkbox-edge-canary-validation Pipeline +---------------------------------------- + +Upon successful snap validation, the ``checkbox-edge-canary-validation`` pipeline begins its operation. + +The :doc:`canary_pipeline` contains the groovy script implementing the pipeline. + +Testing Platforms and Specifications +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +The pipeline concurrently runs 3 distinct jobs on different platforms: + +1. **amd64 ubuntu core 22**: Utilizing machines that consume Testflinger tasks from the ``dearest team`` queue. +2. **amd64 ubuntu core 16**: Also using machines from the ``dearest team`` queue, which corresponds to generic x86_64 machines. +3. **arm64 ubuntu core 22**: Targeting machines in the ``cert-rpi4b4g`` queue, which corresponds to the Raspberry Pi4 4GB model. + +The :doc:`validation_job_example` contains the Jenkins job definition for the amd64 ubuntu core 22 validation. + +For a detailed look on how the job execution is carried out by all the entities in the chain, +refer to the :doc:`validation_pipeline_execution`. + +Canary Test Plan Criteria +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The canary test plan outlines specific tests that are imperative for the new snap's validation. The pipeline's successful conclusion is contingent upon all these tests passing on each of the mentioned platforms. + +Outcome and Information propagation +----------------------------------- + +Should the validation complete without errors, the `beta` reference in the Checkbox repository is set to point at the revision that got validated. +The `beta` branch head is then updated on GitHub. + +Conclusion +---------- + +This Canary Testing process, complete from snap building to testing, ensures that every release of Checkbox in the edge channel is thoroughly vetted and stable. \ No newline at end of file diff --git a/docs/explanation/release_process/canary_pipeline.rst b/docs/explanation/release_process/canary_pipeline.rst new file mode 100644 index 0000000000..d8efd92b25 --- /dev/null +++ b/docs/explanation/release_process/canary_pipeline.rst @@ -0,0 +1,44 @@ +.. _canary_pipeline: + +Canary Pipeline +^^^^^^^^^^^^^^^ +This is an example on how the whole pipeline can be implemented in Jenkins. + +.. code-block:: groovy + + pipeline { + agent any + + stages { + stage('Run validation sub jobs') { + steps { + script { + parallel ( + "Checkbox series-22 for amd64": { + echo 'Running Canary on core22 amd64' + build job: 'checkbox-edge-validation-core22-amd64', wait: true, propagate: true + }, + "Checkbox series-22 for arm64": { + echo 'Running Canary on core22 arm64' + build job: 'checkbox-edge-validation-core22-arm64', wait: true, propagate: true + }, + "Checkbox series-16 for amd64": { + echo 'Running Canary on core16 amd64' + build job: 'checkbox-edge-validation-core16-amd64', wait: true, propagate: true + } + + ) + } + } + } + } + post { + always { + script { + def resultParam = currentBuild.resultIsBetterOrEqualTo('SUCCESS') ? 'edge-validation-succeeded' : 'edge-validation-failed' + // Trigger the job shifting the beta reference + build job: 'checkbox-edge-validation-move-beta', parameters: [string(name: 'RESULT', value: resultParam)] + } + } + } + } \ No newline at end of file diff --git a/docs/explanation/release_process/validation_job_example.rst b/docs/explanation/release_process/validation_job_example.rst new file mode 100644 index 0000000000..0534151be2 --- /dev/null +++ b/docs/explanation/release_process/validation_job_example.rst @@ -0,0 +1,66 @@ +.. _validation_job_example: + +Example of a validation job +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Below is the shell code that can be used as a build step in Jenkins in a job +responsible for validating edge snap of Checkbox. + +.. code-block:: shell + + #!/bin/bash + set -e + set -x + + cat > job.yaml < JOB_ID + testflinger poll $JOB_ID + + TEST_STATUS=$(testflinger results $JOB_ID |jq -r .test_status) + + echo "Test exit status: $TEST_STATUS" + exit $EXITCODE diff --git a/docs/explanation/release_process/validation_pipeline_execution.rst b/docs/explanation/release_process/validation_pipeline_execution.rst new file mode 100644 index 0000000000..f5583692a9 --- /dev/null +++ b/docs/explanation/release_process/validation_pipeline_execution.rst @@ -0,0 +1,101 @@ +Validation pipeline execution +============================= + +When validating various versions of Checkbox multiple workflows are in play. Everything starts with automated creation of an Edge version. + +Creating new Edge version +------------------------- +Every day GitHub checks for new commits that landed in the Checkbox repository, +and if anything new landed a new version is created. + + +.. mermaid:: + + graph TB + + A[GitHub Workflow: Detect New Commits] + C[Mark Version as Edge and Build Snaps] + E{Did Snap Builds Succeed?} + F[Upload New Edge Version to Store] + G[End: Build Failed - No New Edge Version] + + A --> C + C --> E + E -->|Yes| F + E -->|No| G + + +With new Edge version of Checkbox in the store we can start validating it. + +Validating the Edge version +--------------------------- + +On the Certification Jenkins instance, the ``checkbox-edge-validation-detect-new-build`` job checks the store API for new Edge versions of Checkbox. +The job is defined in the |hwcert-jenkins-jobs|_ repository. + +This job is also responsible for checking if all of the necessary snaps were published (for other series and architectures). +Once confirmed, the "Canary Test Plan" is executed. It is defined in the |hwcert-jenkins-jobs|_ repository as too. + +.. mermaid:: + + graph TB + + A[Detect New Edge Version in Store] + B{Check if All Necessary Snaps are Published} + C{Run Canary Test Plan on Devices} + D[No changes in the repository] + E[Move `beta` HEAD to the point at the validated revision] + + A --> B + B -->|No| D + B -->|Yes| C + C -->|Passes| E + C -->|Fails| D + + + + +There are multiple entities participating in the chain of validating a Checkbox snap. + +.. mermaid:: + + sequenceDiagram + + participant Pipeline as Checkbox Validation Pipeline + participant Jenkins as Jenkins Job + participant TServer as Testflinger Server + participant TAgent as Testflinger Agent + participant Docker as Docker container (Checkbox Controller) + participant Device as Device Under Test + participant CAgent as Checkbox Agent + note over Device,CAgent: Same device + + Pipeline->>Jenkins: Trigger Jenkins Job + activate Pipeline + Jenkins->>TServer: Submit Testing Job + activate TServer + loop Poll for Job + TAgent-->>TServer: Check for Available Jobs + end + activate TAgent + TAgent->>Device: Provision Device & Start Checkbox Agent + activate Device + activate CAgent + TAgent->>Docker: Run Checkbox Controller + activate Docker + Docker->>CAgent: Start Canary Test Plan + CAgent-->>Docker: Return Test Results + deactivate CAgent + Docker-->>TAgent: Report Results + deactivate Docker + TAgent-->>TServer: Job Completion Status + deactivate TAgent + TServer-->>Jenkins: Inform Jenkins of Outcome + deactivate TServer + Jenkins-->>Pipeline: Update Pipeline with Job Outcome + deactivate Pipeline + +.. add code format to link text +.. |hwcert-jenkins-jobs| replace:: ``hwcert-jenkins-jobs`` +.. _hwcert-jenkins-jobs: https://github.com/canonical/hwcert-jenkins-jobs +