Cherry pick commits for releasing v2.0.0

.github/workflows/check-release.yaml

@@ -0,0 +1,64 @@
+name: Check Release
+
+on:
+  pull_request:
+    branches:
+      - release-*
+    paths:
+      - VERSION
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  SEMVER_PATTERN: '^v([0-9]+)\.([0-9]+)\.([0-9]+)(-rc\.([0-9]+))?$'
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check whether version matches semver pattern
+        run: |
+          VERSION=$(cat VERSION)
+          if [[ ${VERSION} =~ ${{ env.SEMVER_PATTERN }} ]]; then
+            echo "Version '${VERSION}' matches semver pattern."
+          else
+            echo "Version '${VERSION}' does not match semver pattern."
+            exit 1
+          fi
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
+      - name: Check whether chart version and appVersion matches version
+        run: |
+          VERSION=${VERSION#v}
+          CHART_VERSION=$(cat charts/spark-operator-chart/Chart.yaml | grep version | awk '{print $2}')
+          CHART_APP_VERSION=$(cat charts/spark-operator-chart/Chart.yaml | grep appVersion | awk '{print $2}')
+          if [[ ${CHART_VERSION} == ${VERSION} ]]; then
+            echo "Chart version '${CHART_VERSION}' matches version '${VERSION}'."
+          else
+            echo "Chart version '${CHART_VERSION}' does not match version '${VERSION}'."
+            exit 1
+          fi
+          if [[ ${CHART_APP_VERSION} == ${VERSION} ]]; then
+            echo "Chart appVersion '${CHART_APP_VERSION}' matches version '${VERSION}'."
+          else
+            echo "Chart appVersion '${CHART_APP_VERSION}' does not match version '${VERSION}'."
+            exit 1
+          fi
+
+      - name: Check if tag exists
+        run: |
+          git fetch --tags
+          if git tag -l | grep -q "^${VERSION}$"; then
+            echo "Tag '${VERSION}' already exists."
+            exit 1
+          else
+            echo "Tag '${VERSION}' does not exist."
+          fi

.github/workflows/integration.yaml

@@ -72,26 +72,11 @@ jobs:
       - name: Run unit tests
         run: make unit-test
 
-      - name: Build Spark-Operator Docker Image
-        run: make docker-build IMAGE_TAG=latest
-
-      - name: Check changes in resources used in docker file
-        run: |
-          DOCKERFILE_RESOURCES=$(cat Dockerfile | grep -P -o "COPY [a-zA-Z0-9].*? " | cut -c6-)
-          for resource in $DOCKERFILE_RESOURCES; do
-            # If the resource is different
-            if ! git diff  --quiet origin/master -- $resource; then
-              ## And the appVersion hasn't been updated
-              if ! git diff origin/master -- charts/spark-operator-chart/Chart.yaml | grep +appVersion; then
-                echo "resource used in docker.io/kubeflow/spark-operator has changed in $resource, need to update the appVersion in charts/spark-operator-chart/Chart.yaml"
-                git diff origin/master -- $resource;
-                echo "failing the build... " && false
-              fi
-            fi
-          done
+      - name: Build Spark operator
+        run: make build-operator
 
   build-helm-chart:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Determine branch name
         id: get_branch
@@ -131,7 +116,7 @@ jobs:
       - name: Run chart-testing (lint)
         if: steps.list-changed.outputs.changed == 'true'
         env:
-            BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
+          BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
         run: ct lint --check-version-increment=false --target-branch $BRANCH
 
       - name: Detect CRDs drift between chart and manifest
@@ -163,37 +148,25 @@ jobs:
           minikube image load docker.io/kubeflow/spark-operator:local
           ct install
 
-  integration-test:
-    runs-on: ubuntu-22.04
+  e2e-test:
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
         with:
-          fetch-depth: "0"
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: "go.mod"
+          go-version-file: go.mod
 
-      - name: setup minikube
-        uses: manusa/actions-setup-minikube@v2.11.0
-        with:
-          minikube version: v1.33.0
-          kubernetes version: v1.30.0
-          start args: --memory 6g --cpus=2 --addons ingress
-          github token: ${{ inputs.github-token }}
+      - name: Create a Kind cluster
+        run: make kind-create-cluster
 
-      - name: Build local spark-operator docker image for minikube testing
+      - name: Build and load image to Kind cluster
         run: |
-          docker build -t docker.io/kubeflow/spark-operator:local .
-          minikube image load docker.io/kubeflow/spark-operator:local
-
-      # The integration tests are currently broken see: https://github.com/kubeflow/spark-operator/issues/1416
-      # - name: Run chart-testing (integration test)
-      #   run: make integration-test
+          make kind-load-image IMAGE_TAG=local
 
-      - name: Setup tmate session
-        if: failure()
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 15
+      - name: Run e2e tests
+        run: make e2e-test

.github/workflows/release-charts.yaml → .github/workflows/release-helm-charts.yaml

@@ -2,17 +2,25 @@ name: Release Helm charts
 
 on:
   release:
-    types: [published]
+    types:
+      - published
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  HELM_REGISTRY: ghcr.io
+  HELM_REPOSITORY: ${{ github.repository_owner }}/helm-charts
+
 jobs:
-  build:
+  release_helm_charts:
     permissions:
       contents: write
+      packages: write
+
     runs-on: ubuntu-latest
+
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
@@ -27,10 +35,28 @@ jobs:
         with:
           version: v3.14.4
 
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.HELM_REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Read version from VERSION file
+        run: |
+          VERSION=$(cat VERSION)
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
       - name: Package Helm charts
         run: |
           for chart in $(ls charts); do
-            helm package charts/$chart
+            helm package charts/${chart}
+          done
+
+      - name: Upload charts to GHCR
+        run: |
+          for pkg in $(ls *.tgz); do
+            helm push ${pkg} oci://${{ env.HELM_REGISTRY }}/${{ env.HELM_REPOSITORY }}
           done
 
       - name: Save packaged charts to temp directory
@@ -44,15 +70,15 @@ jobs:
           ref: gh-pages
           fetch-depth: 0
 
-      - name: Copy packages charts
+      - name: Copy packaged charts
         run: |
           cp /tmp/charts/*.tgz .
 
       - name: Update Helm charts repo index
         env:
           CHART_URL: https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}
         run: |
-          helm repo index --merge index.yaml --url $CHART_URL .
+          helm repo index --merge index.yaml --url ${CHART_URL} .
           git add index.yaml
-          git commit -s -m "Update index.yaml" || exit 0
+          git commit -s -m "Add index for Spark operator chart ${VERSION}" || exit 0
           git push