From f8b8143170a63b6ed8afb6f2d9da709900ba040b Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Tue, 27 Aug 2024 14:47:26 +0100 Subject: [PATCH 1/4] changes to support deterministic image in solo Signed-off-by: Jeromy Cannon --- .../templates/network-node-statefulset.yaml | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/charts/fullstack-deployment/templates/network-node-statefulset.yaml b/charts/fullstack-deployment/templates/network-node-statefulset.yaml index e7076b7b..76ff5eb1 100644 --- a/charts/fullstack-deployment/templates/network-node-statefulset.yaml +++ b/charts/fullstack-deployment/templates/network-node-statefulset.yaml @@ -156,9 +156,41 @@ spec: task() { mkdir -p /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar chmod 777 /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar - curl network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133 + } + jshell_command() { + cat << EOF | jshell /dev/stdin + int rc = 1; + try { + final String clusterDomain = "cluster.local"; + final String namespace = "{{ default $.Release.Namespace $.Values.global.namespaceOverride }}"; + String podName; + String serviceName; + String fqdn; + InetAddress serviceAddress; + {{ range $index, $node := $.Values.hedera.nodes }} + podName = "network-{{ $node.name }}-0"; + serviceName = "network-{{ $node.name }}"; + fqdn = String.join(".", podName, serviceName, namespace, "svc", clusterDomain); + serviceAddress = InetAddress.getByName(fqdn); + {{ end }} + rc = 0; + } catch (Exception exception) { + System.out.println(exception); + } + /exit rc + EOF } task + jshell --version + if [ "$?" -ne 0 ]; then + echo "jshell not found, using curl to test DNS resolution"; + curl network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133; + return $?; + else + echo "jshell found, using jshell to test DNS resolution"; + jshell_command; + return $?; + fi failureThreshold: 30 periodSeconds: 10 timeoutSeconds: 5 @@ -537,8 +569,29 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} - {{- if gt $initContainersLength 0 }} initContainers: + - name: init-hedera + image: curlimages/curl:8.9.1 + command: + - "sh" + - "-c" + - | + loop=1 + while [ "$loop" -ne 0 ]; do + loop=0; + {{- range $index, $node := $.Values.hedera.nodes }} + nslookup network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local; + rc=$?; + if [ "$loop" -eq 0 -a "$rc" -eq 0 ]; then + loop=0; + else + loop=1; + fi + {{- end }} + sleep 5; + done + return 0 + {{- if gt $initContainersLength 0 }} {{- toYaml $.Values.hedera.initContainers | nindent 8 }} {{- end }} {{ end }} From 540be5b5a8dc5d7152af6d9cd218e52b681ab1ad Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Tue, 27 Aug 2024 17:48:52 +0100 Subject: [PATCH 2/4] only need mkdir/chmod for startup probe now, that we have init container Signed-off-by: Jeromy Cannon --- .../templates/network-node-statefulset.yaml | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/charts/fullstack-deployment/templates/network-node-statefulset.yaml b/charts/fullstack-deployment/templates/network-node-statefulset.yaml index 76ff5eb1..df541734 100644 --- a/charts/fullstack-deployment/templates/network-node-statefulset.yaml +++ b/charts/fullstack-deployment/templates/network-node-statefulset.yaml @@ -157,40 +157,7 @@ spec: mkdir -p /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar chmod 777 /opt/hgcapp/recordStreams/record{{ $node.accountId }}/sidecar } - jshell_command() { - cat << EOF | jshell /dev/stdin - int rc = 1; - try { - final String clusterDomain = "cluster.local"; - final String namespace = "{{ default $.Release.Namespace $.Values.global.namespaceOverride }}"; - String podName; - String serviceName; - String fqdn; - InetAddress serviceAddress; - {{ range $index, $node := $.Values.hedera.nodes }} - podName = "network-{{ $node.name }}-0"; - serviceName = "network-{{ $node.name }}"; - fqdn = String.join(".", podName, serviceName, namespace, "svc", clusterDomain); - serviceAddress = InetAddress.getByName(fqdn); - {{ end }} - rc = 0; - } catch (Exception exception) { - System.out.println(exception); - } - /exit rc - EOF - } task - jshell --version - if [ "$?" -ne 0 ]; then - echo "jshell not found, using curl to test DNS resolution"; - curl network-{{ $node.name }}-0.network-{{ $node.name }}.{{ default $.Release.Namespace $.Values.global.namespaceOverride }}.svc.cluster.local:13133; - return $?; - else - echo "jshell found, using jshell to test DNS resolution"; - jshell_command; - return $?; - fi failureThreshold: 30 periodSeconds: 10 timeoutSeconds: 5 From 337687928fdccb2427571ff4c2d33b9b49ed46df Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Fri, 6 Sep 2024 10:44:49 +0100 Subject: [PATCH 3/4] helm dependency update Signed-off-by: Jeromy Cannon --- charts/fullstack-deployment/Chart.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/fullstack-deployment/Chart.lock b/charts/fullstack-deployment/Chart.lock index c4e57b20..8aa8ab8e 100644 --- a/charts/fullstack-deployment/Chart.lock +++ b/charts/fullstack-deployment/Chart.lock @@ -4,7 +4,7 @@ dependencies: version: 0.2.0 - name: hedera-mirror repository: https://hashgraph.github.io/hedera-mirror-node/charts - version: 0.110.1 + version: 0.111.1 - name: tenant repository: https://operator.min.io/ version: 5.0.12 @@ -14,5 +14,5 @@ dependencies: - name: haproxy-ingress repository: https://haproxy-ingress.github.io/charts version: 0.14.5 -digest: sha256:0e6897144860871142fd0db8c0a34fc113dc66121aa50c1482e89d932d323af8 -generated: "2024-08-09T12:47:26.726721+01:00" +digest: sha256:ca8cade42bbfa3211fde7112cb0998e8c7c86587c9113b10f2cbd209683113b0 +generated: "2024-09-06T10:44:29.620625+01:00" From d1eada7d317d8299b22b7788ffc33348f7b9d7d9 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Fri, 6 Sep 2024 14:07:44 +0100 Subject: [PATCH 4/4] update test to wait for pod condition ready to avoid errors Signed-off-by: Jeromy Cannon --- .github/workflows/support/ci_test.sh | 5 ++++- charts/fullstack-deployment/tests/test_basic_deployment.bats | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/support/ci_test.sh b/.github/workflows/support/ci_test.sh index 5b0b5d09..6acda66a 100755 --- a/.github/workflows/support/ci_test.sh +++ b/.github/workflows/support/ci_test.sh @@ -75,9 +75,12 @@ echo "Get service and pod information" kubectl get svc -o wide && \ kubectl get pods -o wide && \ -echo "Waiting for network-node pods to be active (first deployment takes ~10m)...." +echo "Waiting for network-node pods to be phase=running (first deployment takes ~10m)...." kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l fullstack.hedera.com/type=network-node --timeout=900s +echo "Waiting for network-node pods to be condition=ready (first deployment takes ~10m)...." +kubectl wait --for=condition=ready pod -l fullstack.hedera.com/type=network-node --timeout=900s + echo "Service Information...." kubectl get svc -o wide diff --git a/charts/fullstack-deployment/tests/test_basic_deployment.bats b/charts/fullstack-deployment/tests/test_basic_deployment.bats index 69f0b31b..e352a58f 100644 --- a/charts/fullstack-deployment/tests/test_basic_deployment.bats +++ b/charts/fullstack-deployment/tests/test_basic_deployment.bats @@ -10,7 +10,10 @@ setup() { log_debug "Expected total nodes: ${TOTAL_NODES}" log_debug "----------------------------------------------------------------------------" + log_debug "Waiting for network node pods to be phase=running..." kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l fullstack.hedera.com/type=network-node --timeout=300s -n "${NAMESPACE}" || return "${EX_ERR}" + log_debug "Waiting for network node pods to be condition=ready..." + kubectl wait --for=condition=ready pod -l fullstack.hedera.com/type=network-node --timeout=300s -n "${NAMESPACE}" || return "${EX_ERR}" local resp="$(get_pod_list network-node)" local nodes=(${resp}) # convert into an array