workaround: run builds with retry (a)

Author: cre4ture

util/android-commands.sh

@@ -482,6 +482,22 @@ generate_and_install_public_key() {
     echo "installed ssh public key on device"
 }
 
+run_with_retry() {
+    tries=$1
+    shift 1
+
+    for i in $(seq 1 $tries); do
+        echo "Try #$i of $tries: run $*"
+        "$@" && echo "Done in try#$i" && return 0
+    done
+
+    exit_code=$?
+
+    echo "Still failing after $tries. Code: $exit_code"
+
+    return $exit_code
+}
+
 snapshot() {
     apk="$1"
     echo "Running snapshot"
@@ -500,7 +516,8 @@ snapshot() {
 
     echo "Installing cargo-nextest"
     # We need to install nextest via cargo currently, since there is no pre-built binary for android x86
-    run_command_via_ssh "export CARGO_TERM_COLOR=always && cargo install cargo-nextest"
+    command="export CARGO_TERM_COLOR=always && cargo install cargo-nextest"
+    run_with_retry 3 run_command_via_ssh "$command"
     return_code=$?
 
     echo "Info about cargo and rust - via SSH Script"
@@ -562,7 +579,7 @@ build() {
     command="export CARGO_TERM_COLOR=always;
              export CARGO_INCREMENTAL=0; \
              cd ~/coreutils && cargo build --features feat_os_unix_android"
-    run_command_via_ssh "$command" || return
+    run_with_retry 3 run_command_via_ssh "$command" || return
 
     echo "Finished build"
 }

util/android-scripts/run-tests.sh

@@ -11,6 +11,22 @@ export CARGO_INCREMENTAL=0
 
 echo "PATH: $PATH"
 
+run_with_retry() {
+    tries=$1
+    shift 1
+
+    for i in $(seq 1 $tries); do
+        echo "Try #$i of $tries: run $*"
+        "$@" && echo "Done in try#$i" && return 0
+    done
+
+    exit_code=$?
+
+    echo "Still failing after $tries. Code: $exit_code"
+
+    return $exit_code
+}
+
 run_tests_in_subprocess() (
 
     # limit virtual memory to 3GB to avoid that OS kills sshd
@@ -32,10 +48,14 @@ run_tests_in_subprocess() (
     watchplus 2 df -h &
     watchplus 2 free -hm &
 
+    nextest_params=(--profile ci --hide-progress-bar --features feat_os_unix_android)
+
     # run tests
     cd ~/coreutils && \
+        run_with_retry 3 timeout --preserve-status --verbose -k 1m 10m \
+            cargo nextest run --no-run "${nextest_params[@]}" &&
         timeout --preserve-status --verbose -k 1m 60m \
-            cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android
+            cargo nextest run "${nextest_params[@]}"
 
     result=$?