ci: cleanup (#31196)

* extract threads limit script

* extract common functions

* mv need_to_generate_test_result need_to_upload_test_result

ci/common/limit-threads.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+# limit jobs to 4gb/thread
+if [[ -f "/proc/meminfo" ]]; then
+  JOBS=$(grep MemTotal /proc/meminfo | awk '{printf "%.0f", ($2 / (4 * 1024 * 1024))}')
+else
+  JOBS=$(sysctl hw.memsize | awk '{printf "%.0f", ($2 / (4 * 1024**3))}')
+fi
+
+NPROC=$(nproc)
+JOBS=$((JOBS > NPROC ? NPROC : JOBS))
+
+export NPROC
+export JOBS

ci/common/shared-functions.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+need_to_upload_test_result() {
+  local branches=(
+    "$EDGE_CHANNEL"
+    "$BETA_CHANNEL"
+    "$STABLE_CHANNEL"
+  )
+
+  for n in "${branches[@]}"; do
+    if [[ "$CI_BRANCH" == "$n" ]]; then
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+exit_if_error() {
+  if [[ "$1" -ne 0 ]]; then
+    exit "$1"
+  fi
+}

ci/test-stable.sh

@@ -12,12 +12,6 @@ annotate() {
   }
 }
 
-exit_if_error() {
-  if [[ "$1" -ne 0 ]]; then
-    exit "$1"
-  fi
-}
-
 # Run the appropriate test based on entrypoint
 testName=$(basename "$0" .sh)
 
@@ -27,41 +21,19 @@ export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 source scripts/ulimit-n.sh
 
-# limit jobs to 4gb/thread
-if [[ -f "/proc/meminfo" ]]; then
-  JOBS=$(grep MemTotal /proc/meminfo | awk '{printf "%.0f", ($2 / (4 * 1024 * 1024))}')
-else
-  JOBS=$(sysctl hw.memsize | awk '{printf "%.0f", ($2 / (4 * 1024**3))}')
-fi
-
-NPROC=$(nproc)
-JOBS=$((JOBS>NPROC ? NPROC : JOBS))
-
+#shellcheck source=ci/common/limit-threads.sh
+source ci/common/limit-threads.sh
 
 # get channel info
 eval "$(ci/channel-info.sh)"
 
-need_to_generate_test_result() {
-  local branches=(
-    "$EDGE_CHANNEL"
-    "$BETA_CHANNEL"
-    "$STABLE_CHANNEL"
-  )
-
-  for n in "${branches[@]}";
-  do
-    if [[ "$CI_BRANCH" == "$n" ]]; then
-      return 0
-    fi
-  done
-
-  return 1
-}
+#shellcheck source=ci/common/shared-functions.sh
+source ci/common/shared-functions.sh
 
 echo "Executing $testName"
 case $testName in
 test-stable)
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --jobs "$JOBS" --all --tests --exclude solana-local-cluster ${V:+--verbose} -- -Z unstable-options --format json --report-time | tee results.json
     exit_if_error "${PIPESTATUS[0]}"
   else
@@ -88,7 +60,7 @@ test-stable-sbf)
 
   # SBF C program system tests
   _ make -C programs/sbf/c tests
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test \
       --manifest-path programs/sbf/Cargo.toml \
       --no-default-features --features=sbf_c,sbf_rust -- -Z unstable-options --format json --report-time | tee results.json
@@ -130,7 +102,7 @@ test-stable-sbf)
 
   # SBF program instruction count assertion
   sbf_target_path=programs/sbf/target
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test \
       --manifest-path programs/sbf/Cargo.toml \
       --no-default-features --features=sbf_c,sbf_rust assert_instruction_count \
@@ -166,7 +138,7 @@ test-stable-perf)
   fi
 
   _ cargo build --bins ${V:+--verbose}
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- -Z unstable-options --format json --report-time | tee results.json
     exit_if_error "${PIPESTATUS[0]}"
   else
@@ -176,7 +148,7 @@ test-stable-perf)
   ;;
 test-local-cluster)
   _ cargo build --release --bins ${V:+--verbose}
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --release --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
     exit_if_error "${PIPESTATUS[0]}"
   else
@@ -186,7 +158,7 @@ test-local-cluster)
   ;;
 test-local-cluster-flakey)
   _ cargo build --release --bins ${V:+--verbose}
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --release --package solana-local-cluster --test local_cluster_flakey ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
     exit_if_error "${PIPESTATUS[0]}"
   else
@@ -196,7 +168,7 @@ test-local-cluster-flakey)
   ;;
 test-local-cluster-slow-1)
   _ cargo build --release --bins ${V:+--verbose}
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --release --package solana-local-cluster --test local_cluster_slow_1 ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
     exit_if_error "${PIPESTATUS[0]}"
   else
@@ -206,7 +178,7 @@ test-local-cluster-slow-1)
   ;;
 test-local-cluster-slow-2)
   _ cargo build --release --bins ${V:+--verbose}
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --release --package solana-local-cluster --test local_cluster_slow_2 ${V:+--verbose} -- --test-threads=1 -Z unstable-options --format json --report-time | tee results.json
     exit_if_error "${PIPESTATUS[0]}"
   else
@@ -228,7 +200,7 @@ test-wasm)
   exit 0
   ;;
 test-docs)
-  if need_to_generate_test_result; then
+  if need_to_upload_test_result; then
     _ cargo test --jobs "$JOBS" --all --doc --exclude solana-local-cluster ${V:+--verbose} -- -Z unstable-options --format json --report-time | tee results.json
     exit "${PIPESTATUS[0]}"
   else

scripts/coverage-in-disk.sh

@@ -69,15 +69,8 @@ if [[ -n $CI || -z $1 ]]; then
     $(git grep -l "proc-macro.*true" :**/Cargo.toml | sed 's|Cargo.toml|src/lib.rs|')
 fi
 
-# limit jobs to 4gb/thread
-if [[ -f "/proc/meminfo" ]]; then
-  JOBS=$(grep MemTotal /proc/meminfo | awk '{printf "%.0f", ($2 / (4 * 1024 * 1024))}')
-else
-  JOBS=$(sysctl hw.memsize | awk '{printf "%.0f", ($2 / (4 * 1024**3))}')
-fi
-
-NPROC=$(nproc)
-JOBS=$((JOBS>NPROC ? NPROC : JOBS))
+#shellcheck source=ci/common/limit-threads.sh
+source ci/common/limit-threads.sh
 
 RUST_LOG=solana=trace _ "$cargo" nightly test --jobs "$JOBS" --target-dir target/cov --no-run "${packages[@]}"
 if RUST_LOG=solana=trace _ "$cargo" nightly test --jobs "$JOBS" --target-dir target/cov "${packages[@]}" 2> target/cov/coverage-stderr.log; then

scripts/coverage.sh

@@ -68,15 +68,8 @@ if [[ -n $CI || -z $1 ]]; then
     $(git grep -l "proc-macro.*true" :**/Cargo.toml | sed 's|Cargo.toml|src/lib.rs|')
 fi
 
-# limit jobs to 4gb/thread
-if [[ -f "/proc/meminfo" ]]; then
-  JOBS=$(grep MemTotal /proc/meminfo | awk '{printf "%.0f", ($2 / (4 * 1024 * 1024))}')
-else
-  JOBS=$(sysctl hw.memsize | awk '{printf "%.0f", ($2 / (4 * 1024**3))}')
-fi
-
-NPROC=$(nproc)
-JOBS=$((JOBS>NPROC ? NPROC : JOBS))
+#shellcheck source=ci/common/limit-threads.sh
+source ci/common/limit-threads.sh
 
 _ "$cargo" nightly test --jobs "$JOBS" --target-dir target/cov --no-run "${packages[@]}"