ci: make dcou partitioned steps not-skewed and cached (#4452)

* wip: ci: Distribute dcou builds more evenly

* Disable sccache altogether

* Make sccache use local disk storage for dcou

* Use _

* Use all system cores

* Update sccache

* Add document and move things a bit

* Revert -Z threads=0

* Update doc

* Some tweaks

* Update Dockerfile

* Update cargo-hack repo

* Try -Z threads=0

* Restore original no newline at eof

* Revert -Z threads

* Use docker image with sccache v0.9.1

ci/docker-run.sh

@@ -57,6 +57,12 @@ if [[ -n $CI ]]; then
       ARGS+=(
         --env "RUSTC_WRAPPER=/usr/local/cargo/bin/sccache"
       )
+      # local disk storage for sccache (experimental; only used by dcou for now)
+      mkdir -p "$HOME/.cache/sccache-for-docker"
+      CONTAINER_HOME="/"
+      ARGS+=(
+        --volume "$HOME/.cache/sccache-for-docker:$CONTAINER_HOME/.cache/sccache"
+      )
 
       # s3
       if [ -n "$AWS_ACCESS_KEY_ID" ]; then

ci/docker/Dockerfile

@@ -78,7 +78,9 @@ RUN \
   rustup component add llvm-tools-preview --toolchain=$RUST_NIGHTLY_VERSION && \
   rustup target add wasm32-unknown-unknown && \
   cargo install cargo-audit && \
-  cargo install cargo-hack && \
+  # uncomment once the dcou-parition related patch is upstreamed...
+  # cargo install cargo-hack && \
+  cargo install --git https://github.com/anza-xyz/cargo-hack.git --rev 5e59c3ec6c661c02601487c0d4b2a2649fe06c9f cargo-hack && \
   cargo install cargo-sort && \
   cargo install mdbook && \
   cargo install mdbook-linkcheck && \

ci/test-dev-context-only-utils.sh

@@ -1,6 +1,29 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+source ./ci/_
+
+(unset RUSTC_WRAPPER; cargo install --force --git https://github.com/anza-xyz/cargo-hack.git --rev 5e59c3ec6c661c02601487c0d4b2a2649fe06c9f cargo-hack)
+
+# Here, experimentally switch the sccache storage from GCS to local disk by
+# `unset`-ing gcs credentials so that sccache automatically falls back to the
+# local disk storage.
+unset SCCACHE_GCS_KEY_PATH SCCACHE_GCS_BUCKET SCCACHE_GCS_RW_MODE SCCACHE_GCS_KEY_PREFIX
+
+# sccache's default is 10G, but our boxes have far more storage. :)
+export SCCACHE_CACHE_SIZE="200G"
+
+# Disable incremental compilation as this is documented as not-compatible with
+# sccache at https://github.com/mozilla/sccache/blob/v0.9.1/README.md#rust
+# > Incrementally compiled crates cannot be cached.
+export CARGO_INCREMENTAL=0
+
+_ sccache --show-stats
 
 scripts/check-dev-context-only-utils.sh check-all-targets "$@"
 scripts/check-dev-context-only-utils.sh check-bins-and-lib "$@"
+
+# This shows final stats while stopping the sccache background server as well
+# for later normal sccache use (if any). Remember that sccache is now
+# temporarily and experimtally running with the local disk storage.
+_ sccache --stop-server

scripts/check-dev-context-only-utils.sh

@@ -151,6 +151,15 @@ fi
 # consistency with other CI steps and for the possibility of new similar lints.
 export RUSTFLAGS="-D warnings -Z threads=8 $RUSTFLAGS"
 
+# As this environment value is used by the rather deep crate of our dep graph
+# (solana-varsion), this could invalidate significant portion of caches when
+# this changes just with a new tiny commit. Technically, it's possible for
+# CI_COMMIT to affect the outcome of compilation via build.rs, but it's
+# extremely unrealistic for such diverting compilation behaviors to be desired
+# as a sane use-case. So, just unset CI_COMMIT unconditionally to increase
+# cache efficiency.
+unset CI_COMMIT
+
 if [[ $mode = "check-bins-and-lib" || $mode = "full" ]]; then
   _ cargo "+${rust_nightly}" hack "$@" check
 fi