ci: Add Anchor and Anchor projects to the downstream build (#22098)

* ci: Add Anchor and Anchor projects to the downstream build

* Separate downstream anchor projects into separate step

* Decrease anchor project build time

ci/buildkite-pipeline.sh

@@ -227,6 +227,31 @@ EOF
       "downstream-projects skipped as no relevant files were modified"
   fi
 
+  # Downstream Anchor projects backwards compatibility
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+             ^scripts/build-downstream-anchor-projects.sh \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "scripts/build-downstream-anchor-projects.sh"
+    name: "downstream-anchor-projects"
+    timeout_in_minutes: 10
+EOF
+  else
+    annotate --style info \
+      "downstream-anchor-projects skipped as no relevant files were modified"
+  fi
+
   # Wasm support
   if affects \
              ^ci/test-wasm.sh \

scripts/build-downstream-anchor-projects.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# Builds known downstream projects against local solana source
+#
+
+set -e
+cd "$(dirname "$0")"/..
+source ci/_
+source scripts/patch-crates.sh
+source scripts/read-cargo-variable.sh
+
+solana_ver=$(readCargoVariable version sdk/Cargo.toml)
+solana_dir=$PWD
+cargo="$solana_dir"/cargo
+cargo_build_bpf="$solana_dir"/cargo-build-bpf
+cargo_test_bpf="$solana_dir"/cargo-test-bpf
+
+mkdir -p target/downstream-projects-anchor
+cd target/downstream-projects-anchor
+
+update_anchor_dependencies() {
+  declare project_root="$1"
+  declare anchor_ver="$2"
+  declare tomls=()
+  while IFS='' read -r line; do tomls+=("$line"); done < <(find "$project_root" -name Cargo.toml)
+
+  sed -i -e "s#\(anchor-lang = \"\)[^\"]*\(\"\)#\1=$anchor_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(anchor-spl = \"\)[^\"]*\(\"\)#\1=$anchor_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(anchor-lang = { version = \"\)[^\"]*\(\"\)#\1=$anchor_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(anchor-spl = { version = \"\)[^\"]*\(\"\)#\1=$anchor_ver\2#g" "${tomls[@]}" || return $?
+}
+
+patch_crates_io_anchor() {
+  declare Cargo_toml="$1"
+  declare anchor_dir="$2"
+  cat >> "$Cargo_toml" <<EOF
+anchor-lang = { path = "$anchor_dir/lang" }
+anchor-spl = { path = "$anchor_dir/spl" }
+EOF
+}
+
+# NOTE This isn't run in a subshell to get $anchor_dir and $anchor_ver
+anchor() {
+  set -x
+  rm -rf anchor
+  git clone https://github.com/project-serum/anchor.git
+  cd anchor
+
+  update_solana_dependencies . "$solana_ver"
+  patch_crates_io_solana Cargo.toml "$solana_dir"
+
+  $cargo build
+  $cargo test
+
+  anchor_dir=$PWD
+  anchor_ver=$(readCargoVariable version "$anchor_dir"/lang/Cargo.toml)
+
+  cd "$solana_dir"/target/downstream-projects-anchor
+}
+
+mango() {
+  (
+    set -x
+    rm -rf mango-v3
+    git clone https://github.com/blockworks-foundation/mango-v3
+    cd mango-v3
+
+    update_solana_dependencies . "$solana_ver"
+    update_anchor_dependencies . "$anchor_ver"
+    patch_crates_io_solana Cargo.toml "$solana_dir"
+    patch_crates_io_anchor Cargo.toml "$anchor_dir"
+
+    $cargo build
+    $cargo test
+    $cargo_build_bpf
+    $cargo_test_bpf
+  )
+}
+
+metaplex() {
+  (
+    set -x
+    rm -rf metaplex-program-library
+    git clone https://github.com/metaplex-foundation/metaplex-program-library
+    cd metaplex-program-library
+
+    update_solana_dependencies . "$solana_ver"
+    update_anchor_dependencies . "$anchor_ver"
+    patch_crates_io_solana Cargo.toml "$solana_dir"
+    patch_crates_io_anchor Cargo.toml "$anchor_dir"
+
+    $cargo build
+    $cargo test
+    $cargo_build_bpf
+    $cargo_test_bpf
+  )
+}
+
+_ anchor
+#_ metaplex
+#_ mango

scripts/build-downstream-projects.sh

@@ -6,6 +6,7 @@
 set -e
 cd "$(dirname "$0")"/..
 source ci/_
+source scripts/patch-crates.sh
 source scripts/read-cargo-variable.sh
 
 solana_ver=$(readCargoVariable version sdk/Cargo.toml)
@@ -17,28 +18,6 @@ cargo_test_bpf="$solana_dir"/cargo-test-bpf
 mkdir -p target/downstream-projects
 cd target/downstream-projects
 
-update_solana_dependencies() {
-  declare tomls=()
-  while IFS='' read -r line; do tomls+=("$line"); done < <(find "$1" -name Cargo.toml)
-
-  sed -i -e "s#\(solana-program = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-program-test = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-sdk = \"\).*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-sdk = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-client = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-  sed -i -e "s#\(solana-client = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
-}
-
-patch_crates_io() {
-  cat >> "$1" <<EOF
-[patch.crates-io]
-solana-client = { path = "$solana_dir/client" }
-solana-program = { path = "$solana_dir/sdk/program" }
-solana-program-test = { path = "$solana_dir/program-test" }
-solana-sdk = { path = "$solana_dir/sdk" }
-EOF
-}
-
 example_helloworld() {
   (
     set -x
@@ -46,8 +25,8 @@ example_helloworld() {
     git clone https://github.com/solana-labs/example-helloworld.git
     cd example-helloworld
 
-    update_solana_dependencies src/program-rust
-    patch_crates_io src/program-rust/Cargo.toml
+    update_solana_dependencies src/program-rust "$solana_ver"
+    patch_crates_io_solana src/program-rust/Cargo.toml "$solana_dir"
     echo "[workspace]" >> src/program-rust/Cargo.toml
 
     $cargo_build_bpf \
@@ -80,9 +59,9 @@ serum_dex() {
     git clone https://github.com/project-serum/serum-dex.git
     cd serum-dex
 
-    update_solana_dependencies .
-    patch_crates_io Cargo.toml
-    patch_crates_io dex/Cargo.toml
+    update_solana_dependencies . "$solana_ver"
+    patch_crates_io_solana Cargo.toml "$solana_dir"
+    patch_crates_io_solana dex/Cargo.toml "$solana_dir"
     cat >> dex/Cargo.toml <<EOF
 [workspace]
 exclude = [
@@ -100,7 +79,6 @@ EOF
   )
 }
 
-
 _ example_helloworld
 _ spl
 _ serum_dex

scripts/patch-crates.sh

@@ -0,0 +1,30 @@
+# source this file
+
+update_solana_dependencies() {
+  declare project_root="$1"
+  declare solana_ver="$2"
+  declare tomls=()
+  while IFS='' read -r line; do tomls+=("$line"); done < <(find "$project_root" -name Cargo.toml)
+
+  sed -i -e "s#\(solana-program = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-program-test = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-sdk = \"\).*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-sdk = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-client = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-client = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-clap-utils = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+  sed -i -e "s#\(solana-clap-utils = { version = \"\)[^\"]*\(\"\)#\1=$solana_ver\2#g" "${tomls[@]}" || return $?
+}
+
+patch_crates_io_solana() {
+  declare Cargo_toml="$1"
+  declare solana_dir="$2"
+  cat >> "$Cargo_toml" <<EOF
+[patch.crates-io]
+solana-clap-utils = { path = "$solana_dir/clap-utils" }
+solana-client = { path = "$solana_dir/client" }
+solana-program = { path = "$solana_dir/sdk/program" }
+solana-program-test = { path = "$solana_dir/program-test" }
+solana-sdk = { path = "$solana_dir/sdk" }
+EOF
+}