cli: also clone program data account (#1495)

.github/workflows/tests.yaml

@@ -319,6 +319,8 @@ jobs:
             path: tests/floats
           - cmd: cd tests/safety-checks && ./test.sh
             path: tests/safety-checks
+          - cmd: cd tests/validator-clone && yarn && anchor test --skip-lint
+            path: tests/validator-clone
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/setup/

CHANGELOG.md

@@ -19,6 +19,7 @@ incremented for features.
 * spl: Add support for revoke instruction ([#1493](https://github.com/project-serum/anchor/pull/1493)).
 * cli: Add support for `anchor idl fetch` to work outside anchor workspace ([#1509](https://github.com/project-serum/anchor/pull/1509)).
 * ts: Add provider parameter to `Spl.token` factory method ([#1597](https://github.com/project-serum/anchor/pull/1597)).
+* cli: [[test.validator.clone]] also clones the program data account of programs owned by the bpf upgradeable loader ([#1481](https://github.com/project-serum/anchor/issues/1481)).
 
 ### Fixes
 
@@ -40,6 +41,7 @@ incremented for features.
 * cli: Fix rust template ([#1488](https://github.com/project-serum/anchor/pull/1488)).
 * lang: Handle array sizes with variable sizes in events and array size casting in IDL parsing ([#1485](https://github.com/project-serum/anchor/pull/1485))
 
+
 ## [0.22.0] - 2022-02-20
 
 ### Features

cli/src/lib.rs

@@ -33,11 +33,13 @@ use solana_sdk::sysvar;
 use solana_sdk::transaction::Transaction;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 use std::process::{Child, Stdio};
+use std::str::FromStr;
 use std::string::ToString;
 use tar::Archive;
 
@@ -1952,7 +1954,8 @@ fn validator_flags(cfg: &WithPath<Config>) -> Result<Vec<String>> {
             }
         }
         if let Some(validator) = &test.validator {
-            for (key, value) in serde_json::to_value(validator)?.as_object().unwrap() {
+            let entries = serde_json::to_value(validator)?;
+            for (key, value) in entries.as_object().unwrap() {
                 if key == "ledger" {
                     // Ledger flag is a special case as it is passed separately to the rest of
                     // these validator flags.
@@ -1966,10 +1969,58 @@ fn validator_flags(cfg: &WithPath<Config>) -> Result<Vec<String>> {
                         flags.push(entry["filename"].as_str().unwrap().to_string());
                     }
                 } else if key == "clone" {
-                    for entry in value.as_array().unwrap() {
+                    // Client for fetching accounts data
+                    let client = if let Some(url) = entries["url"].as_str() {
+                        RpcClient::new(url.to_string())
+                    } else {
+                        return Err(anyhow!(
+                    "Validator url for Solana's JSON RPC should be provided in order to clone accounts from it"
+                ));
+                    };
+
+                    let mut pubkeys = value
+                        .as_array()
+                        .unwrap()
+                        .iter()
+                        .map(|entry| {
+                            let address = entry["address"].as_str().unwrap();
+                            Pubkey::from_str(address)
+                                .map_err(|_| anyhow!("Invalid pubkey {}", address))
+                        })
+                        .collect::<Result<HashSet<Pubkey>>>()?;
+
+                    let accounts_keys = pubkeys.iter().cloned().collect::<Vec<_>>();
+                    let accounts = client
+                        .get_multiple_accounts_with_commitment(
+                            &accounts_keys,
+                            CommitmentConfig::default(),
+                        )?
+                        .value;
+
+                    // Check if there are program accounts
+                    for (account, acc_key) in accounts.iter().zip(accounts_keys) {
+                        if let Some(account) = account {
+                            if account.owner == bpf_loader_upgradeable::id() {
+                                let upgradable: UpgradeableLoaderState = account
+                                    .deserialize_data()
+                                    .map_err(|_| anyhow!("Invalid program account {}", acc_key))?;
+
+                                if let UpgradeableLoaderState::Program {
+                                    programdata_address,
+                                } = upgradable
+                                {
+                                    pubkeys.insert(programdata_address);
+                                }
+                            }
+                        } else {
+                            return Err(anyhow!("Account {} not found", acc_key));
+                        }
+                    }
+
+                    for pubkey in &pubkeys {
                         // Push the clone flag for each array entry
                         flags.push("--clone".to_string());
-                        flags.push(entry["address"].as_str().unwrap().to_string());
+                        flags.push(pubkey.to_string());
                     }
                 } else {
                     // Remaining validator flags are non-array types

tests/package.json

@@ -29,6 +29,7 @@
     "sysvars",
     "tictactoe",
     "typescript",
+    "validator-clone",
     "zero-copy",
     "declare-id"
   ],

tests/validator-clone/Anchor.toml

@@ -0,0 +1,35 @@
+[features]
+seeds = false
+[programs.localnet]
+validator_clone = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"
+
+[registry]
+url = "https://anchor.projectserum.com"
+
+[provider]
+cluster = "localnet"
+wallet = "~/.config/solana/id.json"
+
+[scripts]
+test = "yarn run ts-mocha -p ./tsconfig.json -t 1000000 tests/**/*.ts"
+
+[test.validator]
+url = "https://api.mainnet-beta.solana.com"
+
+[[test.validator.clone]]
+address = "AqH29mZfQFgRpfwaPoTMWSKJ5kqauoc1FwVBRksZyQrt"
+
+[[test.validator.clone]]
+address = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"
+
+[[test.validator.clone]]
+address = "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s"
+
+[[test.validator.clone]]
+address = "So1endDq2YkqhipRh3WViPa8hdiSpxWy6z3Z6tMCpAo"
+
+[[test.validator.clone]]
+address = "mv3ekLzLbnVPNxjSKvqBpU3ZeZXPQdEC3bp5MDEBG68"
+
+[[test.validator.clone]]
+address = "8DKwAVrCEVStDYNPCsmxHtUj8LH9oXNtkVRrBfpNKvhp"

tests/validator-clone/Cargo.toml

@@ -0,0 +1,4 @@
+[workspace]
+members = [
+    "programs/*"
+]

tests/validator-clone/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "validator-clone",
+  "version": "0.22.0",
+  "license": "(MIT OR Apache-2.0)",
+  "homepage": "https://github.com/project-serum/anchor#readme",
+  "bugs": {
+    "url": "https://github.com/project-serum/anchor/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/project-serum/anchor.git"
+  },
+  "engines": {
+    "node": ">=11"
+  },
+  "scripts": {
+    "test": "anchor run test-with-build"
+  }
+}

tests/validator-clone/programs/validator-clone/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "validator-clone"
+version = "0.1.0"
+description = "Created with Anchor"
+edition = "2018"
+
+[lib]
+crate-type = ["cdylib", "lib"]
+name = "validator_clone"
+
+[features]
+no-entrypoint = []
+no-idl = []
+no-log-ix-name = []
+cpi = ["no-entrypoint"]
+default = []
+
+[dependencies]
+anchor-lang = { path = "../../../../lang" }

tests/validator-clone/programs/validator-clone/Xargo.toml

@@ -0,0 +1,2 @@
+[target.bpfel-unknown-unknown.dependencies.std]
+features = []

tests/validator-clone/programs/validator-clone/src/lib.rs

@@ -0,0 +1,15 @@
+use anchor_lang::prelude::*;
+
+declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
+
+#[program]
+pub mod validator_clone {
+    use super::*;
+
+    pub fn initialize(_ctx: Context<Initialize>) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[derive(Accounts)]
+pub struct Initialize {}

tests/validator-clone/tests/validator-clone.ts

@@ -0,0 +1,70 @@
+import * as anchor from "@project-serum/anchor";
+import { Program } from "@project-serum/anchor";
+import assert from "assert";
+import { ValidatorClone } from "../target/types/validator_clone";
+
+describe("validator-clone", () => {
+  // Configure the client to use the local cluster.
+  anchor.setProvider(anchor.Provider.env());
+
+  const program = anchor.workspace.ValidatorClone as Program<ValidatorClone>;
+  const connection = program.provider.connection;
+
+  it("Cloned non-executable account", async () => {
+    // Metadata program upgrade authority
+    const account = "AqH29mZfQFgRpfwaPoTMWSKJ5kqauoc1FwVBRksZyQrt";
+    const [accountInfo] = await anchor.utils.rpc.getMultipleAccounts(
+      connection,
+      [new anchor.web3.PublicKey(account)]
+    );
+    assert.ok(accountInfo !== null, "Account " + account + " not found");
+  });
+
+  it("Cloned bpf2-program account", async () => {
+    // Memo program
+    const account = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr";
+    const [accountInfo] = await anchor.utils.rpc.getMultipleAccounts(
+      connection,
+      [new anchor.web3.PublicKey(account)]
+    );
+    assert.ok(accountInfo !== null, "Account " + account + " not found");
+  });
+
+  it("Cloned bpf3-program accounts and their program data", async () => {
+    const accounts = [
+      // Metadata program
+      "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s",
+      // Metadata program executable data
+      "PwDiXFxQsGra4sFFTT8r1QWRMd4vfumiWC1jfWNfdYT",
+      // Solend program
+      "So1endDq2YkqhipRh3WViPa8hdiSpxWy6z3Z6tMCpAo",
+      // Solend program executable data
+      "DMCvGv1fS5rMcAvEDPDDBawPqbDRSzJh2Bo6qXCmgJkR",
+    ];
+    const accountInfos = await anchor.utils.rpc.getMultipleAccounts(
+      connection,
+      accounts.map((acc) => new anchor.web3.PublicKey(acc))
+    );
+
+    accountInfos.forEach((acc, i) => {
+      assert.ok(acc !== null, "Account " + accounts[i] + " not found");
+    });
+  });
+
+  it("Cloned bpf3-program account and its program data (both explicitly declared)", async () => {
+    const accounts = [
+      // Mango v3 program
+      "mv3ekLzLbnVPNxjSKvqBpU3ZeZXPQdEC3bp5MDEBG68",
+      // Mango v3 program executable data
+      "8DKwAVrCEVStDYNPCsmxHtUj8LH9oXNtkVRrBfpNKvhp",
+    ];
+    const accountInfos = await anchor.utils.rpc.getMultipleAccounts(
+      connection,
+      accounts.map((acc) => new anchor.web3.PublicKey(acc))
+    );
+
+    accountInfos.forEach((acc, i) => {
+      assert.ok(acc !== null, "Account " + accounts[i] + " not found");
+    });
+  });
+});

tests/validator-clone/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "types": ["mocha", "chai"],
+    "typeRoots": ["./node_modules/@types"],
+    "lib": ["es2015"],
+    "module": "commonjs",
+    "target": "es6",
+    "esModuleInterop": true
+  }
+}