Facility to generate a blocktree prune list using ledger tool (#5041)

automerge

Cargo.lock

@@ -2605,7 +2605,10 @@ version = "0.17.0"
 dependencies = [
  "assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana 0.17.0",
  "solana-logger 0.17.0",
  "solana-runtime 0.17.0",

core/src/blocktree_processor.rs

@@ -142,6 +142,7 @@ pub fn process_blocktree(
     genesis_block: &GenesisBlock,
     blocktree: &Blocktree,
     account_paths: Option<String>,
+    verify_ledger: bool,
 ) -> result::Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
     let now = Instant::now();
     info!("processing ledger...");
@@ -205,7 +206,7 @@ pub fn process_blocktree(
         }
 
         if !entries.is_empty() {
-            if !entries.verify(&last_entry_hash) {
+            if verify_ledger && !entries.verify(&last_entry_hash) {
                 warn!(
                     "Ledger proof of history failed at slot: {}, entry: {}",
                     slot, entry_height
@@ -374,7 +375,7 @@ pub mod tests {
         fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 2, 1, blockhash);
 
         let (mut _bank_forks, bank_forks_info, _) =
-            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+            process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
 
         assert_eq!(bank_forks_info.len(), 1);
         assert_eq!(
@@ -433,7 +434,7 @@ pub mod tests {
         blocktree.set_roots(&[4, 1, 0]).unwrap();
 
         let (bank_forks, bank_forks_info, _) =
-            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+            process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
 
         assert_eq!(bank_forks_info.len(), 1); // One fork, other one is ignored b/c not a descendant of the root
 
@@ -507,7 +508,7 @@ pub mod tests {
         blocktree.set_roots(&[0, 1]).unwrap();
 
         let (bank_forks, bank_forks_info, _) =
-            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+            process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
 
         assert_eq!(bank_forks_info.len(), 2); // There are two forks
         assert_eq!(
@@ -588,7 +589,7 @@ pub mod tests {
 
         // Check that we can properly restart the ledger / leader scheduler doesn't fail
         let (bank_forks, bank_forks_info, _) =
-            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+            process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
 
         assert_eq!(bank_forks_info.len(), 1); // There is one fork
         assert_eq!(
@@ -724,7 +725,7 @@ pub mod tests {
             .unwrap();
         let entry_height = genesis_block.ticks_per_slot + entries.len() as u64;
         let (bank_forks, bank_forks_info, _) =
-            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+            process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
 
         assert_eq!(bank_forks_info.len(), 1);
         assert_eq!(bank_forks.root(), 0);
@@ -755,7 +756,7 @@ pub mod tests {
 
         let blocktree = Blocktree::open(&ledger_path).unwrap();
         let (bank_forks, bank_forks_info, _) =
-            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+            process_blocktree(&genesis_block, &blocktree, None, true).unwrap();
 
         assert_eq!(bank_forks_info.len(), 1);
         assert_eq!(

core/src/local_cluster.rs

@@ -177,6 +177,7 @@ impl LocalCluster {
             &leader_voting_keypair,
             &leader_storage_keypair,
             None,
+            true,
             &config.validator_configs[0],
         );
 
@@ -308,6 +309,7 @@ impl LocalCluster {
             &voting_keypair,
             &storage_keypair,
             Some(&self.entry_point_info),
+            true,
             &validator_config,
         );
 
@@ -561,6 +563,7 @@ impl Cluster for LocalCluster {
             &fullnode_info.voting_keypair,
             &fullnode_info.storage_keypair,
             None,
+            true,
             config,
         );
 

core/src/validator.rs

@@ -82,6 +82,7 @@ impl Validator {
         voting_keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         entrypoint_info_option: Option<&ContactInfo>,
+        verify_ledger: bool,
         config: &ValidatorConfig,
     ) -> Self {
         warn!("CUDA is {}abled", if cfg!(cuda) { "en" } else { "dis" });
@@ -102,6 +103,7 @@ impl Validator {
             ledger_path,
             config.account_paths.clone(),
             config.snapshot_path.clone(),
+            verify_ledger,
         );
 
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
@@ -302,6 +304,7 @@ fn get_bank_forks(
     blocktree: &Blocktree,
     account_paths: Option<String>,
     snapshot_path: Option<String>,
+    verify_ledger: bool,
 ) -> (BankForks, Vec<BankForksInfo>, LeaderScheduleCache) {
     if snapshot_path.is_some() {
         let bank_forks =
@@ -319,8 +322,13 @@ fn get_bank_forks(
         }
     }
     let (mut bank_forks, bank_forks_info, leader_schedule_cache) =
-        blocktree_processor::process_blocktree(&genesis_block, &blocktree, account_paths)
-            .expect("process_blocktree failed");
+        blocktree_processor::process_blocktree(
+            &genesis_block,
+            &blocktree,
+            account_paths,
+            verify_ledger,
+        )
+        .expect("process_blocktree failed");
     if snapshot_path.is_some() {
         bank_forks.set_snapshot_config(snapshot_path);
         let _ = bank_forks.add_snapshot(0, 0);
@@ -332,6 +340,7 @@ pub fn new_banks_from_blocktree(
     blocktree_path: &str,
     account_paths: Option<String>,
     snapshot_path: Option<String>,
+    verify_ledger: bool,
 ) -> (
     BankForks,
     Vec<BankForksInfo>,
@@ -348,8 +357,13 @@ pub fn new_banks_from_blocktree(
         Blocktree::open_with_signal(blocktree_path)
             .expect("Expected to successfully open database ledger");
 
-    let (bank_forks, bank_forks_info, leader_schedule_cache) =
-        get_bank_forks(&genesis_block, &blocktree, account_paths, snapshot_path);
+    let (bank_forks, bank_forks_info, leader_schedule_cache) = get_bank_forks(
+        &genesis_block,
+        &blocktree,
+        account_paths,
+        snapshot_path,
+        verify_ledger,
+    );
 
     (
         bank_forks,
@@ -413,6 +427,7 @@ pub fn new_validator_for_tests() -> (Validator, ContactInfo, Keypair, String) {
         &voting_keypair,
         &storage_keypair,
         None,
+        true,
         &ValidatorConfig::default(),
     );
     discover_cluster(&contact_info.gossip, 1).expect("Node startup failed");
@@ -448,6 +463,7 @@ mod tests {
             &voting_keypair,
             &storage_keypair,
             Some(&leader_node.info),
+            true,
             &ValidatorConfig::default(),
         );
         validator.close().unwrap();
@@ -479,6 +495,7 @@ mod tests {
                     &voting_keypair,
                     &storage_keypair,
                     Some(&leader_node.info),
+                    true,
                     &ValidatorConfig::default(),
                 )
             })

core/tests/tvu.rs

@@ -97,7 +97,7 @@ fn test_replay() {
         completed_slots_receiver,
         leader_schedule_cache,
         _,
-    ) = validator::new_banks_from_blocktree(&blocktree_path, None, None);
+    ) = validator::new_banks_from_blocktree(&blocktree_path, None, None, true);
     let working_bank = bank_forks.working_bank();
     assert_eq!(
         working_bank.get_balance(&mint_keypair.pubkey()),

ledger-tool/Cargo.toml

@@ -10,7 +10,10 @@ homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
+serde = "1.0.94"
+serde_derive = "1.0.94"
 serde_json = "1.0.40"
+serde_yaml = "0.8.9"
 solana = { path = "../core", version = "0.17.0" }
 solana-logger = { path = "../logger", version = "0.17.0" }
 solana-runtime = { path = "../runtime", version = "0.17.0" }

ledger-tool/src/main.rs

@@ -2,8 +2,11 @@ use clap::{crate_description, crate_name, crate_version, value_t, App, Arg, SubC
 use solana::blocktree::Blocktree;
 use solana::blocktree_processor::process_blocktree;
 use solana_sdk::genesis_block::GenesisBlock;
+use std::collections::BTreeMap;
+use std::fs::File;
 use std::io::{stdout, Write};
 use std::process::exit;
+use std::str::FromStr;
 
 #[derive(PartialEq)]
 enum LedgerOutputMethod {
@@ -58,6 +61,7 @@ fn output_ledger(blocktree: Blocktree, starting_slot: u64, method: LedgerOutputM
 }
 
 fn main() {
+    const DEFAULT_ROOT_COUNT: &str = "1";
     solana_logger::setup();
     let matches = App::new(crate_name!())
         .about(crate_description!())
@@ -82,6 +86,36 @@ fn main() {
         .subcommand(SubCommand::with_name("print").about("Print the ledger"))
         .subcommand(SubCommand::with_name("json").about("Print the ledger in JSON format"))
         .subcommand(SubCommand::with_name("verify").about("Verify the ledger's PoH"))
+        .subcommand(SubCommand::with_name("prune").about("Prune the ledger at the block height").arg(
+            Arg::with_name("slot_list")
+                .long("slot-list")
+                .value_name("FILENAME")
+                .takes_value(true)
+                .help("The location of the YAML file with a list of rollback slot heights and hashes"),
+        ))
+        .subcommand(SubCommand::with_name("list-roots").about("Output upto last <num-roots> root hashes and their heights starting at the given block height").arg(
+            Arg::with_name("max_height")
+                .long("max-height")
+                .value_name("NUM")
+                .takes_value(true)
+                .required(true)
+                .help("Maximum block height"),
+        ).arg(
+            Arg::with_name("slot_list")
+                .long("slot-list")
+                .value_name("FILENAME")
+                .required(false)
+                .takes_value(true)
+                .help("The location of the output YAML file. A list of rollback slot heights and hashes will be written to the file."),
+        ).arg(
+            Arg::with_name("num_roots")
+                .long("num-roots")
+                .value_name("NUM")
+                .takes_value(true)
+                .default_value(DEFAULT_ROOT_COUNT)
+                .required(false)
+                .help("Number of roots in the output"),
+        ))
         .get_matches();
 
     let ledger_path = matches.value_of("ledger").unwrap();
@@ -113,7 +147,7 @@ fn main() {
         }
         ("verify", _) => {
             println!("Verifying ledger...");
-            match process_blocktree(&genesis_block, &blocktree, None) {
+            match process_blocktree(&genesis_block, &blocktree, None, true) {
                 Ok((_bank_forks, bank_forks_info, _)) => {
                     println!("{:?}", bank_forks_info);
                 }
@@ -123,6 +157,97 @@ fn main() {
                 }
             }
         }
+        ("prune", Some(args_matches)) => {
+            if let Some(prune_file_path) = args_matches.value_of("slot_list") {
+                let prune_file = File::open(prune_file_path.to_string()).unwrap();
+                let slot_hashes: BTreeMap<u64, String> =
+                    serde_yaml::from_reader(prune_file).unwrap();
+
+                let iter = blocktree
+                    .rooted_slot_iterator(0)
+                    .expect("Failed to get rooted slot");
+
+                let potential_hashes: Vec<_> = iter
+                    .filter_map(|(slot, meta)| {
+                        let blockhash = blocktree
+                            .get_slot_entries(slot, meta.last_index, Some(1))
+                            .unwrap()
+                            .first()
+                            .unwrap()
+                            .hash
+                            .to_string();
+
+                        slot_hashes.get(&slot).and_then(|hash| {
+                            if *hash == blockhash {
+                                Some((slot, blockhash))
+                            } else {
+                                None
+                            }
+                        })
+                    })
+                    .collect();
+
+                let (target_slot, target_hash) = potential_hashes
+                    .last()
+                    .expect("Failed to find a valid slot");
+                println!("Prune at slot {:?} hash {:?}", target_slot, target_hash);
+                // ToDo: Do the actual pruning of the database
+            }
+        }
+        ("list-roots", Some(args_matches)) => {
+            let max_height = if let Some(height) = args_matches.value_of("max_height") {
+                usize::from_str(height).expect("Maximum height must be a number")
+            } else {
+                panic!("Maximum height must be provided");
+            };
+            let num_roots = if let Some(roots) = args_matches.value_of("num_roots") {
+                usize::from_str(roots).expect("Number of roots must be a number")
+            } else {
+                usize::from_str(DEFAULT_ROOT_COUNT).unwrap()
+            };
+
+            let iter = blocktree
+                .rooted_slot_iterator(0)
+                .expect("Failed to get rooted slot");
+
+            let slot_hash: Vec<_> = iter
+                .filter_map(|(slot, meta)| {
+                    if slot <= max_height as u64 {
+                        let blockhash = blocktree
+                            .get_slot_entries(slot, meta.last_index, Some(1))
+                            .unwrap()
+                            .first()
+                            .unwrap()
+                            .hash;
+                        Some((slot, blockhash))
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+
+            let mut output_file: Box<Write> = if let Some(path) = args_matches.value_of("slot_list")
+            {
+                match File::create(path) {
+                    Ok(file) => Box::new(file),
+                    _ => Box::new(stdout()),
+                }
+            } else {
+                Box::new(stdout())
+            };
+
+            slot_hash
+                .into_iter()
+                .rev()
+                .enumerate()
+                .for_each(|(i, (slot, hash))| {
+                    if i < num_roots {
+                        output_file
+                            .write_all(format!("{:?}: {:?}\n", slot, hash).as_bytes())
+                            .expect("failed to write");
+                    }
+                });
+        }
         ("", _) => {
             eprintln!("{}", matches.usage());
             exit(1);

validator/src/main.rs

@@ -162,6 +162,12 @@ fn main() {
                 .value_name("PATHS")
                 .takes_value(true)
                 .help("Snapshot path"),
+        )
+        .arg(
+            clap::Arg::with_name("skip_ledger_verify")
+                .long("skip-ledger-verify")
+                .takes_value(false)
+                .help("Skip ledger verification at node bootup"),
         )
          .get_matches();
 
@@ -267,6 +273,8 @@ fn main() {
         node.info.rpc_pubsub = SocketAddr::new(gossip_addr.ip(), port_number + 1);
     };
 
+    let verify_ledger = !matches.is_present("skip_ledger_verify");
+
     let validator = Validator::new(
         node,
         &keypair,
@@ -275,6 +283,7 @@ fn main() {
         &Arc::new(voting_keypair),
         &Arc::new(storage_keypair),
         cluster_entrypoint.as_ref(),
+        verify_ledger,
         &validator_config,
     );