Move genesis related helpers from agave-snapshots to solana_genesis_utils (#8658)

Cargo.lock

@@ -8018,6 +8018,7 @@ dependencies = [
  "solana-frozen-abi",
  "solana-frozen-abi-macro",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-hard-forks",
@@ -8521,6 +8522,7 @@ dependencies = [
  "solana-fee-calculator",
  "solana-genesis",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-inflation",
  "solana-keypair",
  "solana-ledger",
@@ -8581,6 +8583,7 @@ dependencies = [
  "solana-genesis-config",
  "solana-hash",
  "solana-rpc-client",
+ "thiserror 2.0.17",
 ]
 
 [[package]]
@@ -8917,6 +8920,7 @@ dependencies = [
  "solana-frozen-abi",
  "solana-frozen-abi-macro",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-hash",
  "solana-instruction",
  "solana-keypair",
@@ -9069,6 +9073,7 @@ dependencies = [
  "solana-entry",
  "solana-epoch-schedule",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-gossip",
  "solana-hard-forks",
  "solana-hash",
@@ -11027,6 +11032,7 @@ dependencies = [
  "solana-epoch-schedule",
  "solana-feature-gate-interface",
  "solana-fee-calculator",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-inflation",
@@ -11729,6 +11735,7 @@ dependencies = [
  "serial_test",
  "solana-clock",
  "solana-entry",
+ "solana-genesis-utils",
  "solana-gossip",
  "solana-hash",
  "solana-keypair",

core/Cargo.toml

@@ -110,6 +110,7 @@ solana-frozen-abi-macro = { workspace = true, optional = true, features = [
     "frozen-abi",
 ] }
 solana-genesis-config = { workspace = true }
+solana-genesis-utils = { workspace = true }
 solana-geyser-plugin-manager = { workspace = true }
 solana-gossip = { workspace = true, features = ["agave-unstable-api"] }
 solana-hard-forks = { workspace = true }

core/src/validator.rs

@@ -35,12 +35,7 @@ use {
         tvu::{Tvu, TvuConfig, TvuSockets},
     },
     agave_snapshots::{
-        hardened_unpack::{
-            open_genesis_config, OpenGenesisConfigError, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
-        },
-        snapshot_config::SnapshotConfig,
-        snapshot_hash::StartingSnapshotHashes,
-        SnapshotInterval,
+        snapshot_config::SnapshotConfig, snapshot_hash::StartingSnapshotHashes, SnapshotInterval,
     },
     anyhow::{anyhow, Context, Result},
     crossbeam_channel::{bounded, unbounded, Receiver},
@@ -61,6 +56,9 @@ use {
     solana_entry::poh::compute_hash_time,
     solana_epoch_schedule::MAX_LEADER_SCHEDULE_EPOCH_OFFSET,
     solana_genesis_config::GenesisConfig,
+    solana_genesis_utils::{
+        open_genesis_config, OpenGenesisConfigError, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+    },
     solana_geyser_plugin_manager::{
         geyser_plugin_service::GeyserPluginService, GeyserPluginManagerRequest,
     },

dev-bins/Cargo.lock

@@ -152,6 +152,7 @@ dependencies = [
  "solana-entry",
  "solana-feature-gate-interface",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-hash",
@@ -2943,6 +2944,20 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "hyper-rustls"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
+dependencies = [
+ "futures-util",
+ "http 0.2.12",
+ "hyper 0.14.32",
+ "rustls 0.21.12",
+ "tokio",
+ "tokio-rustls 0.24.1",
+]
+
 [[package]]
 name = "hyper-rustls"
 version = "0.27.7"
@@ -3219,6 +3234,19 @@ dependencies = [
  "rayon",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.17.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+dependencies = [
+ "console 0.15.11",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width 0.2.2",
+ "web-time",
+]
+
 [[package]]
 name = "indicatif"
 version = "0.18.0"
@@ -4088,6 +4116,12 @@ dependencies = [
  "syn 2.0.106",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -5016,6 +5050,7 @@ version = "0.11.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
 dependencies = [
+ "async-compression",
  "base64 0.21.7",
  "bytes",
  "encoding_rs",
@@ -5025,6 +5060,7 @@ dependencies = [
  "http 0.2.12",
  "http-body 0.4.6",
  "hyper 0.14.32",
+ "hyper-rustls 0.24.2",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -5034,6 +5070,7 @@ dependencies = [
  "once_cell",
  "percent-encoding 2.3.2",
  "pin-project-lite",
+ "rustls 0.21.12",
  "rustls-pemfile",
  "serde",
  "serde_json",
@@ -5042,11 +5079,14 @@ dependencies = [
  "system-configuration",
  "tokio",
  "tokio-native-tls",
+ "tokio-rustls 0.24.1",
+ "tokio-util 0.7.16",
  "tower-service",
  "url 2.5.7",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
+ "webpki-roots 0.25.4",
  "winreg",
 ]
 
@@ -5066,7 +5106,7 @@ dependencies = [
  "http-body 1.0.1",
  "http-body-util",
  "hyper 1.7.0",
- "hyper-rustls",
+ "hyper-rustls 0.27.7",
  "hyper-util",
  "js-sys",
  "log",
@@ -6405,7 +6445,7 @@ dependencies = [
  "clap 2.34.0",
  "console 0.16.1",
  "humantime",
- "indicatif",
+ "indicatif 0.18.0",
  "pretty-hex",
  "semver",
  "serde",
@@ -6444,7 +6484,7 @@ dependencies = [
  "futures 0.3.31",
  "futures-util",
  "indexmap 2.11.4",
- "indicatif",
+ "indicatif 0.18.0",
  "log",
  "quinn",
  "rayon",
@@ -6687,6 +6727,7 @@ dependencies = [
  "solana-fee-calculator",
  "solana-fee-structure",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-hard-forks",
@@ -6892,6 +6933,18 @@ dependencies = [
  "solana-version",
 ]
 
+[[package]]
+name = "solana-download-utils"
+version = "3.1.0"
+dependencies = [
+ "agave-snapshots",
+ "log",
+ "solana-clock",
+ "solana-file-download",
+ "solana-genesis-config",
+ "solana-runtime",
+]
+
 [[package]]
 name = "solana-ed25519-program"
 version = "3.0.0"
@@ -7061,6 +7114,18 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "solana-file-download"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6884e13cc98f58e609a9b73e3d53f728f0f743b8c15c6768cad6f6382c336c1"
+dependencies = [
+ "console 0.15.11",
+ "indicatif 0.17.11",
+ "log",
+ "reqwest 0.11.27",
+]
+
 [[package]]
 name = "solana-genesis"
 version = "3.1.0"
@@ -7086,6 +7151,7 @@ dependencies = [
  "solana-feature-gate-interface",
  "solana-fee-calculator",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-inflation",
  "solana-keypair",
  "solana-ledger",
@@ -7136,6 +7202,19 @@ dependencies = [
  "solana-time-utils",
 ]
 
+[[package]]
+name = "solana-genesis-utils"
+version = "3.1.0"
+dependencies = [
+ "agave-snapshots",
+ "log",
+ "solana-download-utils",
+ "solana-genesis-config",
+ "solana-hash",
+ "solana-rpc-client",
+ "thiserror 2.0.17",
+]
+
 [[package]]
 name = "solana-geyser-plugin-manager"
 version = "3.1.0"
@@ -7410,6 +7489,7 @@ dependencies = [
  "solana-entry",
  "solana-epoch-schedule",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-hash",
  "solana-instruction",
  "solana-keypair",
@@ -8218,7 +8298,7 @@ dependencies = [
  "bincode",
  "bs58",
  "futures 0.3.31",
- "indicatif",
+ "indicatif 0.18.0",
  "log",
  "reqwest 0.12.24",
  "reqwest-middleware",
@@ -9087,6 +9167,7 @@ dependencies = [
  "solana-epoch-schedule",
  "solana-feature-gate-interface",
  "solana-fee-calculator",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-inflation",
@@ -9170,7 +9251,7 @@ dependencies = [
  "bincode",
  "futures-util",
  "indexmap 2.11.4",
- "indicatif",
+ "indicatif 0.18.0",
  "log",
  "rayon",
  "solana-client-traits",
@@ -9574,6 +9655,7 @@ dependencies = [
  "rayon",
  "solana-clock",
  "solana-entry",
+ "solana-genesis-utils",
  "solana-gossip",
  "solana-hash",
  "solana-ledger",
@@ -11109,6 +11191,12 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "0.25.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
+
 [[package]]
 name = "webpki-roots"
 version = "0.26.11"

dev-bins/Cargo.toml

@@ -93,6 +93,7 @@ solana-feature-gate-interface = "3.0.0"
 solana-fee-calculator = "3.0.0"
 solana-genesis = { path = "../genesis", version = "=3.1.0", features = ["agave-unstable-api"] }
 solana-genesis-config = "3.0.0"
+solana-genesis-utils = { path = "../genesis-utils", version = "=3.1.0", features = ["agave-unstable-api"] }
 solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "=3.1.0", features = ["agave-unstable-api"] }
 solana-gossip = { path = "../gossip", version = "=3.1.0", features = ["agave-unstable-api"] }
 solana-hash = "3.0.0"

genesis-utils/Cargo.toml

@@ -26,3 +26,4 @@ solana-download-utils = { workspace = true }
 solana-genesis-config = { workspace = true }
 solana-hash = { workspace = true }
 solana-rpc-client = { workspace = true }
+thiserror = { workspace = true }

genesis-utils/src/lib.rs

@@ -8,7 +8,7 @@
     )
 )]
 use {
-    agave_snapshots::hardened_unpack::unpack_genesis_archive,
+    agave_snapshots::unpack_genesis_archive,
     log::*,
     solana_download_utils::download_genesis_if_missing,
     solana_genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE},
@@ -17,6 +17,8 @@ use {
     std::net::SocketAddr,
 };
 
+mod open;
+
 fn check_genesis_hash(
     genesis_config: &GenesisConfig,
     expected_genesis_hash: Option<Hash>,
@@ -130,3 +132,5 @@ pub fn download_then_check_genesis_hash(
 
     set_and_verify_expected_genesis_hash(genesis_config, expected_genesis_hash, rpc_client)
 }
+
+pub use open::{open_genesis_config, OpenGenesisConfigError, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE};

genesis-utils/src/open.rs

@@ -0,0 +1,39 @@
+use {
+    agave_snapshots::{hardened_unpack::UnpackError, unpack_genesis_archive},
+    solana_genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE},
+    std::path::Path,
+    thiserror::Error,
+};
+
+pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB
+
+#[derive(Error, Debug)]
+pub enum OpenGenesisConfigError {
+    #[error("unpack error: {0}")]
+    Unpack(#[from] UnpackError),
+    #[error("Genesis load error: {0}")]
+    Load(#[from] std::io::Error),
+}
+
+pub fn open_genesis_config(
+    ledger_path: &Path,
+    max_genesis_archive_unpacked_size: u64,
+) -> Result<GenesisConfig, OpenGenesisConfigError> {
+    match GenesisConfig::load(ledger_path) {
+        Ok(genesis_config) => Ok(genesis_config),
+        Err(load_err) => {
+            log::warn!(
+                "Failed to load genesis_config at {ledger_path:?}: {load_err}. Will attempt to \
+                 unpack genesis archive and then retry loading."
+            );
+
+            let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
+            unpack_genesis_archive(
+                &genesis_package,
+                ledger_path,
+                max_genesis_archive_unpacked_size,
+            )?;
+            GenesisConfig::load(ledger_path).map_err(OpenGenesisConfigError::Load)
+        }
+    }
+}

genesis/Cargo.toml

@@ -45,6 +45,7 @@ solana-epoch-schedule = "=3.0.0"
 solana-feature-gate-interface = "=3.0.0"
 solana-fee-calculator = "=3.0.0"
 solana-genesis-config = "=3.0.0"
+solana-genesis-utils = { workspace = true }
 solana-inflation = "=3.0.0"
 solana-keypair = "=3.0.1"
 solana-ledger = { workspace = true }

genesis/src/main.rs

@@ -3,7 +3,6 @@
 
 use {
     agave_feature_set::FEATURE_NAMES,
-    agave_snapshots::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     base64::{prelude::BASE64_STANDARD, Engine},
     clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg, ArgMatches},
     itertools::Itertools,
@@ -29,6 +28,7 @@ use {
         ValidatorAccountsFile,
     },
     solana_genesis_config::GenesisConfig,
+    solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     solana_inflation::Inflation,
     solana_keypair::{read_keypair_file, Keypair},
     solana_ledger::{blockstore::create_new_ledger, blockstore_options::LedgerColumnOptions},

ledger-tool/Cargo.toml

@@ -56,6 +56,7 @@ solana-cost-model = { workspace = true }
 solana-entry = { workspace = true }
 solana-feature-gate-interface = { workspace = true }
 solana-genesis-config = { workspace = true }
+solana-genesis-utils = { workspace = true }
 solana-geyser-plugin-manager = { workspace = true }
 solana-gossip = { workspace = true }
 solana-hash = { workspace = true }

ledger-tool/src/args.rs

@@ -346,7 +346,7 @@ pub fn hardforks_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Slot>> {
 
 #[cfg(test)]
 mod tests {
-    use {super::*, agave_snapshots::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE};
+    use {super::*, solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE};
 
     #[test]
     fn test_max_genesis_archive_unpacked_size_constant() {

ledger-tool/src/ledger_utils.rs

@@ -1,7 +1,6 @@
 use {
     crate::LEDGER_TOOL_DIRECTORY,
     agave_snapshots::{
-        hardened_unpack::open_genesis_config,
         snapshot_config::{SnapshotConfig, SnapshotUsage},
         snapshot_hash::StartingSnapshotHashes,
     },
@@ -14,6 +13,7 @@ use {
     solana_clock::Slot,
     solana_core::validator::BlockVerificationMethod,
     solana_genesis_config::GenesisConfig,
+    solana_genesis_utils::open_genesis_config,
     solana_geyser_plugin_manager::geyser_plugin_service::{
         GeyserPluginService, GeyserPluginServiceError,
     },

ledger-tool/src/main.rs

@@ -1786,7 +1786,7 @@ fn main() {
                     create_new_ledger(
                         &output_directory,
                         &genesis_config,
-                        agave_snapshots::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+                        solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
                         LedgerColumnOptions::default(),
                     )
                     .unwrap_or_else(|err| {

ledger/Cargo.toml

@@ -75,6 +75,7 @@ solana-frozen-abi-macro = { workspace = true, optional = true, features = [
     "frozen-abi",
 ] }
 solana-genesis-config = { workspace = true }
+solana-genesis-utils = { workspace = true }
 solana-hash = { workspace = true }
 solana-instruction = { workspace = true }
 solana-keypair = { workspace = true }

ledger/src/blockstore.rs

@@ -24,7 +24,7 @@ use {
         transaction_address_lookup_table_scanner::scan_transaction,
     },
     agave_feature_set::FeatureSet,
-    agave_snapshots::hardened_unpack::unpack_genesis_archive,
+    agave_snapshots::unpack_genesis_archive,
     assert_matches::debug_assert_matches,
     bincode::{deserialize, serialize},
     crossbeam_channel::{bounded, Receiver, Sender, TrySendError},
@@ -5219,9 +5219,6 @@ pub mod tests {
             leader_schedule::{FixedSchedule, IdentityKeyedLeaderSchedule},
             shred::{max_ticks_per_n_shreds, MAX_DATA_SHREDS_PER_SLOT},
         },
-        agave_snapshots::hardened_unpack::{
-            open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
-        },
         assert_matches::assert_matches,
         bincode::{serialize, Options},
         crossbeam_channel::unbounded,
@@ -5229,6 +5226,7 @@ pub mod tests {
         solana_account_decoder::parse_token::UiTokenAmount,
         solana_clock::{DEFAULT_MS_PER_SLOT, DEFAULT_TICKS_PER_SLOT},
         solana_entry::entry::{next_entry, next_entry_mut},
+        solana_genesis_utils::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
         solana_hash::Hash,
         solana_message::{compiled_instruction::CompiledInstruction, v0::LoadedAddresses},
         solana_packet::PACKET_DATA_SIZE,

ledger/src/lib.rs

@@ -62,9 +62,9 @@ extern crate log;
 #[cfg(feature = "frozen-abi")]
 extern crate solana_frozen_abi_macro;
 
+mod wire_format_tests;
+
 #[doc(hidden)]
 pub mod macro_reexports {
-    pub use agave_snapshots::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE;
+    pub use solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE;
 }
-
-mod wire_format_tests;

local-cluster/Cargo.toml

@@ -82,6 +82,7 @@ gag = { workspace = true }
 serial_test = { workspace = true }
 solana-core = { workspace = true, features = ["dev-context-only-utils"] }
 solana-download-utils = { workspace = true }
+solana-genesis-utils = { workspace = true }
 solana-ledger = { workspace = true, features = ["dev-context-only-utils"] }
 solana-local-cluster = { path = ".", features = ["agave-unstable-api", "dev-context-only-utils"] }
 solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }

local-cluster/tests/local_cluster.rs

@@ -1,8 +1,6 @@
 #![allow(clippy::arithmetic_side_effects)]
 use {
-    agave_snapshots::{
-        hardened_unpack::open_genesis_config, snapshot_config::SnapshotConfig, SnapshotInterval,
-    },
+    agave_snapshots::{snapshot_config::SnapshotConfig, SnapshotInterval},
     assert_matches::assert_matches,
     crossbeam_channel::{unbounded, Receiver},
     gag::BufferRedirect,
@@ -29,6 +27,7 @@ use {
     solana_download_utils::download_snapshot_archive,
     solana_entry::entry::create_ticks,
     solana_epoch_schedule::{MAX_LEADER_SCHEDULE_EPOCH_OFFSET, MINIMUM_SLOTS_PER_EPOCH},
+    solana_genesis_utils::open_genesis_config,
     solana_gossip::{crds_data::MAX_VOTES, gossip_service::discover_validators},
     solana_hard_forks::HardForks,
     solana_hash::Hash,

programs/sbf/Cargo.lock

@@ -6531,6 +6531,7 @@ dependencies = [
  "solana-fee-calculator",
  "solana-fee-structure",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-hard-forks",
@@ -6956,6 +6957,7 @@ dependencies = [
  "solana-genesis-config",
  "solana-hash",
  "solana-rpc-client",
+ "thiserror 2.0.17",
 ]
 
 [[package]]
@@ -7233,6 +7235,7 @@ dependencies = [
  "solana-entry",
  "solana-epoch-schedule",
  "solana-genesis-config",
+ "solana-genesis-utils",
  "solana-hash",
  "solana-instruction",
  "solana-keypair",
@@ -9649,6 +9652,7 @@ dependencies = [
  "solana-epoch-schedule",
  "solana-feature-gate-interface",
  "solana-fee-calculator",
+ "solana-genesis-utils",
  "solana-geyser-plugin-manager",
  "solana-gossip",
  "solana-inflation",
@@ -10107,6 +10111,7 @@ dependencies = [
  "rayon",
  "solana-clock",
  "solana-entry",
+ "solana-genesis-utils",
  "solana-gossip",
  "solana-hash",
  "solana-ledger",

snapshots/src/hardened_unpack.rs

@@ -1,19 +1,17 @@
 use {
     agave_fs::file_io::{self, FileCreator},
-    bzip2::bufread::BzDecoder,
     crossbeam_channel::Sender,
     log::*,
     rand::{thread_rng, Rng},
-    solana_genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE},
+    solana_genesis_config::DEFAULT_GENESIS_FILE,
     std::{
         fs::{self, File},
-        io::{self, BufReader, Read},
+        io::{self, Read},
         path::{
             Component::{self, CurDir, Normal},
             Path, PathBuf,
         },
         sync::Arc,
-        time::Instant,
     },
     tar::{
         Archive,
@@ -46,7 +44,6 @@ const MAX_SNAPSHOT_ARCHIVE_UNPACKED_APPARENT_SIZE: u64 = 64 * 1024 * 1024 * 1024
 const MAX_SNAPSHOT_ARCHIVE_UNPACKED_ACTUAL_SIZE: u64 = 4 * 1024 * 1024 * 1024 * 1024;
 
 const MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT: u64 = 5_000_000;
-pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB
 const MAX_GENESIS_ARCHIVE_UNPACKED_COUNT: u64 = 100;
 
 // The buffer should be large enough to saturate write I/O bandwidth, while also accommodating:
@@ -449,58 +446,7 @@ fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
     }
 }
 
-#[derive(Error, Debug)]
-pub enum OpenGenesisConfigError {
-    #[error("unpack error: {0}")]
-    Unpack(#[from] UnpackError),
-    #[error("Genesis load error: {0}")]
-    Load(#[from] std::io::Error),
-}
-
-pub fn open_genesis_config(
-    ledger_path: &Path,
-    max_genesis_archive_unpacked_size: u64,
-) -> std::result::Result<GenesisConfig, OpenGenesisConfigError> {
-    match GenesisConfig::load(ledger_path) {
-        Ok(genesis_config) => Ok(genesis_config),
-        Err(load_err) => {
-            warn!(
-                "Failed to load genesis_config at {ledger_path:?}: {load_err}. Will attempt to \
-                 unpack genesis archive and then retry loading."
-            );
-
-            let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
-            unpack_genesis_archive(
-                &genesis_package,
-                ledger_path,
-                max_genesis_archive_unpacked_size,
-            )?;
-            GenesisConfig::load(ledger_path).map_err(OpenGenesisConfigError::Load)
-        }
-    }
-}
-
-pub fn unpack_genesis_archive(
-    archive_filename: &Path,
-    destination_dir: &Path,
-    max_genesis_archive_unpacked_size: u64,
-) -> std::result::Result<(), UnpackError> {
-    info!("Extracting {archive_filename:?}...");
-    let extract_start = Instant::now();
-
-    fs::create_dir_all(destination_dir)?;
-    let tar_bz2 = File::open(archive_filename)?;
-    let tar = BzDecoder::new(BufReader::new(tar_bz2));
-    unpack_genesis(tar, destination_dir, max_genesis_archive_unpacked_size)?;
-    info!(
-        "Extracted {:?} in {:?}",
-        archive_filename,
-        Instant::now().duration_since(extract_start)
-    );
-    Ok(())
-}
-
-fn unpack_genesis(
+pub(super) fn unpack_genesis(
     input: impl Read,
     unpack_dir: &Path,
     max_genesis_archive_unpacked_size: u64,
@@ -541,6 +487,7 @@ mod tests {
     use {
         super::*,
         assert_matches::assert_matches,
+        std::io::BufReader,
         tar::{Builder, Header},
     };
 
@@ -790,9 +737,7 @@ mod tests {
     }
 
     fn finalize_and_unpack_genesis(archive: tar::Builder<Vec<u8>>) -> Result<()> {
-        with_finalize_and_unpack(archive, |a, b| {
-            unpack_genesis(a, b, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE)
-        })
+        with_finalize_and_unpack(archive, |a, b| unpack_genesis(a, b, 1024))
     }
 
     #[test]

snapshots/src/lib.rs

@@ -17,6 +17,8 @@ mod snapshot_version;
 mod unarchive;
 
 pub use {
-    archive_format::*, snapshot_interval::SnapshotInterval, snapshot_version::SnapshotVersion,
-    unarchive::streaming_unarchive_snapshot,
+    archive_format::*,
+    snapshot_interval::SnapshotInterval,
+    snapshot_version::SnapshotVersion,
+    unarchive::{streaming_unarchive_snapshot, unpack_genesis_archive},
 };

snapshots/src/unarchive.rs

@@ -4,12 +4,14 @@ use {
         ArchiveFormat, ArchiveFormatDecompressor,
     },
     agave_fs::buffered_reader,
+    bzip2::bufread::BzDecoder,
     crossbeam_channel::Sender,
     std::{
         fs,
-        io::{self, BufRead},
+        io::{self, BufRead, BufReader},
         path::{Path, PathBuf},
         thread::{self, JoinHandle},
+        time::Instant,
     },
 };
 
@@ -49,6 +51,26 @@ pub fn streaming_unarchive_snapshot(
         .unwrap()
 }
 
+pub fn unpack_genesis_archive(
+    archive_filename: &Path,
+    destination_dir: &Path,
+    max_genesis_archive_unpacked_size: u64,
+) -> Result<(), UnpackError> {
+    log::info!("Extracting {archive_filename:?}...");
+    let extract_start = Instant::now();
+
+    fs::create_dir_all(destination_dir)?;
+    let tar_bz2 = fs::File::open(archive_filename)?;
+    let tar = BzDecoder::new(BufReader::new(tar_bz2));
+    hardened_unpack::unpack_genesis(tar, destination_dir, max_genesis_archive_unpacked_size)?;
+    log::info!(
+        "Extracted {:?} in {:?}",
+        archive_filename,
+        Instant::now().duration_since(extract_start)
+    );
+    Ok(())
+}
+
 fn decompressed_tar_reader(
     archive_format: ArchiveFormat,
     archive_path: impl AsRef<Path>,

test-validator/Cargo.toml

@@ -33,6 +33,7 @@ solana-core = { workspace = true }
 solana-epoch-schedule = { workspace = true }
 solana-feature-gate-interface = { workspace = true }
 solana-fee-calculator = { workspace = true }
+solana-genesis-utils = { workspace = true }
 solana-geyser-plugin-manager = { workspace = true }
 solana-gossip = { workspace = true }
 solana-inflation = { workspace = true }

test-validator/src/lib.rs

@@ -1,10 +1,7 @@
 #![allow(clippy::arithmetic_side_effects)]
 use {
     agave_feature_set::{alpenglow, raise_cpi_nesting_limit_to_8, FeatureSet, FEATURE_NAMES},
-    agave_snapshots::{
-        hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, snapshot_config::SnapshotConfig,
-        SnapshotInterval,
-    },
+    agave_snapshots::{snapshot_config::SnapshotConfig, SnapshotInterval},
     base64::{prelude::BASE64_STANDARD, Engine},
     crossbeam_channel::Receiver,
     log::*,
@@ -24,6 +21,7 @@ use {
     },
     solana_epoch_schedule::EpochSchedule,
     solana_fee_calculator::FeeRateGovernor,
+    solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     solana_geyser_plugin_manager::{
         geyser_plugin_manager::GeyserPluginManager, GeyserPluginManagerRequest,
     },

validator/src/cli.rs

@@ -1,7 +1,6 @@
 use {
     crate::commands,
     agave_snapshots::{
-        hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         snapshot_config::{
             DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
             DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
@@ -26,6 +25,7 @@ use {
     },
     solana_epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
     solana_faucet::faucet::{self, FAUCET_PORT},
+    solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     solana_hash::Hash,
     solana_net_utils::{MINIMUM_VALIDATOR_PORT_RANGE_WIDTH, VALIDATOR_PORT_RANGE},
     solana_quic_definitions::QUIC_PORT_OFFSET,

validator/src/commands/run/execute.rs

@@ -8,7 +8,6 @@ use {
     },
     agave_logger::redirect_stderr_to_file,
     agave_snapshots::{
-        hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         snapshot_config::{SnapshotConfig, SnapshotUsage},
         ArchiveFormat, SnapshotInterval, SnapshotVersion,
     },
@@ -42,6 +41,7 @@ use {
             ValidatorTpuConfig,
         },
     },
+    solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     solana_gossip::{
         cluster_info::{NodeConfig, DEFAULT_CONTACT_SAVE_INTERVAL_MILLIS},
         contact_info::ContactInfo,

wen-restart/Cargo.toml

@@ -28,6 +28,7 @@ prost-types = { workspace = true }
 rayon = { workspace = true }
 solana-clock = { workspace = true }
 solana-entry = { workspace = true }
+solana-genesis-utils = { workspace = true }
 solana-gossip = { workspace = true }
 solana-hash = { workspace = true }
 solana-ledger = { workspace = true }

wen-restart/src/wen_restart.rs

@@ -1410,12 +1410,12 @@ mod tests {
     use {
         crate::wen_restart::{tests::wen_restart_proto::LastVotedForkSlotsAggregateFinal, *},
         agave_snapshots::{
-            hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
             snapshot_config::{SnapshotConfig, SnapshotUsage},
             snapshot_hash::SnapshotHash,
         },
         crossbeam_channel::unbounded,
         solana_entry::entry::create_ticks,
+        solana_genesis_utils::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         solana_gossip::{
             cluster_info::ClusterInfo,
             contact_info::ContactInfo,