Introduce agave-fs crate, move file_io and filesystem ops there (#8654)

Cargo.lock

@@ -119,6 +119,21 @@ dependencies = [
  "solana-svm-feature-set",
 ]
 
+[[package]]
+name = "agave-fs"
+version = "3.1.0"
+dependencies = [
+ "agave-io-uring",
+ "io-uring",
+ "libc",
+ "log",
+ "rand 0.8.5",
+ "slab",
+ "smallvec",
+ "tempfile",
+ "test-case",
+]
+
 [[package]]
 name = "agave-geyser-plugin-interface"
 version = "3.1.0"
@@ -248,6 +263,7 @@ dependencies = [
 name = "agave-snapshots"
 version = "3.1.0"
 dependencies = [
+ "agave-fs",
  "assert_matches",
  "bzip2",
  "crossbeam-channel",
@@ -6939,7 +6955,7 @@ dependencies = [
 name = "solana-accounts-db"
 version = "3.1.0"
 dependencies = [
- "agave-io-uring",
+ "agave-fs",
  "agave-reserved-account-keys",
  "ahash 0.8.11",
  "assert_matches",
@@ -6952,9 +6968,7 @@ dependencies = [
  "crossbeam-channel",
  "dashmap",
  "indexmap 2.11.4",
- "io-uring",
  "itertools 0.12.1",
- "libc",
  "libsecp256k1",
  "log",
  "lz4",
@@ -6970,7 +6984,6 @@ dependencies = [
  "seqlock",
  "serde",
  "serde_bytes",
- "slab",
  "smallvec",
  "solana-account",
  "solana-accounts-db",
@@ -10048,6 +10061,7 @@ name = "solana-runtime"
 version = "3.1.0"
 dependencies = [
  "agave-feature-set",
+ "agave-fs",
  "agave-precompiles",
  "agave-reserved-account-keys",
  "agave-snapshots",

Cargo.toml

@@ -34,6 +34,7 @@ members = [
     "faucet",
     "feature-set",
     "fee",
+    "fs",
     "genesis",
     "genesis-utils",
     "geyser-plugin-interface",
@@ -182,6 +183,7 @@ aes-gcm-siv = "0.11.1"
 agave-banking-stage-ingress-types = { path = "banking-stage-ingress-types", version = "=3.1.0", features = ["agave-unstable-api"] }
 agave-cargo-registry = { path = "cargo-registry", version = "=3.1.0" }
 agave-feature-set = { path = "feature-set", version = "=3.1.0", features = ["agave-unstable-api"] }
+agave-fs = { path = "fs", version = "=3.1.0", features = ["agave-unstable-api"] }
 agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=3.1.0", features = ["agave-unstable-api"] }
 agave-io-uring = { path = "io-uring", version = "=3.1.0", features = ["agave-unstable-api"] }
 agave-precompiles = { path = "precompiles", version = "=3.1.0", features = ["agave-unstable-api"] }

accounts-db/Cargo.toml

@@ -36,6 +36,7 @@ frozen-abi = [
 ]
 
 [dependencies]
+agave-fs = { workspace = true }
 ahash = { workspace = true }
 bincode = { workspace = true }
 blake3 = { workspace = true }
@@ -46,7 +47,6 @@ crossbeam-channel = { workspace = true }
 dashmap = { workspace = true, features = ["rayon", "raw-api"] }
 indexmap = { workspace = true }
 itertools = { workspace = true }
-libc = { workspace = true }
 log = { workspace = true }
 lz4 = { workspace = true }
 memmap2 = { workspace = true }
@@ -58,7 +58,6 @@ rand = { workspace = true }
 rayon = { workspace = true }
 seqlock = { workspace = true }
 serde = { workspace = true, features = ["rc"] }
-slab = { workspace = true }
 smallvec = { workspace = true, features = ["const_generics"] }
 solana-account = { workspace = true, features = ["serde"] }
 solana-address-lookup-table-interface = { workspace = true, features = [
@@ -104,10 +103,6 @@ static_assertions = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 
-[target.'cfg(target_os = "linux")'.dependencies]
-agave-io-uring = { workspace = true }
-io-uring = { workspace = true }
-
 [dev-dependencies]
 agave-reserved-account-keys = { workspace = true }
 assert_matches = { workspace = true }

accounts-db/src/accounts_db.rs

@@ -52,7 +52,6 @@ use {
         active_stats::{ActiveStatItem, ActiveStats},
         ancestors::Ancestors,
         append_vec::{self, aligned_stored_size, STORE_META_OVERHEAD},
-        buffered_reader::RequiredLenBufFileRead,
         contains::Contains,
         is_zero_lamport::IsZeroLamport,
         obsolete_accounts::ObsoleteAccounts,
@@ -62,6 +61,7 @@ use {
         u64_align,
         utils::{self, create_account_shared_data},
     },
+    agave_fs::buffered_reader::RequiredLenBufFileRead,
     dashmap::{DashMap, DashSet},
     log::*,
     rand::{thread_rng, Rng},

accounts-db/src/accounts_file.rs

@@ -6,12 +6,12 @@ use {
         account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData},
         accounts_db::AccountsFileId,
         append_vec::{AppendVec, AppendVecError},
-        buffered_reader::RequiredLenBufFileRead,
         storable_accounts::StorableAccounts,
         tiered_storage::{
             error::TieredStorageError, hot::HOT_FORMAT, index::IndexOffset, TieredStorage,
         },
     },
+    agave_fs::buffered_reader::RequiredLenBufFileRead,
     solana_account::AccountSharedData,
     solana_clock::Slot,
     solana_pubkey::Pubkey,

accounts-db/src/append_vec.rs

@@ -16,15 +16,17 @@ use {
         account_info::Offset,
         account_storage::stored_account_info::{StoredAccountInfo, StoredAccountInfoWithoutData},
         accounts_file::{InternalsForArchive, StorageAccess, StoredAccountsInfo},
+        is_zero_lamport::IsZeroLamport,
+        storable_accounts::StorableAccounts,
+        u64_align,
+        utils::create_account_shared_data,
+    },
+    agave_fs::{
         buffered_reader::{
             BufReaderWithOverflow, BufferedReader, FileBufRead as _, RequiredLenBufFileRead,
             RequiredLenBufRead as _,
         },
         file_io::{read_into_buffer, write_buffer_to_file},
-        is_zero_lamport::IsZeroLamport,
-        storable_accounts::StorableAccounts,
-        u64_align,
-        utils::create_account_shared_data,
     },
     log::*,
     memmap2::MmapMut,

accounts-db/src/lib.rs

@@ -32,10 +32,7 @@ mod append_vec;
 pub mod blockhash_queue;
 mod bucket_map_holder;
 mod bucket_map_holder_stats;
-mod buffered_reader;
 pub mod contains;
-mod file_io;
-mod io_uring;
 pub mod is_loadable;
 mod is_zero_lamport;
 mod obsolete_accounts;
@@ -53,11 +50,7 @@ pub mod tiered_storage;
 pub mod utils;
 pub mod waitable_condvar;
 
-pub use {
-    buffered_reader::large_file_buf_reader,
-    file_io::{file_creator, set_path_permissions, FileCreator},
-    obsolete_accounts::{ObsoleteAccountItem, ObsoleteAccounts},
-};
+pub use obsolete_accounts::{ObsoleteAccountItem, ObsoleteAccounts};
 
 #[macro_use]
 extern crate solana_metrics;

accounts-db/src/utils.rs

@@ -1,6 +1,5 @@
-#[cfg(target_os = "linux")]
-use {crate::io_uring::dir_remover::RingDirRemover, agave_io_uring::io_uring_supported};
 use {
+    agave_fs::dirs,
     log::*,
     solana_account::{AccountSharedData, ReadableAccount, WritableAccount},
     solana_measure::measure_time,
@@ -52,7 +51,7 @@ pub fn create_accounts_run_and_snapshot_dirs(
         // The run/ content cleanup will be done at a later point.  The snapshot/ content persists
         // across the process boot, and will be purged by the account_background_service.
         if fs::remove_dir_all(&account_dir).is_err() {
-            remove_dir_contents(&account_dir);
+            dirs::remove_dir_contents(&account_dir);
         }
         fs::create_dir_all(&run_path)?;
         fs::create_dir_all(&snapshot_path)?;
@@ -111,7 +110,7 @@ pub fn move_and_async_delete_path(path: impl AsRef<Path>) {
         lock.insert(path.as_ref().to_path_buf());
         drop(lock); // unlock before doing sync delete
 
-        remove_dir_contents(&path);
+        dirs::remove_dir_contents(&path);
         IN_PROGRESS_DELETES.lock().unwrap().remove(path.as_ref());
         return;
     }
@@ -122,7 +121,7 @@ pub fn move_and_async_delete_path(path: impl AsRef<Path>) {
         .name("solDeletePath".to_string())
         .spawn(move || {
             trace!("background deleting {}...", path_delete.display());
-            let (result, measure_delete) = measure_time!(remove_dir_all(&path_delete));
+            let (result, measure_delete) = measure_time!(dirs::remove_dir_all(&path_delete));
             if let Err(err) = result {
                 panic!("Failed to async delete '{}': {err}", path_delete.display());
             }
@@ -136,69 +135,6 @@ pub fn move_and_async_delete_path(path: impl AsRef<Path>) {
         .expect("spawn background delete thread");
 }
 
-/// Removes a directory and all its contents.
-pub fn remove_dir_all(path: impl Into<PathBuf> + AsRef<Path>) -> io::Result<()> {
-    #[cfg(target_os = "linux")]
-    {
-        assert!(io_uring_supported());
-        if let Ok(mut remover) = RingDirRemover::new() {
-            return remover.remove_dir_all(path);
-        }
-    }
-
-    fs::remove_dir_all(path)
-}
-
-/// Removes the contents of a directory, but not the directory itself.
-pub fn remove_dir_contents(path: impl AsRef<Path>) {
-    let path = path.as_ref();
-
-    #[cfg(target_os = "linux")]
-    {
-        assert!(io_uring_supported());
-        if let Ok(mut remover) = RingDirRemover::new() {
-            if let Err(e) = remover.remove_dir_contents(path) {
-                warn!("Failed to delete contents of '{}': {e}", path.display());
-            }
-
-            return;
-        }
-    }
-
-    remove_dir_contents_slow(path)
-}
-
-/// Delete the files and subdirectories in a directory.
-/// This is useful if the process does not have permission
-/// to delete the top level directory it might be able to
-/// delete the contents of that directory.
-fn remove_dir_contents_slow(path: impl AsRef<Path>) {
-    match fs::read_dir(&path) {
-        Err(err) => {
-            warn!(
-                "Failed to delete contents of '{}': could not read dir: {err}",
-                path.as_ref().display(),
-            )
-        }
-        Ok(dir_entries) => {
-            for entry in dir_entries.flatten() {
-                let sub_path = entry.path();
-                let result = if sub_path.is_dir() {
-                    fs::remove_dir_all(&sub_path)
-                } else {
-                    fs::remove_file(&sub_path)
-                };
-                if let Err(err) = result {
-                    warn!(
-                        "Failed to delete contents of '{}': {err}",
-                        sub_path.display(),
-                    );
-                }
-            }
-        }
-    }
-}
-
 /// Creates `directories` if they do not exist, and canonicalizes their paths
 pub fn create_and_canonicalize_directories(
     directories: impl IntoIterator<Item = impl AsRef<Path>>,
@@ -249,7 +185,7 @@ mod tests {
 
         // delete a `run/` and `snapshot/` dir, then re-create it
         let account_path_first = account_paths.first().unwrap();
-        remove_dir_contents(account_path_first);
+        dirs::remove_dir_contents(account_path_first);
         assert!(account_path_first.exists());
         assert!(!account_path_first.join(ACCOUNTS_RUN_DIR).exists());
         assert!(!account_path_first.join(ACCOUNTS_SNAPSHOT_DIR).exists());

dev-bins/Cargo.lock

@@ -74,6 +74,18 @@ dependencies = [
  "solana-svm-feature-set",
 ]
 
+[[package]]
+name = "agave-fs"
+version = "3.1.0"
+dependencies = [
+ "agave-io-uring",
+ "io-uring",
+ "libc",
+ "log",
+ "slab",
+ "smallvec",
+]
+
 [[package]]
 name = "agave-geyser-plugin-interface"
 version = "3.1.0"
@@ -234,6 +246,7 @@ dependencies = [
 name = "agave-snapshots"
 version = "3.1.0"
 dependencies = [
+ "agave-fs",
  "bzip2",
  "crossbeam-channel",
  "log",
@@ -5876,7 +5889,7 @@ dependencies = [
 name = "solana-accounts-db"
 version = "3.1.0"
 dependencies = [
- "agave-io-uring",
+ "agave-fs",
  "ahash 0.8.12",
  "bincode",
  "blake3",
@@ -5886,9 +5899,7 @@ dependencies = [
  "crossbeam-channel",
  "dashmap",
  "indexmap 2.11.4",
- "io-uring",
  "itertools 0.12.1",
- "libc",
  "log",
  "lz4",
  "memmap2 0.9.8",
@@ -5900,7 +5911,6 @@ dependencies = [
  "rayon",
  "seqlock",
  "serde",
- "slab",
  "smallvec",
  "solana-account",
  "solana-address-lookup-table-interface",
@@ -8304,6 +8314,7 @@ name = "solana-runtime"
 version = "3.1.0"
 dependencies = [
  "agave-feature-set",
+ "agave-fs",
  "agave-precompiles",
  "agave-reserved-account-keys",
  "agave-snapshots",

fs/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "agave-fs"
+description = "Agave file system utils"
+documentation = "https://docs.rs/agave-fs"
+version = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+edition = { workspace = true }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[lib]
+crate-type = ["lib"]
+name = "agave_fs"
+
+[features]
+agave-unstable-api = []
+
+[dependencies]
+libc = { workspace = true }
+log = { workspace = true }
+slab = { workspace = true }
+smallvec = { workspace = true }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+agave-io-uring = { workspace = true }
+io-uring = { workspace = true }
+
+[dev-dependencies]
+rand = { workspace = true }
+tempfile = { workspace = true }
+test-case = { workspace = true }

accounts-db/src/buffered_reader.rs → fs/src/buffered_reader.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::arithmetic_side_effects)]
+
 //! File I/O buffered readers for AppendVec
 //! Specialized `BufRead`-like types for reading account data.
 //!
@@ -29,11 +31,11 @@ use {
 ///
 /// This should be used when the required size is known at compile time and is within reasonable stack
 /// limits.
-pub(crate) struct Stack<const N: usize>([MaybeUninit<u8>; N]);
+struct Stack<const N: usize>([MaybeUninit<u8>; N]);
 
 impl<const N: usize> Stack<N> {
     #[inline(always)]
-    pub fn new() -> Self {
+    const fn new() -> Self {
         Self([MaybeUninit::uninit(); N])
     }
 }
@@ -56,7 +58,7 @@ impl<const N: usize> Stack<N> {
 
 /// An extension of the `BufRead` trait for file readers that allow tracking file
 /// read position offset.
-pub(crate) trait FileBufRead<'a>: BufRead {
+pub trait FileBufRead<'a>: BufRead {
     /// Activate the given `file` as source of reads of this reader.
     ///
     /// Resets the internal buffer to an empty state and sets the file offset to 0.
@@ -79,7 +81,7 @@ pub(crate) trait FileBufRead<'a>: BufRead {
 /// Unlike the standard `BufRead`, which only guarantees a non-empty buffer,
 /// this trait allows callers to enforce a minimum number of contiguous bytes
 /// to be made available.
-pub(crate) trait RequiredLenBufRead: BufRead {
+pub trait RequiredLenBufRead: BufRead {
     /// Ensures the internal buffer contains at least `required_len` contiguous bytes,
     /// and returns a slice of that buffer.
     ///
@@ -94,7 +96,7 @@ pub(crate) trait RequiredLenBufRead: BufRead {
     fn fill_buf_required(&mut self, required_len: usize) -> io::Result<&[u8]>;
 }
 
-pub(crate) trait RequiredLenBufFileRead<'a>: RequiredLenBufRead + FileBufRead<'a> {}
+pub trait RequiredLenBufFileRead<'a>: RequiredLenBufRead + FileBufRead<'a> {}
 impl<'a, T: RequiredLenBufRead + FileBufRead<'a>> RequiredLenBufFileRead<'a> for T {}
 
 /// read a file a large buffer at a time and provide access to a slice in that buffer
@@ -114,7 +116,8 @@ pub struct BufferedReader<'a, const N: usize> {
 }
 
 impl<'a, const N: usize> BufferedReader<'a, N> {
-    pub fn new() -> Self {
+    #[allow(clippy::new_without_default)]
+    pub const fn new() -> Self {
         Self {
             file_offset_of_next_read: 0,
             buf: Stack::new(),

fs/src/dirs.rs

@@ -0,0 +1,69 @@
+use std::{
+    fs, io,
+    path::{Path, PathBuf},
+};
+#[cfg(target_os = "linux")]
+use {crate::io_uring::dir_remover::RingDirRemover, agave_io_uring::io_uring_supported};
+
+/// Removes a directory and all its contents.
+pub fn remove_dir_all(path: impl Into<PathBuf> + AsRef<Path>) -> io::Result<()> {
+    #[cfg(target_os = "linux")]
+    {
+        assert!(io_uring_supported());
+        if let Ok(mut remover) = RingDirRemover::new() {
+            return remover.remove_dir_all(path);
+        }
+    }
+
+    fs::remove_dir_all(path)
+}
+
+/// Removes the contents of a directory, but not the directory itself.
+pub fn remove_dir_contents(path: impl AsRef<Path>) {
+    let path = path.as_ref();
+
+    #[cfg(target_os = "linux")]
+    {
+        assert!(io_uring_supported());
+        if let Ok(mut remover) = RingDirRemover::new() {
+            if let Err(e) = remover.remove_dir_contents(path) {
+                log::warn!("Failed to delete contents of '{}': {e}", path.display());
+            }
+
+            return;
+        }
+    }
+
+    remove_dir_contents_slow(path)
+}
+
+/// Delete the files and subdirectories in a directory.
+/// This is useful if the process does not have permission
+/// to delete the top level directory it might be able to
+/// delete the contents of that directory.
+fn remove_dir_contents_slow(path: impl AsRef<Path>) {
+    match fs::read_dir(&path) {
+        Err(err) => {
+            log::warn!(
+                "Failed to delete contents of '{}': could not read dir: {err}",
+                path.as_ref().display(),
+            )
+        }
+        Ok(dir_entries) => {
+            for entry in dir_entries.flatten() {
+                let sub_path = entry.path();
+                let result = if sub_path.is_dir() {
+                    fs::remove_dir_all(&sub_path)
+                } else {
+                    fs::remove_file(&sub_path)
+                };
+                if let Err(err) = result {
+                    log::warn!(
+                        "Failed to delete contents of '{}': {err}",
+                        sub_path.display(),
+                    );
+                }
+            }
+        }
+    }
+}

accounts-db/src/file_io.rs → fs/src/file_io.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::arithmetic_side_effects)]
+
 //! File i/o helper functions.
 use std::{
     fs::{self, File, OpenOptions},

accounts-db/src/io_uring/dir_remover.rs → fs/src/io_uring/dir_remover.rs

@@ -46,6 +46,7 @@ impl RingDirRemover {
         self.remove(path, false)
     }
 
+    #[allow(clippy::arithmetic_side_effects)]
     fn remove(&mut self, path: impl Into<PathBuf>, remove_root: bool) -> io::Result<()> {
         let path = path.into();
 
@@ -178,6 +179,7 @@ impl UnlinkOp {
         opcode::UnlinkAt::new(types::Fd(self.dir_fd), self.path.as_ptr() as _).build()
     }
 
+    #[allow(clippy::arithmetic_side_effects)]
     fn complete(
         &mut self,
         comp: &mut Completion<State, Op>,

accounts-db/src/io_uring/file_creator.rs → fs/src/io_uring/file_creator.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::arithmetic_side_effects)]
+
 use {
     crate::{
         file_io::FileCreator,

accounts-db/src/io_uring/memory.rs → fs/src/io_uring/memory.rs

@@ -153,6 +153,7 @@ impl FixedIoBuffer {
 
     /// Split buffer into `chunk_size` sized `IoFixedBuffer` buffers for use as registered
     /// buffer in io_uring operations.
+    #[allow(clippy::arithmetic_side_effects)]
     pub unsafe fn split_buffer_chunks(
         buffer: &mut [u8],
         chunk_size: usize,

accounts-db/src/io_uring/sequential_file_reader.rs → fs/src/io_uring/sequential_file_reader.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::arithmetic_side_effects)]
+
 use {
     super::{
         memory::{FixedIoBuffer, LargeBuffer},

fs/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod buffered_reader;
+pub mod dirs;
+pub mod file_io;
+mod io_uring;

programs/sbf/Cargo.lock

@@ -74,6 +74,18 @@ dependencies = [
  "solana-svm-feature-set",
 ]
 
+[[package]]
+name = "agave-fs"
+version = "3.1.0"
+dependencies = [
+ "agave-io-uring",
+ "io-uring",
+ "libc",
+ "log",
+ "slab",
+ "smallvec",
+]
+
 [[package]]
 name = "agave-geyser-plugin-interface"
 version = "3.1.0"
@@ -150,6 +162,7 @@ dependencies = [
 name = "agave-snapshots"
 version = "3.1.0"
 dependencies = [
+ "agave-fs",
  "bzip2",
  "crossbeam-channel",
  "log",
@@ -5790,7 +5803,7 @@ dependencies = [
 name = "solana-accounts-db"
 version = "3.1.0"
 dependencies = [
- "agave-io-uring",
+ "agave-fs",
  "ahash 0.8.11",
  "bincode",
  "blake3",
@@ -5800,9 +5813,7 @@ dependencies = [
  "crossbeam-channel",
  "dashmap",
  "indexmap 2.11.4",
- "io-uring",
  "itertools 0.12.1",
- "libc",
  "log",
  "lz4",
  "memmap2 0.9.8",
@@ -5814,7 +5825,6 @@ dependencies = [
  "rayon",
  "seqlock",
  "serde",
- "slab",
  "smallvec",
  "solana-account",
  "solana-address-lookup-table-interface",
@@ -8072,6 +8082,7 @@ name = "solana-runtime"
 version = "3.1.0"
 dependencies = [
  "agave-feature-set",
+ "agave-fs",
  "agave-precompiles",
  "agave-reserved-account-keys",
  "agave-snapshots",

runtime/Cargo.toml

@@ -52,6 +52,7 @@ shuttle-test = ["dep:shuttle"]
 
 [dependencies]
 agave-feature-set = { workspace = true }
+agave-fs = { workspace = true }
 agave-precompiles = { workspace = true }
 agave-reserved-account-keys = { workspace = true }
 agave-snapshots = { workspace = true }

runtime/src/snapshot_bank_utils.rs

@@ -31,6 +31,7 @@ use {
         },
         status_cache,
     },
+    agave_fs::dirs,
     agave_snapshots::{
         snapshot_config::SnapshotConfig, snapshot_hash::SnapshotHash, ArchiveFormat,
         SnapshotVersion,
@@ -39,7 +40,6 @@ use {
     solana_accounts_db::{
         accounts_db::{AccountsDbConfig, AtomicAccountsFileId},
         accounts_update_notifier_interface::AccountsUpdateNotifier,
-        utils::remove_dir_contents,
     },
     solana_clock::{Epoch, Slot},
     solana_genesis_config::GenesisConfig,
@@ -346,7 +346,7 @@ pub fn bank_from_snapshot_dir(
     // Clear the contents of the account paths run directories.  When constructing the bank, the appendvec
     // files will be extracted from the snapshot hardlink directories into these run/ directories.
     for path in account_paths {
-        remove_dir_contents(path);
+        dirs::remove_dir_contents(path);
     }
 
     let next_append_vec_id = Arc::new(AtomicAccountsFileId::new(0));

snapshots/Cargo.toml

@@ -20,6 +20,7 @@ name = "agave_snapshots"
 agave-unstable-api = []
 
 [dependencies]
+agave-fs = { workspace = true }
 bzip2 = { workspace = true }
 crossbeam-channel = { workspace = true }
 log = { workspace = true }

snapshots/src/hardened_unpack.rs

@@ -1,9 +1,9 @@
 use {
+    agave_fs::file_io::{self, FileCreator},
     bzip2::bufread::BzDecoder,
     crossbeam_channel::Sender,
     log::*,
     rand::{thread_rng, Rng},
-    solana_accounts_db::{file_creator, set_path_permissions, FileCreator},
     solana_genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE},
     std::{
         fs::{self, File},
@@ -118,7 +118,7 @@ where
     // (decompression multiplies content size, but buffering more than origin isn't necessary).
     let buf_size =
         (memlock_budget_size.min(actual_limit_size as usize)).min(MAX_UNPACK_WRITE_BUF_SIZE);
-    let mut files_creator = file_creator(buf_size, file_path_processor)?;
+    let mut files_creator = file_io::file_creator(buf_size, file_path_processor)?;
 
     let mut archive = Archive::new(input);
     for entry in archive.entries()? {
@@ -217,7 +217,7 @@ fn unpack_entry<'a, R: Read>(
     if should_fallback_to_tar_unpack(&entry) {
         entry.unpack(&dst)?;
         // Sanitize permissions.
-        set_path_permissions(&dst, mode)?;
+        file_io::set_path_permissions(&dst, mode)?;
 
         if !entry.header().entry_type().is_dir() {
             // Process file after setting permissions

snapshots/src/unarchive.rs

@@ -3,6 +3,7 @@ use {
         hardened_unpack::{self, UnpackError},
         ArchiveFormat, ArchiveFormatDecompressor,
     },
+    agave_fs::buffered_reader,
     crossbeam_channel::Sender,
     std::{
         fs,
@@ -54,6 +55,6 @@ fn decompressed_tar_reader(
     buf_size: u64,
 ) -> io::Result<ArchiveFormatDecompressor<impl BufRead>> {
     let buf_reader =
-        solana_accounts_db::large_file_buf_reader(archive_path.as_ref(), buf_size as usize)?;
+        buffered_reader::large_file_buf_reader(archive_path.as_ref(), buf_size as usize)?;
     ArchiveFormatDecompressor::new(archive_format, buf_reader)
 }