perf: move benchmarks to bencher 0.1.5 (#7041)

move benchmarks to bencher 0.1.5

Cargo.lock

@@ -9306,6 +9306,7 @@ version = "3.0.0"
 dependencies = [
  "ahash 0.8.11",
  "assert_matches",
+ "bencher",
  "bincode",
  "bv",
  "bytes",

perf/Cargo.toml

@@ -77,6 +77,7 @@ nix = { workspace = true, features = ["user"] }
 
 [dev-dependencies]
 assert_matches = { workspace = true }
+bencher = { workspace = true }
 rand_chacha = { workspace = true }
 solana-logger = { workspace = true }
 solana-perf = { path = ".", features = ["dev-context-only-utils"] }
@@ -85,11 +86,29 @@ test-case = { workspace = true }
 [target.'cfg(not(any(target_env = "msvc", target_os = "freebsd")))'.dev-dependencies]
 jemallocator = { workspace = true }
 
+[[bench]]
+name = "dedup"
+harness = false
+
+[[bench]]
+name = "recycler"
+harness = false
+
+[[bench]]
+name = "reset"
+harness = false
+
+[[bench]]
+name = "shrink"
+harness = false
+
 [[bench]]
 name = "sigverify"
+harness = false
 
 [[bench]]
 name = "discard"
+harness = false
 
 [lints.rust.unexpected_cfgs]
 level = "warn"

perf/benches/dedup.rs

@@ -1,16 +1,13 @@
 #![allow(clippy::arithmetic_side_effects)]
-#![feature(test)]
-
-extern crate test;
 
 use {
+    bencher::{benchmark_group, benchmark_main, Bencher},
     rand::prelude::*,
     solana_perf::{
         deduper::{self, Deduper},
         packet::{to_packet_batches, PacketBatch},
     },
     std::time::Duration,
-    test::Bencher,
 };
 
 #[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
@@ -26,11 +23,11 @@ fn test_packet_with_size(size: usize, rng: &mut ThreadRng) -> Vec<u8> {
         .collect()
 }
 
-fn do_bench_dedup_packets(bencher: &mut Bencher, mut batches: Vec<PacketBatch>) {
+fn do_bench_dedup_packets(b: &mut Bencher, mut batches: Vec<PacketBatch>) {
     // verify packets
     let mut rng = rand::thread_rng();
     let mut deduper = Deduper::<2, [u8]>::new(&mut rng, /*num_bits:*/ 63_999_979);
-    bencher.iter(|| {
+    b.iter(|| {
         let _ans = deduper::dedup_packets_and_count_discards(&deduper, &mut batches);
         deduper.maybe_reset(
             &mut rng,
@@ -44,9 +41,7 @@ fn do_bench_dedup_packets(bencher: &mut Bencher, mut batches: Vec<PacketBatch>)
     });
 }
 
-#[bench]
-#[ignore]
-fn bench_dedup_same_small_packets(bencher: &mut Bencher) {
+fn bench_dedup_same_small_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
     let small_packet = test_packet_with_size(128, &mut rng);
 
@@ -55,12 +50,10 @@ fn bench_dedup_same_small_packets(bencher: &mut Bencher) {
         128,
     );
 
-    do_bench_dedup_packets(bencher, batches);
+    do_bench_dedup_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_dedup_same_big_packets(bencher: &mut Bencher) {
+fn bench_dedup_same_big_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
     let big_packet = test_packet_with_size(1024, &mut rng);
 
@@ -69,12 +62,10 @@ fn bench_dedup_same_big_packets(bencher: &mut Bencher) {
         128,
     );
 
-    do_bench_dedup_packets(bencher, batches);
+    do_bench_dedup_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_dedup_diff_small_packets(bencher: &mut Bencher) {
+fn bench_dedup_diff_small_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
 
     let batches = to_packet_batches(
@@ -84,12 +75,10 @@ fn bench_dedup_diff_small_packets(bencher: &mut Bencher) {
         128,
     );
 
-    do_bench_dedup_packets(bencher, batches);
+    do_bench_dedup_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_dedup_diff_big_packets(bencher: &mut Bencher) {
+fn bench_dedup_diff_big_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
 
     let batches = to_packet_batches(
@@ -99,12 +88,10 @@ fn bench_dedup_diff_big_packets(bencher: &mut Bencher) {
         128,
     );
 
-    do_bench_dedup_packets(bencher, batches);
+    do_bench_dedup_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_dedup_baseline(bencher: &mut Bencher) {
+fn bench_dedup_baseline(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
 
     let batches = to_packet_batches(
@@ -114,15 +101,13 @@ fn bench_dedup_baseline(bencher: &mut Bencher) {
         128,
     );
 
-    do_bench_dedup_packets(bencher, batches);
+    do_bench_dedup_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_dedup_reset(bencher: &mut Bencher) {
+fn bench_dedup_reset(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
     let mut deduper = Deduper::<2, [u8]>::new(&mut rng, /*num_bits:*/ 63_999_979);
-    bencher.iter(|| {
+    b.iter(|| {
         deduper.maybe_reset(
             &mut rng,
             0.001,                    // false_positive_rate
@@ -130,3 +115,14 @@ fn bench_dedup_reset(bencher: &mut Bencher) {
         );
     });
 }
+
+benchmark_group!(
+    benches,
+    bench_dedup_reset,
+    bench_dedup_baseline,
+    bench_dedup_diff_big_packets,
+    bench_dedup_diff_small_packets,
+    bench_dedup_same_big_packets,
+    bench_dedup_same_small_packets
+);
+benchmark_main!(benches);

perf/benches/discard.rs

@@ -1,10 +1,6 @@
-#![feature(test)]
-
-extern crate test;
-
 use {
+    bencher::{benchmark_group, benchmark_main, Bencher},
     solana_perf::{discard::discard_batches_randomly, packet::to_packet_batches, test_tx::test_tx},
-    test::Bencher,
 };
 
 #[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
@@ -13,8 +9,7 @@ static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
 const NUM: usize = 1000;
 
-#[bench]
-fn bench_discard(bencher: &mut Bencher) {
+fn bench_discard(b: &mut Bencher) {
     solana_logger::setup();
     let tx = test_tx();
     let num_packets = NUM;
@@ -25,9 +20,12 @@ fn bench_discard(bencher: &mut Bencher) {
         10,
     );
 
-    bencher.iter(|| {
+    b.iter(|| {
         let mut discarded = batches.clone();
         discard_batches_randomly(&mut discarded, 100, NUM);
         assert_eq!(discarded.len(), 10);
     })
 }
+
+benchmark_group!(benches, bench_discard);
+benchmark_main!(benches);

perf/benches/recycler.rs

@@ -1,14 +1,9 @@
-#![feature(test)]
-
-extern crate test;
-
 use {
+    bencher::{benchmark_group, benchmark_main, Bencher},
     solana_perf::{packet::PacketBatchRecycler, recycler::Recycler},
-    test::Bencher,
 };
 
-#[bench]
-fn bench_recycler(bencher: &mut Bencher) {
+fn bench_recycler(b: &mut Bencher) {
     solana_logger::setup();
 
     let recycler: PacketBatchRecycler = Recycler::default();
@@ -17,7 +12,10 @@ fn bench_recycler(bencher: &mut Bencher) {
         let _packet = recycler.allocate("");
     }
 
-    bencher.iter(move || {
+    b.iter(move || {
         let _packet = recycler.allocate("");
     });
 }
+
+benchmark_group!(benches, bench_recycler);
+benchmark_main!(benches);

perf/benches/reset.rs

@@ -1,10 +1,9 @@
-#![feature(test)]
-
-extern crate test;
-
 use {
-    std::sync::atomic::{AtomicU64, Ordering},
-    test::Bencher,
+    bencher::{benchmark_group, benchmark_main, Bencher},
+    std::{
+        hint::black_box,
+        sync::atomic::{AtomicU64, Ordering},
+    },
 };
 
 #[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
@@ -16,15 +15,14 @@ const N: usize = 1_000_000;
 // test bench_reset1 ... bench:     436,240 ns/iter (+/- 176,714)
 // test bench_reset2 ... bench:     274,007 ns/iter (+/- 129,552)
 
-#[bench]
-fn bench_reset1(bencher: &mut Bencher) {
+fn bench_reset1(b: &mut Bencher) {
     solana_logger::setup();
 
     let mut v = Vec::with_capacity(N);
     v.resize_with(N, AtomicU64::default);
 
-    bencher.iter(|| {
-        test::black_box({
+    b.iter(|| {
+        black_box({
             for i in &v {
                 i.store(0, Ordering::Relaxed);
             }
@@ -33,18 +31,20 @@ fn bench_reset1(bencher: &mut Bencher) {
     });
 }
 
-#[bench]
-fn bench_reset2(bencher: &mut Bencher) {
+fn bench_reset2(b: &mut Bencher) {
     solana_logger::setup();
 
     let mut v = Vec::with_capacity(N);
     v.resize_with(N, AtomicU64::default);
 
-    bencher.iter(|| {
-        test::black_box({
+    b.iter(|| {
+        black_box({
             v.clear();
             v.resize_with(N, AtomicU64::default);
             0
         });
     });
 }
+
+benchmark_group!(benches, bench_reset2, bench_reset1);
+benchmark_main!(benches);

perf/benches/shrink.rs

@@ -1,16 +1,13 @@
 #![allow(clippy::arithmetic_side_effects)]
-#![feature(test)]
-
-extern crate test;
 
 use {
+    bencher::{benchmark_group, benchmark_main, Bencher},
     rand::prelude::*,
     solana_perf::{
         packet::{to_packet_batches, PacketBatch, PACKETS_PER_BATCH},
         sigverify,
     },
     std::iter,
-    test::Bencher,
 };
 
 #[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
@@ -26,7 +23,7 @@ fn test_packet_with_size(size: usize, rng: &mut ThreadRng) -> Vec<u8> {
         .collect()
 }
 
-fn do_bench_shrink_packets(bencher: &mut Bencher, mut batches: Vec<PacketBatch>) {
+fn do_bench_shrink_packets(b: &mut Bencher, mut batches: Vec<PacketBatch>) {
     let mut batches = iter::repeat_with(|| {
         batches.iter_mut().for_each(|b| {
             b.iter_mut()
@@ -40,16 +37,14 @@ fn do_bench_shrink_packets(bencher: &mut Bencher, mut batches: Vec<PacketBatch>)
     .collect::<Vec<_>>()
     .into_iter()
     .cycle();
-    bencher.iter(|| {
+    b.iter(|| {
         let batches = batches.next().unwrap();
         // verify packets
         sigverify::shrink_batches(batches);
     });
 }
 
-#[bench]
-#[ignore]
-fn bench_shrink_diff_small_packets(bencher: &mut Bencher) {
+fn bench_shrink_diff_small_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
 
     let batches = to_packet_batches(
@@ -59,12 +54,10 @@ fn bench_shrink_diff_small_packets(bencher: &mut Bencher) {
         PACKETS_PER_BATCH,
     );
 
-    do_bench_shrink_packets(bencher, batches);
+    do_bench_shrink_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_shrink_diff_big_packets(bencher: &mut Bencher) {
+fn bench_shrink_diff_big_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
 
     let batches = to_packet_batches(
@@ -74,12 +67,10 @@ fn bench_shrink_diff_big_packets(bencher: &mut Bencher) {
         PACKETS_PER_BATCH,
     );
 
-    do_bench_shrink_packets(bencher, batches);
+    do_bench_shrink_packets(b, batches);
 }
 
-#[bench]
-#[ignore]
-fn bench_shrink_count_packets(bencher: &mut Bencher) {
+fn bench_shrink_count_packets(b: &mut Bencher) {
     let mut rng = rand::thread_rng();
 
     let mut batches = to_packet_batches(
@@ -93,7 +84,15 @@ fn bench_shrink_count_packets(bencher: &mut Bencher) {
             .for_each(|mut p| p.meta_mut().set_discard(thread_rng().gen()))
     });
 
-    bencher.iter(|| {
+    b.iter(|| {
         let _ = sigverify::count_valid_packets(&batches);
     });
 }
+
+benchmark_group!(
+    benches,
+    bench_shrink_count_packets,
+    bench_shrink_diff_big_packets,
+    bench_shrink_diff_small_packets
+);
+benchmark_main!(benches);

perf/benches/sigverify.rs

@@ -1,8 +1,7 @@
-#![feature(test)]
-
-extern crate test;
+#![allow(clippy::arithmetic_side_effects)]
 
 use {
+    bencher::{benchmark_group, benchmark_main, Bencher},
     log::*,
     rand::{thread_rng, Rng},
     solana_perf::{
@@ -11,7 +10,6 @@ use {
         sigverify,
         test_tx::{test_multisig_tx, test_tx},
     },
-    test::Bencher,
 };
 
 #[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
@@ -21,8 +19,7 @@ static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
 const NUM: usize = 256;
 const LARGE_BATCH_PACKET_COUNT: usize = 128;
 
-#[bench]
-fn bench_sigverify_simple(bencher: &mut Bencher) {
+fn bench_sigverify_simple(b: &mut Bencher) {
     let tx = test_tx();
     let num_packets = NUM;
 
@@ -35,7 +32,7 @@ fn bench_sigverify_simple(bencher: &mut Bencher) {
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
     // verify packets
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
@@ -56,82 +53,68 @@ fn gen_batches(
     }
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_low_packets_small_batch(bencher: &mut Bencher) {
+fn bench_sigverify_low_packets_small_batch(b: &mut Bencher) {
     let num_packets = sigverify::VERIFY_PACKET_CHUNK_SIZE - 1;
     let mut batches = gen_batches(false, 1, num_packets);
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_low_packets_large_batch(bencher: &mut Bencher) {
+fn bench_sigverify_low_packets_large_batch(b: &mut Bencher) {
     let num_packets = sigverify::VERIFY_PACKET_CHUNK_SIZE - 1;
     let mut batches = gen_batches(false, LARGE_BATCH_PACKET_COUNT, num_packets);
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_medium_packets_small_batch(bencher: &mut Bencher) {
+fn bench_sigverify_medium_packets_small_batch(b: &mut Bencher) {
     let num_packets = sigverify::VERIFY_PACKET_CHUNK_SIZE * 8;
     let mut batches = gen_batches(false, 1, num_packets);
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_medium_packets_large_batch(bencher: &mut Bencher) {
+fn bench_sigverify_medium_packets_large_batch(b: &mut Bencher) {
     let num_packets = sigverify::VERIFY_PACKET_CHUNK_SIZE * 8;
     let mut batches = gen_batches(false, LARGE_BATCH_PACKET_COUNT, num_packets);
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_high_packets_small_batch(bencher: &mut Bencher) {
+fn bench_sigverify_high_packets_small_batch(b: &mut Bencher) {
     let num_packets = sigverify::VERIFY_PACKET_CHUNK_SIZE * 32;
     let mut batches = gen_batches(false, 1, num_packets);
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_high_packets_large_batch(bencher: &mut Bencher) {
+fn bench_sigverify_high_packets_large_batch(b: &mut Bencher) {
     let num_packets = sigverify::VERIFY_PACKET_CHUNK_SIZE * 32;
     let mut batches = gen_batches(false, LARGE_BATCH_PACKET_COUNT, num_packets);
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
     // verify packets
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-#[ignore]
-fn bench_sigverify_uneven(bencher: &mut Bencher) {
+fn bench_sigverify_uneven(b: &mut Bencher) {
     solana_logger::setup();
     let simple_tx = test_tx();
     let multi_tx = test_multisig_tx();
@@ -171,13 +154,12 @@ fn bench_sigverify_uneven(bencher: &mut Bencher) {
     let recycler = Recycler::default();
     let recycler_out = Recycler::default();
     // verify packets
-    bencher.iter(|| {
+    b.iter(|| {
         sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, num_packets);
     })
 }
 
-#[bench]
-fn bench_get_offsets(bencher: &mut Bencher) {
+fn bench_get_offsets(b: &mut Bencher) {
     let tx = test_tx();
 
     // generate packet vector
@@ -185,7 +167,21 @@ fn bench_get_offsets(bencher: &mut Bencher) {
 
     let recycler = Recycler::default();
     // verify packets
-    bencher.iter(|| {
+    b.iter(|| {
         let _ans = sigverify::generate_offsets(&mut batches, &recycler, false);
     })
 }
+
+benchmark_group!(
+    benches,
+    bench_get_offsets,
+    bench_sigverify_uneven,
+    bench_sigverify_high_packets_large_batch,
+    bench_sigverify_high_packets_small_batch,
+    bench_sigverify_medium_packets_large_batch,
+    bench_sigverify_medium_packets_small_batch,
+    bench_sigverify_low_packets_large_batch,
+    bench_sigverify_low_packets_small_batch,
+    bench_sigverify_simple
+);
+benchmark_main!(benches);