ForwardingStage hookup, BankingStage forwarding removal (#4518)

banking-bench/src/main.rs

@@ -7,7 +7,6 @@ use {
     log::*,
     rand::{thread_rng, Rng},
     rayon::prelude::*,
-    solana_client::connection_cache::ConnectionCache,
     solana_core::{
         banking_stage::{update_bank_forks_and_poh_recorder_for_new_tpu_bank, BankingStage},
         banking_trace::{BankingTracer, Channels, BANKING_TRACE_DIR_DEFAULT_BYTE_LIMIT},
@@ -37,7 +36,6 @@ use {
         transaction::Transaction,
     },
     solana_streamer::socket::SocketAddrSpace,
-    solana_tpu_client::tpu_client::DEFAULT_TPU_CONNECTION_POOL_SIZE,
     std::{
         sync::{atomic::Ordering, Arc, RwLock},
         thread::sleep,
@@ -304,12 +302,6 @@ fn main() {
                 .takes_value(true)
                 .help("Number of threads to use in the banking stage"),
         )
-        .arg(
-            Arg::new("tpu_disable_quic")
-                .long("tpu-disable-quic")
-                .takes_value(false)
-                .help("Disable forwarding messages to TPU using QUIC"),
-        )
         .arg(
             Arg::new("simulate_mint")
                 .long("simulate-mint")
@@ -468,18 +460,6 @@ fn main() {
         gossip_vote_sender,
         gossip_vote_receiver,
     } = banking_tracer.create_channels(false);
-    let tpu_disable_quic = matches.is_present("tpu_disable_quic");
-    let connection_cache = if tpu_disable_quic {
-        ConnectionCache::with_udp(
-            "connection_cache_banking_bench_udp",
-            DEFAULT_TPU_CONNECTION_POOL_SIZE,
-        )
-    } else {
-        ConnectionCache::new_quic(
-            "connection_cache_banking_bench_quic",
-            DEFAULT_TPU_CONNECTION_POOL_SIZE,
-        )
-    };
     let banking_stage = BankingStage::new_num_threads(
         block_production_method,
         transaction_struct,
@@ -492,10 +472,8 @@ fn main() {
         None,
         replay_vote_sender,
         None,
-        Arc::new(connection_cache),
         bank_forks.clone(),
         &prioritization_fee_cache,
-        false,
     );
 
     // This is so that the signal_receiver does not go out of scope after the closure.

core/benches/banking_stage.rs

@@ -18,7 +18,6 @@ use {
     log::*,
     rand::{thread_rng, Rng},
     rayon::prelude::*,
-    solana_client::connection_cache::ConnectionCache,
     solana_core::{
         banking_stage::{
             committer::Committer,
@@ -316,10 +315,8 @@ fn bench_banking(
         None,
         s,
         None,
-        Arc::new(ConnectionCache::new("connection_cache_test")),
         bank_forks,
         &Arc::new(PrioritizationFeeCache::new(0u64)),
-        false,
     );
 
     let chunk_len = verified.len() / CHUNKS;

core/benches/forwarder.rs

@@ -1,177 +0,0 @@
-#![feature(test)]
-extern crate test;
-use {
-    itertools::Itertools,
-    solana_client::connection_cache::ConnectionCache,
-    solana_core::banking_stage::{
-        forwarder::Forwarder,
-        leader_slot_metrics::LeaderSlotMetricsTracker,
-        unprocessed_packet_batches::{DeserializedPacket, UnprocessedPacketBatches},
-        unprocessed_transaction_storage::{ThreadType, UnprocessedTransactionStorage},
-        BankingStageStats,
-    },
-    solana_gossip::cluster_info::{ClusterInfo, Node},
-    solana_ledger::{
-        blockstore::Blockstore,
-        genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo},
-    },
-    solana_perf::{data_budget::DataBudget, packet::Packet},
-    solana_poh::{poh_recorder::create_test_recorder, poh_service::PohService},
-    solana_runtime::{bank::Bank, genesis_utils::bootstrap_validator_stake_lamports},
-    solana_sdk::{poh_config::PohConfig, signature::Keypair, signer::Signer, system_transaction},
-    solana_streamer::socket::SocketAddrSpace,
-    std::sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    tempfile::TempDir,
-    test::Bencher,
-};
-
-struct BenchSetup {
-    exit: Arc<AtomicBool>,
-    poh_service: PohService,
-    forwarder: Forwarder<Arc<ClusterInfo>>,
-    unprocessed_packet_batches: UnprocessedTransactionStorage,
-    tracker: LeaderSlotMetricsTracker,
-    stats: BankingStageStats,
-}
-
-fn setup(num_packets: usize, contentious_transaction: bool) -> BenchSetup {
-    let validator_keypair = Arc::new(Keypair::new());
-    let genesis_config_info = create_genesis_config_with_leader(
-        10_000,
-        &validator_keypair.pubkey(),
-        bootstrap_validator_stake_lamports(),
-    );
-    let GenesisConfigInfo { genesis_config, .. } = &genesis_config_info;
-
-    let (bank, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(genesis_config);
-
-    let ledger_path = TempDir::new().unwrap();
-    let blockstore = Arc::new(
-        Blockstore::open(ledger_path.as_ref())
-            .expect("Expected to be able to open database ledger"),
-    );
-    let poh_config = PohConfig {
-        // limit tick count to avoid clearing working_bank at
-        // PohRecord then PohRecorderError(MaxHeightReached) at BankingStage
-        target_tick_count: Some(bank.max_tick_height().saturating_sub(1)),
-        ..PohConfig::default()
-    };
-
-    let (exit, poh_recorder, poh_service, _entry_receiver) =
-        create_test_recorder(bank, blockstore, Some(poh_config), None);
-
-    let local_node = Node::new_localhost_with_pubkey(&validator_keypair.pubkey());
-    let cluster_info = ClusterInfo::new(
-        local_node.info.clone(),
-        validator_keypair,
-        SocketAddrSpace::Unspecified,
-    );
-    let cluster_info = Arc::new(cluster_info);
-    let min_balance = genesis_config.rent.minimum_balance(0);
-    let hash = genesis_config.hash();
-
-    // packets are deserialized upon receiving, failed packets will not be
-    // forwarded; Therefore need to create real packets here.
-    let keypair = Keypair::new();
-    let packets = (0..num_packets)
-        .map(|_| {
-            let mut transaction =
-                system_transaction::transfer(&keypair, &Keypair::new().pubkey(), min_balance, hash);
-            if !contentious_transaction {
-                transaction.message.account_keys[0] = solana_pubkey::Pubkey::new_unique();
-            }
-            let mut packet = Packet::from_data(None, transaction).unwrap();
-            packet.meta_mut().set_from_staked_node(true);
-            DeserializedPacket::new(packet).unwrap()
-        })
-        .collect_vec();
-
-    let unprocessed_packet_batches = UnprocessedTransactionStorage::new_transaction_storage(
-        UnprocessedPacketBatches::from_iter(packets, num_packets),
-        ThreadType::Transactions,
-    );
-
-    let connection_cache = ConnectionCache::new("connection_cache_test");
-    // use a restrictive data budget to bench everything except actual sending data over
-    // connection.
-    let data_budget = DataBudget::restricted();
-    let forwarder = Forwarder::new(
-        poh_recorder,
-        bank_forks,
-        cluster_info,
-        Arc::new(connection_cache),
-        Arc::new(data_budget),
-    );
-
-    BenchSetup {
-        exit,
-        poh_service,
-        forwarder,
-        unprocessed_packet_batches,
-        tracker: LeaderSlotMetricsTracker::new(0),
-        stats: BankingStageStats::default(),
-    }
-}
-
-#[bench]
-fn bench_forwarder_handle_forwading_contentious_transaction(bencher: &mut Bencher) {
-    let num_packets = 10240;
-    let BenchSetup {
-        exit,
-        poh_service,
-        mut forwarder,
-        mut unprocessed_packet_batches,
-        mut tracker,
-        stats,
-    } = setup(num_packets, true);
-
-    // hold packets so they can be reused for benching
-    let hold = true;
-    bencher.iter(|| {
-        forwarder.handle_forwarding(&mut unprocessed_packet_batches, hold, &mut tracker, &stats);
-        // reset packet.forwarded flag to reuse `unprocessed_packet_batches`
-        if let UnprocessedTransactionStorage::LocalTransactionStorage(unprocessed_packets) =
-            &mut unprocessed_packet_batches
-        {
-            for deserialized_packet in unprocessed_packets.iter_mut() {
-                deserialized_packet.forwarded = false;
-            }
-        }
-    });
-
-    exit.store(true, Ordering::Relaxed);
-    poh_service.join().unwrap();
-}
-
-#[bench]
-fn bench_forwarder_handle_forwading_parallel_transactions(bencher: &mut Bencher) {
-    let num_packets = 10240;
-    let BenchSetup {
-        exit,
-        poh_service,
-        mut forwarder,
-        mut unprocessed_packet_batches,
-        mut tracker,
-        stats,
-    } = setup(num_packets, false);
-
-    // hold packets so they can be reused for benching
-    let hold = true;
-    bencher.iter(|| {
-        forwarder.handle_forwarding(&mut unprocessed_packet_batches, hold, &mut tracker, &stats);
-        // reset packet.forwarded flag to reuse `unprocessed_packet_batches`
-        if let UnprocessedTransactionStorage::LocalTransactionStorage(unprocessed_packets) =
-            &mut unprocessed_packet_batches
-        {
-            for deserialized_packet in unprocessed_packets.iter_mut() {
-                deserialized_packet.forwarded = false;
-            }
-        }
-    });
-
-    exit.store(true, Ordering::Relaxed);
-    poh_service.join().unwrap();
-}

core/benches/sigverify_stage.rs

@@ -157,7 +157,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher, use_same_tx: bool) {
     trace!("start");
     let (packet_s, packet_r) = unbounded();
     let (verified_s, verified_r) = BankingTracer::channel_for_test();
-    let verifier = TransactionSigVerifier::new(verified_s);
+    let verifier = TransactionSigVerifier::new(verified_s, None);
     let stage = SigVerifyStage::new(packet_r, verifier, "solSigVerBench", "bench");
 
     bencher.iter(move || {
@@ -231,7 +231,7 @@ fn prepare_batches(discard_factor: i32) -> (Vec<PacketBatch>, usize) {
 fn bench_shrink_sigverify_stage_core(bencher: &mut Bencher, discard_factor: i32) {
     let (batches0, num_valid_packets) = prepare_batches(discard_factor);
     let (verified_s, _verified_r) = BankingTracer::channel_for_test();
-    let verifier = TransactionSigVerifier::new(verified_s);
+    let verifier = TransactionSigVerifier::new(verified_s, None);
 
     let mut c = 0;
     let mut total_shrink_time = 0;

core/src/banking_simulation.rs

@@ -16,7 +16,6 @@ use {
     crossbeam_channel::{unbounded, Sender},
     itertools::Itertools,
     log::*,
-    solana_client::connection_cache::ConnectionCache,
     solana_gossip::{
         cluster_info::{ClusterInfo, Node},
         contact_info::ContactInfoQuery,
@@ -787,7 +786,6 @@ impl BankingSimulator {
             gossip_vote_receiver,
         } = retracer.create_channels(false);
 
-        let connection_cache = Arc::new(ConnectionCache::new("connection_cache_sim"));
         let (replay_vote_sender, _replay_vote_receiver) = unbounded();
         let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
         let shred_version = compute_shred_version(
@@ -833,10 +831,8 @@ impl BankingSimulator {
             None,
             replay_vote_sender,
             None,
-            connection_cache,
             bank_forks.clone(),
             prioritization_fee_cache,
-            false,
         );
 
         let (&_slot, &raw_base_event_time) = freeze_time_by_slot

core/src/banking_stage.rs

@@ -9,7 +9,6 @@ use {
         committer::Committer,
         consumer::Consumer,
         decision_maker::{BufferedPacketsDecision, DecisionMaker},
-        forwarder::Forwarder,
         latest_unprocessed_votes::{LatestUnprocessedVotes, VoteSource},
         leader_slot_metrics::LeaderSlotMetricsTracker,
         packet_receiver::PacketReceiver,
@@ -30,11 +29,10 @@ use {
     agave_banking_stage_ingress_types::BankingPacketReceiver,
     crossbeam_channel::{unbounded, Receiver, RecvTimeoutError, Sender},
     histogram::Histogram,
-    solana_client::connection_cache::ConnectionCache,
     solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfoQuery},
     solana_ledger::blockstore_processor::TransactionStatusSender,
     solana_measure::measure_us,
-    solana_perf::{data_budget::DataBudget, packet::PACKETS_PER_BATCH},
+    solana_perf::packet::PACKETS_PER_BATCH,
     solana_poh::poh_recorder::{PohRecorder, TransactionRecorder},
     solana_runtime::{
         bank::Bank, bank_forks::BankForks, prioritization_fee_cache::PrioritizationFeeCache,
@@ -64,7 +62,6 @@ use {
 // Below modules are pub to allow use by banking_stage bench
 pub mod committer;
 pub mod consumer;
-pub mod forwarder;
 pub mod leader_slot_metrics;
 pub mod qos_service;
 pub mod unprocessed_packet_batches;
@@ -72,7 +69,6 @@ pub mod unprocessed_transaction_storage;
 
 mod consume_worker;
 mod decision_maker;
-mod forward_packet_batches_by_accounts;
 mod immutable_deserialized_packet;
 mod latest_unprocessed_votes;
 mod leader_slot_timing_metrics;
@@ -115,8 +111,6 @@ pub struct BankingStageStats {
     current_buffered_packets_count: AtomicUsize,
     rebuffered_packets_count: AtomicUsize,
     consumed_buffered_packets_count: AtomicUsize,
-    forwarded_transaction_count: AtomicUsize,
-    forwarded_vote_count: AtomicUsize,
     batch_packet_indexes_len: Histogram,
 
     // Timing
@@ -161,8 +155,6 @@ impl BankingStageStats {
             + self.filter_pending_packets_elapsed.load(Ordering::Relaxed)
             + self.packet_conversion_elapsed.load(Ordering::Relaxed)
             + self.transaction_processing_elapsed.load(Ordering::Relaxed)
-            + self.forwarded_transaction_count.load(Ordering::Relaxed) as u64
-            + self.forwarded_vote_count.load(Ordering::Relaxed) as u64
             + self.batch_packet_indexes_len.entries()
     }
 
@@ -226,16 +218,6 @@ impl BankingStageStats {
                         .swap(0, Ordering::Relaxed),
                     i64
                 ),
-                (
-                    "forwarded_transaction_count",
-                    self.forwarded_transaction_count.swap(0, Ordering::Relaxed),
-                    i64
-                ),
-                (
-                    "forwarded_vote_count",
-                    self.forwarded_vote_count.swap(0, Ordering::Relaxed),
-                    i64
-                ),
                 (
                     "consume_buffered_packets_elapsed",
                     self.consume_buffered_packets_elapsed
@@ -320,21 +302,6 @@ pub struct BankingStage {
     bank_thread_hdls: Vec<JoinHandle<()>>,
 }
 
-#[derive(Debug, Clone)]
-pub enum ForwardOption {
-    NotForward,
-    ForwardTpuVote,
-    ForwardTransaction,
-}
-
-#[derive(Debug, Default)]
-pub struct FilterForwardingResults {
-    pub(crate) total_forwardable_packets: usize,
-    pub(crate) total_dropped_packets: usize,
-    pub(crate) total_packet_conversion_us: u64,
-    pub(crate) total_filter_packets_us: u64,
-}
-
 pub trait LikeClusterInfo: Send + Sync + 'static + Clone {
     fn id(&self) -> Pubkey;
 
@@ -365,10 +332,8 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         replay_vote_sender: ReplayVoteSender,
         log_messages_bytes_limit: Option<usize>,
-        connection_cache: Arc<ConnectionCache>,
         bank_forks: Arc<RwLock<BankForks>>,
         prioritization_fee_cache: &Arc<PrioritizationFeeCache>,
-        enable_forwarding: bool,
     ) -> Self {
         Self::new_num_threads(
             block_production_method,
@@ -382,10 +347,8 @@ impl BankingStage {
             transaction_status_sender,
             replay_vote_sender,
             log_messages_bytes_limit,
-            connection_cache,
             bank_forks,
             prioritization_fee_cache,
-            enable_forwarding,
         )
     }
 
@@ -402,10 +365,8 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         replay_vote_sender: ReplayVoteSender,
         log_messages_bytes_limit: Option<usize>,
-        connection_cache: Arc<ConnectionCache>,
         bank_forks: Arc<RwLock<BankForks>>,
         prioritization_fee_cache: &Arc<PrioritizationFeeCache>,
-        enable_forwarding: bool,
     ) -> Self {
         match block_production_method {
             BlockProductionMethod::CentralScheduler
@@ -426,10 +387,8 @@ impl BankingStage {
                     transaction_status_sender,
                     replay_vote_sender,
                     log_messages_bytes_limit,
-                    connection_cache,
                     bank_forks,
                     prioritization_fee_cache,
-                    enable_forwarding,
                 )
             }
         }
@@ -448,16 +407,10 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         replay_vote_sender: ReplayVoteSender,
         log_messages_bytes_limit: Option<usize>,
-        connection_cache: Arc<ConnectionCache>,
         bank_forks: Arc<RwLock<BankForks>>,
         prioritization_fee_cache: &Arc<PrioritizationFeeCache>,
-        enable_forwarding: bool,
     ) -> Self {
         assert!(num_threads >= MIN_TOTAL_THREADS);
-        // Single thread to generate entries from many banks.
-        // This thread talks to poh_service and broadcasts the entries once they have been recorded.
-        // Once an entry has been recorded, its blockhash is registered with the bank.
-        let data_budget = Arc::new(DataBudget::default());
         // Keeps track of extraneous vote transactions for the vote threads
         let latest_unprocessed_votes = {
             let bank = bank_forks.read().unwrap().working_bank();
@@ -484,16 +437,10 @@ impl BankingStage {
                 id,
                 packet_receiver,
                 decision_maker.clone(),
+                bank_forks.clone(),
                 committer.clone(),
                 transaction_recorder.clone(),
                 log_messages_bytes_limit,
-                Forwarder::new(
-                    poh_recorder.clone(),
-                    bank_forks.clone(),
-                    cluster_info.clone(),
-                    connection_cache.clone(),
-                    data_budget.clone(),
-                ),
                 UnprocessedTransactionStorage::new_vote_storage(
                     latest_unprocessed_votes.clone(),
                     vote_source,
@@ -501,22 +448,11 @@ impl BankingStage {
             ));
         }
 
-        let transaction_struct =
-            if enable_forwarding && !matches!(transaction_struct, TransactionStructure::Sdk) {
-                warn!(
-                "Forwarding only supported for `Sdk` transaction struct. Overriding to use `Sdk`."
-            );
-                TransactionStructure::Sdk
-            } else {
-                transaction_struct
-            };
-
         match transaction_struct {
             TransactionStructure::Sdk => {
                 let receive_and_buffer = SanitizedTransactionReceiveAndBuffer::new(
                     PacketDeserializer::new(non_vote_receiver),
                     bank_forks.clone(),
-                    enable_forwarding,
                 );
                 Self::spawn_scheduler_and_workers(
                     &mut bank_thread_hdls,
@@ -524,14 +460,10 @@ impl BankingStage {
                     use_greedy_scheduler,
                     decision_maker,
                     committer,
-                    cluster_info,
                     poh_recorder,
                     num_threads,
                     log_messages_bytes_limit,
-                    connection_cache,
                     bank_forks,
-                    enable_forwarding,
-                    data_budget,
                 );
             }
             TransactionStructure::View => {
@@ -545,14 +477,10 @@ impl BankingStage {
                     use_greedy_scheduler,
                     decision_maker,
                     committer,
-                    cluster_info,
                     poh_recorder,
                     num_threads,
                     log_messages_bytes_limit,
-                    connection_cache,
                     bank_forks,
-                    enable_forwarding,
-                    data_budget,
                 );
             }
         }
@@ -567,14 +495,10 @@ impl BankingStage {
         use_greedy_scheduler: bool,
         decision_maker: DecisionMaker,
         committer: Committer,
-        cluster_info: &impl LikeClusterInfo,
         poh_recorder: &Arc<RwLock<PohRecorder>>,
         num_threads: u32,
         log_messages_bytes_limit: Option<usize>,
-        connection_cache: Arc<ConnectionCache>,
         bank_forks: Arc<RwLock<BankForks>>,
-        enable_forwarding: bool,
-        data_budget: Arc<DataBudget>,
     ) {
         // Create channels for communication between scheduler and workers
         let num_workers = (num_threads).saturating_sub(NUM_VOTE_PROCESSING_THREADS);
@@ -610,16 +534,6 @@ impl BankingStage {
             )
         }
 
-        let forwarder = enable_forwarding.then(|| {
-            Forwarder::new(
-                poh_recorder.clone(),
-                bank_forks.clone(),
-                cluster_info.clone(),
-                connection_cache.clone(),
-                data_budget.clone(),
-            )
-        });
-
         // Spawn the central scheduler thread
         if use_greedy_scheduler {
             bank_thread_hdls.push(
@@ -637,7 +551,6 @@ impl BankingStage {
                             bank_forks,
                             scheduler,
                             worker_metrics,
-                            forwarder,
                         );
 
                         match scheduler_controller.run() {
@@ -666,7 +579,6 @@ impl BankingStage {
                             bank_forks,
                             scheduler,
                             worker_metrics,
-                            forwarder,
                         );
 
                         match scheduler_controller.run() {
@@ -682,14 +594,14 @@ impl BankingStage {
         }
     }
 
-    fn spawn_thread_local_multi_iterator_thread<T: LikeClusterInfo>(
+    fn spawn_thread_local_multi_iterator_thread(
         id: u32,
         packet_receiver: BankingPacketReceiver,
         mut decision_maker: DecisionMaker,
+        bank_forks: Arc<RwLock<BankForks>>,
         committer: Committer,
         transaction_recorder: TransactionRecorder,
         log_messages_bytes_limit: Option<usize>,
-        mut forwarder: Forwarder<T>,
         unprocessed_transaction_storage: UnprocessedTransactionStorage,
     ) -> JoinHandle<()> {
         let mut packet_receiver = PacketReceiver::new(id, packet_receiver);
@@ -706,7 +618,7 @@ impl BankingStage {
                 Self::process_loop(
                     &mut packet_receiver,
                     &mut decision_maker,
-                    &mut forwarder,
+                    &bank_forks,
                     &consumer,
                     id,
                     unprocessed_transaction_storage,
@@ -716,9 +628,9 @@ impl BankingStage {
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn process_buffered_packets<T: LikeClusterInfo>(
+    fn process_buffered_packets(
         decision_maker: &mut DecisionMaker,
-        forwarder: &mut Forwarder<T>,
+        bank_forks: &RwLock<BankForks>,
         consumer: &Consumer,
         unprocessed_transaction_storage: &mut UnprocessedTransactionStorage,
         banking_stage_stats: &BankingStageStats,
@@ -753,36 +665,26 @@ impl BankingStage {
                     .increment_consume_buffered_packets_us(consume_buffered_packets_us);
             }
             BufferedPacketsDecision::Forward => {
-                let ((), forward_us) = measure_us!(forwarder.handle_forwarding(
-                    unprocessed_transaction_storage,
-                    false,
-                    slot_metrics_tracker,
-                    banking_stage_stats,
-                ));
-                slot_metrics_tracker.increment_forward_us(forward_us);
-                // Take metrics action after forwarding packets to include forwarded
-                // metrics into current slot
-                slot_metrics_tracker.apply_action(metrics_action);
+                // get current working bank from bank_forks, use it to sanitize transaction and
+                // load all accounts from address loader;
+                let current_bank = bank_forks.read().unwrap().working_bank();
+                unprocessed_transaction_storage.cache_epoch_boundary_info(&current_bank);
+                unprocessed_transaction_storage.clear();
             }
             BufferedPacketsDecision::ForwardAndHold => {
-                let ((), forward_and_hold_us) = measure_us!(forwarder.handle_forwarding(
-                    unprocessed_transaction_storage,
-                    true,
-                    slot_metrics_tracker,
-                    banking_stage_stats,
-                ));
-                slot_metrics_tracker.increment_forward_and_hold_us(forward_and_hold_us);
-                // Take metrics action after forwarding packets
-                slot_metrics_tracker.apply_action(metrics_action);
+                // get current working bank from bank_forks, use it to sanitize transaction and
+                // load all accounts from address loader;
+                let current_bank = bank_forks.read().unwrap().working_bank();
+                unprocessed_transaction_storage.cache_epoch_boundary_info(&current_bank);
             }
-            _ => (),
+            BufferedPacketsDecision::Hold => {}
         }
     }
 
-    fn process_loop<T: LikeClusterInfo>(
+    fn process_loop(
         packet_receiver: &mut PacketReceiver,
         decision_maker: &mut DecisionMaker,
-        forwarder: &mut Forwarder<T>,
+        bank_forks: &RwLock<BankForks>,
         consumer: &Consumer,
         id: u32,
         mut unprocessed_transaction_storage: UnprocessedTransactionStorage,
@@ -798,7 +700,7 @@ impl BankingStage {
             {
                 let (_, process_buffered_packets_us) = measure_us!(Self::process_buffered_packets(
                     decision_maker,
-                    forwarder,
+                    bank_forks,
                     consumer,
                     &mut unprocessed_transaction_storage,
                     &banking_stage_stats,
@@ -949,10 +851,8 @@ mod tests {
             None,
             replay_vote_sender,
             None,
-            Arc::new(ConnectionCache::new("connection_cache_test")),
             bank_forks,
             &Arc::new(PrioritizationFeeCache::new(0u64)),
-            false,
         );
         drop(non_vote_sender);
         drop(tpu_vote_sender);
@@ -1008,10 +908,8 @@ mod tests {
             None,
             replay_vote_sender,
             None,
-            Arc::new(ConnectionCache::new("connection_cache_test")),
             bank_forks,
             &Arc::new(PrioritizationFeeCache::new(0u64)),
-            false,
         );
         trace!("sending bank");
         drop(non_vote_sender);
@@ -1093,10 +991,8 @@ mod tests {
             None,
             replay_vote_sender,
             None,
-            Arc::new(ConnectionCache::new("connection_cache_test")),
             bank_forks.clone(), // keep a local-copy of bank-forks so worker threads do not lose weak access to bank-forks
             &Arc::new(PrioritizationFeeCache::new(0u64)),
-            false,
         );
 
         // fund another account so we can send 2 good transactions in a single batch.
@@ -1265,10 +1161,8 @@ mod tests {
                 None,
                 replay_vote_sender,
                 None,
-                Arc::new(ConnectionCache::new("connection_cache_test")),
                 bank_forks,
                 &Arc::new(PrioritizationFeeCache::new(0u64)),
-                false,
             );
 
             // wait for banking_stage to eat the packets
@@ -1455,10 +1349,8 @@ mod tests {
             None,
             replay_vote_sender,
             None,
-            Arc::new(ConnectionCache::new("connection_cache_test")),
             bank_forks,
             &Arc::new(PrioritizationFeeCache::new(0u64)),
-            false,
         );
 
         let keypairs = (0..100).map(|_| Keypair::new()).collect_vec();

core/src/banking_stage/consumer.rs

@@ -214,7 +214,7 @@ impl Consumer {
             packets_to_process_len.saturating_sub(retryable_transaction_indexes.len());
 
         // Out of the buffered packets just retried, collect any still unprocessed
-        // transactions in this batch for forwarding
+        // transactions in this batch
         *rebuffered_packet_count += retryable_transaction_indexes.len();
 
         payload

core/src/banking_stage/forward_packet_batches_by_accounts.rs

@@ -1,429 +0,0 @@
-use {
-    super::immutable_deserialized_packet::ImmutableDeserializedPacket,
-    core::num::NonZeroU64,
-    solana_cost_model::{
-        block_cost_limits,
-        cost_model::CostModel,
-        cost_tracker::{CostTracker, UpdatedCosts},
-    },
-    solana_feature_set::FeatureSet,
-    solana_perf::packet::Packet,
-    solana_runtime_transaction::transaction_with_meta::TransactionWithMeta,
-    std::sync::Arc,
-};
-
-/// `ForwardBatch` to have half of default cost_tracker limits, as smaller batch
-/// allows better granularity in composing forwarding transactions; e.g.,
-/// transactions in each batch are potentially more evenly distributed across accounts.
-const FORWARDED_BLOCK_COMPUTE_RATIO: u32 = 2;
-/// this number divided by`FORWARDED_BLOCK_COMPUTE_RATIO` is the total blocks to forward.
-/// To accommodate transactions without `compute_budget` instruction, which will
-/// have default 200_000 compute units, it has 100 batches as default to forward
-/// up to 12_000 such transaction. (120 such transactions fill up a batch, 100
-/// batches allows 12_000 transactions)
-const DEFAULT_NUMBER_OF_BATCHES: u32 = 100;
-
-/// `ForwardBatch` represents one forwardable batch of transactions with a
-/// limited number of total compute units
-#[derive(Debug, Default)]
-pub struct ForwardBatch {
-    // `forwardable_packets` keeps forwardable packets in a vector in its
-    // original fee prioritized order
-    forwardable_packets: Vec<Arc<ImmutableDeserializedPacket>>,
-}
-
-impl ForwardBatch {
-    pub fn get_forwardable_packets(&self) -> impl Iterator<Item = &Packet> {
-        self.forwardable_packets
-            .iter()
-            .map(|immutable_packet| immutable_packet.original_packet())
-    }
-
-    pub fn len(&self) -> usize {
-        self.forwardable_packets.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.forwardable_packets.is_empty()
-    }
-
-    pub fn reset(&mut self) {
-        self.forwardable_packets.clear();
-    }
-}
-
-/// To avoid forward queue being saturated by transactions for single hot account,
-/// the forwarder will group and send prioritized transactions by account limit
-/// to allow transactions on non-congested accounts to be forwarded alongside higher fee
-/// transactions that saturate those highly demanded accounts.
-#[derive(Debug)]
-pub struct ForwardPacketBatchesByAccounts {
-    // Forwardable packets are staged in number of batches, each batch is limited
-    // by cost_tracker on both account limit and block limits. Those limits are
-    // set as `limit_ratio` of regular block limits to facilitate quicker iteration.
-    forward_batches: Vec<ForwardBatch>,
-
-    // Single cost tracker that imposes the total cu limits for forwarding.
-    cost_tracker: CostTracker,
-
-    // Compute Unit limits for each batch
-    batch_block_limit: NonZeroU64,
-    batch_account_limit: NonZeroU64,
-}
-
-impl ForwardPacketBatchesByAccounts {
-    pub fn new_with_default_batch_limits() -> Self {
-        Self::new(FORWARDED_BLOCK_COMPUTE_RATIO, DEFAULT_NUMBER_OF_BATCHES)
-    }
-
-    pub fn new(limit_ratio: u32, number_of_batches: u32) -> Self {
-        let forward_batches = (0..number_of_batches)
-            .map(|_| ForwardBatch::default())
-            .collect();
-
-        let batch_vote_limit =
-            NonZeroU64::new(block_cost_limits::MAX_VOTE_UNITS.saturating_div(limit_ratio as u64))
-                .expect("batch vote limit must not be zero");
-        let batch_block_limit =
-            NonZeroU64::new(block_cost_limits::MAX_BLOCK_UNITS.saturating_div(limit_ratio as u64))
-                .expect("batch block limit must not be zero");
-        let batch_account_limit = NonZeroU64::new(
-            block_cost_limits::MAX_WRITABLE_ACCOUNT_UNITS.saturating_div(limit_ratio as u64),
-        )
-        .expect("batch account limit must not be zero");
-
-        let mut cost_tracker = CostTracker::default();
-        cost_tracker.set_limits(
-            batch_account_limit
-                .get()
-                .saturating_mul(number_of_batches as u64),
-            batch_block_limit
-                .get()
-                .saturating_mul(number_of_batches as u64),
-            batch_vote_limit
-                .get()
-                .saturating_mul(number_of_batches as u64),
-        );
-        Self {
-            forward_batches,
-            cost_tracker,
-            batch_block_limit,
-            batch_account_limit,
-        }
-    }
-
-    pub fn try_add_packet(
-        &mut self,
-        sanitized_transaction: &impl TransactionWithMeta,
-        immutable_packet: Arc<ImmutableDeserializedPacket>,
-        feature_set: &FeatureSet,
-    ) -> bool {
-        let tx_cost = CostModel::calculate_cost(sanitized_transaction, feature_set);
-
-        if let Ok(updated_costs) = self.cost_tracker.try_add(&tx_cost) {
-            let batch_index = self.get_batch_index_by_updated_costs(&updated_costs);
-
-            if let Some(forward_batch) = self.forward_batches.get_mut(batch_index) {
-                forward_batch.forwardable_packets.push(immutable_packet);
-            } else {
-                // A successfully added tx_cost means it does not exceed block limit, nor vote
-                // limit, nor account limit. batch_index calculated as quotient from division
-                // will not be out of bounds.
-                unreachable!("batch_index out of bounds");
-            }
-            true
-        } else {
-            false
-        }
-    }
-
-    pub fn iter_batches(&self) -> impl Iterator<Item = &ForwardBatch> {
-        self.forward_batches.iter()
-    }
-
-    pub fn reset(&mut self) {
-        for forward_batch in self.forward_batches.iter_mut() {
-            forward_batch.reset();
-        }
-
-        self.cost_tracker.reset();
-    }
-
-    // Successfully added packet should be placed into the batch where no block/vote/account limits
-    // would be exceeded. Eg, if by block limit, it can be put into batch #1; by vote limit, it can
-    // be put into batch #2; and by account limit, it can be put into batch #3; then it should be
-    // put into batch #3 to satisfy all batch limits.
-    fn get_batch_index_by_updated_costs(&self, updated_costs: &UpdatedCosts) -> usize {
-        let batch_index_by_block_cost =
-            updated_costs.updated_block_cost / self.batch_block_limit.get();
-        let batch_index_by_account_limit =
-            updated_costs.updated_costliest_account_cost / self.batch_account_limit.get();
-        batch_index_by_block_cost.max(batch_index_by_account_limit) as usize
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use {
-        super::*,
-        crate::banking_stage::unprocessed_packet_batches::DeserializedPacket,
-        solana_feature_set::FeatureSet,
-        solana_runtime_transaction::runtime_transaction::RuntimeTransaction,
-        solana_sdk::{
-            compute_budget::ComputeBudgetInstruction,
-            message::Message,
-            pubkey::Pubkey,
-            system_instruction,
-            transaction::{SanitizedTransaction, Transaction},
-        },
-    };
-
-    /// build test transaction, return corresponding sanitized_transaction and deserialized_packet,
-    /// and the batch limit_ratio that would only allow one transaction per bucket.
-    fn build_test_transaction_and_packet(
-        priority: u64,
-        write_to_account: &Pubkey,
-    ) -> (
-        RuntimeTransaction<SanitizedTransaction>,
-        DeserializedPacket,
-        u32,
-    ) {
-        let from_account = solana_pubkey::new_rand();
-
-        let transaction = Transaction::new_unsigned(Message::new(
-            &[
-                ComputeBudgetInstruction::set_compute_unit_price(priority),
-                system_instruction::transfer(&from_account, write_to_account, 2),
-            ],
-            Some(&from_account),
-        ));
-        let sanitized_transaction =
-            RuntimeTransaction::from_transaction_for_tests(transaction.clone());
-        let tx_cost = CostModel::calculate_cost(&sanitized_transaction, &FeatureSet::all_enabled());
-        let cost = tx_cost.sum();
-        let deserialized_packet =
-            DeserializedPacket::new(Packet::from_data(None, transaction).unwrap()).unwrap();
-
-        // set limit ratio so each batch can only have one test transaction
-        let limit_ratio: u32 =
-            ((block_cost_limits::MAX_WRITABLE_ACCOUNT_UNITS - cost + 1) / cost) as u32;
-        (sanitized_transaction, deserialized_packet, limit_ratio)
-    }
-
-    #[test]
-    fn test_try_add_packet_to_multiple_batches() {
-        // setup two transactions, one has high priority that writes to hot account, the
-        // other write to non-contentious account with no priority
-        let hot_account = solana_pubkey::new_rand();
-        let other_account = solana_pubkey::new_rand();
-        let (tx_high_priority, packet_high_priority, limit_ratio) =
-            build_test_transaction_and_packet(10, &hot_account);
-        let (tx_low_priority, packet_low_priority, _) =
-            build_test_transaction_and_packet(0, &other_account);
-
-        // setup forwarding with 2 buckets, each only allow one transaction
-        let number_of_batches = 2;
-        let mut forward_packet_batches_by_accounts =
-            ForwardPacketBatchesByAccounts::new(limit_ratio, number_of_batches);
-
-        // Assert initially both batches are empty
-        {
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(0, batches.next().unwrap().len());
-            assert_eq!(0, batches.next().unwrap().len());
-            assert!(batches.next().is_none());
-        }
-
-        // Assert one high-priority packet will be added to 1st bucket successfully
-        {
-            assert!(forward_packet_batches_by_accounts.try_add_packet(
-                &tx_high_priority,
-                packet_high_priority.immutable_section().clone(),
-                &FeatureSet::all_enabled(),
-            ));
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(1, batches.next().unwrap().len());
-            assert_eq!(0, batches.next().unwrap().len());
-            assert!(batches.next().is_none());
-        }
-
-        // Assert second high-priority packet will not fit in first bucket, but will
-        // be added to 2nd bucket
-        {
-            assert!(forward_packet_batches_by_accounts.try_add_packet(
-                &tx_high_priority,
-                packet_high_priority.immutable_section().clone(),
-                &FeatureSet::all_enabled(),
-            ));
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(1, batches.next().unwrap().len());
-            assert_eq!(1, batches.next().unwrap().len());
-        }
-
-        // Assert 3rd high-priority packet can not added since both buckets would
-        // exceed hot-account limit
-        {
-            assert!(!forward_packet_batches_by_accounts.try_add_packet(
-                &tx_high_priority,
-                packet_high_priority.immutable_section().clone(),
-                &FeatureSet::all_enabled(),
-            ));
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(1, batches.next().unwrap().len());
-            assert_eq!(1, batches.next().unwrap().len());
-            assert!(batches.next().is_none());
-        }
-
-        // Assert lower priority packet will be successfully added to first bucket
-        // since non-contentious account is still free
-        {
-            assert!(forward_packet_batches_by_accounts.try_add_packet(
-                &tx_low_priority,
-                packet_low_priority.immutable_section().clone(),
-                &FeatureSet::all_enabled(),
-            ));
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(2, batches.next().unwrap().len());
-            assert_eq!(1, batches.next().unwrap().len());
-            assert!(batches.next().is_none());
-        }
-    }
-
-    #[test]
-    fn test_try_add_packet_to_single_batch() {
-        let (tx, packet, limit_ratio) =
-            build_test_transaction_and_packet(10, &solana_pubkey::new_rand());
-        let number_of_batches = 1;
-        let mut forward_packet_batches_by_accounts =
-            ForwardPacketBatchesByAccounts::new(limit_ratio, number_of_batches);
-
-        // Assert initially batch is empty, and accepting new packets
-        {
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(0, batches.next().unwrap().len());
-            assert!(batches.next().is_none());
-        }
-
-        // Assert can successfully add first packet to forwarding buffer
-        {
-            assert!(forward_packet_batches_by_accounts.try_add_packet(
-                &tx,
-                packet.immutable_section().clone(),
-                &FeatureSet::all_enabled()
-            ));
-
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(1, batches.next().unwrap().len());
-        }
-
-        // Assert cannot add same packet to forwarding buffer again, due to reached account limit;
-        {
-            assert!(!forward_packet_batches_by_accounts.try_add_packet(
-                &tx,
-                packet.immutable_section().clone(),
-                &FeatureSet::all_enabled()
-            ));
-
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(1, batches.next().unwrap().len());
-        }
-
-        // Assert can still add non-contentious packet to same batch
-        {
-            // build a small packet to a non-contentious account with high priority
-            let (tx2, packet2, _) =
-                build_test_transaction_and_packet(100, &solana_pubkey::new_rand());
-
-            assert!(forward_packet_batches_by_accounts.try_add_packet(
-                &tx2,
-                packet2.immutable_section().clone(),
-                &FeatureSet::all_enabled()
-            ));
-
-            let mut batches = forward_packet_batches_by_accounts.iter_batches();
-            assert_eq!(2, batches.next().unwrap().len());
-        }
-    }
-
-    #[test]
-    fn test_get_batch_index_by_updated_costs() {
-        let test_cost = 99;
-
-        // check against block limit only
-        {
-            let mut forward_packet_batches_by_accounts =
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-            forward_packet_batches_by_accounts.batch_block_limit =
-                NonZeroU64::new(test_cost + 1).unwrap();
-
-            assert_eq!(
-                0,
-                forward_packet_batches_by_accounts.get_batch_index_by_updated_costs(
-                    &UpdatedCosts {
-                        updated_block_cost: test_cost,
-                        updated_costliest_account_cost: 0
-                    }
-                )
-            );
-            assert_eq!(
-                1,
-                forward_packet_batches_by_accounts.get_batch_index_by_updated_costs(
-                    &UpdatedCosts {
-                        updated_block_cost: test_cost + 1,
-                        updated_costliest_account_cost: 0
-                    }
-                )
-            );
-        }
-
-        // check against account limit only
-        {
-            let mut forward_packet_batches_by_accounts =
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-            forward_packet_batches_by_accounts.batch_account_limit =
-                NonZeroU64::new(test_cost + 1).unwrap();
-
-            assert_eq!(
-                0,
-                forward_packet_batches_by_accounts.get_batch_index_by_updated_costs(
-                    &UpdatedCosts {
-                        updated_block_cost: 0,
-                        updated_costliest_account_cost: test_cost
-                    }
-                )
-            );
-            assert_eq!(
-                1,
-                forward_packet_batches_by_accounts.get_batch_index_by_updated_costs(
-                    &UpdatedCosts {
-                        updated_block_cost: 0,
-                        updated_costliest_account_cost: test_cost + 1
-                    }
-                )
-            );
-        }
-
-        // by block limit, it can be put into batch #1;
-        // by vote limit, it can be put into batch #2;
-        // by account limit, it can be put into batch #3;
-        // it should be put into batch #3 to satisfy all batch limits.
-        {
-            let mut forward_packet_batches_by_accounts =
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-            forward_packet_batches_by_accounts.batch_block_limit =
-                NonZeroU64::new(test_cost + 1).unwrap();
-            forward_packet_batches_by_accounts.batch_account_limit =
-                NonZeroU64::new(test_cost / 3 + 1).unwrap();
-
-            assert_eq!(
-                2,
-                forward_packet_batches_by_accounts.get_batch_index_by_updated_costs(
-                    &UpdatedCosts {
-                        updated_block_cost: test_cost,
-                        updated_costliest_account_cost: test_cost
-                    }
-                )
-            );
-        }
-    }
-}

core/src/banking_stage/forwarder.rs

@@ -1,592 +0,0 @@
-use {
-    super::{
-        forward_packet_batches_by_accounts::ForwardPacketBatchesByAccounts,
-        leader_slot_metrics::LeaderSlotMetricsTracker,
-        unprocessed_transaction_storage::UnprocessedTransactionStorage, BankingStageStats,
-        ForwardOption,
-    },
-    crate::{
-        banking_stage::{
-            immutable_deserialized_packet::ImmutableDeserializedPacket, LikeClusterInfo,
-        },
-        next_leader::{next_leader, next_leader_tpu_vote},
-    },
-    solana_client::connection_cache::ConnectionCache,
-    solana_connection_cache::client_connection::ClientConnection as TpuConnection,
-    solana_feature_set::FeatureSet,
-    solana_measure::measure_us,
-    solana_net_utils::bind_to_unspecified,
-    solana_perf::{data_budget::DataBudget, packet::Packet},
-    solana_poh::poh_recorder::PohRecorder,
-    solana_runtime::bank_forks::BankForks,
-    solana_runtime_transaction::transaction_with_meta::TransactionWithMeta,
-    solana_sdk::{pubkey::Pubkey, transport::TransportError},
-    solana_streamer::sendmmsg::batch_send,
-    std::{
-        net::{SocketAddr, UdpSocket},
-        sync::{atomic::Ordering, Arc, RwLock},
-    },
-};
-
-pub struct Forwarder<T: LikeClusterInfo> {
-    poh_recorder: Arc<RwLock<PohRecorder>>,
-    bank_forks: Arc<RwLock<BankForks>>,
-    socket: UdpSocket,
-    cluster_info: T,
-    connection_cache: Arc<ConnectionCache>,
-    data_budget: Arc<DataBudget>,
-    forward_packet_batches_by_accounts: ForwardPacketBatchesByAccounts,
-}
-
-impl<T: LikeClusterInfo> Forwarder<T> {
-    pub fn new(
-        poh_recorder: Arc<RwLock<PohRecorder>>,
-        bank_forks: Arc<RwLock<BankForks>>,
-        cluster_info: T,
-        connection_cache: Arc<ConnectionCache>,
-        data_budget: Arc<DataBudget>,
-    ) -> Self {
-        Self {
-            poh_recorder,
-            bank_forks,
-            socket: bind_to_unspecified().unwrap(),
-            cluster_info,
-            connection_cache,
-            data_budget,
-            forward_packet_batches_by_accounts:
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits(),
-        }
-    }
-
-    pub fn clear_batches(&mut self) {
-        self.forward_packet_batches_by_accounts.reset();
-    }
-
-    pub fn try_add_packet(
-        &mut self,
-        sanitized_transaction: &impl TransactionWithMeta,
-        immutable_packet: Arc<ImmutableDeserializedPacket>,
-        feature_set: &FeatureSet,
-    ) -> bool {
-        self.forward_packet_batches_by_accounts.try_add_packet(
-            sanitized_transaction,
-            immutable_packet,
-            feature_set,
-        )
-    }
-
-    pub fn forward_batched_packets(&self, forward_option: &ForwardOption) {
-        self.forward_packet_batches_by_accounts
-            .iter_batches()
-            .filter(|&batch| !batch.is_empty())
-            .for_each(|forwardable_batch| {
-                let _ = self
-                    .forward_packets(forward_option, forwardable_batch.get_forwardable_packets());
-            });
-    }
-
-    /// This function is exclusively used by multi-iterator banking threads to handle forwarding
-    /// logic per thread. Central scheduler Controller uses try_add_packet() ... forward_batched_packets()
-    /// to handle forwarding slight differently.
-    pub fn handle_forwarding(
-        &mut self,
-        unprocessed_transaction_storage: &mut UnprocessedTransactionStorage,
-        hold: bool,
-        slot_metrics_tracker: &mut LeaderSlotMetricsTracker,
-        banking_stage_stats: &BankingStageStats,
-    ) {
-        let forward_option = unprocessed_transaction_storage.forward_option();
-
-        // get current working bank from bank_forks, use it to sanitize transaction and
-        // load all accounts from address loader;
-        let current_bank = self.bank_forks.read().unwrap().working_bank();
-
-        // if we have crossed an epoch boundary, recache any state
-        unprocessed_transaction_storage.cache_epoch_boundary_info(&current_bank);
-
-        // sanitize and filter packets that are no longer valid (could be too old, a duplicate of something
-        // already processed), then add to forwarding buffer.
-        let filter_forwarding_result = unprocessed_transaction_storage
-            .filter_forwardable_packets_and_add_batches(
-                current_bank,
-                &mut self.forward_packet_batches_by_accounts,
-            );
-        slot_metrics_tracker.increment_transactions_from_packets_us(
-            filter_forwarding_result.total_packet_conversion_us,
-        );
-        banking_stage_stats.packet_conversion_elapsed.fetch_add(
-            filter_forwarding_result.total_packet_conversion_us,
-            Ordering::Relaxed,
-        );
-        banking_stage_stats
-            .filter_pending_packets_elapsed
-            .fetch_add(
-                filter_forwarding_result.total_filter_packets_us,
-                Ordering::Relaxed,
-            );
-        banking_stage_stats.dropped_forward_packets_count.fetch_add(
-            filter_forwarding_result.total_dropped_packets,
-            Ordering::Relaxed,
-        );
-
-        self.forward_packet_batches_by_accounts
-            .iter_batches()
-            .filter(|&batch| !batch.is_empty())
-            .for_each(|forward_batch| {
-                slot_metrics_tracker.increment_forwardable_batches_count(1);
-
-                let batched_forwardable_packets_count = forward_batch.len();
-                let (_forward_result, successful_forwarded_packets_count, _leader_pubkey) = self
-                    .forward_buffered_packets(
-                        &forward_option,
-                        forward_batch.get_forwardable_packets(),
-                        banking_stage_stats,
-                    );
-
-                let failed_forwarded_packets_count = batched_forwardable_packets_count
-                    .saturating_sub(successful_forwarded_packets_count);
-
-                if failed_forwarded_packets_count > 0 {
-                    slot_metrics_tracker.increment_failed_forwarded_packets_count(
-                        failed_forwarded_packets_count as u64,
-                    );
-                    slot_metrics_tracker.increment_packet_batch_forward_failure_count(1);
-                }
-
-                if successful_forwarded_packets_count > 0 {
-                    slot_metrics_tracker.increment_successful_forwarded_packets_count(
-                        successful_forwarded_packets_count as u64,
-                    );
-                }
-            });
-        self.clear_batches();
-
-        if !hold {
-            slot_metrics_tracker.increment_cleared_from_buffer_after_forward_count(
-                filter_forwarding_result.total_forwardable_packets as u64,
-            );
-            unprocessed_transaction_storage.clear_forwarded_packets();
-        }
-    }
-
-    /// Forwards all valid, unprocessed packets in the iterator, up to a rate limit.
-    /// Returns whether forwarding succeeded, the number of attempted forwarded packets
-    /// if any, the time spent forwarding in us, and the leader pubkey if any.
-    pub fn forward_packets<'a>(
-        &self,
-        forward_option: &ForwardOption,
-        forwardable_packets: impl Iterator<Item = &'a Packet>,
-    ) -> (
-        std::result::Result<(), TransportError>,
-        usize,
-        u64,
-        Option<Pubkey>,
-    ) {
-        let Some((leader_pubkey, addr)) = self.get_leader_and_addr(forward_option) else {
-            return (Ok(()), 0, 0, None);
-        };
-
-        self.update_data_budget();
-        let packet_vec: Vec<_> = forwardable_packets
-            .filter(|p| !p.meta().forwarded())
-            .filter(|p| p.meta().is_from_staked_node())
-            .filter(|p| self.data_budget.take(p.meta().size))
-            .filter_map(|p| p.data(..).map(|data| data.to_vec()))
-            .collect();
-
-        let packet_vec_len = packet_vec.len();
-        // TODO: see https://github.com/solana-labs/solana/issues/23819
-        // fix this so returns the correct number of succeeded packets
-        // when there's an error sending the batch. This was left as-is for now
-        // in favor of shipping Quic support, which was considered higher-priority
-        let (res, forward_us) = if !packet_vec.is_empty() {
-            measure_us!(self.forward(forward_option, packet_vec, &addr))
-        } else {
-            (Ok(()), 0)
-        };
-
-        (res, packet_vec_len, forward_us, Some(leader_pubkey))
-    }
-
-    /// Forwards all valid, unprocessed packets in the buffer, up to a rate limit. Returns
-    /// the number of successfully forwarded packets in second part of tuple
-    fn forward_buffered_packets<'a>(
-        &self,
-        forward_option: &ForwardOption,
-        forwardable_packets: impl Iterator<Item = &'a Packet>,
-        banking_stage_stats: &BankingStageStats,
-    ) -> (
-        std::result::Result<(), TransportError>,
-        usize,
-        Option<Pubkey>,
-    ) {
-        let (res, num_packets, _forward_us, leader_pubkey) =
-            self.forward_packets(forward_option, forwardable_packets);
-        if let Err(ref err) = res {
-            warn!("failed to forward packets: {err}");
-        }
-
-        if num_packets > 0 {
-            if let ForwardOption::ForwardTpuVote = forward_option {
-                banking_stage_stats
-                    .forwarded_vote_count
-                    .fetch_add(num_packets, Ordering::Relaxed);
-            } else {
-                banking_stage_stats
-                    .forwarded_transaction_count
-                    .fetch_add(num_packets, Ordering::Relaxed);
-            }
-        }
-
-        (res, num_packets, leader_pubkey)
-    }
-
-    /// Get the pubkey and socket address for the leader to forward to
-    fn get_leader_and_addr(&self, forward_option: &ForwardOption) -> Option<(Pubkey, SocketAddr)> {
-        match forward_option {
-            ForwardOption::NotForward => None,
-            ForwardOption::ForwardTransaction => {
-                next_leader(&self.cluster_info, &self.poh_recorder, |node| {
-                    node.tpu_forwards(self.connection_cache.protocol())
-                })
-            }
-            ForwardOption::ForwardTpuVote => {
-                next_leader_tpu_vote(&self.cluster_info, &self.poh_recorder)
-            }
-        }
-    }
-
-    /// Re-fill the data budget if enough time has passed
-    fn update_data_budget(&self) {
-        const INTERVAL_MS: u64 = 100;
-        // 12 MB outbound limit per second
-        const MAX_BYTES_PER_SECOND: usize = 12_000_000;
-        const MAX_BYTES_PER_INTERVAL: usize = MAX_BYTES_PER_SECOND * INTERVAL_MS as usize / 1000;
-        const MAX_BYTES_BUDGET: usize = MAX_BYTES_PER_INTERVAL * 5;
-        self.data_budget.update(INTERVAL_MS, |bytes| {
-            std::cmp::min(
-                bytes.saturating_add(MAX_BYTES_PER_INTERVAL),
-                MAX_BYTES_BUDGET,
-            )
-        });
-    }
-
-    fn forward(
-        &self,
-        forward_option: &ForwardOption,
-        packet_vec: Vec<Vec<u8>>,
-        addr: &SocketAddr,
-    ) -> Result<(), TransportError> {
-        match forward_option {
-            ForwardOption::ForwardTpuVote => {
-                // The vote must be forwarded using only UDP.
-                let pkts = packet_vec.iter().map(|pkt| (pkt, addr));
-                batch_send(&self.socket, pkts).map_err(TransportError::from)
-            }
-            ForwardOption::ForwardTransaction => {
-                let conn = self.connection_cache.get_connection(addr);
-                conn.send_data_batch_async(packet_vec)
-            }
-            ForwardOption::NotForward => panic!("should not forward"),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use {
-        super::*,
-        crate::banking_stage::{
-            tests::{create_slow_genesis_config_with_leader, new_test_cluster_info},
-            unprocessed_packet_batches::{DeserializedPacket, UnprocessedPacketBatches},
-            unprocessed_transaction_storage::ThreadType,
-        },
-        solana_client::rpc_client::SerializableTransaction,
-        solana_gossip::cluster_info::{ClusterInfo, Node},
-        solana_ledger::{blockstore::Blockstore, genesis_utils::GenesisConfigInfo},
-        solana_perf::packet::PacketFlags,
-        solana_poh::{poh_recorder::create_test_recorder, poh_service::PohService},
-        solana_runtime::bank::Bank,
-        solana_sdk::{
-            hash::Hash, poh_config::PohConfig, signature::Keypair, signer::Signer,
-            system_transaction, transaction::VersionedTransaction,
-        },
-        solana_streamer::{
-            nonblocking::testing_utilities::{
-                setup_quic_server_with_sockets, SpawnTestServerResult, TestServerConfig,
-            },
-            quic::rt,
-        },
-        std::{
-            sync::atomic::AtomicBool,
-            time::{Duration, Instant},
-        },
-        tempfile::TempDir,
-        tokio::time::sleep,
-    };
-
-    struct TestSetup {
-        _ledger_dir: TempDir,
-        blockhash: Hash,
-        rent_min_balance: u64,
-
-        bank_forks: Arc<RwLock<BankForks>>,
-        poh_recorder: Arc<RwLock<PohRecorder>>,
-        exit: Arc<AtomicBool>,
-        poh_service: PohService,
-        cluster_info: Arc<ClusterInfo>,
-        local_node: Node,
-    }
-
-    fn setup() -> TestSetup {
-        let validator_keypair = Arc::new(Keypair::new());
-        let genesis_config_info =
-            create_slow_genesis_config_with_leader(10_000, &validator_keypair.pubkey());
-        let GenesisConfigInfo { genesis_config, .. } = &genesis_config_info;
-
-        let (bank, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(genesis_config);
-
-        let ledger_path = TempDir::new().unwrap();
-        let blockstore = Arc::new(
-            Blockstore::open(ledger_path.as_ref())
-                .expect("Expected to be able to open database ledger"),
-        );
-        let poh_config = PohConfig {
-            // limit tick count to avoid clearing working_bank at
-            // PohRecord then PohRecorderError(MaxHeightReached) at BankingStage
-            target_tick_count: Some(bank.max_tick_height() - 1),
-            ..PohConfig::default()
-        };
-
-        let (exit, poh_recorder, poh_service, _entry_receiver) =
-            create_test_recorder(bank, blockstore, Some(poh_config), None);
-
-        let (local_node, cluster_info) = new_test_cluster_info(Some(validator_keypair));
-        let cluster_info = Arc::new(cluster_info);
-
-        TestSetup {
-            _ledger_dir: ledger_path,
-            blockhash: genesis_config.hash(),
-            rent_min_balance: genesis_config.rent.minimum_balance(0),
-
-            bank_forks,
-            poh_recorder,
-            exit,
-            poh_service,
-            cluster_info,
-            local_node,
-        }
-    }
-
-    async fn check_all_received(
-        socket: UdpSocket,
-        expected_num_packets: usize,
-        expected_packet_size: usize,
-        expected_blockhash: &Hash,
-    ) {
-        let SpawnTestServerResult {
-            join_handle,
-            exit,
-            receiver,
-            server_address: _,
-            stats: _,
-        } = setup_quic_server_with_sockets(vec![socket], None, TestServerConfig::default());
-
-        let now = Instant::now();
-        let mut total_packets = 0;
-        while now.elapsed().as_secs() < 5 {
-            if let Ok(packets) = receiver.try_recv() {
-                total_packets += packets.len();
-                for packet in packets.iter() {
-                    assert_eq!(packet.meta().size, expected_packet_size);
-                    let tx: VersionedTransaction = packet.deserialize_slice(..).unwrap();
-                    assert_eq!(
-                        tx.get_recent_blockhash(),
-                        expected_blockhash,
-                        "Unexpected blockhash, tx: {tx:?}, expected blockhash: {expected_blockhash}."
-                    );
-                }
-            } else {
-                sleep(Duration::from_millis(100)).await;
-            }
-            if total_packets >= expected_num_packets {
-                break;
-            }
-        }
-        assert_eq!(total_packets, expected_num_packets);
-
-        exit.store(true, Ordering::Relaxed);
-        join_handle.await.unwrap();
-    }
-
-    #[test]
-    fn test_forwarder_budget() {
-        let TestSetup {
-            blockhash,
-            rent_min_balance,
-            bank_forks,
-            poh_recorder,
-            exit,
-            poh_service,
-            cluster_info,
-            local_node,
-            ..
-        } = setup();
-
-        // Create `PacketBatch` with 1 unprocessed packet
-        let tx = system_transaction::transfer(
-            &Keypair::new(),
-            &solana_pubkey::new_rand(),
-            rent_min_balance,
-            blockhash,
-        );
-        let mut packet = Packet::from_data(None, tx).unwrap();
-        // unstaked transactions will not be forwarded
-        packet.meta_mut().set_from_staked_node(true);
-        let expected_packet_size = packet.meta().size;
-        let deserialized_packet = DeserializedPacket::new(packet).unwrap();
-
-        let test_cases = vec![
-            ("budget-restricted", DataBudget::restricted(), 0),
-            ("budget-available", DataBudget::default(), 1),
-        ];
-        let runtime = rt("solQuicTestRt".to_string());
-        for (_name, data_budget, expected_num_forwarded) in test_cases {
-            let mut forwarder = Forwarder::new(
-                poh_recorder.clone(),
-                bank_forks.clone(),
-                cluster_info.clone(),
-                Arc::new(ConnectionCache::new("connection_cache_test")),
-                Arc::new(data_budget),
-            );
-            let unprocessed_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(
-                    vec![deserialized_packet.clone()].into_iter(),
-                    1,
-                );
-            let stats = BankingStageStats::default();
-            forwarder.handle_forwarding(
-                &mut UnprocessedTransactionStorage::new_transaction_storage(
-                    unprocessed_packet_batches,
-                    ThreadType::Transactions,
-                ),
-                true,
-                &mut LeaderSlotMetricsTracker::new(0),
-                &stats,
-            );
-
-            let recv_socket = &local_node.sockets.tpu_forwards_quic[0];
-            runtime.block_on(check_all_received(
-                (*recv_socket).try_clone().unwrap(),
-                expected_num_forwarded,
-                expected_packet_size,
-                &blockhash,
-            ));
-        }
-
-        exit.store(true, Ordering::Relaxed);
-        poh_service.join().unwrap();
-    }
-
-    #[test]
-    fn test_handle_forwarding() {
-        let TestSetup {
-            blockhash,
-            rent_min_balance,
-            bank_forks,
-            poh_recorder,
-            exit,
-            poh_service,
-            cluster_info,
-            local_node,
-            ..
-        } = setup();
-
-        let keypair = Keypair::new();
-        let pubkey = solana_pubkey::new_rand();
-
-        // forwarded packets will not be forwarded again
-        let forwarded_packet = {
-            let transaction =
-                system_transaction::transfer(&keypair, &pubkey, rent_min_balance, blockhash);
-            let mut packet = Packet::from_data(None, transaction).unwrap();
-            packet.meta_mut().flags |= PacketFlags::FORWARDED;
-            DeserializedPacket::new(packet).unwrap()
-        };
-        // packets from unstaked nodes will not be forwarded
-        let unstaked_packet = {
-            let transaction =
-                system_transaction::transfer(&keypair, &pubkey, rent_min_balance, blockhash);
-            let packet = Packet::from_data(None, transaction).unwrap();
-            DeserializedPacket::new(packet).unwrap()
-        };
-        // packets with incorrect blockhash will be filtered out
-        let incorrect_blockhash_packet = {
-            let transaction =
-                system_transaction::transfer(&keypair, &pubkey, rent_min_balance, Hash::default());
-            let packet = Packet::from_data(None, transaction).unwrap();
-            DeserializedPacket::new(packet).unwrap()
-        };
-
-        // maybe also add packet without stake and packet with incorrect blockhash?
-        let (expected_packet_size, normal_packet) = {
-            let transaction = system_transaction::transfer(&keypair, &pubkey, 1, blockhash);
-            let mut packet = Packet::from_data(None, transaction).unwrap();
-            packet.meta_mut().set_from_staked_node(true);
-            (packet.meta().size, DeserializedPacket::new(packet).unwrap())
-        };
-
-        let mut unprocessed_packet_batches = UnprocessedTransactionStorage::new_transaction_storage(
-            UnprocessedPacketBatches::from_iter(
-                vec![
-                    forwarded_packet,
-                    unstaked_packet,
-                    incorrect_blockhash_packet,
-                    normal_packet,
-                ],
-                4,
-            ),
-            ThreadType::Transactions,
-        );
-        let connection_cache = ConnectionCache::new("connection_cache_test");
-
-        let test_cases = vec![
-            ("fwd-normal", true, 2, 1),
-            ("fwd-no-op", true, 2, 0),
-            ("fwd-no-hold", false, 0, 0),
-        ];
-
-        let mut forwarder = Forwarder::new(
-            poh_recorder,
-            bank_forks,
-            cluster_info,
-            Arc::new(connection_cache),
-            Arc::new(DataBudget::default()),
-        );
-        let runtime = rt("solQuicTestRt".to_string());
-        for (name, hold, expected_num_unprocessed, expected_num_processed) in test_cases {
-            let stats = BankingStageStats::default();
-            forwarder.handle_forwarding(
-                &mut unprocessed_packet_batches,
-                hold,
-                &mut LeaderSlotMetricsTracker::new(0),
-                &stats,
-            );
-
-            let recv_socket = &local_node.sockets.tpu_forwards_quic[0];
-
-            runtime.block_on(check_all_received(
-                (*recv_socket).try_clone().unwrap(),
-                expected_num_processed,
-                expected_packet_size,
-                &blockhash,
-            ));
-
-            let num_unprocessed_packets: usize = unprocessed_packet_batches.len();
-            assert_eq!(num_unprocessed_packets, expected_num_unprocessed, "{name}");
-        }
-
-        exit.store(true, Ordering::Relaxed);
-        poh_service.join().unwrap();
-    }
-}

core/src/banking_stage/latest_unprocessed_votes.rs

@@ -1,8 +1,5 @@
 use {
-    super::{
-        forward_packet_batches_by_accounts::ForwardPacketBatchesByAccounts,
-        immutable_deserialized_packet::{DeserializedPacketError, ImmutableDeserializedPacket},
-    },
+    super::immutable_deserialized_packet::{DeserializedPacketError, ImmutableDeserializedPacket},
     itertools::Itertools,
     rand::{thread_rng, Rng},
     solana_perf::packet::Packet,
@@ -35,7 +32,7 @@ pub enum VoteSource {
     Tpu,
 }
 
-/// Holds deserialized vote messages as well as their source, forward status and slot
+/// Holds deserialized vote messages as well as their source, and slot
 #[derive(Debug, Clone)]
 pub struct LatestValidatorVotePacket {
     vote_source: VoteSource,
@@ -43,7 +40,6 @@ pub struct LatestValidatorVotePacket {
     vote: Option<Arc<ImmutableDeserializedPacket>>,
     slot: Slot,
     hash: Hash,
-    forwarded: bool,
     timestamp: Option<UnixTimestamp>,
 }
 
@@ -108,7 +104,6 @@ impl LatestValidatorVotePacket {
                     hash,
                     vote_pubkey,
                     vote_source,
-                    forwarded: false,
                     timestamp,
                 })
             }
@@ -136,11 +131,6 @@ impl LatestValidatorVotePacket {
         self.timestamp
     }
 
-    pub fn is_forwarded(&self) -> bool {
-        // By definition all gossip votes have been forwarded
-        self.forwarded || matches!(self.vote_source, VoteSource::Gossip)
-    }
-
     pub fn is_vote_taken(&self) -> bool {
         self.vote.is_none()
     }
@@ -399,57 +389,6 @@ impl LatestUnprocessedVotes {
         );
     }
 
-    /// Returns how many packets were forwardable
-    /// Performs a weighted random order based on stake and stops forwarding at the first error
-    /// Votes from validators with 0 stakes are ignored
-    pub fn get_and_insert_forwardable_packets(
-        &self,
-        bank: Arc<Bank>,
-        forward_packet_batches_by_accounts: &mut ForwardPacketBatchesByAccounts,
-    ) -> usize {
-        let pubkeys_by_stake = self.weighted_random_order_by_stake();
-        let mut forwarded_count: usize = 0;
-        for pubkey in pubkeys_by_stake {
-            let Some(vote) = self.get_entry(pubkey) else {
-                continue;
-            };
-
-            let mut vote = vote.write().unwrap();
-            if vote.is_vote_taken() || vote.is_forwarded() {
-                continue;
-            }
-
-            let deserialized_vote_packet = vote.vote.as_ref().unwrap().clone();
-            let Some((sanitized_vote_transaction, _deactivation_slot)) = deserialized_vote_packet
-                .build_sanitized_transaction(
-                    bank.vote_only_bank(),
-                    bank.as_ref(),
-                    bank.get_reserved_account_keys(),
-                )
-            else {
-                continue;
-            };
-
-            let forwarding_successful = forward_packet_batches_by_accounts.try_add_packet(
-                &sanitized_vote_transaction,
-                deserialized_vote_packet,
-                &bank.feature_set,
-            );
-
-            if !forwarding_successful {
-                // To match behavior of regular transactions we stop forwarding votes as soon as one
-                // fails. We are assuming that failure (try_add_packet) means no more space
-                // available.
-                break;
-            }
-
-            vote.forwarded = true;
-            forwarded_count += 1;
-        }
-
-        forwarded_count
-    }
-
     /// Drains all votes yet to be processed sorted by a weighted random ordering by stake
     /// Do not touch votes that are for a different fork from `bank` as we know they will fail,
     /// however the next bank could be built on a different fork and consume these votes.
@@ -495,17 +434,14 @@ impl LatestUnprocessedVotes {
             .unwrap_or(false)
     }
 
-    /// Sometimes we forward and hold the packets, sometimes we forward and clear.
-    /// This also clears all gossip votes since by definition they have been forwarded
-    pub fn clear_forwarded_packets(&self) {
+    pub fn clear(&self) {
         self.latest_vote_per_vote_pubkey
             .read()
             .unwrap()
             .values()
-            .filter(|lock| lock.read().unwrap().is_forwarded())
             .for_each(|lock| {
                 let mut vote = lock.write().unwrap();
-                if vote.is_forwarded() && vote.take_vote().is_some() {
+                if vote.take_vote().is_some() {
                     self.num_unprocessed_votes.fetch_sub(1, Ordering::Relaxed);
                 }
             });
@@ -525,7 +461,6 @@ mod tests {
         solana_perf::packet::{Packet, PacketBatch, PacketFlags},
         solana_runtime::{
             bank::Bank,
-            epoch_stakes::EpochStakes,
             genesis_utils::{self, ValidatorVoteKeypairs},
         },
         solana_sdk::{
@@ -966,126 +901,7 @@ mod tests {
     }
 
     #[test]
-    fn test_forwardable_packets() {
-        let latest_unprocessed_votes = LatestUnprocessedVotes::new_for_tests(&[]);
-        let bank_0 = Bank::new_for_tests(&GenesisConfig::default());
-        let mut bank = Bank::new_from_parent(
-            Arc::new(bank_0),
-            &Pubkey::new_unique(),
-            MINIMUM_SLOTS_PER_EPOCH,
-        );
-        assert_eq!(bank.epoch(), 1);
-        bank.set_epoch_stakes_for_test(
-            bank.epoch().saturating_add(2),
-            EpochStakes::new_for_tests(HashMap::new(), bank.epoch().saturating_add(2)),
-        );
-        let bank = Arc::new(bank);
-        let mut forward_packet_batches_by_accounts =
-            ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-
-        let keypair_a = ValidatorVoteKeypairs::new_rand();
-        let keypair_b = ValidatorVoteKeypairs::new_rand();
-
-        let vote_a = from_slots(vec![(1, 1)], VoteSource::Gossip, &keypair_a, None);
-        let vote_b = from_slots(vec![(2, 1)], VoteSource::Tpu, &keypair_b, None);
-        latest_unprocessed_votes
-            .update_latest_vote(vote_a.clone(), false /* should replenish */);
-        latest_unprocessed_votes
-            .update_latest_vote(vote_b.clone(), false /* should replenish */);
-
-        // Recache on epoch boundary and don't forward 0 stake accounts
-        latest_unprocessed_votes.cache_epoch_boundary_info(&bank);
-        let forwarded = latest_unprocessed_votes
-            .get_and_insert_forwardable_packets(bank, &mut forward_packet_batches_by_accounts);
-        assert_eq!(0, forwarded);
-        assert_eq!(
-            0,
-            forward_packet_batches_by_accounts
-                .iter_batches()
-                .filter(|&batch| !batch.is_empty())
-                .count()
-        );
-
-        let config =
-            genesis_utils::create_genesis_config_with_vote_accounts(100, &[keypair_a], vec![200])
-                .genesis_config;
-        let bank_0 = Bank::new_for_tests(&config);
-        let bank = Bank::new_from_parent(
-            Arc::new(bank_0),
-            &Pubkey::new_unique(),
-            2 * MINIMUM_SLOTS_PER_EPOCH,
-        );
-        let mut forward_packet_batches_by_accounts =
-            ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-
-        // Don't forward votes from gossip
-        latest_unprocessed_votes.cache_epoch_boundary_info(&bank);
-        latest_unprocessed_votes
-            .update_latest_vote(vote_a.clone(), false /* should replenish */);
-        latest_unprocessed_votes
-            .update_latest_vote(vote_b.clone(), false /* should replenish */);
-        let forwarded = latest_unprocessed_votes.get_and_insert_forwardable_packets(
-            Arc::new(bank),
-            &mut forward_packet_batches_by_accounts,
-        );
-
-        assert_eq!(0, forwarded);
-        assert_eq!(
-            0,
-            forward_packet_batches_by_accounts
-                .iter_batches()
-                .filter(|&batch| !batch.is_empty())
-                .count()
-        );
-
-        let config =
-            genesis_utils::create_genesis_config_with_vote_accounts(100, &[keypair_b], vec![200])
-                .genesis_config;
-        let bank_0 = Bank::new_for_tests(&config);
-        let bank = Arc::new(Bank::new_from_parent(
-            Arc::new(bank_0),
-            &Pubkey::new_unique(),
-            3 * MINIMUM_SLOTS_PER_EPOCH,
-        ));
-        let mut forward_packet_batches_by_accounts =
-            ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-
-        // Forward from TPU
-        latest_unprocessed_votes.cache_epoch_boundary_info(&bank);
-        latest_unprocessed_votes.update_latest_vote(vote_a, false /* should replenish */);
-        latest_unprocessed_votes.update_latest_vote(vote_b, false /* should replenish */);
-        let forwarded = latest_unprocessed_votes.get_and_insert_forwardable_packets(
-            bank.clone(),
-            &mut forward_packet_batches_by_accounts,
-        );
-
-        assert_eq!(1, forwarded);
-        assert_eq!(
-            1,
-            forward_packet_batches_by_accounts
-                .iter_batches()
-                .filter(|&batch| !batch.is_empty())
-                .count()
-        );
-
-        // Don't forward again
-        let mut forward_packet_batches_by_accounts =
-            ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-        let forwarded = latest_unprocessed_votes
-            .get_and_insert_forwardable_packets(bank, &mut forward_packet_batches_by_accounts);
-
-        assert_eq!(0, forwarded);
-        assert_eq!(
-            0,
-            forward_packet_batches_by_accounts
-                .iter_batches()
-                .filter(|&batch| !batch.is_empty())
-                .count()
-        );
-    }
-
-    #[test]
-    fn test_clear_forwarded_packets() {
+    fn test_clear() {
         let keypair_a = ValidatorVoteKeypairs::new_rand();
         let keypair_b = ValidatorVoteKeypairs::new_rand();
         let keypair_c = ValidatorVoteKeypairs::new_rand();
@@ -1098,8 +914,7 @@ mod tests {
         ]);
 
         let vote_a = from_slots(vec![(1, 1)], VoteSource::Gossip, &keypair_a, None);
-        let mut vote_b = from_slots(vec![(2, 1)], VoteSource::Tpu, &keypair_b, None);
-        vote_b.forwarded = true;
+        let vote_b = from_slots(vec![(2, 1)], VoteSource::Tpu, &keypair_b, None);
         let vote_c = from_slots(vec![(3, 1)], VoteSource::Tpu, &keypair_c, None);
         let vote_d = from_slots(vec![(4, 1)], VoteSource::Gossip, &keypair_d, None);
 
@@ -1109,8 +924,8 @@ mod tests {
         latest_unprocessed_votes.update_latest_vote(vote_d, false /* should replenish */);
         assert_eq!(4, latest_unprocessed_votes.len());
 
-        latest_unprocessed_votes.clear_forwarded_packets();
-        assert_eq!(1, latest_unprocessed_votes.len());
+        latest_unprocessed_votes.clear();
+        assert_eq!(0, latest_unprocessed_votes.len());
 
         assert_eq!(
             Some(1),

core/src/banking_stage/leader_slot_metrics.rs

@@ -220,24 +220,6 @@ struct LeaderSlotPacketCountMetrics {
     // according to the cost model. These transactions are added back to the buffered queue and are
     // already counted in `self.retrayble_errored_transaction_count`.
     cost_model_throttled_transactions_count: u64,
-
-    // total number of forwardsable packets that failed forwarding
-    failed_forwarded_packets_count: u64,
-
-    // total number of forwardsable packets that were successfully forwarded
-    successful_forwarded_packets_count: u64,
-
-    // total number of attempted forwards that failed. Note this is not a count of the number of packets
-    // that failed, just the total number of batches of packets that failed forwarding
-    packet_batch_forward_failure_count: u64,
-
-    // total number of valid unprocessed packets in the buffer that were removed after being forwarded
-    cleared_from_buffer_after_forward_count: u64,
-
-    // total number of forwardable batches that were attempted for forwarding. A forwardable batch
-    // is defined in `ForwardPacketBatchesByAccounts` in `forward_packet_batches_by_accounts.rs`
-    forwardable_batches_count: u64,
-
     // min prioritization fees for scheduled transactions
     min_prioritization_fees: u64,
     // max prioritization fees for scheduled transactions
@@ -354,31 +336,6 @@ impl LeaderSlotPacketCountMetrics {
                 self.cost_model_throttled_transactions_count,
                 i64
             ),
-            (
-                "failed_forwarded_packets_count",
-                self.failed_forwarded_packets_count,
-                i64
-            ),
-            (
-                "successful_forwarded_packets_count",
-                self.successful_forwarded_packets_count,
-                i64
-            ),
-            (
-                "packet_batch_forward_failure_count",
-                self.packet_batch_forward_failure_count,
-                i64
-            ),
-            (
-                "cleared_from_buffer_after_forward_count",
-                self.cleared_from_buffer_after_forward_count,
-                i64
-            ),
-            (
-                "forwardable_batches_count",
-                self.forwardable_batches_count,
-                i64
-            ),
             (
                 "end_of_slot_unprocessed_buffer_len",
                 self.end_of_slot_unprocessed_buffer_len,
@@ -883,61 +840,6 @@ impl LeaderSlotMetricsTracker {
         }
     }
 
-    pub(crate) fn increment_failed_forwarded_packets_count(&mut self, count: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            saturating_add_assign!(
-                leader_slot_metrics
-                    .packet_count_metrics
-                    .failed_forwarded_packets_count,
-                count
-            );
-        }
-    }
-
-    pub(crate) fn increment_successful_forwarded_packets_count(&mut self, count: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            saturating_add_assign!(
-                leader_slot_metrics
-                    .packet_count_metrics
-                    .successful_forwarded_packets_count,
-                count
-            );
-        }
-    }
-
-    pub(crate) fn increment_packet_batch_forward_failure_count(&mut self, count: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            saturating_add_assign!(
-                leader_slot_metrics
-                    .packet_count_metrics
-                    .packet_batch_forward_failure_count,
-                count
-            );
-        }
-    }
-
-    pub(crate) fn increment_cleared_from_buffer_after_forward_count(&mut self, count: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            saturating_add_assign!(
-                leader_slot_metrics
-                    .packet_count_metrics
-                    .cleared_from_buffer_after_forward_count,
-                count
-            );
-        }
-    }
-
-    pub(crate) fn increment_forwardable_batches_count(&mut self, count: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            saturating_add_assign!(
-                leader_slot_metrics
-                    .packet_count_metrics
-                    .forwardable_batches_count,
-                count
-            );
-        }
-    }
-
     pub(crate) fn increment_retryable_packets_count(&mut self, count: u64) {
         if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
             saturating_add_assign!(
@@ -1008,24 +910,6 @@ impl LeaderSlotMetricsTracker {
         }
     }
 
-    pub(crate) fn increment_forward_us(&mut self, us: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            leader_slot_metrics
-                .timing_metrics
-                .process_buffered_packets_timings
-                .forward_us += us;
-        }
-    }
-
-    pub(crate) fn increment_forward_and_hold_us(&mut self, us: u64) {
-        if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
-            leader_slot_metrics
-                .timing_metrics
-                .process_buffered_packets_timings
-                .forward_and_hold_us += us;
-        }
-    }
-
     pub(crate) fn increment_process_packets_transactions_us(&mut self, us: u64) {
         if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
             leader_slot_metrics

core/src/banking_stage/leader_slot_timing_metrics.rs

@@ -182,8 +182,6 @@ impl OuterLoopTimings {
 pub(crate) struct ProcessBufferedPacketsTimings {
     pub make_decision_us: u64,
     pub consume_buffered_packets_us: u64,
-    pub forward_us: u64,
-    pub forward_and_hold_us: u64,
 }
 impl ProcessBufferedPacketsTimings {
     fn report(&self, id: &str, slot: Slot) {
@@ -197,8 +195,6 @@ impl ProcessBufferedPacketsTimings {
                 self.consume_buffered_packets_us as i64,
                 i64
             ),
-            ("forward_us", self.forward_us as i64, i64),
-            ("forward_and_hold_us", self.forward_and_hold_us as i64, i64),
         );
     }
 }

core/src/banking_stage/transaction_scheduler/greedy_scheduler.rs

@@ -400,7 +400,6 @@ mod test {
     use {
         super::*,
         crate::banking_stage::{
-            immutable_deserialized_packet::ImmutableDeserializedPacket,
             scheduler_messages::{MaxAge, TransactionId},
             transaction_scheduler::{
                 transaction_state::SanitizedTransactionTTL,
@@ -409,7 +408,6 @@ mod test {
         },
         crossbeam_channel::unbounded,
         itertools::Itertools,
-        solana_perf::packet::Packet,
         solana_pubkey::Pubkey,
         solana_runtime_transaction::runtime_transaction::RuntimeTransaction,
         solana_sdk::{
@@ -421,7 +419,7 @@ mod test {
             system_instruction,
             transaction::{SanitizedTransaction, Transaction},
         },
-        std::{borrow::Borrow, sync::Arc},
+        std::borrow::Borrow,
     };
 
     #[allow(clippy::type_complexity)]
@@ -483,12 +481,6 @@ mod test {
                 lamports,
                 compute_unit_price,
             );
-            let packet = Arc::new(
-                ImmutableDeserializedPacket::new(
-                    Packet::from_data(None, transaction.to_versioned_transaction()).unwrap(),
-                )
-                .unwrap(),
-            );
             let transaction_ttl = SanitizedTransactionTTL {
                 transaction,
                 max_age: MaxAge::MAX,
@@ -496,7 +488,6 @@ mod test {
             const TEST_TRANSACTION_COST: u64 = 5000;
             container.insert_new_transaction(
                 transaction_ttl,
-                packet,
                 compute_unit_price,
                 TEST_TRANSACTION_COST,
             );

core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs

@@ -629,10 +629,7 @@ fn try_schedule_transaction<Tx: TransactionWithMeta>(
 mod tests {
     use {
         super::*,
-        crate::banking_stage::{
-            immutable_deserialized_packet::ImmutableDeserializedPacket,
-            transaction_scheduler::transaction_state_container::TransactionStateContainer,
-        },
+        crate::banking_stage::transaction_scheduler::transaction_state_container::TransactionStateContainer,
         crossbeam_channel::{unbounded, Receiver},
         itertools::Itertools,
         solana_runtime_transaction::runtime_transaction::RuntimeTransaction,
@@ -640,14 +637,13 @@ mod tests {
             compute_budget::ComputeBudgetInstruction,
             hash::Hash,
             message::Message,
-            packet::Packet,
             pubkey::Pubkey,
             signature::Keypair,
             signer::Signer,
             system_instruction,
             transaction::{SanitizedTransaction, Transaction},
         },
-        std::{borrow::Borrow, sync::Arc},
+        std::borrow::Borrow,
     };
 
     #[allow(clippy::type_complexity)]
@@ -725,12 +721,6 @@ mod tests {
                 lamports,
                 compute_unit_price,
             );
-            let packet = Arc::new(
-                ImmutableDeserializedPacket::new(
-                    Packet::from_data(None, transaction.to_versioned_transaction()).unwrap(),
-                )
-                .unwrap(),
-            );
             let transaction_ttl = SanitizedTransactionTTL {
                 transaction,
                 max_age: MaxAge::MAX,
@@ -738,7 +728,6 @@ mod tests {
             const TEST_TRANSACTION_COST: u64 = 5000;
             container.insert_new_transaction(
                 transaction_ttl,
-                packet,
                 compute_unit_price,
                 TEST_TRANSACTION_COST,
             );

core/src/banking_stage/transaction_scheduler/receive_and_buffer.rs

@@ -66,8 +66,6 @@ pub(crate) struct SanitizedTransactionReceiveAndBuffer {
     /// Packet/Transaction ingress.
     packet_receiver: PacketDeserializer,
     bank_forks: Arc<RwLock<BankForks>>,
-
-    forwarding_enabled: bool,
 }
 
 impl ReceiveAndBuffer for SanitizedTransactionReceiveAndBuffer {
@@ -93,7 +91,7 @@ impl ReceiveAndBuffer for SanitizedTransactionReceiveAndBuffer {
                 },
                 true,
             ),
-            BufferedPacketsDecision::Forward => (MAX_PACKET_RECEIVE_TIME, self.forwarding_enabled),
+            BufferedPacketsDecision::Forward => (MAX_PACKET_RECEIVE_TIME, false),
             BufferedPacketsDecision::ForwardAndHold => (MAX_PACKET_RECEIVE_TIME, true),
         };
 
@@ -145,15 +143,10 @@ impl ReceiveAndBuffer for SanitizedTransactionReceiveAndBuffer {
 }
 
 impl SanitizedTransactionReceiveAndBuffer {
-    pub fn new(
-        packet_receiver: PacketDeserializer,
-        bank_forks: Arc<RwLock<BankForks>>,
-        forwarding_enabled: bool,
-    ) -> Self {
+    pub fn new(packet_receiver: PacketDeserializer, bank_forks: Arc<RwLock<BankForks>>) -> Self {
         Self {
             packet_receiver,
             bank_forks,
-            forwarding_enabled,
         }
     }
 
@@ -181,7 +174,6 @@ impl SanitizedTransactionReceiveAndBuffer {
         const CHUNK_SIZE: usize = 128;
         let lock_results: [_; CHUNK_SIZE] = core::array::from_fn(|_| Ok(()));
 
-        let mut arc_packets = ArrayVec::<_, CHUNK_SIZE>::new();
         let mut transactions = ArrayVec::<_, CHUNK_SIZE>::new();
         let mut max_ages = ArrayVec::<_, CHUNK_SIZE>::new();
         let mut fee_budget_limits_vec = ArrayVec::<_, CHUNK_SIZE>::new();
@@ -192,32 +184,27 @@ impl SanitizedTransactionReceiveAndBuffer {
             chunk
                 .iter()
                 .filter_map(|packet| {
-                    packet
-                        .build_sanitized_transaction(
-                            vote_only,
-                            root_bank.as_ref(),
-                            root_bank.get_reserved_account_keys(),
-                        )
-                        .map(|(tx, deactivation_slot)| (packet.clone(), tx, deactivation_slot))
+                    packet.build_sanitized_transaction(
+                        vote_only,
+                        root_bank.as_ref(),
+                        root_bank.get_reserved_account_keys(),
+                    )
                 })
                 .inspect(|_| saturating_add_assign!(post_sanitization_count, 1))
-                .filter(|(_packet, tx, _deactivation_slot)| {
+                .filter(|(tx, _deactivation_slot)| {
                     validate_account_locks(
                         tx.message().account_keys(),
                         transaction_account_lock_limit,
                     )
                     .is_ok()
                 })
-                .filter_map(|(packet, tx, deactivation_slot)| {
+                .filter_map(|(tx, deactivation_slot)| {
                     tx.compute_budget_instruction_details()
                         .sanitize_and_convert_to_compute_budget_limits(&working_bank.feature_set)
-                        .map(|compute_budget| {
-                            (packet, tx, deactivation_slot, compute_budget.into())
-                        })
+                        .map(|compute_budget| (tx, deactivation_slot, compute_budget.into()))
                         .ok()
                 })
-                .for_each(|(packet, tx, deactivation_slot, fee_budget_limits)| {
-                    arc_packets.push(packet);
+                .for_each(|(tx, deactivation_slot, fee_budget_limits)| {
                     transactions.push(tx);
                     max_ages.push(calculate_max_age(
                         sanitized_epoch,
@@ -238,18 +225,15 @@ impl SanitizedTransactionReceiveAndBuffer {
             let mut post_transaction_check_count: usize = 0;
             let mut num_dropped_on_capacity: usize = 0;
             let mut num_buffered: usize = 0;
-            for ((((packet, transaction), max_age), fee_budget_limits), _check_result) in
-                arc_packets
-                    .drain(..)
-                    .zip(transactions.drain(..))
-                    .zip(max_ages.drain(..))
-                    .zip(fee_budget_limits_vec.drain(..))
-                    .zip(check_results)
-                    .filter(|(_, check_result)| check_result.is_ok())
-                    .filter(|((((_, tx), _), _), _)| {
-                        Consumer::check_fee_payer_unlocked(&working_bank, tx, &mut error_counts)
-                            .is_ok()
-                    })
+            for (((transaction, max_age), fee_budget_limits), _check_result) in transactions
+                .drain(..)
+                .zip(max_ages.drain(..))
+                .zip(fee_budget_limits_vec.drain(..))
+                .zip(check_results)
+                .filter(|(_, check_result)| check_result.is_ok())
+                .filter(|(((tx, _), _), _)| {
+                    Consumer::check_fee_payer_unlocked(&working_bank, tx, &mut error_counts).is_ok()
+                })
             {
                 saturating_add_assign!(post_transaction_check_count, 1);
 
@@ -260,7 +244,7 @@ impl SanitizedTransactionReceiveAndBuffer {
                     max_age,
                 };
 
-                if container.insert_new_transaction(transaction_ttl, packet, priority, cost) {
+                if container.insert_new_transaction(transaction_ttl, priority, cost) {
                     saturating_add_assign!(num_dropped_on_capacity, 1);
                 }
                 saturating_add_assign!(num_buffered, 1);
@@ -395,7 +379,7 @@ impl TransactionViewReceiveAndBuffer {
         working_bank: &Bank,
         packet_batch_message: BankingPacketBatch,
     ) -> usize {
-        // Do not support forwarding - only add support for this if we really need it.
+        // If not holding packets, just drop them immediately without parsing.
         if matches!(decision, BufferedPacketsDecision::Forward) {
             return 0;
         }
@@ -616,7 +600,6 @@ impl TransactionViewReceiveAndBuffer {
                 transaction: view,
                 max_age,
             },
-            None,
             priority,
             cost,
         ))

core/src/banking_stage/transaction_scheduler/scheduler_controller.rs

@@ -14,28 +14,19 @@ use {
         consume_worker::ConsumeWorkerMetrics,
         consumer::Consumer,
         decision_maker::{BufferedPacketsDecision, DecisionMaker},
-        forwarder::Forwarder,
         transaction_scheduler::transaction_state_container::StateContainer,
-        ForwardOption, LikeClusterInfo, TOTAL_BUFFERED_PACKETS,
+        TOTAL_BUFFERED_PACKETS,
     },
     solana_measure::measure_us,
     solana_runtime::{bank::Bank, bank_forks::BankForks},
-    solana_sdk::{
-        self,
-        clock::{FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, MAX_PROCESSING_AGE},
-        saturating_add_assign,
-    },
+    solana_sdk::{self, clock::MAX_PROCESSING_AGE, saturating_add_assign},
     solana_svm::transaction_error_metrics::TransactionErrorMetrics,
-    std::{
-        sync::{Arc, RwLock},
-        time::{Duration, Instant},
-    },
+    std::sync::{Arc, RwLock},
 };
 
 /// Controls packet and transaction flow into scheduler, and scheduling execution.
-pub(crate) struct SchedulerController<C, R, S>
+pub(crate) struct SchedulerController<R, S>
 where
-    C: LikeClusterInfo,
     R: ReceiveAndBuffer,
     S: Scheduler<R::Transaction>,
 {
@@ -58,13 +49,10 @@ where
     timing_metrics: SchedulerTimingMetrics,
     /// Metric report handles for the worker threads.
     worker_metrics: Vec<Arc<ConsumeWorkerMetrics>>,
-    /// State for forwarding packets to the leader, if enabled.
-    forwarder: Option<Forwarder<C>>,
 }
 
-impl<C, R, S> SchedulerController<C, R, S>
+impl<R, S> SchedulerController<R, S>
 where
-    C: LikeClusterInfo,
     R: ReceiveAndBuffer,
     S: Scheduler<R::Transaction>,
 {
@@ -74,7 +62,6 @@ where
         bank_forks: Arc<RwLock<BankForks>>,
         scheduler: S,
         worker_metrics: Vec<Arc<ConsumeWorkerMetrics>>,
-        forwarder: Option<Forwarder<C>>,
     ) -> Self {
         Self {
             decision_maker,
@@ -86,7 +73,6 @@ where
             count_metrics: SchedulerCountMetrics::default(),
             timing_metrics: SchedulerTimingMetrics::default(),
             worker_metrics,
-            forwarder,
         }
     }
 
@@ -144,7 +130,6 @@ where
         &mut self,
         decision: &BufferedPacketsDecision,
     ) -> Result<(), SchedulerError> {
-        let forwarding_enabled = self.forwarder.is_some();
         match decision {
             BufferedPacketsDecision::Consume(bank_start) => {
                 let (scheduling_summary, schedule_time_us) = measure_us!(self.scheduler.schedule(
@@ -184,30 +169,16 @@ where
                 });
             }
             BufferedPacketsDecision::Forward => {
-                if forwarding_enabled {
-                    let (_, forward_time_us) = measure_us!(self.forward_packets(false));
-                    self.timing_metrics.update(|timing_metrics| {
-                        saturating_add_assign!(timing_metrics.forward_time_us, forward_time_us);
-                    });
-                } else {
-                    let (_, clear_time_us) = measure_us!(self.clear_container());
-                    self.timing_metrics.update(|timing_metrics| {
-                        saturating_add_assign!(timing_metrics.clear_time_us, clear_time_us);
-                    });
-                }
+                let (_, clear_time_us) = measure_us!(self.clear_container());
+                self.timing_metrics.update(|timing_metrics| {
+                    saturating_add_assign!(timing_metrics.clear_time_us, clear_time_us);
+                });
             }
             BufferedPacketsDecision::ForwardAndHold => {
-                if forwarding_enabled {
-                    let (_, forward_time_us) = measure_us!(self.forward_packets(true));
-                    self.timing_metrics.update(|timing_metrics| {
-                        saturating_add_assign!(timing_metrics.forward_time_us, forward_time_us);
-                    });
-                } else {
-                    let (_, clean_time_us) = measure_us!(self.clean_queue());
-                    self.timing_metrics.update(|timing_metrics| {
-                        saturating_add_assign!(timing_metrics.clean_time_us, clean_time_us);
-                    });
-                }
+                let (_, clean_time_us) = measure_us!(self.clean_queue());
+                self.timing_metrics.update(|timing_metrics| {
+                    saturating_add_assign!(timing_metrics.clean_time_us, clean_time_us);
+                });
             }
             BufferedPacketsDecision::Hold => {}
         }
@@ -241,105 +212,6 @@ where
         }
     }
 
-    /// Forward packets to the next leader.
-    fn forward_packets(&mut self, hold: bool) {
-        const MAX_FORWARDING_DURATION: Duration = Duration::from_millis(100);
-        let start = Instant::now();
-        let bank = self.bank_forks.read().unwrap().working_bank();
-        let feature_set = &bank.feature_set;
-        let forwarder = self.forwarder.as_mut().expect("forwarder must exist");
-
-        // Pop from the container in chunks, filter using bank checks, then attempt to forward.
-        // This doubles as a way to clean the queue as well as forwarding transactions.
-        const CHUNK_SIZE: usize = 64;
-        let mut num_forwarded: usize = 0;
-        let mut ids_to_add_back = Vec::new();
-        let mut max_time_reached = false;
-        while !self.container.is_empty() {
-            let mut filter_array = [true; CHUNK_SIZE];
-            let mut ids = Vec::with_capacity(CHUNK_SIZE);
-            let mut txs = Vec::with_capacity(CHUNK_SIZE);
-
-            for _ in 0..CHUNK_SIZE {
-                if let Some(id) = self.container.pop() {
-                    ids.push(id);
-                } else {
-                    break;
-                }
-            }
-            let chunk_size = ids.len();
-            ids.iter().for_each(|id| {
-                let transaction = self.container.get_transaction_ttl(id.id).unwrap();
-                txs.push(&transaction.transaction);
-            });
-
-            // use same filter we use for processing transactions:
-            // age, already processed, fee-check.
-            Self::pre_graph_filter(
-                &txs,
-                &mut filter_array,
-                &bank,
-                MAX_PROCESSING_AGE
-                    .saturating_sub(FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET as usize),
-            );
-
-            for (id, filter_result) in ids.iter().zip(&filter_array[..chunk_size]) {
-                if !*filter_result {
-                    self.container.remove_by_id(id.id);
-                    continue;
-                }
-
-                ids_to_add_back.push(*id); // add back to the queue at end
-                let state = self.container.get_mut_transaction_state(id.id).unwrap();
-                let sanitized_transaction = &state.transaction_ttl().transaction;
-                let immutable_packet = state.packet().expect("forwarding requires packet");
-
-                // If not already forwarded and can be forwarded, add to forwardable packets.
-                if state.should_forward()
-                    && forwarder.try_add_packet(
-                        sanitized_transaction,
-                        immutable_packet.clone(),
-                        feature_set,
-                    )
-                {
-                    saturating_add_assign!(num_forwarded, 1);
-                    state.mark_forwarded();
-                }
-            }
-
-            if start.elapsed() >= MAX_FORWARDING_DURATION {
-                max_time_reached = true;
-                break;
-            }
-        }
-
-        // Forward each batch of transactions
-        forwarder.forward_batched_packets(&ForwardOption::ForwardTransaction);
-        forwarder.clear_batches();
-
-        // If we hit the time limit. Drop everything that was not checked/processed.
-        // If we cannot run these simple checks in time, then we cannot run them during
-        // leader slot.
-        if max_time_reached {
-            while let Some(id) = self.container.pop() {
-                self.container.remove_by_id(id.id);
-            }
-        }
-
-        if hold {
-            self.container
-                .push_ids_into_queue(ids_to_add_back.into_iter());
-        } else {
-            for priority_id in ids_to_add_back {
-                self.container.remove_by_id(priority_id.id);
-            }
-        }
-
-        self.count_metrics.update(|count_metrics| {
-            saturating_add_assign!(count_metrics.num_forwarded, num_forwarded);
-        });
-    }
-
     /// Clears the transaction state container.
     /// This only clears pending transactions, and does **not** clear in-flight transactions.
     fn clear_container(&mut self) {
@@ -469,7 +341,6 @@ mod tests {
         agave_banking_stage_ingress_types::{BankingPacketBatch, BankingPacketReceiver},
         crossbeam_channel::{unbounded, Receiver, Sender},
         itertools::Itertools,
-        solana_gossip::cluster_info::ClusterInfo,
         solana_ledger::{
             blockstore::Blockstore, genesis_utils::GenesisConfigInfo,
             get_tmp_ledger_path_auto_delete, leader_schedule_cache::LeaderScheduleCache,
@@ -511,11 +382,7 @@ mod tests {
         receiver: BankingPacketReceiver,
         bank_forks: Arc<RwLock<BankForks>>,
     ) -> SanitizedTransactionReceiveAndBuffer {
-        SanitizedTransactionReceiveAndBuffer::new(
-            PacketDeserializer::new(receiver),
-            bank_forks,
-            false,
-        )
+        SanitizedTransactionReceiveAndBuffer::new(PacketDeserializer::new(receiver), bank_forks)
     }
 
     fn test_create_transaction_view_receive_and_buffer(
@@ -534,7 +401,7 @@ mod tests {
         create_receive_and_buffer: impl FnOnce(BankingPacketReceiver, Arc<RwLock<BankForks>>) -> R,
     ) -> (
         TestFrame<R::Transaction>,
-        SchedulerController<Arc<ClusterInfo>, R, PrioGraphScheduler<R::Transaction>>,
+        SchedulerController<R, PrioGraphScheduler<R::Transaction>>,
     ) {
         let GenesisConfigInfo {
             mut genesis_config,
@@ -591,7 +458,6 @@ mod tests {
             bank_forks,
             scheduler,
             vec![], // no actual workers with metrics to report, this can be empty
-            None,
         );
 
         (test_frame, scheduler_controller)
@@ -635,11 +501,7 @@ mod tests {
     // In the tests, the decision will not become stale, so it is more convenient
     // to receive first and then schedule.
     fn test_receive_then_schedule<R: ReceiveAndBuffer>(
-        scheduler_controller: &mut SchedulerController<
-            Arc<ClusterInfo>,
-            R,
-            impl Scheduler<R::Transaction>,
-        >,
+        scheduler_controller: &mut SchedulerController<R, impl Scheduler<R::Transaction>>,
     ) {
         let decision = scheduler_controller
             .decision_maker

core/src/banking_stage/transaction_scheduler/scheduler_metrics.rs

@@ -59,8 +59,6 @@ pub struct SchedulerCountMetricsInner {
     pub num_finished: usize,
     /// Number of transactions that were retryable.
     pub num_retryable: usize,
-    /// Number of transactions that were scheduled to be forwarded.
-    pub num_forwarded: usize,
 
     /// Number of transactions that were immediately dropped on receive.
     pub num_dropped_on_receive: usize,
@@ -124,7 +122,6 @@ impl SchedulerCountMetricsInner {
             ),
             ("num_finished", self.num_finished, i64),
             ("num_retryable", self.num_retryable, i64),
-            ("num_forwarded", self.num_forwarded, i64),
             ("num_dropped_on_receive", self.num_dropped_on_receive, i64),
             (
                 "num_dropped_on_sanitization",
@@ -165,7 +162,6 @@ impl SchedulerCountMetricsInner {
             || self.num_schedule_filtered_out != 0
             || self.num_finished != 0
             || self.num_retryable != 0
-            || self.num_forwarded != 0
             || self.num_dropped_on_receive != 0
             || self.num_dropped_on_sanitization != 0
             || self.num_dropped_on_validate_locks != 0
@@ -183,7 +179,6 @@ impl SchedulerCountMetricsInner {
         self.num_schedule_filtered_out = 0;
         self.num_finished = 0;
         self.num_retryable = 0;
-        self.num_forwarded = 0;
         self.num_dropped_on_receive = 0;
         self.num_dropped_on_sanitization = 0;
         self.num_dropped_on_validate_locks = 0;
@@ -275,8 +270,6 @@ pub struct SchedulerTimingMetricsInner {
     pub clear_time_us: u64,
     /// Time spent cleaning expired or processed transactions from the container.
     pub clean_time_us: u64,
-    /// Time spent forwarding transactions.
-    pub forward_time_us: u64,
     /// Time spent receiving completed transactions.
     pub receive_completed_time_us: u64,
 }
@@ -318,7 +311,6 @@ impl SchedulerTimingMetricsInner {
             ("schedule_time_us", self.schedule_time_us, i64),
             ("clear_time_us", self.clear_time_us, i64),
             ("clean_time_us", self.clean_time_us, i64),
-            ("forward_time_us", self.forward_time_us, i64),
             (
                 "receive_completed_time_us",
                 self.receive_completed_time_us,
@@ -339,7 +331,6 @@ impl SchedulerTimingMetricsInner {
         self.schedule_time_us = 0;
         self.clear_time_us = 0;
         self.clean_time_us = 0;
-        self.forward_time_us = 0;
         self.receive_completed_time_us = 0;
     }
 }

core/src/banking_stage/transaction_scheduler/transaction_state.rs

@@ -1,10 +1,4 @@
-use {
-    crate::banking_stage::{
-        immutable_deserialized_packet::ImmutableDeserializedPacket, scheduler_messages::MaxAge,
-    },
-    solana_sdk::packet::{self},
-    std::sync::Arc,
-};
+use crate::banking_stage::scheduler_messages::MaxAge;
 
 /// Simple wrapper type to tie a sanitized transaction to max age slot.
 pub(crate) struct SanitizedTransactionTTL<Tx> {
@@ -36,18 +30,11 @@ pub(crate) enum TransactionState<Tx> {
     /// The transaction is available for scheduling.
     Unprocessed {
         transaction_ttl: SanitizedTransactionTTL<Tx>,
-        packet: Option<Arc<ImmutableDeserializedPacket>>,
         priority: u64,
         cost: u64,
-        should_forward: bool,
     },
     /// The transaction is currently scheduled or being processed.
-    Pending {
-        packet: Option<Arc<ImmutableDeserializedPacket>>,
-        priority: u64,
-        cost: u64,
-        should_forward: bool,
-    },
+    Pending { priority: u64, cost: u64 },
     /// Only used during transition.
     Transitioning,
 }
@@ -56,20 +43,13 @@ impl<Tx> TransactionState<Tx> {
     /// Creates a new `TransactionState` in the `Unprocessed` state.
     pub(crate) fn new(
         transaction_ttl: SanitizedTransactionTTL<Tx>,
-        packet: Option<Arc<ImmutableDeserializedPacket>>,
         priority: u64,
         cost: u64,
     ) -> Self {
-        let should_forward = packet
-            .as_ref()
-            .map(|packet| should_forward_from_meta(packet.original_packet().meta()))
-            .unwrap_or_default();
         Self::Unprocessed {
             transaction_ttl,
-            packet,
             priority,
             cost,
-            should_forward,
         }
     }
 
@@ -93,40 +73,6 @@ impl<Tx> TransactionState<Tx> {
         }
     }
 
-    /// Return whether packet should be attempted to be forwarded.
-    pub(crate) fn should_forward(&self) -> bool {
-        match self {
-            Self::Unprocessed {
-                should_forward: forwarded,
-                ..
-            } => *forwarded,
-            Self::Pending {
-                should_forward: forwarded,
-                ..
-            } => *forwarded,
-            Self::Transitioning => unreachable!(),
-        }
-    }
-
-    /// Mark the packet as forwarded.
-    /// This is used to prevent the packet from being forwarded multiple times.
-    pub(crate) fn mark_forwarded(&mut self) {
-        match self {
-            Self::Unprocessed { should_forward, .. } => *should_forward = false,
-            Self::Pending { should_forward, .. } => *should_forward = false,
-            Self::Transitioning => unreachable!(),
-        }
-    }
-
-    /// Return the packet of the transaction.
-    pub(crate) fn packet(&self) -> Option<&Arc<ImmutableDeserializedPacket>> {
-        match self {
-            Self::Unprocessed { packet, .. } => packet.as_ref(),
-            Self::Pending { packet, .. } => packet.as_ref(),
-            Self::Transitioning => unreachable!(),
-        }
-    }
-
     /// Intended to be called when a transaction is scheduled. This method will
     /// transition the transaction from `Unprocessed` to `Pending` and return the
     /// `SanitizedTransactionTTL` for processing.
@@ -138,17 +84,10 @@ impl<Tx> TransactionState<Tx> {
         match self.take() {
             TransactionState::Unprocessed {
                 transaction_ttl,
-                packet,
                 priority,
                 cost,
-                should_forward: forwarded,
             } => {
-                *self = TransactionState::Pending {
-                    packet,
-                    priority,
-                    cost,
-                    should_forward: forwarded,
-                };
+                *self = TransactionState::Pending { priority, cost };
                 transaction_ttl
             }
             TransactionState::Pending { .. } => {
@@ -170,18 +109,11 @@ impl<Tx> TransactionState<Tx> {
     ) {
         match self.take() {
             TransactionState::Unprocessed { .. } => panic!("already unprocessed"),
-            TransactionState::Pending {
-                packet,
-                priority,
-                cost,
-                should_forward: forwarded,
-            } => {
+            TransactionState::Pending { priority, cost } => {
                 *self = Self::Unprocessed {
                     transaction_ttl,
-                    packet,
                     priority,
                     cost,
-                    should_forward: forwarded,
                 }
             }
             Self::Transitioning => unreachable!(),
@@ -209,21 +141,15 @@ impl<Tx> TransactionState<Tx> {
     }
 }
 
-fn should_forward_from_meta(meta: &packet::Meta) -> bool {
-    !meta.forwarded() && meta.is_from_staked_node()
-}
-
 #[cfg(test)]
 mod tests {
     use {
         super::*,
-        packet::PacketFlags,
         solana_runtime_transaction::runtime_transaction::RuntimeTransaction,
         solana_sdk::{
             compute_budget::ComputeBudgetInstruction,
             hash::Hash,
             message::Message,
-            packet::Packet,
             signature::Keypair,
             signer::Signer,
             system_instruction,
@@ -242,20 +168,12 @@ mod tests {
         let message = Message::new(&ixs, Some(&from_keypair.pubkey()));
         let tx = Transaction::new(&[&from_keypair], message, Hash::default());
 
-        let packet = Arc::new(
-            ImmutableDeserializedPacket::new(Packet::from_data(None, tx.clone()).unwrap()).unwrap(),
-        );
         let transaction_ttl = SanitizedTransactionTTL {
             transaction: RuntimeTransaction::from_transaction_for_tests(tx),
             max_age: MaxAge::MAX,
         };
         const TEST_TRANSACTION_COST: u64 = 5000;
-        TransactionState::new(
-            transaction_ttl,
-            Some(packet),
-            compute_unit_price,
-            TEST_TRANSACTION_COST,
-        )
+        TransactionState::new(transaction_ttl, compute_unit_price, TEST_TRANSACTION_COST)
     }
 
     #[test]
@@ -373,23 +291,4 @@ mod tests {
         ));
         assert_eq!(transaction_ttl.max_age, MaxAge::MAX);
     }
-
-    #[test]
-    fn test_initialize_should_forward() {
-        let meta = packet::Meta::default();
-        assert!(!should_forward_from_meta(&meta));
-
-        let mut meta = packet::Meta::default();
-        meta.flags.set(PacketFlags::FORWARDED, true);
-        assert!(!should_forward_from_meta(&meta));
-
-        let mut meta = packet::Meta::default();
-        meta.set_from_staked_node(true);
-        assert!(should_forward_from_meta(&meta));
-
-        let mut meta = packet::Meta::default();
-        meta.flags.set(PacketFlags::FORWARDED, true);
-        meta.set_from_staked_node(true);
-        assert!(!should_forward_from_meta(&meta));
-    }
 }

core/src/banking_stage/transaction_scheduler/transaction_state_container.rs

@@ -3,10 +3,7 @@ use {
         transaction_priority_id::TransactionPriorityId,
         transaction_state::{SanitizedTransactionTTL, TransactionState},
     },
-    crate::banking_stage::{
-        immutable_deserialized_packet::ImmutableDeserializedPacket,
-        scheduler_messages::TransactionId,
-    },
+    crate::banking_stage::scheduler_messages::TransactionId,
     agave_transaction_view::resolved_transaction_view::ResolvedTransactionView,
     itertools::MinMaxResult,
     min_max_heap::MinMaxHeap,
@@ -67,7 +64,7 @@ pub(crate) trait StateContainer<Tx: TransactionWithMeta> {
     /// Panics if the transaction does not exist.
     fn get_transaction_ttl(&self, id: TransactionId) -> Option<&SanitizedTransactionTTL<Tx>>;
 
-    /// Retries a transaction - inserts transaction back into map (but not packet).
+    /// Retries a transaction - inserts transaction back into map.
     /// This transitions the transaction to `Unprocessed` state.
     fn retry_transaction(
         &mut self,
@@ -179,19 +176,13 @@ impl<Tx: TransactionWithMeta> TransactionStateContainer<Tx> {
     pub(crate) fn insert_new_transaction(
         &mut self,
         transaction_ttl: SanitizedTransactionTTL<Tx>,
-        packet: Arc<ImmutableDeserializedPacket>,
         priority: u64,
         cost: u64,
     ) -> bool {
         let priority_id = {
             let entry = self.get_vacant_map_entry();
             let transaction_id = entry.key();
-            entry.insert(TransactionState::new(
-                transaction_ttl,
-                Some(packet),
-                priority,
-                cost,
-            ));
+            entry.insert(TransactionState::new(transaction_ttl, priority, cost));
             TransactionPriorityId::new(priority, transaction_id)
         };
 
@@ -322,12 +313,12 @@ mod tests {
         super::*,
         crate::banking_stage::scheduler_messages::MaxAge,
         agave_transaction_view::transaction_view::SanitizedTransactionView,
+        solana_perf::packet::Packet,
         solana_runtime_transaction::runtime_transaction::RuntimeTransaction,
         solana_sdk::{
             compute_budget::ComputeBudgetInstruction,
             hash::Hash,
             message::Message,
-            packet::Packet,
             signature::Keypair,
             signer::Signer,
             system_instruction,
@@ -341,7 +332,6 @@ mod tests {
         priority: u64,
     ) -> (
         SanitizedTransactionTTL<RuntimeTransaction<SanitizedTransaction>>,
-        Arc<ImmutableDeserializedPacket>,
         u64,
         u64,
     ) {
@@ -356,18 +346,12 @@ mod tests {
             message,
             Hash::default(),
         ));
-        let packet = Arc::new(
-            ImmutableDeserializedPacket::new(
-                Packet::from_data(None, tx.to_versioned_transaction()).unwrap(),
-            )
-            .unwrap(),
-        );
         let transaction_ttl = SanitizedTransactionTTL {
             transaction: tx,
             max_age: MaxAge::MAX,
         };
         const TEST_TRANSACTION_COST: u64 = 5000;
-        (transaction_ttl, packet, priority, TEST_TRANSACTION_COST)
+        (transaction_ttl, priority, TEST_TRANSACTION_COST)
     }
 
     fn push_to_container(
@@ -375,8 +359,8 @@ mod tests {
         num: usize,
     ) {
         for priority in 0..num as u64 {
-            let (transaction_ttl, packet, priority, cost) = test_transaction(priority);
-            container.insert_new_transaction(transaction_ttl, packet, priority, cost);
+            let (transaction_ttl, priority, cost) = test_transaction(priority);
+            container.insert_new_transaction(transaction_ttl, priority, cost);
         }
     }
 
@@ -446,7 +430,6 @@ mod tests {
                     transaction: view,
                     max_age: MaxAge::MAX,
                 },
-                None,
                 priority,
                 cost,
             ))
@@ -454,9 +437,12 @@ mod tests {
 
         // Push 2 transactions into the queue so buffer is full.
         for priority in [4, 5] {
-            let (_transaction_ttl, packet, priority, cost) = test_transaction(priority);
+            let (transaction_ttl, priority, cost) = test_transaction(priority);
+            let packet =
+                Packet::from_data(None, transaction_ttl.transaction.to_versioned_transaction())
+                    .unwrap();
             let id = container
-                .try_insert_map_only_with_data(packet.original_packet().data(..).unwrap(), |data| {
+                .try_insert_map_only_with_data(packet.data(..).unwrap(), |data| {
                     packet_parser(data, priority, cost)
                 })
                 .unwrap();
@@ -470,9 +456,12 @@ mod tests {
         // Push 5 additional packets in. 5 should be dropped.
         let mut priority_ids = Vec::with_capacity(5);
         for priority in [10, 11, 12, 1, 2] {
-            let (_transaction_ttl, packet, priority, cost) = test_transaction(priority);
+            let (transaction_ttl, priority, cost) = test_transaction(priority);
+            let packet =
+                Packet::from_data(None, transaction_ttl.transaction.to_versioned_transaction())
+                    .unwrap();
             let id = container
-                .try_insert_map_only_with_data(packet.original_packet().data(..).unwrap(), |data| {
+                .try_insert_map_only_with_data(packet.data(..).unwrap(), |data| {
                     packet_parser(data, priority, cost)
                 })
                 .unwrap();
@@ -488,9 +477,12 @@ mod tests {
         // If we attempt to push additional transactions to the queue, they
         // are rejected regardless of their priority.
         let priority = u64::MAX;
-        let (_transaction_ttl, packet, priority, cost) = test_transaction(priority);
+        let (transaction_ttl, priority, cost) = test_transaction(priority);
+        let packet =
+            Packet::from_data(None, transaction_ttl.transaction.to_versioned_transaction())
+                .unwrap();
         let id = container
-            .try_insert_map_only_with_data(packet.original_packet().data(..).unwrap(), |data| {
+            .try_insert_map_only_with_data(packet.data(..).unwrap(), |data| {
                 packet_parser(data, priority, cost)
             })
             .unwrap();

core/src/banking_stage/unprocessed_packet_batches.rs

@@ -15,14 +15,12 @@ use {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct DeserializedPacket {
     immutable_section: Arc<ImmutableDeserializedPacket>,
-    pub forwarded: bool,
 }
 
 impl DeserializedPacket {
     pub fn from_immutable_section(immutable_section: ImmutableDeserializedPacket) -> Self {
         Self {
             immutable_section: Arc::new(immutable_section),
-            forwarded: false,
         }
     }
 
@@ -31,7 +29,6 @@ impl DeserializedPacket {
 
         Ok(Self {
             immutable_section: Arc::new(immutable_section),
-            forwarded: false,
         })
     }
 
@@ -265,30 +262,6 @@ impl UnprocessedPacketBatches {
     pub fn capacity(&self) -> usize {
         self.packet_priority_queue.capacity()
     }
-
-    pub fn is_forwarded(&self, immutable_packet: &ImmutableDeserializedPacket) -> bool {
-        self.message_hash_to_transaction
-            .get(immutable_packet.message_hash())
-            .map_or(true, |p| p.forwarded)
-    }
-
-    pub fn mark_accepted_packets_as_forwarded(
-        &mut self,
-        packets_to_process: &[Arc<ImmutableDeserializedPacket>],
-        accepted_packet_indexes: &[usize],
-    ) {
-        accepted_packet_indexes
-            .iter()
-            .for_each(|accepted_packet_index| {
-                let accepted_packet = packets_to_process[*accepted_packet_index].clone();
-                if let Some(deserialized_packet) = self
-                    .message_hash_to_transaction
-                    .get_mut(accepted_packet.message_hash())
-                {
-                    deserialized_packet.forwarded = true;
-                }
-            });
-    }
 }
 
 #[cfg(test)]

core/src/banking_stage/unprocessed_transaction_storage.rs

@@ -1,7 +1,6 @@
 use {
     super::{
         consumer::Consumer,
-        forward_packet_batches_by_accounts::ForwardPacketBatchesByAccounts,
         immutable_deserialized_packet::ImmutableDeserializedPacket,
         latest_unprocessed_votes::{
             LatestUnprocessedVotes, LatestValidatorVotePacket, VoteBatchInsertionMetrics,
@@ -13,19 +12,15 @@ use {
         unprocessed_packet_batches::{
             DeserializedPacket, PacketBatchInsertionMetrics, UnprocessedPacketBatches,
         },
-        BankingStageStats, FilterForwardingResults, ForwardOption,
+        BankingStageStats,
     },
     itertools::Itertools,
     min_max_heap::MinMaxHeap,
     solana_accounts_db::account_locks::validate_account_locks,
-    solana_feature_set::FeatureSet,
     solana_measure::measure_us,
     solana_runtime::bank::Bank,
     solana_runtime_transaction::runtime_transaction::RuntimeTransaction,
-    solana_sdk::{
-        clock::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, hash::Hash, saturating_add_assign,
-        transaction::SanitizedTransaction,
-    },
+    solana_sdk::{hash::Hash, transaction::SanitizedTransaction},
     solana_svm::transaction_error_metrics::TransactionErrorMetrics,
     std::{
         collections::HashMap,
@@ -316,7 +311,7 @@ impl UnprocessedTransactionStorage {
     }
 
     pub fn should_not_process(&self) -> bool {
-        // The gossip vote thread does not need to process or forward any votes, that is
+        // The gossip vote thread does not need to process any votes, that is
         // handled by the tpu vote thread
         if let Self::VoteStorage(vote_storage) = self {
             return matches!(vote_storage.vote_source, VoteSource::Gossip);
@@ -332,22 +327,6 @@ impl UnprocessedTransactionStorage {
         }
     }
 
-    pub fn forward_option(&self) -> ForwardOption {
-        match self {
-            Self::VoteStorage(vote_storage) => vote_storage.forward_option(),
-            Self::LocalTransactionStorage(transaction_storage) => {
-                transaction_storage.forward_option()
-            }
-        }
-    }
-
-    pub fn clear_forwarded_packets(&mut self) {
-        match self {
-            Self::LocalTransactionStorage(transaction_storage) => transaction_storage.clear(), // Since we set everything as forwarded this is the same
-            Self::VoteStorage(vote_storage) => vote_storage.clear_forwarded_packets(),
-        }
-    }
-
     pub(crate) fn insert_batch(
         &mut self,
         deserialized_packets: Vec<ImmutableDeserializedPacket>,
@@ -362,25 +341,6 @@ impl UnprocessedTransactionStorage {
         }
     }
 
-    pub fn filter_forwardable_packets_and_add_batches(
-        &mut self,
-        bank: Arc<Bank>,
-        forward_packet_batches_by_accounts: &mut ForwardPacketBatchesByAccounts,
-    ) -> FilterForwardingResults {
-        match self {
-            Self::LocalTransactionStorage(transaction_storage) => transaction_storage
-                .filter_forwardable_packets_and_add_batches(
-                    bank,
-                    forward_packet_batches_by_accounts,
-                ),
-            Self::VoteStorage(vote_storage) => vote_storage
-                .filter_forwardable_packets_and_add_batches(
-                    bank,
-                    forward_packet_batches_by_accounts,
-                ),
-        }
-    }
-
     /// The processing function takes a stream of packets ready to process, and returns the indices
     /// of the unprocessed packets that are eligible for retry. A return value of None means that
     /// all packets are unprocessed and eligible for retry.
@@ -415,6 +375,13 @@ impl UnprocessedTransactionStorage {
         }
     }
 
+    pub(crate) fn clear(&mut self) {
+        match self {
+            Self::LocalTransactionStorage(_) => {}
+            Self::VoteStorage(vote_storage) => vote_storage.clear(),
+        }
+    }
+
     pub(crate) fn cache_epoch_boundary_info(&mut self, bank: &Bank) {
         match self {
             Self::LocalTransactionStorage(_) => (),
@@ -436,17 +403,6 @@ impl VoteStorage {
         MAX_NUM_VOTES_RECEIVE
     }
 
-    fn forward_option(&self) -> ForwardOption {
-        match self.vote_source {
-            VoteSource::Tpu => ForwardOption::ForwardTpuVote,
-            VoteSource::Gossip => ForwardOption::NotForward,
-        }
-    }
-
-    fn clear_forwarded_packets(&mut self) {
-        self.latest_unprocessed_votes.clear_forwarded_packets();
-    }
-
     fn insert_batch(
         &mut self,
         deserialized_packets: Vec<ImmutableDeserializedPacket>,
@@ -467,23 +423,6 @@ impl VoteStorage {
         )
     }
 
-    fn filter_forwardable_packets_and_add_batches(
-        &mut self,
-        bank: Arc<Bank>,
-        forward_packet_batches_by_accounts: &mut ForwardPacketBatchesByAccounts,
-    ) -> FilterForwardingResults {
-        if matches!(self.vote_source, VoteSource::Tpu) {
-            let total_forwardable_packets = self
-                .latest_unprocessed_votes
-                .get_and_insert_forwardable_packets(bank, forward_packet_batches_by_accounts);
-            return FilterForwardingResults {
-                total_forwardable_packets,
-                ..FilterForwardingResults::default()
-            };
-        }
-        FilterForwardingResults::default()
-    }
-
     // returns `true` if the end of slot is reached
     fn process_packets<F>(
         &mut self,
@@ -560,6 +499,10 @@ impl VoteStorage {
         scanner.finalize().payload.reached_end_of_slot
     }
 
+    fn clear(&mut self) {
+        self.latest_unprocessed_votes.clear();
+    }
+
     fn cache_epoch_boundary_info(&mut self, bank: &Bank) {
         if matches!(self.vote_source, VoteSource::Gossip) {
             panic!("Gossip vote thread should not be checking epoch boundary");
@@ -603,18 +546,6 @@ impl ThreadLocalUnprocessedPackets {
         self.unprocessed_packet_batches.iter_mut()
     }
 
-    fn forward_option(&self) -> ForwardOption {
-        match self.thread_type {
-            ThreadType::Transactions => ForwardOption::ForwardTransaction,
-            ThreadType::Voting(VoteSource::Tpu) => ForwardOption::ForwardTpuVote,
-            ThreadType::Voting(VoteSource::Gossip) => ForwardOption::NotForward,
-        }
-    }
-
-    fn clear(&mut self) {
-        self.unprocessed_packet_batches.clear();
-    }
-
     fn insert_batch(
         &mut self,
         deserialized_packets: Vec<ImmutableDeserializedPacket>,
@@ -626,123 +557,6 @@ impl ThreadLocalUnprocessedPackets {
         )
     }
 
-    /// Filter out packets that fail to sanitize, or are no longer valid (could be
-    /// too old, a duplicate of something already processed). Doing this in batches to avoid
-    /// checking bank's blockhash and status cache per transaction which could be bad for performance.
-    /// Added valid and sanitized packets to forwarding queue.
-    fn filter_forwardable_packets_and_add_batches(
-        &mut self,
-        bank: Arc<Bank>,
-        forward_buffer: &mut ForwardPacketBatchesByAccounts,
-    ) -> FilterForwardingResults {
-        let mut total_forwardable_packets: usize = 0;
-        let mut total_packet_conversion_us: u64 = 0;
-        let mut total_filter_packets_us: u64 = 0;
-        let mut total_dropped_packets: usize = 0;
-
-        let mut original_priority_queue = self.take_priority_queue();
-        let original_capacity = original_priority_queue.capacity();
-        let mut new_priority_queue = MinMaxHeap::with_capacity(original_capacity);
-
-        // indicates if `forward_buffer` still accept more packets, see details at
-        // `ForwardPacketBatchesByAccounts.rs`.
-        let mut accepting_packets = true;
-        // batch iterate through self.unprocessed_packet_batches in desc priority order
-        new_priority_queue.extend(
-            original_priority_queue
-                .drain_desc()
-                .chunks(UNPROCESSED_BUFFER_STEP_SIZE)
-                .into_iter()
-                .flat_map(|packets_to_process| {
-                    // Only process packets not yet forwarded
-                    let (forwarded_packets, packets_to_forward) =
-                        self.prepare_packets_to_forward(packets_to_process);
-
-                    [
-                        forwarded_packets,
-                        if accepting_packets {
-                            let (
-                                (sanitized_transactions, transaction_to_packet_indexes),
-                                packet_conversion_us,
-                            ) = measure_us!(self.sanitize_unforwarded_packets(
-                                &packets_to_forward,
-                                &bank,
-                                &mut total_dropped_packets
-                            ));
-                            saturating_add_assign!(
-                                total_packet_conversion_us,
-                                packet_conversion_us
-                            );
-
-                            let (forwardable_transaction_indexes, filter_packets_us) =
-                                measure_us!(Self::filter_invalid_transactions(
-                                    &sanitized_transactions,
-                                    &bank,
-                                    &mut total_dropped_packets
-                                ));
-                            saturating_add_assign!(total_filter_packets_us, filter_packets_us);
-                            saturating_add_assign!(
-                                total_forwardable_packets,
-                                forwardable_transaction_indexes.len()
-                            );
-
-                            let accepted_packet_indexes =
-                                Self::add_filtered_packets_to_forward_buffer(
-                                    forward_buffer,
-                                    &packets_to_forward,
-                                    &sanitized_transactions,
-                                    &transaction_to_packet_indexes,
-                                    &forwardable_transaction_indexes,
-                                    &mut total_dropped_packets,
-                                    &bank.feature_set,
-                                );
-                            accepting_packets = accepted_packet_indexes.len()
-                                == forwardable_transaction_indexes.len();
-
-                            self.unprocessed_packet_batches
-                                .mark_accepted_packets_as_forwarded(
-                                    &packets_to_forward,
-                                    &accepted_packet_indexes,
-                                );
-
-                            Self::collect_retained_packets(
-                                &mut self.unprocessed_packet_batches.message_hash_to_transaction,
-                                &packets_to_forward,
-                                &Self::prepare_filtered_packet_indexes(
-                                    &transaction_to_packet_indexes,
-                                    &forwardable_transaction_indexes,
-                                ),
-                            )
-                        } else {
-                            // skip sanitizing and filtering if not longer able to add more packets for forwarding
-                            saturating_add_assign!(total_dropped_packets, packets_to_forward.len());
-                            packets_to_forward
-                        },
-                    ]
-                    .concat()
-                }),
-        );
-
-        // replace packet priority queue
-        self.unprocessed_packet_batches.packet_priority_queue = new_priority_queue;
-        self.verify_priority_queue(original_capacity);
-
-        // Assert unprocessed queue is still consistent
-        assert_eq!(
-            self.unprocessed_packet_batches.packet_priority_queue.len(),
-            self.unprocessed_packet_batches
-                .message_hash_to_transaction
-                .len()
-        );
-
-        FilterForwardingResults {
-            total_forwardable_packets,
-            total_dropped_packets,
-            total_packet_conversion_us,
-            total_filter_packets_us,
-        }
-    }
-
     /// Take self.unprocessed_packet_batches's priority_queue out, leave empty MinMaxHeap in its place.
     fn take_priority_queue(&mut self) -> MinMaxHeap<Arc<ImmutableDeserializedPacket>> {
         std::mem::replace(
@@ -768,105 +582,6 @@ impl ThreadLocalUnprocessedPackets {
         );
     }
 
-    /// sanitize un-forwarded packet into SanitizedTransaction for validation and forwarding.
-    fn sanitize_unforwarded_packets(
-        &mut self,
-        packets_to_process: &[Arc<ImmutableDeserializedPacket>],
-        bank: &Bank,
-        total_dropped_packets: &mut usize,
-    ) -> (Vec<RuntimeTransaction<SanitizedTransaction>>, Vec<usize>) {
-        // Get ref of ImmutableDeserializedPacket
-        let deserialized_packets = packets_to_process.iter().map(|p| &**p);
-        let (transactions, transaction_to_packet_indexes): (Vec<_>, Vec<_>) = deserialized_packets
-            .enumerate()
-            .filter_map(|(packet_index, deserialized_packet)| {
-                deserialized_packet
-                    .build_sanitized_transaction(
-                        bank.vote_only_bank(),
-                        bank,
-                        bank.get_reserved_account_keys(),
-                    )
-                    .map(|(transaction, _deactivation_slot)| (transaction, packet_index))
-            })
-            .unzip();
-
-        let filtered_count = packets_to_process.len().saturating_sub(transactions.len());
-        saturating_add_assign!(*total_dropped_packets, filtered_count);
-
-        (transactions, transaction_to_packet_indexes)
-    }
-
-    /// Checks sanitized transactions against bank, returns valid transaction indexes
-    fn filter_invalid_transactions(
-        transactions: &[RuntimeTransaction<SanitizedTransaction>],
-        bank: &Bank,
-        total_dropped_packets: &mut usize,
-    ) -> Vec<usize> {
-        let filter = vec![Ok(()); transactions.len()];
-        let results = bank.check_transactions_with_forwarding_delay(
-            transactions,
-            &filter,
-            FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET,
-        );
-
-        let filtered_count = transactions.len().saturating_sub(results.len());
-        saturating_add_assign!(*total_dropped_packets, filtered_count);
-
-        results
-            .iter()
-            .enumerate()
-            .filter_map(|(tx_index, result)| result.as_ref().ok().map(|_| tx_index))
-            .collect_vec()
-    }
-
-    fn prepare_filtered_packet_indexes(
-        transaction_to_packet_indexes: &[usize],
-        retained_transaction_indexes: &[usize],
-    ) -> Vec<usize> {
-        retained_transaction_indexes
-            .iter()
-            .map(|tx_index| transaction_to_packet_indexes[*tx_index])
-            .collect_vec()
-    }
-
-    /// try to add filtered forwardable and valid packets to forward buffer;
-    /// returns vector of packet indexes that were accepted for forwarding.
-    fn add_filtered_packets_to_forward_buffer(
-        forward_buffer: &mut ForwardPacketBatchesByAccounts,
-        packets_to_process: &[Arc<ImmutableDeserializedPacket>],
-        transactions: &[RuntimeTransaction<SanitizedTransaction>],
-        transaction_to_packet_indexes: &[usize],
-        forwardable_transaction_indexes: &[usize],
-        total_dropped_packets: &mut usize,
-        feature_set: &FeatureSet,
-    ) -> Vec<usize> {
-        let mut added_packets_count: usize = 0;
-        let mut accepted_packet_indexes = Vec::with_capacity(transaction_to_packet_indexes.len());
-        for forwardable_transaction_index in forwardable_transaction_indexes {
-            let sanitized_transaction = &transactions[*forwardable_transaction_index];
-            let forwardable_packet_index =
-                transaction_to_packet_indexes[*forwardable_transaction_index];
-            let immutable_deserialized_packet =
-                packets_to_process[forwardable_packet_index].clone();
-            if !forward_buffer.try_add_packet(
-                sanitized_transaction,
-                immutable_deserialized_packet,
-                feature_set,
-            ) {
-                break;
-            }
-            accepted_packet_indexes.push(forwardable_packet_index);
-            saturating_add_assign!(added_packets_count, 1);
-        }
-
-        let filtered_count = forwardable_transaction_indexes
-            .len()
-            .saturating_sub(added_packets_count);
-        saturating_add_assign!(*total_dropped_packets, filtered_count);
-
-        accepted_packet_indexes
-    }
-
     fn collect_retained_packets(
         message_hash_to_transaction: &mut HashMap<Hash, DeserializedPacket>,
         packets_to_process: &[Arc<ImmutableDeserializedPacket>],
@@ -959,36 +674,6 @@ impl ThreadLocalUnprocessedPackets {
 
         reached_end_of_slot
     }
-
-    /// Prepare a chunk of packets for forwarding, filter out already forwarded packets while
-    /// counting tracers.
-    /// Returns Vec of unforwarded packets, and Vec<bool> of same size each indicates corresponding
-    /// packet is tracer packet.
-    fn prepare_packets_to_forward(
-        &self,
-        packets_to_forward: impl Iterator<Item = Arc<ImmutableDeserializedPacket>>,
-    ) -> (
-        Vec<Arc<ImmutableDeserializedPacket>>,
-        Vec<Arc<ImmutableDeserializedPacket>>,
-    ) {
-        let mut forwarded_packets: Vec<Arc<ImmutableDeserializedPacket>> = vec![];
-        let forwardable_packets = packets_to_forward
-            .into_iter()
-            .filter_map(|immutable_deserialized_packet| {
-                if !self
-                    .unprocessed_packet_batches
-                    .is_forwarded(&immutable_deserialized_packet)
-                {
-                    Some(immutable_deserialized_packet)
-                } else {
-                    forwarded_packets.push(immutable_deserialized_packet);
-                    None
-                }
-            })
-            .collect();
-
-        (forwarded_packets, forwardable_packets)
-    }
 }
 
 #[cfg(test)]
@@ -996,14 +681,12 @@ mod tests {
     use {
         super::*,
         itertools::iproduct,
-        solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo},
         solana_perf::packet::{Packet, PacketFlags},
         solana_runtime::genesis_utils,
         solana_sdk::{
             hash::Hash,
             signature::{Keypair, Signer},
             system_transaction,
-            transaction::Transaction,
         },
         solana_vote::vote_transaction::new_tower_sync_transaction,
         solana_vote_program::vote_state::TowerSync,
@@ -1061,126 +744,6 @@ mod tests {
         assert_eq!(non_retryable_indexes, vec![(0, 1), (4, 5), (6, 8)]);
     }
 
-    #[test]
-    fn test_filter_and_forward_with_account_limits() {
-        solana_logger::setup();
-        let GenesisConfigInfo {
-            genesis_config,
-            mint_keypair,
-            ..
-        } = create_genesis_config(10);
-        let (current_bank, _bank_forks) = Bank::new_with_bank_forks_for_tests(&genesis_config);
-
-        let simple_transactions: Vec<Transaction> = (0..256)
-            .map(|_id| {
-                // packets are deserialized upon receiving, failed packets will not be
-                // forwarded; Therefore we need to create real packets here.
-                let key1 = Keypair::new();
-                system_transaction::transfer(
-                    &mint_keypair,
-                    &key1.pubkey(),
-                    genesis_config.rent.minimum_balance(0),
-                    genesis_config.hash(),
-                )
-            })
-            .collect_vec();
-
-        let mut packets: Vec<DeserializedPacket> = simple_transactions
-            .iter()
-            .enumerate()
-            .map(|(packets_id, transaction)| {
-                let mut p = Packet::from_data(None, transaction).unwrap();
-                p.meta_mut().port = packets_id as u16;
-                DeserializedPacket::new(p).unwrap()
-            })
-            .collect_vec();
-
-        // all packets are forwarded
-        {
-            let buffered_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(packets.clone(), packets.len());
-            let mut transaction_storage = UnprocessedTransactionStorage::new_transaction_storage(
-                buffered_packet_batches,
-                ThreadType::Transactions,
-            );
-            let mut forward_packet_batches_by_accounts =
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-
-            let FilterForwardingResults {
-                total_forwardable_packets,
-                ..
-            } = transaction_storage.filter_forwardable_packets_and_add_batches(
-                current_bank.clone(),
-                &mut forward_packet_batches_by_accounts,
-            );
-            assert_eq!(total_forwardable_packets, 256);
-
-            // packets in a batch are forwarded in arbitrary order; verify the ports match after
-            // sorting
-            let expected_ports: Vec<_> = (0..256).collect();
-            let mut forwarded_ports: Vec<_> = forward_packet_batches_by_accounts
-                .iter_batches()
-                .flat_map(|batch| batch.get_forwardable_packets().map(|p| p.meta().port))
-                .collect();
-            forwarded_ports.sort_unstable();
-            assert_eq!(expected_ports, forwarded_ports);
-        }
-
-        // some packets are forwarded
-        {
-            let num_already_forwarded = 16;
-            for packet in &mut packets[0..num_already_forwarded] {
-                packet.forwarded = true;
-            }
-            let buffered_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(packets.clone(), packets.len());
-            let mut transaction_storage = UnprocessedTransactionStorage::new_transaction_storage(
-                buffered_packet_batches,
-                ThreadType::Transactions,
-            );
-            let mut forward_packet_batches_by_accounts =
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-            let FilterForwardingResults {
-                total_forwardable_packets,
-                ..
-            } = transaction_storage.filter_forwardable_packets_and_add_batches(
-                current_bank.clone(),
-                &mut forward_packet_batches_by_accounts,
-            );
-            assert_eq!(
-                total_forwardable_packets,
-                packets.len() - num_already_forwarded
-            );
-        }
-
-        // some packets are invalid (already processed)
-        {
-            let num_already_processed = 16;
-            for tx in &simple_transactions[0..num_already_processed] {
-                assert_eq!(current_bank.process_transaction(tx), Ok(()));
-            }
-            let buffered_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(packets.clone(), packets.len());
-            let mut transaction_storage = UnprocessedTransactionStorage::new_transaction_storage(
-                buffered_packet_batches,
-                ThreadType::Transactions,
-            );
-            let mut forward_packet_batches_by_accounts =
-                ForwardPacketBatchesByAccounts::new_with_default_batch_limits();
-            let FilterForwardingResults {
-                total_forwardable_packets,
-                ..
-            } = transaction_storage.filter_forwardable_packets_and_add_batches(
-                current_bank,
-                &mut forward_packet_batches_by_accounts,
-            );
-            assert_eq!(
-                total_forwardable_packets,
-                packets.len() - num_already_processed
-            );
-        }
-    }
-
     #[test]
     fn test_unprocessed_transaction_storage_insert() -> Result<(), Box<dyn Error>> {
         let keypair = Keypair::new();
@@ -1309,96 +872,4 @@ mod tests {
         assert_eq!(1, transaction_storage.len());
         Ok(())
     }
-
-    #[test]
-    fn test_prepare_packets_to_forward() {
-        solana_logger::setup();
-        let GenesisConfigInfo {
-            genesis_config,
-            mint_keypair,
-            ..
-        } = create_genesis_config(10);
-
-        let simple_transactions: Vec<Transaction> = (0..256)
-            .map(|_id| {
-                // packets are deserialized upon receiving, failed packets will not be
-                // forwarded; Therefore we need to create real packets here.
-                let key1 = Keypair::new();
-                system_transaction::transfer(
-                    &mint_keypair,
-                    &key1.pubkey(),
-                    genesis_config.rent.minimum_balance(0),
-                    genesis_config.hash(),
-                )
-            })
-            .collect_vec();
-
-        let mut packets: Vec<DeserializedPacket> = simple_transactions
-            .iter()
-            .enumerate()
-            .map(|(packets_id, transaction)| {
-                let mut p = Packet::from_data(None, transaction).unwrap();
-                p.meta_mut().port = packets_id as u16;
-                DeserializedPacket::new(p).unwrap()
-            })
-            .collect_vec();
-
-        // test preparing buffered packets for forwarding
-        let test_prepareing_buffered_packets_for_forwarding =
-            |buffered_packet_batches: UnprocessedPacketBatches| -> usize {
-                let mut total_packets_to_forward: usize = 0;
-
-                let mut unprocessed_transactions = ThreadLocalUnprocessedPackets {
-                    unprocessed_packet_batches: buffered_packet_batches,
-                    thread_type: ThreadType::Transactions,
-                };
-
-                let mut original_priority_queue = unprocessed_transactions.take_priority_queue();
-                let _ = original_priority_queue
-                    .drain_desc()
-                    .chunks(128usize)
-                    .into_iter()
-                    .flat_map(|packets_to_process| {
-                        let (_, packets_to_forward) =
-                            unprocessed_transactions.prepare_packets_to_forward(packets_to_process);
-                        total_packets_to_forward += packets_to_forward.len();
-                        packets_to_forward
-                    })
-                    .collect::<MinMaxHeap<Arc<ImmutableDeserializedPacket>>>();
-                total_packets_to_forward
-            };
-
-        {
-            let buffered_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(packets.clone(), packets.len());
-            let total_packets_to_forward =
-                test_prepareing_buffered_packets_for_forwarding(buffered_packet_batches);
-            assert_eq!(total_packets_to_forward, 256);
-        }
-
-        // some packets are forwarded
-        {
-            let num_already_forwarded = 16;
-            for packet in &mut packets[0..num_already_forwarded] {
-                packet.forwarded = true;
-            }
-            let buffered_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(packets.clone(), packets.len());
-            let total_packets_to_forward =
-                test_prepareing_buffered_packets_for_forwarding(buffered_packet_batches);
-            assert_eq!(total_packets_to_forward, 256 - num_already_forwarded);
-        }
-
-        // all packets are forwarded
-        {
-            for packet in &mut packets {
-                packet.forwarded = true;
-            }
-            let buffered_packet_batches: UnprocessedPacketBatches =
-                UnprocessedPacketBatches::from_iter(packets.clone(), packets.len());
-            let total_packets_to_forward =
-                test_prepareing_buffered_packets_for_forwarding(buffered_packet_batches);
-            assert_eq!(total_packets_to_forward, 0);
-        }
-    }
 }

core/src/sigverify.rs

@@ -13,27 +13,36 @@ use {
         sigverify_stage::{SigVerifier, SigVerifyServiceError},
     },
     agave_banking_stage_ingress_types::BankingPacketBatch,
+    crossbeam_channel::Sender,
     solana_perf::{cuda_runtime::PinnedVec, packet::PacketBatch, recycler::Recycler, sigverify},
 };
 
 pub struct TransactionSigVerifier {
-    packet_sender: BankingPacketSender,
+    banking_stage_sender: BankingPacketSender,
+    forward_stage_sender: Option<Sender<(BankingPacketBatch, bool)>>,
     recycler: Recycler<TxOffset>,
     recycler_out: Recycler<PinnedVec<u8>>,
     reject_non_vote: bool,
 }
 
 impl TransactionSigVerifier {
-    pub fn new_reject_non_vote(packet_sender: BankingPacketSender) -> Self {
-        let mut new_self = Self::new(packet_sender);
+    pub fn new_reject_non_vote(
+        packet_sender: BankingPacketSender,
+        forward_stage_sender: Option<Sender<(BankingPacketBatch, bool)>>,
+    ) -> Self {
+        let mut new_self = Self::new(packet_sender, forward_stage_sender);
         new_self.reject_non_vote = true;
         new_self
     }
 
-    pub fn new(packet_sender: BankingPacketSender) -> Self {
+    pub fn new(
+        banking_stage_sender: BankingPacketSender,
+        forward_stage_sender: Option<Sender<(BankingPacketBatch, bool)>>,
+    ) -> Self {
         init();
         Self {
-            packet_sender,
+            banking_stage_sender,
+            forward_stage_sender,
             recycler: Recycler::warmed(50, 4096),
             recycler_out: Recycler::warmed(50, 4096),
             reject_non_vote: false,
@@ -48,8 +57,15 @@ impl SigVerifier for TransactionSigVerifier {
         &mut self,
         packet_batches: Vec<PacketBatch>,
     ) -> Result<(), SigVerifyServiceError<Self::SendType>> {
-        self.packet_sender
-            .send(BankingPacketBatch::new(packet_batches))?;
+        let banking_packet_batch = BankingPacketBatch::new(packet_batches);
+        if let Some(forward_stage_sender) = &self.forward_stage_sender {
+            self.banking_stage_sender
+                .send(banking_packet_batch.clone())?;
+            let _ = forward_stage_sender.try_send((banking_packet_batch, self.reject_non_vote));
+        } else {
+            self.banking_stage_sender.send(banking_packet_batch)?;
+        }
+
         Ok(())
     }
 

core/src/sigverify_stage.rs

@@ -498,7 +498,7 @@ mod tests {
         trace!("start");
         let (packet_s, packet_r) = unbounded();
         let (verified_s, verified_r) = BankingTracer::channel_for_test();
-        let verifier = TransactionSigVerifier::new(verified_s);
+        let verifier = TransactionSigVerifier::new(verified_s, None);
         let stage = SigVerifyStage::new(packet_r, verifier, "solSigVerTest", "test");
 
         let now = Instant::now();

core/src/tpu.rs

@@ -17,6 +17,7 @@ use {
             VerifiedVoteSender, VoteTracker,
         },
         fetch_stage::FetchStage,
+        forwarding_stage::ForwardingStage,
         sigverify::TransactionSigVerifier,
         sigverify_stage::SigVerifyStage,
         staked_nodes_updater_service::StakedNodesUpdaterService,
@@ -24,13 +25,14 @@ use {
         validator::{BlockProductionMethod, GeneratorConfig, TransactionStructure},
     },
     bytes::Bytes,
-    crossbeam_channel::{unbounded, Receiver},
+    crossbeam_channel::{bounded, unbounded, Receiver},
     solana_client::connection_cache::ConnectionCache,
     solana_gossip::cluster_info::ClusterInfo,
     solana_ledger::{
         blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
         entry_notifier_service::EntryNotifierSender,
     },
+    solana_perf::data_budget::DataBudget,
     solana_poh::poh_recorder::{PohRecorder, WorkingBankEntry},
     solana_rpc::{
         optimistically_confirmed_bank_tracker::BankNotificationSender,
@@ -39,6 +41,7 @@ use {
     solana_runtime::{
         bank_forks::BankForks,
         prioritization_fee_cache::PrioritizationFeeCache,
+        root_bank_cache::RootBankCache,
         vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender},
     },
     solana_sdk::{clock::Slot, pubkey::Pubkey, quic::NotifyKeyUpdate, signature::Keypair},
@@ -51,7 +54,7 @@ use {
         collections::HashMap,
         net::{SocketAddr, UdpSocket},
         sync::{atomic::AtomicBool, Arc, RwLock},
-        thread,
+        thread::{self, JoinHandle},
         time::Duration,
     },
     tokio::sync::mpsc::Sender as AsyncSender,
@@ -72,6 +75,7 @@ pub struct Tpu {
     sigverify_stage: SigVerifyStage,
     vote_sigverify_stage: SigVerifyStage,
     banking_stage: BankingStage,
+    forwarding_stage: JoinHandle<()>,
     cluster_info_vote_listener: ClusterInfoVoteListener,
     broadcast_stage: BroadcastStage,
     tpu_quic_t: thread::JoinHandle<()>,
@@ -219,13 +223,20 @@ impl Tpu {
         )
         .unwrap();
 
+        let (forward_stage_sender, forward_stage_receiver) = bounded(1024);
         let sigverify_stage = {
-            let verifier = TransactionSigVerifier::new(non_vote_sender);
+            let verifier = TransactionSigVerifier::new(
+                non_vote_sender,
+                enable_block_production_forwarding.then(|| forward_stage_sender.clone()),
+            );
             SigVerifyStage::new(packet_receiver, verifier, "solSigVerTpu", "tpu-verifier")
         };
 
         let vote_sigverify_stage = {
-            let verifier = TransactionSigVerifier::new_reject_non_vote(tpu_vote_sender);
+            let verifier = TransactionSigVerifier::new_reject_non_vote(
+                tpu_vote_sender,
+                Some(forward_stage_sender),
+            );
             SigVerifyStage::new(
                 vote_packet_receiver,
                 verifier,
@@ -260,10 +271,16 @@ impl Tpu {
             transaction_status_sender,
             replay_vote_sender,
             log_messages_bytes_limit,
-            connection_cache.clone(),
             bank_forks.clone(),
             prioritization_fee_cache,
-            enable_block_production_forwarding,
+        );
+
+        let forwarding_stage = ForwardingStage::spawn(
+            forward_stage_receiver,
+            connection_cache.clone(),
+            RootBankCache::new(bank_forks.clone()),
+            (cluster_info.clone(), poh_recorder.clone()),
+            DataBudget::default(),
         );
 
         let (entry_receiver, tpu_entry_notifier) =
@@ -298,6 +315,7 @@ impl Tpu {
                 sigverify_stage,
                 vote_sigverify_stage,
                 banking_stage,
+                forwarding_stage,
                 cluster_info_vote_listener,
                 broadcast_stage,
                 tpu_quic_t,
@@ -318,6 +336,7 @@ impl Tpu {
             self.vote_sigverify_stage.join(),
             self.cluster_info_vote_listener.join(),
             self.banking_stage.join(),
+            self.forwarding_stage.join(),
             self.staked_nodes_updater_service.join(),
             self.tpu_quic_t.join(),
             self.tpu_forwards_quic_t.join(),

docs/art/tpu.bob

@@ -1,19 +1,25 @@
 
-                                        .-------------.
-                                        | PoH Service |
-                                        `--------+----`
-                                            ^    |
-             .------------------------------|----|--------------------.
-             | TPU                          |    v                    |
-             |  .-------.  .-----------.  .-+-------.  .-----------.  |  .------------.
- .---------. |  | Fetch |  | SigVerify |  | Banking |  | Broadcast |  |  | Downstream |
- | Clients |--->| Stage |->| Stage     |->| Stage   |->| Stage     |---->| Validators |
- `---------` |  |       |  |           |  |         |  |           |  |  |            |
-             |  `-------`  `-----------`  `----+----`  `-----------`  |  `------------`
-             |                                 |                      |
-             `---------------------------------|----------------------`
-                                               |
-                                               v
-                                            .------.
-                                            | Bank |
-                                            `------`
+                                          .-------------.
+                                          | PoH Service |
+                                          `--------+----`
+                                              ^    |
+             .--------------------------------|----|--------------------.
+             | TPU                            |    v                    |
+             |  .-------.  .-----------.    .-+-------.  .-----------.  |  .------------.
+ .---------. |  | Fetch |  | SigVerify |    | Banking |  | Broadcast |  |  | Downstream |
+ | Clients |--->| Stage |->| Stage     |-+->| Stage   |->| Stage     |---->| Validators |
+ `---------` |  |       |  |           | |  |         |  |           |  |  |            |
+             |  `-------`  `-----------` |  `----+----`  `-----------`  |  `------------`
+             |                           v       |                      |
+             |                  .--------+---.   |                      |
+             |                  | Forwarding |   |                      |
+             |                  | Stage      |   |                      |
+             |                  |            |   |                      |
+             |                  `------------`   |                      |
+             |                                   |                      |
+             `-----------------------------------|----------------------`
+                                                 |
+                                                 v
+                                              .------.
+                                              | Bank |
+                                              `------`

docs/src/validator/tpu.md

@@ -31,9 +31,14 @@ transactions when running into this situation.
 to remove excessive packets before then filtering packets with invalid
 signatures by setting the packet's discard flag.
 
-* banking stage: decides whether to forward, hold or process packets
-received. Once it detects the node is the block producer it processes
-held packets and newly received packets with a Bank at the tip slot.
+* banking stage: receives and buffers packet when the node is close to
+becoming the leader. Once it detects the node is the block producer it
+processes held packets and newly received packets with a Bank at the tip slot.
+
+* forwarding stage: forwards received packets to a node that is or will soon
+be leader. Sorts packets by priority and forwards them. Non-vote transactions
+are only forwarded if the node has the option enabled (stake overrides) but
+will always forward tpu votes.
 
 * broadcast stage: receives the valid transactions formed into Entry's from
 banking stage and packages them into shreds to send to network peers through

vortexor/src/vortexor.rs

@@ -96,7 +96,7 @@ impl Vortexor {
         tpu_receiver: Receiver<solana_perf::packet::PacketBatch>,
         non_vote_sender: TracedSender,
     ) -> SigVerifyStage {
-        let verifier = TransactionSigVerifier::new(non_vote_sender);
+        let verifier = TransactionSigVerifier::new(non_vote_sender, None);
         SigVerifyStage::new(
             tpu_receiver,
             verifier,