ci: add packets verification to Github Actions (#5335)

test verify packets

.github/workflows/verify-packets.yml

@@ -0,0 +1,44 @@
+name: Verify Packets
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths:
+      - .github/workflows/verify-packets.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gossip:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install required packages
+        run: |
+          sudo apt update
+          sudo apt install -y \
+            libclang-dev \
+            libprotobuf-dev \
+            libssl-dev \
+            libudev-dev \
+            pkg-config \
+            zlib1g-dev \
+            llvm \
+            clang \
+            cmake \
+            make \
+            protobuf-compiler \
+            git-lfs
+
+      - name: Run packet verify
+        run: |
+          ./ci/test-verify-packets-gossip.sh

Cargo.lock

@@ -1330,6 +1330,15 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
+[[package]]
+name = "byteorder_slice"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b294e30387378958e8bf8f4242131b930ea615ff81e8cac2440cea0a6013190"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "bytes"
 version = "1.10.0"
@@ -2083,6 +2092,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive-into-owned"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d94d81e3819a7b06a8638f448bc6339371ca9b6076a99d4a43eece3c4c923"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "derive-where"
 version = "1.2.7"
@@ -3134,6 +3154,12 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
+[[package]]
+name = "hxdmp"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a17b27f28a7466846baca75f0a5244e546e44178eb7f1c07a3820f413e91c6b0"
+
 [[package]]
 name = "hyper"
 version = "0.14.32"
@@ -4569,6 +4595,17 @@ dependencies = [
  "digest 0.10.7",
 ]
 
+[[package]]
+name = "pcap-file"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc1f139757b058f9f37b76c48501799d12c9aa0aa4c0d4c980b062ee925d1b2"
+dependencies = [
+ "byteorder_slice",
+ "derive-into-owned",
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "pem"
 version = "1.1.1"
@@ -8005,6 +8042,7 @@ dependencies = [
 name = "solana-gossip"
 version = "2.3.0"
 dependencies = [
+ "anyhow",
  "arrayvec",
  "assert_matches",
  "bincode",
@@ -8251,6 +8289,7 @@ dependencies = [
 name = "solana-ledger"
 version = "2.3.0"
 dependencies = [
+ "anyhow",
  "assert_matches",
  "bincode",
  "bitflags 2.9.0",
@@ -8296,6 +8335,7 @@ dependencies = [
  "solana-logger",
  "solana-measure",
  "solana-metrics",
+ "solana-net-utils",
  "solana-perf",
  "solana-program-runtime",
  "solana-pubkey",
@@ -8585,9 +8625,11 @@ dependencies = [
  "bincode",
  "bytes",
  "clap 3.2.23",
+ "hxdmp",
  "itertools 0.12.1",
  "log",
  "nix",
+ "pcap-file",
  "rand 0.8.5",
  "serde",
  "serde_derive",

ci/test-verify-packets-gossip.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+if ! git lfs --version &>/dev/null; then
+  echo "Git LFS is not installed. Please install Git LFS to proceed."
+  exit 1
+fi
+
+rm -rf "$here"/solana-packets
+git clone https://github.com/anza-xyz/solana-packets.git "$here"/solana-packets
+GOSSIP_WIRE_FORMAT_PACKETS="$here/solana-packets/GOSSIP_PACKETS" cargo test --package solana-gossip -- wire_format_tests::tests::test_gossip_wire_format --exact --show-output

gossip/Cargo.toml

@@ -76,12 +76,14 @@ static_assertions = { workspace = true }
 thiserror = { workspace = true }
 
 [dev-dependencies]
+anyhow = { workspace = true }
 bs58 = { workspace = true }
 criterion = { workspace = true }
 num_cpus = { workspace = true }
 rand0-7 = { workspace = true }
 rand_chacha0-2 = { workspace = true }
 serial_test = { workspace = true }
+solana-net-utils = { workspace = true, features = ["dev-context-only-utils"] }
 solana-perf = { workspace = true, features = ["dev-context-only-utils"] }
 solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }
 solana-sdk = { workspace = true }

gossip/src/lib.rs

@@ -46,3 +46,5 @@ extern crate solana_frozen_abi_macro;
 
 #[macro_use]
 extern crate solana_metrics;
+
+mod wire_format_tests;

gossip/src/wire_format_tests.rs

@@ -0,0 +1,65 @@
+#![allow(clippy::arithmetic_side_effects)]
+
+#[cfg(test)]
+mod tests {
+
+    use {
+        crate::protocol::Protocol,
+        serde::Serialize,
+        solana_net_utils::tooling_for_tests::{hexdump, validate_packet_format},
+        solana_sanitize::Sanitize,
+        std::path::PathBuf,
+    };
+
+    fn parse_gossip(bytes: &[u8]) -> anyhow::Result<Protocol> {
+        let pkt: Protocol = solana_perf::packet::deserialize_from_with_limit(bytes)?;
+        pkt.sanitize()?;
+        Ok(pkt)
+    }
+
+    fn serialize<T: Serialize>(pkt: T) -> Vec<u8> {
+        bincode::serialize(&pkt).unwrap()
+    }
+
+    fn find_differences(a: &[u8], b: &[u8]) -> Option<usize> {
+        if a.len() != b.len() {
+            return Some(a.len().min(b.len()));
+        }
+        for (idx, (e1, e2)) in a.iter().zip(b).enumerate() {
+            if e1 != e2 {
+                return Some(idx);
+            }
+        }
+        None
+    }
+
+    /// Test the ability of gossip parsers to understand and re-serialize a corpus of
+    /// packets captured from mainnet.
+    ///
+    /// This test requires external files and is not run by default.
+    /// Export the "GOSSIP_WIRE_FORMAT_PACKETS" variable to run this test
+    #[test]
+    fn test_gossip_wire_format() {
+        solana_logger::setup();
+        let path_base = match std::env::var_os("GOSSIP_WIRE_FORMAT_PACKETS") {
+            Some(p) => PathBuf::from(p),
+            None => {
+                eprintln!("Test requires GOSSIP_WIRE_FORMAT_PACKETS env variable, skipping!");
+                return;
+            }
+        };
+        for entry in
+            std::fs::read_dir(path_base).expect("Expecting env var to point to a directory")
+        {
+            let entry = entry.expect("Expecting a readable file");
+            validate_packet_format(
+                &entry.path(),
+                parse_gossip,
+                serialize,
+                hexdump,
+                find_differences,
+            )
+            .unwrap();
+        }
+    }
+}

ledger/Cargo.toml

@@ -10,6 +10,7 @@ license = { workspace = true }
 edition = { workspace = true }
 
 [dependencies]
+anyhow = { workspace = true }
 assert_matches = { workspace = true }
 bincode = { workspace = true }
 bitflags = { workspace = true, features = ["serde"] }
@@ -55,6 +56,7 @@ solana-frozen-abi-macro = { workspace = true, optional = true, features = [
 ] }
 solana-measure = { workspace = true }
 solana-metrics = { workspace = true }
+solana-net-utils = { workspace = true }
 solana-perf = { workspace = true }
 solana-program-runtime = { workspace = true, features = ["metrics"] }
 solana-pubkey = { workspace = true }
@@ -96,6 +98,7 @@ bs58 = { workspace = true }
 criterion = { workspace = true }
 solana-account-decoder = { workspace = true }
 solana-logger = { workspace = true }
+solana-net-utils = { workspace = true, features = ["dev-context-only-utils"] }
 solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }
 solana-vote = { workspace = true, features = ["dev-context-only-utils"] }
 spl-pod = { workspace = true }

ledger/src/lib.rs

@@ -51,3 +51,5 @@ extern crate solana_frozen_abi_macro;
 pub mod macro_reexports {
     pub use solana_accounts_db::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE;
 }
+
+mod wire_format_tests;

ledger/src/wire_format_tests.rs

@@ -0,0 +1,93 @@
+#![allow(clippy::arithmetic_side_effects)]
+
+#[cfg(test)]
+mod tests {
+    use {
+        crate::shred::Shred,
+        solana_net_utils::tooling_for_tests::{hexdump, validate_packet_format},
+        std::path::PathBuf,
+    };
+
+    fn parse_turbine(bytes: &[u8]) -> anyhow::Result<Shred> {
+        let shred = Shred::new_from_serialized_shred(bytes.to_owned())
+            .map_err(|_e| anyhow::anyhow!("Can not deserialize"))?;
+        shred
+            .sanitize()
+            .map_err(|_e| anyhow::anyhow!("Failed sanitize"))?;
+        Ok(shred)
+    }
+
+    fn serialize(pkt: Shred) -> Vec<u8> {
+        pkt.payload().to_vec()
+    }
+
+    fn find_differences(a: &[u8], b: &[u8]) -> Option<usize> {
+        if a.len() != b.len() {
+            return Some(a.len());
+        }
+        for (idx, (e1, e2)) in a.iter().zip(b).enumerate() {
+            if e1 != e2 {
+                return Some(idx);
+            }
+        }
+        None
+    }
+
+    fn show_packet(bytes: &[u8]) -> anyhow::Result<()> {
+        let shred = parse_turbine(bytes)?;
+        let merkle_root = shred.merkle_root();
+        let chained_merkle_root = shred.chained_merkle_root();
+        let rtx_sign = shred.retransmitter_signature();
+
+        println!("=== {} bytes ===", bytes.len());
+        println!(
+            "Shred ID={ID:?} ErasureSetID={ESI:?}",
+            ID = shred.id(),
+            ESI = shred.erasure_set()
+        );
+        println!(
+            "Shred merkle root {:X?}, chained root {:X?}, rtx_sign {:X?}",
+            merkle_root.map(|v| v.as_ref().to_vec()),
+            chained_merkle_root.map(|v| v.as_ref().to_vec()),
+            rtx_sign.map(|v| v.as_ref().to_vec())
+        );
+        println!(
+            "Data shreds: {:?}, Coding shreds: {:?}",
+            shred.num_data_shreds(),
+            shred.num_coding_shreds()
+        );
+        hexdump(bytes)?;
+        println!("===");
+        Ok(())
+    }
+
+    /// Test the ability of turbine parser to understand and re-serialize a corpus of
+    /// packets captured from mainnet.
+    ///
+    /// This test requires external files and is not run by default.
+    /// Export the "TURBINE_WIRE_FORMAT_PACKETS" env variable to run this test.
+    #[test]
+    fn test_turbine_wire_format() {
+        solana_logger::setup();
+        let path_base = match std::env::var_os("TURBINE_WIRE_FORMAT_PACKETS") {
+            Some(p) => PathBuf::from(p),
+            None => {
+                eprintln!("Test requires TURBINE_WIRE_FORMAT_PACKETS env variable, skipping!");
+                return;
+            }
+        };
+        for entry in
+            std::fs::read_dir(path_base).expect("Expecting env var to point to a directory")
+        {
+            let entry = entry.expect("Expecting a readable file");
+            validate_packet_format(
+                &entry.path(),
+                parse_turbine,
+                serialize,
+                show_packet,
+                find_differences,
+            )
+            .unwrap();
+        }
+    }
+}

net-utils/Cargo.toml

@@ -15,9 +15,11 @@ anyhow = { workspace = true }
 bincode = { workspace = true }
 bytes = { workspace = true }
 clap = { version = "3.1.5", features = ["cargo"], optional = true }
+hxdmp = { version = "0.2.1", optional = true }
 itertools = { workspace = true }
 log = { workspace = true }
 nix = { workspace = true, features = ["socket"] }
+pcap-file = { version = "2.0.0", optional = true }
 rand = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }
@@ -34,7 +36,7 @@ solana-logger = { workspace = true }
 [features]
 default = []
 clap = ["dep:clap", "dep:solana-logger", "dep:solana-version"]
-dev-context-only-utils = []
+dev-context-only-utils = ["dep:pcap-file", "dep:hxdmp"]
 
 [lib]
 name = "solana_net_utils"

net-utils/src/lib.rs

@@ -2,6 +2,9 @@
 mod ip_echo_client;
 mod ip_echo_server;
 
+#[cfg(feature = "dev-context-only-utils")]
+pub mod tooling_for_tests;
+
 pub use ip_echo_server::{
     ip_echo_server, IpEchoServer, DEFAULT_IP_ECHO_SERVER_THREADS, MAX_PORT_COUNT_PER_MESSAGE,
     MINIMUM_IP_ECHO_SERVER_THREADS,

net-utils/src/tooling_for_tests.rs

@@ -0,0 +1,122 @@
+#![allow(clippy::arithmetic_side_effects)]
+use {
+    anyhow::Context,
+    log::{debug, error, info},
+    pcap_file::pcapng::PcapNgReader,
+    std::{fs::File, io::Write, path::PathBuf},
+};
+
+/// Prints a hexdump of a given byte buffer into stderr
+pub fn hexdump(bytes: &[u8]) -> anyhow::Result<()> {
+    hxdmp::hexdump(bytes, &mut std::io::stderr())?;
+    std::io::stderr().write_all(b"\n")?;
+    Ok(())
+}
+
+/// Reads all packets from PCAPNG file
+pub struct PcapReader {
+    reader: PcapNgReader<File>,
+}
+
+impl PcapReader {
+    pub fn new(filename: &PathBuf) -> anyhow::Result<Self> {
+        let file_in = File::open(filename).with_context(|| format!("opening file {filename:?}"))?;
+        let reader = PcapNgReader::new(file_in).context("pcap reader creation")?;
+
+        Ok(PcapReader { reader })
+    }
+}
+
+impl Iterator for PcapReader {
+    type Item = Vec<u8>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            let block = match self.reader.next_block() {
+                Some(block) => block.ok()?,
+                None => return None,
+            };
+            let data = match block {
+                pcap_file::pcapng::Block::Packet(ref block) => {
+                    &block.data[0..block.original_len as usize]
+                }
+                pcap_file::pcapng::Block::SimplePacket(ref block) => {
+                    &block.data[0..block.original_len as usize]
+                }
+                pcap_file::pcapng::Block::EnhancedPacket(ref block) => {
+                    &block.data[0..block.original_len as usize]
+                }
+                _ => {
+                    debug!("Skipping unknown block in pcap file");
+                    continue;
+                }
+            };
+
+            let pkt_payload = data;
+            return Some(pkt_payload.to_vec());
+        }
+    }
+}
+
+/// Helper function to validate packet parsing capabilities across agave.
+/// It will read all packets from file identified by `filename`, then parse them
+/// using parse_packet, re-serialize using `serialize_packet`, and finally compare
+/// whether the original packet matches the reserialized version.
+/// If parser returns errors, the test fails and offending packet is reported.
+/// If any differences are found using `custom_compare`, they are reported as errors.
+///
+/// Note that no matter how many packets are present in a given file, one can never be 100%
+/// certain this will catch all wire format issues.
+pub fn validate_packet_format<T>(
+    filename: &PathBuf,
+    parse_packet: fn(&[u8]) -> anyhow::Result<T>,
+    serialize_packet: fn(T) -> Vec<u8>,
+    show_packet: fn(&[u8]) -> anyhow::Result<()>,
+    custom_compare: fn(&[u8], &[u8]) -> Option<usize>,
+) -> anyhow::Result<usize>
+where
+    T: Sized,
+{
+    info!(
+        "Validating packet format for {} using samples from {filename:?}",
+        std::any::type_name::<T>()
+    );
+    let reader = PcapReader::new(filename)?;
+    let mut number = 0;
+    let mut errors = 0;
+    for data in reader.into_iter() {
+        number += 1;
+        match parse_packet(&data) {
+            Ok(pkt) => {
+                let reconstructed_bytes = serialize_packet(pkt);
+                let diff = custom_compare(&reconstructed_bytes, &data);
+                if let Some(pos) = diff {
+                    errors += 1;
+                    error!(
+                        "Reserialization differences found for packet {number} in {filename:?}!"
+                    );
+                    error!("Differences start at byte {pos}");
+                    error!("Original packet:");
+                    show_packet(&data)?;
+                    error!("Reserialized:");
+                    show_packet(&reconstructed_bytes)?;
+                    break;
+                }
+            }
+            Err(e) => {
+                errors += 1;
+                error!("Found packet {number} that failed to parse with error {e}");
+                error!("Problematic packet:");
+                show_packet(&data)?;
+                break;
+            }
+        }
+    }
+    if errors > 0 {
+        error!("Packet format checks passed for {number} packets, failed for {errors} packets.");
+        Err(anyhow::anyhow!("Failed checks for {errors} packets"))
+    } else {
+        info!("Packet format checks passed for {number} packets.");
+        Ok(number)
+    }
+}

programs/sbf/Cargo.lock

@@ -6483,6 +6483,7 @@ dependencies = [
 name = "solana-ledger"
 version = "2.3.0"
 dependencies = [
+ "anyhow",
  "assert_matches",
  "bincode",
  "bitflags 2.9.0",
@@ -6523,6 +6524,7 @@ dependencies = [
  "solana-feature-set",
  "solana-measure",
  "solana-metrics",
+ "solana-net-utils",
  "solana-perf",
  "solana-program-runtime",
  "solana-pubkey",

svm/examples/Cargo.lock

@@ -6307,6 +6307,7 @@ dependencies = [
 name = "solana-ledger"
 version = "2.3.0"
 dependencies = [
+ "anyhow",
  "assert_matches",
  "bincode",
  "bitflags 2.9.0",
@@ -6347,6 +6348,7 @@ dependencies = [
  "solana-feature-set",
  "solana-measure",
  "solana-metrics",
+ "solana-net-utils",
  "solana-perf",
  "solana-program-runtime",
  "solana-pubkey",