v2.0: ledger-tool: Make blockstore slot functional with no tx metadata (backport of #2423) (#3887)

ledger-tool: Make blockstore slot functional with no tx metadata (#2423)

A previous commit unified the code to output a slot between the
bigtable block and blockstore slot commands. In doing so, support for
blockstore slot when tx metadata is absent was unintentionally broken

This re-adds support for using the blockstore slot command when the
blockstore does not contain tx metadata

(cherry picked from commit ee0667d36932a46ee2e2e36419567479f202a8f0)

Co-authored-by: steviez <steven@anza.xyz>

ledger-tool/src/blockstore.rs

@@ -1024,7 +1024,6 @@ fn do_blockstore_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) -
             let blockstore =
                 crate::open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
             for slot in slots {
-                println!("Slot {slot}");
                 output_slot(
                     &blockstore,
                     slot,

ledger-tool/src/output.rs

@@ -20,11 +20,12 @@ use {
         hash::Hash,
         native_token::lamports_to_sol,
         pubkey::Pubkey,
+        transaction::VersionedTransaction,
     },
     solana_transaction_status::{
-        BlockEncodingOptions, ConfirmedBlock, EncodeError, EncodedConfirmedBlock,
+        BlockEncodingOptions, ConfirmedBlock, Encodable, EncodedConfirmedBlock,
         EncodedTransactionWithStatusMeta, EntrySummary, Rewards, TransactionDetails,
-        UiTransactionEncoding, VersionedConfirmedBlockWithEntries,
+        UiTransactionEncoding, VersionedConfirmedBlock, VersionedConfirmedBlockWithEntries,
         VersionedTransactionWithStatusMeta,
     },
     std::{
@@ -348,24 +349,82 @@ impl EncodedConfirmedBlockWithEntries {
 pub(crate) fn encode_confirmed_block(
     confirmed_block: ConfirmedBlock,
 ) -> Result<EncodedConfirmedBlock> {
-    let encoded_block = confirmed_block
-        .encode_with_options(
-            UiTransactionEncoding::Base64,
-            BlockEncodingOptions {
-                transaction_details: TransactionDetails::Full,
-                show_rewards: true,
-                max_supported_transaction_version: Some(0),
-            },
-        )
-        .map_err(|err| match err {
-            EncodeError::UnsupportedTransactionVersion(version) => LedgerToolError::Generic(
-                format!("Failed to process unsupported transaction version ({version}) in block"),
-            ),
-        })?;
+    let encoded_block = confirmed_block.encode_with_options(
+        UiTransactionEncoding::Base64,
+        BlockEncodingOptions {
+            transaction_details: TransactionDetails::Full,
+            show_rewards: true,
+            max_supported_transaction_version: Some(0),
+        },
+    )?;
+
     let encoded_block: EncodedConfirmedBlock = encoded_block.into();
     Ok(encoded_block)
 }
 
+fn encode_versioned_transactions(block: BlockWithoutMetadata) -> EncodedConfirmedBlock {
+    let transactions = block
+        .transactions
+        .into_iter()
+        .map(|transaction| EncodedTransactionWithStatusMeta {
+            transaction: transaction.encode(UiTransactionEncoding::Base64),
+            meta: None,
+            version: None,
+        })
+        .collect();
+
+    EncodedConfirmedBlock {
+        previous_blockhash: Hash::default().to_string(),
+        blockhash: block.blockhash,
+        parent_slot: block.parent_slot,
+        transactions,
+        rewards: Rewards::default(),
+        num_partitions: None,
+        block_time: None,
+        block_height: None,
+    }
+}
+
+pub enum BlockContents {
+    VersionedConfirmedBlock(VersionedConfirmedBlock),
+    BlockWithoutMetadata(BlockWithoutMetadata),
+}
+
+// A VersionedConfirmedBlock analogue for use when the transaction metadata
+// fields are unavailable. Also supports non-full blocks
+pub struct BlockWithoutMetadata {
+    pub blockhash: String,
+    pub parent_slot: Slot,
+    pub transactions: Vec<VersionedTransaction>,
+}
+
+impl BlockContents {
+    pub fn transactions(&self) -> Box<dyn Iterator<Item = &VersionedTransaction> + '_> {
+        match self {
+            BlockContents::VersionedConfirmedBlock(block) => Box::new(
+                block
+                    .transactions
+                    .iter()
+                    .map(|VersionedTransactionWithStatusMeta { transaction, .. }| transaction),
+            ),
+            BlockContents::BlockWithoutMetadata(block) => Box::new(block.transactions.iter()),
+        }
+    }
+}
+
+impl TryFrom<BlockContents> for EncodedConfirmedBlock {
+    type Error = LedgerToolError;
+
+    fn try_from(block_contents: BlockContents) -> Result<Self> {
+        match block_contents {
+            BlockContents::VersionedConfirmedBlock(block) => {
+                encode_confirmed_block(ConfirmedBlock::from(block))
+            }
+            BlockContents::BlockWithoutMetadata(block) => Ok(encode_versioned_transactions(block)),
+        }
+    }
+}
+
 pub fn output_slot(
     blockstore: &Blockstore,
     slot: Slot,
@@ -374,26 +433,77 @@ pub fn output_slot(
     verbose_level: u64,
     all_program_ids: &mut HashMap<Pubkey, u64>,
 ) -> Result<()> {
-    if blockstore.is_dead(slot) {
-        if allow_dead_slots {
-            if *output_format == OutputFormat::Display {
-                println!(" Slot is dead");
-            }
-        } else {
-            return Err(LedgerToolError::from(BlockstoreError::DeadSlot));
+    let is_root = blockstore.is_root(slot);
+    let is_dead = blockstore.is_dead(slot);
+    if *output_format == OutputFormat::Display && verbose_level <= 1 {
+        if is_root && is_dead {
+            eprintln!("Slot {slot} is marked as both a root and dead, this shouldn't be possible");
         }
+        println!(
+            "Slot {slot}{}",
+            if is_root {
+                " (root)"
+            } else if is_dead {
+                " (dead)"
+            } else {
+                ""
+            }
+        );
+    }
+
+    if is_dead && !allow_dead_slots {
+        return Err(LedgerToolError::from(BlockstoreError::DeadSlot));
     }
 
     let Some(meta) = blockstore.meta(slot)? else {
         return Ok(());
     };
-    let VersionedConfirmedBlockWithEntries { block, entries } = blockstore
-        .get_complete_block_with_entries(
-            slot,
-            /*require_previous_blockhash:*/ false,
-            /*populate_entries:*/ true,
-            allow_dead_slots,
-        )?;
+    let (block_contents, entries) = match blockstore.get_complete_block_with_entries(
+        slot,
+        /*require_previous_blockhash:*/ false,
+        /*populate_entries:*/ true,
+        allow_dead_slots,
+    ) {
+        Ok(VersionedConfirmedBlockWithEntries { block, entries }) => {
+            (BlockContents::VersionedConfirmedBlock(block), entries)
+        }
+        Err(_) => {
+            // Transaction metadata could be missing, try to fetch just the
+            // entries and leave the metadata fields empty
+            let entries = blockstore.get_slot_entries(slot, /*shred_start_index:*/ 0)?;
+
+            let blockhash = entries
+                .last()
+                .filter(|_| meta.is_full())
+                .map(|entry| entry.hash)
+                .unwrap_or(Hash::default());
+            let parent_slot = meta.parent_slot.unwrap_or(0);
+
+            let mut entry_summaries = Vec::with_capacity(entries.len());
+            let mut starting_transaction_index = 0;
+            let transactions = entries
+                .into_iter()
+                .flat_map(|entry| {
+                    entry_summaries.push(EntrySummary {
+                        num_hashes: entry.num_hashes,
+                        hash: entry.hash,
+                        num_transactions: entry.transactions.len() as u64,
+                        starting_transaction_index,
+                    });
+                    starting_transaction_index += entry.transactions.len();
+
+                    entry.transactions
+                })
+                .collect();
+
+            let block = BlockWithoutMetadata {
+                blockhash: blockhash.to_string(),
+                parent_slot,
+                transactions,
+            };
+            (BlockContents::BlockWithoutMetadata(block), entry_summaries)
+        }
+    };
 
     if verbose_level == 0 {
         if *output_format == OutputFormat::Display {
@@ -417,24 +527,23 @@ pub fn output_slot(
             for entry in entries.iter() {
                 num_hashes += entry.num_hashes;
             }
+            let blockhash = entries
+                .last()
+                .filter(|_| meta.is_full())
+                .map(|entry| entry.hash)
+                .unwrap_or(Hash::default());
 
-            let blockhash = if let Some(entry) = entries.last() {
-                entry.hash
-            } else {
-                Hash::default()
-            };
-
-            let transactions = block.transactions.len();
+            let mut num_transactions = 0;
             let mut program_ids = HashMap::new();
-            for VersionedTransactionWithStatusMeta { transaction, .. } in block.transactions.iter()
-            {
+
+            for transaction in block_contents.transactions() {
+                num_transactions += 1;
                 for program_id in get_program_ids(transaction) {
                     *program_ids.entry(*program_id).or_insert(0) += 1;
                 }
             }
-
             println!(
-                "  Transactions: {transactions}, hashes: {num_hashes}, block_hash: {blockhash}",
+                "  Transactions: {num_transactions}, hashes: {num_hashes}, block_hash: {blockhash}",
             );
             for (pubkey, count) in program_ids.iter() {
                 *all_program_ids.entry(*pubkey).or_insert(0) += count;
@@ -443,7 +552,7 @@ pub fn output_slot(
             output_sorted_program_ids(program_ids);
         }
     } else {
-        let encoded_block = encode_confirmed_block(ConfirmedBlock::from(block))?;
+        let encoded_block = EncodedConfirmedBlock::try_from(block_contents)?;
         let cli_block = CliBlockWithEntries {
             encoded_confirmed_block: EncodedConfirmedBlockWithEntries::try_from(
                 encoded_block,
@@ -477,7 +586,7 @@ pub fn output_ledger(
     let num_slots = num_slots.unwrap_or(Slot::MAX);
     let mut num_printed = 0;
     let mut all_program_ids = HashMap::new();
-    for (slot, slot_meta) in slot_iterator {
+    for (slot, _slot_meta) in slot_iterator {
         if only_rooted && !blockstore.is_root(slot) {
             continue;
         }
@@ -485,17 +594,6 @@ pub fn output_ledger(
             break;
         }
 
-        match output_format {
-            OutputFormat::Display => {
-                println!("Slot {} root?: {}", slot, blockstore.is_root(slot))
-            }
-            OutputFormat::Json => {
-                serde_json::to_writer(stdout(), &slot_meta)?;
-                stdout().write_all(b",\n")?;
-            }
-            _ => unreachable!(),
-        }
-
         if let Err(err) = output_slot(
             &blockstore,
             slot,

ledger-tool/tests/basic.rs

@@ -101,7 +101,6 @@ fn ledger_tool_copy_test(src_shred_compaction: &str, dst_shred_compaction: &str)
         assert!(src_slot_output.status.success());
         assert!(dst_slot_output.status.success());
         assert!(!src_slot_output.stdout.is_empty());
-        assert_eq!(src_slot_output.stdout, dst_slot_output.stdout);
     }
 }
 

transaction-status/src/lib.rs

@@ -27,7 +27,7 @@ use {
         },
         transaction_context::TransactionReturnData,
     },
-    std::fmt,
+    std::{collections::HashSet, fmt},
     thiserror::Error,
 };
 
@@ -1136,6 +1136,38 @@ impl EncodableWithMeta for VersionedTransaction {
     }
 }
 
+impl Encodable for VersionedTransaction {
+    type Encoded = EncodedTransaction;
+    fn encode(&self, encoding: UiTransactionEncoding) -> Self::Encoded {
+        match encoding {
+            UiTransactionEncoding::Binary => EncodedTransaction::LegacyBinary(
+                bs58::encode(bincode::serialize(self).unwrap()).into_string(),
+            ),
+            UiTransactionEncoding::Base58 => EncodedTransaction::Binary(
+                bs58::encode(bincode::serialize(self).unwrap()).into_string(),
+                TransactionBinaryEncoding::Base58,
+            ),
+            UiTransactionEncoding::Base64 => EncodedTransaction::Binary(
+                BASE64_STANDARD.encode(bincode::serialize(self).unwrap()),
+                TransactionBinaryEncoding::Base64,
+            ),
+            UiTransactionEncoding::Json | UiTransactionEncoding::JsonParsed => {
+                EncodedTransaction::Json(UiTransaction {
+                    signatures: self.signatures.iter().map(ToString::to_string).collect(),
+                    message: match &self.message {
+                        VersionedMessage::Legacy(message) => {
+                            message.encode(UiTransactionEncoding::JsonParsed)
+                        }
+                        VersionedMessage::V0(message) => {
+                            message.encode(UiTransactionEncoding::JsonParsed)
+                        }
+                    },
+                })
+            }
+        }
+    }
+}
+
 impl Encodable for Transaction {
     type Encoded = EncodedTransaction;
     fn encode(&self, encoding: UiTransactionEncoding) -> Self::Encoded {
@@ -1240,6 +1272,40 @@ impl Encodable for Message {
     }
 }
 
+impl Encodable for v0::Message {
+    type Encoded = UiMessage;
+    fn encode(&self, encoding: UiTransactionEncoding) -> Self::Encoded {
+        if encoding == UiTransactionEncoding::JsonParsed {
+            let account_keys = AccountKeys::new(&self.account_keys, None);
+            let loaded_addresses = LoadedAddresses::default();
+            let loaded_message =
+                LoadedMessage::new_borrowed(self, &loaded_addresses, &HashSet::new());
+            UiMessage::Parsed(UiParsedMessage {
+                account_keys: parse_v0_message_accounts(&loaded_message),
+                recent_blockhash: self.recent_blockhash.to_string(),
+                instructions: self
+                    .instructions
+                    .iter()
+                    .map(|instruction| UiInstruction::parse(instruction, &account_keys, None))
+                    .collect(),
+                address_table_lookups: None,
+            })
+        } else {
+            UiMessage::Raw(UiRawMessage {
+                header: self.header,
+                account_keys: self.account_keys.iter().map(ToString::to_string).collect(),
+                recent_blockhash: self.recent_blockhash.to_string(),
+                instructions: self
+                    .instructions
+                    .iter()
+                    .map(|ix| UiCompiledInstruction::from(ix, None))
+                    .collect(),
+                address_table_lookups: None,
+            })
+        }
+    }
+}
+
 impl EncodableWithMeta for v0::Message {
     type Encoded = UiMessage;
     fn encode_with_meta(