feat(fortuna): add metrics for retry count, gas/fee multipliers, and fee estimates (#2258)

* feat(fortuna): add metrics for retry count, gas/fee multipliers, and fee estimates

Added histogram metrics to track:
- Number of retries for successful transactions
- Final gas multiplier for successful transactions
- Final fee multiplier for successful transactions
- Priority fee estimate before thresholds
- Estimated keeper fee before thresholds

Co-Authored-By: Jayant Krishnamurthy <jayant@dourolabs.xyz>

* cleanup

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Jayant Krishnamurthy <jayant@dourolabs.xyz>
Co-authored-by: Jayant Krishnamurthy <jayantkrishnamurthy@gmail.com>

apps/fortuna/Cargo.lock

@@ -1503,7 +1503,7 @@ dependencies = [
 
 [[package]]
 name = "fortuna"
-version = "7.2.0"
+version = "7.3.0"
 dependencies = [
  "anyhow",
  "axum",

apps/fortuna/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fortuna"
-version = "7.2.0"
+version = "7.3.0"
 edition = "2021"
 
 [dependencies]

apps/fortuna/src/keeper.rs

@@ -69,6 +69,7 @@ pub struct KeeperMetrics {
     pub balance: Family<AccountLabel, Gauge<f64, AtomicU64>>,
     pub collected_fee: Family<AccountLabel, Gauge<f64, AtomicU64>>,
     pub current_fee: Family<AccountLabel, Gauge<f64, AtomicU64>>,
+    pub target_provider_fee: Family<AccountLabel, Gauge<f64, AtomicU64>>,
     pub total_gas_spent: Family<AccountLabel, Gauge<f64, AtomicU64>>,
     pub total_gas_fee_spent: Family<AccountLabel, Gauge<f64, AtomicU64>>,
     pub requests: Family<AccountLabel, Counter>,
@@ -78,6 +79,10 @@ pub struct KeeperMetrics {
     pub requests_reprocessed: Family<AccountLabel, Counter>,
     pub reveals: Family<AccountLabel, Counter>,
     pub request_duration_ms: Family<AccountLabel, Histogram>,
+    pub retry_count: Family<AccountLabel, Histogram>,
+    pub final_gas_multiplier: Family<AccountLabel, Histogram>,
+    pub final_fee_multiplier: Family<AccountLabel, Histogram>,
+    pub gas_price_estimate: Family<AccountLabel, Gauge<f64, AtomicU64>>,
 }
 
 impl Default for KeeperMetrics {
@@ -88,6 +93,7 @@ impl Default for KeeperMetrics {
             balance: Family::default(),
             collected_fee: Family::default(),
             current_fee: Family::default(),
+            target_provider_fee: Family::default(),
             total_gas_spent: Family::default(),
             total_gas_fee_spent: Family::default(),
             requests: Family::default(),
@@ -105,6 +111,18 @@ impl Default for KeeperMetrics {
                     .into_iter(),
                 )
             }),
+            retry_count: Family::new_with_constructor(|| {
+                Histogram::new(vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 15.0, 20.0].into_iter())
+            }),
+            final_gas_multiplier: Family::new_with_constructor(|| {
+                Histogram::new(
+                    vec![100.0, 125.0, 150.0, 200.0, 300.0, 400.0, 500.0, 600.0].into_iter(),
+                )
+            }),
+            final_fee_multiplier: Family::new_with_constructor(|| {
+                Histogram::new(vec![100.0, 110.0, 120.0, 140.0, 160.0, 180.0, 200.0].into_iter())
+            }),
+            gas_price_estimate: Family::default(),
         }
     }
 }
@@ -174,6 +192,12 @@ impl KeeperMetrics {
             keeper_metrics.current_fee.clone(),
         );
 
+        writable_registry.register(
+            "target_provider_fee",
+            "Target fee in ETH -- differs from current_fee in that this is the goal, and current_fee is the on-chain value.",
+            keeper_metrics.target_provider_fee.clone(),
+        );
+
         writable_registry.register(
             "total_gas_spent",
             "Total gas spent revealing requests",
@@ -198,6 +222,30 @@ impl KeeperMetrics {
             keeper_metrics.request_duration_ms.clone(),
         );
 
+        writable_registry.register(
+            "retry_count",
+            "Number of retries for successful transactions",
+            keeper_metrics.retry_count.clone(),
+        );
+
+        writable_registry.register(
+            "final_gas_multiplier",
+            "Final gas multiplier percentage for successful transactions",
+            keeper_metrics.final_gas_multiplier.clone(),
+        );
+
+        writable_registry.register(
+            "final_fee_multiplier",
+            "Final fee multiplier percentage for successful transactions",
+            keeper_metrics.final_fee_multiplier.clone(),
+        );
+
+        writable_registry.register(
+            "gas_price_estimate",
+            "Gas price estimate for the blockchain (in gwei)",
+            keeper_metrics.gas_price_estimate.clone(),
+        );
+
         keeper_metrics
     }
 }
@@ -327,6 +375,7 @@ pub async fn run_keeper_threads(
     spawn(
         adjust_fee_wrapper(
             contract.clone(),
+            chain_state.clone(),
             chain_state.provider_address,
             ADJUST_FEE_INTERVAL,
             chain_eth_config.legacy_tx,
@@ -346,6 +395,7 @@ pub async fn run_keeper_threads(
             u64::try_from(100 + chain_eth_config.max_profit_pct)
                 .expect("max_profit_pct must be >= -100"),
             chain_eth_config.fee,
+            metrics.clone(),
         )
         .in_current_span(),
     );
@@ -479,6 +529,25 @@ pub async fn process_event_with_backoff(
                 .request_duration_ms
                 .get_or_create(&account_label)
                 .observe(duration.as_millis() as f64);
+
+            // Track retry count, gas multiplier, and fee multiplier for successful transactions
+            let num_retries = num_retries.load(std::sync::atomic::Ordering::Relaxed);
+            metrics
+                .retry_count
+                .get_or_create(&account_label)
+                .observe(num_retries as f64);
+
+            let gas_multiplier = escalation_policy.get_gas_multiplier_pct(num_retries);
+            metrics
+                .final_gas_multiplier
+                .get_or_create(&account_label)
+                .observe(gas_multiplier as f64);
+
+            let fee_multiplier = escalation_policy.get_fee_multiplier_pct(num_retries);
+            metrics
+                .final_fee_multiplier
+                .get_or_create(&account_label)
+                .observe(fee_multiplier as f64);
         }
         Err(e) => {
             // In case the callback did not succeed, we double-check that the request is still on-chain.
@@ -1133,6 +1202,7 @@ pub async fn send_and_confirm(contract_call: PythContractCall) -> Result<()> {
 #[allow(clippy::too_many_arguments)]
 pub async fn adjust_fee_wrapper(
     contract: Arc<InstrumentedSignablePythContract>,
+    chain_state: BlockchainState,
     provider_address: Address,
     poll_interval: Duration,
     legacy_tx: bool,
@@ -1141,6 +1211,7 @@ pub async fn adjust_fee_wrapper(
     target_profit_pct: u64,
     max_profit_pct: u64,
     min_fee_wei: u128,
+    metrics: Arc<KeeperMetrics>,
 ) {
     // The maximum balance of accrued fees + provider wallet balance. None if we haven't observed a value yet.
     let mut high_water_pnl: Option<U256> = None;
@@ -1149,6 +1220,7 @@ pub async fn adjust_fee_wrapper(
     loop {
         if let Err(e) = adjust_fee_if_necessary(
             contract.clone(),
+            chain_state.id.clone(),
             provider_address,
             legacy_tx,
             gas_limit,
@@ -1158,6 +1230,7 @@ pub async fn adjust_fee_wrapper(
             min_fee_wei,
             &mut high_water_pnl,
             &mut sequence_number_of_last_fee_update,
+            metrics.clone(),
         )
         .in_current_span()
         .await
@@ -1232,6 +1305,7 @@ pub async fn update_commitments_if_necessary(
 #[allow(clippy::too_many_arguments)]
 pub async fn adjust_fee_if_necessary(
     contract: Arc<InstrumentedSignablePythContract>,
+    chain_id: ChainId,
     provider_address: Address,
     legacy_tx: bool,
     gas_limit: u64,
@@ -1241,6 +1315,7 @@ pub async fn adjust_fee_if_necessary(
     min_fee_wei: u128,
     high_water_pnl: &mut Option<U256>,
     sequence_number_of_last_fee_update: &mut Option<u64>,
+    metrics: Arc<KeeperMetrics>,
 ) -> Result<()> {
     let provider_info = contract
         .get_provider_info(provider_address)
@@ -1256,6 +1331,17 @@ pub async fn adjust_fee_if_necessary(
     let max_callback_cost: u128 = estimate_tx_cost(contract.clone(), legacy_tx, gas_limit.into())
         .await
         .map_err(|e| anyhow!("Could not estimate transaction cost. error {:?}", e))?;
+
+    let account_label = AccountLabel {
+        chain_id: chain_id.clone(),
+        address: provider_address.to_string(),
+    };
+
+    metrics
+        .gas_price_estimate
+        .get_or_create(&account_label)
+        .set((max_callback_cost / u128::from(gas_limit)) as f64 / 1e9);
+
     let target_fee_min = std::cmp::max(
         (max_callback_cost * u128::from(min_profit_pct)) / 100,
         min_fee_wei,
@@ -1264,6 +1350,11 @@ pub async fn adjust_fee_if_necessary(
         (max_callback_cost * u128::from(target_profit_pct)) / 100,
         min_fee_wei,
     );
+    metrics
+        .target_provider_fee
+        .get_or_create(&account_label)
+        .set(((max_callback_cost * u128::from(target_profit_pct)) / 100) as f64 / 1e18);
+
     let target_fee_max = std::cmp::max(
         (max_callback_cost * u128::from(max_profit_pct)) / 100,
         min_fee_wei,