浏览代码

Merge branch 'main' into pyth-stylus-update-and-initialize

Ayush Suresh 4 月之前
父节点
当前提交
e4ea5a083e
共有 100 个文件被更改,包括 3465 次插入791 次删除
  1. 37 0
      .github/workflows/ci-pyth-lazer-agent.yml
  2. 5 1
      .github/workflows/ci-starknet-contract.yml
  3. 55 0
      .github/workflows/docker-pyth-lazer-agent.yml
  4. 58 0
      .github/workflows/docker-quorum.yml
  5. 3 0
      README.md
  6. 2 1
      apps/entropy-tester/package.json
  7. 12 1
      apps/entropy-tester/src/index.ts
  8. 14 1
      apps/fortuna/Cargo.lock
  9. 2 2
      apps/fortuna/Cargo.toml
  10. 3 1
      apps/fortuna/src/chain/ethereum.rs
  11. 80 38
      apps/fortuna/src/eth_utils/utils.rs
  12. 68 6
      apps/fortuna/src/keeper/process_event.rs
  13. 11 10
      apps/fortuna/src/main.rs
  14. 186 57
      apps/hermes/server/Cargo.lock
  15. 45 1
      apps/hermes/server/Cargo.toml
  16. 2 2
      apps/hermes/server/Dockerfile
  17. 1 0
      apps/hermes/server/build.rs
  18. 1 1
      apps/hermes/server/src/api.rs
  19. 1 0
      apps/hermes/server/src/api/rest.rs
  20. 3 2
      apps/hermes/server/src/api/rest/get_vaa_ccip.rs
  21. 1 0
      apps/hermes/server/src/api/types.rs
  22. 4 1
      apps/hermes/server/src/config.rs
  23. 1 1
      apps/hermes/server/src/config/cache.rs
  24. 5 0
      apps/hermes/server/src/config/pythnet.rs
  25. 10 6
      apps/hermes/server/src/main.rs
  26. 95 3
      apps/hermes/server/src/network/pythnet.rs
  27. 13 3
      apps/hermes/server/src/network/wormhole.rs
  28. 7 4
      apps/hermes/server/src/serde.rs
  29. 2 2
      apps/hermes/server/src/state.rs
  30. 33 10
      apps/hermes/server/src/state/aggregate.rs
  31. 6 1
      apps/hermes/server/src/state/aggregate/metrics.rs
  32. 8 0
      apps/hermes/server/src/state/aggregate/wormhole_merkle.rs
  33. 2 2
      apps/hermes/server/src/state/benchmarks.rs
  34. 7 6
      apps/hermes/server/src/state/cache.rs
  35. 4 1
      apps/hermes/server/src/state/metrics.rs
  36. 4 4
      apps/hermes/server/src/state/wormhole.rs
  37. 4 4
      apps/insights/src/components/PriceFeed/chart.tsx
  38. 1 1
      apps/price_pusher/README.md
  39. 1 1
      apps/price_pusher/package.json
  40. 21 6
      apps/price_pusher/src/solana/command.ts
  41. 11 31
      apps/price_pusher/src/solana/solana.ts
  42. 0 0
      apps/pyth-lazer-agent/.dockerignore
  43. 0 0
      apps/pyth-lazer-agent/.gitignore
  44. 227 165
      apps/pyth-lazer-agent/Cargo.lock
  45. 5 1
      apps/pyth-lazer-agent/Cargo.toml
  46. 2 2
      apps/pyth-lazer-agent/Dockerfile
  47. 2 3
      apps/pyth-lazer-agent/config/config.toml
  48. 0 0
      apps/pyth-lazer-agent/rust-toolchain.toml
  49. 1 2
      apps/pyth-lazer-agent/src/config.rs
  50. 0 0
      apps/pyth-lazer-agent/src/http_server.rs
  51. 268 0
      apps/pyth-lazer-agent/src/lazer_publisher.rs
  52. 1 1
      apps/pyth-lazer-agent/src/main.rs
  53. 0 0
      apps/pyth-lazer-agent/src/publisher_handle.rs
  54. 288 0
      apps/pyth-lazer-agent/src/relayer_session.rs
  55. 0 0
      apps/pyth-lazer-agent/src/websocket_utils.rs
  56. 121 1
      apps/quorum/Cargo.lock
  57. 3 1
      apps/quorum/Cargo.toml
  58. 18 0
      apps/quorum/Dockerfile
  59. 522 19
      apps/quorum/src/api.rs
  60. 1 0
      apps/quorum/src/main.rs
  61. 61 0
      apps/quorum/src/metrics_server.rs
  62. 102 10
      apps/quorum/src/server.rs
  63. 61 17
      apps/quorum/src/ws.rs
  64. 1 1
      contract_manager/package.json
  65. 0 88
      contract_manager/src/core/contracts/evm.ts
  66. 0 158
      contract_manager/src/core/contracts/evm_abis.ts
  67. 9 9
      contract_manager/src/node/utils/shell.ts
  68. 0 5
      contract_manager/src/node/utils/store.ts
  69. 8 1
      contract_manager/store/chains/EvmChains.json
  70. 1 1
      contract_manager/store/contracts/AptosPriceFeedContracts.json
  71. 1 1
      contract_manager/store/contracts/AptosWormholeContracts.json
  72. 1 1
      contract_manager/store/contracts/CosmWasmPriceFeedContracts.json
  73. 1 1
      contract_manager/store/contracts/CosmWasmWormholeContracts.json
  74. 5 60
      contract_manager/store/contracts/EvmEntropyContracts.json
  75. 0 7
      contract_manager/store/contracts/EvmExpressRelayContracts.json
  76. 6 1
      contract_manager/store/contracts/EvmPriceFeedContracts.json
  77. 6 1
      contract_manager/store/contracts/EvmWormholeContracts.json
  78. 1 1
      contract_manager/store/contracts/FuelPriceFeedContracts.json
  79. 1 1
      contract_manager/store/contracts/FuelWormholeContracts.json
  80. 1 1
      contract_manager/store/contracts/IotaPriceFeedContracts.json
  81. 1 1
      contract_manager/store/contracts/IotaWormholeContracts.json
  82. 1 1
      contract_manager/store/contracts/NearPriceFeedContracts.json
  83. 1 1
      contract_manager/store/contracts/NearWormholeContracts.json
  84. 1 1
      contract_manager/store/contracts/StarknetPriceFeedContracts.json
  85. 1 1
      contract_manager/store/contracts/StarknetWormholeContracts.json
  86. 1 1
      contract_manager/store/contracts/SuiPriceFeedContracts.json
  87. 1 1
      contract_manager/store/contracts/SuiWormholeContracts.json
  88. 1 1
      contract_manager/store/contracts/TonPriceFeedContracts.json
  89. 1 1
      contract_manager/store/contracts/TonWormholeContracts.json
  90. 243 0
      doc/code-guidelines.md
  91. 514 0
      doc/rust-code-guidelines.md
  92. 5 5
      governance/xc_admin/packages/xc_admin_cli/src/index.ts
  93. 1 0
      governance/xc_admin/packages/xc_admin_common/src/chains.ts
  94. 3 1
      governance/xc_admin/packages/xc_admin_common/src/multisig_transaction/index.ts
  95. 4 1
      lazer/contracts/solana/package.json
  96. 2 0
      lazer/contracts/solana/programs/pyth-lazer-solana-contract/src/lib.rs
  97. 50 0
      lazer/contracts/solana/scripts/add_ed25519_signer.ts
  98. 4 4
      lazer/contracts/solana/scripts/check_trusted_signer.ts
  99. 76 0
      lazer/contracts/solana/scripts/verify_ed25519_message.ts
  100. 0 0
      lazer/contracts/solana/src/ed25519.ts

+ 37 - 0
.github/workflows/ci-pyth-lazer-agent.yml

@@ -0,0 +1,37 @@
+name: "pyth-lazer-agent Rust Test Suite"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - .github/workflows/ci-pyth-lazer-agent.yml
+      - apps/pyth-lazer-agent/**
+
+jobs:
+  pyth-lazer-agent-rust-test-suite:
+    name: pyth-lazer-agent Rust Test Suite
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: apps/pyth-lazer-agent
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: 1.87.0
+          components: clippy,rustfmt
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "apps/pyth-lazer-agent -> target"
+      - name: Format check
+        run: cargo fmt --all -- --check
+        if: success() || failure()
+      - name: Clippy check
+        run: cargo clippy --all-targets -- --deny warnings
+        if: success() || failure()
+      - name: test
+        run: cargo test
+        if: success() || failure()

+ 5 - 1
.github/workflows/ci-starknet-contract.yml

@@ -25,8 +25,12 @@ jobs:
           tool-versions: target_chains/starknet/contracts/.tool-versions
       - name: Install Starkli
         run: curl https://get.starkli.sh | sh && . ~/.config/.starkli/env && starkliup -v $(awk '/starkli/{print $2}' .tool-versions)
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.85.0
       - name: Install Devnet
-        run: cargo install starknet-devnet --version $(awk '/starknet-devnet/{print $2}' .tool-versions)
+        run: cargo +1.85.0 install starknet-devnet --version $(awk '/starknet-devnet/{print $2}' .tool-versions)
       - name: Check formatting
         run: scarb fmt --check
       - name: Run tests

+ 55 - 0
.github/workflows/docker-pyth-lazer-agent.yml

@@ -0,0 +1,55 @@
+name: Build and Push pyth-lazer-agent Image
+on:
+  push:
+    tags:
+      - pyth-lazer-agent-v*
+  pull_request:
+    paths:
+      - "apps/pyth-lazer-agent/**"
+  workflow_dispatch:
+    inputs:
+      dispatch_description:
+        description: "Dispatch description"
+        required: true
+        type: string
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: pyth-network/pyth-lazer-agent
+jobs:
+  pyth-lazer-agent-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set image tag to version of the git tag
+        if: ${{ startsWith(github.ref, 'refs/tags/pyth-lazer-agent-v') }}
+        run: |
+          PREFIX="refs/tags/pyth-lazer-agent-"
+          VERSION="${GITHUB_REF:${#PREFIX}}"
+          echo "IMAGE_TAG=${VERSION}" >> "${GITHUB_ENV}"
+      - name: Set image tag to the git commit hash
+        if: ${{ !startsWith(github.ref, 'refs/tags/pyth-lazer-agent-v') }}
+        run: |
+          echo "IMAGE_TAG=${{ github.sha }}" >> "${GITHUB_ENV}"
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: metadata_pyth_lazer_agent
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build and push server docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: "./apps/pyth-lazer-agent/Dockerfile"
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.metadata_pyth_lazer_agent.outputs.tags }}
+          labels: ${{ steps.metadata_pyth_lazer_agent.outputs.labels }}

+ 58 - 0
.github/workflows/docker-quorum.yml

@@ -0,0 +1,58 @@
+name: Build and Push Quorum Image
+on:
+  push:
+    tags:
+      - quorum-v*
+  pull_request:
+    paths:
+      - "apps/quorum/**"
+  workflow_dispatch:
+    inputs:
+      dispatch_description:
+        description: "Dispatch description"
+        required: true
+        type: string
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: pyth-network/quorum
+jobs:
+  quorum-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set image tag to version of the git tag
+        if: ${{ startsWith(github.ref, 'refs/tags/quorum-v') }}
+        run: |
+          PREFIX="refs/tags/quorum-"
+          VERSION="${GITHUB_REF:${#PREFIX}}"
+          echo "IMAGE_TAG=${VERSION}" >> "${GITHUB_ENV}"
+      - name: Set image tag to the git commit hash
+        if: ${{ !startsWith(github.ref, 'refs/tags/quorum-v') }}
+        run: |
+          SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-7)
+          echo "IMAGE_TAG=sha-${SHORT_SHA}" >> "${GITHUB_ENV}"
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: metadata_quorum
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.IMAGE_TAG }}
+      - name: Build and push server docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: "./apps/quorum/Dockerfile"
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.metadata_quorum.outputs.tags }}
+          labels: ${{ steps.metadata_quorum.outputs.labels }}

+ 3 - 0
README.md

@@ -56,6 +56,8 @@ Use the [Conventional Commits](https://www.conventionalcommits.org) format for y
 In the PR description, please include a summary of the changes and any relevant context. Also, please make sure
 to update the package versions following the [Semantic Versioning](https://semver.org/) rules.
 
+See also: [Code guidelines](doc/code-guidelines.md)
+
 ### Releases
 
 The repository has several CI workflows that automatically release new versions of the various components when a new Github release is published.
@@ -66,6 +68,7 @@ The general process for creating a new release is:
 2. Submit a PR with the changes and merge them in to main.
 3. Create a new release on github. Configure the release to create a new tag when published. Set the tag name and version for the component you wish to release -- see the [Releases](https://github.com/pyth-network/pyth-crosschain/releases) page to identify the relevant tag.
 4. Publish the release. This step will automatically trigger a Github Action to build the package and release it. This step will e.g., publish packages to NPM, or build and push docker images.
+   - Note that when publishing a public package, you should prune the auto-generated Github release notes to only include changes relevant to the release. Otherwise, the changelog will include commits from unrelated projects in the monorepo since the previous release.
 
 Note that all javascript packages are released together using a tag of the form `pyth-js-v<number>`. (The `number` is arbitrary.)
 If you have a javascript package that shouldn't be published, simply add `"private": "true"` to the `package.json` file

+ 2 - 1
apps/entropy-tester/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@pythnetwork/entropy-tester",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Utility to test entropy provider callbacks",
+  "private": true,
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

+ 12 - 1
apps/entropy-tester/src/index.ts

@@ -181,7 +181,18 @@ export const main = function () {
           // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
           while (true) {
             try {
-              await testLatency(contract, privateKey, child);
+              await Promise.race([
+                testLatency(contract, privateKey, child),
+                new Promise((_, reject) =>
+                  setTimeout(() => {
+                    reject(
+                      new Error(
+                        "Timeout: 120s passed but testLatency function was not resolved",
+                      ),
+                    );
+                  }, 120_000),
+                ),
+              ]);
             } catch (error) {
               child.error(error, "Error testing latency");
             }

+ 14 - 1
apps/fortuna/Cargo.lock

@@ -1647,7 +1647,7 @@ dependencies = [
 
 [[package]]
 name = "fortuna"
-version = "7.6.3"
+version = "7.6.5"
 dependencies = [
  "anyhow",
  "axum",
@@ -4750,6 +4750,16 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-serde"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.17"
@@ -4760,12 +4770,15 @@ dependencies = [
  "nu-ansi-term",
  "once_cell",
  "regex",
+ "serde",
+ "serde_json",
  "sharded-slab",
  "smallvec",
  "thread_local",
  "tracing",
  "tracing-core",
  "tracing-log",
+ "tracing-serde",
 ]
 
 [[package]]

+ 2 - 2
apps/fortuna/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fortuna"
-version = "7.6.3"
+version = "7.6.5"
 edition = "2021"
 
 [lib]
@@ -32,7 +32,7 @@ sha3 = "0.10.8"
 tokio = { version = "1.33.0", features = ["full"] }
 tower-http = { version = "0.4.0", features = ["cors"] }
 tracing = { version = "0.1.37", features = ["log"] }
-tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] }
 utoipa = { version = "3.4.0", features = ["axum_extras"] }
 utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
 once_cell = "1.18.0"

+ 3 - 1
apps/fortuna/src/chain/ethereum.rs

@@ -33,7 +33,9 @@ use {
 // contract in the same repo.
 abigen!(
     PythRandom,
-    "../../target_chains/ethereum/entropy_sdk/solidity/abis/IEntropy.json"
+    "../../target_chains/ethereum/entropy_sdk/solidity/abis/IEntropy.json";
+    PythRandomErrors,
+    "../../target_chains/ethereum/entropy_sdk/solidity/abis/EntropyErrors.json"
 );
 
 pub type MiddlewaresWrapper<T> = LegacyTxMiddleware<

+ 80 - 38
apps/fortuna/src/eth_utils/utils.rs

@@ -4,11 +4,15 @@ use {
     backoff::ExponentialBackoff,
     ethabi::ethereum_types::U64,
     ethers::{
-        contract::ContractCall,
+        contract::{ContractCall, ContractError},
         middleware::Middleware,
-        types::{TransactionReceipt, U256},
+        providers::ProviderError,
+        types::{transaction::eip2718::TypedTransaction, TransactionReceipt, U256},
+    },
+    std::{
+        fmt::Display,
+        sync::{atomic::AtomicU64, Arc},
     },
-    std::sync::{atomic::AtomicU64, Arc},
     tokio::time::{timeout, Duration},
     tracing,
 };
@@ -151,12 +155,17 @@ pub async fn estimate_tx_cost<T: Middleware + 'static>(
 /// the transaction exceeds this limit, the transaction is not submitted.
 /// Note however that any gas_escalation policy is applied to the estimate, so the actual gas used may exceed the limit.
 /// The transaction is retried until it is confirmed on chain or the maximum number of retries is reached.
+/// You can pass an `error_mapper` function that will be called on each retry with the number of retries and the error.
+/// This lets you customize the backoff behavior based on the error type.
 pub async fn submit_tx_with_backoff<T: Middleware + NonceManaged + 'static>(
     middleware: Arc<T>,
     call: ContractCall<T, ()>,
     gas_limit: U256,
     escalation_policy: EscalationPolicy,
-) -> Result<SubmitTxResult> {
+    error_mapper: Option<
+        impl Fn(u64, backoff::Error<SubmitTxError<T>>) -> backoff::Error<SubmitTxError<T>>,
+    >,
+) -> Result<SubmitTxResult, SubmitTxError<T>> {
     let start_time = std::time::Instant::now();
 
     tracing::info!("Started processing event");
@@ -176,14 +185,19 @@ pub async fn submit_tx_with_backoff<T: Middleware + NonceManaged + 'static>(
 
             let gas_multiplier_pct = escalation_policy.get_gas_multiplier_pct(num_retries);
             let fee_multiplier_pct = escalation_policy.get_fee_multiplier_pct(num_retries);
-            submit_tx(
+            let result = submit_tx(
                 middleware.clone(),
                 &call,
                 padded_gas_limit,
                 gas_multiplier_pct,
                 fee_multiplier_pct,
             )
-            .await
+            .await;
+            if let Some(ref mapper) = error_mapper {
+                result.map_err(|e| mapper(num_retries, e))
+            } else {
+                result
+            }
         },
         |e, dur| {
             let retry_number = num_retries.load(std::sync::atomic::Ordering::Relaxed);
@@ -210,6 +224,51 @@ pub async fn submit_tx_with_backoff<T: Middleware + NonceManaged + 'static>(
     })
 }
 
+pub enum SubmitTxError<T: Middleware + NonceManaged + 'static> {
+    GasUsageEstimateError(ContractError<T>),
+    GasLimitExceeded { estimate: U256, limit: U256 },
+    GasPriceEstimateError(<T as Middleware>::Error),
+    SubmissionError(TypedTransaction, <T as Middleware>::Error),
+    ConfirmationTimeout(TypedTransaction),
+    ConfirmationError(TypedTransaction, ProviderError),
+    ReceiptError(TypedTransaction, TransactionReceipt),
+}
+
+impl<T> Display for SubmitTxError<T>
+where
+    T: Middleware + NonceManaged + 'static,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SubmitTxError::GasUsageEstimateError(e) => {
+                write!(f, "Error estimating gas for reveal: {:?}", e)
+            }
+            SubmitTxError::GasLimitExceeded { estimate, limit } => write!(
+                f,
+                "Gas estimate for reveal with callback is higher than the gas limit {} > {}",
+                estimate, limit
+            ),
+            SubmitTxError::GasPriceEstimateError(e) => write!(f, "Gas price estimate error: {}", e),
+            SubmitTxError::SubmissionError(tx, e) => write!(
+                f,
+                "Error submitting the reveal transaction. Tx:{:?}, Error:{:?}",
+                tx, e
+            ),
+            SubmitTxError::ConfirmationTimeout(tx) => {
+                write!(f, "Tx stuck in mempool. Resetting nonce. Tx:{:?}", tx)
+            }
+            SubmitTxError::ConfirmationError(tx, e) => write!(
+                f,
+                "Error waiting for transaction receipt. Tx:{:?} Error:{:?}",
+                tx, e
+            ),
+            SubmitTxError::ReceiptError(tx, _) => {
+                write!(f, "Reveal transaction reverted on-chain. Tx:{:?}", tx,)
+            }
+        }
+    }
+}
+
 /// Submit a transaction to the blockchain. It estimates the gas for the transaction,
 /// pads both the gas and fee estimates using the provided multipliers, and submits the transaction.
 /// It will return a permanent or transient error depending on the error type and whether
@@ -221,24 +280,23 @@ pub async fn submit_tx<T: Middleware + NonceManaged + 'static>(
     // A value of 100 submits the tx with the same gas/fee as the estimate.
     gas_estimate_multiplier_pct: u64,
     fee_estimate_multiplier_pct: u64,
-) -> Result<TransactionReceipt, backoff::Error<anyhow::Error>> {
+) -> Result<TransactionReceipt, backoff::Error<SubmitTxError<T>>> {
     let gas_estimate_res = call.estimate_gas().await;
 
     let gas_estimate = gas_estimate_res.map_err(|e| {
         // we consider the error transient even if it is a contract revert since
         // it can be because of routing to a lagging RPC node. Retrying such errors will
         // incur a few additional RPC calls, but it is fine.
-        backoff::Error::transient(anyhow!("Error estimating gas for reveal: {:?}", e))
+        backoff::Error::transient(SubmitTxError::GasUsageEstimateError(e))
     })?;
 
     // The gas limit on the simulated transaction is the maximum expected tx gas estimate,
     // but we are willing to pad the gas a bit to ensure reliable submission.
     if gas_estimate > gas_limit {
-        return Err(backoff::Error::permanent(anyhow!(
-            "Gas estimate for reveal with callback is higher than the gas limit {} > {}",
-            gas_estimate,
-            gas_limit
-        )));
+        return Err(backoff::Error::permanent(SubmitTxError::GasLimitExceeded {
+            estimate: gas_estimate,
+            limit: gas_limit,
+        }));
     }
 
     // Pad the gas estimate after checking it against the simulation gas limit.
@@ -247,13 +305,11 @@ pub async fn submit_tx<T: Middleware + NonceManaged + 'static>(
     let call = call.clone().gas(gas_estimate);
     let mut transaction = call.tx.clone();
 
-    // manually fill the tx with the gas info, so we can log the details in case of error
+    // manually fill the tx with the gas price info, so we can log the details in case of error
     client
         .fill_transaction(&mut transaction, None)
         .await
-        .map_err(|e| {
-            backoff::Error::transient(anyhow!("Error filling the reveal transaction: {:?}", e))
-        })?;
+        .map_err(|e| backoff::Error::transient(SubmitTxError::GasPriceEstimateError(e)))?;
 
     // Apply the fee escalation policy. Note: the unwrap_or_default should never default as we have a gas oracle
     // in the client that sets the gas price.
@@ -271,11 +327,7 @@ pub async fn submit_tx<T: Middleware + NonceManaged + 'static>(
         .send_transaction(transaction.clone(), None)
         .await
         .map_err(|e| {
-            backoff::Error::transient(anyhow!(
-                "Error submitting the reveal transaction. Tx:{:?}, Error:{:?}",
-                transaction,
-                e
-            ))
+            backoff::Error::transient(SubmitTxError::SubmissionError(transaction.clone(), e))
         })?;
 
     let reset_nonce = || {
@@ -292,34 +344,24 @@ pub async fn submit_tx<T: Middleware + NonceManaged + 'static>(
         // in this case ethers internal polling will not reduce the number of retries
         // and keep retrying indefinitely. So we set a manual timeout here and reset the nonce.
         reset_nonce();
-        backoff::Error::transient(anyhow!(
-            "Tx stuck in mempool. Resetting nonce. Tx:{:?}",
-            transaction
-        ))
+        backoff::Error::transient(SubmitTxError::ConfirmationTimeout(transaction.clone()))
     })?;
 
     let receipt = pending_receipt
         .map_err(|e| {
-            backoff::Error::transient(anyhow!(
-                "Error waiting for transaction receipt. Tx:{:?} Error:{:?}",
-                transaction,
-                e
-            ))
+            backoff::Error::transient(SubmitTxError::ConfirmationError(transaction.clone(), e))
         })?
         .ok_or_else(|| {
             // RPC may not return an error on tx submission if the nonce is too high.
             // But we will never get a receipt. So we reset the nonce manager to get the correct nonce.
             reset_nonce();
-            backoff::Error::transient(anyhow!(
-                "Can't verify the reveal, probably dropped from mempool. Resetting nonce. Tx:{:?}",
-                transaction
-            ))
+            backoff::Error::transient(SubmitTxError::ConfirmationTimeout(transaction.clone()))
         })?;
 
     if receipt.status == Some(U64::from(0)) {
-        return Err(backoff::Error::transient(anyhow!(
-            "Reveal transaction reverted on-chain. Tx:{:?}",
-            transaction
+        return Err(backoff::Error::transient(SubmitTxError::ReceiptError(
+            transaction.clone(),
+            receipt.clone(),
         )));
     }
 

+ 68 - 6
apps/fortuna/src/keeper/process_event.rs

@@ -1,12 +1,14 @@
 use {
     super::keeper_metrics::AccountLabel,
     crate::{
-        chain::reader::RequestedWithCallbackEvent,
-        eth_utils::utils::submit_tx_with_backoff,
+        chain::{ethereum::PythRandomErrorsErrors, reader::RequestedWithCallbackEvent},
+        eth_utils::utils::{submit_tx_with_backoff, SubmitTxError},
         history::{RequestEntryState, RequestStatus},
         keeper::block::ProcessParams,
     },
     anyhow::{anyhow, Result},
+    ethers::{abi::AbiDecode, contract::ContractError},
+    std::time::Duration,
     tracing,
 };
 
@@ -74,12 +76,43 @@ pub async fn process_event_with_backoff(
         event.user_random_number,
         provider_revelation,
     );
+    let error_mapper = |num_retries, e| {
+        if let backoff::Error::Transient {
+            err: SubmitTxError::GasUsageEstimateError(ContractError::Revert(revert)),
+            ..
+        } = &e
+        {
+            if let Ok(PythRandomErrorsErrors::NoSuchRequest(_)) =
+                PythRandomErrorsErrors::decode(revert)
+            {
+                let err =
+                    SubmitTxError::GasUsageEstimateError(ContractError::Revert(revert.clone()));
+                // Slow down the retries if the request is not found.
+                // This probably means that the request is already fulfilled via another process.
+                // After 5 retries, we return the error permanently.
+                if num_retries >= 5 {
+                    return backoff::Error::Permanent(err);
+                }
+                let retry_after_seconds = match num_retries {
+                    0 => 5,
+                    1 => 10,
+                    _ => 60,
+                };
+                return backoff::Error::Transient {
+                    err,
+                    retry_after: Some(Duration::from_secs(retry_after_seconds)),
+                };
+            }
+        }
+        e
+    };
 
     let success = submit_tx_with_backoff(
         contract.client(),
         contract_call,
         gas_limit,
         escalation_policy,
+        Some(error_mapper),
     )
     .await;
 
@@ -160,16 +193,45 @@ pub async fn process_event_with_backoff(
                 .get_request(event.provider_address, event.sequence_number)
                 .await;
 
-            tracing::error!("Failed to process event: {:?}. Request: {:?}", e, req);
-
             // We only count failures for cases where we are completely certain that the callback failed.
-            if req.is_ok_and(|x| x.is_some()) {
+            if req.as_ref().is_ok_and(|x| x.is_some()) {
+                tracing::error!("Failed to process event: {}. Request: {:?}", e, req);
                 metrics
                     .requests_processed_failure
                     .get_or_create(&account_label)
                     .inc();
+                // Do not display the internal error, it might include RPC details.
+                let reason = match e {
+                    SubmitTxError::GasUsageEstimateError(ContractError::Revert(revert)) => {
+                        format!("Reverted: {}", revert)
+                    }
+                    SubmitTxError::GasLimitExceeded { limit, estimate } => format!(
+                        "Gas limit exceeded: limit = {}, estimate = {}",
+                        limit, estimate
+                    ),
+                    SubmitTxError::GasUsageEstimateError(_) => {
+                        "Unable to estimate gas usage".to_string()
+                    }
+                    SubmitTxError::GasPriceEstimateError(_) => {
+                        "Unable to estimate gas price".to_string()
+                    }
+                    SubmitTxError::SubmissionError(_, _) => {
+                        "Error submitting the transaction on-chain".to_string()
+                    }
+                    SubmitTxError::ConfirmationTimeout(tx) => format!(
+                        "Transaction was submitted, but never confirmed. Hash: {}",
+                        tx.sighash()
+                    ),
+                    SubmitTxError::ConfirmationError(tx, _) => format!(
+                        "Transaction was submitted, but never confirmed. Hash: {}",
+                        tx.sighash()
+                    ),
+                    SubmitTxError::ReceiptError(tx, _) => {
+                        format!("Reveal transaction failed on-chain. Hash: {}", tx.sighash())
+                    }
+                };
                 status.state = RequestEntryState::Failed {
-                    reason: format!("Error revealing: {:?}", e),
+                    reason,
                     provider_random_number: Some(provider_revelation),
                 };
                 history.add(&status);

+ 11 - 10
apps/fortuna/src/main.rs

@@ -17,16 +17,17 @@ use {
 #[tracing::instrument]
 async fn main() -> Result<()> {
     // Initialize a Tracing Subscriber
-    tracing::subscriber::set_global_default(
-        tracing_subscriber::fmt()
-            .compact()
-            .with_file(false)
-            .with_line_number(true)
-            .with_thread_ids(true)
-            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-            .with_ansi(std::io::stderr().is_terminal())
-            .finish(),
-    )?;
+    let fmt_builder = tracing_subscriber::fmt()
+        .with_file(false)
+        .with_line_number(true)
+        .with_thread_ids(true)
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .with_ansi(std::io::stderr().is_terminal());
+    if std::io::stderr().is_terminal() {
+        tracing::subscriber::set_global_default(fmt_builder.compact().finish())?;
+    } else {
+        tracing::subscriber::set_global_default(fmt_builder.json().finish())?;
+    }
 
     match config::Options::parse() {
         config::Options::GetRequest(opts) => command::get_request(&opts).await,

+ 186 - 57
apps/hermes/server/Cargo.lock

@@ -342,7 +342,7 @@ dependencies = [
  "nom",
  "num-traits",
  "rusticata-macros",
- "thiserror",
+ "thiserror 1.0.58",
  "time",
 ]
 
@@ -472,7 +472,7 @@ dependencies = [
  "bitflags 1.3.2",
  "bytes",
  "futures-util",
- "http",
+ "http 0.2.12",
  "http-body",
  "hyper",
  "itoa",
@@ -504,7 +504,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http",
+ "http 0.2.12",
  "http-body",
  "mime",
  "rustversion",
@@ -861,9 +861,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.6.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
 
 [[package]]
 name = "caps"
@@ -872,7 +872,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b"
 dependencies = [
  "libc",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -1756,6 +1756,18 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasi 0.14.2+wasi-0.2.4",
+]
+
 [[package]]
 name = "gimli"
 version = "0.28.1"
@@ -1804,7 +1816,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "futures-util",
- "http",
+ "http 0.2.12",
  "indexmap 2.2.6",
  "slab",
  "tokio",
@@ -1868,7 +1880,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermes"
-version = "0.9.3"
+version = "0.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1913,6 +1925,7 @@ dependencies = [
  "strum",
  "tokio",
  "tokio-stream",
+ "tokio-tungstenite 0.26.2",
  "tonic",
  "tonic-build",
  "tower-http",
@@ -2003,6 +2016,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
 [[package]]
 name = "http-body"
 version = "0.4.6"
@@ -2010,7 +2034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
 dependencies = [
  "bytes",
- "http",
+ "http 0.2.12",
  "pin-project-lite",
 ]
 
@@ -2049,7 +2073,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http",
+ "http 0.2.12",
  "http-body",
  "httparse",
  "httpdate",
@@ -2069,7 +2093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
 dependencies = [
  "futures-util",
- "http",
+ "http 0.2.12",
  "hyper",
  "rustls 0.21.10",
  "tokio",
@@ -2296,9 +2320,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.172"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
 
 [[package]]
 name = "libredox"
@@ -3239,7 +3263,7 @@ dependencies = [
  "pyth-sdk 0.8.0",
  "serde",
  "solana-program",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -3258,7 +3282,7 @@ dependencies = [
  "sha3 0.10.8",
  "slow_primes",
  "strum",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -3297,7 +3321,7 @@ dependencies = [
  "quinn-udp",
  "rustc-hash",
  "rustls 0.20.9",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
  "tracing",
  "webpki",
@@ -3316,7 +3340,7 @@ dependencies = [
  "rustls 0.20.9",
  "rustls-native-certs",
  "slab",
- "thiserror",
+ "thiserror 1.0.58",
  "tinyvec",
  "tracing",
  "webpki",
@@ -3353,6 +3377,12 @@ dependencies = [
  "proc-macro2 1.0.92",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+
 [[package]]
 name = "radium"
 version = "0.7.0"
@@ -3383,6 +3413,16 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "rand"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
+dependencies = [
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.3",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.2.2"
@@ -3403,6 +3443,16 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.3",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.5.1"
@@ -3421,6 +3471,15 @@ dependencies = [
  "getrandom 0.2.12",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+dependencies = [
+ "getrandom 0.3.3",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.2.0"
@@ -3497,7 +3556,7 @@ checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
 dependencies = [
  "getrandom 0.2.12",
  "libredox",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -3566,7 +3625,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http",
+ "http 0.2.12",
  "http-body",
  "hyper",
  "hyper-rustls",
@@ -4044,7 +4103,7 @@ dependencies = [
  "futures",
  "percent-encoding",
  "serde",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4090,7 +4149,7 @@ dependencies = [
  "itoa",
  "serde",
  "serde_bytes",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4281,7 +4340,7 @@ dependencies = [
  "spl-token",
  "spl-token-2022",
  "spl-token-metadata-interface",
- "thiserror",
+ "thiserror 1.0.58",
  "zstd",
 ]
 
@@ -4303,7 +4362,7 @@ dependencies = [
  "solana-program",
  "solana-program-runtime",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4318,7 +4377,7 @@ dependencies = [
  "solana-perf",
  "solana-remote-wallet",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
  "tiny-bip39",
  "uriparse",
  "url",
@@ -4353,7 +4412,7 @@ dependencies = [
  "solana-thin-client",
  "solana-tpu-client",
  "solana-udp-client",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
 ]
 
@@ -4388,7 +4447,7 @@ dependencies = [
  "solana-measure",
  "solana-metrics",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
 ]
 
@@ -4422,7 +4481,7 @@ dependencies = [
  "sha2 0.10.8",
  "solana-frozen-abi-macro",
  "subtle",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4570,7 +4629,7 @@ dependencies = [
  "solana-frozen-abi",
  "solana-frozen-abi-macro",
  "solana-sdk-macro",
- "thiserror",
+ "thiserror 1.0.58",
  "tiny-bip39",
  "wasm-bindgen",
  "zeroize",
@@ -4601,7 +4660,7 @@ dependencies = [
  "solana-metrics",
  "solana-sdk",
  "solana_rbpf",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4621,7 +4680,7 @@ dependencies = [
  "solana-account-decoder",
  "solana-rpc-client-api",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
  "tokio-stream",
  "tokio-tungstenite 0.17.2",
@@ -4653,7 +4712,7 @@ dependencies = [
  "solana-rpc-client-api",
  "solana-sdk",
  "solana-streamer",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
 ]
 
@@ -4682,7 +4741,7 @@ dependencies = [
  "qstring",
  "semver",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
  "uriparse",
 ]
 
@@ -4731,7 +4790,7 @@ dependencies = [
  "solana-transaction-status",
  "solana-version",
  "spl-token-2022",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4744,7 +4803,7 @@ dependencies = [
  "solana-clap-utils",
  "solana-rpc-client",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4795,7 +4854,7 @@ dependencies = [
  "solana-logger",
  "solana-program",
  "solana-sdk-macro",
- "thiserror",
+ "thiserror 1.0.58",
  "uriparse",
  "wasm-bindgen",
 ]
@@ -4841,7 +4900,7 @@ dependencies = [
  "solana-metrics",
  "solana-perf",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
  "x509-parser",
 ]
@@ -4882,7 +4941,7 @@ dependencies = [
  "solana-rpc-client",
  "solana-rpc-client-api",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
 ]
 
@@ -4909,7 +4968,7 @@ dependencies = [
  "spl-memo",
  "spl-token",
  "spl-token-2022",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4923,7 +4982,7 @@ dependencies = [
  "solana-net-utils",
  "solana-sdk",
  "solana-streamer",
- "thiserror",
+ "thiserror 1.0.58",
  "tokio",
 ]
 
@@ -4962,7 +5021,7 @@ dependencies = [
  "solana-program",
  "solana-program-runtime",
  "solana-sdk",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -4990,7 +5049,7 @@ dependencies = [
  "solana-program",
  "solana-sdk",
  "subtle",
- "thiserror",
+ "thiserror 1.0.58",
  "zeroize",
 ]
 
@@ -5009,7 +5068,7 @@ dependencies = [
  "rand 0.8.5",
  "rustc-demangle",
  "scroll",
- "thiserror",
+ "thiserror 1.0.58",
  "winapi",
 ]
 
@@ -5057,7 +5116,7 @@ dependencies = [
  "solana-program",
  "spl-token",
  "spl-token-2022",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -5092,7 +5151,7 @@ dependencies = [
  "quote 1.0.35",
  "sha2 0.10.8",
  "syn 2.0.89",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -5127,7 +5186,7 @@ dependencies = [
  "num-traits",
  "solana-program",
  "spl-program-error-derive",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -5168,7 +5227,7 @@ dependencies = [
  "num-traits",
  "num_enum 0.6.1",
  "solana-program",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -5190,7 +5249,7 @@ dependencies = [
  "spl-token-metadata-interface",
  "spl-transfer-hook-interface",
  "spl-type-length-value",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -5402,7 +5461,16 @@ version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.58",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
+dependencies = [
+ "thiserror-impl 2.0.12",
 ]
 
 [[package]]
@@ -5416,6 +5484,17 @@ dependencies = [
  "syn 2.0.89",
 ]
 
+[[package]]
+name = "thiserror-impl"
+version = "2.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
+dependencies = [
+ "proc-macro2 1.0.92",
+ "quote 1.0.35",
+ "syn 2.0.89",
+]
+
 [[package]]
 name = "thread_local"
 version = "1.1.8"
@@ -5470,7 +5549,7 @@ dependencies = [
  "rand 0.7.3",
  "rustc-hash",
  "sha2 0.9.9",
- "thiserror",
+ "thiserror 1.0.58",
  "unicode-normalization",
  "wasm-bindgen",
  "zeroize",
@@ -5602,6 +5681,20 @@ dependencies = [
  "tungstenite 0.20.1",
 ]
 
+[[package]]
+name = "tokio-tungstenite"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084"
+dependencies = [
+ "futures-util",
+ "log",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+ "tungstenite 0.26.2",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.10"
@@ -5665,7 +5758,7 @@ dependencies = [
  "base64 0.21.7",
  "bytes",
  "h2",
- "http",
+ "http 0.2.12",
  "http-body",
  "hyper",
  "hyper-timeout",
@@ -5726,7 +5819,7 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-util",
- "http",
+ "http 0.2.12",
  "http-body",
  "http-range-header",
  "pin-project-lite",
@@ -5836,13 +5929,13 @@ dependencies = [
  "base64 0.13.1",
  "byteorder",
  "bytes",
- "http",
+ "http 0.2.12",
  "httparse",
  "log",
  "rand 0.8.5",
  "rustls 0.20.9",
  "sha-1",
- "thiserror",
+ "thiserror 1.0.58",
  "url",
  "utf-8",
  "webpki",
@@ -5858,16 +5951,34 @@ dependencies = [
  "byteorder",
  "bytes",
  "data-encoding",
- "http",
+ "http 0.2.12",
  "httparse",
  "log",
  "rand 0.8.5",
  "sha1",
- "thiserror",
+ "thiserror 1.0.58",
  "url",
  "utf-8",
 ]
 
+[[package]]
+name = "tungstenite"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13"
+dependencies = [
+ "bytes",
+ "data-encoding",
+ "http 1.3.1",
+ "httparse",
+ "log",
+ "native-tls",
+ "rand 0.9.1",
+ "sha1",
+ "thiserror 2.0.12",
+ "utf-8",
+]
+
 [[package]]
 name = "typenum"
 version = "1.17.0"
@@ -6094,6 +6205,15 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasi"
+version = "0.14.2+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.92"
@@ -6455,6 +6575,15 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
+dependencies = [
+ "bitflags 2.5.0",
+]
+
 [[package]]
 name = "wormhole-sdk"
 version = "0.1.0"
@@ -6466,7 +6595,7 @@ dependencies = [
  "serde",
  "serde_wormhole",
  "sha3 0.10.8",
- "thiserror",
+ "thiserror 1.0.58",
 ]
 
 [[package]]
@@ -6492,7 +6621,7 @@ dependencies = [
  "nom",
  "oid-registry",
  "rusticata-macros",
- "thiserror",
+ "thiserror 1.0.58",
  "time",
 ]
 

+ 45 - 1
apps/hermes/server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name        = "hermes"
-version     = "0.9.3"
+version     = "0.10.0-alpha"
 description = "Hermes is an agent that provides Verified Prices from the Pythnet Pyth Oracle."
 edition     = "2021"
 
@@ -44,6 +44,7 @@ sha3               = { version = "0.10.4" }
 strum              = { version = "0.24.1", features = ["derive"] }
 tokio              = { version = "1.26.0", features = ["full"] }
 tokio-stream       = { version = "0.1.15", features = ["full"] }
+tokio-tungstenite  = { version = "0.26.2", features = ["native-tls"] }
 tonic              = { version = "0.10.1", features = ["tls"] }
 tower-http         = { version = "0.4.0", features = ["cors"] }
 tracing            = { version = "0.1.37", features = ["log"] }
@@ -74,3 +75,46 @@ panic                  = 'abort'
 
 [profile.dev]
 panic                  = 'abort'
+
+
+[lints.rust]
+unsafe_code = "deny"
+
+[lints.clippy]
+# See [Rust code guidelines](../../doc/rust-code-guidelines.md)
+
+wildcard_dependencies = "deny"
+
+collapsible_if = "allow"
+collapsible_else_if = "allow"
+
+allow_attributes_without_reason = "warn"
+
+# Panics
+expect_used = "warn"
+fallible_impl_from = "warn"
+indexing_slicing = "warn"
+panic = "warn"
+panic_in_result_fn = "warn"
+string_slice = "warn"
+todo = "warn"
+unchecked_duration_subtraction = "warn"
+unreachable = "warn"
+unwrap_in_result = "warn"
+unwrap_used = "warn"
+
+# Correctness
+cast_lossless = "warn"
+cast_possible_truncation = "warn"
+cast_possible_wrap = "warn"
+cast_sign_loss = "warn"
+collection_is_never_read = "warn"
+match_wild_err_arm = "warn"
+path_buf_push_overwrite = "warn"
+read_zero_byte_vec = "warn"
+same_name_method = "warn"
+suspicious_operation_groupings = "warn"
+suspicious_xor_used_as_pow = "warn"
+unused_self = "warn"
+used_underscore_binding = "warn"
+while_float = "warn"

+ 2 - 2
apps/hermes/server/Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /src/apps/hermes/server
 
 RUN --mount=type=cache,target=/root/.cargo/registry cargo build --release
 
-FROM rust:1.82.0
-
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
 # Copy artifacts from other images
 COPY --from=build /src/apps/hermes/server/target/release/hermes /usr/local/bin/

+ 1 - 0
apps/hermes/server/build.rs

@@ -13,6 +13,7 @@ fn main() {
     // Build the wormhole and google protobufs using Rust's prost_build crate.
     // The generated Rust code is placed in the OUT_DIR (env var set by cargo).
     // `network/wormhole.rs` then includes the generated code into the source while compilation is happening.
+    #[allow(clippy::expect_used, reason = "failing at build time is fine")]
     tonic_build::configure()
         .build_server(false)
         .compile(

+ 1 - 1
apps/hermes/server/src/api.rs

@@ -140,7 +140,7 @@ where
     // Initialize Axum Router. Note the type here is a `Router<State>` due to the use of the
     // `with_state` method which replaces `Body` with `State` in the type signature.
     let app = Router::new();
-    #[allow(deprecated)]
+    #[allow(deprecated, reason = "serving deprecated API endpoints")]
     let app = app
         .merge(SwaggerUi::new("/docs").url("/docs/openapi.json", ApiDoc::openapi()))
         .route("/", get(rest::index))

+ 1 - 0
apps/hermes/server/src/api/rest.rs

@@ -119,6 +119,7 @@ where
     }
 }
 #[cfg(test)]
+#[allow(clippy::unwrap_used, reason = "tests")]
 mod tests {
     use {
         super::*,

+ 3 - 2
apps/hermes/server/src/api/rest/get_vaa_ccip.rs

@@ -51,15 +51,16 @@ pub async fn get_vaa_ccip<S>(
 where
     S: Aggregates,
 {
+    let data: [u8; 40] = *params.data;
     let price_id: PriceIdentifier = PriceIdentifier::new(
-        params.data[0..32]
+        data[0..32]
             .try_into()
             .map_err(|_| RestError::InvalidCCIPInput)?,
     );
     validate_price_ids(&state, &[price_id], false).await?;
 
     let publish_time = UnixTimestamp::from_be_bytes(
-        params.data[32..40]
+        data[32..40]
             .try_into()
             .map_err(|_| RestError::InvalidCCIPInput)?,
     );

+ 1 - 0
apps/hermes/server/src/api/types.rs

@@ -396,6 +396,7 @@ impl Display for AssetType {
 }
 
 #[cfg(test)]
+#[allow(clippy::unwrap_used, reason = "tests")]
 mod tests {
     use super::*;
 

+ 4 - 1
apps/hermes/server/src/config.rs

@@ -14,7 +14,10 @@ mod wormhole;
 #[command(author = crate_authors!())]
 #[command(about = crate_description!())]
 #[command(version = crate_version!())]
-#[allow(clippy::large_enum_variant)]
+#[allow(
+    clippy::large_enum_variant,
+    reason = "performance is not a concern for config"
+)]
 pub enum Options {
     /// Run the Hermes Price Service.
     Run(RunOptions),

+ 1 - 1
apps/hermes/server/src/config/cache.rs

@@ -14,5 +14,5 @@ pub struct Options {
     #[arg(long = "cache-size-slots")]
     #[arg(env = "CACHE_SIZE_SLOTS")]
     #[arg(default_value = "1600")]
-    pub size_slots: u64,
+    pub size_slots: usize,
 }

+ 5 - 0
apps/hermes/server/src/config/pythnet.rs

@@ -21,4 +21,9 @@ pub struct Options {
     #[arg(default_value = DEFAULT_PYTHNET_ORACLE_PROGRAM_ADDR)]
     #[arg(env = "PYTHNET_ORACLE_PROGRAM_ADDR")]
     pub oracle_program_addr: Pubkey,
+
+    /// Address of a PythNet quorum websocket RPC endpoint.
+    #[arg(long = "pythnet-quorum-ws-addr")]
+    #[arg(env = "PYTHNET_QUORUM_WS_ADDR")]
+    pub quorum_ws_addr: Option<String>,
 }

+ 10 - 6
apps/hermes/server/src/main.rs

@@ -1,5 +1,5 @@
 use {
-    anyhow::Result,
+    anyhow::{Context, Result},
     clap::{CommandFactory, Parser},
     futures::future::join_all,
     lazy_static::lazy_static,
@@ -53,9 +53,13 @@ async fn init() -> Result<()> {
             // Listen for Ctrl+C so we can set the exit flag and wait for a graceful shutdown.
             spawn(async move {
                 tracing::info!("Registered shutdown signal handler...");
-                tokio::signal::ctrl_c().await.unwrap();
-                tracing::info!("Shut down signal received, waiting for tasks...");
-                let _ = EXIT.send(true);
+                match tokio::signal::ctrl_c().await {
+                    Ok(()) => {
+                        tracing::info!("Shut down signal received, waiting for tasks...");
+                        let _ = EXIT.send(true);
+                    }
+                    Err(err) => tracing::warn!("failed to register shutdown signal handler: {err}"),
+                }
             });
 
             // Spawn all worker tasks, and wait for all to complete (which will happen if a shutdown
@@ -83,8 +87,8 @@ async fn init() -> Result<()> {
                         let defaults = arg
                             .get_default_values()
                             .iter()
-                            .map(|v| v.to_str().unwrap())
-                            .collect::<Vec<_>>()
+                            .map(|v| v.to_str().context("non-utf8 default arg value"))
+                            .collect::<Result<Vec<_>>>()?
                             .join(",");
 
                         println!(

+ 95 - 3
apps/hermes/server/src/network/pythnet.rs

@@ -14,8 +14,8 @@ use {
         },
     },
     anyhow::{anyhow, bail, Result},
-    borsh::BorshDeserialize,
-    futures::stream::StreamExt,
+    borsh::{BorshDeserialize, BorshSerialize},
+    futures::{stream::StreamExt, SinkExt},
     pyth_sdk::PriceIdentifier,
     pyth_sdk_solana::state::load_product_account,
     solana_account_decoder::UiAccountEncoding,
@@ -29,6 +29,10 @@ use {
     },
     std::{collections::BTreeMap, sync::Arc, time::Duration},
     tokio::time::Instant,
+    tokio_tungstenite::{
+        connect_async,
+        tungstenite::{client::IntoClientRequest, Message},
+    },
 };
 
 /// Using a Solana RPC endpoint, fetches the target GuardianSet based on an index.
@@ -432,10 +436,98 @@ where
         })
     };
 
+    let task_quorum_listener = match opts.pythnet.quorum_ws_addr {
+        Some(pythnet_quorum_ws_addr) => {
+            let store = state.clone();
+            let mut exit = crate::EXIT.subscribe();
+            tokio::spawn(async move {
+                loop {
+                    let current_time = Instant::now();
+                    tokio::select! {
+                        _ = exit.changed() => break,
+                        Err(err) = run_quorom_listener(store.clone(), pythnet_quorum_ws_addr.clone()) => {
+                            tracing::error!(error = ?err, "Error in Pythnet quorum network listener.");
+                            if current_time.elapsed() < Duration::from_secs(30) {
+                                tracing::error!("Pythnet quorum listener restarting too quickly. Sleep 1s.");
+                                tokio::time::sleep(Duration::from_secs(1)).await;
+                            }
+                        }
+                    }
+                }
+                tracing::info!("Shutting down Pythnet quorum listener...");
+            })
+        }
+        None => tokio::spawn(async {
+            tracing::warn!(
+                "Pythnet quorum websocket address not provided, skipping quorum listener."
+            );
+        }),
+    };
+
     let _ = tokio::join!(
         task_listener,
         task_guardian_watcher,
-        task_price_feeds_metadata_updater
+        task_price_feeds_metadata_updater,
+        task_quorum_listener,
     );
     Ok(())
 }
+
+const QUORUM_PING_INTERVAL: Duration = Duration::from_secs(10);
+
+#[tracing::instrument(skip(state))]
+async fn run_quorom_listener<S>(state: Arc<S>, pythnet_quorum_ws_endpoint: String) -> Result<()>
+where
+    S: Wormhole,
+    S: Send + Sync + 'static,
+{
+    let mut ping_interval = tokio::time::interval(QUORUM_PING_INTERVAL);
+    let mut responded_to_ping = true; // Start with a true to not close the connection immediately
+    let request = pythnet_quorum_ws_endpoint.into_client_request()?;
+    let (mut ws_stream, _) = connect_async(request).await?;
+
+    loop {
+        tokio::select! {
+            message = ws_stream.next() => {
+                let vaa_bytes = match message.ok_or_else(|| anyhow!("PythNet quorum stream terminated."))?? {
+                    Message::Frame(_) => continue,
+                    Message::Text(message) => {
+                        match message.try_to_vec() {
+                            Ok(bytes) => bytes,
+                            Err(e) => {
+                                tracing::error!(error = ?e, "Failed to convert PythNet quorum text message to bytes.");
+                                continue;
+                            }
+                        }
+                    },
+                    Message::Binary(bytes) => bytes.to_vec(),
+                    Message::Ping(_) => continue,
+                    Message::Pong(_) => {
+                        responded_to_ping = true;
+                        continue;
+                    }
+                    Message::Close(_) => break,
+                };
+                tokio::spawn({
+                    let state = state.clone();
+                    async move {
+                        if let Err(e) = state.process_message(vaa_bytes).await {
+                            tracing::debug!(error = ?e, "Skipped VAA.");
+                        }
+                    }
+                });
+            },
+            _  = ping_interval.tick() => {
+                if !responded_to_ping {
+                    return Err(anyhow!("PythNet quorum subscriber did not respond to ping. Closing connection."));
+                }
+                responded_to_ping = false; // Reset the flag for the next ping
+                if let Err(e) = ws_stream.send(Message::Ping(vec![].into())).await {
+                    tracing::error!(error = ?e, "Failed to send PythNet quorum ping message.");
+                    return Err(anyhow!("Failed to send PythNet quorum ping message."));
+                }
+            },
+        }
+    }
+    Err(anyhow!("Pyth quorum stream terminated."))
+}

+ 13 - 3
apps/hermes/server/src/network/wormhole.rs

@@ -43,7 +43,10 @@ impl std::fmt::Display for GuardianSet {
 
 /// BridgeData extracted from wormhole bridge account, due to no API.
 #[derive(borsh::BorshDeserialize)]
-#[allow(dead_code)]
+#[allow(
+    dead_code,
+    reason = "we have to deserialize all fields but we don't use all of them"
+)]
 pub struct BridgeData {
     pub guardian_set_index: u32,
     pub last_lamports: u64,
@@ -52,7 +55,10 @@ pub struct BridgeData {
 
 /// BridgeConfig extracted from wormhole bridge account, due to no API.
 #[derive(borsh::BorshDeserialize)]
-#[allow(dead_code)]
+#[allow(
+    dead_code,
+    reason = "we have to deserialize all fields but we don't use all of them"
+)]
 pub struct BridgeConfig {
     pub guardian_set_expiration_time: u32,
     pub fee: u64,
@@ -75,7 +81,11 @@ pub struct GuardianSetData {
 ///
 /// The following module structure must match the protobuf definitions, so that the generated code
 /// can correctly reference modules from each other.
-#[allow(clippy::enum_variant_names)]
+#[allow(
+    clippy::enum_variant_names,
+    clippy::allow_attributes_without_reason,
+    reason = "generated code"
+)]
 mod proto {
     pub mod node {
         pub mod v1 {

+ 7 - 4
apps/hermes/server/src/serde.rs

@@ -17,13 +17,16 @@ pub mod hex {
         R: FromHex,
         <R as hex::FromHex>::Error: std::fmt::Display,
     {
-        let s: String = Deserialize::deserialize(d)?;
-        let p = s.starts_with("0x") || s.starts_with("0X");
-        let s = if p { &s[2..] } else { &s[..] };
-        hex::serde::deserialize(s.into_deserializer())
+        let full: String = Deserialize::deserialize(d)?;
+        let hex = full
+            .strip_prefix("0x")
+            .or_else(|| full.strip_prefix("0X"))
+            .unwrap_or(&full);
+        hex::serde::deserialize(hex.into_deserializer())
     }
 
     #[cfg(test)]
+    #[allow(clippy::unwrap_used, reason = "tests")]
     mod tests {
         use serde::Deserialize;
 

+ 2 - 2
apps/hermes/server/src/state.rs

@@ -56,7 +56,7 @@ struct State {
 
 pub fn new(
     update_tx: Sender<AggregationEvent>,
-    cache_size: u64,
+    cache_size: usize,
     benchmarks_endpoint: Option<Url>,
     readiness_staleness_threshold: Duration,
     readiness_max_allowed_slot_lag: Slot,
@@ -87,7 +87,7 @@ pub mod test {
     };
 
     pub async fn setup_state(
-        cache_size: u64,
+        cache_size: usize,
     ) -> (Arc<impl Aggregates>, Receiver<AggregationEvent>) {
         let (update_tx, update_rx) = tokio::sync::broadcast::channel(1000);
         let state = super::new(update_tx, cache_size, None, Duration::from_secs(30), 10);

+ 33 - 10
apps/hermes/server/src/state/aggregate.rs

@@ -1,3 +1,4 @@
+use anyhow::Context;
 use log::warn;
 #[cfg(test)]
 use mock_instant::{SystemTime, UNIX_EPOCH};
@@ -164,8 +165,9 @@ pub struct AccumulatorMessages {
 }
 
 impl AccumulatorMessages {
+    #[allow(clippy::cast_possible_truncation, reason = "intended truncation")]
     pub fn ring_index(&self) -> u32 {
-        (self.slot % self.ring_size as u64) as u32
+        (self.slot % u64::from(self.ring_size)) as u32
     }
 }
 
@@ -492,7 +494,12 @@ where
         let state_data = self.into().data.read().await;
         let price_feeds_metadata = PriceFeedMeta::retrieve_price_feeds_metadata(self)
             .await
-            .unwrap();
+            .unwrap_or_else(|err| {
+                tracing::error!(
+                    "unexpected failure of PriceFeedMeta::retrieve_price_feeds_metadata(self): {err}"
+                );
+                Vec::new()
+            });
 
         let current_time = SystemTime::now();
 
@@ -529,8 +536,8 @@ where
                 latest_completed_unix_timestamp: state_data.latest_completed_update_time.and_then(
                     |t| {
                         t.duration_since(UNIX_EPOCH)
-                            .map(|d| d.as_secs() as i64)
                             .ok()
+                            .and_then(|d| d.as_secs().try_into().ok())
                     },
                 ),
                 price_feeds_metadata_len: price_feeds_metadata.len(),
@@ -547,7 +554,11 @@ fn build_message_states(
     let wormhole_merkle_message_states_proofs =
         construct_message_states_proofs(&accumulator_messages, &wormhole_merkle_state)?;
 
-    let current_time: UnixTimestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as _;
+    let current_time: UnixTimestamp = SystemTime::now()
+        .duration_since(UNIX_EPOCH)?
+        .as_secs()
+        .try_into()
+        .context("timestamp overflow")?;
 
     accumulator_messages
         .raw_messages
@@ -663,7 +674,12 @@ where
         ));
     } else {
         // Use the publish time from the first end message
-        end_messages[0].message.publish_time() - window_seconds as i64
+        end_messages
+            .first()
+            .context("no messages found")?
+            .message
+            .publish_time()
+            - i64::try_from(window_seconds).context("window size overflow")?
     };
     let start_time = RequestTime::FirstAfter(start_timestamp);
 
@@ -807,6 +823,12 @@ fn calculate_twap(start_message: &TwapMessage, end_message: &TwapMessage) -> Res
 }
 
 #[cfg(test)]
+#[allow(
+    clippy::unwrap_used,
+    clippy::indexing_slicing,
+    clippy::cast_possible_wrap,
+    reason = "tests"
+)]
 mod test {
     use {
         super::*,
@@ -892,11 +914,11 @@ mod test {
     ) -> PriceFeedMessage {
         PriceFeedMessage {
             feed_id: [seed; 32],
-            price: seed as _,
-            conf: seed as _,
+            price: seed.into(),
+            conf: seed.into(),
             exponent: 0,
-            ema_conf: seed as _,
-            ema_price: seed as _,
+            ema_conf: seed.into(),
+            ema_price: seed.into(),
             publish_time,
             prev_publish_time,
         }
@@ -1239,7 +1261,7 @@ mod test {
                         PriceIdentifier::new([100; 32]),
                         PriceIdentifier::new([200; 32])
                     ],
-                    RequestTime::FirstAfter(slot as i64),
+                    RequestTime::FirstAfter(slot.into()),
                 )
                 .await
                 .is_err());
@@ -1631,6 +1653,7 @@ mod test {
     }
 }
 #[cfg(test)]
+#[allow(clippy::unwrap_used, reason = "tests")]
 /// Unit tests for the core TWAP calculation logic in `calculate_twap`
 mod calculate_twap_unit_tests {
     use super::*;

+ 6 - 1
apps/hermes/server/src/state/aggregate/metrics.rs

@@ -106,7 +106,12 @@ impl Metrics {
 
         // Clear out old slots
         while self.first_observed_time_of_slot.len() > MAX_SLOT_OBSERVATIONS {
-            let oldest_slot = *self.first_observed_time_of_slot.keys().next().unwrap();
+            #[allow(clippy::expect_used, reason = "len checked above")]
+            let oldest_slot = *self
+                .first_observed_time_of_slot
+                .keys()
+                .next()
+                .expect("first_observed_time_of_slot is empty");
             self.first_observed_time_of_slot.remove(&oldest_slot);
         }
     }

+ 8 - 0
apps/hermes/server/src/state/aggregate/wormhole_merkle.rs

@@ -142,6 +142,13 @@ pub fn construct_update_data(mut messages: Vec<RawMessageWithMerkleProof>) -> Re
 }
 
 #[cfg(test)]
+#[allow(
+    clippy::unwrap_used,
+    clippy::cast_possible_wrap,
+    clippy::panic,
+    clippy::indexing_slicing,
+    reason = "tests"
+)]
 mod test {
     use {
         super::*,
@@ -175,6 +182,7 @@ mod test {
     }
 
     #[test]
+
     fn test_construct_update_data_works_on_mixed_slot_and_big_size() {
         let mut messages = vec![];
 

+ 2 - 2
apps/hermes/server/src/state/benchmarks.rs

@@ -6,7 +6,7 @@ use {
         State,
     },
     crate::api::types::PriceUpdate,
-    anyhow::Result,
+    anyhow::{Context, Result},
     base64::{engine::general_purpose::STANDARD as base64_standard_engine, Engine as _},
     pyth_sdk::PriceIdentifier,
     reqwest::Url,
@@ -89,7 +89,7 @@ where
             .as_ref()
             .ok_or_else(|| anyhow::anyhow!("Benchmarks endpoint is not set"))?
             .join(&format!("/v1/updates/price/{}", publish_time))
-            .unwrap();
+            .context("failed to construct price endpoint")?;
 
         let mut request = reqwest::Client::new()
             .get(endpoint)

+ 7 - 6
apps/hermes/server/src/state/cache.rs

@@ -74,8 +74,8 @@ impl MessageState {
 }
 
 #[derive(Clone, Copy)]
-#[allow(dead_code)]
 pub enum MessageStateFilter {
+    #[allow(dead_code, reason = "can be useful later")]
     All,
     Only(MessageType),
 }
@@ -94,11 +94,11 @@ pub struct CacheState {
     accumulator_messages_cache: AccumulatorMessagesCache,
     wormhole_merkle_state_cache: WormholeMerkleStateCache,
     message_cache: MessageCache,
-    cache_size: u64,
+    cache_size: usize,
 }
 
 impl CacheState {
-    pub fn new(size: u64) -> Self {
+    pub fn new(size: usize) -> Self {
         Self {
             accumulator_messages_cache: Arc::new(RwLock::new(BTreeMap::new())),
             wormhole_merkle_state_cache: Arc::new(RwLock::new(BTreeMap::new())),
@@ -164,7 +164,7 @@ where
             cache.insert(time, message_state);
 
             // Remove the earliest message states if the cache size is exceeded
-            while cache.len() > self.into().cache_size as usize {
+            while cache.len() > self.into().cache_size {
                 cache.pop_first();
             }
         }
@@ -238,7 +238,7 @@ where
 
         // Messages don't exist, store them
         cache.insert(slot, accumulator_messages);
-        while cache.len() > self.into().cache_size as usize {
+        while cache.len() > self.into().cache_size {
             cache.pop_first();
         }
         Ok(true)
@@ -264,7 +264,7 @@ where
 
         // State doesn't exist, store it
         cache.insert(slot, wormhole_merkle_state);
-        while cache.len() > self.into().cache_size as usize {
+        while cache.len() > self.into().cache_size {
             cache.pop_first();
         }
         Ok(true)
@@ -344,6 +344,7 @@ async fn retrieve_message_state(
 }
 
 #[cfg(test)]
+#[allow(clippy::unwrap_used, reason = "tests")]
 mod test {
     use {
         super::*,

+ 4 - 1
apps/hermes/server/src/state/metrics.rs

@@ -51,7 +51,10 @@ where
     async fn encode(&self) -> String {
         let registry = self.into().registry.read().await;
         let mut buffer = String::new();
-        encode(&mut buffer, &registry).unwrap();
+        if let Err(err) = encode(&mut buffer, &registry) {
+            tracing::error!("failed to encode metrics: {err}");
+            return String::new();
+        }
         buffer
     }
 }

+ 4 - 4
apps/hermes/server/src/state/wormhole.rs

@@ -4,7 +4,7 @@ use {
         State,
     },
     crate::network::wormhole::GuardianSet,
-    anyhow::{anyhow, ensure, Result},
+    anyhow::{anyhow, ensure, Context, Result},
     chrono::DateTime,
     pythnet_sdk::{
         wire::v1::{WormholeMessage, WormholePayload},
@@ -104,8 +104,8 @@ where
         let vaa = serde_wormhole::from_slice::<Vaa<&RawMessage>>(&vaa_bytes)?;
 
         // Log VAA Processing.
-        let vaa_timestamp = DateTime::from_timestamp(vaa.timestamp as i64, 0)
-            .ok_or(anyhow!("Failed to parse VAA Tiestamp"))?
+        let vaa_timestamp = DateTime::from_timestamp(vaa.timestamp.into(), 0)
+            .context("Failed to parse VAA Timestamp")?
             .format("%Y-%m-%dT%H:%M:%S.%fZ")
             .to_string();
 
@@ -219,7 +219,7 @@ fn verify_vaa<'a>(
 
         // The address is the last 20 bytes of the Keccak256 hash of the public key
         let address: [u8; 32] = Keccak256::new_with_prefix(&pubkey[1..]).finalize().into();
-        let address: [u8; 20] = address[address.len() - 20..].try_into()?;
+        let address: [u8; 20] = address[32 - 20..].try_into()?;
 
         // Confirm the recovered address matches an address in the guardian set.
         if guardian_set.keys.get(signer_id) == Some(&address) {

+ 4 - 4
apps/insights/src/components/PriceFeed/chart.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { useLogger } from "@pythnetwork/component-library/useLogger";
-import { useResizeObserver } from "@react-hookz/web";
+import { useResizeObserver, useMountEffect } from "@react-hookz/web";
 import type { IChartApi, ISeriesApi, UTCTimestamp } from "lightweight-charts";
 import { LineSeries, LineStyle, createChart } from "lightweight-charts";
 import { useTheme } from "next-themes";
@@ -101,10 +101,10 @@ const useChartElem = (symbol: string, feedId: string) => {
     }
   }, [logger, symbol]);
 
-  useEffect(() => {
+  useMountEffect(() => {
     const chartElem = chartContainerRef.current;
     if (chartElem === null) {
-      return;
+      throw new Error("Chart element was null on mount");
     } else {
       const chart = createChart(chartElem, {
         layout: {
@@ -146,7 +146,7 @@ const useChartElem = (symbol: string, feedId: string) => {
         chart.remove();
       };
     }
-  }, [backfillData, priceFormatter]);
+  });
 
   useEffect(() => {
     if (current && chartRef.current) {

+ 1 - 1
apps/price_pusher/README.md

@@ -159,7 +159,7 @@ pnpm run start solana \
   --endpoint https://api.mainnet-beta.solana.com \
   --keypair-file ./id.json \
   --shard-id 1 \
-  --jito-endpoint mainnet.block-engine.jito.wtf \
+  --jito-endpoints mainnet.block-engine.jito.wtf,ny.mainnet.block-engine.jito.wtf \
   --jito-keypair-file ./jito.json \
   --jito-tip-lamports 100000 \
   --jito-bundle-size 5 \

+ 1 - 1
apps/price_pusher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/price-pusher",
-  "version": "9.3.4",
+  "version": "9.3.5",
   "description": "Pyth Price Pusher",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",

+ 21 - 6
apps/price_pusher/src/solana/command.ts

@@ -49,8 +49,8 @@ export default {
       type: "number",
       default: 50000,
     } as Options,
-    "jito-endpoint": {
-      description: "Jito endpoint",
+    "jito-endpoints": {
+      description: "Jito endpoint(s) - comma-separated list of endpoints",
       type: "string",
       optional: true,
     } as Options,
@@ -117,7 +117,7 @@ export default {
       pythContractAddress,
       pushingFrequency,
       pollingFrequency,
-      jitoEndpoint,
+      jitoEndpoints,
       jitoKeypairFile,
       jitoTipLamports,
       dynamicJitoTips,
@@ -209,7 +209,18 @@ export default {
         Uint8Array.from(JSON.parse(fs.readFileSync(jitoKeypairFile, "ascii"))),
       );
 
-      const jitoClient = searcherClient(jitoEndpoint, jitoKeypair);
+      const jitoEndpointsList = jitoEndpoints
+        .split(",")
+        .map((endpoint: string) => endpoint.trim());
+      const jitoClients: SearcherClient[] = jitoEndpointsList.map(
+        (endpoint: string) => {
+          logger.info(
+            `Constructing Jito searcher client from endpoint ${endpoint}`,
+          );
+          return searcherClient(endpoint, jitoKeypair);
+        },
+      );
+
       solanaPricePusher = new SolanaPricePusherJito(
         pythSolanaReceiver,
         hermesClient,
@@ -218,13 +229,17 @@ export default {
         jitoTipLamports,
         dynamicJitoTips,
         maxJitoTipLamports,
-        jitoClient,
+        jitoClients,
         jitoBundleSize,
         updatesPerJitoBundle,
+        // Set max retry time to pushing frequency, since we want to stop retrying before the next push attempt
+        pushingFrequency * 1000,
         lookupTableAccount,
       );
 
-      onBundleResult(jitoClient, logger.child({ module: "JitoClient" }));
+      jitoClients.forEach((client, index) => {
+        onBundleResult(client, logger.child({ module: `JitoClient-${index}` }));
+      });
     } else {
       solanaPricePusher = new SolanaPricePusher(
         pythSolanaReceiver,

+ 11 - 31
apps/price_pusher/src/solana/solana.ts

@@ -166,9 +166,10 @@ export class SolanaPricePusherJito implements IPricePusher {
     private defaultJitoTipLamports: number,
     private dynamicJitoTips: boolean,
     private maxJitoTipLamports: number,
-    private searcherClient: SearcherClient,
+    private searcherClients: SearcherClient[],
     private jitoBundleSize: number,
     private updatesPerJitoBundle: number,
+    private maxRetryTimeMs: number,
     private addressLookupTableAccount?: AddressLookupTableAccount,
   ) {}
 
@@ -194,10 +195,6 @@ export class SolanaPricePusherJito implements IPricePusher {
     }
   }
 
-  private async sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
   async updatePriceFeed(priceIds: string[]): Promise<void> {
     const recentJitoTip = await this.getRecentJitoTipLamports();
     const jitoTip =
@@ -243,32 +240,15 @@ export class SolanaPricePusherJito implements IPricePusher {
         jitoBundleSize: this.jitoBundleSize,
       });
 
-      let retries = 60;
-      while (retries > 0) {
-        try {
-          await sendTransactionsJito(
-            transactions,
-            this.searcherClient,
-            this.pythSolanaReceiver.wallet,
-          );
-          break;
-        } catch (err: any) {
-          if (err.code === 8 && err.details?.includes("Rate limit exceeded")) {
-            this.logger.warn("Rate limit hit, waiting before retry...");
-            await this.sleep(1100); // Wait slightly more than 1 second
-            retries--;
-            if (retries === 0) {
-              this.logger.error("Max retries reached for rate limit");
-              throw err;
-            }
-          } else {
-            throw err;
-          }
-        }
-      }
-
-      // Add a delay between bundles to avoid rate limiting
-      await this.sleep(1100);
+      await sendTransactionsJito(
+        transactions,
+        this.searcherClients,
+        this.pythSolanaReceiver.wallet,
+        {
+          maxRetryTimeMs: this.maxRetryTimeMs,
+        },
+        this.logger,
+      );
     }
   }
 }

+ 0 - 0
pyth-lazer-agent/.dockerignore → apps/pyth-lazer-agent/.dockerignore


+ 0 - 0
pyth-lazer-agent/.gitignore → apps/pyth-lazer-agent/.gitignore


文件差异内容过多而无法显示
+ 227 - 165
apps/pyth-lazer-agent/Cargo.lock


+ 5 - 1
pyth-lazer-agent/Cargo.toml → apps/pyth-lazer-agent/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyth-lazer-agent"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2024"
 
 [dependencies]
@@ -9,6 +9,7 @@ pyth-lazer-protocol = "0.7.2"
 
 anyhow = "1.0.98"
 backoff = "0.4.0"
+base64 = "0.22.1"
 bincode = { version = "2.0.1", features = ["serde"] }
 clap = { version = "4.5.32", features = ["derive"] }
 config = "0.15.11"
@@ -32,3 +33,6 @@ tokio-util = { version = "0.7.14", features = ["compat"] }
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] }
 url = { version = "2.5.4", features = ["serde"] }
+
+[dev-dependencies]
+tempfile = "3.20.0"

+ 2 - 2
pyth-lazer-agent/Dockerfile → apps/pyth-lazer-agent/Dockerfile

@@ -2,14 +2,14 @@ FROM rust:slim-bookworm AS builder
 
 RUN apt update && apt install -y curl libssl-dev pkg-config build-essential && apt clean all
 
-ADD . /pyth-lazer-agent
+ADD apps/pyth-lazer-agent /pyth-lazer-agent
 WORKDIR /pyth-lazer-agent
 
 RUN cargo build --release
 
 FROM debian:12-slim
 
-RUN apt update && apt install -y libssl-dev && apt clean all
+RUN apt update && apt install -y libssl-dev ca-certificates && apt clean all
 
 COPY --from=builder /pyth-lazer-agent/target/release/pyth-lazer-agent /pyth-lazer-agent/
 COPY --from=builder /pyth-lazer-agent/config/* /pyth-lazer-agent/config/

+ 2 - 3
pyth-lazer-agent/config/config.toml → apps/pyth-lazer-agent/config/config.toml

@@ -1,5 +1,4 @@
 relayer_urls = ["ws://relayer-0.pyth-lazer.dourolabs.app/v1/transaction", "ws://relayer-0.pyth-lazer.dourolabs.app/v1/transaction"]
-authorization_token = "token1"
 publish_keypair_path = "/path/to/solana/id.json"
-listen_address = "0.0.0.0:1776"
-publish_interval_duration = "0.5ms"
+listen_address = "0.0.0.0:8910"
+publish_interval_duration = "25ms"

+ 0 - 0
pyth-lazer-agent/rust-toolchain.toml → apps/pyth-lazer-agent/rust-toolchain.toml


+ 1 - 2
pyth-lazer-agent/src/config.rs → apps/pyth-lazer-agent/src/config.rs

@@ -12,8 +12,7 @@ use url::Url;
 pub struct Config {
     pub listen_address: SocketAddr,
     pub relayer_urls: Vec<Url>,
-    #[derivative(Debug = "ignore")]
-    pub authorization_token: String,
+    pub authorization_token: Option<String>,
     #[derivative(Debug = "ignore")]
     pub publish_keypair_path: PathBuf,
     #[serde(with = "humantime_serde", default = "default_publish_interval")]

+ 0 - 0
pyth-lazer-agent/src/http_server.rs → apps/pyth-lazer-agent/src/http_server.rs


+ 268 - 0
apps/pyth-lazer-agent/src/lazer_publisher.rs

@@ -0,0 +1,268 @@
+use crate::config::{CHANNEL_CAPACITY, Config};
+use crate::relayer_session::RelayerSessionTask;
+use anyhow::{Context, Result, bail};
+use base64::Engine;
+use base64::prelude::BASE64_STANDARD;
+use ed25519_dalek::{Signer, SigningKey};
+use protobuf::well_known_types::timestamp::Timestamp;
+use protobuf::{Message, MessageField};
+use pyth_lazer_publisher_sdk::publisher_update::{FeedUpdate, PublisherUpdate};
+use pyth_lazer_publisher_sdk::transaction::lazer_transaction::Payload;
+use pyth_lazer_publisher_sdk::transaction::signature_data::Data::Ed25519;
+use pyth_lazer_publisher_sdk::transaction::{
+    Ed25519SignatureData, LazerTransaction, SignatureData, SignedLazerTransaction,
+};
+use solana_keypair::read_keypair_file;
+use std::path::PathBuf;
+use tokio::sync::broadcast;
+use tokio::{
+    select,
+    sync::mpsc::{self, Receiver, Sender},
+    time::interval,
+};
+use tracing::error;
+
+#[derive(Clone)]
+pub struct LazerPublisher {
+    sender: Sender<FeedUpdate>,
+}
+
+impl LazerPublisher {
+    fn load_signing_key(publish_keypair_path: &PathBuf) -> Result<SigningKey> {
+        // Read the keypair from the file using Solana SDK because it's the same key used by the Pythnet publisher
+        let publish_keypair = match read_keypair_file(publish_keypair_path) {
+            Ok(k) => k,
+            Err(e) => {
+                tracing::error!(
+                    error = ?e,
+                    publish_keypair_path = publish_keypair_path.display().to_string(),
+                    "Reading publish keypair returned an error. ",
+                );
+                bail!("Reading publish keypair returned an error.");
+            }
+        };
+
+        SigningKey::from_keypair_bytes(&publish_keypair.to_bytes())
+            .context("Failed to create signing key from keypair")
+    }
+
+    pub async fn new(config: &Config) -> Self {
+        let signing_key = match Self::load_signing_key(&config.publish_keypair_path) {
+            Ok(signing_key) => signing_key,
+            Err(e) => {
+                tracing::error!("Failed to load signing key: {e:?}");
+                // Can't proceed on key failure
+                panic!("Failed to load signing key: {e:?}");
+            }
+        };
+
+        let authorization_token =
+            if let Some(authorization_token) = config.authorization_token.clone() {
+                // If authorization_token is configured, use it.
+                authorization_token
+            } else {
+                // Otherwise, use the base64 pubkey.
+                BASE64_STANDARD.encode(signing_key.verifying_key().to_bytes())
+            };
+
+        let (relayer_sender, _) = broadcast::channel(CHANNEL_CAPACITY);
+        for url in config.relayer_urls.iter() {
+            let mut task = RelayerSessionTask {
+                url: url.clone(),
+                token: authorization_token.clone(),
+                receiver: relayer_sender.subscribe(),
+            };
+            tokio::spawn(async move { task.run().await });
+        }
+
+        let (sender, receiver) = mpsc::channel(CHANNEL_CAPACITY);
+        let mut task = LazerPublisherTask {
+            config: config.clone(),
+            receiver,
+            pending_updates: Vec::new(),
+            relayer_sender,
+            signing_key,
+        };
+        tokio::spawn(async move { task.run().await });
+        Self { sender }
+    }
+
+    pub async fn push_feed_update(&self, feed_update: FeedUpdate) -> Result<()> {
+        self.sender.send(feed_update).await?;
+        Ok(())
+    }
+}
+
+struct LazerPublisherTask {
+    // connection state
+    config: Config,
+    receiver: Receiver<FeedUpdate>,
+    pending_updates: Vec<FeedUpdate>,
+    relayer_sender: broadcast::Sender<SignedLazerTransaction>,
+    signing_key: SigningKey,
+}
+
+impl LazerPublisherTask {
+    pub async fn run(&mut self) {
+        let mut publish_interval = interval(self.config.publish_interval_duration);
+        loop {
+            select! {
+                Some(feed_update) = self.receiver.recv() => {
+                    self.pending_updates.push(feed_update);
+                }
+                _ = publish_interval.tick() => {
+                    if let Err(err) = self.batch_transaction().await {
+                        error!("Failed to publish updates: {}", err);
+                    }
+                }
+            }
+        }
+    }
+
+    async fn batch_transaction(&mut self) -> Result<()> {
+        if self.pending_updates.is_empty() {
+            return Ok(());
+        }
+
+        let publisher_update = PublisherUpdate {
+            updates: self.pending_updates.drain(..).collect(),
+            publisher_timestamp: MessageField::some(Timestamp::now()),
+            special_fields: Default::default(),
+        };
+        let lazer_transaction = LazerTransaction {
+            payload: Some(Payload::PublisherUpdate(publisher_update)),
+            special_fields: Default::default(),
+        };
+        let buf = match lazer_transaction.write_to_bytes() {
+            Ok(buf) => buf,
+            Err(e) => {
+                tracing::warn!("Failed to encode Lazer transaction to bytes: {:?}", e);
+                bail!("Failed to encode Lazer transaction")
+            }
+        };
+        let signature = self.signing_key.sign(&buf);
+        let signature_data = SignatureData {
+            data: Some(Ed25519(Ed25519SignatureData {
+                signature: Some(signature.to_bytes().into()),
+                public_key: Some(self.signing_key.verifying_key().to_bytes().into()),
+                special_fields: Default::default(),
+            })),
+            special_fields: Default::default(),
+        };
+        let signed_lazer_transaction = SignedLazerTransaction {
+            signature_data: MessageField::some(signature_data),
+            payload: Some(buf),
+            special_fields: Default::default(),
+        };
+        match self.relayer_sender.send(signed_lazer_transaction.clone()) {
+            Ok(_) => (),
+            Err(e) => {
+                tracing::error!("Error sending transaction to relayer receivers: {e}");
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::config::{CHANNEL_CAPACITY, Config};
+    use crate::lazer_publisher::LazerPublisherTask;
+    use ed25519_dalek::SigningKey;
+    use protobuf::well_known_types::timestamp::Timestamp;
+    use protobuf::{Message, MessageField};
+    use pyth_lazer_publisher_sdk::publisher_update::feed_update::Update;
+    use pyth_lazer_publisher_sdk::publisher_update::{FeedUpdate, PriceUpdate};
+    use pyth_lazer_publisher_sdk::transaction::{LazerTransaction, lazer_transaction};
+    use std::io::Write;
+    use std::path::PathBuf;
+    use std::time::Duration;
+    use tempfile::NamedTempFile;
+    use tokio::sync::broadcast::error::TryRecvError;
+    use tokio::sync::{broadcast, mpsc};
+    use url::Url;
+
+    fn get_private_key() -> SigningKey {
+        SigningKey::from_keypair_bytes(&[
+            105, 175, 146, 91, 32, 145, 164, 199, 37, 111, 139, 255, 44, 225, 5, 247, 154, 170,
+            238, 70, 47, 15, 9, 48, 102, 87, 180, 50, 50, 38, 148, 243, 62, 148, 219, 72, 222, 170,
+            8, 246, 176, 33, 205, 29, 118, 11, 220, 163, 214, 204, 46, 49, 132, 94, 170, 173, 244,
+            39, 179, 211, 177, 70, 252, 31,
+        ])
+        .unwrap()
+    }
+
+    fn get_private_key_file() -> NamedTempFile {
+        let private_key_string = "[105,175,146,91,32,145,164,199,37,111,139,255,44,225,5,247,154,170,238,70,47,15,9,48,102,87,180,50,50,38,148,243,62,148,219,72,222,170,8,246,176,33,205,29,118,11,220,163,214,204,46,49,132,94,170,173,244,39,179,211,177,70,252,31]";
+        let mut temp_file = NamedTempFile::new().unwrap();
+        temp_file
+            .as_file_mut()
+            .write_all(private_key_string.as_bytes())
+            .unwrap();
+        temp_file.flush().unwrap();
+        temp_file
+    }
+
+    #[tokio::test]
+    async fn test_lazer_exporter_task() {
+        let signing_key_file = get_private_key_file();
+        let signing_key = get_private_key();
+
+        let config = Config {
+            listen_address: "0.0.0.0:12345".parse().unwrap(),
+            relayer_urls: vec![Url::parse("http://127.0.0.1:12346").unwrap()],
+            authorization_token: None,
+            publish_keypair_path: PathBuf::from(signing_key_file.path()),
+            publish_interval_duration: Duration::from_millis(25),
+        };
+
+        let (relayer_sender, mut relayer_receiver) = broadcast::channel(CHANNEL_CAPACITY);
+        let (sender, receiver) = mpsc::channel(CHANNEL_CAPACITY);
+        let mut task = LazerPublisherTask {
+            config: config.clone(),
+            receiver,
+            pending_updates: Vec::new(),
+            relayer_sender,
+            signing_key,
+        };
+        tokio::spawn(async move { task.run().await });
+
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+        match relayer_receiver.try_recv() {
+            Err(TryRecvError::Empty) => (),
+            _ => panic!("channel should be empty"),
+        }
+
+        let feed_update = FeedUpdate {
+            feed_id: Some(1),
+            source_timestamp: MessageField::some(Timestamp::now()),
+            update: Some(Update::PriceUpdate(PriceUpdate {
+                price: Some(100_000 * 100_000_000),
+                ..PriceUpdate::default()
+            })),
+            special_fields: Default::default(),
+        };
+        sender.send(feed_update.clone()).await.unwrap();
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+
+        match relayer_receiver.try_recv() {
+            Ok(transaction) => {
+                let lazer_transaction =
+                    LazerTransaction::parse_from_bytes(transaction.payload.unwrap().as_slice())
+                        .unwrap();
+                let publisher_update =
+                    if let lazer_transaction::Payload::PublisherUpdate(publisher_update) =
+                        lazer_transaction.payload.unwrap()
+                    {
+                        publisher_update
+                    } else {
+                        panic!("expected publisher_update")
+                    };
+                assert_eq!(publisher_update.updates.len(), 1);
+                assert_eq!(publisher_update.updates[0], feed_update);
+            }
+            _ => panic!("channel should have a transaction waiting"),
+        }
+    }
+}

+ 1 - 1
pyth-lazer-agent/src/main.rs → apps/pyth-lazer-agent/src/main.rs

@@ -16,7 +16,7 @@ mod websocket_utils;
 #[derive(Parser)]
 #[command(version)]
 struct Cli {
-    #[clap(short, long, default_value = "config.toml")]
+    #[clap(short, long, default_value = "config/config.toml")]
     config: String,
 }
 

+ 0 - 0
pyth-lazer-agent/src/publisher_handle.rs → apps/pyth-lazer-agent/src/publisher_handle.rs


+ 288 - 0
apps/pyth-lazer-agent/src/relayer_session.rs

@@ -0,0 +1,288 @@
+use anyhow::{Result, bail};
+use backoff::ExponentialBackoffBuilder;
+use backoff::backoff::Backoff;
+use futures_util::stream::{SplitSink, SplitStream};
+use futures_util::{SinkExt, StreamExt};
+use http::HeaderValue;
+use protobuf::Message;
+use pyth_lazer_publisher_sdk::transaction::SignedLazerTransaction;
+use std::time::{Duration, Instant};
+use tokio::net::TcpStream;
+use tokio::select;
+use tokio::sync::broadcast;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::{
+    MaybeTlsStream, WebSocketStream, connect_async_with_config,
+    tungstenite::Message as TungsteniteMessage,
+};
+use url::Url;
+
+type RelayerWsSender = SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, TungsteniteMessage>;
+type RelayerWsReceiver = SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>;
+
+async fn connect_to_relayer(
+    mut url: Url,
+    token: &str,
+) -> Result<(RelayerWsSender, RelayerWsReceiver)> {
+    tracing::info!("connecting to the relayer at {}", url);
+    url.set_path("/v1/transaction");
+    let mut req = url.clone().into_client_request()?;
+    let headers = req.headers_mut();
+    headers.insert(
+        "Authorization",
+        HeaderValue::from_str(&format!("Bearer {token}"))?,
+    );
+    let (ws_stream, _) = connect_async_with_config(req, None, true).await?;
+    Ok(ws_stream.split())
+}
+
+struct RelayerWsSession {
+    ws_sender: RelayerWsSender,
+}
+
+impl RelayerWsSession {
+    async fn send_transaction(
+        &mut self,
+        signed_lazer_transaction: SignedLazerTransaction,
+    ) -> Result<()> {
+        tracing::debug!(
+            "Sending SignedLazerTransaction: {:?}",
+            signed_lazer_transaction
+        );
+        let buf = signed_lazer_transaction.write_to_bytes()?;
+        self.ws_sender
+            .send(TungsteniteMessage::from(buf.clone()))
+            .await?;
+        self.ws_sender.flush().await?;
+        Ok(())
+    }
+}
+
+pub struct RelayerSessionTask {
+    // connection state
+    pub url: Url,
+    pub token: String,
+    pub receiver: broadcast::Receiver<SignedLazerTransaction>,
+}
+
+impl RelayerSessionTask {
+    pub async fn run(&mut self) {
+        let initial_interval = Duration::from_millis(100);
+        let max_interval = Duration::from_secs(5);
+        let mut backoff = ExponentialBackoffBuilder::new()
+            .with_initial_interval(initial_interval)
+            .with_max_interval(max_interval)
+            .with_max_elapsed_time(None)
+            .build();
+
+        const FAILURE_RESET_TIME: Duration = Duration::from_secs(300);
+        let mut first_failure_time = Instant::now();
+        let mut failure_count = 0;
+
+        loop {
+            match self.run_relayer_connection().await {
+                Ok(()) => {
+                    tracing::info!("relayer session graceful shutdown");
+                    return;
+                }
+                Err(e) => {
+                    if first_failure_time.elapsed() > FAILURE_RESET_TIME {
+                        failure_count = 0;
+                        first_failure_time = Instant::now();
+                        backoff.reset();
+                    }
+
+                    failure_count += 1;
+                    let next_backoff = backoff.next_backoff().unwrap_or(max_interval);
+                    tracing::warn!(
+                        "relayer session url: {} ended with error: {:?}, failure_count: {}; retrying in {:?}",
+                        self.url,
+                        e,
+                        failure_count,
+                        next_backoff
+                    );
+                    tokio::time::sleep(next_backoff).await;
+                }
+            }
+        }
+    }
+
+    pub async fn run_relayer_connection(&mut self) -> Result<()> {
+        // Establish relayer connection
+        // Relayer will drop the connection if no data received in 5s
+        let (relayer_ws_sender, mut relayer_ws_receiver) =
+            connect_to_relayer(self.url.clone(), &self.token).await?;
+        let mut relayer_ws_session = RelayerWsSession {
+            ws_sender: relayer_ws_sender,
+        };
+
+        loop {
+            select! {
+                recv_result = self.receiver.recv() => {
+                    match recv_result {
+                        Ok(transaction) => {
+                            if let Err(e) = relayer_ws_session.send_transaction(transaction).await {
+                                tracing::error!("Error publishing transaction to Lazer relayer: {e:?}");
+                                bail!("Failed to publish transaction to Lazer relayer: {e:?}");
+                            }
+                        },
+                        Err(e) => {
+                            match e {
+                                broadcast::error::RecvError::Closed => {
+                                    tracing::error!("transaction broadcast channel closed");
+                                    bail!("transaction broadcast channel closed");
+                                }
+                                broadcast::error::RecvError::Lagged(skipped_count) => {
+                                    tracing::warn!("transaction broadcast channel lagged by {skipped_count} messages");
+                                }
+                            }
+                        }
+                    }
+                }
+                // Handle messages from the relayers, such as errors if we send a bad update
+                msg = relayer_ws_receiver.next() => {
+                    match msg {
+                        Some(Ok(msg)) => {
+                            tracing::debug!("Received message from relayer: {msg:?}");
+                        }
+                        Some(Err(e)) => {
+                            tracing::error!("Error receiving message from at relayer: {e:?}");
+                        }
+                        None => {
+                            tracing::warn!("relayer connection closed url: {}", self.url);
+                            bail!("relayer connection closed");
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::relayer_session::RelayerSessionTask;
+    use ed25519_dalek::{Signer, SigningKey};
+    use futures_util::StreamExt;
+    use protobuf::well_known_types::timestamp::Timestamp;
+    use protobuf::{Message, MessageField};
+    use pyth_lazer_publisher_sdk::publisher_update::feed_update::Update;
+    use pyth_lazer_publisher_sdk::publisher_update::{FeedUpdate, PriceUpdate, PublisherUpdate};
+    use pyth_lazer_publisher_sdk::transaction::lazer_transaction::Payload;
+    use pyth_lazer_publisher_sdk::transaction::signature_data::Data::Ed25519;
+    use pyth_lazer_publisher_sdk::transaction::{
+        Ed25519SignatureData, LazerTransaction, SignatureData, SignedLazerTransaction,
+    };
+    use std::net::SocketAddr;
+    use tokio::net::TcpListener;
+    use tokio::sync::{broadcast, mpsc};
+    use url::Url;
+
+    pub const RELAYER_CHANNEL_CAPACITY: usize = 1000;
+
+    fn get_private_key() -> SigningKey {
+        SigningKey::from_keypair_bytes(&[
+            105, 175, 146, 91, 32, 145, 164, 199, 37, 111, 139, 255, 44, 225, 5, 247, 154, 170,
+            238, 70, 47, 15, 9, 48, 102, 87, 180, 50, 50, 38, 148, 243, 62, 148, 219, 72, 222, 170,
+            8, 246, 176, 33, 205, 29, 118, 11, 220, 163, 214, 204, 46, 49, 132, 94, 170, 173, 244,
+            39, 179, 211, 177, 70, 252, 31,
+        ])
+        .unwrap()
+    }
+
+    pub async fn run_mock_relayer(
+        addr: SocketAddr,
+        back_sender: mpsc::Sender<SignedLazerTransaction>,
+    ) {
+        let listener = TcpListener::bind(addr).await.unwrap();
+
+        tokio::spawn(async move {
+            let Ok((stream, _)) = listener.accept().await else {
+                panic!("failed to accept mock relayer websocket connection");
+            };
+            let ws_stream = tokio_tungstenite::accept_async(stream)
+                .await
+                .expect("handshake failed");
+            let (_, mut read) = ws_stream.split();
+            while let Some(msg) = read.next().await {
+                if let Ok(msg) = msg {
+                    if msg.is_binary() {
+                        tracing::info!("Received binary message: {msg:?}");
+                        let transaction =
+                            SignedLazerTransaction::parse_from_bytes(msg.into_data().as_ref())
+                                .unwrap();
+                        back_sender.clone().send(transaction).await.unwrap();
+                    }
+                } else {
+                    tracing::error!("Received a malformed message: {msg:?}");
+                }
+            }
+        });
+    }
+
+    #[tokio::test]
+    async fn test_relayer_session() {
+        let (back_sender, mut back_receiver) = mpsc::channel(RELAYER_CHANNEL_CAPACITY);
+        let relayer_addr = "127.0.0.1:12346".parse().unwrap();
+        run_mock_relayer(relayer_addr, back_sender).await;
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+
+        let (relayer_sender, relayer_receiver) = broadcast::channel(RELAYER_CHANNEL_CAPACITY);
+
+        let mut relayer_session_task = RelayerSessionTask {
+            // connection state
+            url: Url::parse("ws://127.0.0.1:12346").unwrap(),
+            token: "token1".to_string(),
+            receiver: relayer_receiver,
+        };
+        tokio::spawn(async move { relayer_session_task.run().await });
+        tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
+
+        let transaction = get_signed_lazer_transaction();
+        relayer_sender
+            .send(transaction.clone())
+            .expect("relayer_sender.send failed");
+        tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
+        let received_transaction = back_receiver
+            .recv()
+            .await
+            .expect("back_receiver.recv failed");
+        assert_eq!(transaction, received_transaction);
+    }
+
+    fn get_signed_lazer_transaction() -> SignedLazerTransaction {
+        let publisher_update = PublisherUpdate {
+            updates: vec![FeedUpdate {
+                feed_id: Some(1),
+                source_timestamp: MessageField::some(Timestamp::now()),
+                update: Some(Update::PriceUpdate(PriceUpdate {
+                    price: Some(1_000_000_000i64),
+                    ..PriceUpdate::default()
+                })),
+                special_fields: Default::default(),
+            }],
+            publisher_timestamp: MessageField::some(Timestamp::now()),
+            special_fields: Default::default(),
+        };
+        let lazer_transaction = LazerTransaction {
+            payload: Some(Payload::PublisherUpdate(publisher_update)),
+            special_fields: Default::default(),
+        };
+        let buf = lazer_transaction.write_to_bytes().unwrap();
+        let signing_key = get_private_key();
+        let signature = signing_key.sign(&buf);
+        let signature_data = SignatureData {
+            data: Some(Ed25519(Ed25519SignatureData {
+                signature: Some(signature.to_bytes().into()),
+                public_key: Some(signing_key.verifying_key().to_bytes().into()),
+                special_fields: Default::default(),
+            })),
+            special_fields: Default::default(),
+        };
+        SignedLazerTransaction {
+            signature_data: MessageField::some(signature_data),
+            payload: Some(buf),
+            special_fields: Default::default(),
+        }
+    }
+}

+ 0 - 0
pyth-lazer-agent/src/websocket_utils.rs → apps/pyth-lazer-agent/src/websocket_utils.rs


+ 121 - 1
apps/quorum/Cargo.lock

@@ -413,6 +413,26 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "axum-prometheus"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e1a6651f119707ec6c416f38fdb5be223707627fe6d47f912850634c106215"
+dependencies = [
+ "axum",
+ "bytes",
+ "futures-core",
+ "http 1.3.1",
+ "http-body 1.0.1",
+ "matchit",
+ "metrics",
+ "metrics-exporter-prometheus",
+ "pin-project",
+ "tokio",
+ "tower",
+ "tower-http",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.75"
@@ -1300,6 +1320,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
 [[package]]
 name = "foreign-types"
 version = "0.3.2"
@@ -1544,6 +1570,9 @@ name = "hashbrown"
 version = "0.15.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
+dependencies = [
+ "foldhash",
+]
 
 [[package]]
 name = "heck"
@@ -2132,6 +2161,46 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5"
+dependencies = [
+ "ahash",
+ "portable-atomic",
+]
+
+[[package]]
+name = "metrics-exporter-prometheus"
+version = "0.16.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034"
+dependencies = [
+ "base64 0.22.1",
+ "indexmap",
+ "metrics",
+ "metrics-util",
+ "quanta",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.15.3",
+ "metrics",
+ "quanta",
+ "rand 0.9.1",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -2513,6 +2582,26 @@ dependencies = [
  "num",
 ]
 
+[[package]]
+name = "pin-project"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.16"
@@ -2671,15 +2760,17 @@ dependencies = [
 
 [[package]]
 name = "quorum"
-version = "0.1.0"
+version = "0.2.1"
 dependencies = [
  "anyhow",
  "axum",
+ "axum-prometheus",
  "borsh 1.5.7",
  "clap",
  "futures",
  "hex",
  "lazy_static",
+ "metrics",
  "secp256k1",
  "serde",
  "serde_json",
@@ -2809,6 +2900,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_xoshiro"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
+dependencies = [
+ "rand_core 0.9.3",
+]
+
 [[package]]
 name = "raw-cpuid"
 version = "11.5.0"
@@ -3443,6 +3543,12 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a"
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -5775,6 +5881,20 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "tower-http"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
+dependencies = [
+ "bitflags 2.9.1",
+ "bytes",
+ "http 1.3.1",
+ "pin-project-lite",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "tower-layer"
 version = "0.3.3"

+ 3 - 1
apps/quorum/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "quorum"
-version = "0.1.0"
+version = "0.2.1"
 edition = "2021"
 
 [dependencies]
@@ -23,3 +23,5 @@ time = "0.3.41"
 serde_json = "1.0.140"
 futures = "0.3.31"
 serde_wormhole = "0.1.0"
+axum-prometheus = "0.8.0"
+metrics = "0.24.2"

+ 18 - 0
apps/quorum/Dockerfile

@@ -0,0 +1,18 @@
+FROM rust:1.87.0 AS build
+
+# Install OS packages
+RUN apt-get update && apt-get install --yes \
+    build-essential curl clang libssl-dev
+
+# Build
+WORKDIR /src
+COPY ./apps/quorum apps/quorum
+
+WORKDIR /src/apps/quorum
+
+RUN --mount=type=cache,target=/root/.cargo/registry cargo build --release
+
+# Copy artifacts from other images
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY --from=build /src/apps/quorum/target/release/quorum /usr/local/bin/

+ 522 - 19
apps/quorum/src/api.rs

@@ -3,6 +3,7 @@ use axum::{
     routing::{get, post},
     Json, Router,
 };
+use axum_prometheus::{EndpointLabel, PrometheusMetricLayerBuilder};
 use secp256k1::{
     ecdsa::{RecoverableSignature, RecoveryId},
     Message, Secp256k1,
@@ -10,7 +11,10 @@ use secp256k1::{
 use serde::Deserialize;
 use serde_wormhole::RawMessage;
 use sha3::{Digest, Keccak256};
-use std::{net::SocketAddr, time::Duration};
+use std::{
+    net::SocketAddr,
+    time::{Duration, Instant},
+};
 use wormhole_sdk::{
     vaa::{Body, Header, Signature},
     GuardianAddress, GuardianSetInfo, Vaa,
@@ -26,16 +30,24 @@ pub type Payload<'a> = &'a RawMessage;
 pub async fn run(listen_address: SocketAddr, state: State) -> anyhow::Result<()> {
     tracing::info!("Starting server...");
 
+    let (prometheus_layer, _) = PrometheusMetricLayerBuilder::new()
+        .with_metrics_from_fn(|| state.metrics_recorder.clone())
+        .with_endpoint_label_type(EndpointLabel::MatchedPathWithFallbackFn(|_| {
+            "unknown".to_string()
+        }))
+        .build_pair();
+
     let routes = Router::new()
         .route("/live", get(|| async { "OK" }))
         .route("/observation", post(post_observation))
         .route("/ws", get(ws_route_handler))
+        .layer(prometheus_layer)
         .with_state(state);
     let listener = tokio::net::TcpListener::bind(&listen_address).await?;
 
     axum::serve(listener, routes)
         .with_graceful_shutdown(async {
-            let _ = crate::server::EXIT.subscribe().changed().await;
+            crate::server::wait_for_exit().await;
             tracing::info!("Shutting down server...");
         })
         .await?;
@@ -60,15 +72,6 @@ impl Observation {
     fn get_body(&self) -> Result<Body<Payload>, serde_wormhole::Error> {
         serde_wormhole::from_slice(self.body.as_slice())
     }
-    pub fn is_expired(&self, observation_lifetime: u32) -> bool {
-        match self.get_body() {
-            Ok(body) => is_body_expired(&body, observation_lifetime),
-            Err(_) => {
-                tracing::warn!("Failed to deserialize observation body");
-                true
-            }
-        }
-    }
 }
 
 fn verify_observation(
@@ -76,13 +79,14 @@ fn verify_observation(
     guardian_set: GuardianSetInfo,
     observation_lifetime: u32,
 ) -> anyhow::Result<usize> {
-    if observation.is_expired(observation_lifetime) {
-        return Err(anyhow::anyhow!("Observation is expired"));
-    }
-
     let body = observation
         .get_body()
         .map_err(|e| anyhow::anyhow!("Failed to deserialize observation body: {}", e))?;
+
+    if is_body_expired(&body, observation_lifetime) {
+        return Err(anyhow::anyhow!("Observation is expired"));
+    }
+
     let digest = body.digest()?;
     let secp = Secp256k1::new();
     let recid = RecoveryId::try_from(observation.signature[64] as i32)?;
@@ -112,10 +116,18 @@ async fn run_expiration_loop(state: axum::extract::State<State>, observation: Ob
         if !verification.contains_key(&observation.body) {
             break;
         }
+        drop(verification); // Explicitly drop the read lock before acquiring a write lock
+
+        let body = match observation.get_body() {
+            Ok(body) => body,
+            Err(e) => {
+                tracing::warn!(error = ?e, "Failed to deserialize observation body");
+                break;
+            }
+        };
 
-        if observation.is_expired(state.observation_lifetime) {
-            let mut verification = state.verification.write().await;
-            verification.remove(&observation.body);
+        if is_body_expired(&body, state.observation_lifetime) {
+            state.verification.write().await.remove(&observation.body);
             break;
         }
     }
@@ -130,6 +142,11 @@ async fn handle_observation(
         state.guardian_set.clone(),
         state.observation_lifetime,
     )?;
+    metrics::counter!(
+        "verified_observations_total",
+        &[("gaurdian_index", verifier_index.to_string())]
+    )
+    .increment(1);
     let new_signature = Signature {
         signature: params.signature,
         index: verifier_index.try_into()?,
@@ -141,6 +158,7 @@ async fn handle_observation(
         .and_modify(|sigs| {
             if sigs.iter().all(|sig| sig.index != new_signature.index) {
                 sigs.push(new_signature);
+                sigs.sort_by(|a, b| a.index.cmp(&b.index));
             }
         })
         .or_insert_with(|| vec![new_signature])
@@ -149,7 +167,7 @@ async fn handle_observation(
     let body = params
         .get_body()
         .map_err(|e| anyhow::anyhow!("Failed to deserialize observation body: {}", e))?;
-    if signatures.len() > (state.guardian_set.addresses.len() * 2) / 3 + 1 {
+    if signatures.len() > (state.guardian_set.addresses.len() * 2) / 3 {
         let vaa: Vaa<Payload> = (
             Header {
                 version: 1,
@@ -159,6 +177,7 @@ async fn handle_observation(
             body,
         )
             .into();
+        metrics::counter!("new_vaa_total").increment(1);
         if let Err(e) = state
             .ws
             .broadcast_sender
@@ -183,10 +202,494 @@ async fn post_observation(
     tokio::spawn({
         let state = state.clone();
         async move {
+            let start = Instant::now();
+            let mut status = "success";
             if let Err(e) = handle_observation(state, params).await {
+                status = "error";
                 tracing::warn!(error = ?e, "Failed to handle observation");
             }
+            metrics::histogram!("handle_observation_duration_seconds", &[("status", status)])
+                .record(start.elapsed().as_secs_f64());
         }
     });
     Json(())
 }
+
+#[cfg(test)]
+mod test {
+    use std::{collections::HashMap, sync::Arc};
+
+    use crate::server::tests::get_state;
+    use secp256k1::{
+        rand::{self, seq::SliceRandom},
+        Secp256k1,
+    };
+    use serde::Serialize;
+    use tokio::{
+        sync::RwLock,
+        time::{sleep, timeout},
+    };
+
+    use super::*;
+
+    fn sign<P: Serialize>(body: &Body<P>, secret_key: &secp256k1::SecretKey) -> [u8; 65] {
+        let secp = Secp256k1::new();
+        let digest = body.digest().unwrap();
+        let message = Message::from_digest(digest.secp256k_hash);
+        let (recid, signature) = secp
+            .sign_ecdsa_recoverable(message, secret_key)
+            .serialize_compact();
+        let mut sig_bytes = [0u8; 65];
+        sig_bytes[..64].copy_from_slice(&signature);
+        sig_bytes[64] = recid as u8;
+        sig_bytes
+    }
+
+    const OBSERVERATION_LIFETIME: u32 = 3;
+    const SAMPLE_PAYLOAD: [u8; 5] = [4; 5];
+
+    fn get_sample_body<'a>(expiration_offset: i64) -> Body<&'a RawMessage> {
+        let body = Body {
+            timestamp: (OffsetDateTime::now_utc().unix_timestamp() + expiration_offset) as u32,
+            nonce: 0,
+            sequence: 1,
+            consistency_level: 2,
+            emitter_chain: wormhole_sdk::Chain::Ethereum,
+            emitter_address: wormhole_sdk::Address([3; 32]),
+            payload: RawMessage::new(&SAMPLE_PAYLOAD),
+        };
+        body
+    }
+
+    fn get_new_keypair() -> (secp256k1::SecretKey, [u8; 20]) {
+        let secp = Secp256k1::new();
+        let (secret_key, public_key) = secp.generate_keypair(&mut rand::rng());
+        let pubkey_uncompressed = public_key.serialize_uncompressed();
+        let pubkey_hash: [u8; 32] = Keccak256::new_with_prefix(&pubkey_uncompressed[1..])
+            .finalize()
+            .into();
+        let pubkey_evm: [u8; 20] = pubkey_hash[pubkey_hash.len() - 20..]
+            .try_into()
+            .expect("Invalid address length");
+        (secret_key, pubkey_evm)
+    }
+
+    fn get_new_keypairs(n: usize) -> Vec<(secp256k1::SecretKey, [u8; 20])> {
+        (0..n).map(|_| get_new_keypair()).collect()
+    }
+
+    fn get_guardian_sets(n: usize) -> (GuardianSetInfo, Vec<secp256k1::SecretKey>) {
+        let keys = get_new_keypairs(n);
+        let addresses: Vec<GuardianAddress> = keys
+            .iter()
+            .map(|(_, addr)| GuardianAddress(*addr))
+            .collect();
+        (
+            GuardianSetInfo { addresses },
+            keys.into_iter().map(|(key, _)| key).collect(),
+        )
+    }
+
+    #[test]
+    fn test_body_is_expired() {
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 + 1));
+        assert!(is_body_expired(&body, OBSERVERATION_LIFETIME));
+    }
+
+    #[test]
+    fn test_body_is_not_expired() {
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        assert!(!is_body_expired(&body, OBSERVERATION_LIFETIME));
+    }
+
+    #[test]
+    fn test_observation_get_body() {
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let observation = Observation {
+            signature: [0; 65],
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        assert_eq!(observation.get_body().unwrap(), body);
+    }
+
+    #[test]
+    fn test_verify_observation() {
+        let (guardian_set, keys) = get_guardian_sets(10);
+        let signer_index = 7;
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let observation = Observation {
+            signature: sign(&body, &keys[signer_index]),
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        let result = verify_observation(&observation, guardian_set.clone(), OBSERVERATION_LIFETIME);
+        assert_eq!(result.unwrap(), signer_index);
+    }
+
+    #[test]
+    fn test_verify_observation_is_expired() {
+        let (guardian_set, keys) = get_guardian_sets(10);
+        let signer_index = 7;
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 + 1));
+        let observation = Observation {
+            signature: sign(&body, &keys[signer_index]),
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        let result = verify_observation(&observation, guardian_set.clone(), OBSERVERATION_LIFETIME);
+        assert_eq!(result.unwrap_err().to_string(), "Observation is expired");
+    }
+
+    #[test]
+    fn test_verify_observation_invalid_body() {
+        let (guardian_set, keys) = get_guardian_sets(10);
+        let signer_index = 7;
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let mut body_bytes = serde_wormhole::to_vec(&body).unwrap();
+        body_bytes.truncate(10); // remove most of the data
+        let observation = Observation {
+            signature: sign(&body, &keys[signer_index]),
+            body: body_bytes,
+        };
+        let result = verify_observation(&observation, guardian_set.clone(), OBSERVERATION_LIFETIME);
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Failed to deserialize observation body: unexpected end of input"
+        );
+    }
+
+    #[test]
+    fn test_verify_observation_invalid_signature() {
+        let (guardian_set, _) = get_guardian_sets(10);
+        let random_key = get_new_keypair().0;
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let observation = Observation {
+            signature: sign(&body, &random_key),
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        let result = verify_observation(&observation, guardian_set.clone(), OBSERVERATION_LIFETIME);
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Signature does not match any guardian address"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_expiration_loop_expired() {
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let (guardian_set, keys) = get_guardian_sets(10);
+        let observation = Observation {
+            signature: sign(&body, &keys[0]),
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        let body = serde_wormhole::to_vec(&body).unwrap();
+        let state = get_state(
+            Arc::new(RwLock::new(HashMap::new())),
+            guardian_set,
+            OBSERVERATION_LIFETIME,
+        );
+        let result = timeout(
+            Duration::from_secs((OBSERVERATION_LIFETIME * 3) as u64),
+            async {
+                state.verification.write().await.insert(
+                    body.clone(),
+                    vec![Signature {
+                        signature: observation.signature,
+                        index: 0,
+                    }],
+                );
+                assert_eq!(
+                    state
+                        .verification
+                        .read()
+                        .await
+                        .get(&body.clone())
+                        .unwrap()
+                        .len(),
+                    1
+                );
+                run_expiration_loop(axum::extract::State(state.clone()), observation).await;
+            },
+        )
+        .await;
+
+        assert!(result.is_ok(), "Test failed due to timeout");
+        assert_eq!(
+            state.verification.read().await.len(),
+            0,
+            "Verification map should be empty after expiration loop"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_expiration_loop_remove_before_expiration() {
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let (guardian_set, keys) = get_guardian_sets(10);
+        let observation = Observation {
+            signature: sign(&body, &keys[0]),
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        let body = serde_wormhole::to_vec(&body).unwrap();
+        let state = get_state(
+            Arc::new(RwLock::new(HashMap::new())),
+            guardian_set,
+            OBSERVERATION_LIFETIME,
+        );
+        let result = timeout(
+            Duration::from_secs((OBSERVERATION_LIFETIME + 1) as u64),
+            async {
+                state.verification.write().await.insert(
+                    body.clone(),
+                    vec![Signature {
+                        signature: observation.signature,
+                        index: 0,
+                    }],
+                );
+                assert_eq!(
+                    state
+                        .verification
+                        .read()
+                        .await
+                        .get(&body.clone())
+                        .unwrap()
+                        .len(),
+                    1
+                );
+                state.verification.write().await.remove(&body.clone());
+                run_expiration_loop(axum::extract::State(state.clone()), observation).await;
+            },
+        )
+        .await;
+
+        assert!(result.is_ok(), "Test failed due to timeout");
+        assert_eq!(
+            state.verification.read().await.len(),
+            0,
+            "Verification map should be empty after expiration loop"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_expiration_loop_higher_expiration_than_timeout() {
+        let timeout_duration = (OBSERVERATION_LIFETIME + 2) as u64;
+        let mut body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+
+        // We should make sure the loop is going to run at least once
+        // So we need to set time duration to be higher than the observation lifetime
+        // And to make sure we are not going to remove the observation before the timeout
+        // We need to set the timestamp to be in the future
+        body.timestamp = (OffsetDateTime::now_utc().unix_timestamp() + 2_i64) as u32;
+
+        let (guardian_set, keys) = get_guardian_sets(10);
+        let observation = Observation {
+            signature: sign(&body, &keys[0]),
+            body: serde_wormhole::to_vec(&body).unwrap(),
+        };
+        let body = serde_wormhole::to_vec(&body).unwrap();
+        let state = get_state(
+            Arc::new(RwLock::new(HashMap::new())),
+            guardian_set,
+            OBSERVERATION_LIFETIME,
+        );
+        let result = timeout(Duration::from_secs(timeout_duration), async {
+            state.verification.write().await.insert(
+                body.clone(),
+                vec![Signature {
+                    signature: observation.signature,
+                    index: 0,
+                }],
+            );
+            assert_eq!(
+                state
+                    .verification
+                    .read()
+                    .await
+                    .get(&body.clone())
+                    .unwrap()
+                    .len(),
+                1
+            );
+            run_expiration_loop(axum::extract::State(state.clone()), observation).await;
+        })
+        .await;
+
+        assert!(
+            result.is_err(),
+            "Test should have timed out, as the observation is not expired yet"
+        );
+        assert_eq!(state.verification.read().await.len(), 1,
+            "Verification map should not be empty after expiration loop, as the observation is not expired yet");
+    }
+
+    #[tokio::test]
+    async fn test_handle_observation() {
+        let total_observations = 19;
+        let quorum = (total_observations * 2) / 3 + 1;
+        assert!(
+            quorum < total_observations,
+            "Quorum should be less than total observations"
+        );
+
+        let sample_body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let (guardian_set, keys) = get_guardian_sets(19);
+
+        // Shuffle the keys to ensure randomness in the test
+        let mut keys = keys.iter().enumerate().collect::<Vec<_>>();
+        keys.shuffle(&mut rand::rng());
+        let observations: Vec<Observation> = keys
+            .iter()
+            .map(|key| Observation {
+                signature: sign(&sample_body, key.1),
+                body: serde_wormhole::to_vec(&sample_body).unwrap(),
+            })
+            .collect();
+
+        let signatures: Vec<Signature> = observations
+            .iter()
+            .enumerate()
+            .map(|(i, obs)| Signature {
+                signature: obs.signature,
+                index: keys[i].0 as u8,
+            })
+            .collect();
+        let body = serde_wormhole::to_vec(&sample_body).unwrap();
+        let state = get_state(
+            Arc::new(RwLock::new(HashMap::new())),
+            guardian_set,
+            OBSERVERATION_LIFETIME,
+        );
+
+        let mut subscriber = state.ws.broadcast_sender.subscribe();
+        for (i, observation) in observations.iter().enumerate() {
+            if i > quorum {
+                // It should be once removed from the verification map
+                assert_eq!(
+                    state
+                        .verification
+                        .read()
+                        .await
+                        .get(&body.clone())
+                        .unwrap()
+                        .len(),
+                    i - quorum
+                );
+            } else if i < quorum && i > 0 {
+                assert_eq!(
+                    state
+                        .verification
+                        .read()
+                        .await
+                        .get(&body.clone())
+                        .unwrap()
+                        .len(),
+                    i
+                );
+            }
+            assert!(
+                handle_observation(axum::extract::State(state.clone()), observation.clone())
+                    .await
+                    .is_ok()
+            );
+
+            if i == quorum - 1 {
+                // Ensure we have reached the quorum
+                let update = subscriber
+                    .try_recv()
+                    .expect("Failed to receive update from subscriber");
+                let UpdateEvent::NewVaa(vaa) = update else {
+                    panic!("Expected NewVaa event, got {:?}", update);
+                };
+                let vaa: Vaa<&RawMessage> =
+                    serde_wormhole::from_slice(&vaa).expect("Failed to deserialize VAA");
+                // Check if the vaa signatures are sorted
+                for i in 0..vaa.signatures.len() {
+                    if i > 0 {
+                        assert!(
+                            vaa.signatures[i].index > vaa.signatures[i - 1].index,
+                            "Signature should be sorted"
+                        );
+                    }
+                }
+
+                let mut expected_signatures = signatures[0..quorum].to_vec();
+                expected_signatures.sort();
+                let expected_vaa: Vaa<&RawMessage> = (
+                    Header {
+                        version: 1,
+                        guardian_set_index: 0,
+                        signatures: expected_signatures,
+                    },
+                    sample_body.clone(),
+                )
+                    .into();
+
+                assert_eq!(
+                    vaa, expected_vaa,
+                    "VAA should match the expected VAA with the correct signatures"
+                );
+            }
+        }
+
+        // Ensure no new VAA is sent
+        let result = subscriber.try_recv();
+        assert!(
+            result.is_err(),
+            "No new VAA should be sent after reaching the quorum"
+        );
+
+        // Wait for the observation to expire
+        sleep(Duration::from_secs((OBSERVERATION_LIFETIME * 2 + 1) as u64)).await;
+        assert_eq!(
+            state.verification.read().await.len(),
+            0,
+            "Verification map should be empty after handling all observations"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_handle_observation_no_quorum() {
+        let body = get_sample_body(-(OBSERVERATION_LIFETIME as i64 - 1));
+        let (guardian_set, keys) = get_guardian_sets(19);
+
+        let observations: Vec<Observation> = (0..12)
+            .map(|i| Observation {
+                signature: sign(&body, &keys[i]),
+                body: serde_wormhole::to_vec(&body).unwrap(),
+            })
+            .collect();
+
+        let body = serde_wormhole::to_vec(&body).unwrap();
+        let state = get_state(
+            Arc::new(RwLock::new(HashMap::new())),
+            guardian_set,
+            OBSERVERATION_LIFETIME,
+        );
+
+        assert_eq!(state.verification.read().await.len(), 0);
+        for (i, observation) in observations.iter().enumerate() {
+            if i != 0 {
+                assert_eq!(
+                    state
+                        .verification
+                        .read()
+                        .await
+                        .get(&body.clone())
+                        .unwrap()
+                        .len(),
+                    i
+                );
+            }
+            assert!(
+                handle_observation(axum::extract::State(state.clone()), observation.clone())
+                    .await
+                    .is_ok()
+            );
+        }
+        assert_eq!(state.verification.read().await.len(), 1,
+            "Verification map should not be empty after handling all observations, as there is no quorum yet");
+
+        // Wait for the observation to expire
+        sleep(Duration::from_secs((OBSERVERATION_LIFETIME * 2 + 1) as u64)).await;
+
+        assert_eq!(state.verification.read().await.len(), 0,
+            "Verification map should not be empty after handling all observations, as there is no quorum yet");
+    }
+}

+ 1 - 0
apps/quorum/src/main.rs

@@ -4,6 +4,7 @@ use std::io::IsTerminal;
 use crate::server::RunOptions;
 
 mod api;
+mod metrics_server;
 mod pythnet;
 mod server;
 mod ws;

+ 61 - 0
apps/quorum/src/metrics_server.rs

@@ -0,0 +1,61 @@
+use std::{future::Future, time::Duration};
+
+use axum::{routing::get, Router};
+use axum_prometheus::{
+    metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle},
+    PrometheusMetricLayerBuilder,
+};
+
+use crate::server::{wait_for_exit, RunOptions, State};
+
+pub const DEFAULT_METRICS_BUCKET: &[f64; 20] = &[
+    0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.25, 1.5, 2.0,
+    3.0, 5.0, 10.0,
+];
+
+pub fn setup_metrics_recorder() -> anyhow::Result<PrometheusHandle> {
+    PrometheusBuilder::new()
+        .set_buckets(DEFAULT_METRICS_BUCKET)?
+        .install_recorder()
+        .map_err(|err| anyhow::anyhow!("Failed to set up metrics recorder: {:?}", err))
+}
+
+const METRIC_COLLECTION_INTERVAL: Duration = Duration::from_secs(1);
+pub async fn metric_collector<F, Fut>(service_name: String, update_metrics: F)
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = ()> + Send + 'static,
+{
+    let mut metric_interval = tokio::time::interval(METRIC_COLLECTION_INTERVAL);
+    loop {
+        tokio::select! {
+            _ = metric_interval.tick() => {
+                update_metrics().await;
+            }
+            _ = wait_for_exit() => {
+                tracing::info!("Received exit signal, stopping metric collector for {}...", service_name);
+                break;
+            }
+        }
+    }
+    tracing::info!("Shutting down metric collector for {}...", service_name);
+}
+
+pub async fn run(run_options: RunOptions, state: State) -> anyhow::Result<()> {
+    tracing::info!("Starting Metrics Server...");
+
+    let (_, metric_handle) = PrometheusMetricLayerBuilder::new()
+        .with_metrics_from_fn(|| state.metrics_recorder.clone())
+        .build_pair();
+    let app = Router::new();
+    let app = app.route("/metrics", get(|| async move { metric_handle.render() }));
+
+    let listener = tokio::net::TcpListener::bind(&run_options.server.metrics_addr).await?;
+    axum::serve(listener, app)
+        .with_graceful_shutdown(async {
+            let _ = wait_for_exit().await;
+            tracing::info!("Shutting down metrics server...");
+        })
+        .await?;
+    Ok(())
+}

+ 102 - 10
apps/quorum/src/server.rs

@@ -1,18 +1,26 @@
+use axum_prometheus::metrics_exporter_prometheus::PrometheusHandle;
 use clap::{crate_authors, crate_description, crate_name, crate_version, Args, Parser};
 use lazy_static::lazy_static;
 use solana_client::client_error::reqwest::Url;
 use solana_sdk::pubkey::Pubkey;
-use std::{collections::HashMap, net::SocketAddr, ops::Deref, sync::Arc};
-use tokio::sync::{watch, RwLock};
+use std::{
+    collections::HashMap, future::Future, net::SocketAddr, ops::Deref, sync::Arc, time::Duration,
+};
+use tokio::{
+    sync::{watch, RwLock},
+    time::sleep,
+};
 use wormhole_sdk::{vaa::Signature, GuardianSetInfo};
 
 use crate::{
     api::{self},
+    metrics_server::{self, metric_collector, setup_metrics_recorder},
     pythnet::fetch_guardian_set,
     ws::WsState,
 };
 
 const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:9000";
+const DEFAULT_METRICS_ADDR: &str = "127.0.0.1:9001";
 
 #[derive(Args, Clone, Debug)]
 #[command(next_help_heading = "Server Options")]
@@ -23,6 +31,11 @@ pub struct ServerOptions {
     #[arg(default_value = DEFAULT_LISTEN_ADDR)]
     #[arg(env = "LISTEN_ADDR")]
     pub listen_addr: SocketAddr,
+    /// Address and port the metrics will bind to.
+    #[arg(long = "metrics-addr")]
+    #[arg(default_value = DEFAULT_METRICS_ADDR)]
+    #[arg(env = "METRICS_ADDR")]
+    pub metrics_addr: SocketAddr,
 }
 
 // `Options` is a structup definition to provide clean command-line args for Hermes.
@@ -66,11 +79,22 @@ lazy_static! {
     /// - The `Receiver` side of a watch channel performs the detection based on if the change
     ///   happened after the subscribe, so it means all listeners should always be notified
     ///   correctly.
-    pub static ref EXIT: watch::Sender<bool> = watch::channel(false).0;
+
+    static ref EXIT: watch::Sender<bool> = watch::channel(false).0;
+}
+
+pub async fn wait_for_exit() {
+    let mut rx = EXIT.subscribe();
+    // Check if the exit flag is already set, if so, we don't need to wait.
+    if !(*rx.borrow()) {
+        // Wait until the exit flag is set.
+        let _ = rx.changed().await;
+    }
 }
 
 #[derive(Clone)]
 pub struct State(Arc<StateInner>);
+
 pub struct StateInner {
     pub verification: Arc<RwLock<HashMap<Vec<u8>, Vec<Signature>>>>,
 
@@ -80,6 +104,8 @@ pub struct StateInner {
     pub observation_lifetime: u32,
 
     pub ws: WsState,
+
+    pub metrics_recorder: PrometheusHandle,
 }
 impl Deref for State {
     type Target = Arc<StateInner>;
@@ -92,17 +118,42 @@ impl Deref for State {
 const DEFAULT_OBSERVATION_LIFETIME: u32 = 10; // In seconds
 const WEBSOCKET_NOTIFICATION_CHANNEL_SIZE: usize = 1000;
 
+async fn fault_tolerant_handler<F, Fut>(name: String, f: F)
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = anyhow::Result<()>> + Send + 'static,
+    Fut::Output: Send + 'static,
+{
+    loop {
+        let res = tokio::spawn(f()).await;
+        match res {
+            Ok(result) => match result {
+                Ok(_) => break, // This will happen on graceful shutdown
+                Err(err) => {
+                    tracing::error!("{} returned error: {:?}", name, err);
+                    sleep(Duration::from_millis(500)).await;
+                }
+            },
+            Err(err) => {
+                tracing::error!("{} is panicked or canceled: {:?}", name, err);
+                EXIT.send_modify(|exit| *exit = true);
+                break;
+            }
+        }
+    }
+}
+
 pub async fn run(run_options: RunOptions) -> anyhow::Result<()> {
     // Listen for Ctrl+C so we can set the exit flag and wait for a graceful shutdown.
     tokio::spawn(async move {
         tracing::info!("Registered shutdown signal handler...");
         tokio::signal::ctrl_c().await.unwrap();
         tracing::info!("Shut down signal received, waiting for tasks...");
-        let _ = EXIT.send(true);
+        EXIT.send_modify(|exit| *exit = true);
     });
 
     let guardian_set = fetch_guardian_set(
-        run_options.pythnet_url,
+        run_options.pythnet_url.clone(),
         run_options.wormhole_pid,
         run_options.guardian_set_index,
     )
@@ -117,13 +168,54 @@ pub async fn run(run_options: RunOptions) -> anyhow::Result<()> {
         observation_lifetime: run_options.observation_lifetime,
 
         ws: WsState::new(WEBSOCKET_NOTIFICATION_CHANNEL_SIZE),
+
+        metrics_recorder: setup_metrics_recorder()?,
     }));
 
-    tokio::join!(async {
-        if let Err(e) = api::run(run_options.server.listen_addr, state).await {
-            tracing::error!(error = ?e, "Failed to start API server");
-        }
-    });
+    tokio::join!(
+        fault_tolerant_handler("API server".to_string(), || api::run(
+            run_options.server.listen_addr,
+            state.clone()
+        )),
+        fault_tolerant_handler("metrics server".to_string(), || metrics_server::run(
+            run_options.clone(),
+            state.clone()
+        )),
+        metric_collector("state".to_string(), || {
+            let state = state.clone();
+            async move {
+                let verification = state.verification.read().await;
+                metrics::gauge!("pending_vaas").set(verification.len() as f64);
+                metrics::gauge!("pending_verified_observations")
+                    .set(verification.values().flatten().count() as f64);
+            }
+        }),
+    );
 
     Ok(())
 }
+
+#[cfg(test)]
+pub mod tests {
+    use axum_prometheus::metrics_exporter_prometheus::PrometheusBuilder;
+
+    use super::*;
+
+    pub fn get_state(
+        verification: Arc<RwLock<HashMap<Vec<u8>, Vec<Signature>>>>,
+        guardian_set: GuardianSetInfo,
+        observation_lifetime: u32,
+    ) -> State {
+        State(Arc::new(StateInner {
+            verification,
+            guardian_set,
+            observation_lifetime,
+
+            guardian_set_index: 0,
+
+            ws: WsState::new(1),
+
+            metrics_recorder: PrometheusBuilder::new().build_recorder().handle(),
+        }))
+    }
+}

+ 61 - 17
apps/quorum/src/ws.rs

@@ -1,5 +1,5 @@
 use {
-    crate::server::{State, EXIT},
+    crate::server::{wait_for_exit, State},
     anyhow::{anyhow, Result},
     axum::{
         extract::{
@@ -16,7 +16,7 @@ use {
         sync::atomic::{AtomicUsize, Ordering},
         time::Duration,
     },
-    tokio::sync::{broadcast, watch},
+    tokio::sync::broadcast,
 };
 
 pub struct WsState {
@@ -54,6 +54,7 @@ async fn websocket_handler(state: axum::extract::State<State>, stream: WebSocket
 #[derive(Clone, PartialEq, Debug)]
 pub enum UpdateEvent {
     NewVaa(Vec<u8>),
+    Ping,
 }
 
 pub type SubscriberId = usize;
@@ -68,7 +69,6 @@ pub struct Subscriber {
     sender: SplitSink<WebSocket, Message>,
     ping_interval: tokio::time::Interval,
     responded_to_ping: bool,
-    exit: watch::Receiver<bool>,
 }
 
 const PING_INTERVAL_DURATION: Duration = Duration::from_secs(30);
@@ -88,7 +88,6 @@ impl Subscriber {
             sender,
             ping_interval: tokio::time::interval(PING_INTERVAL_DURATION),
             responded_to_ping: true, // We start with true so we don't close the connection immediately
-            exit: EXIT.subscribe(),
         }
     }
 
@@ -119,10 +118,9 @@ impl Subscriber {
                     return Err(anyhow!("Subscriber did not respond to ping. Closing connection."));
                 }
                 self.responded_to_ping = false;
-                self.sender.send(Message::Ping(vec![].into())).await?;
-                Ok(())
+                self.handle_update(UpdateEvent::Ping).await
             },
-            _ = self.exit.changed() => {
+            _ = wait_for_exit() => {
                 self.sender.close().await?;
                 self.closed = true;
                 Err(anyhow!("Application is shutting down. Closing connection."))
@@ -136,13 +134,35 @@ impl Subscriber {
     }
 
     async fn handle_update(&mut self, event: UpdateEvent) -> Result<()> {
-        match event.clone() {
-            UpdateEvent::NewVaa(vaa) => self.handle_new_vaa(vaa).await,
-        }
+        let start = std::time::Instant::now();
+        let update_name;
+        let result = match event.clone() {
+            UpdateEvent::NewVaa(vaa) => {
+                update_name = "new_vaa";
+                self.handle_new_vaa(vaa).await
+            }
+            UpdateEvent::Ping => {
+                update_name = "ping";
+                self.sender.send(Message::Ping(vec![].into())).await?;
+                Ok(())
+            }
+        };
+        let status = match &result {
+            Ok(_) => "success",
+            Err(_) => "error",
+        };
+        let label = [("status", status), ("name", update_name)];
+        metrics::counter!("ws_server_update_total", &label).increment(1);
+        metrics::histogram!("ws_server_update_duration_seconds", &label,)
+            .record(start.elapsed().as_secs_f64());
+        result
     }
 
     async fn handle_client_message(&mut self, message: Message) -> Result<()> {
-        match message {
+        let start = std::time::Instant::now();
+        let message_type;
+
+        let result: anyhow::Result<()> = match message {
             Message::Close(_) => {
                 // Closing the connection. We don't remove it from the subscribers
                 // list, instead when the Subscriber struct is dropped the channel
@@ -151,15 +171,39 @@ impl Subscriber {
                 // Send the close message to gracefully shut down the connection
                 // Otherwise the client might get an abnormal Websocket closure
                 // error.
+                message_type = "close";
                 self.sender.close().await?;
                 self.closed = true;
-                return Ok(());
+                Ok(())
+            }
+            Message::Text(_) => {
+                message_type = "text";
+                Ok(())
+            }
+            Message::Binary(_) => {
+                message_type = "binary";
+                Ok(())
+            }
+            Message::Ping(_) => {
+                message_type = "ping";
+                Ok(())
+            }
+            Message::Pong(_) => {
+                message_type = "pong";
+                self.responded_to_ping = true;
+                Ok(())
             }
-            Message::Text(_) => {}
-            Message::Binary(_) => {}
-            Message::Ping(_) => {}
-            Message::Pong(_) => self.responded_to_ping = true,
         };
-        Ok(())
+
+        let status = match &result {
+            Ok(_) => "success",
+            Err(_) => "error",
+        };
+        let label = [("status", status), ("message_type", message_type)];
+        metrics::counter!("ws_client_message_total", &label).increment(1);
+        metrics::histogram!("ws_client_message_duration_seconds", &label,)
+            .record(start.elapsed().as_secs_f64());
+
+        result
     }
 }

+ 1 - 1
contract_manager/package.json

@@ -31,7 +31,7 @@
   ],
   "scripts": {
     "build": "tsc",
-    "shell": "ts-node ./src/shell.ts",
+    "shell": "ts-node ./src/node/utils/shell.ts",
     "fix:lint": "eslint src/ scripts/ --fix --max-warnings 0",
     "fix:format": "prettier --write \"src/**/*.ts\" \"scripts/**/*.ts\"",
     "test:lint": "eslint src/ scripts/ --max-warnings 0",

+ 0 - 88
contract_manager/src/core/contracts/evm.ts

@@ -7,7 +7,6 @@ import { WormholeContract } from "./wormhole";
 import { TokenQty } from "../token";
 import {
   EXECUTOR_ABI,
-  EXPRESS_RELAY_ABI,
   EXTENDED_ENTROPY_ABI,
   EXTENDED_PYTH_ABI,
   WORMHOLE_ABI,
@@ -413,93 +412,6 @@ export class EvmEntropyContract extends Storable {
   }
 }
 
-export class EvmExpressRelayContract extends Storable {
-  static type = "EvmExpressRelayContract";
-
-  constructor(
-    public chain: EvmChain,
-    public address: string,
-  ) {
-    super();
-  }
-
-  getId(): string {
-    return `${this.chain.getId()}_${this.address}`;
-  }
-
-  getChain(): EvmChain {
-    return this.chain;
-  }
-
-  getType(): string {
-    return EvmExpressRelayContract.type;
-  }
-
-  async getVersion(): Promise<string> {
-    const contract = this.getContract();
-    return contract.methods.version().call();
-  }
-
-  static fromJson(
-    chain: Chain,
-    parsed: { type: string; address: string },
-  ): EvmExpressRelayContract {
-    if (parsed.type !== EvmExpressRelayContract.type)
-      throw new Error("Invalid type");
-    if (!(chain instanceof EvmChain))
-      throw new Error(`Wrong chain type ${chain}`);
-    return new EvmExpressRelayContract(chain, parsed.address);
-  }
-
-  async generateSetRelayerPayload(relayer: string): Promise<Buffer> {
-    const contract = this.getContract();
-    const data = contract.methods.setRelayer(relayer).encodeABI();
-    return this.chain.generateExecutorPayload(
-      await this.getOwner(),
-      this.address,
-      data,
-    );
-  }
-
-  async getOwner(): Promise<string> {
-    const contract = this.getContract();
-    return contract.methods.owner().call();
-  }
-
-  async getExecutorContract(): Promise<EvmExecutorContract> {
-    const owner = await this.getOwner();
-    return new EvmExecutorContract(this.chain, owner);
-  }
-
-  async getPendingOwner(): Promise<string> {
-    const contract = this.getContract();
-    return contract.methods.pendingOwner().call();
-  }
-
-  async getRelayer(): Promise<string> {
-    const contract = this.getContract();
-    return contract.methods.getRelayer().call();
-  }
-
-  async getRelayerSubwallets(): Promise<string[]> {
-    const contract = this.getContract();
-    return contract.methods.getRelayerSubwallets().call();
-  }
-
-  toJson() {
-    return {
-      chain: this.chain.getId(),
-      address: this.address,
-      type: EvmExpressRelayContract.type,
-    };
-  }
-
-  getContract() {
-    const web3 = this.chain.getWeb3();
-    return new web3.eth.Contract(EXPRESS_RELAY_ABI, this.address);
-  }
-}
-
 export class EvmExecutorContract {
   constructor(
     public chain: EvmChain,

+ 0 - 158
contract_manager/src/core/contracts/evm_abis.ts

@@ -63,164 +63,6 @@ export const OWNABLE_ABI = [
   },
 ] as any; // eslint-disable-line  @typescript-eslint/no-explicit-any
 
-export const EXPRESS_RELAY_ABI = [
-  {
-    type: "function",
-    name: "getAdmin",
-    inputs: [],
-    outputs: [
-      {
-        name: "",
-        type: "address",
-        internalType: "address",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "getFeeProtocol",
-    inputs: [
-      {
-        name: "feeRecipient",
-        type: "address",
-        internalType: "address",
-      },
-    ],
-    outputs: [
-      {
-        name: "",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "getFeeProtocolDefault",
-    inputs: [],
-    outputs: [
-      {
-        name: "",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "getFeeRelayer",
-    inputs: [],
-    outputs: [
-      {
-        name: "",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "getFeeSplitPrecision",
-    inputs: [],
-    outputs: [
-      {
-        name: "",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "getRelayer",
-    inputs: [],
-    outputs: [
-      {
-        name: "",
-        type: "address",
-        internalType: "address",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "getRelayerSubwallets",
-    inputs: [],
-    outputs: [
-      {
-        name: "",
-        type: "address[]",
-        internalType: "address[]",
-      },
-    ],
-    stateMutability: "view",
-  },
-  {
-    type: "function",
-    name: "setFeeProtocol",
-    inputs: [
-      {
-        name: "feeRecipient",
-        type: "address",
-        internalType: "address",
-      },
-      {
-        name: "feeSplit",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    outputs: [],
-    stateMutability: "nonpayable",
-  },
-  {
-    type: "function",
-    name: "setFeeProtocolDefault",
-    inputs: [
-      {
-        name: "feeSplit",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    outputs: [],
-    stateMutability: "nonpayable",
-  },
-  {
-    type: "function",
-    name: "setFeeRelayer",
-    inputs: [
-      {
-        name: "feeSplit",
-        type: "uint256",
-        internalType: "uint256",
-      },
-    ],
-    outputs: [],
-    stateMutability: "nonpayable",
-  },
-  {
-    type: "function",
-    name: "setRelayer",
-    inputs: [
-      {
-        name: "relayer",
-        type: "address",
-        internalType: "address",
-      },
-    ],
-    outputs: [],
-    stateMutability: "nonpayable",
-  },
-  ...OWNABLE_ABI,
-] as any; // eslint-disable-line  @typescript-eslint/no-explicit-any
-
 export const EXTENDED_ENTROPY_ABI = [
   {
     inputs: [],

+ 9 - 9
contract_manager/src/node/utils/shell.ts

@@ -5,14 +5,14 @@ const service = tsNode.create({ ...repl.evalAwarePartialHost });
 repl.setService(service);
 repl.start();
 repl.evalCode(
-  "import { loadHotWallet, Vault } from './src/governance';" +
-    "import { SuiChain, CosmWasmChain, AptosChain, EvmChain, StarknetChain } from './src/chains';" +
-    "import { SuiPriceFeedContract } from './src/contracts/sui';" +
-    "import { CosmWasmWormholeContract, CosmWasmPriceFeedContract } from './src/contracts/cosmwasm';" +
-    "import { EvmWormholeContract, EvmPriceFeedContract, EvmEntropyContract, EvmExpressRelayContract } from './src/contracts/evm';" +
-    "import { AptosWormholeContract, AptosPriceFeedContract } from './src/contracts/aptos';" +
-    "import { StarknetPriceFeedContract } from './src/contracts/starknet';" +
-    "import { DefaultStore } from './src/store';" +
-    "import { toPrivateKey } from './src/base';" +
+  "import { loadHotWallet, Vault } from './src/node/utils/governance';" +
+    "import { SuiChain, CosmWasmChain, AptosChain, EvmChain, StarknetChain } from './src/core/chains';" +
+    "import { SuiPriceFeedContract } from './src/core/contracts/sui';" +
+    "import { CosmWasmWormholeContract, CosmWasmPriceFeedContract } from './src/core/contracts/cosmwasm';" +
+    "import { EvmWormholeContract, EvmPriceFeedContract, EvmEntropyContract } from './src/core/contracts/evm';" +
+    "import { AptosWormholeContract, AptosPriceFeedContract } from './src/core/contracts/aptos';" +
+    "import { StarknetPriceFeedContract } from './src/core/contracts/starknet';" +
+    "import { DefaultStore } from './src/node/utils/store';" +
+    "import { toPrivateKey } from './src/core/base';" +
     "DefaultStore",
 );

+ 0 - 5
contract_manager/src/node/utils/store.ts

@@ -24,7 +24,6 @@ import {
   FuelWormholeContract,
   WormholeContract,
   FuelPriceFeedContract,
-  EvmExpressRelayContract,
   TonPriceFeedContract,
   TonWormholeContract,
   IotaWormholeContract,
@@ -50,7 +49,6 @@ export class Store {
   public entropy_contracts: Record<string, EvmEntropyContract> = {};
   public pulse_contracts: Record<string, EvmPulseContract> = {};
   public wormhole_contracts: Record<string, WormholeContract> = {};
-  public express_relay_contracts: Record<string, EvmExpressRelayContract> = {};
   public tokens: Record<string, Token> = {};
   public vaults: Record<string, Vault> = {};
 
@@ -168,7 +166,6 @@ export class Store {
       [AptosPriceFeedContract.type]: AptosPriceFeedContract,
       [AptosWormholeContract.type]: AptosWormholeContract,
       [EvmEntropyContract.type]: EvmEntropyContract,
-      [EvmExpressRelayContract.type]: EvmExpressRelayContract,
       [EvmWormholeContract.type]: EvmWormholeContract,
       [FuelPriceFeedContract.type]: FuelPriceFeedContract,
       [FuelWormholeContract.type]: FuelWormholeContract,
@@ -202,8 +199,6 @@ export class Store {
           );
         if (chainContract instanceof EvmEntropyContract) {
           this.entropy_contracts[chainContract.getId()] = chainContract;
-        } else if (chainContract instanceof EvmExpressRelayContract) {
-          this.express_relay_contracts[chainContract.getId()] = chainContract;
         } else if (chainContract instanceof WormholeContract) {
           this.wormhole_contracts[chainContract.getId()] = chainContract;
         } else {

+ 8 - 1
contract_manager/store/chains/EvmChains.json

@@ -1073,7 +1073,7 @@
   {
     "id": "hyperevm_testnet",
     "mainnet": false,
-    "rpcUrl": "https://api.hyperliquid-testnet.xyz/evm",
+    "rpcUrl": "https://rpc.hyperliquid-testnet.xyz/evm",
     "networkId": 998,
     "type": "EvmChain"
   },
@@ -1258,5 +1258,12 @@
     "rpcUrl": "https://testnet.rpc.hemi.network/rpc",
     "networkId": 743111,
     "type": "EvmChain"
+  },
+  {
+    "id": "injective_evm_testnet",
+    "mainnet": false,
+    "rpcUrl": "https://k8s.testnet.json-rpc.injective.network/",
+    "networkId": 1439,
+    "type": "EvmChain"
   }
 ]

+ 1 - 1
contract_manager/store/contracts/AptosPriceFeedContracts.json

@@ -41,4 +41,4 @@
     "wormholeStateId": "0x9236893d6444b208b7e0b3e8d4be4ace90b6d17817ab7d1584e46a33ef5c50c9",
     "type": "AptosPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/AptosWormholeContracts.json

@@ -34,4 +34,4 @@
     "address": "0x9236893d6444b208b7e0b3e8d4be4ace90b6d17817ab7d1584e46a33ef5c50c9",
     "type": "AptosWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/CosmWasmPriceFeedContracts.json

@@ -84,4 +84,4 @@
     "address": "xion18nsqwhfwnqzs4vkxdr02x40awm0gz9pl0wn4ecsl8qqra2vxqppq57qx5a",
     "type": "CosmWasmPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/CosmWasmWormholeContracts.json

@@ -84,4 +84,4 @@
     "address": "xion1zfdqgkd9lcqwc4ywkeg2pr2v2p5xxa7n2s9layq2623pvhp4xv0sr4659c",
     "type": "CosmWasmWormholeContract"
   }
-]
+]

+ 5 - 60
contract_manager/store/contracts/EvmEntropyContracts.json

@@ -1,19 +1,4 @@
 [
-  {
-    "chain": "lightlink_pegasus_testnet",
-    "address": "0x8250f4aF4B972684F7b336503E2D6dFeDeB1487a",
-    "type": "EvmEntropyContract"
-  },
-  {
-    "chain": "chiliz_spicy",
-    "address": "0xD458261E832415CFd3BAE5E416FdF3230ce6F134",
-    "type": "EvmEntropyContract"
-  },
-  {
-    "chain": "mode_testnet",
-    "address": "0x98046Bd286715D3B0BC227Dd7a956b83D8978603",
-    "type": "EvmEntropyContract"
-  },
   {
     "chain": "arbitrum_sepolia",
     "address": "0x549Ebba8036Ab746611B4fFA1423eb0A4Df61440",
@@ -24,16 +9,6 @@
     "address": "0x98046Bd286715D3B0BC227Dd7a956b83D8978603",
     "type": "EvmEntropyContract"
   },
-  {
-    "chain": "lightlink_phoenix",
-    "address": "0x98046Bd286715D3B0BC227Dd7a956b83D8978603",
-    "type": "EvmEntropyContract"
-  },
-  {
-    "chain": "chiliz",
-    "address": "0x0708325268dF9F66270F1401206434524814508b",
-    "type": "EvmEntropyContract"
-  },
   {
     "chain": "arbitrum",
     "address": "0x7698E925FfC29655576D0b361D75Af579e20AdAc",
@@ -54,56 +29,21 @@
     "address": "0x4821932D0CDd71225A6d914706A621e0389D7061",
     "type": "EvmEntropyContract"
   },
-  {
-    "chain": "mode",
-    "address": "0x8D254a21b3C86D32F7179855531CE99164721933",
-    "type": "EvmEntropyContract"
-  },
   {
     "chain": "blast",
     "address": "0x5744Cbf430D99456a0A8771208b674F27f8EF0Fb",
     "type": "EvmEntropyContract"
   },
-  {
-    "chain": "zetachain_testnet",
-    "address": "0x4374e5a8b9C22271E9EB878A2AA31DE97DF15DAF",
-    "type": "EvmEntropyContract"
-  },
-  {
-    "chain": "zetachain",
-    "address": "0x36825bf3Fbdf5a29E2d5148bfe7Dcf7B5639e320",
-    "type": "EvmEntropyContract"
-  },
   {
     "chain": "base",
     "address": "0x6E7D74FA7d5c90FEF9F0512987605a6d546181Bb",
     "type": "EvmEntropyContract"
   },
-  {
-    "chain": "taiko_hekla",
-    "address": "0x98046Bd286715D3B0BC227Dd7a956b83D8978603",
-    "type": "EvmEntropyContract"
-  },
   {
     "chain": "sei_evm_mainnet",
     "address": "0x98046Bd286715D3B0BC227Dd7a956b83D8978603",
     "type": "EvmEntropyContract"
   },
-  {
-    "chain": "merlin",
-    "address": "0x36825bf3Fbdf5a29E2d5148bfe7Dcf7B5639e320",
-    "type": "EvmEntropyContract"
-  },
-  {
-    "chain": "taiko_mainnet",
-    "address": "0x26DD80569a8B23768A1d80869Ed7339e07595E85",
-    "type": "EvmEntropyContract"
-  },
-  {
-    "chain": "merlin_testnet",
-    "address": "0x5744Cbf430D99456a0A8771208b674F27f8EF0Fb",
-    "type": "EvmEntropyContract"
-  },
   {
     "chain": "etherlink_testnet",
     "address": "0x23f0e8FAeE7bbb405E7A7C3d60138FCfd43d7509",
@@ -233,5 +173,10 @@
     "chain": "soneium",
     "address": "0x0708325268dF9F66270F1401206434524814508b",
     "type": "EvmEntropyContract"
+  },
+  {
+    "chain": "hyperevm_testnet",
+    "address": "0x23f0e8FAeE7bbb405E7A7C3d60138FCfd43d7509",
+    "type": "EvmEntropyContract"
   }
 ]

+ 0 - 7
contract_manager/store/contracts/EvmExpressRelayContracts.json

@@ -1,7 +0,0 @@
-[
-  {
-    "chain": "mode",
-    "address": "0x5Cc070844E98F4ceC5f2fBE1592fB1ed73aB7b48",
-    "type": "EvmExpressRelayContract"
-  }
-]

+ 6 - 1
contract_manager/store/contracts/EvmPriceFeedContracts.json

@@ -843,5 +843,10 @@
     "chain": "mezo",
     "address": "0x2880aB155794e7179c9eE2e38200202908C17B43",
     "type": "EvmPriceFeedContract"
+  },
+  {
+    "chain": "injective_evm_testnet",
+    "address": "0xDd24F84d36BF92C65F92307595335bdFab5Bbd21",
+    "type": "EvmPriceFeedContract"
   }
-]
+]

+ 6 - 1
contract_manager/store/contracts/EvmWormholeContracts.json

@@ -833,5 +833,10 @@
     "chain": "mezo",
     "address": "0xb27e5ca259702f209a29225d0eDdC131039C9933",
     "type": "EvmWormholeContract"
+  },
+  {
+    "chain": "injective_evm_testnet",
+    "address": "0x23f0e8FAeE7bbb405E7A7C3d60138FCfd43d7509",
+    "type": "EvmWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/FuelPriceFeedContracts.json

@@ -9,4 +9,4 @@
     "address": "0x1c86fdd9e0e7bc0d2ae1bf6817ef4834ffa7247655701ee1b031b52a24c523da",
     "type": "FuelPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/FuelWormholeContracts.json

@@ -9,4 +9,4 @@
     "address": "0x1c86fdd9e0e7bc0d2ae1bf6817ef4834ffa7247655701ee1b031b52a24c523da",
     "type": "FuelWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/IotaPriceFeedContracts.json

@@ -11,4 +11,4 @@
     "wormholeStateId": "0xd43b448afc9dd01deb18273ec39d8f27ddd4dd46b0922383874331771b70df73",
     "type": "IotaPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/IotaWormholeContracts.json

@@ -9,4 +9,4 @@
     "stateId": "0xd43b448afc9dd01deb18273ec39d8f27ddd4dd46b0922383874331771b70df73",
     "type": "IotaWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/NearPriceFeedContracts.json

@@ -15,4 +15,4 @@
     "lastExecutedGovernanceSequence": 100,
     "type": "NearPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/NearWormholeContracts.json

@@ -9,4 +9,4 @@
     "address": "wormhole.wormhole.testnet",
     "type": "NearWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/StarknetPriceFeedContracts.json

@@ -9,4 +9,4 @@
     "address": "0x062ab68d8e23a7aa0d5bf4d25380c2d54f2dd8f83012e047851c3706b53d64d1",
     "type": "StarknetPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/StarknetWormholeContracts.json

@@ -9,4 +9,4 @@
     "address": "0x06fb1af6d323188105e6f10212316139dbe71650e1703af35331ceaad7aaf3bd",
     "type": "StarknetWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/SuiPriceFeedContracts.json

@@ -17,4 +17,4 @@
     "wormholeStateId": "0xcf185fbc1af3a437a600587e0b39e5fede163336ffbb7ff24dca9b6eb19d2656",
     "type": "SuiPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/SuiWormholeContracts.json

@@ -14,4 +14,4 @@
     "stateId": "0xcf185fbc1af3a437a600587e0b39e5fede163336ffbb7ff24dca9b6eb19d2656",
     "type": "SuiWormholeContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/TonPriceFeedContracts.json

@@ -9,4 +9,4 @@
     "address": "EQBU6k8HH6yX4Jf3d18swWbnYr31D3PJI7PgjXT-flsKHqql",
     "type": "TonPriceFeedContract"
   }
-]
+]

+ 1 - 1
contract_manager/store/contracts/TonWormholeContracts.json

@@ -9,4 +9,4 @@
     "address": "EQBU6k8HH6yX4Jf3d18swWbnYr31D3PJI7PgjXT-flsKHqql",
     "type": "TonWormholeContract"
   }
-]
+]

+ 243 - 0
doc/code-guidelines.md

@@ -0,0 +1,243 @@
+# Code Guidelines
+
+# Language specific
+
+[Rust code guidelines](rust-code-guidelines.md)
+
+# Make your services as resilient to errors as possible
+
+- Perform more benchmarking/add benchmarking tests through our codebase. Currently
+  there are portions of the codebase that we have unknown performance for that
+  may become more important as we scale. Most languages have benchmark test
+  capability, rust has it built in for example.
+- Implement error recovery even for unlikely cases. Think about how the service can continue to work after different failures.
+- The service should continue to work (if possible) if its dependencies are unavailable or broken.
+- Avoid the possibility of leaving the service in a state where it no longer able to start or work properly. The service should be able to recover from things like invalid files or unexpected database state. If that is not possible, provide clear error messages that explain what should be done to fix it.
+- Minimize the number of dependencies required for a service to start.
+- It should be possible to run multiple instances of each service at the same time.
+
+# Set up essential tooling
+
+- Use strongest lint settings. It is better to have at minimum pedantic warnings
+  on all projects. Good examples of bad settings: allowing `any` globally in
+  typescript, ignoring integer clippy type warnings in Rust, etc.
+- Add extensive logging, metrics and tracing capability early, much of our code is missing
+  metrics, good log handling, or ability to do introspection on code that has
+  failed in retrospect. Good example: hermes launch.
+
+# Keep the code readable and maintainable
+
+- Make heavy use of types to define behaviour. In general introducing a type can be
+  thought of as introducing a unit test. For example:
+
+      ```rust
+      struct PositiveTime(i64);
+
+      impl TryFrom<i64> for PositiveTime {
+          type Err = ();
+          fn try_from(n: i64) -> Result<Self, Self::Err> {
+              if n < 0 {
+                  return Err(());
+              }
+              return Ok(Self(n));
+          }
+      }
+
+      ```
+
+      This can be thought of reducing the valid range of i64 to one we prefer
+      (given that i64 is the native Linux time type but often we do not want these)
+      that we can enforce a compile-time. The benefit in types over unit tests is
+      simply use-at-site of a type ensure behaviour everywhere and reducing the
+      amount of unwanted behaviour in a codebase.
+
+      Currently we do not try hard enough to isolate behaviours through types.
+
+- Avoid monolithic event handlers, and avoid state handling in logic. Some
+  stateful code in our repos mixes the logic handling with the state handle
+  code which produces very long, hard to reason about code which ends up as
+  a rather large inline state machine:
+
+      Good:
+
+      ```tsx
+      function handleEvent(e, state) {
+          switch(e.type) {
+              case Event.Websocket: handleWebsocketEvent(e, state.websockets);
+              case Event.PythNet:   handlePythnetEvent(e, state.pyth_handle);
+              case ...
+          }
+      }
+
+      ```
+
+      Bad:
+
+      ```tsx
+      function handleEvent(e) {
+          // Many inlined state tracking vars. Not much better than globals.
+          var latstPythNetupdateTime = DateTime.now();
+          var clientsWaiting         = {};
+          var ...
+
+          switch(e.type) {
+             // lots of inline handling
+          }
+      }
+
+      ```
+
+- Avoid catch-all modules, I.E: `types/`, `utils/`
+- Favor Immutability and Idempotency. Both are a huge source of reducing logic bugs.
+- State should whenever possible flow top-down, I.E: create at entry point and
+  flow to other components. Global state should be avoided and no state should be
+  hidden in separate modules.
+
+      Good:
+
+      ```tsx
+      // main.ts
+      function main() {
+          const db = db.init();
+          initDb(db);
+      }
+
+      ```
+
+      Bad:
+
+      ```tsx
+      // main.ts
+      const { db } = require('db');
+      function() {
+          initDb(); // Databaes not passed, implies global use.
+      }
+
+      ```
+
+- For types/functions that are only used once, keep them close to the
+  definition. If they are re-used, try and lift them only up to a common
+  parent, in the following example types/functions only lift as far
+  as they are useful:
+
+      Example File Hierarchy:
+
+      ```
+      lib/routes.rs:validateUserId()
+      lib/routes/user.rs:type RequestUser
+      lib/routes/user/register.rs:generateRandomUsername()
+
+      ```
+
+      Good:
+
+      ```tsx
+      // Definition only applies to this function, keep locality.
+      type FeedResponse = {
+          id:   FeedId,
+          feed: Feed,
+      };
+
+      // Note the distinction between FeedResponse/Feed for DDD.
+      function getFeed(id: FeedId, db: Db): FeedResponse {
+          let feed: Feed = db.execute(FEED_QUERY, [id]);
+          return { id, feed: feed, }
+      }
+
+      ```
+
+      Bad:
+
+      ```tsx
+      import { FeedResponse } from 'types';
+      function getFeed(id: FeedId, db: Db): FeedResponse {
+          let feed = db.execute(FEED_QUERY, [id]);
+          return { id, feed: feed, }
+      }
+
+      ```
+
+- Map functionality into submodules when a module defines a category of handlers.
+  This help emphasise where code re-use should happen, for example:
+
+      Good:
+
+      ```
+      src/routes/user/register.ts
+      src/routes/user/login.ts
+      src/routes/user/add_role.ts
+      src/routes/index.ts
+
+      ```
+
+      Bad:
+
+      ```tsx
+      // src/index.ts
+      function register() { ... }
+      function login() { ... }
+      function addRole() { ... }
+      function index() { ... }
+
+      ```
+
+      Not only does this make large unwieldy files but it encourages things like
+      `types/` catch alls, or unnecessary sharing of functionality. For example
+      imagine a `usernameAsBase58` function thrown into this file, that then
+      looks useful within an unrelated to users function, it can be tempting to
+      abuse the utility function or move it to a vague catch-all location. Focus
+      on clear, API boundaries even within our own codebase.
+
+- When possible use layered architecture (onion/hexagonal/domain driven design) where
+  we separate API processing, business logic, and data logic. The benefit of this
+  is it defines API layers within the application itself:
+
+      Good:
+
+      ```tsx
+      // web/user/register.ts
+      import { registerUser, User } from 'api/user/register.ts';
+
+      // Note locality: one place use functions stay near, no utils/
+      function verifyUsername( ...
+      function verifyPassword( ...
+
+      // Locality again.
+      type RegisterRequest = {
+          ...
+      };
+
+      function register(req: RegisterRequest): void {
+          // Validation Logic Only
+          verifyUsername(req.username);
+          verifyPassword(req.password);
+
+          // Business Logic Separate
+          registerUser({
+              username: req.username,
+              password: req.password,
+          });
+      }
+
+      ```
+
+      ```tsx
+      // api/user/register.ts
+      import { storeUser, DbUser } from 'db/user';
+
+      function registerUser(user: User) {
+          const user = fetchByUsername(user.username);
+          if (user) {
+              throw "User Exists;
+          }
+
+          // Note again that the type used here differs from User (DbUser) which
+          // prevents code breakage (such as if the schema is updated but the
+          // code is not.
+          storeUser({
+              username: user.username,
+              password: hash(user.password),
+          });
+      }
+
+      ```

+ 514 - 0
doc/rust-code-guidelines.md

@@ -0,0 +1,514 @@
+# Rust code guidelines
+
+Note: general [Code Guidelines](code-guidelines.md) also apply.
+
+# Tooling
+
+**Rust toolchain version:** always pin a specific version using `rust-toolchain` or `rust-toolchain.toml` file. Use stable versions unless nightly is absolutely required. Periodically update the pinned version when possible.
+
+`Cargo.lock` should be checked in to git and used in CI and for building packages/binaries. Versions of other tools used in CI should be pinned too.
+
+**Formatting:** use `rustfmt` with default configuration. Use `taplo` with default configuration for `Cargo.toml` files.
+
+**Linting: use `clippy` with the following configuration:**
+
+- (expand to see clippy configuration)
+
+  ```toml
+  [lints.rust]
+  unsafe_code = "deny"
+
+  [lints.clippy]
+  wildcard_dependencies = "deny"
+
+  collapsible_if = "allow"
+  collapsible_else_if = "allow"
+
+  allow_attributes_without_reason = "warn"
+
+  # Panics
+  expect_used = "warn"
+  fallible_impl_from = "warn"
+  indexing_slicing = "warn"
+  panic = "warn"
+  panic_in_result_fn = "warn"
+  string_slice = "warn"
+  todo = "warn"
+  unchecked_duration_subtraction = "warn"
+  unreachable = "warn"
+  unwrap_in_result = "warn"
+  unwrap_used = "warn"
+
+  # Correctness
+  cast_lossless = "warn"
+  cast_possible_truncation = "warn"
+  cast_possible_wrap = "warn"
+  cast_sign_loss = "warn"
+  collection_is_never_read = "warn"
+  match_wild_err_arm = "warn"
+  path_buf_push_overwrite = "warn"
+  read_zero_byte_vec = "warn"
+  same_name_method = "warn"
+  suspicious_operation_groupings = "warn"
+  suspicious_xor_used_as_pow = "warn"
+  unused_self = "warn"
+  used_underscore_binding = "warn"
+  while_float = "warn"
+  ```
+
+The recommendations on this page should help with dealing with these lints.
+
+Refer also to [Clippy lints documentation](https://rust-lang.github.io/rust-clippy/master/index.html) for more information about the lints.
+
+If the lint is a false positive, put `#[allow(lint_name, reason = "...")]` on the relevant line or block and and specify the reason why the code is correct.
+
+Many of the lints (e.g. most of the panic-related lints) can be allowed globally for tests and other non-production code.
+
+# Essential crates
+
+- `tracing` for logging and tracing
+- `opentelemetry` for metrics
+- `anyhow` for error handling in services
+- `thiserror` for error handling in libraries
+- `backoff` for retrying (take note of the distinction between transient and permanent errors).
+- `axum` and `utoipa` for API server implementation
+- `reqwest` for HTTP client
+- `chrono` for date and time manipulation
+- `config` for flexible config loading
+- `humantime` for printing and parsing duration
+- `tokio` for async runtime
+- `itertools` for extra operations on iterators
+- `derive_more` and `strum` for more derives
+- `proptest`, `mry` for testing
+- `criterion` for benchmarking
+
+# Avoiding panics
+
+Panics are unrecoverable errors that unwind the current thread. If a panic occurs in the main thread, the whole program may terminate. If a panic occurs in a task spawned on an async runtime, that task will terminate.
+
+Panics are dangerous because they can arise from concise code constructions (such as slice indexing with `a[n]`) and have vast effects on the program. Minimize the possibility of panics with the help of the relevant `clippy` lints. Handle the unexpected case properly no matter how unlikely it seems.
+
+**Common sources of panics:**
+
+| Source of panic | Non-panicking alternatives |
+| --------------- | -------------------------- |
+
+| `.unwrap()`, `.expect(...)`,
+`panic!()`, `assert*!()` | `Result`-based handling using `anyhow` crate:
+`.context()?` , `.with_context()?` ,
+`bail!()`, `ensure!()`. |
+| `unimplemented!()`, `todo!()`, `unreachable!()` | — |
+| Indexing out of bounds on slices, strings, `Vec`, `VecDeque`: `&arr[x]` , `&arr[x..y]` | `.get()`, `.get_mut()` |
+| Indexing with a range if min > max: `&arr[to..from]` | `.get()`, `.get_mut()` |
+| Indexing a `HashMap` or `BTreeMap` on a non-existing key: `&map[key]` | `.get()`, `.get_mut()`, `.entry()` |
+| Indexing a non-ASCII string not at char boundaries: `&"😢😢😢"[0..1]` | `.get()`, `.get_mut()` |
+| `Vec` methods: `.insert()`, `.remove()`, `.drain()`, `.split_off()` and many more | There are checked alternatives for some but not all of them. |
+| Division by zero: `a / b`, `a % b` | `.checked_div()`, `.checked_rem()`, `/ NonZero*` |
+
+Think about the cases that could cause your code to panic. Try to rewrite the code in a way that avoids a possible panic.
+
+Here are some tips to solve `clippy` warnings and avoid panics:
+
+- **Use `Result` type and return the error to the caller.** Use `.context()` or `.with_context()` from `anyhow` crate to add relevant information to the error. Use `.ok_or_else()` or `.context()` to convert `Option` to `Result`.
+
+  ⛔ Bad:
+
+  ```rust
+  pub fn best_bid(response: &Response) -> String {
+      response.bids[0].clone()
+  }
+  ```
+
+  ✅ Good:
+
+  ```rust
+  pub fn best_bid(response: &Response) -> anyhow::Result<String> {
+      Ok(response.bids.first().context("expected 1 item in bids, got 0")?.clone())
+  }
+  ```
+
+- **If propagation with `?` doesn’t work because of error type, convert the error type with `.map_err()`.**
+
+  ✅ Good (`binary_search` returns `Err(index)` instead of a well-formed error, so we create our own error value):
+
+  ```rust
+      items
+          .binary_search(&needle)
+          .map_err(|_| anyhow!("item not found: {:?}", needle))?;
+  ```
+
+  ✅ Good (`?` can’t convert from `Box<dyn Error>` to `anyhow::Error`, but there is a function that converts it):
+
+  ```rust
+  let info = message.info().map_err(anyhow::Error::from_boxed)?;
+  ```
+
+- **If the error is in a non-Result function inside an iterator chain (e.g. `.map()`) or a combinator (e.g. `.unwrap_or_else()`), consider rewriting it as a plain `for` loop or `match`/`if let` to allow error propagation.** (If you’re determined to make iterators work, `fallible-iterator` crate may also be useful.)
+
+  ⛔ Bad (unwraps the error just because `?` doesn’t work):
+
+  ```rust
+  fn check_files(paths: &[&Path]) -> anyhow::Result<()> {
+      let good_paths: Vec<&Path> = paths
+          .iter()
+          .copied()
+          .filter(|path| fs::read_to_string(path).unwrap().contains("magic"))
+          .collect();
+      //...
+  }
+  ```
+
+  ✅ Good:
+
+  ```rust
+  fn check_files(paths: &[&Path]) -> anyhow::Result<()> {
+      let mut good_paths = Vec::new();
+      for path in paths {
+          if fs::read_to_string(path)?.contains("magic") {
+              good_paths.push(path);
+          }
+      }
+      //...
+  }
+  ```
+
+- **Log the error and return early or skip an item.**
+
+  ⛔ Bad (panics if we add too many publishers):
+
+  ```rust
+  let publisher_count = u16::try_from(prices.len())
+      .expect("too many publishers");
+  ```
+
+  ✅ Good:
+
+  ```rust
+  let Ok(publisher_count) = u16::try_from(prices.len()) else {
+      error!("too many publishers ({})", prices.len());
+      return Default::default();
+  };
+  ```
+
+  ⛔ Bad (terminates on error) (and yes, [it can fail](https://www.greyblake.com/blog/when-serde-json-to-string-fails/)):
+
+  ```rust
+  loop {
+      //...
+      yield Ok(
+          serde_json::to_string(&response).expect("should not fail") + "\n"
+      );
+  }
+  ```
+
+  ✅ Good (`return` instead of `continue` could also be reasonable):
+
+  ```rust
+  loop {
+      //...
+      match serde_json::to_string(&response) {
+          Ok(json) => {
+              yield Ok(json + "\n");
+          }
+          Err(err) => {
+              error!("json serialization error for {:?}: {}", response, err);
+              continue;
+          }
+      }
+  }
+  ```
+
+- **Supply a sensible default:**
+
+  ⛔ Bad:
+
+  ```rust
+  let interval_us: u64 = snapshot_interval
+      .as_micros()
+      .try_into()
+      .expect("snapshot_interval overflow");
+  ```
+
+  ✅ Better:
+
+  ```rust
+  let interval_us =
+      u64::try_from(snapshot_interval.as_micros()).unwrap_or_else(|_| {
+          error!(
+              "invalid snapshot_interval in config: {:?}, defaulting to 1 min",
+              snapshot_interval
+          );
+          60_000_000 // 1 min
+      });
+  ```
+
+- **Avoid checking a condition and then unwrapping.** Instead, combine the check and access to the value using `match`, `if let`, and combinators such as `map_or`, `some_or`, etc.
+
+  🟡 Not recommended:
+
+  ```rust
+  if !values.is_empty() {
+  		process_one(&values[0]);
+  }
+  ```
+
+  ✅ Good:
+
+  ```rust
+  if let Some(first_value) = values.first() {
+      process_one(first_value);
+  }
+  ```
+
+  🟡 Not recommended:
+
+  ```rust
+  .filter(|price| {
+      median_price.is_none() || price < &median_price.expect("should not fail")
+  })
+  ```
+
+  ✅ Good:
+
+  ```rust
+  .filter(|price| median_price.is_none_or(|median_price| price < &median_price))
+  ```
+
+  🟡 Not recommended:
+
+  ```rust
+  if data.len() < header_len {
+      bail!("data too short");
+  }
+  let header = &data[..header_len];
+  let payload = &data[header_len..];
+  ```
+
+  ✅ Good:
+
+  ```rust
+  let (header, payload) = data
+      .split_at_checked(header_len)
+      .context("data too short")?;
+  ```
+
+- **Avoid the panic by introducing a constant or move the panic inside the constant initialization.** Panicking in const initializers is safe because it happens at compile time.
+
+  🟡 Not recommended:
+
+  ```rust
+  price.unwrap_or(Price::new(PRICE_FEED_EPS).expect("should never fail"))
+  ```
+
+  ✅ Good:
+
+  ```rust
+  #[allow(clippy::unwrap_used, reason = "safe in const")]
+  const MIN_POSITIVE_PRICE: Price =
+      Price(NonZeroI64::new(PRICE_FEED_EPS).unwrap());
+  ```
+
+- **If it’s not possible to refactor the code, allow the lint and specify the reason why the code cannot fail. Prefer `.expect()` over `.unwrap()` and specify the failure in the argument.** This is reasonable if the failure is truly impossible or if a failure would be so critical that the service cannot continue working.
+
+  🟡 Not recommended:
+
+  ```rust
+  Some(prices[prices.len() / 2])
+  ```
+
+  ✅ Good:
+
+  ```rust
+  #[allow(
+      clippy::indexing_slicing,
+      reason = "prices are not empty, prices.len() / 2 < prices.len()"
+  )]
+  Some(prices[prices.len() / 2])
+  ```
+
+  🟡 Not recommended:
+
+  ```rust
+  let expiry_time = expiry_times.get(self.active_index).unwrap();
+  ```
+
+  ✅ Better:
+
+  ```rust
+  #[allow(
+      clippy::expect_used,
+      reason = "we only assign valid indexes to `self.active_index`"
+  )]
+  let expiry_time = expiry_times
+      .get(self.active_index)
+      .expect("invalid active index");
+  ```
+
+# Avoid unchecked arithmetic operations
+
+Be aware that some basic arithmetic operators (`+`, `-`, `*`, unary negation with `-`) and some functions (`.abs()`, `.pow()`, `.next_multiple_of()`, etc.) can overflow. These operators and functions will panic only in debug mode (more specifically, when debug assertions are enabled). In release mode, they will quietly produce an invalid value. For example, `200u8 + 150u8` evaluates to 94 in release mode. Be especially careful when subtracting unsigned integers because even small values can produce an overflow: `2u32 - 4u32` evaluates to 4294967294.
+
+Consider using an alternative function that produces a more reasonable value:
+
+- Use `.checked_*()` functions that return `None` in case of overflow.
+- Use `.saturating_*()` functions that will cap on MIN or MAX value.
+
+Use `.wrapping_*()` functions if you expect the value to wrap around.
+
+⛔ Bad:
+
+```rust
+charge_payment(amount + fee);
+```
+
+✅ Good:
+
+```rust
+let total = amount
+    .checked_add(fee)
+    .with_context(|| {
+        format!("total amount overflow: amount = {amount}, fee = {fee}")
+    })?;
+charge_payment(total);
+```
+
+<aside>
+💡
+
+You can catch unchecked arithmetic operations with `clippy::arithmetic_side_effects` lint, but we do not enable this lint. For now. 😈
+
+</aside>
+
+# Avoid implicit wrapping and truncation with `as`
+
+Limit the use of `as` keyword for converting values between numeric types. While `as` is often the most convenient option, it’s quite bad at expressing intent. `as` can do conversions with different semantics. In the case of integers, it can do both **lossless conversions** (e.g. from `u8` to `u32`; from `u32` to `i64`) and **lossy conversions** (e.g. `258_u32 as u8` evaluates to 2; `200_u8 as i8` evaluates to -56).
+
+- Always use `.into()` and `T::from()` instead of `as` for lossless conversions. This makes the intent more clear.
+- Only use `as` for lossy conversions when you specifically intend to perform a lossy conversion and are aware of how it affects the value. Add an `#[allow]` attribute and specify the reason.
+- See if you can choose a more suitable type to avoid the conversion entirely.
+- When a lossless conversion is necessary but it’s not possible using `From` and `Into`, use `.try_into()` or `T::try_from()` instead of `as` and handle the error case appropriately.
+- When correctness of the resulting value is not super important (e.g. it’s used for metrics) and the failure is unlikely, you can use `.shrink()` or `T::shrink_from()` provided by `truncate-integer` crate. These functions perform a saturating conversion, i.e. they return a min or max value when the value cannot be represented in the new type. In most cases it’s better than what `as` would produce.
+
+⛔ Bad (if `count` is negative, it will attempt a huge allocation which can crash the process):
+
+```rust
+let count: i32 = /*...*/;
+vec.resize(count as usize);
+
+```
+
+✅ Good:
+
+```rust
+vec.resize(
+    count
+        .try_into()
+        .with_context(|| format!("invalid count: {count}"))?
+);
+```
+
+⛔ Bad (truncates on overflow, producing an invalid value):
+
+```rust
+ABC_DURATION.record(started_at.elapsed().as_micros() as u64, &[]);
+```
+
+✅ Better (saturates to max value, using `truncate-integer` crate):
+
+```rust
+ABC_DURATION.record(started_at.elapsed().as_micros().shrink(), &[]);
+```
+
+# Error handling
+
+End-user applications and services can use a single error type throughout the code. We use `anyhow` for that. Fine-grained error types are often not beneficial in these applications because most of the time you just catch and log the error at a certain level.
+
+Libraries, on the other hand, often need to return a detailed error to the caller. This is where fine-grained error types come in handy. You should make sure that they are well formed:
+
+- Error types should implement `Debug`, `Display` and `Error`. When implementing `Error`, the `source()` method should be properly implemented. The easiest way to do these things is the `thiserror` crate and the `#[error]` and `#[source]` attributes it provides.
+- If the location of the error can be ambiguous, consider adding a backtrace to it. For custom error types you have to do it manually by storing `std::backtrace::Backtrace` inside the error.
+
+Other recommendations:
+
+- The error message should contain relevant context that can help understand the issue.
+
+  ⛔ Bad:
+
+  ```rust
+  let timestamp = args.timestamp.try_into()?;
+  ```
+
+  ✅ Good (using `anyhow::Context`):
+
+  ```rust
+  let timestamp = args
+      .timestamp
+      .try_into()
+      .with_context(|| {
+          format!("invalid timestamp received from Hermes: {:?}", args.timestamp)
+      })?;
+  ```
+
+  ⛔ Bad:
+
+  ```rust
+  let exponent = config.feeds.get(feed_id).context("missing feed")?.exponent;
+  ```
+
+  ✅ Good:
+
+  ```rust
+  let exponent = config
+      .feeds
+      .get(feed_id)
+      .with_context(|| {
+  			format!("missing feed {:?} in config", feed_id)
+      })?
+      .exponent;
+  ```
+
+- When wrapping another error or converting error type, preserve the error source instead of converting it to `String`. Especially avoid calling `to_string()` on `anyhow::Error` or formatting it with `{}` because it erases context and backtrace information.
+
+  ⛔ Bad (loses information about source errors and backtrace):
+
+  ```rust
+  if let Err(err) = parse(data) {
+      bail!("parsing failed: {err}");
+  }
+  // OR
+  parse(data).map_err(|err| format!("parsing failed: {err}"))?;
+  ```
+
+  ⛔ Better but still bad (preserves source errors and backtrace, but the error will have two backtraces now):
+
+  ```rust
+  if let Err(err) = parse(data) {
+      bail!("parsing failed for {data:?}: {err:?}");
+  }
+  ```
+
+  ✅ Good (returns an error with proper source and backtrace):
+
+  ```rust
+  parse(data).with_context(|| format!("failed to parse {data:?}"))?;
+  ```
+
+  ⛔ Bad (loses information about source errors and backtrace):
+
+  ```rust
+  warn!(%err, ?data, "parsing failed");
+  // OR
+  warn!("parsing failed: {}", err);
+  ```
+
+  ✅ Better (returns full information about the error in a structured way):
+
+  ```rust
+  warn!(?err, ?data, "parsing failed");
+  ```
+
+# Other recommendations
+
+- Avoid writing unsafe code. Unsafe code is hard to get right and is only needed for really low level stuff.
+- Prefer default requirements (e.g. `time = "0.1.12"`) when specifying dependencies and to allow semver-compatible upgrades. Avoid using `<=` requirements because they break semver. Never use `*` requirement.
+- Avoid using macros if the same result can be achieved without a macro.

+ 5 - 5
governance/xc_admin/packages/xc_admin_cli/src/index.ts

@@ -57,7 +57,7 @@ import {
 } from "@pythnetwork/pyth-solana-receiver";
 import {
   SOLANA_LAZER_PROGRAM_ID,
-  SOLANA_STORAGE_ID,
+  SOLANA_LAZER_STORAGE_ID,
 } from "@pythnetwork/pyth-lazer-sdk";
 
 import { LedgerNodeWallet } from "./ledger";
@@ -978,7 +978,7 @@ multisigCommand(
       .update(trustedSigner, expiryTime)
       .accounts({
         topAuthority: await vault.getVaultAuthorityPDA(targetCluster),
-        storage: SOLANA_STORAGE_ID,
+        storage: new PublicKey(SOLANA_LAZER_STORAGE_ID),
       })
       .instruction();
 
@@ -1010,7 +1010,7 @@ multisigCommand(
     const trustedSigner = Buffer.from(options.signer, "hex");
     const expiryTime = new BN(options.expiryTime);
 
-    const programId = SOLANA_LAZER_PROGRAM_ID;
+    const programId = new PublicKey(SOLANA_LAZER_PROGRAM_ID);
     const programDataAccount = PublicKey.findProgramAddressSync(
       [programId.toBuffer()],
       BPF_UPGRADABLE_LOADER,
@@ -1039,7 +1039,7 @@ multisigCommand(
     // Create Anchor program instance
     const lazerProgram = new Program(
       lazerIdl as Idl,
-      SOLANA_LAZER_PROGRAM_ID,
+      programId,
       vault.getAnchorProvider(),
     );
 
@@ -1048,7 +1048,7 @@ multisigCommand(
       .updateEcdsaSigner(trustedSigner, expiryTime)
       .accounts({
         topAuthority: await vault.getVaultAuthorityPDA(targetCluster),
-        storage: SOLANA_STORAGE_ID,
+        storage: new PublicKey(SOLANA_LAZER_STORAGE_ID),
       })
       .instruction();
 

+ 1 - 0
governance/xc_admin/packages/xc_admin_common/src/chains.ts

@@ -245,6 +245,7 @@ export const RECEIVER_CHAINS = {
   worldchain_testnet: 50123,
   mezo_testnet: 50124,
   hemi_testnet: 50125,
+  injective_evm_testnet: 50126,
 };
 
 // If there is any overlapping value the receiver chain will replace the wormhole

+ 3 - 1
governance/xc_admin/packages/xc_admin_common/src/multisig_transaction/index.ts

@@ -166,7 +166,9 @@ export class MultisigParser {
       return SolanaStakingMultisigInstruction.fromTransactionInstruction(
         instruction,
       );
-    } else if (instruction.programId.equals(SOLANA_LAZER_PROGRAM_ID)) {
+    } else if (
+      instruction.programId.equals(new PublicKey(SOLANA_LAZER_PROGRAM_ID))
+    ) {
       return LazerMultisigInstruction.fromInstruction(instruction);
     } else {
       return UnrecognizedProgram.fromTransactionInstruction(instruction);

+ 4 - 1
lazer/contracts/solana/package.json

@@ -10,7 +10,10 @@
     "check-trusted-signer": "pnpm ts-node scripts/check_trusted_signer.ts"
   },
   "dependencies": {
-    "@coral-xyz/anchor": "^0.30.1"
+    "@coral-xyz/anchor": "^0.30.1",
+    "@pythnetwork/pyth-lazer-sdk": "workspace:*",
+    "@solana/web3.js": "^1.98.0",
+    "@solana/buffer-layout": "^4.0.1"
   },
   "devDependencies": {
     "@types/bn.js": "^5.1.0",

+ 2 - 0
lazer/contracts/solana/programs/pyth-lazer-solana-contract/src/lib.rs

@@ -1,3 +1,5 @@
+#![allow(unexpected_cfgs)] // anchor macro triggers it
+
 mod signature;
 
 use {

+ 50 - 0
lazer/contracts/solana/scripts/add_ed25519_signer.ts

@@ -0,0 +1,50 @@
+import * as anchor from "@coral-xyz/anchor";
+import { Program } from "@coral-xyz/anchor";
+import { PythLazerSolanaContract } from "../target/types/pyth_lazer_solana_contract";
+import * as pythLazerSolanaContractIdl from "../target/idl/pyth_lazer_solana_contract.json";
+import yargs from "yargs/yargs";
+import { readFileSync } from "fs";
+import NodeWallet from "@coral-xyz/anchor/dist/cjs/nodewallet";
+
+// Add a trusted signer or change its expiry time.
+//
+// Example:
+// pnpm ts-node scripts/add_ed25519_signer.ts --url 'https://api.testnet.solana.com' \
+//    --keypair-path .../key.json --trusted-signer HaXscpSUcbCLSnPQB8Z7H6idyANxp1mZAXTbHeYpfrJJ \
+//    --expiry-time-seconds 2057930841
+async function main() {
+  let argv = await yargs(process.argv.slice(2))
+    .options({
+      url: { type: "string", demandOption: true },
+      "keypair-path": { type: "string", demandOption: true },
+      "trusted-signer": { type: "string", demandOption: true },
+      "expiry-time-seconds": { type: "number", demandOption: true },
+    })
+    .parse();
+
+  const keypair = anchor.web3.Keypair.fromSecretKey(
+    new Uint8Array(JSON.parse(readFileSync(argv.keypairPath, "ascii"))),
+  );
+
+  const wallet = new NodeWallet(keypair);
+  const connection = new anchor.web3.Connection(argv.url, {
+    commitment: "confirmed",
+  });
+  const provider = new anchor.AnchorProvider(connection, wallet);
+
+  const program: Program<PythLazerSolanaContract> = new Program(
+    pythLazerSolanaContractIdl as PythLazerSolanaContract,
+    provider,
+  );
+
+  await program.methods
+    .update(
+      new anchor.web3.PublicKey(argv.trustedSigner),
+      new anchor.BN(argv.expiryTimeSeconds),
+    )
+    .accounts({})
+    .rpc();
+  console.log("signer updated");
+}
+
+main();

+ 4 - 4
lazer/contracts/solana/scripts/check_trusted_signer.ts

@@ -40,11 +40,11 @@ async function main() {
 
   // Print storage info
   console.log("Storage Account Info:");
-  console.log("--------------------");
+  console.log("---------------------");
   console.log("Top Authority:", storage.topAuthority.toBase58());
   console.log("Treasury:", storage.treasury.toBase58());
-  console.log("\nTrusted Signers:");
-  console.log("----------------");
+  console.log("\nTrusted Ed25519 Signers:");
+  console.log("------------------------");
 
   const trustedSigners = storage.trustedSigners.slice(
     0,
@@ -67,7 +67,7 @@ async function main() {
   }
 
   console.log("\nTrusted ECDSA Signers:");
-  console.log("----------------");
+  console.log("----------------------");
 
   const trustedEcdsaSigners = storage.trustedEcdsaSigners.slice(
     0,

+ 76 - 0
lazer/contracts/solana/scripts/verify_ed25519_message.ts

@@ -0,0 +1,76 @@
+import * as anchor from "@coral-xyz/anchor";
+import { Program } from "@coral-xyz/anchor";
+import { PythLazerSolanaContract } from "../target/types/pyth_lazer_solana_contract";
+import * as pythLazerSolanaContractIdl from "../target/idl/pyth_lazer_solana_contract.json";
+import yargs from "yargs/yargs";
+import { readFileSync } from "fs";
+import NodeWallet from "@coral-xyz/anchor/dist/cjs/nodewallet";
+import { createEd25519Instruction } from "../src/ed25519";
+import {
+  sendAndConfirmTransaction,
+  SendTransactionError,
+  SYSVAR_INSTRUCTIONS_PUBKEY,
+  Transaction,
+} from "@solana/web3.js";
+
+async function main() {
+  let argv = await yargs(process.argv.slice(2))
+    .options({
+      url: { type: "string", demandOption: true },
+      "keypair-path": { type: "string", demandOption: true },
+      message: { type: "string", demandOption: true },
+    })
+    .parse();
+
+  const keypair = anchor.web3.Keypair.fromSecretKey(
+    new Uint8Array(JSON.parse(readFileSync(argv.keypairPath, "ascii"))),
+  );
+
+  const wallet = new NodeWallet(keypair);
+  const connection = new anchor.web3.Connection(argv.url, {
+    commitment: "confirmed",
+  });
+  const provider = new anchor.AnchorProvider(connection, wallet);
+
+  const program: Program<PythLazerSolanaContract> = new Program(
+    pythLazerSolanaContractIdl as PythLazerSolanaContract,
+    provider,
+  );
+
+  const instructionMessage = Buffer.from(argv.message, "hex");
+  const ed25519Instruction = createEd25519Instruction(
+    instructionMessage,
+    1,
+    12,
+  );
+  const lazerInstruction = await program.methods
+    .verifyMessage(instructionMessage, 0, 0)
+    .accounts({
+      payer: wallet.publicKey,
+      instructionsSysvar: SYSVAR_INSTRUCTIONS_PUBKEY,
+    })
+    .instruction();
+
+  const transaction = new Transaction().add(
+    ed25519Instruction,
+    lazerInstruction,
+  );
+  console.log("transaction:", transaction);
+
+  try {
+    const signature = await sendAndConfirmTransaction(
+      connection,
+      transaction,
+      [wallet.payer],
+      {
+        skipPreflight: true,
+      },
+    );
+    console.log("Transaction confirmed with signature:", signature);
+  } catch (e) {
+    console.log("error", e);
+    console.log(e.getLogs());
+  }
+}
+
+main();

+ 0 - 0
lazer/sdk/js/src/ed25519.ts → lazer/contracts/solana/src/ed25519.ts


部分文件因为文件数量过多而无法显示