backup write support

apps/hip-3-pusher/src/pusher/config.py

@@ -1,4 +1,5 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+from typing import Optional
 
 STALE_TIMEOUT_SECONDS = 5
 
@@ -27,6 +28,7 @@ class HermesConfig(BaseModel):
 
 class HyperliquidConfig(BaseModel):
     hyperliquid_ws_urls: list[str]
+    backup_push_urls: Optional[list[str]] = Field(default_factory=list)
     market_name: str
     market_symbol: str
     use_testnet: bool

apps/hip-3-pusher/src/pusher/kms_signer.py

@@ -7,7 +7,6 @@ from eth_keys.backends.native.ecdsa import N as SECP256K1_N
 from eth_keys.datatypes import Signature
 from eth_utils import keccak, to_hex
 from hyperliquid.exchange import Exchange
-from hyperliquid.utils.constants import TESTNET_API_URL, MAINNET_API_URL
 from hyperliquid.utils.signing import get_timestamp_ms, action_hash, construct_phantom_agent, l1_payload
 from loguru import logger
 from pathlib import Path
@@ -27,10 +26,9 @@ def _init_client():
 
 
 class KMSSigner:
-    def __init__(self, config: Config):
+    def __init__(self, config: Config, publisher_exchanges: list[Exchange]):
         self.use_testnet = config.hyperliquid.use_testnet
-        url = TESTNET_API_URL if self.use_testnet else MAINNET_API_URL
-        self.oracle_publisher_exchange: Exchange = Exchange(wallet=None, base_url=url)
+        self.publisher_exchanges = publisher_exchanges
 
         # AWS client and public key load
         self.client = _init_client()
@@ -79,11 +77,20 @@ class KMSSigner:
             nonce=timestamp,
             is_mainnet= self.use_testnet,
         )
-        return self.oracle_publisher_exchange._post_action(
-            action=action,
-            signature=signature,
-            nonce=timestamp,
-        )
+        return self._send_update(action, signature, timestamp)
+
+    def _send_update(self, action, signature, timestamp):
+        for exchange in self.publisher_exchanges:
+            try:
+                return exchange._post_action(
+                    action=action,
+                    signature=signature,
+                    nonce=timestamp,
+                )
+            except Exception as e:
+                logger.exception("perp_deploy_set_oracle exception for endpoint: {} error: {}", exchange.base_url, e)
+
+        return None
 
     def sign_l1_action(self, action, nonce, is_mainnet):
         hash = action_hash(action, vault_address=None, nonce=nonce, expires_after=None)

apps/hip-3-pusher/src/pusher/publisher.py

@@ -21,22 +21,21 @@ class Publisher:
     """
     def __init__(self, config: Config, price_state: PriceState, metrics: Metrics):
         self.publish_interval = float(config.hyperliquid.publish_interval)
-        self.kms_signer = None
-        self.enable_kms = False
         self.use_testnet = config.hyperliquid.use_testnet
+        self.push_urls = [TESTNET_API_URL if self.use_testnet else MAINNET_API_URL] + config.hyperliquid.backup_push_urls
 
-        if config.kms.enable_kms:
-            self.enable_kms = True
-            oracle_account = None
-            self.kms_signer = KMSSigner(config)
-        else:
-            oracle_pusher_key_path = config.hyperliquid.oracle_pusher_key_path
-            oracle_pusher_key = Path(oracle_pusher_key_path).read_text().strip()
+        self.kms_signer = None
+        self.enable_kms = False
+        oracle_account = None
+        if not config.kms.enable_kms:
+            oracle_pusher_key = Path(config.hyperliquid.oracle_pusher_key_path).read_text().strip()
             oracle_account: LocalAccount = Account.from_key(oracle_pusher_key)
             logger.info("oracle pusher local pubkey: {}", oracle_account.address)
+        self.publisher_exchanges = [Exchange(wallet=oracle_account, base_url=url) for url in self.push_urls]
+        if config.kms.enable_kms:
+            self.enable_kms = True
+            self.kms_signer = KMSSigner(config, self.publisher_exchanges)
 
-        url = TESTNET_API_URL if self.use_testnet else MAINNET_API_URL
-        self.oracle_publisher_exchange: Exchange = Exchange(wallet=oracle_account, base_url=url)
         self.market_name = config.hyperliquid.market_name
         self.market_symbol = config.hyperliquid.market_symbol
         self.enable_publish = config.hyperliquid.enable_publish
@@ -72,6 +71,7 @@ class Publisher:
         # TODO: "Each update can change oraclePx and markPx by at most 1%."
         # TODO: "The markPx cannot be updated such that open interest would be 10x the open interest cap."
 
+        push_response = None
         if self.enable_publish:
             if self.enable_kms:
                 push_response = self.kms_signer.set_oracle(
@@ -81,18 +81,38 @@ class Publisher:
                     external_perp_pxs=external_perp_pxs,
                 )
             else:
-                push_response = self.oracle_publisher_exchange.perp_deploy_set_oracle(
-                    dex=self.market_name,
+                push_response = self._send_update(
                     oracle_pxs=oracle_pxs,
                     all_mark_pxs=mark_pxs,
                     external_perp_pxs=external_perp_pxs,
                 )
 
-            # TODO: Look at specific error responses and log/alert accordingly
-            logger.debug("publish: push response: {} {}", push_response, type(push_response))
-            status = push_response.get("status", "")
-            if status == "ok":
-                self.metrics.successful_push_counter.add(1, self.metrics_labels)
-            elif status == "err":
-                self.metrics.failed_push_counter.add(1, self.metrics_labels)
-                logger.error("publish: publish error: {}", push_response)
+        self._handle_response(push_response)
+
+    def _send_update(self, oracle_pxs, all_mark_pxs, external_perp_pxs):
+        for exchange in self.publisher_exchanges:
+            try:
+                return exchange.perp_deploy_set_oracle(
+                    dex=self.market_name,
+                    oracle_pxs=oracle_pxs,
+                    all_mark_pxs=all_mark_pxs,
+                    external_perp_pxs=external_perp_pxs,
+                )
+            except Exception as e:
+                logger.exception("perp_deploy_set_oracle exception for endpoint: {} error: {}", exchange.base_url, e)
+
+        return None
+
+    def _handle_response(self, response):
+        if response is None:
+            logger.error("Push API call failed")
+            self.metrics.failed_push_counter.add(1, self.metrics_labels)
+            return
+
+        logger.debug("publish: push response: {} {}", response, type(response))
+        status = response.get("status")
+        if status == "ok":
+            self.metrics.successful_push_counter.add(1, self.metrics_labels)
+        elif status == "err":
+            self.metrics.failed_push_counter.add(1, self.metrics_labels)
+            logger.error("publish: publish error: {}", response)