feat: support multiple JITO endpoints with round-robin retry (#2664)

* feat: support multiple JITO endpoints with round-robin retry

Co-Authored-By: Ali Behjati <ali@dourolabs.xyz>

* fix: rename jito-endpoint to jito-endpoints

Co-Authored-By: Ali Behjati <ali@dourolabs.xyz>

* fix: update readme, logs, version

* fix: simplify retry logic, avoid retrying on top of next push attempt

* chore: bump ver

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Ali Behjati <ali@dourolabs.xyz>
Co-authored-by: Tejas Badadare <tejasbadadare@gmail.com>

apps/price_pusher/README.md

@@ -159,7 +159,7 @@ pnpm run start solana \
   --endpoint https://api.mainnet-beta.solana.com \
   --keypair-file ./id.json \
   --shard-id 1 \
-  --jito-endpoint mainnet.block-engine.jito.wtf \
+  --jito-endpoints mainnet.block-engine.jito.wtf,ny.mainnet.block-engine.jito.wtf \
   --jito-keypair-file ./jito.json \
   --jito-tip-lamports 100000 \
   --jito-bundle-size 5 \

apps/price_pusher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/price-pusher",
-  "version": "9.3.4",
+  "version": "9.3.5",
   "description": "Pyth Price Pusher",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",

apps/price_pusher/src/solana/command.ts

@@ -49,8 +49,8 @@ export default {
       type: "number",
       default: 50000,
     } as Options,
-    "jito-endpoint": {
-      description: "Jito endpoint",
+    "jito-endpoints": {
+      description: "Jito endpoint(s) - comma-separated list of endpoints",
       type: "string",
       optional: true,
     } as Options,
@@ -117,7 +117,7 @@ export default {
       pythContractAddress,
       pushingFrequency,
       pollingFrequency,
-      jitoEndpoint,
+      jitoEndpoints,
       jitoKeypairFile,
       jitoTipLamports,
       dynamicJitoTips,
@@ -209,7 +209,18 @@ export default {
         Uint8Array.from(JSON.parse(fs.readFileSync(jitoKeypairFile, "ascii"))),
       );
 
-      const jitoClient = searcherClient(jitoEndpoint, jitoKeypair);
+      const jitoEndpointsList = jitoEndpoints
+        .split(",")
+        .map((endpoint: string) => endpoint.trim());
+      const jitoClients: SearcherClient[] = jitoEndpointsList.map(
+        (endpoint: string) => {
+          logger.info(
+            `Constructing Jito searcher client from endpoint ${endpoint}`,
+          );
+          return searcherClient(endpoint, jitoKeypair);
+        },
+      );
+
       solanaPricePusher = new SolanaPricePusherJito(
         pythSolanaReceiver,
         hermesClient,
@@ -218,13 +229,17 @@ export default {
         jitoTipLamports,
         dynamicJitoTips,
         maxJitoTipLamports,
-        jitoClient,
+        jitoClients,
         jitoBundleSize,
         updatesPerJitoBundle,
+        // Set max retry time to pushing frequency, since we want to stop retrying before the next push attempt
+        pushingFrequency * 1000,
         lookupTableAccount,
       );
 
-      onBundleResult(jitoClient, logger.child({ module: "JitoClient" }));
+      jitoClients.forEach((client, index) => {
+        onBundleResult(client, logger.child({ module: `JitoClient-${index}` }));
+      });
     } else {
       solanaPricePusher = new SolanaPricePusher(
         pythSolanaReceiver,

apps/price_pusher/src/solana/solana.ts

@@ -166,9 +166,10 @@ export class SolanaPricePusherJito implements IPricePusher {
     private defaultJitoTipLamports: number,
     private dynamicJitoTips: boolean,
     private maxJitoTipLamports: number,
-    private searcherClient: SearcherClient,
+    private searcherClients: SearcherClient[],
     private jitoBundleSize: number,
     private updatesPerJitoBundle: number,
+    private maxRetryTimeMs: number,
     private addressLookupTableAccount?: AddressLookupTableAccount,
   ) {}
 
@@ -194,10 +195,6 @@ export class SolanaPricePusherJito implements IPricePusher {
     }
   }
 
-  private async sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
   async updatePriceFeed(priceIds: string[]): Promise<void> {
     const recentJitoTip = await this.getRecentJitoTipLamports();
     const jitoTip =
@@ -243,32 +240,15 @@ export class SolanaPricePusherJito implements IPricePusher {
         jitoBundleSize: this.jitoBundleSize,
       });
 
-      let retries = 60;
-      while (retries > 0) {
-        try {
-          await sendTransactionsJito(
-            transactions,
-            this.searcherClient,
-            this.pythSolanaReceiver.wallet,
-          );
-          break;
-        } catch (err: any) {
-          if (err.code === 8 && err.details?.includes("Rate limit exceeded")) {
-            this.logger.warn("Rate limit hit, waiting before retry...");
-            await this.sleep(1100); // Wait slightly more than 1 second
-            retries--;
-            if (retries === 0) {
-              this.logger.error("Max retries reached for rate limit");
-              throw err;
-            }
-          } else {
-            throw err;
-          }
-        }
-      }
-
-      // Add a delay between bundles to avoid rate limiting
-      await this.sleep(1100);
+      await sendTransactionsJito(
+        transactions,
+        this.searcherClients,
+        this.pythSolanaReceiver.wallet,
+        {
+          maxRetryTimeMs: this.maxRetryTimeMs,
+        },
+        this.logger,
+      );
     }
   }
 }

pnpm-lock.yaml

@@ -2889,6 +2889,9 @@ importers:
       jito-ts:
         specifier: ^3.0.1
         version: 3.0.1(bufferutil@4.0.9)(encoding@0.1.13)(utf-8-validate@5.0.10)
+      ts-log:
+        specifier: ^2.2.7
+        version: 2.2.7
     devDependencies:
       '@solana/wallet-adapter-react':
         specifier: ^0.15.28

target_chains/solana/sdk/js/solana_utils/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/solana-utils",
-  "version": "0.4.4",
+  "version": "0.4.5",
   "description": "Utility functions for Solana",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",
@@ -49,6 +49,7 @@
     "@coral-xyz/anchor": "^0.29.0",
     "@solana/web3.js": "^1.90.0",
     "bs58": "^5.0.0",
-    "jito-ts": "^3.0.1"
+    "jito-ts": "^3.0.1",
+    "ts-log": "^2.2.7"
   }
 }

target_chains/solana/sdk/js/solana_utils/src/jito.ts

@@ -1,3 +1,4 @@
+import { dummyLogger, Logger } from "ts-log";
 import { Wallet } from "@coral-xyz/anchor";
 import {
   PublicKey,
@@ -42,9 +43,27 @@ export async function sendTransactionsJito(
     tx: VersionedTransaction;
     signers?: Signer[] | undefined;
   }[],
-  searcherClient: SearcherClient,
+  searcherClients: SearcherClient | SearcherClient[],
   wallet: Wallet,
+  options: {
+    maxRetryTimeMs?: number; // Max time to retry sending transactions
+    delayBetweenCyclesMs?: number; // Delay between cycles of sending transactions to all searcher clients
+  } = {},
+  logger: Logger = dummyLogger, // Optional logger to track progress of retries
 ): Promise<string> {
+  const clients = Array.isArray(searcherClients)
+    ? searcherClients
+    : [searcherClients];
+
+  if (clients.length === 0) {
+    throw new Error("No searcher clients provided");
+  }
+
+  const maxRetryTimeMs = options.maxRetryTimeMs || 60000; // Default to 60 seconds
+  const delayBetweenCyclesMs = options.delayBetweenCyclesMs || 1000; // Default to 1 second
+
+  const startTime = Date.now();
+
   const signedTransactions = [];
 
   for (const transaction of transactions) {
@@ -64,7 +83,56 @@ export async function sendTransactionsJito(
   );
 
   const bundle = new Bundle(signedTransactions, 2);
-  await searcherClient.sendBundle(bundle);
 
-  return firstTransactionSignature;
+  let lastError: Error | null = null;
+  let totalAttempts = 0;
+
+  while (Date.now() - startTime < maxRetryTimeMs) {
+    // Try all clients in this cycle
+    for (let i = 0; i < clients.length; i++) {
+      const currentClient = clients[i];
+      totalAttempts++;
+
+      try {
+        await currentClient.sendBundle(bundle);
+        logger.info(
+          { clientIndex: i, totalAttempts },
+          `Successfully sent bundle to Jito client after ${totalAttempts} attempts`,
+        );
+        return firstTransactionSignature;
+      } catch (err: any) {
+        lastError = err;
+        logger.error(
+          { clientIndex: i, totalAttempts, err: err.message },
+          `Attempt ${totalAttempts}: Error sending bundle to Jito client ${i}`,
+        );
+      }
+
+      // Check if we've run out of time
+      if (Date.now() - startTime >= maxRetryTimeMs) {
+        break;
+      }
+    }
+
+    // If we've tried all clients and still have time, wait before next cycle
+    const timeRemaining = maxRetryTimeMs - (Date.now() - startTime);
+    if (timeRemaining > delayBetweenCyclesMs) {
+      await new Promise((resolve) => setTimeout(resolve, delayBetweenCyclesMs));
+    }
+  }
+
+  const totalTimeMs = Date.now() - startTime;
+  const errorMsg = `Failed to send transactions via JITO after ${totalAttempts} attempts over ${totalTimeMs}ms (max: ${maxRetryTimeMs}ms)`;
+
+  logger.error(
+    {
+      totalAttempts,
+      totalTimeMs,
+      maxRetryTimeMs,
+      lastError: lastError?.message,
+    },
+    errorMsg,
+  );
+
+  throw lastError || new Error(errorMsg);
 }