Add health probe (#107)

devnet/p2w-terra-relay.yaml

@@ -43,8 +43,10 @@ spec:
             - containerPort: 4200
               name: rest-api
               protocol: TCP
-          tcpSocket:
-            port: 2000
+          readinessProbe:
+            httpGet:
+              path: '/health'
+              port: 4200
           env:
             - name: SPY_SERVICE_HOST
               value: spy:7072
@@ -69,13 +71,13 @@ spec:
               value: '8081'
             - name: BAL_QUERY_INTERVAL
               value: '60000'
-            - name: READINESS_PORT
-              value: '2000'
             - name: RETRY_MAX_ATTEMPTS
               value: '4'
             - name: RETRY_DELAY_IN_MS
               value: '250'
             - name: MAX_MSGS_PER_BATCH
               value: '1'
+            - name: MAX_HEALTHY_NO_RELAY_DURATION_IN_SECONDS
+              value: '120'
             - name: LOG_LEVEL
               value: debug

third_party/pyth/p2w-terra-relay/.env.sample

@@ -31,11 +31,11 @@ TERRA_COIN=uluna
 REST_PORT=4200
 PROM_PORT=8081
 BAL_QUERY_INTERVAL=60000
-#READINESS_PORT=2000
 
 RETRY_MAX_ATTEMPTS=4
 RETRY_DELAY_IN_MS=250
 MAX_MSGS_PER_BATCH=1
+MAX_HEALTHY_NO_RELAY_DURATION_IN_SECONDS=120
 
 # The default is to log the console with level info.
 LOG_DIR=/var/pyth_relay/logs

third_party/pyth/p2w-terra-relay/src/index.ts

@@ -60,17 +60,4 @@ if (
     worker.run(promClient);
     rest.run();
   }
-
-  if (process.env.READINESS_PORT) {
-    const readinessPort: number = parseInt(process.env.READINESS_PORT);
-    const Net = require("net");
-    const readinessServer = new Net.Server();
-    readinessServer.listen(readinessPort, function () {
-      logger.info("listening for readiness requests on port " + readinessPort);
-    });
-
-    readinessServer.on("connection", function (socket: any) {
-      //logger.debug("readiness connection");
-    });
-  }
 }

third_party/pyth/p2w-terra-relay/src/rest.ts

@@ -1,6 +1,6 @@
 import { Request, Response } from "express";
 import { logger } from "./helpers";
-import { getStatus, getPriceData } from "./worker";
+import { getStatus, getPriceData, isHealthy } from "./worker";
 
 let restPort: number = 0;
 
@@ -35,8 +35,16 @@ export async function run() {
       res.json(result);
     });
 
+    app.get("/health", async (req: Request, res: Response) => {
+      if (isHealthy()) {
+        res.sendStatus(200);
+      } else {
+        res.sendStatus(503);
+      }
+    });
+
     app.get("/", (req: Request, res: Response) =>
-      res.json(["/status", "/queryterra/<price_id>"])
+      res.json(["/status", "/queryterra/<price_id>", "/health"])
     );
   })();
 }

third_party/pyth/p2w-terra-relay/src/worker.ts

@@ -45,6 +45,8 @@ let walletTimeStamp: Date;
 let maxPerBatch: number = 1;
 let maxAttempts: number = 2;
 let retryDelayInMs: number = 0;
+let maxHealthyNoRelayDurationInSeconds: number = 120;
+let lastSuccessfulRelayTime: Date;
 
 export function init(runWorker: boolean, relay: Relay): boolean {
   if (!runWorker) return true;
@@ -93,6 +95,20 @@ export function init(runWorker: boolean, relay: Relay): boolean {
     return false;
   }
 
+  if (process.env.MAX_HEALTHY_NO_RELAY_DURATION_IN_SECONDS) {
+    maxHealthyNoRelayDurationInSeconds = parseInt(process.env.MAX_HEALTHY_NO_RELAY_DURATION_IN_SECONDS);
+  }
+
+  if (maxHealthyNoRelayDurationInSeconds <= 0) {
+    logger.error(
+      "Environment variable MAX_HEALTHY_NO_RELAY_DURATION_IN_SECONDS has an invalid value of " +
+        maxHealthyNoRelayDurationInSeconds +
+        ", must be positive."
+    );
+
+    return false;
+  }
+
   return true;
 }
 
@@ -392,6 +408,10 @@ async function finalizeEventsAlreadyLocked(
         relayResult
     );
   }
+
+  if (relayResult.is_ok()) {
+    lastSuccessfulRelayTime = new Date();
+  }
 }
 
 async function updateBalance() {
@@ -474,3 +494,18 @@ export async function getPriceData(priceId: string): Promise<any> {
 
   return result;
 }
+
+export function isHealthy(): boolean {
+  if (lastSuccessfulRelayTime === undefined) {
+    return false;
+  }
+
+  const currentDate = new Date();
+  const timeDiffMs = currentDate.getTime() - lastSuccessfulRelayTime.getTime();
+
+  if (timeDiffMs > maxHealthyNoRelayDurationInSeconds * 1000) {
+    return false;
+  }
+
+  return true;
+}