[TWAP] Update Hermes binary response format, add TwapMessage parsing to price-service-sdk (#2158)

* feat: change hermes twap binary response, add twap data model and parsing to price-service-sdk

* feat: reexport parseTwapMessage

* test: add parse twap msg test

* test: update parse twap test

* refactor: pack both start & end updatedatas into a single binaryupdate

* refactor: snake -> camel case

apps/hermes/server/Cargo.lock

@@ -1868,7 +1868,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermes"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "anyhow",
  "async-trait",

apps/hermes/server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name        = "hermes"
-version     = "0.8.0"
+version     = "0.8.1"
 description = "Hermes is an agent that provides Verified Prices from the Pythnet Pyth Oracle."
 edition     = "2021"
 

apps/hermes/server/src/api/rest/v2/latest_twaps.rs

@@ -131,22 +131,17 @@ where
     })?;
 
     let twap_update_data = twaps_with_update_data.update_data;
-    let binary: Vec<BinaryUpdate> = twap_update_data
+    let encoded_data = twap_update_data
         .into_iter()
-        .map(|data_vec| {
-            let encoded_data = data_vec
-                .into_iter()
-                .map(|data| match params.encoding {
-                    EncodingType::Base64 => base64_standard_engine.encode(data),
-                    EncodingType::Hex => hex::encode(data),
-                })
-                .collect();
-            BinaryUpdate {
-                encoding: params.encoding,
-                data: encoded_data,
-            }
+        .map(|data| match params.encoding {
+            EncodingType::Base64 => base64_standard_engine.encode(data),
+            EncodingType::Hex => hex::encode(data),
         })
         .collect();
+    let binary = BinaryUpdate {
+        encoding: params.encoding,
+        data: encoded_data,
+    };
 
     let parsed: Option<Vec<ParsedPriceFeedTwap>> = if params.parsed {
         Some(

apps/hermes/server/src/api/types.rs

@@ -281,9 +281,9 @@ impl From<PriceFeedTwap> for ParsedPriceFeedTwap {
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 pub struct TwapsResponse {
-    /// Each BinaryUpdate contains the start & end cumulative price updates used to
+    /// Contains the start & end cumulative price updates used to
     /// calculate a given price feed's TWAP.
-    pub binary: Vec<BinaryUpdate>,
+    pub binary: BinaryUpdate,
 
     /// The calculated TWAPs for each price ID
     #[serde(skip_serializing_if = "Option::is_none")]

apps/hermes/server/src/state/aggregate.rs

@@ -206,7 +206,7 @@ pub struct PublisherStakeCapsWithUpdateData {
 #[derive(Debug)]
 pub struct TwapsWithUpdateData {
     pub twaps: Vec<PriceFeedTwap>,
-    pub update_data: Vec<Vec<Vec<u8>>>,
+    pub update_data: Vec<Vec<u8>>,
 }
 
 #[derive(Debug, Serialize)]
@@ -652,7 +652,6 @@ where
     }
 
     let mut twaps = Vec::new();
-    let mut update_data = Vec::new();
 
     // Iterate through start and end messages together
     for (start_message, end_message) in start_messages.iter().zip(end_messages.iter()) {
@@ -676,34 +675,27 @@ where
                         end_timestamp: end_twap.publish_time,
                         down_slots_ratio,
                     });
-
-                    // Combine messages for update data
-                    let mut messages = Vec::new();
-                    messages.push(start_message.clone().into());
-                    messages.push(end_message.clone().into());
-
-                    if let Ok(update) = construct_update_data(messages) {
-                        update_data.push(update);
-                    } else {
-                        tracing::warn!(
-                            "Failed to construct update data for price feed {:?}",
-                            start_twap.feed_id
-                        );
-                        continue;
-                    }
                 }
                 Err(e) => {
-                    tracing::warn!(
+                    return Err(anyhow!(
                         "Failed to calculate TWAP for price feed {:?}: {}",
                         start_twap.feed_id,
                         e
-                    );
-                    continue;
+                    ));
                 }
             }
         }
     }
 
+    // Construct update data.
+    // update_data[0] contains the start VAA and merkle proofs
+    // update_data[1] contains the end VAA and merkle proofs
+    let mut update_data =
+        construct_update_data(start_messages.into_iter().map(Into::into).collect())?;
+    update_data.extend(construct_update_data(
+        end_messages.into_iter().map(Into::into).collect(),
+    )?);
+
     Ok(TwapsWithUpdateData { twaps, update_data })
 }
 
@@ -1316,10 +1308,8 @@ mod test {
         assert_eq!(twap_2.start_timestamp, 100);
         assert_eq!(twap_2.end_timestamp, 200);
 
-        // Verify update data contains both start and end messages for both feeds
+        // update_data should have 2 elements, one for the start block and one for the end block.
         assert_eq!(result.update_data.len(), 2);
-        assert_eq!(result.update_data[0].len(), 2); // Should contain 2 messages
-        assert_eq!(result.update_data[1].len(), 2); // Should contain 2 messages
     }
     #[tokio::test]
 

price_service/sdk/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/price-service-sdk",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Pyth price service SDK",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",

price_service/sdk/js/src/AccumulatorUpdateData.ts

@@ -5,12 +5,12 @@ const MAJOR_VERSION = 1;
 const MINOR_VERSION = 0;
 const KECCAK160_HASH_SIZE = 20;
 const PRICE_FEED_MESSAGE_VARIANT = 0;
+const TWAP_MESSAGE_VARIANT = 1;
 
 export type AccumulatorUpdateData = {
   vaa: Buffer;
   updates: { message: Buffer; proof: number[][] }[];
 };
-
 export type PriceFeedMessage = {
   feedId: Buffer;
   price: BN;
@@ -22,6 +22,17 @@ export type PriceFeedMessage = {
   emaConf: BN;
 };
 
+export type TwapMessage = {
+  feedId: Buffer;
+  cumulativePrice: BN;
+  cumulativeConf: BN;
+  numDownSlots: BN;
+  exponent: number;
+  publishTime: BN;
+  prevPublishTime: BN;
+  publishSlot: BN;
+};
+
 export function isAccumulatorUpdateData(updateBytes: Buffer): boolean {
   return (
     updateBytes.toString("hex").slice(0, 8) === ACCUMULATOR_MAGIC &&
@@ -29,6 +40,7 @@ export function isAccumulatorUpdateData(updateBytes: Buffer): boolean {
     updateBytes[5] === MINOR_VERSION
   );
 }
+
 export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage {
   let cursor = 0;
   const variant = message.readUInt8(cursor);
@@ -64,6 +76,41 @@ export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage {
   };
 }
 
+export function parseTwapMessage(message: Buffer): TwapMessage {
+  let cursor = 0;
+  const variant = message.readUInt8(cursor);
+  if (variant !== TWAP_MESSAGE_VARIANT) {
+    throw new Error("Not a twap message");
+  }
+  cursor += 1;
+  const feedId = message.subarray(cursor, cursor + 32);
+  cursor += 32;
+  const cumulativePrice = new BN(message.subarray(cursor, cursor + 16), "be");
+  cursor += 16;
+  const cumulativeConf = new BN(message.subarray(cursor, cursor + 16), "be");
+  cursor += 16;
+  const numDownSlots = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  const exponent = message.readInt32BE(cursor);
+  cursor += 4;
+  const publishTime = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  const prevPublishTime = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  const publishSlot = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  return {
+    feedId,
+    cumulativePrice,
+    cumulativeConf,
+    numDownSlots,
+    exponent,
+    publishTime,
+    prevPublishTime,
+    publishSlot,
+  };
+}
+
 /**
  * An AccumulatorUpdateData contains a VAA and a list of updates. This function returns a new serialized AccumulatorUpdateData with only the updates in the range [start, end).
  */

price_service/sdk/js/src/index.ts

@@ -15,6 +15,7 @@ export {
   parseAccumulatorUpdateData,
   AccumulatorUpdateData,
   parsePriceFeedMessage,
+  parseTwapMessage,
 } from "./AccumulatorUpdateData";
 
 /**