Add a test pyth instance

Change-Id: Ifa5b50fb80f01f386fc8079eec3a0564df8072e1

Tiltfile

@@ -98,6 +98,15 @@ k8s_resource("guardian", resource_deps = ["proto-gen", "solana-devnet"], port_fo
     port_forward(7071, name = "Public REST [:7071]"),
 ])
 
+docker_build(
+    ref = "pyth",
+    context = ".",
+    dockerfile = "third_party/pyth/Dockerfile"
+)
+k8s_yaml_with_ns("./devnet/pyth.yaml")
+
+k8s_resource("pyth", resource_deps = ["solana-devnet"])
+
 # publicRPC proxy that allows grpc over http1, for local development
 
 k8s_yaml_with_ns("./devnet/envoy-proxy.yaml")

devnet/pyth.yaml

@@ -0,0 +1,55 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pyth
+  labels:
+    app: pyth
+spec:
+  clusterIP: None
+  selector:
+    app: pyth
+  ports:
+    - port: 8898
+      name: pyth-tx
+      protocol: TCP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pyth
+spec:
+  selector:
+    matchLabels:
+      app: pyth
+  serviceName: pyth
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: pyth
+    spec:
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 0
+      containers:
+        - name: pyth-publisher
+          image: pyth
+          command:
+            - python3
+            - /opt/pyth/pyth_publisher.py
+          readinessProbe:
+            tcpSocket:
+              port: 2000
+            periodSeconds: 1
+            failureThreshold: 300
+        - name: pyth-tx
+          image: pyth
+          command:
+            - ./pyth_tx
+            - -r
+            - solana-devnet
+          ports:
+            - containerPort: 8898
+              name: pyth-tx
+              protocol: TCP

devnet/solana-devnet.yaml

@@ -50,6 +50,9 @@ spec:
             - --bpf-program
             - metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s
             - /opt/solana/deps/spl_token_metadata.so
+            - --bpf-program
+            - gMYYig2utAxVoXnM9UhtTWrt8e7x2SVBZqsWZJeT5Gw # Derived from pyth_program.json
+            - /opt/solana/deps/pyth_oracle.so
             - --log
           ports:
             - containerPort: 8001

solana/Dockerfile

@@ -1,9 +1,18 @@
 #syntax=docker/dockerfile:1.2@sha256:e2a8561e419ab1ba6b2fe6cbdf49fd92b95912df1cf7d313c3e2230a333fdbcc
 FROM docker.io/library/rust:1.49@sha256:a50165ea96983c21832578afb1c8c028674c965bc1ed43b607871b1f362e06a5
 
-RUN apt-get update && apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang
-RUN rustup component add rustfmt
-RUN rustup default nightly-2021-08-01
+RUN apt-get update && \
+    apt-get install -y \
+    clang \
+    libssl-dev \
+    libudev-dev \
+    llvm \
+    pkg-config \
+    zlib1g-dev \
+    && \
+    rm -rf /var/lib/apt/lists/* && \
+    rustup component add rustfmt && \
+    rustup default nightly-2021-08-01
 
 RUN sh -c "$(curl -sSfL https://release.solana.com/v1.7.8/install)"
 
@@ -14,6 +23,17 @@ RUN cargo init --lib /tmp/decoy-crate && \
     cd /tmp/decoy-crate && cargo build-bpf && \
     rm -rf /tmp/decoy-crate
 
+# Cache Pyth sources
+# This comes soon after mainnet-v2.1
+ENV PYTH_SRC_REV=31e3188bbf52ec1a25f71e4ab969378b27415b0a
+ENV PYTH_DIR=/usr/src/pyth/pyth-client
+
+WORKDIR $PYTH_DIR
+ADD https://github.com/pyth-network/pyth-client/archive/$PYTH_SRC_REV.tar.gz .
+
+# GitHub appends revision to dir in archive
+RUN tar -xvf *.tar.gz && rm -rf *.tar.gz && mv pyth-client-$PYTH_SRC_REV pyth-client
+
 # Add bridge contract sources
 WORKDIR /usr/src/bridge
 
@@ -36,5 +56,10 @@ RUN --mount=type=cache,target=bridge/target \
     cp modules/token_bridge/target/deploy/token_bridge.so /opt/solana/deps/token_bridge.so && \
     cp modules/token_bridge/token-metadata/spl_token_metadata.so /opt/solana/deps/spl_token_metadata.so
 
+# Build the Pyth Solana program
+WORKDIR $PYTH_DIR/pyth-client/program
+RUN make SOLANA=~/.local/share/solana/install/active_release/bin OUT_DIR=../target && \
+    cp ../target/oracle.so /opt/solana/deps/pyth_oracle.so
+
 ENV RUST_LOG="solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=trace,solana_bpf_loader=debug,solana_rbpf=debug"
 ENV RUST_BACKTRACE=1

solana/keys/pyth_program.json

@@ -0,0 +1 @@
+[151,156,152,229,131,186,5,254,107,42,234,87,191,209,182,237,170,57,174,150,37,14,5,58,100,237,114,141,46,22,155,104,10,20,225,112,227,95,250,0,102,170,119,34,187,74,144,163,181,123,233,253,191,6,2,70,127,227,138,51,98,209,205,172]

solana/keys/pyth_publisher.json

@@ -0,0 +1 @@
+[62,189,176,181,215,49,125,17,130,43,109,83,115,112,151,110,117,239,235,54,205,209,6,255,76,27,210,115,206,166,217,165,250,48,211,191,77,246,195,18,170,246,162,103,141,129,14,143,127,4,243,114,79,112,11,46,90,174,215,2,63,42,134,56]

solana/pyth2wormhole/Cargo.lock

@@ -251,9 +251,11 @@ dependencies = [
  "borsh",
  "byteorder",
  "primitive-types",
+ "serde",
  "sha3",
  "solana-program",
  "solitaire",
+ "wasm-bindgen",
 ]
 
 [[package]]

third_party/pyth/Dockerfile

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1.2
+# Wormhole-specific setup for pyth
+FROM pythfoundation/pyth-client:devnet-v2.2@sha256:2ce8de6a43b2fafafd15ebdb723c530a0319860dc40c9fdb97149d5aa270fdde
+
+USER root
+
+# At the time of this writing, debian is fussy about performing an
+# apt-get update. Please add one if repos go stale
+RUN apt-get install -y netcat-openbsd python3 && \
+    rm -rf /var/lib/apt/lists/*
+
+ADD solana/keys /opt/solana/keys
+
+ENV PYTH_KEY_STORE=/home/pyth/.pythd
+
+# Prepare keys
+WORKDIR $PYTH_KEY_STORE
+
+RUN cp /opt/solana/keys/pyth_publisher.json publish_key_pair.json && \
+    cp /opt/solana/keys/pyth_program.json program_key_pair.json && \
+    chown pyth:pyth -R . && \
+    chmod go-rwx -R .
+
+# Rebuild Pyth sources from scratch
+# This comes soon after mainnet-v2.1
+ENV PYTH_SRC_REV=31e3188bbf52ec1a25f71e4ab969378b27415b0a
+ENV PYTH_SRC_ROOT=/home/pyth/pyth-client
+
+ADD https://github.com/drozdziak1/pyth-client/archive/$PYTH_SRC_REV.tar.gz .
+RUN tar -xvf *.tar.gz && \
+    rm -rf $PYTH_SRC_ROOT *.tar.gz && \
+    mv pyth-client-$PYTH_SRC_REV $PYTH_SRC_ROOT/
+
+WORKDIR $PYTH_SRC_ROOT/build
+
+RUN cmake .. && make
+
+# Prepare setup script
+ADD third_party/pyth/ /opt/pyth/
+RUN chmod a+rx /opt/pyth/*.py
+USER pyth
+
+ENV PYTH=$PYTH_SRC_ROOT/build/pyth
+ENV READINESS_PORT=2000

third_party/pyth/pyth_publisher.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+from pyth_utils import *
+
+import random
+import sys
+import threading
+import time
+
+# Accept connections from readiness probe
+def publisher_readiness():
+    run_or_die(["nc", "-k", "-l", "-p", READINESS_PORT])
+
+# Update the specified price with random values
+def publisher_random_update(price_pubkey):
+    value = random.randrange(1024)
+    confidence = 1
+    pyth_run_or_die("upd_price_val", args=[price_pubkey, str(value), str(confidence), "trading"])
+    print("Price updated!")
+
+# Fund the publisher
+sol_run_or_die("airdrop", [str(SOL_AIRDROP_AMT),
+                           "--keypair", PYTH_PUBLISHER_KEYPAIR,
+                           "--commitment", "finalized",
+                           ])
+
+# Create a mapping
+pyth_run_or_die("init_mapping")
+
+# Add a product
+prod_pubkey = pyth_run_or_die("add_product", capture_output=True).stdout.strip()
+print(f"Added product {prod_pubkey}")
+
+# Add a price
+price_pubkey = pyth_run_or_die(
+    "add_price",
+    args=[prod_pubkey, "price"],
+    confirm=False,
+    capture_output=True
+).stdout.strip()
+
+print(f"Added price {price_pubkey}")
+
+publisher_pubkey = sol_run_or_die("address", args=["--keypair", PYTH_PUBLISHER_KEYPAIR], capture_output=True).stdout.strip()
+
+# Become a publisher
+pyth_run_or_die("add_publisher", args=[publisher_pubkey, price_pubkey], confirm=False, debug=True, capture_output=True)
+print(f"Added publisher {publisher_pubkey}")
+
+# Update the price as the newly added publisher
+publisher_random_update(price_pubkey)
+
+print(f"Updated price {price_pubkey}. Mock updates ready to roll. Updating every {str(PYTH_PUBLISHER_INTERVAL)} seconds")
+
+# Spin off the readiness probe endpoint into a separate thread
+readiness_thread = threading.Thread(target=publisher_readiness)
+
+readiness_thread.start()
+
+while True:
+    print(f"Updating price {price_pubkey}")
+    publisher_random_update(price_pubkey)
+    time.sleep(PYTH_PUBLISHER_INTERVAL)
+    sys.stdout.flush()
+
+readiness_thread.join()

third_party/pyth/pyth_utils.py

@@ -0,0 +1,59 @@
+import os
+import sys
+import subprocess
+
+PYTH=os.environ.get("PYTH", "./pyth")
+PYTH_KEY_STORE = os.environ.get("PYTH_KEY_STORE", "/home/pyth/.pythd")
+PYTH_PROGRAM_KEYPAIR = f"{PYTH_KEY_STORE}/program_key_pair.json"
+PYTH_PROGRAM_SO_PATH=os.environ.get("PYTH_PROGRAM_SO", "../target/oracle.so")
+PYTH_PUBLISHER_KEYPAIR = f"{PYTH_KEY_STORE}/publish_key_pair.json"
+PYTH_PUBLISHER_INTERVAL = float(os.environ.get("PYTH_PUBLISHER_INTERVAL", "5"))
+
+SOL_AIRDROP_AMT = 100
+SOL_RPC_HOST = "solana-devnet"
+SOL_RPC_PORT = 8899
+SOL_RPC_URL = f"http://{SOL_RPC_HOST}:{str(SOL_RPC_PORT)}"
+
+READINESS_PORT=os.environ.get("READINESS_PORT", "2000")
+
+# pretend we're set -e
+def run_or_die(args, die=True, **kwargs):
+    args_readable = ' '.join(args)
+    print(f"CMD RUN\t{args_readable}", file=sys.stderr)
+    sys.stderr.flush()
+    ret = subprocess.run(args, text=True, **kwargs)
+
+    if ret.returncode is not 0:
+        print(f"CMD FAIL {ret.returncode}\t{args_readable}", file=sys.stderr)
+
+        out = ret.stdout if ret.stdout is not None else "<not captured>"
+        err = ret.stderr if ret.stderr is not None else "<not captured>"
+
+        print(f"CMD STDOUT\n{out}", file=sys.stderr)
+        print(f"CMD STDERR\n{err}", file=sys.stderr)
+
+        if die:
+            sys.exit(ret.returncode)
+        else:
+            print(f"CMD DIE FALSE", file=sys.stderr)
+
+    else:
+        print(f"CMD OK\t{args_readable}", file=sys.stderr)
+    sys.stderr.flush()
+    return ret
+
+# Pyth boilerplate in front of run_or_die
+def pyth_run_or_die(subcommand, args=[], debug=False, confirm=True, **kwargs):
+    return run_or_die([PYTH, subcommand]
+                      + args
+                      + (["-d"] if debug else [])
+                      + ([] if confirm else ["-n"]) # Note: not all pyth subcommands accept -n
+                      + ["-k", PYTH_KEY_STORE]
+                      + ["-r", SOL_RPC_HOST]
+                      + ["-c", "finalized"], **kwargs)
+
+# Solana boilerplate in front of run_or_die
+def sol_run_or_die(subcommand, args=[], **kwargs):
+    return run_or_die(["solana", subcommand]
+                         + args
+                         + ["--url", SOL_RPC_URL], **kwargs)