Merge pull request #2921 from pyth-network/feat/perf-caching

feat(insights): pyth metadata caching

apps/insights/next.config.js

@@ -3,7 +3,6 @@ const config = {
     useCache: true,
     reactCompiler: true,
   },
-
   reactStrictMode: true,
 
   pageExtensions: ["ts", "tsx", "mdx"],

apps/insights/package.json

@@ -28,10 +28,12 @@
     "@pythnetwork/known-publishers": "workspace:*",
     "@react-hookz/web": "catalog:",
     "@solana/web3.js": "catalog:",
+    "async-cache-dedupe": "catalog:",
     "bs58": "catalog:",
     "clsx": "catalog:",
     "cryptocurrency-icons": "catalog:",
     "dnum": "catalog:",
+    "ioredis": "^5.7.0",
     "lightweight-charts": "catalog:",
     "motion": "catalog:",
     "next": "catalog:",
@@ -44,7 +46,8 @@
     "superjson": "catalog:",
     "swr": "catalog:",
     "zod": "catalog:",
-    "zod-validation-error": "catalog:"
+    "zod-validation-error": "catalog:",
+    "zod-search-params": "catalog:"
   },
   "devDependencies": {
     "@cprussin/eslint-config": "catalog:",

apps/insights/src/app/api/pyth/get-feeds-for-publisher/[publisher]/route.ts

@@ -0,0 +1,54 @@
+import { NextRequest } from "next/server";
+import { stringify } from "superjson";
+import { z } from "zod";
+import { parseSearchParams } from "zod-search-params";
+
+import {
+  Cluster,
+  CLUSTER_NAMES,
+  ClusterToName,
+  toCluster,
+} from "../../../../../services/pyth";
+import { getFeeds } from "../../../../../services/pyth/get-feeds";
+
+export const GET = async (
+  request: NextRequest,
+  { params }: { params: Promise<{ publisher: string }> },
+) => {
+  const { publisher } = await params;
+  const searchParams = request.nextUrl.searchParams;
+  const parsedSearchParams = parseSearchParams(queryParamsSchema, searchParams);
+
+  if (!parsedSearchParams) {
+    return new Response("Invalid params", {
+      status: 400,
+    });
+  }
+
+  const { cluster } = parsedSearchParams;
+
+  if (!publisher) {
+    return new Response("Publisher is required", {
+      status: 400,
+    });
+  }
+
+  const feeds = await getFeeds(cluster);
+
+  const filteredFeeds = feeds.filter((feed) =>
+    feed.price.priceComponents.some((c) => c.publisher === publisher),
+  );
+
+  return new Response(stringify(filteredFeeds), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+};
+
+const queryParamsSchema = z.object({
+  cluster: z
+    .enum(CLUSTER_NAMES)
+    .transform((value) => toCluster(value))
+    .default(ClusterToName[Cluster.Pythnet]),
+});

apps/insights/src/app/api/pyth/get-feeds/[symbol]/route.ts

@@ -0,0 +1,45 @@
+import { NextRequest } from "next/server";
+import { stringify } from "superjson";
+import { z } from "zod";
+import { parseSearchParams } from "zod-search-params";
+
+import {
+  Cluster,
+  CLUSTER_NAMES,
+  ClusterToName,
+  toCluster,
+} from "../../../../../services/pyth";
+import { getFeeds } from "../../../../../services/pyth/get-feeds";
+
+export const GET = async (
+  request: NextRequest,
+  { params }: { params: Promise<{ symbol: string }> },
+) => {
+  const { symbol } = await params;
+  const searchParams = request.nextUrl.searchParams;
+  const parsedSearchParams = parseSearchParams(queryParamsSchema, searchParams);
+
+  if (!parsedSearchParams) {
+    return new Response("Invalid params", {
+      status: 400,
+    });
+  }
+
+  const { cluster } = parsedSearchParams;
+
+  const feeds = await getFeeds(cluster);
+  const feed = feeds.find((feed) => feed.symbol === symbol);
+
+  return new Response(stringify(feed), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+};
+
+const queryParamsSchema = z.object({
+  cluster: z
+    .enum(CLUSTER_NAMES)
+    .transform((value) => toCluster(value))
+    .default(ClusterToName[Cluster.Pythnet]),
+});

apps/insights/src/app/api/pyth/get-feeds/route.ts

@@ -0,0 +1,41 @@
+import { NextRequest } from "next/server";
+import { stringify } from "superjson";
+import { z } from "zod";
+import { parseSearchParams } from "zod-search-params";
+
+import { CLUSTER_NAMES, toCluster } from "../../../../services/pyth";
+import { getFeeds } from "../../../../services/pyth/get-feeds";
+
+export const GET = async (request: NextRequest) => {
+  // get cluster from query params
+  const searchParams = request.nextUrl.searchParams;
+  const parsedSearchParams = parseSearchParams(queryParamsSchema, searchParams);
+
+  if (!parsedSearchParams) {
+    return new Response("Invalid params", {
+      status: 400,
+    });
+  }
+
+  const { excludePriceComponents, cluster } = parsedSearchParams;
+
+  const feeds = await getFeeds(cluster);
+  const filteredFeeds = excludePriceComponents
+    ? feeds.map((feed) => {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { price, ...rest } = feed;
+        return rest;
+      })
+    : feeds;
+
+  return new Response(stringify(filteredFeeds), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+};
+
+const queryParamsSchema = z.object({
+  cluster: z.enum(CLUSTER_NAMES).transform((value) => toCluster(value)),
+  excludePriceComponents: z.boolean(),
+});

apps/insights/src/app/api/pyth/get-publishers/[symbol]/route.ts

@@ -0,0 +1,45 @@
+import { NextRequest, NextResponse } from "next/server";
+import { z } from "zod";
+import { parseSearchParams } from "zod-search-params";
+
+import {
+  Cluster,
+  CLUSTER_NAMES,
+  ClusterToName,
+  toCluster,
+} from "../../../../../services/pyth";
+import { getPublishersForCluster } from "../../../../../services/pyth/get-publishers-for-cluster";
+
+export const GET = async (
+  request: NextRequest,
+  { params }: { params: Promise<{ symbol: string }> },
+) => {
+  const { symbol } = await params;
+  const searchParams = request.nextUrl.searchParams;
+  const parsedSearchParams = parseSearchParams(queryParamsSchema, searchParams);
+
+  if (!parsedSearchParams) {
+    return new Response("Invalid params", {
+      status: 400,
+    });
+  }
+
+  const { cluster } = parsedSearchParams;
+
+  if (!symbol) {
+    return new Response("Symbol is required", {
+      status: 400,
+    });
+  }
+
+  const map = await getPublishersForCluster(cluster);
+
+  return NextResponse.json(map[symbol] ?? []);
+};
+
+const queryParamsSchema = z.object({
+  cluster: z
+    .enum(CLUSTER_NAMES)
+    .transform((value) => toCluster(value))
+    .default(ClusterToName[Cluster.Pythnet]),
+});

apps/insights/src/app/component-score-history/route.ts

@@ -16,13 +16,13 @@ export const GET = async (req: NextRequest) => {
   );
   if (parsed.success) {
     const { cluster, publisherKey, symbol, from, to } = parsed.data;
-    const data = await getFeedScoreHistory(
+    const data = await getFeedScoreHistory({
       cluster,
       publisherKey,
       symbol,
       from,
       to,
-    );
+    });
     return Response.json(data);
   } else {
     return new Response(fromError(parsed.error).toString(), {

apps/insights/src/app/historical-prices/route.ts

@@ -6,7 +6,10 @@ export async function GET(req: NextRequest) {
   const symbol = req.nextUrl.searchParams.get("symbol");
   const until = req.nextUrl.searchParams.get("until");
   if (symbol && until) {
-    const res = await getHistoricalPrices(decodeURIComponent(symbol), until);
+    const res = await getHistoricalPrices({
+      symbol: decodeURIComponent(symbol),
+      until,
+    });
     return Response.json(res);
   } else {
     return new Response("Must provide `symbol` and `until`", { status: 400 });

apps/insights/src/app/price-feeds/[slug]/layout.ts

@@ -2,7 +2,8 @@ import type { Metadata } from "next";
 import { notFound } from "next/navigation";
 import type { ReactNode } from "react";
 
-import { Cluster, getFeeds } from "../../../services/pyth";
+import { Cluster } from "../../../services/pyth";
+import { getFeeds } from "../../../services/pyth/get-feeds";
 
 export { PriceFeedLayout as default } from "../../../components/PriceFeed/layout";
 

apps/insights/src/components/Overview/index.tsx

@@ -9,7 +9,8 @@ import PriceFeedsLight from "./price-feeds-light.svg";
 import PublishersDark from "./publishers-dark.svg";
 import PublishersLight from "./publishers-light.svg";
 import { TabList } from "./tab-list";
-import { Cluster, getFeeds } from "../../services/pyth";
+import { Cluster } from "../../services/pyth";
+import { getFeeds } from "../../services/pyth/get-feeds";
 import {
   totalVolumeTraded,
   activeChains,
@@ -25,6 +26,7 @@ import { FormattedNumber } from "../FormattedNumber";
 export const Overview = async () => {
   const priceFeeds = await getFeeds(Cluster.Pythnet);
   const today = new Date();
+
   const feedCounts = [
     ...activeFeeds.map(({ date, numFeeds }) => ({
       x: date,
@@ -37,6 +39,7 @@ export const Overview = async () => {
       y: priceFeeds.length,
     },
   ];
+
   return (
     <div className={styles.overview}>
       <h1 className={styles.header}>Overview</h1>

apps/insights/src/components/PriceFeed/get-feed.tsx

@@ -1,20 +1,21 @@
 import { notFound } from "next/navigation";
 
-import { Cluster, getFeeds } from "../../services/pyth";
+import { getFeedForSymbolRequest, getFeedsRequest } from "../../server/pyth";
+import { Cluster } from "../../services/pyth";
 
 export const getFeed = async (params: Promise<{ slug: string }>) => {
-  "use cache";
+  const feeds = await getFeedsRequest(Cluster.Pythnet);
 
-  const [{ slug }, feeds] = await Promise.all([params, getPythnetFeeds()]);
+  const { slug } = await params;
   const symbol = decodeURIComponent(slug);
+  const feed = await getFeedForSymbolRequest({
+    symbol,
+    cluster: Cluster.Pythnet,
+  });
+
   return {
     feeds,
-    feed: feeds.find((item) => item.symbol === symbol) ?? notFound(),
+    feed: feed ?? notFound(),
     symbol,
   } as const;
 };
-
-const getPythnetFeeds = async () => {
-  "use cache";
-  return getFeeds(Cluster.Pythnet);
-};

apps/insights/src/components/PriceFeed/publishers.tsx

@@ -1,13 +1,12 @@
 import { lookup as lookupPublisher } from "@pythnetwork/known-publishers";
 import { notFound } from "next/navigation";
 
-import { getRankingsBySymbol } from "../../services/clickhouse";
 import {
-  Cluster,
-  ClusterToName,
-  getFeeds,
-  getPublishersForFeed,
-} from "../../services/pyth";
+  getFeedForSymbolRequest,
+  getPublishersForFeedRequest,
+} from "../../server/pyth";
+import { getRankingsBySymbol } from "../../services/clickhouse";
+import { Cluster, ClusterToName } from "../../services/pyth";
 import { getStatus } from "../../status";
 import { PublisherIcon } from "../PublisherIcon";
 import { PublisherTag } from "../PublisherTag";
@@ -22,21 +21,15 @@ type Props = {
 export const Publishers = async ({ params }: Props) => {
   const { slug } = await params;
   const symbol = decodeURIComponent(slug);
-  const [
-    pythnetFeeds,
-    pythtestConformanceFeeds,
-    pythnetPublishers,
-    pythtestConformancePublishers,
-  ] = await Promise.all([
-    getFeeds(Cluster.Pythnet),
-    getFeeds(Cluster.PythtestConformance),
-    getPublishers(Cluster.Pythnet, symbol),
-    getPublishers(Cluster.PythtestConformance, symbol),
-  ]);
-  const feed = pythnetFeeds.find((feed) => feed.symbol === symbol);
-  const testFeed = pythtestConformanceFeeds.find(
-    (feed) => feed.symbol === symbol,
-  );
+
+  const [feed, testFeed, pythnetPublishers, pythtestConformancePublishers] =
+    await Promise.all([
+      getFeedForSymbolRequest({ symbol, cluster: Cluster.Pythnet }),
+      getFeedForSymbolRequest({ symbol, cluster: Cluster.PythtestConformance }),
+      getPublishers(Cluster.Pythnet, symbol),
+      getPublishers(Cluster.PythtestConformance, symbol),
+    ]);
+
   const publishers = [...pythnetPublishers, ...pythtestConformancePublishers];
   const metricsTime = pythnetPublishers.find(
     (publisher) => publisher.ranking !== undefined,
@@ -87,25 +80,23 @@ export const PublishersLoading = () => <PublishersCard isLoading />;
 
 const getPublishers = async (cluster: Cluster, symbol: string) => {
   const [publishers, rankings] = await Promise.all([
-    getPublishersForFeed(cluster, symbol),
+    getPublishersForFeedRequest(cluster, symbol),
     getRankingsBySymbol(symbol),
   ]);
 
-  return (
-    publishers?.map((publisher) => {
-      const ranking = rankings.find(
-        (ranking) =>
-          ranking.publisher === publisher &&
-          ranking.cluster === ClusterToName[cluster],
-      );
+  return publishers.map((publisher) => {
+    const ranking = rankings.find(
+      (ranking) =>
+        ranking.publisher === publisher &&
+        ranking.cluster === ClusterToName[cluster],
+    );
 
-      return {
-        ranking,
-        publisher,
-        status: getStatus(ranking),
-        cluster,
-        knownPublisher: lookupPublisher(publisher),
-      };
-    }) ?? []
-  );
+    return {
+      ranking,
+      publisher,
+      status: getStatus(ranking),
+      cluster,
+      knownPublisher: lookupPublisher(publisher),
+    };
+  });
 };

apps/insights/src/components/PriceFeeds/index.tsx

@@ -19,7 +19,8 @@ import { AssetClassTable } from "./asset-class-table";
 import { ComingSoonList } from "./coming-soon-list";
 import styles from "./index.module.scss";
 import { PriceFeedsCard } from "./price-feeds-card";
-import { Cluster, getFeeds } from "../../services/pyth";
+import { Cluster } from "../../services/pyth";
+import { getFeeds } from "../../services/pyth/get-feeds";
 import { priceFeeds as priceFeedsStaticConfig } from "../../static-data/price-feeds";
 import { activeChains } from "../../static-data/stats";
 import { Cards } from "../Cards";

apps/insights/src/components/Publisher/get-price-feeds.tsx

@@ -1,11 +1,11 @@
+import { getFeedsForPublisherRequest } from "../../server/pyth";
 import { getRankingsByPublisher } from "../../services/clickhouse";
-import type { Cluster } from "../../services/pyth";
-import { ClusterToName, getFeedsForPublisher } from "../../services/pyth";
+import { Cluster, ClusterToName } from "../../services/pyth";
 import { getStatus } from "../../status";
 
 export const getPriceFeeds = async (cluster: Cluster, key: string) => {
   const [feeds, rankings] = await Promise.all([
-    getFeedsForPublisher(cluster, key),
+    getFeedsForPublisherRequest(cluster, key),
     getRankingsByPublisher(key),
   ]);
   return feeds.map((feed) => {

apps/insights/src/components/Publisher/layout.tsx

@@ -16,9 +16,9 @@ import type { ReactNode } from "react";
 import { Suspense } from "react";
 
 import {
+  getPublishers,
   getPublisherRankingHistory,
   getPublisherAverageScoreHistory,
-  getPublishers,
 } from "../../services/clickhouse";
 import { getPublisherCaps } from "../../services/hermes";
 import { ClusterToName, parseCluster, Cluster } from "../../services/pyth";
@@ -150,10 +150,10 @@ const RankingCard = async ({
   cluster: Cluster;
   publisherKey: string;
 }) => {
-  const rankingHistory = await getPublisherRankingHistory(
+  const rankingHistory = await getPublisherRankingHistory({
     cluster,
-    publisherKey,
-  );
+    key: publisherKey,
+  });
   return <RankingCardImpl rankingHistory={rankingHistory} />;
 };
 
@@ -234,10 +234,11 @@ const ScoreCard = async ({
   cluster: Cluster;
   publisherKey: string;
 }) => {
-  const averageScoreHistory = await getPublisherAverageScoreHistory(
+  const averageScoreHistory = await getPublisherAverageScoreHistory({
     cluster,
-    publisherKey,
-  );
+    key: publisherKey,
+  });
+
   return <ScoreCardImpl averageScoreHistory={averageScoreHistory} />;
 };
 

apps/insights/src/components/Publishers/index.tsx

@@ -31,7 +31,6 @@ export const Publishers = async () => {
       getPublishers(Cluster.PythtestConformance),
       getOisStats(),
     ]);
-
   const rankingTime = pythnetPublishers[0]?.timestamp;
   const scoreTime = pythnetPublishers[0]?.scoreTime;
 
@@ -173,7 +172,6 @@ const getOisStats = async () => {
       getDistributedRewards(),
       getPublisherCaps(),
     ]);
-
   return {
     totalStaked:
       sumDelegations(delState.delState) + sumDelegations(delState.selfDelState),

apps/insights/src/components/Root/index.tsx

@@ -5,13 +5,14 @@ import type { ReactNode } from "react";
 import { Suspense } from "react";
 
 import {
+  AMPLITUDE_API_KEY,
   ENABLE_ACCESSIBILITY_REPORTING,
   GOOGLE_ANALYTICS_ID,
-  AMPLITUDE_API_KEY,
 } from "../../config/server";
 import { LivePriceDataProvider } from "../../hooks/use-live-price-data";
 import { getPublishers } from "../../services/clickhouse";
-import { Cluster, getFeeds } from "../../services/pyth";
+import { Cluster } from "../../services/pyth";
+import { getFeeds } from "../../services/pyth/get-feeds";
 import { PriceFeedIcon } from "../PriceFeedIcon";
 import { PublisherIcon } from "../PublisherIcon";
 import { SearchButton as SearchButtonImpl } from "./search-button";
@@ -57,7 +58,6 @@ const SearchButton = async () => {
 };
 
 const getPublishersForSearchDialog = async (cluster: Cluster) => {
-  "use cache";
   const publishers = await getPublishers(cluster);
   return publishers.map((publisher) => {
     const knownPublisher = lookupPublisher(publisher.key);
@@ -75,7 +75,6 @@ const getPublishersForSearchDialog = async (cluster: Cluster) => {
 };
 
 const getFeedsForSearchDialog = async (cluster: Cluster) => {
-  "use cache";
   const feeds = await getFeeds(cluster);
 
   return feeds.map((feed) => ({

apps/insights/src/config/server.ts

@@ -2,12 +2,13 @@
 // and load all env variables.
 /* eslint-disable n/no-process-env */
 
+import { Redis } from "ioredis";
 import "server-only";
 
 /**
  * Throw if the env var `key` is not set (at either runtime or build time).
  */
-const demand = (key: string): string => {
+export const demand = (key: string): string => {
   const value = process.env[key];
   if (value === undefined || value === "") {
     throw new MissingEnvironmentError(key);
@@ -30,6 +31,7 @@ const getEnvOrDefault = (key: string, defaultValue: string) =>
  * Indicates that this server is the live customer-facing production server.
  */
 export const IS_PRODUCTION_SERVER = process.env.VERCEL_ENV === "production";
+export const IS_PREVIEW_SERVER = process.env.VERCEL_ENV === "preview";
 
 const defaultInProduction = IS_PRODUCTION_SERVER
   ? getEnvOrDefault
@@ -56,3 +58,23 @@ export const SOLANA_RPC =
 
 export const ENABLE_ACCESSIBILITY_REPORTING =
   !IS_PRODUCTION_SERVER && !process.env.DISABLE_ACCESSIBILITY_REPORTING;
+
+let redisClient: Redis | undefined;
+
+export function getRedis(): Redis {
+  const url = demand("REDIS_URL");
+  if (redisClient) {
+    return redisClient;
+  }
+  redisClient = new Redis(url);
+  return redisClient;
+}
+
+export const VERCEL_AUTOMATION_BYPASS_SECRET = demand(
+  "VERCEL_AUTOMATION_BYPASS_SECRET",
+);
+
+export const VERCEL_REQUEST_HEADERS = {
+  // this is a way to bypass vercel protection for the internal api route
+  "x-vercel-protection-bypass": VERCEL_AUTOMATION_BYPASS_SECRET,
+};

apps/insights/src/server/pyth.ts

@@ -0,0 +1,94 @@
+import { parse } from "superjson";
+import { z } from "zod";
+
+import { VERCEL_REQUEST_HEADERS } from "../config/server";
+import { Cluster, ClusterToName, priceFeedsSchema } from "../services/pyth";
+import { absoluteUrl } from "../utils/absolute-url";
+import { DEFAULT_CACHE_TTL } from "../utils/cache";
+
+export async function getPublishersForFeedRequest(
+  cluster: Cluster,
+  symbol: string,
+) {
+  const url = await absoluteUrl(
+    `/api/pyth/get-publishers/${encodeURIComponent(symbol)}`,
+  );
+  url.searchParams.set("cluster", ClusterToName[cluster]);
+
+  const data = await fetch(url, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: VERCEL_REQUEST_HEADERS,
+  });
+  const parsedData: unknown = await data.json();
+  return z.array(z.string()).parse(parsedData);
+}
+
+export async function getFeedsForPublisherRequest(
+  cluster: Cluster,
+  publisher: string,
+) {
+  const url = await absoluteUrl(
+    `/api/pyth/get-feeds-for-publisher/${encodeURIComponent(publisher)}`,
+  );
+  url.searchParams.set("cluster", ClusterToName[cluster]);
+
+  const data = await fetch(url, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: VERCEL_REQUEST_HEADERS,
+  });
+  const rawData = await data.text();
+  const parsedData = parse(rawData);
+  return priceFeedsSchema.parse(parsedData);
+}
+
+export const getFeedsRequest = async (cluster: Cluster) => {
+  const url = await absoluteUrl(`/api/pyth/get-feeds`);
+  url.searchParams.set("cluster", ClusterToName[cluster]);
+  url.searchParams.set("excludePriceComponents", "true");
+
+  const data = await fetch(url, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: VERCEL_REQUEST_HEADERS,
+  });
+  const rawData = await data.text();
+  const parsedData = parse(rawData);
+
+  return priceFeedsSchema.element
+    .omit({ price: true })
+    .array()
+    .parse(parsedData);
+};
+
+export const getFeedForSymbolRequest = async ({
+  symbol,
+  cluster = Cluster.Pythnet,
+}: {
+  symbol: string;
+  cluster?: Cluster;
+}): Promise<z.infer<typeof priceFeedsSchema.element> | undefined> => {
+  const url = await absoluteUrl(
+    `/api/pyth/get-feeds/${encodeURIComponent(symbol)}`,
+  );
+  url.searchParams.set("cluster", ClusterToName[cluster]);
+
+  const data = await fetch(url, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: VERCEL_REQUEST_HEADERS,
+  });
+
+  if (!data.ok) {
+    return undefined;
+  }
+
+  const rawData = await data.text();
+  const parsedData = parse(rawData);
+  return priceFeedsSchema.element.parse(parsedData);
+};

apps/insights/src/services/clickhouse.ts

@@ -6,10 +6,11 @@ import { z } from "zod";
 
 import { Cluster, ClusterToName } from "./pyth";
 import { CLICKHOUSE } from "../config/server";
+import { redisCache } from "../utils/cache";
 
 const client = createClient(CLICKHOUSE);
 
-export const getPublishers = async (cluster: Cluster) =>
+const _getPublishers = async (cluster: Cluster) =>
   safeQuery(
     z.array(
       z.strictObject({
@@ -72,7 +73,7 @@ export const getPublishers = async (cluster: Cluster) =>
     },
   );
 
-export const getRankingsByPublisher = async (publisherKey: string) =>
+const _getRankingsByPublisher = async (publisherKey: string) =>
   safeQuery(rankingsSchema, {
     query: `
       WITH first_rankings AS (
@@ -108,7 +109,7 @@ export const getRankingsByPublisher = async (publisherKey: string) =>
     query_params: { publisherKey },
   });
 
-export const getRankingsBySymbol = async (symbol: string) =>
+const _getRankingsBySymbol = async (symbol: string) =>
   safeQuery(rankingsSchema, {
     query: `
       WITH first_rankings AS (
@@ -159,7 +160,7 @@ const rankingsSchema = z.array(
   }),
 );
 
-export const getYesterdaysPrices = async (symbols: string[]) =>
+const _getYesterdaysPrices = async (symbols: string[]) =>
   safeQuery(
     z.array(
       z.object({
@@ -182,10 +183,13 @@ export const getYesterdaysPrices = async (symbols: string[]) =>
     },
   );
 
-export const getPublisherRankingHistory = async (
-  cluster: Cluster,
-  key: string,
-) =>
+const _getPublisherRankingHistory = async ({
+  cluster,
+  key,
+}: {
+  cluster: Cluster;
+  key: string;
+}) =>
   safeQuery(
     z.array(
       z.strictObject({
@@ -209,13 +213,19 @@ export const getPublisherRankingHistory = async (
     },
   );
 
-export const getFeedScoreHistory = async (
-  cluster: Cluster,
-  publisherKey: string,
-  symbol: string,
-  from: string,
-  to: string,
-) =>
+const _getFeedScoreHistory = async ({
+  cluster,
+  publisherKey,
+  symbol,
+  from,
+  to,
+}: {
+  cluster: Cluster;
+  publisherKey: string;
+  symbol: string;
+  from: string;
+  to: string;
+}) =>
   safeQuery(
     z.array(
       z.strictObject({
@@ -253,11 +263,15 @@ export const getFeedScoreHistory = async (
     },
   );
 
-export const getFeedPriceHistory = async (
-  cluster: Cluster,
-  publisherKey: string,
-  symbol: string,
-) =>
+const _getFeedPriceHistory = async ({
+  cluster,
+  publisherKey,
+  symbol,
+}: {
+  cluster: Cluster;
+  publisherKey: string;
+  symbol: string;
+}) =>
   safeQuery(
     z.array(
       z.strictObject({
@@ -287,10 +301,13 @@ export const getFeedPriceHistory = async (
     },
   );
 
-export const getPublisherAverageScoreHistory = async (
-  cluster: Cluster,
-  key: string,
-) =>
+export const _getPublisherAverageScoreHistory = async ({
+  cluster,
+  key,
+}: {
+  cluster: Cluster;
+  key: string;
+}) =>
   safeQuery(
     z.array(
       z.strictObject({
@@ -318,7 +335,13 @@ export const getPublisherAverageScoreHistory = async (
     },
   );
 
-export const getHistoricalPrices = async (symbol: string, until: string) =>
+const _getHistoricalPrices = async ({
+  symbol,
+  until,
+}: {
+  symbol: string;
+  until: string;
+}) =>
   safeQuery(
     z.array(
       z.strictObject({
@@ -353,3 +376,48 @@ const safeQuery = async <Output, Def extends ZodTypeDef, Input>(
 
   return schema.parse(result.data);
 };
+
+export const getRankingsBySymbol = redisCache.define(
+  "getRankingsBySymbol",
+  _getRankingsBySymbol,
+).getRankingsBySymbol;
+
+export const getRankingsByPublisher = redisCache.define(
+  "getRankingsByPublisher",
+  _getRankingsByPublisher,
+).getRankingsByPublisher;
+
+export const getPublishers = redisCache.define(
+  "getPublishers",
+  _getPublishers,
+).getPublishers;
+
+export const getPublisherAverageScoreHistory = redisCache.define(
+  "getPublisherAverageScoreHistory",
+  _getPublisherAverageScoreHistory,
+).getPublisherAverageScoreHistory;
+
+export const getPublisherRankingHistory = redisCache.define(
+  "getPublisherRankingHistory",
+  _getPublisherRankingHistory,
+).getPublisherRankingHistory;
+
+export const getFeedScoreHistory = redisCache.define(
+  "getFeedScoreHistory",
+  _getFeedScoreHistory,
+).getFeedScoreHistory;
+
+export const getHistoricalPrices = redisCache.define(
+  "getHistoricalPrices",
+  _getHistoricalPrices,
+).getHistoricalPrices;
+
+export const getFeedPriceHistory = redisCache.define(
+  "getFeedPriceHistory",
+  _getFeedPriceHistory,
+).getFeedPriceHistory;
+
+export const getYesterdaysPrices = redisCache.define(
+  "getYesterdaysPrices",
+  _getYesterdaysPrices,
+).getYesterdaysPrices;

apps/insights/src/services/pyth/get-feeds.ts

@@ -0,0 +1,29 @@
+import { Cluster, priceFeedsSchema } from ".";
+import { getPythMetadataCached } from "./get-metadata";
+import { redisCache } from "../../utils/cache";
+
+const _getFeeds = async (cluster: Cluster) => {
+  const unfilteredData = await getPythMetadataCached(cluster);
+  const filtered = unfilteredData.symbols
+    .filter(
+      (symbol) =>
+        unfilteredData.productFromSymbol.get(symbol)?.display_symbol !==
+        undefined,
+    )
+    .map((symbol) => ({
+      symbol,
+      product: unfilteredData.productFromSymbol.get(symbol),
+      price: {
+        ...unfilteredData.productPrice.get(symbol),
+        priceComponents:
+          unfilteredData.productPrice
+            .get(symbol)
+            ?.priceComponents.map(({ publisher }) => ({
+              publisher: publisher.toBase58(),
+            })) ?? [],
+      },
+    }));
+  return priceFeedsSchema.parse(filtered);
+};
+
+export const getFeeds = redisCache.define("getFeeds", _getFeeds).getFeeds;

apps/insights/src/services/pyth/get-metadata.ts

@@ -0,0 +1,11 @@
+import { clients, Cluster } from ".";
+import { memoryOnlyCache } from "../../utils/cache";
+
+const getPythMetadata = async (cluster: Cluster) => {
+  return clients[cluster].getData();
+};
+
+export const getPythMetadataCached = memoryOnlyCache.define(
+  "getPythMetadata",
+  getPythMetadata,
+).getPythMetadata;

apps/insights/src/services/pyth/get-publishers-for-cluster.ts

@@ -0,0 +1,19 @@
+import { Cluster } from ".";
+import { getPythMetadataCached } from "./get-metadata";
+import { redisCache } from "../../utils/cache";
+
+const _getPublishersForCluster = async (cluster: Cluster) => {
+  const data = await getPythMetadataCached(cluster);
+  const result: Record<string, string[]> = {};
+  for (const key of data.productPrice.keys()) {
+    const price = data.productPrice.get(key);
+    result[key] =
+      price?.priceComponents.map(({ publisher }) => publisher.toBase58()) ?? [];
+  }
+  return result;
+};
+
+export const getPublishersForCluster = redisCache.define(
+  "getPublishersForCluster",
+  _getPublishersForCluster,
+).getPublishersForCluster;

apps/insights/src/services/pyth.ts → apps/insights/src/services/pyth/index.ts

@@ -7,7 +7,7 @@ import type { PythPriceCallback } from "@pythnetwork/client/lib/PythConnection";
 import { Connection, PublicKey } from "@solana/web3.js";
 import { z } from "zod";
 
-import { PYTHNET_RPC, PYTHTEST_CONFORMANCE_RPC } from "../config/isomorphic";
+import { PYTHNET_RPC, PYTHTEST_CONFORMANCE_RPC } from "../../config/isomorphic";
 
 export enum Cluster {
   Pythnet,
@@ -56,57 +56,12 @@ const mkClient = (cluster: Cluster) =>
     getPythProgramKeyForCluster(ClusterToName[cluster]),
   );
 
-const clients = {
+export const clients = {
   [Cluster.Pythnet]: mkClient(Cluster.Pythnet),
   [Cluster.PythtestConformance]: mkClient(Cluster.PythtestConformance),
 } as const;
 
-export const getPublishersForFeed = async (
-  cluster: Cluster,
-  symbol: string,
-) => {
-  const data = await clients[cluster].getData();
-  return data.productPrice
-    .get(symbol)
-    ?.priceComponents.map(({ publisher }) => publisher.toBase58());
-};
-
-export const getFeeds = async (cluster: Cluster) => {
-  const data = await clients[cluster].getData();
-  return priceFeedsSchema.parse(
-    data.symbols.map((symbol) => ({
-      symbol,
-      product: data.productFromSymbol.get(symbol),
-      price: data.productPrice.get(symbol),
-    })),
-  );
-};
-
-export const getFeedsForPublisher = async (
-  cluster: Cluster,
-  publisher: string,
-) => {
-  const data = await clients[cluster].getData();
-  return priceFeedsSchema.parse(
-    data.symbols
-      .filter(
-        (symbol) =>
-          data.productFromSymbol.get(symbol)?.display_symbol !== undefined,
-      )
-      .map((symbol) => ({
-        symbol,
-        product: data.productFromSymbol.get(symbol),
-        price: data.productPrice.get(symbol),
-      }))
-      .filter(({ price }) =>
-        price?.priceComponents.some(
-          (component) => component.publisher.toBase58() === publisher,
-        ),
-      ),
-  );
-};
-
-const priceFeedsSchema = z.array(
+export const priceFeedsSchema = z.array(
   z.object({
     symbol: z.string(),
     product: z.object({
@@ -133,6 +88,11 @@ const priceFeedsSchema = z.array(
       minPublishers: z.number(),
       lastSlot: z.bigint(),
       validSlot: z.bigint(),
+      priceComponents: z.array(
+        z.object({
+          publisher: z.string(),
+        }),
+      ),
     }),
   }),
 );

apps/insights/src/utils/absolute-url.ts

@@ -0,0 +1,51 @@
+import { headers } from "next/headers";
+
+import {
+  demand,
+  IS_PREVIEW_SERVER,
+  IS_PRODUCTION_SERVER,
+} from "../config/server";
+
+/**
+ * Returns an absolute URL for the given pathname.
+ *
+ * @param pathname - The pathname to make absolute.
+ * @returns A URL object with the absolute URL.
+ */
+export async function absoluteUrl(pathname: string) {
+  let origin: string | undefined;
+
+  try {
+    // note that using headers() makes the context dynamic (disables full static optimization)
+    const nextHeaders = await headers();
+    // this can be comma-separated, so we take the first one
+    const xfHost = nextHeaders.get("x-forwarded-host")?.split(",")[0]?.trim();
+    const host = xfHost ?? nextHeaders.get("host") ?? undefined;
+
+    // this can be comma-separated, so we take the first one
+    const proto =
+      nextHeaders.get("x-forwarded-proto")?.split(",")[0]?.trim() ??
+      (host?.startsWith("localhost") ? "http" : "https");
+
+    // if we have a host and a proto, we can construct the origin
+    if (host && proto) origin = `${proto}://${host}`;
+  } catch {
+    // headers() is unavailable
+  }
+
+  // Fallbacks for requests where headers() is not available
+  if (!origin) {
+    if (IS_PRODUCTION_SERVER) {
+      const productionUrl = demand("VERCEL_PROJECT_PRODUCTION_URL");
+      origin = `https://${productionUrl}`;
+    } else if (IS_PREVIEW_SERVER) {
+      const previewUrl = demand("VERCEL_URL");
+      origin = `https://${previewUrl}`;
+    } else {
+      origin = "http://localhost:3003";
+    }
+  }
+
+  const path = pathname.startsWith("/") ? pathname : `/${pathname}`;
+  return new URL(origin + path);
+}

apps/insights/src/utils/cache.ts

@@ -0,0 +1,30 @@
+import type { Cache as ACDCache } from "async-cache-dedupe";
+import { createCache } from "async-cache-dedupe";
+import { stringify, parse } from "superjson";
+
+import { getRedis } from "../config/server";
+
+const transformer = {
+  serialize: stringify,
+  deserialize: parse,
+};
+
+export const DEFAULT_CACHE_TTL = 86_400; // 24 hours
+export const DEFAULT_CACHE_STALE = 86_400; // 24 hours
+
+export const redisCache: ACDCache = createCache({
+  transformer,
+  stale: DEFAULT_CACHE_STALE,
+  ttl: DEFAULT_CACHE_TTL,
+  storage: {
+    type: "redis",
+    options: {
+      client: getRedis(),
+    },
+  },
+});
+
+export const memoryOnlyCache: ACDCache = createCache({
+  ttl: DEFAULT_CACHE_TTL,
+  stale: DEFAULT_CACHE_STALE,
+});

apps/insights/turbo.json

@@ -4,7 +4,9 @@
   "tasks": {
     "build:vercel": {
       "env": [
+        "VERCEL_AUTOMATION_BYPASS_SECRET",
         "VERCEL_ENV",
+        "VERCEL_URL",
         "GOOGLE_ANALYTICS_ID",
         "AMPLITUDE_API_KEY",
         "CLICKHOUSE_URL",
@@ -13,7 +15,9 @@
         "SOLANA_RPC",
         "DISABLE_ACCESSIBILITY_REPORTING",
         "NEXT_PUBLIC_PYTHNET_RPC",
-        "NEXT_PUBLIC_PYTHTEST_CONFORMANCE_RPC"
+        "NEXT_PUBLIC_PYTHTEST_CONFORMANCE_RPC",
+        "REDIS_URL",
+        "VERCEL_PROJECT_PRODUCTION_URL"
       ]
     },
     "fix:lint": {

pnpm-workspace.yaml

@@ -118,7 +118,7 @@ catalog:
   lucide-react: ^0.487.0
   modern-normalize: ^3.0.1
   motion: ^12.9.2
-  next: ^15.3.2
+  next: ^15.4.5
   next-themes: ^0.4.6
   nuqs: ^2.4.1
   pino: ^9.6.0
@@ -155,6 +155,8 @@ catalog:
   wagmi: ^2.14.16
   zod: ^3.24.2
   zod-validation-error: ^3.4.0
+  async-cache-dedupe: ^3.0.0
+  zod-search-params: ^0.1.6
 
 onlyBuiltDependencies:
   - "@apollo/protobufjs"