feat: more endpoint fetching

apps/insights/src/app/api/pyth/get-feeds/[symbol]/route.ts

@@ -1,12 +1,18 @@
+import { NextResponse } from "next/server";
 import { stringify } from 'superjson';
 
 import { getFeedsCached } from "../../../../../server/pyth/get-feeds";
 import { Cluster } from "../../../../../services/pyth";
 
-export const GET = async (_: Request, { params }: { params: Promise<{ symbol: string }> }) => {
+export const GET = async (request: Request, { params }: { params: Promise<{ symbol: string }> }) => {
   const { symbol } = await params;
-
-  const feeds = await getFeedsCached(Cluster.Pythnet);
+  const { searchParams } = new URL(request.url);
+  const cluster = Number.parseInt(searchParams.get("cluster") ?? Cluster.Pythnet.toString()) as Cluster;
+  // check if cluster is valid
+  if (cluster && !Object.values(Cluster).includes(cluster)) {
+    return NextResponse.json({ error: "Invalid cluster" }, { status: 400 });
+  }
+  const feeds = await getFeedsCached(cluster);
   const feed = feeds.find((feed) => feed.symbol === symbol);
   return new Response(stringify(feed), {
     headers: {

apps/insights/src/components/PriceFeed/get-feed.tsx

@@ -2,12 +2,14 @@ import { parse } from "superjson";
 import { z } from "zod";
 
 import { PUBLIC_URL, VERCEL_AUTOMATION_BYPASS_SECRET } from '../../config/server';
+import { getFeedForSymbolCached } from '../../server/pyth';
 import { Cluster, priceFeedsSchema } from "../../services/pyth";
+import { DEFAULT_CACHE_TTL } from '../../utils/cache';
 
 export const getFeed = async (params: Promise<{ slug: string }>) => {
   const data = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds?cluster=${Cluster.Pythnet.toString()}&excludePriceComponents=true`, {
     next: {
-      revalidate: 1000 * 60 * 60 * 24,
+      revalidate: DEFAULT_CACHE_TTL,
     },
     headers: {
       'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
@@ -18,20 +20,11 @@ export const getFeed = async (params: Promise<{ slug: string }>) => {
 
   const { slug } = await params;
   const symbol = decodeURIComponent(slug);
-  const feed = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds/${encodeURIComponent(symbol)}`, {
-    next: {
-      revalidate: 1000 * 60 * 60 * 24,
-    },
-     headers: {
-      'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
-    },
-  });
-  const feedJson = await feed.text();
-  const feedData: z.infer<typeof priceFeedsSchema>[0] = parse(feedJson);
+  const feed = await getFeedForSymbolCached({symbol, cluster: Cluster.Pythnet});
 
   return {
     feeds,
-    feed: feedData,
+    feed,
     symbol,
   } as const;
 };

apps/insights/src/components/PriceFeed/publishers.tsx

@@ -2,8 +2,7 @@ import { lookup as lookupPublisher } from "@pythnetwork/known-publishers";
 import { notFound } from "next/navigation";
 
 import { getRankingsBySymbolCached } from '../../server/clickhouse';
-import { getPublishersForFeedCached } from "../../server/pyth";
-import { getFeeds } from "../../server/pyth/get-feeds";
+import { getFeedForSymbolCached, getPublishersForFeedCached } from "../../server/pyth";
 import {
   Cluster,
   ClusterToName,
@@ -23,25 +22,22 @@ type Props = {
 export const Publishers = async ({ params }: Props) => {
   const { slug } = await params;
   const symbol = decodeURIComponent(slug);
+  
   const start = Date.now();
   const [
-    pythnetFeeds,
-    pythtestConformanceFeeds,
+    feed,
+    testFeed,
     pythnetPublishers,
     pythtestConformancePublishers,
   ] = await Promise.all([
-    getFeeds(Cluster.Pythnet),
-    getFeeds(Cluster.PythtestConformance),
+    getFeedForSymbolCached({symbol, cluster: Cluster.Pythnet}),
+    getFeedForSymbolCached({symbol, cluster: Cluster.PythtestConformance}),
     getPublishers(Cluster.Pythnet, symbol),
     getPublishers(Cluster.PythtestConformance, symbol),
   ]);
   const end = Date.now();
   // eslint-disable-next-line no-console
   console.info(`Publishers took ${(end - start).toString()}ms`);
-  const feed = pythnetFeeds.find((feed) => feed.symbol === symbol);
-  const testFeed = pythtestConformanceFeeds.find(
-    (feed) => feed.symbol === symbol,
-  );
   const publishers = [...pythnetPublishers, ...pythtestConformancePublishers];
   const metricsTime = pythnetPublishers.find(
     (publisher) => publisher.ranking !== undefined,

apps/insights/src/config/server.ts

@@ -69,7 +69,10 @@ export function getRedis(): Redis {
   if (!host || !port) {
     throw new Error('REDIS_HOST, and REDIS_PORT must be set');
   }
-  redisClient ??= new Redis({ 
+  if(redisClient) {
+    return redisClient;
+  }
+  redisClient = new Redis({ 
     username: 'default',
     password: password ?? '',
     host,

apps/insights/src/server/clickhouse.ts

@@ -3,45 +3,30 @@ import { redisCache } from '../utils/cache';
 
 export const getRankingsBySymbolCached = redisCache.define(
   "getRankingsBySymbol",
-  {
-    ttl: 1000 * 60 * 60 * 24,
-  },
   getRankingsBySymbol,
 ).getRankingsBySymbol;
 
 export const getRankingsByPublisherCached = redisCache.define(
   "getRankingsByPublisher",
-  {
-    ttl: 1000 * 60 * 60 * 24,
-  },
   getRankingsByPublisher,
 ).getRankingsByPublisher;
 
 
 export const getPublishersCached = redisCache.define(
   "getPublishers",
-  {
-    ttl: 1000 * 60 * 60 * 24,
-  },
   getPublishers,
 ).getPublishers;
 
 export const getPublisherAverageScoreHistoryCached = redisCache.define(
   "getPublisherAverageScoreHistory",
-  {
-    ttl: 1000 * 60 * 60 * 24,
-  },
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   getPublisherAverageScoreHistory,
-).getPublisherAverageScoreHistory as typeof getPublisherAverageScoreHistory;
+).getPublisherAverageScoreHistory
 
 export const getPublisherRankingHistoryCached = redisCache.define(
   "getPublisherRankingHistory",
-  {
-    ttl: 1000 * 60 * 60 * 24,
-  },
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error
   getPublisherRankingHistory,
-).getPublisherRankingHistory as typeof getPublisherRankingHistory;
+).getPublisherRankingHistory

apps/insights/src/server/pyth.ts

@@ -3,6 +3,7 @@ import { z } from "zod";
 
 import { PUBLIC_URL, VERCEL_AUTOMATION_BYPASS_SECRET } from '../config/server';
 import { Cluster, priceFeedsSchema } from "../services/pyth";
+import { DEFAULT_CACHE_TTL } from "../utils/cache";
 
 // Convenience helpers matching your previous functions
 export async function getPublishersForFeedCached(
@@ -11,7 +12,7 @@ export async function getPublishersForFeedCached(
 ) {
   const data = await fetch(`${PUBLIC_URL}/api/pyth/get-publishers/${encodeURIComponent(symbol)}?cluster=${cluster.toString()}`, {
     next: {
-      revalidate: 1000 * 60 * 60 * 24,
+      revalidate: DEFAULT_CACHE_TTL,
     },
     headers: {
       'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
@@ -26,7 +27,7 @@ export async function getFeedsForPublisherCached(
 ) {
   const data = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds-for-publisher/${encodeURIComponent(publisher)}?cluster=${cluster.toString()}`, {
     next: {
-      revalidate: 1000 * 60 * 60 * 24,
+      revalidate: DEFAULT_CACHE_TTL,
     },
     headers: {
       'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
@@ -34,4 +35,36 @@ export async function getFeedsForPublisherCached(
   });
   const rawData = await data.text();
   return parse<z.infer<typeof priceFeedsSchema>>(rawData);
+}
+
+export const getFeedsCached = async (cluster: Cluster) => {
+  const data = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds?cluster=${cluster.toString()}&excludePriceComponents=true`, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: {
+      'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
+    },
+  });
+  const dataJson = await data.text();
+  const feeds: z.infer<typeof priceFeedsSchema> = parse(dataJson);
+  return feeds;
+}
+
+export const getFeedForSymbolCached = async ({symbol, cluster = Cluster.Pythnet}: {symbol: string, cluster?: Cluster}): Promise<z.infer<typeof priceFeedsSchema>[0] | undefined> => {
+  const data = await fetch(`${PUBLIC_URL}/api/pyth/get-feeds/${encodeURIComponent(symbol)}?cluster=${cluster.toString()}`, {
+    next: {
+      revalidate: DEFAULT_CACHE_TTL,
+    },
+    headers: {
+      'x-vercel-protection-bypass': VERCEL_AUTOMATION_BYPASS_SECRET,
+    },
+  });
+  
+  if(!data.ok) {
+    return undefined;
+  }
+  const dataJson = await data.text();
+  const feed: z.infer<typeof priceFeedsSchema>[0] = parse(dataJson);
+  return feed;
 }

apps/insights/src/server/pyth/get-feeds.ts

@@ -2,7 +2,7 @@ import { z } from 'zod';
 
 import { getPythMetadata } from './get-metadata';
 import { Cluster, priceFeedsSchema } from "../../services/pyth";
-import { redisCache } from '../../utils/cache';
+import { DEFAULT_CACHE_TTL, redisCache } from '../../utils/cache';
 
 const _getFeeds = async (cluster: Cluster) => {
   const unfilteredData = await getPythMetadata(cluster);
@@ -32,7 +32,7 @@ const _getFeeds = async (cluster: Cluster) => {
 export const getFeedsCached = redisCache.define(
   "getFeeds",
   {
-    ttl: 1000 * 60 * 60 * 24,
+    ttl: DEFAULT_CACHE_TTL,
   },
   _getFeeds,
 ).getFeeds;

apps/insights/src/server/pyth/get-metadata.ts

@@ -1,6 +1,6 @@
 
 import { clients, Cluster } from '../../services/pyth';
-import { memoryOnlyCache } from '../../utils/cache';
+import { DEFAULT_CACHE_TTL, memoryOnlyCache } from '../../utils/cache';
 
 const _getPythMetadata = async (cluster: Cluster) => {
   // Fetch fresh data from Pyth client
@@ -9,7 +9,7 @@ const _getPythMetadata = async (cluster: Cluster) => {
 
 export const getPythMetadata = memoryOnlyCache.define(
   "getPythMetadata",
-  {ttl: 1000 * 60 * 60 * 24},
+  {ttl: DEFAULT_CACHE_TTL},
   _getPythMetadata,
 ).getPythMetadata;
 

apps/insights/src/server/pyth/get-publishers-for-cluster.ts

@@ -1,6 +1,6 @@
 import { getPythMetadata } from './get-metadata';
 import { Cluster } from '../../services/pyth';
-import { redisCache } from '../../utils/cache';
+import { DEFAULT_CACHE_TTL, redisCache } from '../../utils/cache';
 
 const _computePublishers = async (cluster: Cluster) => {
   const data = await getPythMetadata(cluster);
@@ -16,7 +16,7 @@ const _computePublishers = async (cluster: Cluster) => {
 export const getPublishersForCluster = redisCache.define(
   "getPublishersForCluster",
   {
-    ttl: 1000 * 60 * 60 * 24,
+    ttl: DEFAULT_CACHE_TTL,
   },
   _computePublishers,
 ).getPublishersForCluster;

apps/insights/src/utils/cache.ts

@@ -1,15 +1,21 @@
 import type { Cache as ACDCache } from "async-cache-dedupe";
 import { createCache } from "async-cache-dedupe";
-import { serialize, deserialize } from "superjson";
+import { stringify, parse } from "superjson";
 
 import { getRedis } from '../config/server';
 
-// L2-backed cache: in-memory LRU (L1) + Redis (L2)
+const transformer = {
+  serialize: stringify,
+  deserialize: parse,
+};
+
+export const DEFAULT_CACHE_TTL = 86_400; // 24 hours
+export const DEFAULT_CACHE_STALE = 86_400; // 24 hours
+
 export const redisCache: ACDCache = createCache({
-  transformer: {
-    serialize,
-    deserialize,
-  },
+  transformer,
+  stale: DEFAULT_CACHE_STALE,
+  ttl: DEFAULT_CACHE_TTL,
   storage: {
     type: "redis",
     options: {
@@ -19,6 +25,6 @@ export const redisCache: ACDCache = createCache({
 });
 
 export const memoryOnlyCache: ACDCache = createCache({
-  ttl: 5000,
-  stale: 2000,
-});
+  ttl: DEFAULT_CACHE_TTL,
+  stale: DEFAULT_CACHE_STALE,
+});