Merge pull request #2320 from pyth-network/cprussin/fix-insights-cache

fix(insights): remove data caching & pregeneration, add revalidate

apps/insights/src/app/layout.ts

@@ -1,2 +1,4 @@
 export { Root as default } from "../components/Root";
 export { metadata, viewport } from "../metadata";
+
+export const revalidate = 3600;

apps/insights/src/app/price-feeds/[slug]/layout.ts

@@ -1,13 +1,9 @@
 import type { Metadata } from "next";
 
-import { Cluster, getFeeds } from "../../../services/pyth";
 export { PriceFeedLayout as default } from "../../../components/PriceFeed/layout";
 
 export const metadata: Metadata = {
   title: "Price Feeds",
 };
 
-export const generateStaticParams = async () => {
-  const feeds = await getFeeds(Cluster.Pythnet);
-  return feeds.map(({ symbol }) => ({ slug: encodeURIComponent(symbol) }));
-};
+export const revalidate = 3600;

apps/insights/src/app/price-feeds/[slug]/page.ts

@@ -1 +1,3 @@
 export { ChartPage as default } from "../../../components/PriceFeed/chart-page";
+
+export const revalidate = 3600;

apps/insights/src/app/price-feeds/[slug]/publishers/page.tsx

@@ -1 +1,3 @@
 export { Publishers as default } from "../../../../components/PriceFeed/publishers";
+
+export const revalidate = 3600;

apps/insights/src/app/price-feeds/page.ts

@@ -5,3 +5,5 @@ export { PriceFeeds as default } from "../../components/PriceFeeds";
 export const metadata: Metadata = {
   title: "Price Feeds",
 };
+
+export const revalidate = 3600;

apps/insights/src/app/publishers/[key]/layout.ts

@@ -1,13 +1,9 @@
 import type { Metadata } from "next";
 
 export { PublishersLayout as default } from "../../../components/Publisher/layout";
-import { getPublishers } from "../../../services/clickhouse";
 
 export const metadata: Metadata = {
   title: "Publishers",
 };
 
-export const generateStaticParams = async () => {
-  const publishers = await getPublishers();
-  return publishers.map(({ key }) => ({ key }));
-};
+export const revalidate = 3600;

apps/insights/src/app/publishers/[key]/page.ts

@@ -1 +1,3 @@
 export { Performance as default } from "../../../components/Publisher/performance";
+
+export const revalidate = 3600;

apps/insights/src/app/publishers/[key]/price-feeds/page.ts

@@ -1 +1,3 @@
 export { PriceFeeds as default } from "../../../../components/Publisher/price-feeds";
+
+export const revalidate = 3600;

apps/insights/src/app/publishers/page.ts

@@ -5,3 +5,5 @@ export { Publishers as default } from "../../components/Publishers";
 export const metadata: Metadata = {
   title: "Publishers",
 };
+
+export const revalidate = 3600;

apps/insights/src/cache.ts

@@ -1,16 +0,0 @@
-import { unstable_cache } from "next/cache";
-import { parse, stringify } from "superjson";
-
-export const cache = <T, P extends unknown[]>(
-  fn: (...params: P) => Promise<T>,
-  keys?: Parameters<typeof unstable_cache>[1],
-  opts?: Parameters<typeof unstable_cache>[2],
-) => {
-  const cachedFn = unstable_cache(
-    async (params: P): Promise<string> => stringify(await fn(...params)),
-    keys,
-    opts,
-  );
-
-  return async (...params: P): Promise<T> => parse(await cachedFn(params));
-};

apps/insights/src/services/clickhouse.ts

@@ -4,34 +4,29 @@ import { createClient } from "@clickhouse/client";
 import { z, type ZodSchema, type ZodTypeDef } from "zod";
 
 import { Cluster, ClusterToName } from "./pyth";
-import { cache } from "../cache";
 import { CLICKHOUSE } from "../config/server";
 
 const client = createClient(CLICKHOUSE);
 
-const ONE_MINUTE_IN_SECONDS = 60;
-const ONE_HOUR_IN_SECONDS = 60 * ONE_MINUTE_IN_SECONDS;
-
-export const getPublishers = cache(
-  async () =>
-    safeQuery(
-      z.array(
-        z.strictObject({
-          key: z.string(),
-          rank: z.number(),
-          activeFeeds: z
-            .string()
-            .transform((value) => Number.parseInt(value, 10)),
-          inactiveFeeds: z
-            .string()
-            .transform((value) => Number.parseInt(value, 10)),
-          averageScore: z.number(),
-          timestamp: z.string().transform((value) => new Date(`${value} UTC`)),
-          scoreTime: z.string().transform((value) => new Date(value)),
-        }),
-      ),
-      {
-        query: `
+export const getPublishers = async () =>
+  safeQuery(
+    z.array(
+      z.strictObject({
+        key: z.string(),
+        rank: z.number(),
+        activeFeeds: z
+          .string()
+          .transform((value) => Number.parseInt(value, 10)),
+        inactiveFeeds: z
+          .string()
+          .transform((value) => Number.parseInt(value, 10)),
+        averageScore: z.number(),
+        timestamp: z.string().transform((value) => new Date(`${value} UTC`)),
+        scoreTime: z.string().transform((value) => new Date(value)),
+      }),
+    ),
+    {
+      query: `
           WITH score_data AS (
             SELECT
               publisher,
@@ -68,19 +63,13 @@ export const getPublishers = cache(
           )
           ORDER BY rank ASC, timestamp
         `,
-        query_params: { cluster: "pythnet" },
-      },
-    ),
-  ["publishers"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+      query_params: { cluster: "pythnet" },
+    },
+  );
 
-export const getRankingsByPublisher = cache(
-  async (publisherKey: string) =>
-    safeQuery(rankingsSchema, {
-      query: `
+export const getRankingsByPublisher = async (publisherKey: string) =>
+  safeQuery(rankingsSchema, {
+    query: `
       SELECT
           time,
           symbol,
@@ -100,18 +89,12 @@ export const getRankingsByPublisher = cache(
           cluster ASC,
           publisher ASC
       `,
-      query_params: { publisherKey },
-    }),
-  ["rankingsByPublisher"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+    query_params: { publisherKey },
+  });
 
-export const getRankingsBySymbol = cache(
-  async (symbol: string) =>
-    safeQuery(rankingsSchema, {
-      query: `
+export const getRankingsBySymbol = async (symbol: string) =>
+  safeQuery(rankingsSchema, {
+    query: `
         SELECT
           time,
           symbol,
@@ -131,13 +114,8 @@ export const getRankingsBySymbol = cache(
           cluster ASC,
           publisher ASC
       `,
-      query_params: { symbol },
-    }),
-  ["rankingsBySymbol"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+    query_params: { symbol },
+  });
 
 const rankingsSchema = z.array(
   z.strictObject({
@@ -153,17 +131,16 @@ const rankingsSchema = z.array(
   }),
 );
 
-export const getYesterdaysPrices = cache(
-  async (symbols: string[]) =>
-    safeQuery(
-      z.array(
-        z.object({
-          symbol: z.string(),
-          price: z.number(),
-        }),
-      ),
-      {
-        query: `
+export const getYesterdaysPrices = async (symbols: string[]) =>
+  safeQuery(
+    z.array(
+      z.object({
+        symbol: z.string(),
+        price: z.number(),
+      }),
+    ),
+    {
+      query: `
           SELECT symbol, price
           FROM prices
           WHERE cluster = 'pythnet'
@@ -173,26 +150,20 @@ export const getYesterdaysPrices = cache(
           ORDER BY time ASC
           LIMIT 1 BY symbol
         `,
-        query_params: { symbols },
-      },
-    ),
-  ["yesterdays-prices"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+      query_params: { symbols },
+    },
+  );
 
-export const getPublisherRankingHistory = cache(
-  async (key: string) =>
-    safeQuery(
-      z.array(
-        z.strictObject({
-          timestamp: z.string().transform((value) => new Date(value)),
-          rank: z.number(),
-        }),
-      ),
-      {
-        query: `
+export const getPublisherRankingHistory = async (key: string) =>
+  safeQuery(
+    z.array(
+      z.strictObject({
+        timestamp: z.string().transform((value) => new Date(value)),
+        rank: z.number(),
+      }),
+    ),
+    {
+      query: `
           SELECT * FROM (
             SELECT timestamp, rank
             FROM publishers_ranking
@@ -203,27 +174,27 @@ export const getPublisherRankingHistory = cache(
           )
           ORDER BY timestamp ASC
         `,
-        query_params: { key },
-      },
-    ),
-  ["publisher-ranking-history"],
-  { revalidate: ONE_HOUR_IN_SECONDS },
-);
+      query_params: { key },
+    },
+  );
 
-export const getFeedScoreHistory = cache(
-  async (cluster: Cluster, publisherKey: string, symbol: string) =>
-    safeQuery(
-      z.array(
-        z.strictObject({
-          time: z.string().transform((value) => new Date(value)),
-          score: z.number(),
-          uptimeScore: z.number(),
-          deviationScore: z.number(),
-          stalledScore: z.number(),
-        }),
-      ),
-      {
-        query: `
+export const getFeedScoreHistory = async (
+  cluster: Cluster,
+  publisherKey: string,
+  symbol: string,
+) =>
+  safeQuery(
+    z.array(
+      z.strictObject({
+        time: z.string().transform((value) => new Date(value)),
+        score: z.number(),
+        uptimeScore: z.number(),
+        deviationScore: z.number(),
+        stalledScore: z.number(),
+      }),
+    ),
+    {
+      query: `
           SELECT * FROM (
             SELECT
               time,
@@ -240,31 +211,29 @@ export const getFeedScoreHistory = cache(
           )
           ORDER BY time ASC
         `,
-        query_params: {
-          cluster: ClusterToName[cluster],
-          publisherKey,
-          symbol,
-        },
+      query_params: {
+        cluster: ClusterToName[cluster],
+        publisherKey,
+        symbol,
       },
-    ),
-  ["feed-score-history"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+    },
+  );
 
-export const getFeedPriceHistory = cache(
-  async (cluster: Cluster, publisherKey: string, symbol: string) =>
-    safeQuery(
-      z.array(
-        z.strictObject({
-          time: z.string().transform((value) => new Date(value)),
-          price: z.number(),
-          confidence: z.number(),
-        }),
-      ),
-      {
-        query: `
+export const getFeedPriceHistory = async (
+  cluster: Cluster,
+  publisherKey: string,
+  symbol: string,
+) =>
+  safeQuery(
+    z.array(
+      z.strictObject({
+        time: z.string().transform((value) => new Date(value)),
+        price: z.number(),
+        confidence: z.number(),
+      }),
+    ),
+    {
+      query: `
           SELECT * FROM (
             SELECT time, price, confidence
             FROM prices
@@ -276,30 +245,24 @@ export const getFeedPriceHistory = cache(
           )
           ORDER BY time ASC
         `,
-        query_params: {
-          cluster: ClusterToName[cluster],
-          publisherKey,
-          symbol,
-        },
+      query_params: {
+        cluster: ClusterToName[cluster],
+        publisherKey,
+        symbol,
       },
-    ),
-  ["feed-price-history"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+    },
+  );
 
-export const getPublisherAverageScoreHistory = cache(
-  async (key: string) =>
-    safeQuery(
-      z.array(
-        z.strictObject({
-          time: z.string().transform((value) => new Date(value)),
-          averageScore: z.number(),
-        }),
-      ),
-      {
-        query: `
+export const getPublisherAverageScoreHistory = async (key: string) =>
+  safeQuery(
+    z.array(
+      z.strictObject({
+        time: z.string().transform((value) => new Date(value)),
+        averageScore: z.number(),
+      }),
+    ),
+    {
+      query: `
           SELECT * FROM (
             SELECT
               time,
@@ -313,27 +276,21 @@ export const getPublisherAverageScoreHistory = cache(
           )
           ORDER BY time ASC
         `,
-        query_params: { key },
-      },
-    ),
-  ["publisher-average-score-history"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+      query_params: { key },
+    },
+  );
 
-export const getHistoricalPrices = cache(
-  async (symbol: string, until: string) =>
-    safeQuery(
-      z.array(
-        z.strictObject({
-          timestamp: z.number(),
-          price: z.number(),
-          confidence: z.number(),
-        }),
-      ),
-      {
-        query: `
+export const getHistoricalPrices = async (symbol: string, until: string) =>
+  safeQuery(
+    z.array(
+      z.strictObject({
+        timestamp: z.number(),
+        price: z.number(),
+        confidence: z.number(),
+      }),
+    ),
+    {
+      query: `
           SELECT toUnixTimestamp(time) AS timestamp, avg(price) AS price, avg(confidence) AS confidence
           FROM prices
           WHERE cluster = 'pythnet'
@@ -345,12 +302,9 @@ export const getHistoricalPrices = cache(
           GROUP BY time
           ORDER BY time ASC
         `,
-        query_params: { symbol, until },
-      },
-    ),
-  ["price-history"],
-  {},
-);
+      query_params: { symbol, until },
+    },
+  );
 
 const safeQuery = async <Output, Def extends ZodTypeDef, Input>(
   schema: ZodSchema<Output, Def, Input>,

apps/insights/src/services/hermes.ts

@@ -2,17 +2,7 @@ import "server-only";
 
 import { HermesClient } from "@pythnetwork/hermes-client";
 
-import { cache } from "../cache";
-
-const ONE_MINUTE_IN_SECONDS = 60;
-const ONE_HOUR_IN_SECONDS = 60 * ONE_MINUTE_IN_SECONDS;
-
 const client = new HermesClient("https://hermes.pyth.network");
 
-export const getPublisherCaps = cache(
-  async () => client.getLatestPublisherCaps({ parsed: true }),
-  ["publisher-caps"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+export const getPublisherCaps = async () =>
+  client.getLatestPublisherCaps({ parsed: true });

apps/insights/src/services/pyth.ts

@@ -8,11 +8,6 @@ import type { PythPriceCallback } from "@pythnetwork/client/lib/PythConnection";
 import { Connection, PublicKey } from "@solana/web3.js";
 import { z } from "zod";
 
-import { cache } from "../cache";
-
-const ONE_MINUTE_IN_SECONDS = 60;
-const ONE_HOUR_IN_SECONDS = 60 * ONE_MINUTE_IN_SECONDS;
-
 export enum Cluster {
   Pythnet,
   PythtestConformance,
@@ -55,67 +50,55 @@ const clients = {
   [Cluster.PythtestConformance]: mkClient(Cluster.PythtestConformance),
 } as const;
 
-export const getPublishersForFeed = cache(
-  async (cluster: Cluster, symbol: string) => {
-    const data = await clients[cluster].getData();
-    return data.productPrice
-      .get(symbol)
-      ?.priceComponents.map(({ publisher }) => publisher.toBase58());
-  },
-  ["publishers-for-feed"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+export const getPublishersForFeed = async (
+  cluster: Cluster,
+  symbol: string,
+) => {
+  const data = await clients[cluster].getData();
+  return data.productPrice
+    .get(symbol)
+    ?.priceComponents.map(({ publisher }) => publisher.toBase58());
+};
 
-export const getFeeds = cache(
-  async (cluster: Cluster) => {
-    const data = await clients[cluster].getData();
-    return priceFeedsSchema.parse(
-      data.symbols
-        .filter(
-          (symbol) =>
-            data.productFromSymbol.get(symbol)?.display_symbol !== undefined,
-        )
-        .map((symbol) => ({
-          symbol,
-          product: data.productFromSymbol.get(symbol),
-          price: data.productPrice.get(symbol),
-        })),
-    );
-  },
-  ["pyth-data"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+export const getFeeds = async (cluster: Cluster) => {
+  const data = await clients[cluster].getData();
+  return priceFeedsSchema.parse(
+    data.symbols
+      .filter(
+        (symbol) =>
+          data.productFromSymbol.get(symbol)?.display_symbol !== undefined,
+      )
+      .map((symbol) => ({
+        symbol,
+        product: data.productFromSymbol.get(symbol),
+        price: data.productPrice.get(symbol),
+      })),
+  );
+};
 
-export const getFeedsForPublisher = cache(
-  async (cluster: Cluster, publisher: string) => {
-    const data = await clients[cluster].getData();
-    return priceFeedsSchema.parse(
-      data.symbols
-        .filter(
-          (symbol) =>
-            data.productFromSymbol.get(symbol)?.display_symbol !== undefined,
-        )
-        .map((symbol) => ({
-          symbol,
-          product: data.productFromSymbol.get(symbol),
-          price: data.productPrice.get(symbol),
-        }))
-        .filter(({ price }) =>
-          price?.priceComponents.some(
-            (component) => component.publisher.toBase58() === publisher,
-          ),
+export const getFeedsForPublisher = async (
+  cluster: Cluster,
+  publisher: string,
+) => {
+  const data = await clients[cluster].getData();
+  return priceFeedsSchema.parse(
+    data.symbols
+      .filter(
+        (symbol) =>
+          data.productFromSymbol.get(symbol)?.display_symbol !== undefined,
+      )
+      .map((symbol) => ({
+        symbol,
+        product: data.productFromSymbol.get(symbol),
+        price: data.productPrice.get(symbol),
+      }))
+      .filter(({ price }) =>
+        price?.priceComponents.some(
+          (component) => component.publisher.toBase58() === publisher,
         ),
-    );
-  },
-  ["pyth-data"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+      ),
+  );
+};
 
 const priceFeedsSchema = z.array(
   z.object({

apps/insights/src/services/staking.ts

@@ -7,69 +7,41 @@ import {
 } from "@pythnetwork/staking-sdk";
 import { Connection } from "@solana/web3.js";
 
-import { cache } from "../cache";
 import { SOLANA_RPC } from "../config/server";
 
-const ONE_MINUTE_IN_SECONDS = 60;
-const ONE_HOUR_IN_SECONDS = 60 * ONE_MINUTE_IN_SECONDS;
-
 const connection = new Connection(SOLANA_RPC);
 const client = new PythStakingClient({ connection });
 
-export const getPublisherPoolData = cache(
-  async () => {
-    const poolData = await client.getPoolDataAccount();
-    const publisherData = extractPublisherData(poolData);
-    return publisherData.map(
-      ({ totalDelegation, totalDelegationDelta, pubkey, apyHistory }) => ({
-        totalDelegation,
-        totalDelegationDelta,
-        pubkey: pubkey.toBase58(),
-        apyHistory: apyHistory.map(({ epoch, apy }) => ({
-          date: epochToDate(epoch + 1n),
-          apy,
-        })),
-      }),
-    );
-  },
-  ["publisher-pool-data"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
-
-export const getDelState = cache(
-  async () => {
-    const poolData = await client.getPoolDataAccount();
-    return {
-      delState: poolData.delState,
-      selfDelState: poolData.selfDelState,
-    };
-  },
-  ["ois-del-state"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
-
-export const getClaimableRewards = cache(
-  async () => {
-    const poolData = await client.getPoolDataAccount();
-    return poolData.claimableRewards;
-  },
-  ["ois-claimable-rewards"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
-
-export const getDistributedRewards = cache(
-  async () => {
-    const rewardCustodyAccount = await client.getRewardCustodyAccount();
-    return rewardCustodyAccount.amount;
-  },
-  ["distributed-rewards"],
-  {
-    revalidate: ONE_HOUR_IN_SECONDS,
-  },
-);
+export const getPublisherPoolData = async () => {
+  const poolData = await client.getPoolDataAccount();
+  const publisherData = extractPublisherData(poolData);
+  return publisherData.map(
+    ({ totalDelegation, totalDelegationDelta, pubkey, apyHistory }) => ({
+      totalDelegation,
+      totalDelegationDelta,
+      pubkey: pubkey.toBase58(),
+      apyHistory: apyHistory.map(({ epoch, apy }) => ({
+        date: epochToDate(epoch + 1n),
+        apy,
+      })),
+    }),
+  );
+};
+
+export const getDelState = async () => {
+  const poolData = await client.getPoolDataAccount();
+  return {
+    delState: poolData.delState,
+    selfDelState: poolData.selfDelState,
+  };
+};
+
+export const getClaimableRewards = async () => {
+  const poolData = await client.getPoolDataAccount();
+  return poolData.claimableRewards;
+};
+
+export const getDistributedRewards = async () => {
+  const rewardCustodyAccount = await client.getRewardCustodyAccount();
+  return rewardCustodyAccount.amount;
+};