bench: Add a script to sync benchmark results for all versions (#2477)

tests/bench/Anchor.toml

@@ -3,13 +3,14 @@ cluster = "localnet"
 wallet = "~/.config/solana/id.json"
 
 [programs.localnet]
-bench = "Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS"
+bench = "Bench11111111111111111111111111111111111111"
 
 [workspace]
 members = ["programs/bench"]
 
 [scripts]
-test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json -t 1000000 tests/**/*.ts"
-update-bench = "yarn run ts-node scripts/update-bench.ts"
+test = "yarn run ts-mocha -t 1000000 -p ./tsconfig.json tests/**/*.ts"
+sync = "yarn run ts-node scripts/sync.ts"
+sync-markdown = "yarn run ts-node scripts/sync-markdown.ts"
 generate-ix = "yarn run ts-node scripts/generate-ix.ts"
 bump-version = "yarn run ts-node scripts/bump-version.ts"

tests/bench/README.md

@@ -4,21 +4,25 @@ The bench program and its tests are used to measure the performance of Anchor pr
 
 ## How
 
-Create a program -> Write tests that measure usage -> Compare the results -> Save the new result
-
-The script will check whether there is a difference between the current result and the last saved result(in `bench.json`) at the end of the tests. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly.
+We run the same tests that measure some metric for each Anchor version starting from `0.27.0`. If the difference between the results is greater than 1%, the new data will be saved in `bench.json` and Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) will be updated accordingly.
 
 ## Scripts
 
+| :memo: TL;DR                                                                                                                   |
+| :----------------------------------------------------------------------------------------------------------------------------- |
+| If you've made changes to programs or tests in this directory, run `anchor run sync`, otherwise run `anchor test --skip-lint`. |
+
 `anchor test --skip-lint`: Run all tests and update benchmark files when necessary. This is the only command that needs to be run for most use cases.
 
 ---
 
 The following scripts are useful when making changes to how benchmarking works.
 
-`anchor run update-bench`: Update Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`.
+`anchor run sync`: Sync all benchmark files by running tests for each version. If you've made changes to the bench program or its tests, you should run this command to sync the results.
+
+`anchor run sync-markdown`: Sync Markdown files in [/bench](https://github.com/coral-xyz/anchor/tree/master/bench) based on the data from `bench.json`.
 
-`anchor run generate-ix`: Generate instructions with repetitive accounts.
+`anchor run generate-ix`: Generate program instructions with repetitive accounts.
 
 ---
 

tests/bench/programs/bench/Cargo.toml

@@ -14,3 +14,8 @@ cpi = ["no-entrypoint"]
 [dependencies]
 anchor-lang = { path = "../../../../lang" }
 anchor-spl = { path = "../../../../spl" }
+
+# TODO: Remove this and store lock files for each version instead.
+# Latest solana version(1.14.17) as of 2023-05-01 comes with rustc 1.62.0-dev but MSRV for latest
+# version of this crate is 1.64.0. See https://github.com/solana-labs/solana/pull/31418
+winnow = "=0.4.1"

tests/bench/programs/bench/src/lib.rs

@@ -1,9 +1,11 @@
 //! This program is used to measure the performance of Anchor programs.
+//!
+//! If you are making a change to this program, run `anchor run sync`.
 
 use anchor_lang::prelude::*;
 use anchor_spl::token_interface::{Mint, TokenAccount, TokenInterface};
 
-declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
+declare_id!("Bench11111111111111111111111111111111111111");
 
 #[program]
 pub mod bench {

tests/bench/scripts/update-bench.ts → tests/bench/scripts/sync-markdown.ts

@@ -1,4 +1,4 @@
-/** Update Markdown files in /bench */
+/** Sync Markdown files in /bench based on the data from bench.json */
 
 import { BenchData, Markdown } from "./utils";
 
@@ -33,19 +33,27 @@ import { BenchData, Markdown } from "./utils";
         bench.compareComputeUnits(
           newComputeUnitsResult,
           oldComputeUnitsResult,
-          (ixName, newComputeUnits, oldComputeUnits) => {
-            const percentChange = (
-              (newComputeUnits / oldComputeUnits - 1) *
-              100
-            ).toFixed(2);
+          ({ ixName, newComputeUnits, oldComputeUnits }) => {
+            if (newComputeUnits === null) {
+              // Deleted instruction
+              return;
+            }
 
             let changeText;
-            if (isNaN(oldComputeUnits)) {
+            if (oldComputeUnits === null) {
+              // New instruction
               changeText = "N/A";
-            } else if (+percentChange > 0) {
-              changeText = `🔴 **+${percentChange}%**`;
             } else {
-              changeText = `🟢 **${percentChange}%**`;
+              const percentChange = (
+                (newComputeUnits / oldComputeUnits - 1) *
+                100
+              ).toFixed(2);
+
+              if (+percentChange > 0) {
+                changeText = `🔴 **+${percentChange}%**`;
+              } else {
+                changeText = `🟢 **${percentChange}%**`;
+              }
             }
 
             table.insert(ixName, newComputeUnits.toString(), changeText);

tests/bench/scripts/sync.ts

@@ -0,0 +1,66 @@
+/**
+ * Sync all saved data by re-running the tests for each version.
+ *
+ * This script should be used when the bench program or its tests has changed
+ * and all data needs to be updated.
+ */
+
+import path from "path";
+import { spawnSync } from "child_process";
+
+import { ANCHOR_VERSION_ARG, BenchData, Toml } from "./utils";
+
+(async () => {
+  const bench = await BenchData.open();
+
+  const cargoToml = await Toml.open(
+    path.join("..", "programs", "bench", "Cargo.toml")
+  );
+  const anchorToml = await Toml.open(path.join("..", "Anchor.toml"));
+
+  for (const version of bench.getVersions()) {
+    console.log(`Updating '${version}'...`);
+
+    const isUnreleased = version === "unreleased";
+
+    // Update the anchor dependency versions
+    for (const dependency of ["lang", "spl"]) {
+      cargoToml.replaceValue(`anchor-${dependency}`, () => {
+        return isUnreleased
+          ? `{ path = "../../../../${dependency}" }`
+          : `"${version}"`;
+      });
+    }
+
+    // Save Cargo.toml
+    await cargoToml.save();
+
+    // Update `anchor test` command to pass version in Anchor.toml
+    anchorToml.replaceValue(
+      "test",
+      (cmd) => {
+        return cmd.includes(ANCHOR_VERSION_ARG)
+          ? cmd.replace(
+              new RegExp(`\\s*${ANCHOR_VERSION_ARG}\\s+(.+)`),
+              (arg, ver) => (isUnreleased ? "" : arg.replace(ver, version))
+            )
+          : `${cmd} ${ANCHOR_VERSION_ARG} ${version}`;
+      },
+      { insideQuotes: true }
+    );
+
+    // Save Anchor.toml
+    await anchorToml.save();
+
+    // Run the command to update the current version's results
+    const result = spawnSync("anchor", ["test", "--skip-lint"]);
+    console.log(result.output.toString());
+
+    // Check for failure
+    if (result.status !== 0) {
+      console.error("Please fix the error and re-run this command.");
+      process.exitCode = 1;
+      return;
+    }
+  }
+})();

tests/bench/scripts/utils.ts

@@ -2,6 +2,9 @@ import * as fs from "fs/promises";
 import path from "path";
 import { spawnSync } from "child_process";
 
+/** Version that is used in bench data file */
+export type Version = "unreleased" | (`${number}.${number}.${number}` & {});
+
 /** Persistent benchmark data(mapping of `Version -> Data`) */
 type Bench = {
   [key: string]: {
@@ -21,7 +24,10 @@ export type ComputeUnits = { [key: string]: number };
 export const THRESHOLD_PERCENTAGE = 1;
 
 /** Path to the benchmark Markdown files */
-export const BENCH_DIR_PATH = "../../bench";
+export const BENCH_DIR_PATH = path.join("..", "..", "bench");
+
+/** Command line argument for Anchor version */
+export const ANCHOR_VERSION_ARG = "--anchor-version";
 
 /** Utility class to handle benchmark data related operations */
 export class BenchData {
@@ -56,43 +62,74 @@ export class BenchData {
   }
 
   /** Get the stored results based on version */
-  get(version: string) {
+  get(version: Version) {
     return this.#data[version];
   }
 
-  /** Get unreleased version results */
-  getUnreleased() {
-    return this.get("unreleased");
-  }
-
   /** Get all versions */
   getVersions() {
-    return Object.keys(this.#data);
+    return Object.keys(this.#data) as Version[];
   }
 
   /** Compare and update compute units changes */
   compareComputeUnits(
     newComputeUnitsResult: ComputeUnits,
     oldComputeUnitsResult: ComputeUnits,
-    changeCb: (
-      ixName: string,
-      newComputeUnits: number,
-      oldComputeUnits: number
-    ) => void,
+    changeCb: (args: {
+      ixName: string;
+      newComputeUnits: number | null;
+      oldComputeUnits: number | null;
+    }) => void,
     noChangeCb?: (ixName: string, computeUnits: number) => void
   ) {
     let needsUpdate = false;
 
+    const checkIxs = (
+      comparedFrom: ComputeUnits,
+      comparedTo: ComputeUnits,
+      cb: (ixName: string, computeUnits: number) => void
+    ) => {
+      for (const ixName in comparedFrom) {
+        if (comparedTo[ixName] === undefined) {
+          cb(ixName, comparedFrom[ixName]);
+        }
+      }
+    };
+
+    // New instruction
+    checkIxs(
+      newComputeUnitsResult,
+      oldComputeUnitsResult,
+      (ixName, computeUnits) => {
+        console.log(`New instruction '${ixName}'`);
+        changeCb({
+          ixName,
+          newComputeUnits: computeUnits,
+          oldComputeUnits: null,
+        });
+        needsUpdate = true;
+      }
+    );
+
+    // Deleted instruction
+    checkIxs(
+      oldComputeUnitsResult,
+      newComputeUnitsResult,
+      (ixName, computeUnits) => {
+        console.log(`Deleted instruction '${ixName}'`);
+        changeCb({
+          ixName,
+          newComputeUnits: null,
+          oldComputeUnits: computeUnits,
+        });
+        needsUpdate = true;
+      }
+    );
+
     // Compare compute units changes
     for (const ixName in newComputeUnitsResult) {
       const oldComputeUnits = oldComputeUnitsResult[ixName];
       const newComputeUnits = newComputeUnitsResult[ixName];
-      if (!oldComputeUnits) {
-        console.log(`New instruction '${ixName}'`);
-        needsUpdate = true;
-        changeCb(ixName, newComputeUnits, NaN);
-        continue;
-      }
 
       const percentage = THRESHOLD_PERCENTAGE / 100;
       const oldMaximumAllowedDelta = oldComputeUnits * percentage;
@@ -119,8 +156,12 @@ export class BenchData {
           `Compute units change '${ixName}' (${oldComputeUnits} -> ${newComputeUnits})`
         );
 
+        changeCb({
+          ixName,
+          newComputeUnits,
+          oldComputeUnits,
+        });
         needsUpdate = true;
-        changeCb(ixName, newComputeUnits, oldComputeUnits);
       } else {
         noChangeCb?.(ixName, newComputeUnits);
       }
@@ -131,14 +172,14 @@ export class BenchData {
 
   /** Bump benchmark data version to the given version */
   bumpVersion(newVersion: string) {
-    const versions = Object.keys(this.#data);
-    const unreleasedVersion = versions[versions.length - 1];
-
     if (this.#data[newVersion]) {
       console.error(`Version '${newVersion}' already exists!`);
       process.exit(1);
     }
 
+    const versions = this.getVersions();
+    const unreleasedVersion = versions[versions.length - 1];
+
     // Add the new version
     this.#data[newVersion] = this.get(unreleasedVersion);
 
@@ -296,3 +337,56 @@ class MarkdownTable {
     );
   }
 }
+
+/** Utility class to handle TOML related operations */
+export class Toml {
+  /** TOML filepath */
+  #path: string;
+
+  /** TOML text */
+  #text: string;
+
+  constructor(path: string, text: string) {
+    this.#path = path;
+    this.#text = text;
+  }
+
+  /** Open the TOML file */
+  static async open(tomlPath: string) {
+    tomlPath = path.join(__dirname, tomlPath);
+    const text = await fs.readFile(tomlPath, {
+      encoding: "utf8",
+    });
+    return new Toml(tomlPath, text);
+  }
+
+  /** Save the TOML file */
+  async save() {
+    await fs.writeFile(this.#path, this.#text);
+  }
+
+  /** Replace the value for the given key */
+  replaceValue(
+    key: string,
+    cb: (previous: string) => string,
+    opts?: { insideQuotes: boolean }
+  ) {
+    this.#text = this.#text.replace(
+      new RegExp(`${key}\\s*=\\s*${opts?.insideQuotes ? `"(.*)"` : "(.*)"}`),
+      (line, value) => line.replace(value, cb(value))
+    );
+  }
+}
+
+/**
+ * Get Anchor version from the passed arguments.
+ *
+ * Defaults to `unreleased`.
+ */
+export const getVersionFromArgs = () => {
+  const args = process.argv;
+  const anchorVersionArgIndex = args.indexOf(ANCHOR_VERSION_ARG);
+  return anchorVersionArgIndex === -1
+    ? "unreleased"
+    : (args[anchorVersionArgIndex + 1] as Version);
+};

tests/bench/tests/compute-units.ts

@@ -3,7 +3,7 @@ import * as token from "@coral-xyz/spl-token";
 import { spawnSync } from "child_process";
 
 import { Bench, IDL } from "../target/types/bench";
-import { BenchData, ComputeUnits } from "../scripts/utils";
+import { BenchData, ComputeUnits, getVersionFromArgs } from "../scripts/utils";
 
 describe(IDL.name, () => {
   // Configure the client to use the local cluster
@@ -226,12 +226,17 @@ describe(IDL.name, () => {
     const bench = await BenchData.open();
 
     // Compare and update compute units changes
-    const oldComputeUnits = bench.getUnreleased().computeUnits;
+    const version = getVersionFromArgs();
+    const oldComputeUnits = bench.get(version).computeUnits;
     const { needsUpdate } = bench.compareComputeUnits(
       computeUnits,
       oldComputeUnits,
-      (ixName, newComputeUnits) => {
-        oldComputeUnits[ixName] = newComputeUnits;
+      ({ ixName, newComputeUnits: newValue }) => {
+        if (newValue === null) {
+          delete oldComputeUnits[ixName];
+        } else {
+          oldComputeUnits[ixName] = newValue;
+        }
       }
     );
 
@@ -239,10 +244,13 @@ describe(IDL.name, () => {
       console.log("Updating benchmark files...");
 
       // Save bench data file
-      // (needs to happen before running the `update-bench` script)
+      // (needs to happen before running the `sync-markdown` script)
       await bench.save();
 
-      spawnSync("anchor", ["run", "update-bench"]);
+      // Only update markdown files on `unreleased` version
+      if (version === "unreleased") {
+        spawnSync("anchor", ["run", "sync-markdown"]);
+      }
     }
   });
 });