Merge pull request #54 from solana-program/febo/external-accounts

Add support for external accounts

.changeset/orange-rats-agree.md

@@ -0,0 +1,5 @@
+---
+"create-solana-program": patch
+---
+
+Add support for external accounts

template/anchor/base/program/Cargo.toml.njk

@@ -9,6 +9,7 @@ publish = false
 [package.metadata.solana]
 program-id = "{{ programAddress }}"
 program-dependencies = []
+account-dependencies = []
 
 [lib]
 crate-type = ["cdylib", "lib"]

template/base/scripts/program/dump.mjs

@@ -1,6 +1,7 @@
 #!/usr/bin/env zx
 import 'zx/globals';
 import {
+  getExternalAccountAddresses,
   getExternalProgramAddresses,
   getExternalProgramOutputDir,
 } from '../utils.mjs';
@@ -10,11 +11,17 @@ const rpc = process.env.RPC ?? 'https://api.mainnet-beta.solana.com';
 const outputDir = getExternalProgramOutputDir();
 await dump();
 
-/** Dump external programs binaries if needed. */
+/** Dump external programs binaries and accounts if needed. */
 async function dump() {
-  // Ensure we have some external programs to dump.
-  const addresses = getExternalProgramAddresses();
-  if (addresses.length === 0) return;
+  // Ensure we have some external accounts to dump.
+  const programs = getExternalProgramAddresses();
+  const accounts = getExternalAccountAddresses();
+  const external = [
+    ...programs.map((program) => [program, 'so']),
+    ...accounts.map((account) => [account, 'json']),
+  ];
+
+  if (external.length === 0) return;
   echo(`Dumping external accounts to '${outputDir}':`);
 
   // Create the output directory if needed.
@@ -22,12 +29,12 @@ async function dump() {
 
   // Copy the binaries from the chain or warn if they are different.
   await Promise.all(
-    addresses.map(async (address) => {
-      const binary = `${address}.so`;
+    external.map(async ([address, extension]) => {
+      const binary = `${address}.${extension}`;
       const hasBinary = await fs.exists(`${outputDir}/${binary}`);
 
       if (!hasBinary) {
-        await copyFromChain(address, binary);
+        await copyFromChain(address, extension);
         echo(`Wrote account data to ${outputDir}/${binary}`);
         return;
       }
@@ -47,29 +54,36 @@ async function dump() {
       }
 
       if (hasShaChecksum) {
-        await copyFromChain(address, `onchain-${binary}`);
-        const [onChainHash, localHash] = await Promise.all([
-          $`${sha} ${options} -b ${outputDir}/onchain-${binary} | cut -d ' ' -f 1`.quiet(),
-          $`${sha} ${options} -b ${outputDir}/${binary} | cut -d ' ' -f 1`.quiet(),
-        ]);
+        try {
+          await copyFromChain(address, extension, 'onchain-');
+          const [onChainHash, localHash] = await Promise.all([
+            $`${sha} ${options} -b ${outputDir}/onchain-${binary} | cut -d ' ' -f 1`.quiet(),
+            $`${sha} ${options} -b ${outputDir}/${binary} | cut -d ' ' -f 1`.quiet(),
+          ]);
+
+          if (onChainHash.toString() !== localHash.toString()) {
+            echo(
+              chalk.yellow('[ WARNING ]'),
+              `on-chain and local binaries are different for '${address}'`
+            );
+          } else {
+            echo(
+              chalk.green('[ SKIPPED ]'),
+              `on-chain and local binaries are the same for '${address}'`
+            );
+          }
 
-        if (onChainHash.toString() !== localHash.toString()) {
+          await $`rm ${outputDir}/onchain-${binary}`.quiet();
+        } catch (error) {
           echo(
             chalk.yellow('[ WARNING ]'),
-            `on-chain and local binaries are different for '${binary}'`
-          );
-        } else {
-          echo(
-            chalk.green('[ SKIPPED ]'),
-            `on-chain and local binaries are the same for '${binary}'`
+            `skipped check for '${address}' (error copying data from '${rpc}')`
           );
         }
-
-        await $`rm ${outputDir}/onchain-${binary}`.quiet();
       } else {
         echo(
           chalk.yellow('[ WARNING ]'),
-          `skipped check for '${binary}' (missing 'sha256sum' command)`
+          `skipped check for '${address}' (missing 'sha256sum' command)`
         );
       }
     })
@@ -77,10 +91,11 @@ async function dump() {
 }
 
 /** Helper function to copy external programs or accounts binaries from the chain. */
-async function copyFromChain(address, binary) {
-  switch (binary.split('.').pop()) {
-    case 'bin':
-      return $`solana account -u ${rpc} ${address} -o ${outputDir}/${binary} >/dev/null`.quiet();
+async function copyFromChain(address, extension, prefix = '') {
+  const binary = `${prefix}${address}.${extension}`;
+  switch (extension) {
+    case 'json':
+      return $`solana account -u ${rpc} ${address} -o ${outputDir}/${binary} --output json >/dev/null`.quiet();
     case 'so':
       return $`solana program dump -u ${rpc} ${address} ${outputDir}/${binary} >/dev/null`.quiet();
     default:

template/base/scripts/utils.mjs

@@ -26,6 +26,14 @@ export function getExternalProgramAddresses() {
   return Array.from(new Set(addresses));
 }
 
+export function getExternalAccountAddresses() {
+  const addresses = getProgramFolders().flatMap(
+    (folder) =>
+      getCargo(folder).package?.metadata?.solana?.['account-dependencies'] ?? []
+  );
+  return Array.from(new Set(addresses));
+}
+
 let didWarnAboutMissingPrograms = false;
 export function getProgramFolders() {
   let programs;

template/clients/base/scripts/start-validator.mjs

@@ -4,6 +4,7 @@ import fs from 'node:fs';
 import 'zx/globals';
 import {
   getCargo,
+  getExternalAccountAddresses,
   getExternalProgramAddresses,
   getExternalProgramOutputDir,
   getProgramFolders,
@@ -21,10 +22,18 @@ if (!restart && isValidatorRunning) {
 
 // Initial message.
 const verb = isValidatorRunning ? 'Restarting' : 'Starting';
+
+// Get programs and accounts.
 const programs = [...getPrograms(), ...getExternalPrograms()];
 const programPluralized = programs.length === 1 ? 'program' : 'programs';
+const accounts = [...getExternalAccounts()];
+const accountsPluralized = accounts.length === 1 ? 'account' : 'accounts';
+
 echo(
-  `${verb} local validator with ${programs.length} custom ${programPluralized}...`
+  `${verb} local validator with ${programs.length} custom ${programPluralized}` +
+    (accounts.length > 0
+      ? ` and ${accounts.length} external ${accountsPluralized}...`
+      : `...`)
 );
 
 // Kill the validator if it's already running.
@@ -41,6 +50,11 @@ programs.forEach(({ programId, deployPath }) => {
   args.push(/* Load BPF program */ '--bpf-program', programId, deployPath);
 });
 
+// Load accounts.
+accounts.forEach(({ account, deployPath }) => {
+  args.push(/* Load account */ '--account', account, deployPath);
+});
+
 // Start the validator in detached mode.
 const cliLogs = path.join(os.tmpdir(), 'validator-cli.log');
 fs.writeFileSync(cliLogs, '', () => {});
@@ -98,3 +112,11 @@ function getExternalPrograms() {
     deployPath: path.join(binaryDir, `${address}.so`),
   }));
 }
+
+function getExternalAccounts() {
+  const binaryDir = getExternalProgramOutputDir();
+  return getExternalAccountAddresses().map((address) => ({
+    account: address,
+    deployPath: path.join(binaryDir, `${address}.json`),
+  }));
+}

template/shank/base/program/Cargo.toml.njk

@@ -9,6 +9,7 @@ publish = false
 [package.metadata.solana]
 program-id = "{{ programAddress }}"
 program-dependencies = []
+account-dependencies = []
 
 [lib]
 crate-type = ["cdylib", "lib"]