Idl: Limit account size to 60kb, allow closing idl accounts (#2329)

CHANGELOG.md

@@ -13,6 +13,8 @@ The minor version will be incremented upon a breaking change and the patch versi
 ### Features
 
 - cli: Add `env` option to verifiable builds ([#2325](https://github.com/coral-xyz/anchor/pull/2325)).
+- cli: Add `idl close` command to close a program's IDL account ([#2329](https://github.com/coral-xyz/anchor/pull/2329)).
+- cli: `idl init` now supports very large IDL files ([#2329](https://github.com/coral-xyz/anchor/pull/2329)).
 - spl: Add `transfer_checked` function ([#2353](https://github.com/coral-xyz/anchor/pull/2353)).
 
 ### Fixes

cli/src/lib.rs

@@ -308,6 +308,9 @@ pub enum IdlCommand {
         #[clap(short, long)]
         filepath: String,
     },
+    Close {
+        program_id: Pubkey,
+    },
     /// Writes an IDL into a buffer account. This can be used with SetBuffer
     /// to perform an upgrade.
     WriteBuffer {
@@ -1565,7 +1568,9 @@ fn fetch_idl(cfg_override: &ConfigOverride, idl_addr: Pubkey) -> Result<Idl> {
     let mut d: &[u8] = &account.data[8..];
     let idl_account: IdlAccount = AnchorDeserialize::deserialize(&mut d)?;
 
-    let mut z = ZlibDecoder::new(&idl_account.data[..]);
+    let compressed_len: usize = idl_account.data_len.try_into().unwrap();
+    let compressed_bytes = &account.data[44..44 + compressed_len];
+    let mut z = ZlibDecoder::new(compressed_bytes);
     let mut s = Vec::new();
     z.read_to_end(&mut s)?;
     serde_json::from_slice(&s[..]).map_err(Into::into)
@@ -1596,6 +1601,7 @@ fn idl(cfg_override: &ConfigOverride, subcmd: IdlCommand) -> Result<()> {
             program_id,
             filepath,
         } => idl_init(cfg_override, program_id, filepath),
+        IdlCommand::Close { program_id } => idl_close(cfg_override, program_id),
         IdlCommand::WriteBuffer {
             program_id,
             filepath,
@@ -1638,6 +1644,17 @@ fn idl_init(cfg_override: &ConfigOverride, program_id: Pubkey, idl_filepath: Str
     })
 }
 
+fn idl_close(cfg_override: &ConfigOverride, program_id: Pubkey) -> Result<()> {
+    with_workspace(cfg_override, |cfg| {
+        let idl_address = IdlAccount::address(&program_id);
+        idl_close_account(cfg, &program_id, idl_address)?;
+
+        println!("Idl account closed: {:?}", idl_address);
+
+        Ok(())
+    })
+}
+
 fn idl_write_buffer(
     cfg_override: &ConfigOverride,
     program_id: Pubkey,
@@ -1811,6 +1828,44 @@ fn idl_erase_authority(cfg_override: &ConfigOverride, program_id: Pubkey) -> Res
     Ok(())
 }
 
+fn idl_close_account(cfg: &Config, program_id: &Pubkey, idl_address: Pubkey) -> Result<()> {
+    let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string())
+        .map_err(|_| anyhow!("Unable to read keypair file"))?;
+    let url = cluster_url(cfg, &cfg.test_validator);
+    let client = RpcClient::new(url);
+
+    // Instruction accounts.
+    let accounts = vec![
+        AccountMeta::new(idl_address, false),
+        AccountMeta::new_readonly(keypair.pubkey(), true),
+        AccountMeta::new(keypair.pubkey(), true),
+    ];
+    // Instruction.
+    let ix = Instruction {
+        program_id: *program_id,
+        accounts,
+        data: { serialize_idl_ix(anchor_lang::idl::IdlInstruction::Close {})? },
+    };
+    // Send transaction.
+    let latest_hash = client.get_latest_blockhash()?;
+    let tx = Transaction::new_signed_with_payer(
+        &[ix],
+        Some(&keypair.pubkey()),
+        &[&keypair],
+        latest_hash,
+    );
+    client.send_and_confirm_transaction_with_spinner_and_config(
+        &tx,
+        CommitmentConfig::confirmed(),
+        RpcSendTransactionConfig {
+            skip_preflight: true,
+            ..RpcSendTransactionConfig::default()
+        },
+    )?;
+
+    Ok(())
+}
+
 // Write the idl to the account buffer, chopping up the IDL into pieces
 // and sending multiple transactions in the event the IDL doesn't fit into
 // a single transaction.
@@ -2834,9 +2889,22 @@ fn create_idl_account(
 
     // Run `Create instruction.
     {
-        let data = serialize_idl_ix(anchor_lang::idl::IdlInstruction::Create {
-            data_len: (idl_data.len() as u64) * 2, // Double for future growth.
-        })?;
+        let pda_max_growth = 60_000;
+        let idl_header_size = 44;
+        let idl_data_len = idl_data.len() as u64;
+        // We're only going to support up to 6 instructions in one transaction
+        // because will anyone really have a >60kb IDL?
+        if idl_data_len > pda_max_growth {
+            return Err(anyhow!(
+                "Your IDL is over 60kb and this isn't supported right now"
+            ));
+        }
+        // Double for future growth.
+        let data_len = (idl_data_len * 2).min(pda_max_growth - idl_header_size);
+
+        let num_additional_instructions = data_len / 10000;
+        let mut instructions = Vec::new();
+        let data = serialize_idl_ix(anchor_lang::idl::IdlInstruction::Create { data_len })?;
         let program_signer = Pubkey::find_program_address(&[], program_id).0;
         let accounts = vec![
             AccountMeta::new_readonly(keypair.pubkey(), true),
@@ -2846,14 +2914,27 @@ fn create_idl_account(
             AccountMeta::new_readonly(*program_id, false),
             AccountMeta::new_readonly(solana_program::sysvar::rent::ID, false),
         ];
-        let ix = Instruction {
+        instructions.push(Instruction {
             program_id: *program_id,
             accounts,
             data,
-        };
+        });
+
+        for _ in 0..num_additional_instructions {
+            let data = serialize_idl_ix(anchor_lang::idl::IdlInstruction::Resize { data_len })?;
+            instructions.push(Instruction {
+                program_id: *program_id,
+                accounts: vec![
+                    AccountMeta::new(idl_address, false),
+                    AccountMeta::new_readonly(keypair.pubkey(), true),
+                    AccountMeta::new_readonly(solana_program::system_program::ID, false),
+                ],
+                data,
+            });
+        }
         let latest_hash = client.get_latest_blockhash()?;
         let tx = Transaction::new_signed_with_payer(
-            &[ix],
+            &instructions,
             Some(&keypair.pubkey()),
             &[&keypair],
             latest_hash,

lang/src/error.rs

@@ -40,6 +40,9 @@ pub enum ErrorCode {
     /// 1001 - Invalid program given to the IDL instruction
     #[msg("Invalid program given to the IDL instruction")]
     IdlInstructionInvalidProgram,
+    /// 1002 - IDL Account must be empty in order to resize
+    #[msg("IDL account must be empty in order to resize, try closing first")]
+    IdlAccountNotEmpty,
 
     // Constraints
     /// 2000 - A mut constraint was violated

lang/src/idl.rs

@@ -45,6 +45,9 @@ pub enum IdlInstruction {
     SetBuffer,
     // Sets a new authority on the IdlAccount.
     SetAuthority { new_authority: Pubkey },
+    Close,
+    // Increases account size for accounts that need over 10kb.
+    Resize { data_len: u64 },
 }
 
 // Accounts for the Create instruction.
@@ -60,6 +63,17 @@ pub struct IdlAccounts<'info> {
     pub authority: Signer<'info>,
 }
 
+// Accounts for resize account instruction
+#[derive(Accounts)]
+pub struct IdlResizeAccount<'info> {
+    #[account(mut, has_one = authority)]
+    #[allow(deprecated)]
+    pub idl: ProgramAccount<'info, IdlAccount>,
+    #[account(mut, constraint = authority.key != &ERASED_AUTHORITY)]
+    pub authority: Signer<'info>,
+    pub system_program: Program<'info, System>,
+}
+
 // Accounts for creating an idl buffer.
 #[derive(Accounts)]
 pub struct IdlCreateBuffer<'info> {
@@ -85,6 +99,18 @@ pub struct IdlSetBuffer<'info> {
     pub authority: Signer<'info>,
 }
 
+// Accounts for closing the canonical Idl buffer.
+#[derive(Accounts)]
+pub struct IdlCloseAccount<'info> {
+    #[account(mut, has_one = authority, close = sol_destination)]
+    #[allow(deprecated)]
+    pub account: ProgramAccount<'info, IdlAccount>,
+    #[account(constraint = authority.key != &ERASED_AUTHORITY)]
+    pub authority: Signer<'info>,
+    #[account(mut)]
+    pub sol_destination: AccountInfo<'info>,
+}
+
 // The account holding a program's IDL. This is stored on chain so that clients
 // can fetch it and generate a client with nothing but a program's ID.
 //
@@ -95,8 +121,9 @@ pub struct IdlSetBuffer<'info> {
 pub struct IdlAccount {
     // Address that can modify the IDL.
     pub authority: Pubkey,
-    // Compressed idl bytes.
-    pub data: Vec<u8>,
+    // Length of compressed idl bytes.
+    pub data_len: u32,
+    // Followed by compressed idl bytes.
 }
 
 impl IdlAccount {
@@ -109,3 +136,22 @@ impl IdlAccount {
         "anchor:idl"
     }
 }
+
+use std::cell::{Ref, RefMut};
+
+pub trait IdlTrailingData<'info> {
+    fn trailing_data(self) -> Ref<'info, [u8]>;
+    fn trailing_data_mut(self) -> RefMut<'info, [u8]>;
+}
+
+#[allow(deprecated)]
+impl<'a, 'info: 'a> IdlTrailingData<'a> for &'a ProgramAccount<'info, IdlAccount> {
+    fn trailing_data(self) -> Ref<'a, [u8]> {
+        let info = self.as_ref();
+        Ref::map(info.try_borrow_data().unwrap(), |d| &d[44..])
+    }
+    fn trailing_data_mut(self) -> RefMut<'a, [u8]> {
+        let info = self.as_ref();
+        RefMut::map(info.try_borrow_mut_data().unwrap(), |d| &mut d[44..])
+    }
+}

lang/syn/src/codegen/program/handlers.rs

@@ -32,6 +32,22 @@ pub fn generate(program: &Program) -> proc_macro2::TokenStream {
                         __idl_create_account(program_id, &mut accounts, data_len)?;
                         accounts.exit(program_id)?;
                     },
+                    anchor_lang::idl::IdlInstruction::Resize { data_len } => {
+                        let mut bumps = std::collections::BTreeMap::new();
+                        let mut reallocs = std::collections::BTreeSet::new();
+                        let mut accounts =
+                            anchor_lang::idl::IdlResizeAccount::try_accounts(program_id, &mut accounts, &[], &mut bumps, &mut reallocs)?;
+                        __idl_resize_account(program_id, &mut accounts, data_len)?;
+                        accounts.exit(program_id)?;
+                    },
+                    anchor_lang::idl::IdlInstruction::Close => {
+                        let mut bumps = std::collections::BTreeMap::new();
+                        let mut reallocs = std::collections::BTreeSet::new();
+                        let mut accounts =
+                            anchor_lang::idl::IdlCloseAccount::try_accounts(program_id, &mut accounts, &[], &mut bumps, &mut reallocs)?;
+                        __idl_close_account(program_id, &mut accounts)?;
+                        accounts.exit(program_id)?;
+                    },
                     anchor_lang::idl::IdlInstruction::CreateBuffer => {
                         let mut bumps = std::collections::BTreeMap::new();
                         let mut reallocs = std::collections::BTreeSet::new();
@@ -95,7 +111,7 @@ pub fn generate(program: &Program) -> proc_macro2::TokenStream {
                 let owner = accounts.program.key;
                 let to = Pubkey::create_with_seed(&base, seed, owner).unwrap();
                 // Space: account discriminator || authority pubkey || vec len || vec data
-                let space = 8 + 32 + 4 + data_len as usize;
+                let space = std::cmp::min(8 + 32 + 4 + data_len as usize, 10_000);
                 let rent = Rent::get()?;
                 let lamports = rent.minimum_balance(space);
                 let seeds = &[&[nonce][..]];
@@ -140,6 +156,64 @@ pub fn generate(program: &Program) -> proc_macro2::TokenStream {
                 Ok(())
             }
 
+            #[inline(never)]
+            pub fn __idl_resize_account(
+                program_id: &Pubkey,
+                accounts: &mut anchor_lang::idl::IdlResizeAccount,
+                data_len: u64,
+            ) -> anchor_lang::Result<()> {
+                #[cfg(not(feature = "no-log-ix-name"))]
+                anchor_lang::prelude::msg!("Instruction: IdlResizeAccount");
+
+                let data_len: usize = data_len as usize;
+
+                // We're not going to support increasing the size of accounts that already contain data
+                // because that would be messy and possibly dangerous
+                if accounts.idl.data_len != 0 {
+                    return Err(anchor_lang::error::ErrorCode::IdlAccountNotEmpty.into());
+                }
+
+                let new_account_space = accounts.idl.to_account_info().data_len().checked_add(std::cmp::min(
+                    data_len
+                        .checked_sub(accounts.idl.to_account_info().data_len())
+                        .expect("data_len should always be >= the current account space"),
+                    10_000,
+                ))
+                .unwrap();
+
+                if new_account_space > accounts.idl.to_account_info().data_len() {
+                    let sysvar_rent = Rent::get()?;
+                    let new_rent_minimum = sysvar_rent.minimum_balance(new_account_space);
+                    anchor_lang::system_program::transfer(
+                        anchor_lang::context::CpiContext::new(
+                            accounts.system_program.to_account_info(),
+                            anchor_lang::system_program::Transfer {
+                                from: accounts.authority.to_account_info(),
+                                to: accounts.idl.to_account_info().clone(),
+                            },
+                        ),
+                        new_rent_minimum
+                            .checked_sub(accounts.idl.to_account_info().lamports())
+                            .unwrap(),
+                    )?;
+                    accounts.idl.to_account_info().realloc(new_account_space, false)?;
+                }
+
+                Ok(())
+
+            }
+
+            #[inline(never)]
+            pub fn __idl_close_account(
+                program_id: &Pubkey,
+                accounts: &mut anchor_lang::idl::IdlCloseAccount,
+            ) -> anchor_lang::Result<()> {
+                #[cfg(not(feature = "no-log-ix-name"))]
+                anchor_lang::prelude::msg!("Instruction: IdlCloseAccount");
+
+                Ok(())
+            }
+
             #[inline(never)]
             pub fn __idl_create_buffer(
                 program_id: &Pubkey,
@@ -162,8 +236,16 @@ pub fn generate(program: &Program) -> proc_macro2::TokenStream {
                 #[cfg(not(feature = "no-log-ix-name"))]
                 anchor_lang::prelude::msg!("Instruction: IdlWrite");
 
-                let mut idl = &mut accounts.idl;
-                idl.data.extend(idl_data);
+                let prev_len: usize = ::std::convert::TryInto::<usize>::try_into(accounts.idl.data_len).unwrap();
+                let new_len: usize = prev_len + idl_data.len();
+                accounts.idl.data_len = accounts.idl.data_len.checked_add(::std::convert::TryInto::<u32>::try_into(idl_data.len()).unwrap()).unwrap();
+
+                use anchor_lang::idl::IdlTrailingData;
+                let mut idl_bytes = accounts.idl.trailing_data_mut();
+                let idl_expansion = &mut idl_bytes[prev_len..new_len];
+                require_eq!(idl_expansion.len(), idl_data.len());
+                idl_expansion.copy_from_slice(&idl_data[..]);
+
                 Ok(())
             }
 
@@ -188,7 +270,16 @@ pub fn generate(program: &Program) -> proc_macro2::TokenStream {
                 #[cfg(not(feature = "no-log-ix-name"))]
                 anchor_lang::prelude::msg!("Instruction: IdlSetBuffer");
 
-                accounts.idl.data = accounts.buffer.data.clone();
+                accounts.idl.data_len = accounts.buffer.data_len;
+
+                use anchor_lang::idl::IdlTrailingData;
+                let buffer_len = ::std::convert::TryInto::<usize>::try_into(accounts.buffer.data_len).unwrap();
+                let mut target = accounts.idl.trailing_data_mut();
+                let source = &accounts.buffer.trailing_data()[..buffer_len];
+                require_gte!(target.len(), buffer_len);
+                target[..buffer_len].copy_from_slice(source);
+                // zero the remainder of target?
+
                 Ok(())
             }
         }

tests/anchor-cli-idl/test.sh

@@ -3,6 +3,27 @@
 # Write a keypair for program deploy
 mkdir -p target/deploy
 cp keypairs/idl_commands_one-keypair.json target/deploy
+# Generate over 20kb bytes of random data (base64 encoded), surround it with quotes, and store it in a variable
+RANDOM_DATA=$(openssl rand -base64 $((10*1680)) | sed 's/.*/"&",/')
+
+# Create the JSON object with the "docs" field containing random data
+echo '{
+  "version": "0.1.0",
+  "name": "idl_commands_one",
+  "instructions": [
+    {
+      "name": "initialize",
+      "docs" : [
+        '"$RANDOM_DATA"'
+        "trailing comma begone"
+      ],
+      "accounts": [],
+      "args": []
+    }
+  ]
+}' > testLargeIdl.json
+
+
 
 echo "Building programs"
 
@@ -23,4 +44,4 @@ echo "Running tests"
 
 anchor test --skip-deploy --skip-local-validator
 
-trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT

tests/anchor-cli-idl/tests/idl.ts

@@ -4,6 +4,7 @@ import { IdlCommandsOne } from "../target/types/idl_commands_one";
 import { IdlCommandsTwo } from "../target/types/idl_commands_two";
 import { assert } from "chai";
 import { execSync } from "child_process";
+import * as fs from "fs";
 
 describe("Test CLI IDL commands", () => {
   // Configure the client to use the local cluster.
@@ -62,4 +63,25 @@ describe("Test CLI IDL commands", () => {
 
     assert.equal(authority, provider.wallet.publicKey.toString());
   });
+
+  it("Can close IDL account", async () => {
+    execSync(`anchor idl close ${programOne.programId}`, { stdio: "inherit" });
+    const idl = await anchor.Program.fetchIdl(programOne.programId, provider);
+    assert.isNull(idl);
+  });
+
+  it("Can initialize super massive IDL account", async () => {
+    execSync(
+      `anchor idl init --filepath testLargeIdl.json ${programOne.programId}`,
+      { stdio: "inherit" }
+    );
+    const idlActual = await anchor.Program.fetchIdl(
+      programOne.programId,
+      provider
+    );
+    const idlExpected = JSON.parse(
+      fs.readFileSync("testLargeIdl.json", "utf8")
+    );
+    assert.deepEqual(idlActual, idlExpected);
+  });
 });