Large fixed storage arrays should be sparse on Solana

In Solidity, storage can be huge. For example:

	contract foo {
		int[1e13] bar;
	}

The contract storage mechanism on Solana does not allow for such a large
amount of memory; Solana uses a simple contiguous chunk of memory. So,
to allow for these types of arrays, we must use sparse arrays.

Signed-off-by: Sean Young <sean@mess.org>

src/codegen/expression.rs

@@ -1573,33 +1573,47 @@ fn array_subscript(
         let slot_ty = ns.storage_type();
 
         if ns.target == Target::Solana {
-            let index = cast(
-                &index_loc,
-                Expression::Variable(index_loc, coerced_ty, pos),
-                &slot_ty,
-                false,
-                ns,
-                &mut Vec::new(),
-            )
-            .unwrap();
+            if ty.array_length().is_some() && ty.is_sparse_solana(ns) {
+                let index = cast(
+                    &index_loc,
+                    Expression::Variable(index_loc, coerced_ty, pos),
+                    &Type::Uint(256),
+                    false,
+                    ns,
+                    &mut Vec::new(),
+                )
+                .unwrap();
+
+                Expression::Subscript(*loc, array_ty.clone(), Box::new(array), Box::new(index))
+            } else {
+                let index = cast(
+                    &index_loc,
+                    Expression::Variable(index_loc, coerced_ty, pos),
+                    &slot_ty,
+                    false,
+                    ns,
+                    &mut Vec::new(),
+                )
+                .unwrap();
 
-            if ty.array_length().is_some() {
-                // fixed length array
-                let elem_size = elem_ty.deref_any().size_of(ns);
+                if ty.array_length().is_some() {
+                    // fixed length array
+                    let elem_size = elem_ty.deref_any().size_of(ns);
 
-                Expression::Add(
-                    *loc,
-                    elem_ty,
-                    Box::new(array),
-                    Box::new(Expression::Multiply(
+                    Expression::Add(
                         *loc,
-                        slot_ty.clone(),
-                        Box::new(index),
-                        Box::new(Expression::NumberLiteral(*loc, slot_ty, elem_size)),
-                    )),
-                )
-            } else {
-                Expression::Subscript(*loc, array_ty.clone(), Box::new(array), Box::new(index))
+                        elem_ty,
+                        Box::new(array),
+                        Box::new(Expression::Multiply(
+                            *loc,
+                            slot_ty.clone(),
+                            Box::new(index),
+                            Box::new(Expression::NumberLiteral(*loc, slot_ty, elem_size)),
+                        )),
+                    )
+                } else {
+                    Expression::Subscript(*loc, array_ty.clone(), Box::new(array), Box::new(index))
+                }
             }
         } else {
             let elem_size = elem_ty.storage_slots(ns);

src/emit/solana.rs

@@ -538,7 +538,7 @@ impl SolanaTarget {
     }
 
     /// Generate sparse lookup
-    fn sparse_lookup<'b>(
+    fn sparse_lookup_function<'b>(
         &self,
         contract: &Contract<'b>,
         key_ty: &ast::Type,
@@ -838,6 +838,68 @@ impl SolanaTarget {
 
         function
     }
+
+    /// Do a lookup/subscript in a sparse array or mapping; this will call a function
+    fn sparse_lookup<'b>(
+        &self,
+        contract: &Contract<'b>,
+        function: FunctionValue<'b>,
+        key_ty: &ast::Type,
+        value_ty: &ast::Type,
+        slot: IntValue<'b>,
+        index: IntValue<'b>,
+    ) -> IntValue<'b> {
+        let offset = contract.build_alloca(function, contract.context.i32_type(), "offset");
+
+        let current_block = contract.builder.get_insert_block().unwrap();
+
+        let lookup = self.sparse_lookup_function(contract, key_ty, value_ty);
+
+        contract.builder.position_at_end(current_block);
+
+        let rc = contract
+            .builder
+            .build_call(
+                lookup,
+                &[
+                    slot.into(),
+                    index.into(),
+                    offset.into(),
+                    contract.accounts.unwrap().into(),
+                ],
+                "mapping_lookup_res",
+            )
+            .try_as_basic_value()
+            .left()
+            .unwrap()
+            .into_int_value();
+
+        // either load the result from offset or return failure
+        let is_rc_zero = contract.builder.build_int_compare(
+            IntPredicate::EQ,
+            rc,
+            rc.get_type().const_zero(),
+            "is_rc_zero",
+        );
+
+        let rc_not_zero = contract.context.append_basic_block(function, "rc_not_zero");
+        let rc_zero = contract.context.append_basic_block(function, "rc_zero");
+
+        contract
+            .builder
+            .build_conditional_branch(is_rc_zero, rc_zero, rc_not_zero);
+
+        contract.builder.position_at_end(rc_not_zero);
+
+        self.return_code(contract, rc);
+
+        contract.builder.position_at_end(rc_zero);
+
+        contract
+            .builder
+            .build_load(offset, "offset")
+            .into_int_value()
+    }
 }
 
 impl<'a> TargetRuntime<'a> for SolanaTarget {
@@ -1116,56 +1178,14 @@ impl<'a> TargetRuntime<'a> for SolanaTarget {
         };
 
         if let ast::Type::Mapping(key, value) = ty.deref_any() {
-            let offset = contract.build_alloca(function, contract.context.i32_type(), "offset");
-
-            let current_block = contract.builder.get_insert_block().unwrap();
-
-            let lookup = self.sparse_lookup(contract, key, value);
-
-            contract.builder.position_at_end(current_block);
-
-            let rc = contract
-                .builder
-                .build_call(
-                    lookup,
-                    &[
-                        slot.into(),
-                        index.into(),
-                        offset.into(),
-                        contract.accounts.unwrap().into(),
-                    ],
-                    "mapping_lookup_res",
-                )
-                .try_as_basic_value()
-                .left()
-                .unwrap()
-                .into_int_value();
-
-            // either load the result from offset or return failure
-            let is_rc_zero = contract.builder.build_int_compare(
-                IntPredicate::EQ,
-                rc,
-                rc.get_type().const_zero(),
-                "is_rc_zero",
-            );
-
-            let rc_not_zero = contract.context.append_basic_block(function, "rc_not_zero");
-            let rc_zero = contract.context.append_basic_block(function, "rc_zero");
-
-            contract
-                .builder
-                .build_conditional_branch(is_rc_zero, rc_zero, rc_not_zero);
-
-            contract.builder.position_at_end(rc_not_zero);
-
-            self.return_code(contract, rc);
+            self.sparse_lookup(contract, function, key, value, slot, index)
+        } else if ty.is_sparse_solana(contract.ns) {
+            // sparse array
+            let elem_ty = ty.storage_array_elem().deref_into();
 
-            contract.builder.position_at_end(rc_zero);
+            let key = ast::Type::Uint(256);
 
-            contract
-                .builder
-                .build_load(offset, "offset")
-                .into_int_value()
+            self.sparse_lookup(contract, function, &key, &elem_ty, slot, index)
         } else {
             // 3rd member of account is data pointer
             let data = unsafe {

src/sema/contracts.rs

@@ -5,12 +5,12 @@ use num_traits::Zero;
 use std::collections::HashMap;
 use std::collections::HashSet;
 
-use super::ast;
 use super::expression::match_constructor_to_args;
 use super::functions;
 use super::statements;
 use super::symtable::Symtable;
 use super::variables;
+use super::{ast, SOLANA_FIRST_OFFSET};
 use crate::{emit, Target};
 
 impl ast::Contract {
@@ -296,7 +296,7 @@ fn layout_contract(contract_no: usize, ns: &mut ast::Namespace) {
     let mut override_needed: HashMap<String, Vec<(usize, usize)>> = HashMap::new();
 
     let mut slot = if ns.target == Target::Solana {
-        BigInt::from(8)
+        BigInt::from(SOLANA_FIRST_OFFSET)
     } else {
         BigInt::zero()
     };

src/sema/mod.rs

@@ -35,6 +35,8 @@ pub type ArrayDimension = Option<(pt::Loc, BigInt)>;
 
 // small prime number
 pub const SOLANA_BUCKET_SIZE: u64 = 251;
+pub const SOLANA_FIRST_OFFSET: u64 = 8;
+pub const SOLANA_SPARSE_ARRAY_SIZE: u64 = 1024;
 
 /// Load a file file from the cache, parse and resolve it. The file must be present in
 /// the cache. This function is recursive for imports.

src/sema/types.rs

@@ -1,9 +1,13 @@
-use super::ast::{
-    Contract, Diagnostic, EnumDecl, EventDecl, Namespace, Parameter, StructDecl, Symbol, Tag, Type,
-};
 use super::diagnostics::any_errors;
 use super::tags::resolve_tags;
 use super::SOLANA_BUCKET_SIZE;
+use super::{
+    ast::{
+        Contract, Diagnostic, EnumDecl, EventDecl, Namespace, Parameter, StructDecl, Symbol, Tag,
+        Type,
+    },
+    SOLANA_SPARSE_ARRAY_SIZE,
+};
 use crate::parser::pt;
 use crate::Target;
 use num_bigint::BigInt;
@@ -950,7 +954,7 @@ impl Type {
     /// be very large
     pub fn storage_slots(&self, ns: &Namespace) -> BigInt {
         if ns.target == Target::Solana {
-            if let Type::Mapping(_, _) = self {
+            if self.is_sparse_solana(ns) {
                 BigInt::from(SOLANA_BUCKET_SIZE) * ns.storage_type().storage_slots(ns)
             } else {
                 self.size_of(ns)
@@ -1147,4 +1151,19 @@ impl Type {
             _ => unreachable!(),
         }
     }
+
+    /// Is this type sparse on Solana
+    pub fn is_sparse_solana(&self, ns: &Namespace) -> bool {
+        match self.deref_any() {
+            Type::Mapping(_, _) => true,
+            Type::Array(ty, dims) => {
+                if let Some(len) = &dims[0] {
+                    ty.storage_slots(ns) * len >= BigInt::from(SOLANA_SPARSE_ARRAY_SIZE)
+                } else {
+                    false
+                }
+            }
+            _ => false,
+        }
+    }
 }

tests/solana_tests/mappings.rs

@@ -126,3 +126,126 @@ fn less_simple_mapping() {
         ])]
     );
 }
+
+#[test]
+fn sparse_array() {
+    let mut vm = build_solidity(
+        r#"
+        struct S {
+            string f1;
+            int64[] f2;
+        }
+
+        contract foo {
+            S[1e9] map;
+
+            function set_string(uint index, string s) public {
+                map[index].f1 = s;
+            }
+
+            function add_int(uint index, int64 n) public {
+                map[index].f2.push(n);
+            }
+
+            function get(uint index) public returns (S) {
+                return map[index];
+            }
+
+            function rm(uint index) public {
+                delete map[index];
+            }
+        }"#,
+    );
+
+    vm.constructor(&[]);
+
+    vm.function(
+        "set_string",
+        &[
+            Token::Uint(ethereum_types::U256::from(909090909)),
+            Token::String(String::from("This is a string which should be a little longer than 32 bytes so we the the abi encoder")),
+        ],
+    );
+
+    vm.function(
+        "add_int",
+        &[
+            Token::Uint(ethereum_types::U256::from(909090909)),
+            Token::Int(ethereum_types::U256::from(102)),
+        ],
+    );
+
+    let returns = vm.function("get", &[Token::Uint(ethereum_types::U256::from(909090909))]);
+
+    assert_eq!(
+        returns,
+        vec![Token::Tuple(vec![
+            Token::String(String::from("This is a string which should be a little longer than 32 bytes so we the the abi encoder")),
+            Token::Array(vec![Token::Int(ethereum_types::U256::from(102))]),
+        ])]
+    );
+}
+
+#[test]
+fn massive_sparse_array() {
+    let mut vm = build_solidity(
+        r#"
+        struct S {
+            string f1;
+            int64[] f2;
+        }
+
+        contract foo {
+            S[1e24] map;
+
+            function set_string(uint index, string s) public {
+                map[index].f1 = s;
+            }
+
+            function add_int(uint index, int64 n) public {
+                map[index].f2.push(n);
+            }
+
+            function get(uint index) public returns (S) {
+                return map[index];
+            }
+
+            function rm(uint index) public {
+                delete map[index];
+            }
+        }"#,
+    );
+
+    vm.constructor(&[]);
+
+    vm.function(
+        "set_string",
+        &[
+            Token::Uint(ethereum_types::U256::from(786868768768678687686877u128)),
+            Token::String(String::from("This is a string which should be a little longer than 32 bytes so we the the abi encoder")),
+        ],
+    );
+
+    vm.function(
+        "add_int",
+        &[
+            Token::Uint(ethereum_types::U256::from(786868768768678687686877u128)),
+            Token::Int(ethereum_types::U256::from(102)),
+        ],
+    );
+
+    let returns = vm.function(
+        "get",
+        &[Token::Uint(ethereum_types::U256::from(
+            786868768768678687686877u128,
+        ))],
+    );
+
+    assert_eq!(
+        returns,
+        vec![Token::Tuple(vec![
+            Token::String(String::from("This is a string which should be a little longer than 32 bytes so we the the abi encoder")),
+            Token::Array(vec![Token::Int(ethereum_types::U256::from(102))]),
+        ])]
+    );
+}