Enable push/pop on memory arrays on Solana

Signed-off-by: Sean Young <sean@mess.org>

CHANGELOG.md

@@ -2,6 +2,13 @@
 All notable changes to [Solang](https://github.com/hyperledger/solang/)
 will be documented here.
 
+## Unreleased
+
+### Added
+- On Solana, programs now use a custom heap implementation, just like on
+  Substrate. As result, it is now possible to `.push()` and `.pop()` on
+  dynamic arrays in memory.
+
 ## v0.1.12 Cairo
 
 ### Added

Cargo.toml

@@ -59,7 +59,7 @@ wasmi = "0.11"
 rand = "0.7"
 sha2 = "0.10"
 # solana_rbpf makes api changes in patch versions
-solana_rbpf = "=0.2.31"
+solana_rbpf = "=0.2.32"
 byteorder = "1.4"
 assert_cmd = "2.0"
 bincode = "1.3"

src/emit/ethabiencoder.rs

@@ -643,10 +643,9 @@ impl<'a, 'b> EncoderBuilder<'a, 'b> {
         self,
         binary: &Binary<'a>,
         function: FunctionValue<'a>,
-        output: PointerValue<'a>,
+        mut output: PointerValue<'a>,
         ns: &ast::Namespace,
     ) {
-        let mut output = output;
         let mut ty_iter = self.tys.iter();
 
         for arg in self.packed.iter() {
@@ -679,7 +678,6 @@ impl<'a, 'b> EncoderBuilder<'a, 'b> {
                 "",
             );
 
-            let mut output = output;
             let mut offset = self.offset;
             let mut dynamic = unsafe { binary.builder.build_gep(output, &[self.offset], "") };
 

src/emit/mod.rs

@@ -3632,6 +3632,13 @@ pub trait TargetRuntime<'a> {
                         let size = bin.builder.build_int_mul(elem_size, new_len, "");
                         let size = bin.builder.build_int_add(size, vec_size, "");
 
+                        let realloc_size = if ns.target == Target::Solana {
+                            bin.builder
+                                .build_int_z_extend(size, bin.context.i64_type(), "")
+                        } else {
+                            size
+                        };
+
                         // Reallocate and reassign the array pointer
                         let new = bin
                             .builder
@@ -3645,7 +3652,7 @@ pub trait TargetRuntime<'a> {
                                             "a",
                                         )
                                         .into(),
-                                    size.into(),
+                                    realloc_size.into(),
                                 ],
                                 "",
                             )
@@ -3814,11 +3821,19 @@ pub trait TargetRuntime<'a> {
                             bin.context.i8_type().ptr_type(AddressSpace::Generic),
                             "a",
                         );
+
+                        let realloc_size = if ns.target == Target::Solana {
+                            bin.builder
+                                .build_int_z_extend(size, bin.context.i64_type(), "")
+                        } else {
+                            size
+                        };
+
                         let new = bin
                             .builder
                             .build_call(
                                 bin.module.get_function("__realloc").unwrap(),
-                                &[a.into(), size.into()],
+                                &[a.into(), realloc_size.into()],
                                 "",
                             )
                             .try_as_basic_value()

src/sema/expression.rs

@@ -6190,14 +6190,6 @@ fn method_call_pos_args(
 
     if matches!(var_ty, Type::Array(..) | Type::DynamicBytes) {
         if func.name == "push" {
-            if ns.target == Target::Solana {
-                diagnostics.push(Diagnostic::error(
-                    func.loc,
-                    format!("'push()' not supported on 'bytes' on target {}", ns.target),
-                ));
-                return Err(());
-            }
-
             let elem_ty = var_ty.array_elem();
 
             let val = match args.len() {
@@ -6238,14 +6230,6 @@ fn method_call_pos_args(
             ));
         }
         if func.name == "pop" {
-            if ns.target == Target::Solana {
-                diagnostics.push(Diagnostic::error(
-                    func.loc,
-                    format!("'pop()' not supported on 'bytes' on target {}", ns.target),
-                ));
-                return Err(());
-            }
-
             if !args.is_empty() {
                 diagnostics.push(Diagnostic::error(
                     func.loc,

stdlib/heap.c

@@ -132,7 +132,8 @@ void *__realloc(void *m, size_t size)
     {
         // merge with next
         cur->next = next->next;
-        cur->next->prev = cur;
+        if (cur->next)
+            cur->next->prev = cur;
         cur->length += next->length + sizeof(struct chunk);
         // resplit ..
         shrink_chunk(cur, size);

tests/contract_testcases/solana/expressions/pushpop.dot

@@ -1,10 +1,20 @@
 strict digraph "tests/contract_testcases/solana/expressions/pushpop.sol" {
 	contract [label="contract foo\ntests/contract_testcases/solana/expressions/pushpop.sol:2:9-8:10"]
 	test [label="function test\ncontract: foo\ntests/contract_testcases/solana/expressions/pushpop.sol:3:13-35\nsignature test()\nvisibility public\nmutability nonpayable"]
+	var_decl [label="variable decl bytes x\ntests/contract_testcases/solana/expressions/pushpop.sol:4:17-24"]
+	expr [label="expression\ntests/contract_testcases/solana/expressions/pushpop.sol:6:17-25"]
+	builtins [label="builtin ArrayPush\ntests/contract_testcases/solana/expressions/pushpop.sol:6:17-25"]
+	variable [label="variable: x\nbytes\ntests/contract_testcases/solana/expressions/pushpop.sol:6:17-18"]
 	diagnostic [label="found contract 'foo'\nlevel Debug\ntests/contract_testcases/solana/expressions/pushpop.sol:2:9-8:10"]
-	diagnostic_5 [label="'push()' not supported on 'bytes' on target solana\nlevel Error\ntests/contract_testcases/solana/expressions/pushpop.sol:6:19-23"]
+	diagnostic_9 [label="function can be declared 'pure'\nlevel Warning\ntests/contract_testcases/solana/expressions/pushpop.sol:3:13-35"]
+	diagnostic_10 [label="local variable 'x' has been assigned, but never read\nlevel Warning\ntests/contract_testcases/solana/expressions/pushpop.sol:4:23-24"]
 	contracts -> contract
 	contract -> test [label="function"]
+	test -> var_decl [label="body"]
+	var_decl -> expr [label="next"]
+	expr -> builtins [label="expr"]
+	builtins -> variable [label="arg #0"]
 	diagnostics -> diagnostic [label="Debug"]
-	diagnostics -> diagnostic_5 [label="Error"]
+	diagnostics -> diagnostic_9 [label="Warning"]
+	diagnostics -> diagnostic_10 [label="Warning"]
 }

tests/contract_testcases/solana/expressions/pushpop_01.dot

@@ -1,10 +1,18 @@
 strict digraph "tests/contract_testcases/solana/expressions/pushpop_01.sol" {
 	contract [label="contract foo\ntests/contract_testcases/solana/expressions/pushpop_01.sol:2:9-8:10"]
 	test [label="function test\ncontract: foo\ntests/contract_testcases/solana/expressions/pushpop_01.sol:3:13-35\nsignature test()\nvisibility public\nmutability nonpayable"]
+	var_decl [label="variable decl bytes x\ntests/contract_testcases/solana/expressions/pushpop_01.sol:4:17-24"]
+	expr [label="expression\ntests/contract_testcases/solana/expressions/pushpop_01.sol:6:17-24"]
+	builtins [label="builtin ArrayPop\ntests/contract_testcases/solana/expressions/pushpop_01.sol:6:17-24"]
+	variable [label="variable: x\nbytes\ntests/contract_testcases/solana/expressions/pushpop_01.sol:6:17-18"]
 	diagnostic [label="found contract 'foo'\nlevel Debug\ntests/contract_testcases/solana/expressions/pushpop_01.sol:2:9-8:10"]
-	diagnostic_5 [label="'pop()' not supported on 'bytes' on target solana\nlevel Error\ntests/contract_testcases/solana/expressions/pushpop_01.sol:6:19-22"]
+	diagnostic_9 [label="function can be declared 'pure'\nlevel Warning\ntests/contract_testcases/solana/expressions/pushpop_01.sol:3:13-35"]
 	contracts -> contract
 	contract -> test [label="function"]
+	test -> var_decl [label="body"]
+	var_decl -> expr [label="next"]
+	expr -> builtins [label="expr"]
+	builtins -> variable [label="arg #0"]
 	diagnostics -> diagnostic [label="Debug"]
-	diagnostics -> diagnostic_5 [label="Error"]
+	diagnostics -> diagnostic_9 [label="Warning"]
 }

tests/solana_tests/arrays.rs

@@ -1024,3 +1024,337 @@ fn initialization_with_literal() {
         vec![Token::Uint(Uint::from(563)), Token::Uint(Uint::from(895))]
     );
 }
+
+#[test]
+fn dynamic_array_push() {
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            function test() public {
+                int[] bar = (new int[])(1);
+
+                bar[0] = 128;
+                bar.push(64);
+
+                assert(bar.length == 2);
+                assert(bar[1] == 64);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            function test() public {
+                bytes bar = (new bytes)(1);
+
+                bar[0] = 128;
+                bar.push(64);
+
+                assert(bar.length == 2);
+                assert(bar[1] == 64);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            struct s {
+                int32 f1;
+                bool f2;
+            }
+            function test() public {
+                s[] bar = new s[](1);
+
+                bar[0] = s({f1: 0, f2: false});
+                bar.push(s({f1: 1, f2: true}));
+
+                assert(bar.length == 2);
+                assert(bar[1].f1 == 1);
+                assert(bar[1].f2 == true);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            enum enum1 { val1, val2, val3 }
+            function test() public {
+                enum1[] bar = new enum1[](1);
+
+                bar[0] = enum1.val1;
+                bar.push(enum1.val2);
+
+                assert(bar.length == 2);
+                assert(bar[1] == enum1.val2);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    // push() returns a reference to the thing
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            struct s {
+                int32 f1;
+                bool f2;
+            }
+
+            function test() public {
+                s[] bar = new s[](0);
+                s memory n = bar.push();
+                n.f1 = 102;
+                n.f2 = true;
+
+                assert(bar[0].f1 == 102);
+                assert(bar[0].f2 == true);
+            }
+        }"#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+}
+
+#[test]
+fn dynamic_array_pop() {
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            function test() public {
+                int[] bar = new int[](1);
+
+                bar[0] = 128;
+
+                assert(bar.length == 1);
+                assert(128 == bar.pop());
+                assert(bar.length == 0);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            function test() public {
+                bytes bar = new bytes(1);
+
+                bar[0] = 128;
+
+                assert(bar.length == 1);
+                assert(128 == bar.pop());
+                assert(bar.length == 0);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            struct s {
+                int32 f1;
+                bool f2;
+            }
+            function test() public {
+                s[] bar = new s[](1);
+
+                bar[0] = s(128, true);
+
+                assert(bar.length == 1);
+
+                s baz = bar.pop();
+                assert(baz.f1 == 128);
+                assert(baz.f2 == true);
+                assert(bar.length == 0);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            enum enum1 { val1, val2, val3 }
+            function test() public {
+                enum1[] bar = new enum1[](1);
+
+                bar[0] = enum1.val2;
+
+                assert(bar.length == 1);
+                assert(enum1.val2 == bar.pop());
+                assert(bar.length == 0);
+            }
+        }
+        "#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+}
+
+#[test]
+#[should_panic]
+fn dynamic_array_pop_empty_array() {
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            function test() public {
+                int[] bar = new int[](0);
+                bar.pop();
+            }
+        }"#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+}
+
+#[test]
+#[should_panic]
+fn dynamic_array_pop_bounds() {
+    let mut runtime = build_solidity(
+        r#"
+        pragma solidity 0;
+
+        contract foo {
+            function test() public {
+                int[] bar = new int[](1);
+                bar[0] = 12;
+                bar.pop();
+
+                assert(bar[0] == 12);
+            }
+        }"#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+}
+
+#[test]
+fn dynamic_array_push_pop_loop() {
+    let mut runtime = build_solidity(
+        r#"
+        contract foo {
+            function test() public {
+                uint32[] bar1 = new uint32[](0);
+                uint32[] bar2 = new uint32[](0);
+
+                // each time we call a system call, the heap is checked
+                // for consistency. So do a print() after each operation
+                for (uint64 i = 1; i < 160; i++) {
+                    if ((i % 10) == 0) {
+                        bar1.pop();
+                        print("bar1.pop");
+                        bar2.pop();
+                        print("bar2.pop");
+                    } else {
+                        uint32 v = bar1.length;
+                        bar1.push(v);
+                        print("bar1.push");
+                        bar2.push(v);
+                        print("bar2.push");
+                    }
+                }
+
+                assert(bar1.length == bar2.length);
+
+                for (uint32 i = 0; i < bar1.length; i++) {
+                    assert(bar1[i] == i);
+                    assert(bar2[i] == i);
+                }
+            }
+        }"#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+
+    let mut runtime = build_solidity(
+        r#"
+        contract foo {
+            function test() public {
+                bytes bar1 = new bytes(0);
+                bytes bar2 = new bytes(0);
+
+                // each time we call a system call, the heap is checked
+                // for consistency. So do a print() after each operation
+                for (uint64 i = 1; i < 160; i++) {
+                    if ((i % 10) == 0) {
+                        bar1.pop();
+                        print("bar1.pop");
+                        bar2.pop();
+                        print("bar2.pop");
+                    } else {
+                        uint8 v = uint8(bar1.length);
+                        bar1.push(v);
+                        print("bar1.push");
+                        bar2.push(v);
+                        print("bar2.push");
+                    }
+                }
+
+                assert(bar1.length == bar2.length);
+
+                for (uint32 i = 0; i < bar1.length; i++) {
+                    uint8 v = uint8(i);
+                    print("{}.{}.{}".format(v, bar1[i], bar2[i]));
+                    assert(bar1[i] == v);
+                    assert(bar2[i] == v);
+                }
+            }
+        }"#,
+    );
+
+    runtime.constructor("foo", &[]);
+    runtime.function("test", &[], &[], None);
+}

tests/solana_tests/yul.rs

@@ -76,10 +76,12 @@ contract testing  {
         &[],
         None,
     );
+
     assert_eq!(
         returns,
         vec![
-            Token::Uint(U256::from(12884901920u64)),
+            // the heap is 0x300000000. The header 32 bytes (sizeof(chunk) in heap.c)
+            Token::Uint(U256::from(0x300000020u64)),
             Token::Uint(U256::from(4))
         ]
     );