Merge pull request #1761 from pguyot/w29/fix-list-subtraction

Operator --: fix memory corruption

These changes are made under both the "Apache 2.0" and the "GNU Lesser General
Public License 2.1 or later" license terms (dual license).

SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later

Author: bettio

src/libAtomVM/nifs.c

@@ -5799,14 +5799,21 @@ static term nif_erlang_lists_subtract(Context *ctx, int argc, term argv[])
         return list1;
     }
 
-    if (UNLIKELY(memory_ensure_free_with_roots(ctx, (last_filtered_idx + 1) * CONS_SIZE, last_filtered_idx + 1, cons, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) {
-        free(cons);
-        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
-    }
-
-    term result = term_nil();
+    term result;
     if (last_filtered_idx < len - 1) {
+        // GC including new tail which is at last_filtered_idx + 1 (last_filtered_idx was filtered)
+        if (UNLIKELY(memory_ensure_free_with_roots(ctx, (last_filtered_idx + 1) * CONS_SIZE, last_filtered_idx + 2, cons, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) {
+            free(cons);
+            RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+        }
         result = cons[last_filtered_idx + 1];
+    } else {
+        // if last_filtered_idx == len -1, tail is nil
+        if (UNLIKELY(memory_ensure_free_with_roots(ctx, (last_filtered_idx + 1) * CONS_SIZE, last_filtered_idx, cons, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) {
+            free(cons);
+            RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+        }
+        result = term_nil();
     }
 
     for (int i = last_filtered_idx - 1; i >= 0; i--) {

tests/libs/estdlib/test_lists_subtraction.erl

@@ -38,7 +38,10 @@ test() ->
         {[1, 2, 3, 4], [4, 3, 2, 1], []},
         {[1, 2, 3, 4], [1, 2, 3, 4], []},
         {[1], [1.0], [1]},
-        {[1.0], [1.0], []}
+        {[1.0], [1.0], []},
+        {[1, 2, 3, 4, 5, 6, 7, 8, 9], ?MODULE:sub([1, 2, 3, 4], [1, 2, 5, 6, 7, 8, 9]), [
+            1, 2, 5, 6, 7, 8, 9
+        ]}
     ],
     ok = expect_failure(fun() -> ?MODULE:sub([a, b | c], [a]) end, badarg),
     ok = expect_failure(fun() -> ?MODULE:sub([], [a, b | c]) end, badarg),