Reject pathological regular expressions (#1222)

Fixes CVE-2025-62495.

Fixes: #1221

Author: bnoordhuis

libregexp.c

@@ -391,6 +391,13 @@ static int re_parse_out_of_memory(REParseState *s)
     return re_parse_error(s, "out of memory");
 }
 
+static int lre_check_size(REParseState *s)
+{
+    if (s->byte_code.size < 64*1024*1024)
+        return 0;
+    return re_parse_out_of_memory(s);
+}
+
 /* If allow_overflow is false, return -1 in case of
    overflow. Otherwise return INT32_MAX. */
 static int parse_digits(const uint8_t **pp, bool allow_overflow)
@@ -1176,6 +1183,8 @@ static int re_parse_term(REParseState *s, bool is_backward_dir)
     bool greedy, add_zero_advance_check, is_neg, is_backward_lookahead;
     CharRange cr_s, *cr = &cr_s;
 
+    if (lre_check_size(s))
+        return -1;
     last_atom_start = -1;
     last_capture_count = 0;
     p = s->buf_ptr;
@@ -1667,6 +1676,8 @@ static int re_parse_alternative(REParseState *s, bool is_backward_dir)
     int ret;
     size_t start, term_start, end, term_size;
 
+    if (lre_check_size(s))
+        return -1;
     start = s->byte_code.size;
     for(;;) {
         p = s->buf_ptr;
@@ -1701,7 +1712,8 @@ static int re_parse_disjunction(REParseState *s, bool is_backward_dir)
 
     if (lre_check_stack_overflow(s->opaque, 0))
         return re_parse_error(s, "stack overflow");
-
+    if (lre_check_size(s))
+        return -1;
     start = s->byte_code.size;
     if (re_parse_alternative(s, is_backward_dir))
         return -1;

tests/bug1221.js

@@ -0,0 +1,16 @@
+import {assert} from "./assert.js"
+
+let caught = 0
+for (let i = 1; i <= 32; i++) {
+    const prefix = "(:?".repeat(i)
+    const suffix = "|)+".repeat(i)
+    const between = "(?:a|)+"
+    try {
+        new RegExp(prefix + between + suffix)
+    } catch (e) {
+        assert(e instanceof SyntaxError)
+        assert(e.message, "out of memory")
+        caught++
+    }
+}
+assert(caught, 12) // subject to change