perf: SIMD base64 via aklomp/base64 + ByteArr/RangeArr/asciiSafe

bench/resources/go_suite/base64_stress.jsonnet

@@ -0,0 +1,22 @@
+{
+  local largeStr = std.repeat("Lorem ipsum dolor sit amet, consectetur adipiscing elit. ", 100),
+  local encoded = std.base64(largeStr),
+  local decoded = std.base64Decode(encoded),
+  local encodedArr = std.base64(std.makeArray(1000, function(i) i % 256)),
+  local decodedBytes = std.base64DecodeBytes(encodedArr),
+
+  local encoded2 = std.base64(decoded),
+  local decoded2 = std.base64Decode(encoded2),
+  local encodedArr2 = std.base64(std.makeArray(2000, function(i) (i * 7 + 13) % 256)),
+  local decodedBytes2 = std.base64DecodeBytes(encodedArr2),
+
+  local encoded3 = std.base64(decoded2),
+  local decoded3 = std.base64Decode(encoded3),
+  local encodedArr3 = std.base64(std.makeArray(3000, function(i) (i * 13 + 37) % 256)),
+  local decodedBytes3 = std.base64DecodeBytes(encodedArr3),
+
+  roundtrip_ok: decoded3 == largeStr,
+  byte_roundtrip_ok: std.length(decodedBytes3) == 3000,
+  encoded_len: std.length(encoded3),
+  decoded_len: std.length(decoded3)
+}

bench/resources/go_suite/base64_stress.jsonnet.golden

@@ -0,0 +1,6 @@
+{
+   "byte_roundtrip_ok": true,
+   "decoded_len": 5700,
+   "encoded_len": 7600,
+   "roundtrip_ok": true
+}

build.mill

@@ -278,6 +278,48 @@ object sjsonnet extends VersionFileModule {
     def nativeLTO = LTO.Full
     def nativeMultithreading = None
 
+    // Build aklomp/base64 as a static library for SIMD-accelerated base64.
+    // We pin to a specific upstream commit (no git submodule) and clone on
+    // demand into the task's sandboxed dest. The pinned SHA below is the single
+    // source of truth — bump it here when upgrading the upstream library.
+    def aklompBase64Repo = "https://github.com/aklomp/base64.git"
+    def aklompBase64Commit = "9e8ed65048ff0f703fad3deb03bf66ac7f78a4d7" // v0.5.2-26-g9e8ed65
+    def aklompBase64Source = Task {
+      val cloneDir = Task.ctx().dest / "base64-src"
+      os.remove.all(cloneDir)
+      os.proc("git", "clone", "--filter=blob:none", aklompBase64Repo, cloneDir.toString).call()
+      os.proc("git", "checkout", "--detach", aklompBase64Commit).call(cwd = cloneDir)
+      PathRef(cloneDir)
+    }
+    def buildBase64Lib = Task {
+      val srcDir = aklompBase64Source().path
+      val buildDir = Task.ctx().dest / "base64-build"
+      os.makeDir.all(buildDir)
+      os.proc(
+        "cmake",
+        srcDir.toString,
+        "-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
+        "-DBASE64_WITH_OpenMP=OFF",
+        "-DBASE64_BUILD_TESTS=OFF",
+        "-DBASE64_BUILD_CLI=OFF",
+        "-DCMAKE_BUILD_TYPE=Release"
+      ).call(cwd = buildDir)
+      os.proc("cmake", "--build", buildDir.toString, "--config", "Release").call()
+      PathRef(buildDir)
+    }
+
+    def nativeLinkingOptions = Task {
+      super.nativeLinkingOptions() ++ Seq(
+        (buildBase64Lib().path / "libbase64.a").toString
+      )
+    }
+
+    def nativeCompileOptions = Task {
+      super.nativeCompileOptions() ++ Seq(
+        s"-I${aklompBase64Source().path / "include"}"
+      )
+    }
+
     object test extends ScalaNativeTests with CrossTests {
       def releaseMode = ReleaseMode.Debug
       def nativeMultithreading = None
@@ -286,6 +328,16 @@ object sjsonnet extends VersionFileModule {
         "SCALANATIVE_THREAD_STACK_SIZE" -> stackSize
       )
       def nativeLTO = LTO.None
+      def nativeLinkingOptions = Task {
+        super.nativeLinkingOptions() ++ Seq(
+          (SjsonnetNativeModule.this.buildBase64Lib().path / "libbase64.a").toString
+        )
+      }
+      def nativeCompileOptions = Task {
+        super.nativeCompileOptions() ++ Seq(
+          s"-I${SjsonnetNativeModule.this.aklompBase64Source().path / "include"}"
+        )
+      }
     }
   }
 

sjsonnet/src-js/sjsonnet/stdlib/PlatformBase64.scala

@@ -0,0 +1,16 @@
+package sjsonnet.stdlib
+
+/**
+ * Scala.js implementation of base64 encode/decode. Delegates to java.util.Base64 (provided by
+ * Scala.js stdlib emulation).
+ */
+object PlatformBase64 {
+
+  def encodeToString(input: Array[Byte]): String =
+    java.util.Base64.getEncoder.encodeToString(input)
+
+  def decode(input: String): Array[Byte] = {
+    Base64Validation.requireStrictPadding(input)
+    java.util.Base64.getDecoder.decode(input)
+  }
+}

sjsonnet/src-jvm/sjsonnet/stdlib/PlatformBase64.scala

@@ -0,0 +1,16 @@
+package sjsonnet.stdlib
+
+/**
+ * JVM implementation of base64 encode/decode. Delegates to java.util.Base64 which has HotSpot
+ * intrinsics for high performance.
+ */
+object PlatformBase64 {
+
+  def encodeToString(input: Array[Byte]): String =
+    java.util.Base64.getEncoder.encodeToString(input)
+
+  def decode(input: String): Array[Byte] = {
+    Base64Validation.requireStrictPadding(input)
+    java.util.Base64.getDecoder.decode(input)
+  }
+}

sjsonnet/src-native/sjsonnet/stdlib/PlatformBase64.scala

@@ -0,0 +1,128 @@
+package sjsonnet.stdlib
+
+import scala.scalanative.unsafe._
+import scala.scalanative.unsigned._
+import scala.scalanative.libc.string.memcpy
+
+/**
+ * Scala Native implementation of base64 encode/decode.
+ *
+ * Uses the aklomp/base64 C library (BSD-2-Clause) which provides SIMD-accelerated base64 via
+ * runtime CPU detection:
+ *   - x86_64: SSSE3 / SSE4.1 / SSE4.2 / AVX / AVX2 / AVX-512
+ *   - AArch64: NEON
+ *   - Fallback: optimized generic C implementation
+ *
+ * The static library is built by CMake and linked via nativeLinkingOptions.
+ *
+ * Both aklomp/base64 and C++ jsonnet (the reference implementation) use strict RFC 4648 mode:
+ * padding is required, unpadded input is rejected. This differs from java.util.Base64 on JVM which
+ * is more lenient (accepts unpadded input) — that JVM leniency is a pre-existing sjsonnet bug, not
+ * something we replicate here.
+ */
+@extern
+private[stdlib] object libbase64 {
+  def base64_encode(
+      src: Ptr[CChar],
+      srclen: CSize,
+      out: Ptr[CChar],
+      outlen: Ptr[CSize],
+      flags: CInt
+  ): Unit = extern
+
+  def base64_decode(
+      src: Ptr[CChar],
+      srclen: CSize,
+      out: Ptr[CChar],
+      outlen: Ptr[CSize],
+      flags: CInt
+  ): CInt = extern
+}
+
+object PlatformBase64 {
+
+  private val DECODE_TABLE: Array[Int] = {
+    val t = Array.fill[Int](256)(-1)
+    var i = 0
+    while (i < 26) { t('A' + i) = i; i += 1 }
+    i = 0
+    while (i < 26) { t('a' + i) = i + 26; i += 1 }
+    i = 0
+    while (i < 10) { t('0' + i) = i + 52; i += 1 }
+    t('+') = 62
+    t('/') = 63
+    t
+  }
+
+  /**
+   * Diagnose why base64 decode failed and throw a JVM-compatible error message. Only called on the
+   * error path (after aklomp/base64 returns failure), so zero overhead on the hot path.
+   *
+   * Error messages match java.util.Base64.Decoder behavior for golden test compatibility:
+   *   - Invalid character: "Illegal base64 character XX" (hex)
+   *   - Wrong length/padding: "Last unit does not have enough valid bits"
+   */
+  private def throwDecodeError(srcBytes: Array[Byte]): Nothing = {
+    val len = srcBytes.length
+
+    var i = 0
+    while (i < len) {
+      val b = srcBytes(i) & 0xff
+      if (b != '='.toInt) {
+        if (DECODE_TABLE(b) < 0) {
+          throw new IllegalArgumentException(
+            "Illegal base64 character " + Integer.toHexString(b)
+          )
+        }
+      }
+      i += 1
+    }
+
+    throw new IllegalArgumentException(
+      "Last unit does not have enough valid bits"
+    )
+  }
+
+  def encodeToString(input: Array[Byte]): String = {
+    if (input.length == 0) return ""
+    val maxOutLen = ((input.length.toLong + 2) / 3) * 4
+    if (maxOutLen > Int.MaxValue)
+      throw new IllegalArgumentException("Input too large for base64 encoding")
+    val outSize = maxOutLen.toInt
+    Zone.acquire { implicit z =>
+      val srcPtr = alloc[Byte](input.length.toUSize)
+      memcpy(srcPtr, input.at(0), input.length.toUSize)
+      val outPtr = alloc[Byte]((outSize + 1).toUSize)
+      val outLenPtr = alloc[CSize](1.toUSize)
+      libbase64.base64_encode(srcPtr, input.length.toUSize, outPtr, outLenPtr, 0)
+      val actualLen = (!outLenPtr).toInt
+      val result = new Array[Byte](actualLen)
+      memcpy(result.at(0), outPtr, actualLen.toUSize)
+      new String(result, "US-ASCII")
+    }
+  }
+
+  def decode(input: String): Array[Byte] = {
+    if (input.isEmpty) return Array.emptyByteArray
+    val srcBytes = input.getBytes("US-ASCII")
+    val maxOutLen = ((srcBytes.length.toLong / 4) * 3) + 3
+    if (maxOutLen > Int.MaxValue)
+      throw new IllegalArgumentException("Input too large for base64 decoding")
+    val outSize = maxOutLen.toInt
+    Zone.acquire { implicit z =>
+      val srcPtr = alloc[Byte](srcBytes.length.toUSize)
+      memcpy(srcPtr, srcBytes.at(0), srcBytes.length.toUSize)
+      val outPtr = alloc[Byte]((outSize + 1).toUSize)
+      val outLenPtr = alloc[CSize](1.toUSize)
+      val ret =
+        libbase64.base64_decode(srcPtr, srcBytes.length.toUSize, outPtr, outLenPtr, 0)
+      if (ret != 1) {
+        throwDecodeError(srcBytes)
+      }
+      val actualLen = (!outLenPtr).toInt
+      val result = new Array[Byte](actualLen)
+      memcpy(result.at(0), outPtr, actualLen.toUSize)
+      result
+    }
+  }
+}

sjsonnet/src/sjsonnet/BaseByteRenderer.scala

@@ -242,6 +242,27 @@ class BaseByteRenderer[T <: java.io.OutputStream](
     else visitLongString(str)
   }
 
+  /**
+   * Fast path for strings known to be ASCII-safe (no escaping needed, all chars 0x20-0x7E). Skips
+   * SWAR scanning and UTF-8 encoding — writes bytes directly from chars.
+   */
+  private[sjsonnet] def renderAsciiSafeString(str: String): Unit = {
+    val len = str.length
+    elemBuilder.ensureLength(len + 2)
+    val arr = elemBuilder.arr
+    var pos = elemBuilder.length
+    arr(pos) = '"'.toByte
+    pos += 1
+    var i = 0
+    while (i < len) {
+      arr(pos) = str.charAt(i).toByte
+      pos += 1
+      i += 1
+    }
+    arr(pos) = '"'.toByte
+    elemBuilder.length = pos + 1
+  }
+
   /**
    * Zero-allocation fast path for short ASCII strings (the vast majority of JSON keys/values). Uses
    * getChars to bulk-copy into a reusable char buffer, then scans the buffer directly (avoiding

sjsonnet/src/sjsonnet/ByteRenderer.scala

@@ -192,7 +192,9 @@ class ByteRenderer(out: OutputStream = new java.io.ByteArrayOutputStream(), inde
     val vt: Int = v.valTag.toInt
     (vt: @scala.annotation.switch) match {
       case 0 => // TAG_STR
-        renderQuotedString(v.asInstanceOf[Val.Str].str)
+        val s = v.asInstanceOf[Val.Str]
+        if (s._asciiSafe) renderAsciiSafeString(s.str)
+        else renderQuotedString(s.str)
       case 1 => // TAG_NUM
         renderDouble(v.asDouble)
       case 2 => // TAG_TRUE
@@ -420,18 +422,32 @@ class ByteRenderer(out: OutputStream = new java.io.ByteArrayOutputStream(), inde
     depth += 1
     resetEmpty()
 
-    var i = 0
-    while (i < len) {
-      val childVal = xs.value(i)
+    // Fast path for byte-backed arrays: emit numbers directly without per-element dispatch
+    xs match {
+      case ba: Val.ByteArr =>
+        val bytes = ba.rawBytes
+        var i = 0
+        while (i < len) {
+          markNonEmpty()
+          flushBuffer()
+          renderDouble((bytes(i) & 0xff).toDouble)
+          commaBuffered = true
+          i += 1
+        }
+      case _ =>
+        var i = 0
+        while (i < len) {
+          val childVal = xs.value(i)
 
-      markNonEmpty()
-      flushBuffer()
+          markNonEmpty()
+          flushBuffer()
 
-      // Render element directly — no flush overhead
-      materializeChild(childVal, matDepth, ctx)
+          // Render element directly — no flush overhead
+          materializeChild(childVal, matDepth, ctx)
 
-      commaBuffered = true
-      i += 1
+          commaBuffered = true
+          i += 1
+        }
     }
 
     // Inline of visitEnd — close bracket

sjsonnet/src/sjsonnet/Materializer.scala

@@ -232,6 +232,25 @@ abstract class Materializer {
       depth: Int,
       ctx: Materializer.MaterializeContext)(implicit evaluator: EvalScope): T = {
     storePos(xs.pos)
+    // Fast path for byte-backed arrays: skip per-element value() + type dispatch
+    xs match {
+      case ba: Val.ByteArr =>
+        val bytes = ba.rawBytes
+        val len = bytes.length
+        val av = visitor.visitArray(len, -1)
+        var i = 0
+        while (i < len) {
+          av.visitValue(
+            av.subVisitor
+              .asInstanceOf[Visitor[T, T]]
+              .visitFloat64((bytes(i) & 0xff).toDouble, -1),
+            -1
+          )
+          i += 1
+        }
+        return av.visitEnd(-1)
+      case _ =>
+    }
     val len = xs.length
     val av = visitor.visitArray(len, -1)
     var i = 0
@@ -357,6 +376,22 @@ abstract class Materializer {
         case frame: Materializer.MaterializeArrFrame[T @unchecked] =>
           val arr = frame.arr
           val av = frame.arrVisitor
+          // Fast path for byte-backed arrays: emit all elements directly
+          if (frame.index == 0) {
+            arr match {
+              case ba: Val.ByteArr =>
+                val bytes = ba.rawBytes
+                val len = bytes.length
+                var i = 0
+                while (i < len) {
+                  val sub = av.subVisitor.asInstanceOf[Visitor[T, T]]
+                  av.visitValue(sub.visitFloat64((bytes(i) & 0xff).toDouble, -1), -1)
+                  i += 1
+                }
+                frame.index = len // mark as done
+              case _ =>
+            }
+          }
           if (frame.index < arr.length) {
             val childVal = arr.value(frame.index)
             frame.index += 1