From d4122d3329c4a59a0d9cf0437a2c05087cdb4f0f Mon Sep 17 00:00:00 2001 From: Tamay Eser Uysal Date: Thu, 22 Oct 2026 20:07:12 +0200 Subject: [PATCH 1/6] Add the brotli decoder intrinsic over libbrotlidec and link it from build.mjs when a unit includes it gea_node_brotli.hpp decodes whole buffers and, through a handle table over BrotliDecoderState, streams (open/write/finish/close); plugin/index.mjs lists the intrinsics. build.mjs links -lbrotlidec -lbrotlicommon only when a unit includes gea_node_brotli.hpp, finding the library through BROTLI_DIR, Homebrew, vcpkg or the system, and names the package to install when it is missing. The build also refuses a libbrotlidec affected by CVE-2026-111497 (decoder out-of-bounds write, fixed upstream in brotli 1.2.1, 1.1.1 and 1.0.10): it compiles and runs a BrotliDecoderVersion() check against the library it is about to link and stops with an upgrade hint below the fixed release of that line. --- plugin/index.mjs | 11 +++- runtime/node/gea_node_brotli.hpp | 108 +++++++++++++++++++++++++++++++ scripts/build.mjs | 89 ++++++++++++++++++++++++- scripts/test-native-brotli.mjs | 89 +++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 5 deletions(-) create mode 100644 runtime/node/gea_node_brotli.hpp create mode 100644 scripts/test-native-brotli.mjs diff --git a/plugin/index.mjs b/plugin/index.mjs index 46183df..909c131 100644 --- a/plugin/index.mjs +++ b/plugin/index.mjs @@ -288,7 +288,12 @@ const nodeOnlyHostFunctions = new Map([ ['__gea_node_crypto_timing_safe_equal', 'gea::node::crypto::timingSafeEqual'], ['__gea_node_crypto_get_fips', 'gea::node::crypto::getFips'], ['__gea_node_zlib_inflate', 'gea::node::zlib::inflate'], - ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'] + ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'], + ['__gea_node_brotli_decompress', 'gea::node::brotli::decompress'], + ['__gea_node_brotli_stream_open', 'gea::node::brotli::streamOpen'], + ['__gea_node_brotli_stream_write', 'gea::node::brotli::streamWrite'], + ['__gea_node_brotli_stream_finish', 'gea::node::brotli::streamFinish'], + ['__gea_node_brotli_stream_close', 'gea::node::brotli::streamClose'] ]) /** @@ -442,7 +447,9 @@ const hostPreambles = () => ? [runtimeHeader, '#include "gea_node_crypto.hpp"'] : spelling.startsWith('gea::node::zlib::') ? [runtimeHeader, '#include "gea_node_zlib.hpp"'] - : [runtimeHeader] + : spelling.startsWith('gea::node::brotli::') + ? [runtimeHeader, '#include "gea_node_brotli.hpp"'] + : [runtimeHeader] ]) ) diff --git a/runtime/node/gea_node_brotli.hpp b/runtime/node/gea_node_brotli.hpp new file mode 100644 index 0000000..940ddba --- /dev/null +++ b/runtime/node/gea_node_brotli.hpp @@ -0,0 +1,108 @@ +#pragma once + +#include "gea_runtime.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace gea::node::brotli { +using Bytes = gea::TypedArray; + +inline std::vector decompress(const Bytes& bytes) { + BrotliDecoderState* state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (state == nullptr) gea::host::throwRuntimeError("Error", "Cannot allocate memory"); + static const std::uint8_t empty = 0; + const std::uint8_t* nextIn = bytes.size() == 0 ? &empty : reinterpret_cast(bytes.data()); + std::size_t availableIn = bytes.size(); + std::vector output; + constexpr std::size_t chunk = 64 * 1024; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + const std::size_t used = output.size(); + output.resize(used + chunk); + std::uint8_t* nextOut = output.data() + used; + std::size_t availableOut = chunk; + result = BrotliDecoderDecompressStream(state, &availableIn, &nextIn, &availableOut, &nextOut, nullptr); + output.resize(used + (chunk - availableOut)); + } + if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + BrotliDecoderDestroyInstance(state); + gea::host::throwRuntimeError("Error", "unexpected end of file"); + } + if (result != BROTLI_DECODER_RESULT_SUCCESS) { + BrotliDecoderDestroyInstance(state); + gea::host::throwRuntimeError("Error", "Decompression failed"); + } + BrotliDecoderDestroyInstance(state); + return output; +} + +struct Stream { + BrotliDecoderState* state = nullptr; + bool finished = false; + + Stream() = default; + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + ~Stream() { + if (state != nullptr) BrotliDecoderDestroyInstance(state); + } +}; + +inline std::unordered_map>& streams() { + static std::unordered_map> table; + return table; +} + +inline Stream& streamAt(double handle) { + const auto found = streams().find(static_cast(handle)); + if (found == streams().end()) gea::host::throwRuntimeError("Error", "brotli binding closed"); + return *found->second; +} + +inline double streamOpen() { + static std::uint64_t last = 0; + auto stream = std::make_unique(); + stream->state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (stream->state == nullptr) gea::host::throwRuntimeError("Error", "Cannot allocate memory"); + streams().emplace(++last, std::move(stream)); + return static_cast(last); +} + +inline std::vector streamWrite(double handle, const Bytes& bytes) { + Stream& stream = streamAt(handle); + std::vector output; + if (stream.finished) return output; + static const std::uint8_t empty = 0; + const std::uint8_t* nextIn = bytes.size() == 0 ? &empty : reinterpret_cast(bytes.data()); + std::size_t availableIn = bytes.size(); + constexpr std::size_t chunk = 64 * 1024; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + const std::size_t used = output.size(); + output.resize(used + chunk); + std::uint8_t* nextOut = output.data() + used; + std::size_t availableOut = chunk; + result = BrotliDecoderDecompressStream(stream.state, &availableIn, &nextIn, &availableOut, &nextOut, nullptr); + output.resize(used + (chunk - availableOut)); + } + if (result == BROTLI_DECODER_RESULT_ERROR) gea::host::throwRuntimeError("Error", "Decompression failed"); + if (result == BROTLI_DECODER_RESULT_SUCCESS) stream.finished = true; + return output; +} + +inline void streamFinish(double handle) { + if (!streamAt(handle).finished) gea::host::throwRuntimeError("Error", "unexpected end of file"); +} + +inline void streamClose(double handle) { + streams().erase(static_cast(handle)); +} + +} diff --git a/scripts/build.mjs b/scripts/build.mjs index 00771eb..939abc2 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -43,7 +43,7 @@ // leaves a record. Exit codes: 1 nothing emitted, 2 link failed. import fs from 'node:fs' import path from 'node:path' -import { spawn, spawnSync } from 'node:child_process' +import { execFileSync, spawn, spawnSync } from 'node:child_process' import crypto from 'node:crypto' import os from 'node:os' import { fileURLToPath } from 'node:url' @@ -896,7 +896,7 @@ await buildExecutable(emittedUnits, displayPath) // or worse silently mis-read, by a unit compiled with a different language // dialect, optimization level or visibility -- with `-x c++-header` in place of // `-c`. `pch` on a compile phase is that PCH's `{ file, codegen }`. -function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, pch = null }) { +function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, needsBrotli, pch = null }) { const compiling = phase !== 'link' const linking = phase === 'all' || phase === 'link' const runtimeDir = path.join(repo, 'runtime') @@ -1014,6 +1014,7 @@ function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, p // the compiler and node-compat runtime roots. ...(compiling ? [`-I${compilerRuntimeDir}`, `-I${runtimeDir}`, `-I${path.join(runtimeDir, 'node')}`, `-I${outDir}`] : []), ...(compiling && needsZlib && windowsLibraries() ? [`-I${path.join(windowsLibraries(), 'include')}`] : []), + ...(compiling && needsBrotli ? brotliIncludeFlags() : []), // Escape hatch for one-off builds that need a flag the pipeline does not // model -- `-g` for a profile or an allocation census, `-fsanitize=...` for // a leak hunt. Whitespace-separated, appended last so it can override an @@ -1079,6 +1080,7 @@ function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, p ...(process.platform === 'linux' ? ['-pthread'] : []), ...(linking && process.platform === 'linux' && needsCrypto ? ['-lcrypto'] : []), ...(linking && needsZlib ? zlibLinkFlags() : []), + ...(linking && needsBrotli ? brotliLinkFlags() : []), '-o', output ] @@ -1185,6 +1187,85 @@ function zlibLinkFlags() { return windows ? [`-L${path.join(windows, 'lib')}`, '-lzs'] : ['-lz'] } +function brotliPrefix() { + if (brotliPrefix.cache !== undefined) return brotliPrefix.cache + const candidates = [] + if (process.env.BROTLI_DIR) candidates.push(path.resolve(process.env.BROTLI_DIR)) + else if (process.platform === 'win32') { + if (windowsLibraries()) candidates.push(windowsLibraries()) + } else if (process.platform === 'darwin') { + try { + candidates.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) + } catch {} + candidates.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') + } + brotliPrefix.cache = candidates.find((prefix) => fs.existsSync(path.join(prefix, 'include', 'brotli', 'decode.h'))) ?? null + if (brotliPrefix.cache === null && (process.platform !== 'linux' || process.env.BROTLI_DIR)) { + const install = + process.platform === 'darwin' + ? 'Install with `brew install brotli`' + : process.platform === 'win32' + ? 'Install with `vcpkg install brotli --triplet x64-windows-static`' + : 'Install libbrotli-dev (Debian/Ubuntu) or brotli-devel (Fedora)' + throw new Error( + `brotli (libbrotlidec) not found: node:zlib's brotli decoder needs its headers and library. ${install}, or set BROTLI_DIR to a prefix holding include/brotli/decode.h and lib/.` + ) + } + return brotliPrefix.cache +} + +function brotliVersionIsPatched(version) { + const major = version >>> 24 + const minor = (version >>> 12) & 0xfff + const patch = version & 0xfff + if (major !== 1) return major > 1 + if (minor === 0) return patch >= 10 + if (minor === 1 || minor === 2) return patch >= 1 + return minor > 2 +} + +function assertPatchedBrotli(cxx) { + if (assertPatchedBrotli.checked) return + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gea-brotli-version-')) + try { + const source = path.join(dir, 'version.cpp') + const executable = path.join(dir, process.platform === 'win32' ? 'version.exe' : 'version') + fs.writeFileSync(source, '#include \n#include \nint main() { std::printf("%u\\n", static_cast(BrotliDecoderVersion())); }\n') + const compiled = spawnSync(cxx, [...brotliIncludeFlags(), source, '-o', executable, ...brotliLinkFlags()], { encoding: 'utf8' }) + if (compiled.status !== 0) throw new Error(`could not build the libbrotlidec version check:\n${compiled.stderr}`) + const prefix = brotliPrefix() + const libraryPath = prefix === null ? {} : { DYLD_LIBRARY_PATH: path.join(prefix, 'lib'), LD_LIBRARY_PATH: path.join(prefix, 'lib') } + const ran = spawnSync(executable, [], { encoding: 'utf8', env: { ...process.env, ...libraryPath } }) + const version = Number.parseInt(ran.stdout, 10) + if (ran.status !== 0 || !Number.isInteger(version)) throw new Error(`could not run the libbrotlidec version check:\n${ran.stderr}`) + if (!brotliVersionIsPatched(version)) { + const found = `${version >>> 24}.${(version >>> 12) & 0xfff}.${version & 0xfff}` + throw new Error( + `libbrotlidec ${found} is affected by CVE-2026-111497 (decoder out-of-bounds write): node:zlib's brotli decoder needs brotli 1.2.1, 1.1.1 or 1.0.10 or later on the same line. Upgrade brotli, or set BROTLI_DIR to a prefix holding a patched build.` + ) + } + assertPatchedBrotli.checked = true + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +} + +function brotliIncludeFlags() { + const prefix = brotliPrefix() + return prefix === null ? [] : [`-I${path.join(prefix, 'include')}`] +} + +function brotliLinkFlags() { + const prefix = brotliPrefix() + const lib = prefix === null ? null : path.join(prefix, 'lib') + return [ + ...(lib === null ? [] : [`-L${lib}`]), + ...(lib !== null && process.platform !== 'win32' ? [`-Wl,-rpath,${lib}`] : []), + '-lbrotlidec', + '-lbrotlicommon' + ] +} + // Full-LTO link flags for clang: the linker that reads LLVM bitcode -- gold's // plugin on Linux (see `cxxFlags`), lld-link on Windows, ld64 itself on macOS. function fullLtoLinkFlags() { @@ -1705,12 +1786,14 @@ async function linkUnits(units, displayPath, exePath) { // zlib the same way, on every platform: node:zlib's codecs are the system // libz, which macOS ships too but does not link by default. const needsZlib = units.some((unit) => unit.source.includes('#include "gea_node_zlib.hpp"')) + const needsBrotli = units.some((unit) => unit.source.includes('#include "gea_node_brotli.hpp"')) // The C++ compiler is the caller's; the default is clang++, which is what the // published benchmarks were built with (clang 18 on the Ubuntu bench host; // g++ 13 there is 4-10% slower on the same emitted source). const cxx = process.env.CXX ?? 'clang++' const cxxIsClang = /clang/.test(path.basename(cxx)) || (!/g\+\+|gcc/.test(path.basename(cxx)) && process.platform === 'darwin') - const common = { cxxIsClang, needsCrypto, needsZlib } + const common = { cxxIsClang, needsCrypto, needsZlib, needsBrotli } + if (needsBrotli) assertPatchedBrotli(cxx) const linkStarted = process.hrtime.bigint() const fail = (stderr, fields = {}) => { diff --git a/scripts/test-native-brotli.mjs b/scripts/test-native-brotli.mjs new file mode 100644 index 0000000..ce9c91f --- /dev/null +++ b/scripts/test-native-brotli.mjs @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict' +import { brotliCompressSync } from 'node:zlib' +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync } from 'node:fs' +import { dirname, resolve } from 'node:path' + +import { compilerRuntimeInclude } from './resolve-compiler.mjs' + +const root = resolve(import.meta.dirname, '..') +const executable = resolve(root, 'apps/nest-node-probe/dist/native-brotli') + (process.platform === 'win32' ? '.exe' : '') +mkdirSync(dirname(executable), { recursive: true }) +const bytes = (value) => `bytes("${value.toString('hex')}")` +const inputs = [ + Buffer.alloc(0), + Buffer.from('abc'), + Buffer.from(Array.from({ length: 300000 }, (_, i) => (i * 37) & 255)), + Buffer.from('{"hello":"nest","n":1} '.repeat(20000)) +] +const operations = inputs.map((input) => `std::cout << hex(gea::node::brotli::decompress(${bytes(brotliCompressSync(input))})) << '\\n';`) +const trailing = Buffer.concat([brotliCompressSync(inputs[1]), Buffer.from('xyz')]) +operations.push(`std::cout << hex(gea::node::brotli::decompress(${bytes(trailing)})) << '\\n';`) +const truncated = brotliCompressSync(inputs[3]).subarray(0, 40) +const prefixes = [] +if (process.env.BROTLI_DIR) prefixes.push(resolve(process.env.BROTLI_DIR)) +else if (process.platform === 'win32') prefixes.push(...[process.env.VCPKG_ROOT, 'C:\\vcpkg'].filter(Boolean).map((r) => resolve(r, 'installed', 'x64-windows-static'))) +else if (process.platform === 'darwin') { + try { + prefixes.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) + } catch {} + prefixes.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') +} +const prefix = prefixes.find((p) => existsSync(resolve(p, 'include', 'brotli', 'decode.h'))) +const brotliFlags = [ + ...(prefix ? [`-I${resolve(prefix, 'include')}`, `-L${resolve(prefix, 'lib')}`] : []), + ...(prefix && process.platform !== 'win32' ? [`-Wl,-rpath,${resolve(prefix, 'lib')}`] : []), + '-lbrotlidec', + '-lbrotlicommon' +] +const code = String.raw` +#include "gea_node_buffer.hpp" +#include "gea_node_brotli.hpp" +#include +#include +gea::TypedArray bytes(const std::string& hex) { + const auto source = gea::node::buffer::from(hex, "hex"); + gea::TypedArray result(source->size()); + std::copy(source->data(), source->data() + source->size(), result.data()); + return result; +} +std::string hex(const std::vector& source) { + gea::TypedArray value(source.size()); + std::copy(source.begin(), source.end(), value.data()); + return gea::node::buffer::toString(value, "hex"); +} +template void throws(F action, const std::string& expected) { + bool caught = false; + try { action(); } catch (const gea::Value& error) { + caught = true; + assert(gea::host::instanceOfError(error, "Error")); + const auto native = gea::detail::unboxAs>(error, gea::Value::Tag::Object, "brotli error"); + const std::string message = native->message; + assert(message == expected); + std::cout << "throws " << message << '\n'; + } + assert(caught); +} +int main() { + ${operations.join('\n ')} + throws([] { gea::node::brotli::decompress(bytes("0102030405")); }, "unexpected end of file"); + throws([] { gea::node::brotli::decompress(${bytes(truncated)}); }, "unexpected end of file"); + throws([] { gea::node::brotli::decompress(bytes("ff")); }, "Decompression failed"); +} +` +execFileSync( + process.env.CXX ?? 'clang++', + ['-std=c++20', '-fsanitize=undefined,address', `-I${compilerRuntimeInclude()}`, `-I${resolve(root, 'runtime/node')}`, '-x', 'c++', '-', ...brotliFlags, '-o', executable], + { input: code, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 256 * 1024 * 1024 } +) +const lines = execFileSync(executable, { encoding: 'utf8', timeout: 60000, maxBuffer: 256 * 1024 * 1024 }) + .trimEnd() + .split(/\r?\n/) +let at = 0 +for (const input of inputs) assert.equal(lines[at++], input.toString('hex'), 'decompress') +assert.equal(lines[at++], inputs[1].toString('hex'), 'trailing bytes ignored') +assert.equal(lines[at++], 'throws unexpected end of file') +assert.equal(lines[at++], 'throws unexpected end of file') +assert.equal(lines[at++], 'throws Decompression failed') +assert.equal(at, lines.length) +console.log(`NATIVE_BROTLI_OK:${lines.length}`) From 5f5b70d3ae87f751a5357ff0238e4b4c25d08dd6 Mon Sep 17 00:00:00 2001 From: Tamay Eser Uysal Date: Thu, 22 Oct 2026 20:07:13 +0200 Subject: [PATCH 2/6] Add node:zlib streams (Gunzip, Inflate, InflateRaw, Unzip, Gzip, Deflate, DeflateRaw) and brotliDecompress with their facades and compatibility rows The stream classes are Transforms over native zlib and brotli stream handles: _transform pushes the output of each chunk as it arrives, _flush finishes the stream, and every exit path releases the handle. stream.ts follows Node where these streams depend on it: - Writable and Duplex convert string chunks to Buffers (decodeStrings, default true, off in object mode). - autoDestroy (default true): a Writable is destroyed after 'finish', a Readable after 'end', a Duplex once both have happened, so 'close' fires. - An error from _write, _final or a write after end destroys the stream (emits 'error' then 'close'); with autoDestroy false it only emits 'error'. - write() and end() after destroy call back with ERR_STREAM_DESTROYED. http.IncomingMessage._destroy tears down the connection only when the request did not complete, as Node does, so the autoDestroy that follows a read body no longer closes the connection before the response is written. scripts/test-zlib-streams.mjs compares a compiled probe with Node: each decoder over split input, truncated and corrupt input, round trips, output before end, 'close' after success and after errors, and string writes. --- .../correctness/zlib-streams.ts | 206 ++++++++++++++++++ plugin/index.mjs | 4 + runtime/node/gea_node_zlib.hpp | 145 +++++++++++- runtime/node/generated/facades/zlib.ts | 108 +-------- .../node/generated/node24-compatibility.json | 132 ++++++++--- runtime/node/zlib.ts | 170 ++++++++++++++- scripts/test-zlib-streams.mjs | 29 +++ 7 files changed, 652 insertions(+), 142 deletions(-) create mode 100644 apps/nest-node-probe/correctness/zlib-streams.ts create mode 100644 scripts/test-zlib-streams.mjs diff --git a/apps/nest-node-probe/correctness/zlib-streams.ts b/apps/nest-node-probe/correctness/zlib-streams.ts new file mode 100644 index 0000000..cc74399 --- /dev/null +++ b/apps/nest-node-probe/correctness/zlib-streams.ts @@ -0,0 +1,206 @@ +import { Buffer } from 'node:buffer' +import { PassThrough, type Transform } from 'node:stream' +import { + brotliDecompressSync, + createBrotliDecompress, + createDeflate, + createDeflateRaw, + createGunzip, + createGzip, + createInflate, + createInflateRaw, + createUnzip, + gunzipSync +} from 'node:zlib' + +const log = (line: string): void => { + console.log(line) +} + +const TEXT = 'The quick brown fox jumps over the lazy dog. '.repeat(40) +const GZIP = '1f8b08000000000000130bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200e6c3956408070000' +const DEFLATE = '789c0bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200880a8637' +const DEFLATE_RAW = '0bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200' +const BROTLI = '1b0707882c0e78d3d0955d9710bb172ba9cad092cc8cad415ce6f236c8199e9e0a7b830d387048206f24bf41a715ce1c1e27aa2938c2a9749f0603' +const GZIP_MEMBERS = '1f8b08000000000000134bcb2c2a2e510000fc7af11c060000001f8b08000000000000132b4e4dcecf4b010069111fb606000000' + +interface Case { + name: string + decoder: string + input: Buffer + expected: string +} + +function decoder(kind: string): Transform { + switch (kind) { + case 'gunzip': + return createGunzip() + case 'inflate': + return createInflate() + case 'inflateRaw': + return createInflateRaw() + case 'unzip': + return createUnzip() + case 'brotli': + return createBrotliDecompress() + default: + throw new Error(`unknown decoder ${kind}`) + } +} + +const cases: Case[] = [ + { name: 'gunzip', decoder: 'gunzip', input: Buffer.from(GZIP, 'hex'), expected: TEXT }, + { name: 'inflate', decoder: 'inflate', input: Buffer.from(DEFLATE, 'hex'), expected: TEXT }, + { name: 'inflateRaw', decoder: 'inflateRaw', input: Buffer.from(DEFLATE_RAW, 'hex'), expected: TEXT }, + { name: 'unzip gzip', decoder: 'unzip', input: Buffer.from(GZIP, 'hex'), expected: TEXT }, + { name: 'unzip zlib', decoder: 'unzip', input: Buffer.from(DEFLATE, 'hex'), expected: TEXT }, + { name: 'brotli', decoder: 'brotli', input: Buffer.from(BROTLI, 'hex'), expected: TEXT }, + { name: 'gunzip members', decoder: 'gunzip', input: Buffer.from(GZIP_MEMBERS, 'hex'), expected: 'first second' }, + { name: 'gunzip garbage', decoder: 'gunzip', input: Buffer.from('0102030405', 'hex'), expected: '' }, + { name: 'gunzip truncated', decoder: 'gunzip', input: Buffer.from(GZIP, 'hex').subarray(0, 40), expected: '' }, + { name: 'inflate garbage', decoder: 'inflate', input: Buffer.from('0102030405', 'hex'), expected: '' }, + { name: 'brotli truncated', decoder: 'brotli', input: Buffer.from(BROTLI, 'hex').subarray(0, 40), expected: '' }, + { name: 'brotli corrupt', decoder: 'brotli', input: Buffer.from('ff', 'hex'), expected: '' }, + { name: 'brotli empty', decoder: 'brotli', input: Buffer.from('3b', 'hex'), expected: '' } +] + +let at = 0 + +function runDecoder(item: Case, done: () => void): void { + const source = new PassThrough() + const stream = decoder(item.decoder) + const chunks: Buffer[] = [] + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const output = Buffer.concat(chunks) + log(`${item.name}: end ${output.length} ${output.toString() === item.expected ? 'match' : 'MISMATCH'}`) + }) + stream.on('error', (error: Error) => { + log(`${item.name}: error ${error.message}`) + }) + stream.on('close', () => { + log(`${item.name}: close destroyed ${stream.destroyed}`) + done() + }) + source.pipe(stream) + const third = Math.max(1, Math.floor(item.input.length / 3)) + source.write(item.input.subarray(0, third)) + source.write(item.input.subarray(third, 2 * third)) + source.end(item.input.subarray(2 * third)) +} + +function roundTrip(name: string, encoder: Transform, decode: Transform, done: () => void): void { + const chunks: Buffer[] = [] + encoder.pipe(decode) + decode.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + decode.on('end', () => { + const output = Buffer.concat(chunks) + log(`${name}: end ${output.length} ${output.toString() === TEXT ? 'match' : 'MISMATCH'}`) + done() + }) + encoder.write(Buffer.from(TEXT.slice(0, 600))) + encoder.write(Buffer.from(TEXT.slice(600, 1200))) + encoder.end(Buffer.from(TEXT.slice(1200))) +} + +function noise(length: number): Buffer { + const bytes = Buffer.alloc(length) + let seed = 7 + for (let i = 0; i < length; i++) { + seed = (seed * 1103515245 + 12345) % 2147483648 + bytes[i] = seed % 256 + } + return bytes +} + +function beforeEnd(name: string, stream: Transform, input: Buffer, done: () => void): void { + let seen = false + let ended = false + const finish = (): void => { + if (ended) return + ended = true + log(`${name}: output before end ${seen ? 'yes' : 'no'}`) + stream.end() + } + const timer = setTimeout(finish, 2000) + stream.on('data', () => { + if (seen) return + seen = true + clearTimeout(timer) + finish() + }) + stream.on('end', () => done()) + stream.on('error', (error: Error) => { + log(`${name}: error ${error.message}`) + done() + }) + stream.write(input) +} + +function stringWrite(done: () => void): void { + const encoder = createGzip() + const chunks: Buffer[] = [] + encoder.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + encoder.on('end', () => { + log(`gzip string write: ${gunzipSync(Buffer.concat(chunks)).toString()}`) + }) + encoder.on('close', () => done()) + encoder.write('first ') + encoder.end('second', 'utf8') +} + +function next(): void { + if (at < cases.length) { + const item = cases[at] + at++ + runDecoder(item, next) + return + } + if (at === cases.length) { + at++ + roundTrip('gzip round trip', createGzip(), createGunzip(), next) + return + } + if (at === cases.length + 1) { + at++ + roundTrip('deflate round trip', createDeflate({ level: 9 }), createInflate(), next) + return + } + if (at === cases.length + 2) { + at++ + roundTrip('deflateRaw round trip', createDeflateRaw(), createInflateRaw(), next) + return + } + if (at === cases.length + 3) { + at++ + beforeEnd('gunzip streams', createGunzip(), Buffer.from(GZIP, 'hex'), next) + return + } + if (at === cases.length + 4) { + at++ + beforeEnd('brotli streams', createBrotliDecompress(), Buffer.from(BROTLI, 'hex'), next) + return + } + if (at === cases.length + 5) { + at++ + beforeEnd('gzip streams', createGzip(), noise(262144), next) + return + } + if (at === cases.length + 6) { + at++ + stringWrite(next) + return + } + log(`brotliDecompressSync: ${brotliDecompressSync(Buffer.from(BROTLI, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) + log(`gunzipSync: ${gunzipSync(Buffer.from(GZIP, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) + log('done') +} + +log('start') +next() diff --git a/plugin/index.mjs b/plugin/index.mjs index 909c131..a9b6be8 100644 --- a/plugin/index.mjs +++ b/plugin/index.mjs @@ -289,6 +289,10 @@ const nodeOnlyHostFunctions = new Map([ ['__gea_node_crypto_get_fips', 'gea::node::crypto::getFips'], ['__gea_node_zlib_inflate', 'gea::node::zlib::inflate'], ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'], + ['__gea_node_zlib_stream_open', 'gea::node::zlib::streamOpen'], + ['__gea_node_zlib_stream_write', 'gea::node::zlib::streamWrite'], + ['__gea_node_zlib_stream_finish', 'gea::node::zlib::streamFinish'], + ['__gea_node_zlib_stream_close', 'gea::node::zlib::streamClose'], ['__gea_node_brotli_decompress', 'gea::node::brotli::decompress'], ['__gea_node_brotli_stream_open', 'gea::node::brotli::streamOpen'], ['__gea_node_brotli_stream_write', 'gea::node::brotli::streamWrite'], diff --git a/runtime/node/gea_node_zlib.hpp b/runtime/node/gea_node_zlib.hpp index f66020e..65d82a4 100644 --- a/runtime/node/gea_node_zlib.hpp +++ b/runtime/node/gea_node_zlib.hpp @@ -1,9 +1,9 @@ #pragma once // node:zlib's codecs, over the platform's own zlib -- the library node itself -// vendors. TypeScript owns the API shape (`runtime/node/zlib.ts`); only whole -// byte buffers cross this boundary, as native typed-array views, so a -// compressed payload is never boxed. +// vendors. TypeScript owns the API shape (`runtime/node/zlib.ts`); only byte +// buffers and numeric stream handles cross this boundary, as native +// typed-array views and doubles, so a compressed payload is never boxed. // // `windowBits` selects the framing exactly as zlib defines it, and node's // classes map onto it one to one: 15 is the zlib wrapper (Deflate/Inflate), @@ -16,7 +16,9 @@ #include #include #include +#include #include +#include #include namespace gea::node::zlib { @@ -49,6 +51,13 @@ inline Bytef* input(const Bytes& bytes) { return bytes.size() == 0 ? &empty : const_cast(reinterpret_cast(bytes.data())); } +inline int deflateLevel(int bits, double level) { + if (bits == 47) gea::host::throwRuntimeError("RangeError", "Unsupported zlib framing"); + if (!std::isfinite(level) || level < -1.0 || level > 9.0 || std::trunc(level) != level) + gea::host::throwRuntimeError("RangeError", "The value of \"options.level\" is out of range. It must be >= -1 and <= 9."); + return static_cast(level); +} + inline std::vector inflate(const Bytes& bytes, double windowBits) { const int bits = framing(windowBits); z_stream stream{}; @@ -100,11 +109,9 @@ inline std::vector inflate(const Bytes& bytes, double windowBits) inline std::vector deflate(const Bytes& bytes, double windowBits, double level) { const int bits = framing(windowBits); - if (bits == 47) gea::host::throwRuntimeError("RangeError", "Unsupported zlib framing"); - if (!std::isfinite(level) || level < -1.0 || level > 9.0 || std::trunc(level) != level) - gea::host::throwRuntimeError("RangeError", "The value of \"options.level\" is out of range. It must be >= -1 and <= 9."); + const int compression = deflateLevel(bits, level); z_stream stream{}; - if (deflateInit2(&stream, static_cast(level), Z_DEFLATED, bits, 8, Z_DEFAULT_STRATEGY) != Z_OK) fail(Z_MEM_ERROR, stream); + if (deflateInit2(&stream, compression, Z_DEFLATED, bits, 8, Z_DEFAULT_STRATEGY) != Z_OK) fail(Z_MEM_ERROR, stream); std::vector output(deflateBound(&stream, static_cast(bytes.size()))); std::size_t remaining = bytes.size(); Bytef* next = input(bytes); @@ -132,4 +139,128 @@ inline std::vector deflate(const Bytes& bytes, double windowBits, return output; } +struct Stream { + z_stream z{}; + int bits = 0; + bool deflating = false; + bool ended = false; + bool betweenMembers = false; + + Stream() = default; + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + ~Stream() { + if (deflating) deflateEnd(&z); + else inflateEnd(&z); + } +}; + +inline std::unordered_map>& streams() { + static std::unordered_map> table; + return table; +} + +inline Stream& streamAt(double handle) { + const auto found = streams().find(static_cast(handle)); + if (found == streams().end()) gea::host::throwRuntimeError("Error", "zlib binding closed"); + return *found->second; +} + +inline double streamOpen(double mode, double windowBits, double level) { + static std::uint64_t last = 0; + const int bits = framing(windowBits); + auto stream = std::make_unique(); + stream->bits = bits; + if (mode == 1.0) { + const int compression = deflateLevel(bits, level); + if (deflateInit2(&stream->z, compression, Z_DEFLATED, bits, 8, Z_DEFAULT_STRATEGY) != Z_OK) fail(Z_MEM_ERROR, stream->z); + stream->deflating = true; + } else if (inflateInit2(&stream->z, bits) != Z_OK) { + fail(Z_MEM_ERROR, stream->z); + } + streams().emplace(++last, std::move(stream)); + return static_cast(last); +} + +inline std::vector deflateInto(Stream& stream, Bytef* next, std::size_t size, int flush) { + std::vector output; + constexpr std::size_t chunk = 64 * 1024; + std::size_t remaining = size; + do { + const uInt take = static_cast(std::min(remaining, std::numeric_limits::max())); + stream.z.next_in = next; + stream.z.avail_in = take; + do { + const std::size_t used = output.size(); + output.resize(used + chunk); + stream.z.next_out = reinterpret_cast(output.data() + used); + stream.z.avail_out = static_cast(chunk); + const int code = ::deflate(&stream.z, remaining == take ? flush : Z_NO_FLUSH); + output.resize(used + (chunk - stream.z.avail_out)); + if (code == Z_STREAM_ERROR) fail(code, stream.z); + } while (stream.z.avail_out == 0); + next += take - stream.z.avail_in; + remaining -= take - stream.z.avail_in; + } while (remaining > 0); + return output; +} + +inline std::vector inflateInto(Stream& stream, const Bytes& bytes) { + std::vector output; + constexpr std::size_t chunk = 64 * 1024; + std::size_t remaining = bytes.size(); + Bytef* next = input(bytes); + while (remaining > 0 && !stream.ended) { + const uInt take = static_cast(std::min(remaining, std::numeric_limits::max())); + stream.z.next_in = next; + stream.z.avail_in = take; + int code = Z_OK; + do { + const std::size_t used = output.size(); + output.resize(used + chunk); + stream.z.next_out = reinterpret_cast(output.data() + used); + stream.z.avail_out = static_cast(chunk); + code = ::inflate(&stream.z, Z_NO_FLUSH); + output.resize(used + (chunk - stream.z.avail_out)); + if (code != Z_OK && code != Z_STREAM_END && code != Z_BUF_ERROR) fail(code, stream.z); + } while (code == Z_OK && stream.z.avail_out == 0); + const std::size_t consumed = take - stream.z.avail_in; + next += consumed; + remaining -= consumed; + if (code == Z_STREAM_END) { + if (stream.bits != 31 && stream.bits != 47) { + stream.ended = true; + break; + } + if (inflateReset(&stream.z) != Z_OK) fail(Z_STREAM_ERROR, stream.z); + stream.betweenMembers = true; + continue; + } + if (consumed > 0) stream.betweenMembers = false; + } + return output; +} + +inline std::vector streamWrite(double handle, const Bytes& bytes) { + Stream& stream = streamAt(handle); + return stream.deflating ? deflateInto(stream, input(bytes), bytes.size(), Z_NO_FLUSH) : inflateInto(stream, bytes); +} + +inline std::vector streamFinish(double handle) { + Stream& stream = streamAt(handle); + if (stream.deflating) { + static Bytef empty = 0; + return deflateInto(stream, &empty, 0, Z_FINISH); + } + if (!stream.ended && !stream.betweenMembers) { + stream.z.msg = nullptr; + fail(Z_BUF_ERROR, stream.z); + } + return {}; +} + +inline void streamClose(double handle) { + streams().erase(static_cast(handle)); +} + } // namespace gea::node::zlib diff --git a/runtime/node/generated/facades/zlib.ts b/runtime/node/generated/facades/zlib.ts index e8e4775..02cf683 100644 --- a/runtime/node/generated/facades/zlib.ts +++ b/runtime/node/generated/facades/zlib.ts @@ -1,7 +1,7 @@ // Generated from the pinned Node 24 declaration inventory. Do not edit. // Canonical builtin: node:zlib import { nodeNotImplemented } from "../../not-implemented" -export { constants, deflate, deflateRawSync, deflateSync, gunzip, gunzipSync, gzip, gzipSync, inflate, inflateRawSync, inflateSync, unzipSync } from "../../zlib" +export { brotliDecompress, BrotliDecompress, brotliDecompressSync, constants, createBrotliDecompress, createDeflate, createDeflateRaw, createGunzip, createGzip, createInflate, createInflateRaw, createUnzip, deflate, Deflate, DeflateRaw, deflateRawSync, deflateSync, gunzip, Gunzip, gunzipSync, gzip, Gzip, gzipSync, inflate, Inflate, InflateRaw, inflateRawSync, inflateSync, Unzip, unzipSync } from "../../zlib" export function brotliCompress(...args: unknown[]): never { void args @@ -20,23 +20,6 @@ export function brotliCompressSync(...args: unknown[]): never { return nodeNotImplemented("node:zlib", "brotliCompressSync") } -export function brotliDecompress(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "brotliDecompress") -} - -export class BrotliDecompress { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "BrotliDecompress") - } -} - -export function brotliDecompressSync(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "brotliDecompressSync") -} - export function crc32(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "crc32") @@ -47,46 +30,6 @@ export function createBrotliCompress(...args: unknown[]): never { return nodeNotImplemented("node:zlib", "createBrotliCompress") } -export function createBrotliDecompress(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createBrotliDecompress") -} - -export function createDeflate(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createDeflate") -} - -export function createDeflateRaw(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createDeflateRaw") -} - -export function createGunzip(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createGunzip") -} - -export function createGzip(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createGzip") -} - -export function createInflate(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createInflate") -} - -export function createInflateRaw(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createInflateRaw") -} - -export function createUnzip(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createUnzip") -} - export function createZstdCompress(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "createZstdCompress") @@ -97,70 +40,21 @@ export function createZstdDecompress(...args: unknown[]): never { return nodeNotImplemented("node:zlib", "createZstdDecompress") } -export class Deflate { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Deflate") - } -} - export function deflateRaw(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "deflateRaw") } -export class DeflateRaw { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "DeflateRaw") - } -} - -export class Gunzip { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Gunzip") - } -} - -export class Gzip { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Gzip") - } -} - -export class Inflate { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Inflate") - } -} - export function inflateRaw(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "inflateRaw") } -export class InflateRaw { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "InflateRaw") - } -} - export function unzip(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "unzip") } -export class Unzip { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Unzip") - } -} - export const Z_ASCII: unknown = undefined export const Z_BEST_COMPRESSION: unknown = undefined diff --git a/runtime/node/generated/node24-compatibility.json b/runtime/node/generated/node24-compatibility.json index 58ea856..b4e5e4e 100644 --- a/runtime/node/generated/node24-compatibility.json +++ b/runtime/node/generated/node24-compatibility.json @@ -20938,25 +20938,38 @@ { "name": "brotliDecompress", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(input: InputType, callback: ZlibCallback): void" + ] }, { "name": "BrotliDecompress", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): BrotliDecompress" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, { "name": "brotliDecompressSync", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(input: InputType): Buffer" + ] }, { "name": "constants", @@ -20977,42 +20990,74 @@ { "name": "createBrotliDecompress", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): BrotliDecompress" + ] }, { "name": "createDeflate", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(options?: ZlibOptions): Deflate" + ] }, { "name": "createDeflateRaw", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(options?: ZlibOptions): DeflateRaw" + ] }, { "name": "createGunzip", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): Gunzip" + ] }, { "name": "createGzip", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(options?: ZlibOptions): Gzip" + ] }, { "name": "createInflate", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): Inflate" + ] }, { "name": "createInflateRaw", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): InflateRaw" + ] }, { "name": "createUnzip", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): Unzip" + ] }, { "name": "createZstdCompress", @@ -21036,13 +21081,18 @@ { "name": "Deflate", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(options?: ZlibOptions): Deflate" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21054,13 +21104,18 @@ { "name": "DeflateRaw", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(options?: ZlibOptions): DeflateRaw" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21094,13 +21149,18 @@ { "name": "Gunzip", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): Gunzip" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21125,13 +21185,18 @@ { "name": "Gzip", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(options?: ZlibOptions): Gzip" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21156,13 +21221,18 @@ { "name": "Inflate", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): Inflate" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21174,13 +21244,18 @@ { "name": "InflateRaw", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): InflateRaw" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21210,13 +21285,18 @@ { "name": "Unzip", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): Unzip" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, diff --git a/runtime/node/zlib.ts b/runtime/node/zlib.ts index 390178d..7a2e45d 100644 --- a/runtime/node/zlib.ts +++ b/runtime/node/zlib.ts @@ -1,12 +1,14 @@ -// node:zlib's one-shot codecs, over the platform zlib (`gea_node_zlib.hpp`). +// node:zlib's codecs, over the platform zlib (`gea_node_zlib.hpp`). // // The synchronous functions are node's `*Sync` family; the callback forms run // the same codec and report on a later turn, as node's do. Streams -// (`createGzip` and friends) are not provided. Buffer never crosses a dynamic +// (`createGzip` and friends) hold a native codec handle and push each chunk's +// output as it is produced. Buffer never crosses a dynamic // carrier: a database driver's wire compression and saslprep's code-point // tables both hand over a whole Buffer and take a whole Buffer back. import { Buffer } from './buffer' +import { Transform, type DestroyCallback, type TransformCallback } from './stream' // The driver always supplies Buffer here. Keeping the alias exact to that // source use prevents the Buffer | Uint8Array API union from degrading the @@ -23,6 +25,24 @@ export type ZlibCallback = (error: Error | null, result: Buffer) => void declare function __gea_node_zlib_inflate(input: Buffer, windowBits: number): Buffer /** @gea-host-inert */ declare function __gea_node_zlib_deflate(input: Buffer, windowBits: number, level: number): Buffer +/** @gea-host-inert */ +declare function __gea_node_brotli_decompress(input: Buffer): Buffer +/** @gea-host-inert */ +declare function __gea_node_zlib_stream_open(mode: number, windowBits: number, level: number): number +/** @gea-host-inert */ +declare function __gea_node_zlib_stream_write(handle: number, input: Buffer): Buffer +/** @gea-host-inert */ +declare function __gea_node_zlib_stream_finish(handle: number): Buffer +/** @gea-host-inert */ +declare function __gea_node_zlib_stream_close(handle: number): void +/** @gea-host-inert */ +declare function __gea_node_brotli_stream_open(): number +/** @gea-host-inert */ +declare function __gea_node_brotli_stream_write(handle: number, input: Buffer): Buffer +/** @gea-host-inert */ +declare function __gea_node_brotli_stream_finish(handle: number): void +/** @gea-host-inert */ +declare function __gea_node_brotli_stream_close(handle: number): void // zlib's `windowBits` for each of node's framings. const ZLIB = 15 @@ -95,3 +115,149 @@ export function gunzip(input: InputType, callback: ZlibCallback): void { export function gzip(input: InputType, options: ZlibOptions, callback: ZlibCallback): void { settle(callback, () => gzipSync(input, options)) } + +export function brotliDecompressSync(input: InputType): Buffer { + return __gea_node_brotli_decompress(input) +} + +export function brotliDecompress(input: InputType, callback: ZlibCallback): void { + settle(callback, () => brotliDecompressSync(input)) +} + +const MODE_INFLATE = 0 +const MODE_DEFLATE = 1 +const MODE_BROTLI_DECOMPRESS = 2 + +const failureOf = (error: unknown): Error => (error instanceof Error ? error : new Error(String(error))) + +class ZlibBase extends Transform { + private handle_: number + private brotli_: boolean + + constructor(mode: number, windowBits: number, level: number) { + super() + this.brotli_ = mode === MODE_BROTLI_DECOMPRESS + this.handle_ = this.brotli_ ? __gea_node_brotli_stream_open() : __gea_node_zlib_stream_open(mode, windowBits, level) + } + + override _transform(chunk: Buffer, _encoding: unknown, callback: TransformCallback): void { + let output: Buffer + try { + output = this.brotli_ ? __gea_node_brotli_stream_write(this.handle_, chunk) : __gea_node_zlib_stream_write(this.handle_, chunk) + } catch (error) { + this.release_() + callback(failureOf(error)) + return + } + if (output.length > 0) this.push(output) + callback(null) + } + + override _flush(callback: TransformCallback): void { + let output: Buffer + try { + if (this.brotli_) { + __gea_node_brotli_stream_finish(this.handle_) + output = Buffer.alloc(0) + } else output = __gea_node_zlib_stream_finish(this.handle_) + } catch (error) { + this.release_() + callback(failureOf(error)) + return + } + this.release_() + callback(null, output.length > 0 ? output : undefined) + } + + override _destroy(error: Error | null, callback: DestroyCallback): void { + this.release_() + callback(error) + } + + private release_(): void { + if (this.handle_ === 0) return + if (this.brotli_) __gea_node_brotli_stream_close(this.handle_) + else __gea_node_zlib_stream_close(this.handle_) + this.handle_ = 0 + } +} + +export class Gunzip extends ZlibBase { + constructor() { + super(MODE_INFLATE, GZIP, 0) + } +} + +export class Inflate extends ZlibBase { + constructor() { + super(MODE_INFLATE, ZLIB, 0) + } +} + +export class InflateRaw extends ZlibBase { + constructor() { + super(MODE_INFLATE, RAW, 0) + } +} + +export class Unzip extends ZlibBase { + constructor() { + super(MODE_INFLATE, DETECT, 0) + } +} + +export class BrotliDecompress extends ZlibBase { + constructor() { + super(MODE_BROTLI_DECOMPRESS, 0, 0) + } +} + +export class Gzip extends ZlibBase { + constructor(options?: ZlibOptions) { + super(MODE_DEFLATE, GZIP, levelOf(options)) + } +} + +export class Deflate extends ZlibBase { + constructor(options?: ZlibOptions) { + super(MODE_DEFLATE, ZLIB, levelOf(options)) + } +} + +export class DeflateRaw extends ZlibBase { + constructor(options?: ZlibOptions) { + super(MODE_DEFLATE, RAW, levelOf(options)) + } +} + +export function createGunzip(): Gunzip { + return new Gunzip() +} + +export function createInflate(): Inflate { + return new Inflate() +} + +export function createInflateRaw(): InflateRaw { + return new InflateRaw() +} + +export function createUnzip(): Unzip { + return new Unzip() +} + +export function createBrotliDecompress(): BrotliDecompress { + return new BrotliDecompress() +} + +export function createGzip(options?: ZlibOptions): Gzip { + return new Gzip(options) +} + +export function createDeflate(options?: ZlibOptions): Deflate { + return new Deflate(options) +} + +export function createDeflateRaw(options?: ZlibOptions): DeflateRaw { + return new DeflateRaw(options) +} diff --git a/scripts/test-zlib-streams.mjs b/scripts/test-zlib-streams.mjs new file mode 100644 index 0000000..458f6de --- /dev/null +++ b/scripts/test-zlib-streams.mjs @@ -0,0 +1,29 @@ +import assert from 'node:assert/strict' +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdirSync } from 'node:fs' +import { resolve } from 'node:path' + +const root = resolve(import.meta.dirname, '..') +const output = resolve(root, 'apps/nest-node-probe/dist') +const source = resolve(root, 'apps/nest-node-probe/correctness/zlib-streams.ts') +const executable = resolve(output, 'zlib-streams') +const options = { cwd: root, encoding: 'utf8', env: { ...process.env, TMPDIR: output } } +mkdirSync(output, { recursive: true }) + +execFileSync(process.execPath, [resolve(root, 'scripts/build.mjs'), source, '--out', output, '--exe', executable], { + ...options, + stdio: 'inherit' +}) +const expected = spawnSync(process.execPath, ['--experimental-strip-types', source], { ...options, timeout: 15000 }) +const actual = spawnSync(executable, [], { ...options, timeout: 15000 }) +assert.equal(expected.status, 0, `node status\n${expected.stderr}`) +assert.equal(actual.status, 0, `compiled status\n${actual.stderr}`) +const expectedLines = expected.stdout.trim().split('\n') +const actualLines = actual.stdout.trim().split('\n') +assert.equal(actualLines.length, expectedLines.length, `line count\n--- node\n${expected.stdout}\n--- compiled\n${actual.stdout}`) +for (let i = 0; i < expectedLines.length; i++) assert.equal(actualLines[i], expectedLines[i], `line ${i + 1}`) +assert.ok(expectedLines.includes('brotli: end 1800 match') && expectedLines.includes('gunzip garbage: error incorrect header check'), 'the oracle covered the decoders') +assert.ok(expectedLines.includes('gzip string write: first second'), 'the oracle wrote strings') +for (const name of ['gunzip', 'gunzip truncated', 'brotli corrupt']) assert.ok(expectedLines.includes(`${name}: close destroyed true`), `the oracle closed ${name}`) +for (const name of ['gunzip', 'brotli', 'gzip']) assert.ok(expectedLines.includes(`${name} streams: output before end yes`), `the oracle streamed ${name}`) +console.log(`ZLIB_STREAMS_PARITY_OK:${expectedLines.length}`) From f9c3c1260dfbcaeb9968333f850051074671ef7d Mon Sep 17 00:00:00 2001 From: Tamay Eser Uysal Date: Fri, 23 Oct 2026 01:56:44 +0200 Subject: [PATCH 3/6] End zlib and brotli streams at trailing input and read gzip zero padding, as Node does A decoder whose stream has ended with input left over pushes null, so 'end' fires before end() as in Node. Gunzip stops at a zero byte after a member on both the stream and the one-shot path instead of reading it as the next header. --- .../correctness/zlib-streams.ts | 91 +++++++++++++++++++ plugin/index.mjs | 2 + runtime/node/gea_node_brotli.hpp | 7 ++ runtime/node/gea_node_zlib.hpp | 19 +++- runtime/node/zlib.ts | 7 ++ 5 files changed, 123 insertions(+), 3 deletions(-) diff --git a/apps/nest-node-probe/correctness/zlib-streams.ts b/apps/nest-node-probe/correctness/zlib-streams.ts index cc74399..aba29d4 100644 --- a/apps/nest-node-probe/correctness/zlib-streams.ts +++ b/apps/nest-node-probe/correctness/zlib-streams.ts @@ -23,6 +23,9 @@ const DEFLATE = '789c0bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2 const DEFLATE_RAW = '0bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200' const BROTLI = '1b0707882c0e78d3d0955d9710bb172ba9cad092cc8cad415ce6f236c8199e9e0a7b830d387048206f24bf41a715ce1c1e27aa2938c2a9749f0603' const GZIP_MEMBERS = '1f8b08000000000000134bcb2c2a2e510000fc7af11c060000001f8b08000000000000132b4e4dcecf4b010069111fb606000000' +const GZIP_FIRST = '1f8b08000000000000134bcb2c2a2e510000fc7af11c06000000' +const GZIP_SECOND = '1f8b08000000000000132b4e4dcecf4b010069111fb606000000' +const ZEROS = '00000000000000000000000000000000' interface Case { name: string @@ -57,6 +60,11 @@ const cases: Case[] = [ { name: 'brotli', decoder: 'brotli', input: Buffer.from(BROTLI, 'hex'), expected: TEXT }, { name: 'gunzip members', decoder: 'gunzip', input: Buffer.from(GZIP_MEMBERS, 'hex'), expected: 'first second' }, { name: 'gunzip garbage', decoder: 'gunzip', input: Buffer.from('0102030405', 'hex'), expected: '' }, + { name: 'gunzip zero padding', decoder: 'gunzip', input: Buffer.from(GZIP + ZEROS, 'hex'), expected: TEXT }, + { name: 'gunzip zero then garbage', decoder: 'gunzip', input: Buffer.from(GZIP + '00010203', 'hex'), expected: TEXT }, + { name: 'gunzip trailing garbage', decoder: 'gunzip', input: Buffer.from(GZIP + '010203', 'hex'), expected: TEXT }, + { name: 'gunzip padded members', decoder: 'gunzip', input: Buffer.from(GZIP_FIRST + '0000' + GZIP_SECOND, 'hex'), expected: 'first ' }, + { name: 'unzip zero padding', decoder: 'unzip', input: Buffer.from(GZIP + ZEROS, 'hex'), expected: TEXT }, { name: 'gunzip truncated', decoder: 'gunzip', input: Buffer.from(GZIP, 'hex').subarray(0, 40), expected: '' }, { name: 'inflate garbage', decoder: 'inflate', input: Buffer.from('0102030405', 'hex'), expected: '' }, { name: 'brotli truncated', decoder: 'brotli', input: Buffer.from(BROTLI, 'hex').subarray(0, 40), expected: '' }, @@ -155,6 +163,60 @@ function stringWrite(done: () => void): void { encoder.end('second', 'utf8') } +function earlyEnd(name: string, stream: Transform, first: Buffer, second: Buffer, done: () => void): void { + const chunks: Buffer[] = [] + let ended = false + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + stream.on('end', () => { + ended = true + }) + stream.on('error', (error: Error) => { + log(`${name}: error ${error.message}`) + }) + stream.on('close', () => { + log(`${name}: close ${Buffer.concat(chunks).length}`) + done() + }) + stream.write(first) + setTimeout(() => { + log(`${name}: ended before end() ${ended}`) + stream.end(second) + }, 20) +} + +function concurrent(done: () => void): void { + const results: string[] = ['', '', '', ''] + let open = results.length + for (let index = 0; index < results.length; index++) { + const stream = index % 2 === 0 ? createGunzip() : createBrotliDecompress() + const chunks: Buffer[] = [] + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + stream.on('end', () => { + results[index] = Buffer.concat(chunks).toString() === TEXT ? 'match' : 'MISMATCH' + }) + stream.on('close', () => { + open-- + if (open === 0) { + log(`concurrent streams: ${results.join(' ')}`) + done() + } + }) + stream.end(Buffer.from(index % 2 === 0 ? GZIP : BROTLI, 'hex')) + } +} + +function syncGunzip(name: string, hex: string): void { + try { + log(`gunzipSync ${name}: ${gunzipSync(Buffer.from(hex, 'hex')).toString().length}`) + } catch (error) { + log(`gunzipSync ${name}: error ${(error as Error).message}`) + } +} + function next(): void { if (at < cases.length) { const item = cases[at] @@ -197,6 +259,35 @@ function next(): void { stringWrite(next) return } + if (at === cases.length + 7) { + at++ + earlyEnd('inflate trailing', createInflate(), Buffer.from(DEFLATE + '61626364', 'hex'), Buffer.from('efgh'), next) + return + } + if (at === cases.length + 8) { + at++ + earlyEnd('brotli trailing', createBrotliDecompress(), Buffer.from(BROTLI + '6162', 'hex'), Buffer.from('cd'), next) + return + } + if (at === cases.length + 9) { + at++ + earlyEnd('gunzip zero padding then zero', createGunzip(), Buffer.from(GZIP + ZEROS, 'hex'), Buffer.from('0001', 'hex'), next) + return + } + if (at === cases.length + 10) { + at++ + earlyEnd('gzip whole', createGunzip(), Buffer.from(GZIP, 'hex'), Buffer.alloc(0), next) + return + } + if (at === cases.length + 11) { + at++ + concurrent(next) + return + } + syncGunzip('zero padding', GZIP + ZEROS) + syncGunzip('zero then garbage', GZIP + '00010203') + syncGunzip('trailing garbage', GZIP + '010203') + syncGunzip('padded members', GZIP_FIRST + '0000' + GZIP_SECOND) log(`brotliDecompressSync: ${brotliDecompressSync(Buffer.from(BROTLI, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) log(`gunzipSync: ${gunzipSync(Buffer.from(GZIP, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) log('done') diff --git a/plugin/index.mjs b/plugin/index.mjs index a9b6be8..b9c72a0 100644 --- a/plugin/index.mjs +++ b/plugin/index.mjs @@ -291,11 +291,13 @@ const nodeOnlyHostFunctions = new Map([ ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'], ['__gea_node_zlib_stream_open', 'gea::node::zlib::streamOpen'], ['__gea_node_zlib_stream_write', 'gea::node::zlib::streamWrite'], + ['__gea_node_zlib_stream_leftover', 'gea::node::zlib::streamLeftover'], ['__gea_node_zlib_stream_finish', 'gea::node::zlib::streamFinish'], ['__gea_node_zlib_stream_close', 'gea::node::zlib::streamClose'], ['__gea_node_brotli_decompress', 'gea::node::brotli::decompress'], ['__gea_node_brotli_stream_open', 'gea::node::brotli::streamOpen'], ['__gea_node_brotli_stream_write', 'gea::node::brotli::streamWrite'], + ['__gea_node_brotli_stream_leftover', 'gea::node::brotli::streamLeftover'], ['__gea_node_brotli_stream_finish', 'gea::node::brotli::streamFinish'], ['__gea_node_brotli_stream_close', 'gea::node::brotli::streamClose'] ]) diff --git a/runtime/node/gea_node_brotli.hpp b/runtime/node/gea_node_brotli.hpp index 940ddba..74dc17e 100644 --- a/runtime/node/gea_node_brotli.hpp +++ b/runtime/node/gea_node_brotli.hpp @@ -46,6 +46,7 @@ inline std::vector decompress(const Bytes& bytes) { struct Stream { BrotliDecoderState* state = nullptr; bool finished = false; + bool leftover = false; Stream() = default; Stream(const Stream&) = delete; @@ -78,6 +79,7 @@ inline double streamOpen() { inline std::vector streamWrite(double handle, const Bytes& bytes) { Stream& stream = streamAt(handle); std::vector output; + stream.leftover = bytes.size() > 0; if (stream.finished) return output; static const std::uint8_t empty = 0; const std::uint8_t* nextIn = bytes.size() == 0 ? &empty : reinterpret_cast(bytes.data()); @@ -94,9 +96,14 @@ inline std::vector streamWrite(double handle, const Bytes& bytes) } if (result == BROTLI_DECODER_RESULT_ERROR) gea::host::throwRuntimeError("Error", "Decompression failed"); if (result == BROTLI_DECODER_RESULT_SUCCESS) stream.finished = true; + stream.leftover = availableIn > 0; return output; } +inline bool streamLeftover(double handle) { + return streamAt(handle).leftover; +} + inline void streamFinish(double handle) { if (!streamAt(handle).finished) gea::host::throwRuntimeError("Error", "unexpected end of file"); } diff --git a/runtime/node/gea_node_zlib.hpp b/runtime/node/gea_node_zlib.hpp index 65d82a4..8532fd9 100644 --- a/runtime/node/gea_node_zlib.hpp +++ b/runtime/node/gea_node_zlib.hpp @@ -88,7 +88,7 @@ inline std::vector inflate(const Bytes& bytes, double windowBits) if (code == Z_STREAM_END) { // node's Gunzip reads every member of a multi-member gzip file; bytes // after a zlib or raw stream are ignored, as node ignores them. - if (remaining == 0 || (bits != 31 && bits != 47)) break; + if (remaining == 0 || (bits != 31 && bits != 47) || *next == 0) break; if (inflateReset(&stream) != Z_OK) { inflateEnd(&stream); fail(Z_STREAM_ERROR, stream); @@ -145,6 +145,7 @@ struct Stream { bool deflating = false; bool ended = false; bool betweenMembers = false; + bool leftover = false; Stream() = default; Stream(const Stream&) = delete; @@ -211,6 +212,11 @@ inline std::vector inflateInto(Stream& stream, const Bytes& bytes) std::size_t remaining = bytes.size(); Bytef* next = input(bytes); while (remaining > 0 && !stream.ended) { + if (stream.betweenMembers) { + if (*next == 0) break; + if (inflateReset(&stream.z) != Z_OK) fail(Z_STREAM_ERROR, stream.z); + stream.betweenMembers = false; + } const uInt take = static_cast(std::min(remaining, std::numeric_limits::max())); stream.z.next_in = next; stream.z.avail_in = take; @@ -232,12 +238,15 @@ inline std::vector inflateInto(Stream& stream, const Bytes& bytes) stream.ended = true; break; } - if (inflateReset(&stream.z) != Z_OK) fail(Z_STREAM_ERROR, stream.z); stream.betweenMembers = true; continue; } - if (consumed > 0) stream.betweenMembers = false; + if (consumed == 0) { + stream.z.msg = nullptr; + fail(Z_BUF_ERROR, stream.z); + } } + stream.leftover = remaining > 0; return output; } @@ -246,6 +255,10 @@ inline std::vector streamWrite(double handle, const Bytes& bytes) return stream.deflating ? deflateInto(stream, input(bytes), bytes.size(), Z_NO_FLUSH) : inflateInto(stream, bytes); } +inline bool streamLeftover(double handle) { + return streamAt(handle).leftover; +} + inline std::vector streamFinish(double handle) { Stream& stream = streamAt(handle); if (stream.deflating) { diff --git a/runtime/node/zlib.ts b/runtime/node/zlib.ts index 7a2e45d..75520fb 100644 --- a/runtime/node/zlib.ts +++ b/runtime/node/zlib.ts @@ -32,6 +32,8 @@ declare function __gea_node_zlib_stream_open(mode: number, windowBits: number, l /** @gea-host-inert */ declare function __gea_node_zlib_stream_write(handle: number, input: Buffer): Buffer /** @gea-host-inert */ +declare function __gea_node_zlib_stream_leftover(handle: number): boolean +/** @gea-host-inert */ declare function __gea_node_zlib_stream_finish(handle: number): Buffer /** @gea-host-inert */ declare function __gea_node_zlib_stream_close(handle: number): void @@ -40,6 +42,8 @@ declare function __gea_node_brotli_stream_open(): number /** @gea-host-inert */ declare function __gea_node_brotli_stream_write(handle: number, input: Buffer): Buffer /** @gea-host-inert */ +declare function __gea_node_brotli_stream_leftover(handle: number): boolean +/** @gea-host-inert */ declare function __gea_node_brotli_stream_finish(handle: number): void /** @gea-host-inert */ declare function __gea_node_brotli_stream_close(handle: number): void @@ -142,14 +146,17 @@ class ZlibBase extends Transform { override _transform(chunk: Buffer, _encoding: unknown, callback: TransformCallback): void { let output: Buffer + let leftover: boolean try { output = this.brotli_ ? __gea_node_brotli_stream_write(this.handle_, chunk) : __gea_node_zlib_stream_write(this.handle_, chunk) + leftover = this.brotli_ ? __gea_node_brotli_stream_leftover(this.handle_) : __gea_node_zlib_stream_leftover(this.handle_) } catch (error) { this.release_() callback(failureOf(error)) return } if (output.length > 0) this.push(output) + if (leftover) this.push(null) callback(null) } From 7891d0b2825c0655466ca2d6ff954129ee363a79 Mon Sep 17 00:00:00 2001 From: Tamay Eser Uysal Date: Fri, 23 Oct 2026 02:00:18 +0200 Subject: [PATCH 4/6] Return zlib and brotli results as Buffers inflate, deflate, brotli decompress and the stream writes hand their bytes back through bufferResult, so gzipSync, gunzipSync, deflateSync, inflateSync, brotliDecompressSync, the callback forms and stream chunks are what Buffer.isBuffer accepts, as in Node. --- .../correctness/zlib-streams.ts | 22 +++++++++++++++++-- runtime/node/gea_node_brotli.hpp | 11 +++++----- runtime/node/gea_node_zlib.hpp | 19 ++++++++-------- scripts/test-native-brotli.mjs | 7 +++--- scripts/test-native-zlib.mjs | 7 +++--- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/apps/nest-node-probe/correctness/zlib-streams.ts b/apps/nest-node-probe/correctness/zlib-streams.ts index aba29d4..34e585d 100644 --- a/apps/nest-node-probe/correctness/zlib-streams.ts +++ b/apps/nest-node-probe/correctness/zlib-streams.ts @@ -10,7 +10,11 @@ import { createInflate, createInflateRaw, createUnzip, - gunzipSync + deflateSync, + gunzip, + gunzipSync, + gzipSync, + inflateSync } from 'node:zlib' const log = (line: string): void => { @@ -290,7 +294,21 @@ function next(): void { syncGunzip('padded members', GZIP_FIRST + '0000' + GZIP_SECOND) log(`brotliDecompressSync: ${brotliDecompressSync(Buffer.from(BROTLI, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) log(`gunzipSync: ${gunzipSync(Buffer.from(GZIP, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) - log('done') + const plain = Buffer.from(TEXT) + log( + `isBuffer: gzipSync ${Buffer.isBuffer(gzipSync(plain))} gunzipSync ${Buffer.isBuffer(gunzipSync(gzipSync(plain)))} deflateSync ${Buffer.isBuffer(deflateSync(plain))} inflateSync ${Buffer.isBuffer(inflateSync(deflateSync(plain)))} brotliDecompressSync ${Buffer.isBuffer(brotliDecompressSync(Buffer.from(BROTLI, 'hex')))}` + ) + const chunkStream = createGunzip() + chunkStream.once('data', (chunk: Buffer) => { + log(`isBuffer: stream chunk ${Buffer.isBuffer(chunk)}`) + }) + chunkStream.on('close', () => { + gunzip(Buffer.from(GZIP, 'hex'), (error: Error | null, result: Buffer) => { + log(`isBuffer: gunzip callback ${error === null && Buffer.isBuffer(result)}`) + log('done') + }) + }) + chunkStream.end(Buffer.from(GZIP, 'hex')) } log('start') diff --git a/runtime/node/gea_node_brotli.hpp b/runtime/node/gea_node_brotli.hpp index 74dc17e..6bdfa45 100644 --- a/runtime/node/gea_node_brotli.hpp +++ b/runtime/node/gea_node_brotli.hpp @@ -1,6 +1,7 @@ #pragma once #include "gea_runtime.h" +#include "gea_node_buffer.hpp" #include @@ -14,7 +15,7 @@ namespace gea::node::brotli { using Bytes = gea::TypedArray; -inline std::vector decompress(const Bytes& bytes) { +inline gea::Ref> decompress(const Bytes& bytes) { BrotliDecoderState* state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); if (state == nullptr) gea::host::throwRuntimeError("Error", "Cannot allocate memory"); static const std::uint8_t empty = 0; @@ -40,7 +41,7 @@ inline std::vector decompress(const Bytes& bytes) { gea::host::throwRuntimeError("Error", "Decompression failed"); } BrotliDecoderDestroyInstance(state); - return output; + return gea::node::buffer::detail::bufferResult(std::move(output)); } struct Stream { @@ -76,11 +77,11 @@ inline double streamOpen() { return static_cast(last); } -inline std::vector streamWrite(double handle, const Bytes& bytes) { +inline gea::Ref> streamWrite(double handle, const Bytes& bytes) { Stream& stream = streamAt(handle); std::vector output; stream.leftover = bytes.size() > 0; - if (stream.finished) return output; + if (stream.finished) return gea::node::buffer::detail::bufferResult(std::move(output)); static const std::uint8_t empty = 0; const std::uint8_t* nextIn = bytes.size() == 0 ? &empty : reinterpret_cast(bytes.data()); std::size_t availableIn = bytes.size(); @@ -97,7 +98,7 @@ inline std::vector streamWrite(double handle, const Bytes& bytes) if (result == BROTLI_DECODER_RESULT_ERROR) gea::host::throwRuntimeError("Error", "Decompression failed"); if (result == BROTLI_DECODER_RESULT_SUCCESS) stream.finished = true; stream.leftover = availableIn > 0; - return output; + return gea::node::buffer::detail::bufferResult(std::move(output)); } inline bool streamLeftover(double handle) { diff --git a/runtime/node/gea_node_zlib.hpp b/runtime/node/gea_node_zlib.hpp index 8532fd9..c051ae2 100644 --- a/runtime/node/gea_node_zlib.hpp +++ b/runtime/node/gea_node_zlib.hpp @@ -10,6 +10,7 @@ // 31 is gzip (Gzip/Gunzip), -15 is raw deflate (DeflateRaw/InflateRaw), and // 47 detects zlib or gzip from the header (Unzip). #include "gea_runtime.h" +#include "gea_node_buffer.hpp" #include @@ -58,7 +59,7 @@ inline int deflateLevel(int bits, double level) { return static_cast(level); } -inline std::vector inflate(const Bytes& bytes, double windowBits) { +inline gea::Ref> inflate(const Bytes& bytes, double windowBits) { const int bits = framing(windowBits); z_stream stream{}; if (inflateInit2(&stream, bits) != Z_OK) fail(Z_MEM_ERROR, stream); @@ -104,10 +105,10 @@ inline std::vector inflate(const Bytes& bytes, double windowBits) } } inflateEnd(&stream); - return output; + return gea::node::buffer::detail::bufferResult(std::move(output)); } -inline std::vector deflate(const Bytes& bytes, double windowBits, double level) { +inline gea::Ref> deflate(const Bytes& bytes, double windowBits, double level) { const int bits = framing(windowBits); const int compression = deflateLevel(bits, level); z_stream stream{}; @@ -136,7 +137,7 @@ inline std::vector deflate(const Bytes& bytes, double windowBits, } deflateEnd(&stream); output.resize(written); - return output; + return gea::node::buffer::detail::bufferResult(std::move(output)); } struct Stream { @@ -250,26 +251,26 @@ inline std::vector inflateInto(Stream& stream, const Bytes& bytes) return output; } -inline std::vector streamWrite(double handle, const Bytes& bytes) { +inline gea::Ref> streamWrite(double handle, const Bytes& bytes) { Stream& stream = streamAt(handle); - return stream.deflating ? deflateInto(stream, input(bytes), bytes.size(), Z_NO_FLUSH) : inflateInto(stream, bytes); + return gea::node::buffer::detail::bufferResult(stream.deflating ? deflateInto(stream, input(bytes), bytes.size(), Z_NO_FLUSH) : inflateInto(stream, bytes)); } inline bool streamLeftover(double handle) { return streamAt(handle).leftover; } -inline std::vector streamFinish(double handle) { +inline gea::Ref> streamFinish(double handle) { Stream& stream = streamAt(handle); if (stream.deflating) { static Bytef empty = 0; - return deflateInto(stream, &empty, 0, Z_FINISH); + return gea::node::buffer::detail::bufferResult(deflateInto(stream, &empty, 0, Z_FINISH)); } if (!stream.ended && !stream.betweenMembers) { stream.z.msg = nullptr; fail(Z_BUF_ERROR, stream.z); } - return {}; + return gea::node::buffer::detail::bufferResult({}); } inline void streamClose(double handle) { diff --git a/scripts/test-native-brotli.mjs b/scripts/test-native-brotli.mjs index ce9c91f..79e32b2 100644 --- a/scripts/test-native-brotli.mjs +++ b/scripts/test-native-brotli.mjs @@ -47,10 +47,9 @@ gea::TypedArray bytes(const std::string& hex) { std::copy(source->data(), source->data() + source->size(), result.data()); return result; } -std::string hex(const std::vector& source) { - gea::TypedArray value(source.size()); - std::copy(source.begin(), source.end(), value.data()); - return gea::node::buffer::toString(value, "hex"); +std::string hex(const gea::Ref>& source) { + assert(gea::node::buffer::isBuffer(*source)); + return gea::node::buffer::toString(*source, "hex"); } template void throws(F action, const std::string& expected) { bool caught = false; diff --git a/scripts/test-native-zlib.mjs b/scripts/test-native-zlib.mjs index b2f5198..d865239 100644 --- a/scripts/test-native-zlib.mjs +++ b/scripts/test-native-zlib.mjs @@ -60,10 +60,9 @@ gea::TypedArray bytes(const std::string& hex) { std::copy(source->data(), source->data() + source->size(), result.data()); return result; } -std::string hex(const std::vector& source) { - gea::TypedArray value(source.size()); - std::copy(source.begin(), source.end(), value.data()); - return gea::node::buffer::toString(value, "hex"); +std::string hex(const gea::Ref>& source) { + assert(gea::node::buffer::isBuffer(*source)); + return gea::node::buffer::toString(*source, "hex"); } template void throws(F action) { bool caught = false; From 8913772060b19ce943fe8ab102445d0a99363609 Mon Sep 17 00:00:00 2001 From: Tamay Eser Uysal Date: Fri, 23 Oct 2026 07:59:49 +0200 Subject: [PATCH 5/6] Emit zlib and brotli stream output in 16 KiB pieces and stop once destroyed A write was expanded in full before anything was pushed, so a consumer enforcing a decoded-size limit (raw-body, under body-parser) saw the output only after the whole expansion was in memory: 1,681 bytes of brotli became one 1 GiB chunk. Each native read now yields at most 16 KiB and keeps the unread input and pending output in the codec; _transform and _flush push piece by piece until the codec needs input and stop when the stream is destroyed, so a 'data' handler's destroy() ends decoding after its limit, as in Node. --- .../correctness/zlib-streams.ts | 42 ++++++ plugin/index.mjs | 4 + runtime/node/gea_node_brotli.hpp | 48 ++++--- runtime/node/gea_node_zlib.hpp | 127 +++++++++++------- runtime/node/zlib.ts | 65 ++++++--- 5 files changed, 201 insertions(+), 85 deletions(-) diff --git a/apps/nest-node-probe/correctness/zlib-streams.ts b/apps/nest-node-probe/correctness/zlib-streams.ts index 34e585d..10d4bc2 100644 --- a/apps/nest-node-probe/correctness/zlib-streams.ts +++ b/apps/nest-node-probe/correctness/zlib-streams.ts @@ -30,6 +30,7 @@ const GZIP_MEMBERS = '1f8b08000000000000134bcb2c2a2e510000fc7af11c060000001f8b08 const GZIP_FIRST = '1f8b08000000000000134bcb2c2a2e510000fc7af11c06000000' const GZIP_SECOND = '1f8b08000000000000132b4e4dcecf4b010069111fb606000000' const ZEROS = '00000000000000000000000000000000' +const BROTLI_GIB_OF_ZEROS = 'cbffff3ff82700e2b14020f7fe8fffff7ff04f00c4611180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffe3ffff1ffc1300715800a07bffc7ffff3ff82700e2b00040f7fe8fffff7ff04f00c4610180eefd1fffffffe09f0088c30200ddfb3ffeffffc13f0110870500baf77ffcffff837f02200e0b0074effff8ffff07ff04401c1600e8defff1ffff0ffe0980382c00d0bdffcbffff3ffc1300715800a07bff07' interface Case { name: string @@ -213,6 +214,37 @@ function concurrent(done: () => void): void { } } +function bomb(name: string, stream: Transform, payload: Buffer, done: () => void): void { + const limit = 1024 * 1024 + let received = 0 + let largest = 0 + let offset = 0 + stream.on('data', (chunk: Buffer) => { + received += chunk.length + if (chunk.length > largest) largest = chunk.length + if (received >= limit && !stream.destroyed) stream.destroy() + }) + stream.on('error', (error: Error) => { + log(`${name}: error ${error.message}`) + }) + stream.on('close', () => { + log(`${name}: received ${received} largest ${largest}`) + done() + }) + const pump = (): void => { + if (stream.destroyed) return + if (offset >= payload.length) { + stream.end() + return + } + const end = Math.min(offset + 65536, payload.length) + stream.write(payload.subarray(offset, end)) + offset = end + setImmediate(pump) + } + pump() +} + function syncGunzip(name: string, hex: string): void { try { log(`gunzipSync ${name}: ${gunzipSync(Buffer.from(hex, 'hex')).toString().length}`) @@ -288,6 +320,16 @@ function next(): void { concurrent(next) return } + if (at === cases.length + 12) { + at++ + bomb('brotli bomb destroyed at 1 MiB', createBrotliDecompress(), Buffer.from(BROTLI_GIB_OF_ZEROS, 'hex'), next) + return + } + if (at === cases.length + 13) { + at++ + bomb('gunzip bomb destroyed at 1 MiB', createGunzip(), gzipSync(Buffer.alloc(16 * 1024 * 1024)), next) + return + } syncGunzip('zero padding', GZIP + ZEROS) syncGunzip('zero then garbage', GZIP + '00010203') syncGunzip('trailing garbage', GZIP + '010203') diff --git a/plugin/index.mjs b/plugin/index.mjs index b9c72a0..c5efbe7 100644 --- a/plugin/index.mjs +++ b/plugin/index.mjs @@ -291,12 +291,16 @@ const nodeOnlyHostFunctions = new Map([ ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'], ['__gea_node_zlib_stream_open', 'gea::node::zlib::streamOpen'], ['__gea_node_zlib_stream_write', 'gea::node::zlib::streamWrite'], + ['__gea_node_zlib_stream_read', 'gea::node::zlib::streamRead'], + ['__gea_node_zlib_stream_pending', 'gea::node::zlib::streamPending'], ['__gea_node_zlib_stream_leftover', 'gea::node::zlib::streamLeftover'], ['__gea_node_zlib_stream_finish', 'gea::node::zlib::streamFinish'], ['__gea_node_zlib_stream_close', 'gea::node::zlib::streamClose'], ['__gea_node_brotli_decompress', 'gea::node::brotli::decompress'], ['__gea_node_brotli_stream_open', 'gea::node::brotli::streamOpen'], ['__gea_node_brotli_stream_write', 'gea::node::brotli::streamWrite'], + ['__gea_node_brotli_stream_read', 'gea::node::brotli::streamRead'], + ['__gea_node_brotli_stream_pending', 'gea::node::brotli::streamPending'], ['__gea_node_brotli_stream_leftover', 'gea::node::brotli::streamLeftover'], ['__gea_node_brotli_stream_finish', 'gea::node::brotli::streamFinish'], ['__gea_node_brotli_stream_close', 'gea::node::brotli::streamClose'] diff --git a/runtime/node/gea_node_brotli.hpp b/runtime/node/gea_node_brotli.hpp index 6bdfa45..c082e69 100644 --- a/runtime/node/gea_node_brotli.hpp +++ b/runtime/node/gea_node_brotli.hpp @@ -48,6 +48,9 @@ struct Stream { BrotliDecoderState* state = nullptr; bool finished = false; bool leftover = false; + bool draining = false; + std::vector input; + std::size_t offset = 0; Stream() = default; Stream(const Stream&) = delete; @@ -77,30 +80,43 @@ inline double streamOpen() { return static_cast(last); } -inline gea::Ref> streamWrite(double handle, const Bytes& bytes) { +constexpr std::size_t kChunk = 16 * 1024; + +inline void streamWrite(double handle, const Bytes& bytes) { + Stream& stream = streamAt(handle); + const auto* data = reinterpret_cast(bytes.data()); + stream.input.assign(data, data + bytes.size()); + stream.offset = 0; + stream.leftover = stream.finished && bytes.size() > 0; +} + +inline gea::Ref> streamRead(double handle) { Stream& stream = streamAt(handle); std::vector output; - stream.leftover = bytes.size() > 0; if (stream.finished) return gea::node::buffer::detail::bufferResult(std::move(output)); + output.resize(kChunk); static const std::uint8_t empty = 0; - const std::uint8_t* nextIn = bytes.size() == 0 ? &empty : reinterpret_cast(bytes.data()); - std::size_t availableIn = bytes.size(); - constexpr std::size_t chunk = 64 * 1024; - BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; - while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { - const std::size_t used = output.size(); - output.resize(used + chunk); - std::uint8_t* nextOut = output.data() + used; - std::size_t availableOut = chunk; - result = BrotliDecoderDecompressStream(stream.state, &availableIn, &nextIn, &availableOut, &nextOut, nullptr); - output.resize(used + (chunk - availableOut)); - } + const std::uint8_t* nextIn = stream.offset < stream.input.size() ? stream.input.data() + stream.offset : ∅ + std::size_t availableIn = stream.input.size() - stream.offset; + std::uint8_t* nextOut = output.data(); + std::size_t availableOut = kChunk; + const BrotliDecoderResult result = BrotliDecoderDecompressStream(stream.state, &availableIn, &nextIn, &availableOut, &nextOut, nullptr); if (result == BROTLI_DECODER_RESULT_ERROR) gea::host::throwRuntimeError("Error", "Decompression failed"); - if (result == BROTLI_DECODER_RESULT_SUCCESS) stream.finished = true; - stream.leftover = availableIn > 0; + stream.offset = stream.input.size() - availableIn; + stream.draining = result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + if (result == BROTLI_DECODER_RESULT_SUCCESS) { + stream.finished = true; + stream.leftover = availableIn > 0; + } + output.resize(kChunk - availableOut); return gea::node::buffer::detail::bufferResult(std::move(output)); } +inline bool streamPending(double handle) { + const Stream& stream = streamAt(handle); + return !stream.finished && stream.draining; +} + inline bool streamLeftover(double handle) { return streamAt(handle).leftover; } diff --git a/runtime/node/gea_node_zlib.hpp b/runtime/node/gea_node_zlib.hpp index c051ae2..2aea597 100644 --- a/runtime/node/gea_node_zlib.hpp +++ b/runtime/node/gea_node_zlib.hpp @@ -147,6 +147,12 @@ struct Stream { bool ended = false; bool betweenMembers = false; bool leftover = false; + bool stopped = false; + bool draining = false; + bool finishing = false; + bool finished = false; + std::vector input; + std::size_t offset = 0; Stream() = default; Stream(const Stream&) = delete; @@ -184,57 +190,62 @@ inline double streamOpen(double mode, double windowBits, double level) { return static_cast(last); } -inline std::vector deflateInto(Stream& stream, Bytef* next, std::size_t size, int flush) { - std::vector output; - constexpr std::size_t chunk = 64 * 1024; - std::size_t remaining = size; - do { - const uInt take = static_cast(std::min(remaining, std::numeric_limits::max())); - stream.z.next_in = next; - stream.z.avail_in = take; - do { - const std::size_t used = output.size(); - output.resize(used + chunk); - stream.z.next_out = reinterpret_cast(output.data() + used); - stream.z.avail_out = static_cast(chunk); - const int code = ::deflate(&stream.z, remaining == take ? flush : Z_NO_FLUSH); - output.resize(used + (chunk - stream.z.avail_out)); - if (code == Z_STREAM_ERROR) fail(code, stream.z); - } while (stream.z.avail_out == 0); - next += take - stream.z.avail_in; - remaining -= take - stream.z.avail_in; - } while (remaining > 0); +constexpr std::size_t kChunk = 16 * 1024; + +inline std::size_t remainingOf(const Stream& stream) { + return stream.input.size() - stream.offset; +} + +inline Bytef* nextOf(Stream& stream) { + static Bytef empty = 0; + return stream.offset < stream.input.size() ? stream.input.data() + stream.offset : ∅ +} + +inline std::vector deflateStep(Stream& stream, int flush) { + std::vector output(kChunk); + const std::size_t remaining = remainingOf(stream); + const std::size_t take = std::min(remaining, std::numeric_limits::max()); + stream.z.next_in = nextOf(stream); + stream.z.avail_in = static_cast(take); + stream.z.next_out = reinterpret_cast(output.data()); + stream.z.avail_out = static_cast(kChunk); + const int code = ::deflate(&stream.z, take == remaining ? flush : Z_NO_FLUSH); + if (code == Z_STREAM_ERROR) fail(code, stream.z); + stream.offset += take - stream.z.avail_in; + stream.draining = stream.z.avail_out == 0; + if (code == Z_STREAM_END) stream.finished = true; + output.resize(kChunk - stream.z.avail_out); return output; } -inline std::vector inflateInto(Stream& stream, const Bytes& bytes) { - std::vector output; - constexpr std::size_t chunk = 64 * 1024; - std::size_t remaining = bytes.size(); - Bytef* next = input(bytes); - while (remaining > 0 && !stream.ended) { +inline std::vector inflateStep(Stream& stream) { + std::vector output(kChunk); + std::size_t used = 0; + while (used < kChunk && !stream.ended && !stream.stopped) { + const std::size_t remaining = remainingOf(stream); + if (remaining == 0 && !stream.draining) break; if (stream.betweenMembers) { - if (*next == 0) break; + if (*nextOf(stream) == 0) { + stream.stopped = true; + break; + } if (inflateReset(&stream.z) != Z_OK) fail(Z_STREAM_ERROR, stream.z); stream.betweenMembers = false; } - const uInt take = static_cast(std::min(remaining, std::numeric_limits::max())); - stream.z.next_in = next; - stream.z.avail_in = take; - int code = Z_OK; - do { - const std::size_t used = output.size(); - output.resize(used + chunk); - stream.z.next_out = reinterpret_cast(output.data() + used); - stream.z.avail_out = static_cast(chunk); - code = ::inflate(&stream.z, Z_NO_FLUSH); - output.resize(used + (chunk - stream.z.avail_out)); - if (code != Z_OK && code != Z_STREAM_END && code != Z_BUF_ERROR) fail(code, stream.z); - } while (code == Z_OK && stream.z.avail_out == 0); + const std::size_t take = std::min(remaining, std::numeric_limits::max()); + stream.z.next_in = nextOf(stream); + stream.z.avail_in = static_cast(take); + stream.z.next_out = reinterpret_cast(output.data() + used); + stream.z.avail_out = static_cast(kChunk - used); + const int code = ::inflate(&stream.z, Z_NO_FLUSH); + if (code != Z_OK && code != Z_STREAM_END && code != Z_BUF_ERROR) fail(code, stream.z); const std::size_t consumed = take - stream.z.avail_in; - next += consumed; - remaining -= consumed; + const std::size_t produced = (kChunk - used) - stream.z.avail_out; + stream.offset += consumed; + used += produced; + stream.draining = stream.z.avail_out == 0; if (code == Z_STREAM_END) { + stream.draining = false; if (stream.bits != 31 && stream.bits != 47) { stream.ended = true; break; @@ -242,18 +253,36 @@ inline std::vector inflateInto(Stream& stream, const Bytes& bytes) stream.betweenMembers = true; continue; } - if (consumed == 0) { + if (consumed == 0 && produced == 0) { + if (remaining == 0) break; stream.z.msg = nullptr; fail(Z_BUF_ERROR, stream.z); } } - stream.leftover = remaining > 0; + if (stream.ended || stream.stopped) stream.leftover = remainingOf(stream) > 0; + output.resize(used); return output; } -inline gea::Ref> streamWrite(double handle, const Bytes& bytes) { +inline void streamWrite(double handle, const Bytes& bytes) { + Stream& stream = streamAt(handle); + const auto* data = reinterpret_cast(bytes.data()); + stream.input.assign(data, data + bytes.size()); + stream.offset = 0; + stream.stopped = false; + stream.leftover = false; +} + +inline gea::Ref> streamRead(double handle) { Stream& stream = streamAt(handle); - return gea::node::buffer::detail::bufferResult(stream.deflating ? deflateInto(stream, input(bytes), bytes.size(), Z_NO_FLUSH) : inflateInto(stream, bytes)); + return gea::node::buffer::detail::bufferResult(stream.deflating ? deflateStep(stream, Z_NO_FLUSH) : inflateStep(stream)); +} + +inline bool streamPending(double handle) { + const Stream& stream = streamAt(handle); + if (stream.finishing) return stream.deflating && !stream.finished; + if (stream.deflating) return stream.draining || remainingOf(stream) > 0; + return !stream.ended && !stream.stopped && (stream.draining || remainingOf(stream) > 0); } inline bool streamLeftover(double handle) { @@ -262,10 +291,8 @@ inline bool streamLeftover(double handle) { inline gea::Ref> streamFinish(double handle) { Stream& stream = streamAt(handle); - if (stream.deflating) { - static Bytef empty = 0; - return gea::node::buffer::detail::bufferResult(deflateInto(stream, &empty, 0, Z_FINISH)); - } + stream.finishing = true; + if (stream.deflating) return gea::node::buffer::detail::bufferResult(deflateStep(stream, Z_FINISH)); if (!stream.ended && !stream.betweenMembers) { stream.z.msg = nullptr; fail(Z_BUF_ERROR, stream.z); diff --git a/runtime/node/zlib.ts b/runtime/node/zlib.ts index 75520fb..555c73b 100644 --- a/runtime/node/zlib.ts +++ b/runtime/node/zlib.ts @@ -30,7 +30,11 @@ declare function __gea_node_brotli_decompress(input: Buffer): Buffer /** @gea-host-inert */ declare function __gea_node_zlib_stream_open(mode: number, windowBits: number, level: number): number /** @gea-host-inert */ -declare function __gea_node_zlib_stream_write(handle: number, input: Buffer): Buffer +declare function __gea_node_zlib_stream_write(handle: number, input: Buffer): void +/** @gea-host-inert */ +declare function __gea_node_zlib_stream_read(handle: number): Buffer +/** @gea-host-inert */ +declare function __gea_node_zlib_stream_pending(handle: number): boolean /** @gea-host-inert */ declare function __gea_node_zlib_stream_leftover(handle: number): boolean /** @gea-host-inert */ @@ -40,7 +44,11 @@ declare function __gea_node_zlib_stream_close(handle: number): void /** @gea-host-inert */ declare function __gea_node_brotli_stream_open(): number /** @gea-host-inert */ -declare function __gea_node_brotli_stream_write(handle: number, input: Buffer): Buffer +declare function __gea_node_brotli_stream_write(handle: number, input: Buffer): void +/** @gea-host-inert */ +declare function __gea_node_brotli_stream_read(handle: number): Buffer +/** @gea-host-inert */ +declare function __gea_node_brotli_stream_pending(handle: number): boolean /** @gea-host-inert */ declare function __gea_node_brotli_stream_leftover(handle: number): boolean /** @gea-host-inert */ @@ -145,35 +153,54 @@ class ZlibBase extends Transform { } override _transform(chunk: Buffer, _encoding: unknown, callback: TransformCallback): void { - let output: Buffer - let leftover: boolean try { - output = this.brotli_ ? __gea_node_brotli_stream_write(this.handle_, chunk) : __gea_node_zlib_stream_write(this.handle_, chunk) - leftover = this.brotli_ ? __gea_node_brotli_stream_leftover(this.handle_) : __gea_node_zlib_stream_leftover(this.handle_) + if (this.brotli_) __gea_node_brotli_stream_write(this.handle_, chunk) + else __gea_node_zlib_stream_write(this.handle_, chunk) } catch (error) { this.release_() callback(failureOf(error)) return } - if (output.length > 0) this.push(output) - if (leftover) this.push(null) - callback(null) + this.drain_(false, callback) } override _flush(callback: TransformCallback): void { - let output: Buffer - try { - if (this.brotli_) { - __gea_node_brotli_stream_finish(this.handle_) - output = Buffer.alloc(0) - } else output = __gea_node_zlib_stream_finish(this.handle_) - } catch (error) { + this.drain_(true, callback) + } + + private drain_(finishing: boolean, callback: TransformCallback): void { + let more = true + while (more) { + let piece: Buffer + try { + piece = this.step_(finishing) + more = this.brotli_ ? __gea_node_brotli_stream_pending(this.handle_) : __gea_node_zlib_stream_pending(this.handle_) + } catch (error) { + this.release_() + callback(failureOf(error)) + return + } + if (piece.length > 0) this.push(piece) + if (this.destroyed) { + callback(null) + return + } + } + if (finishing) { this.release_() - callback(failureOf(error)) + callback(null) return } - this.release_() - callback(null, output.length > 0 ? output : undefined) + const leftover = this.brotli_ ? __gea_node_brotli_stream_leftover(this.handle_) : __gea_node_zlib_stream_leftover(this.handle_) + if (leftover) this.push(null) + callback(null) + } + + private step_(finishing: boolean): Buffer { + if (!this.brotli_) return finishing ? __gea_node_zlib_stream_finish(this.handle_) : __gea_node_zlib_stream_read(this.handle_) + if (!finishing) return __gea_node_brotli_stream_read(this.handle_) + __gea_node_brotli_stream_finish(this.handle_) + return Buffer.alloc(0) } override _destroy(error: Error | null, callback: DestroyCallback): void { From dfcbcc1bce12f2224ce3186a54320c5c9b7760d4 Mon Sep 17 00:00:00 2001 From: Tamay Eser Uysal Date: Fri, 23 Oct 2026 08:02:00 +0200 Subject: [PATCH 6/6] Share the brotli and vcpkg library lookup between build.mjs and the brotli test test-native-brotli.mjs repeated brotliPrefix() and, on Windows, accepted a vcpkg triplet directory without the lib/ check build.mjs makes. Both now import windowsLibraries() and brotliPrefix() from scripts/native-libraries.mjs. --- scripts/build.mjs | 43 +------------------------------- scripts/native-libraries.mjs | 45 ++++++++++++++++++++++++++++++++++ scripts/test-native-brotli.mjs | 12 ++------- 3 files changed, 48 insertions(+), 52 deletions(-) create mode 100644 scripts/native-libraries.mjs diff --git a/scripts/build.mjs b/scripts/build.mjs index 939abc2..ebe49e3 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -53,6 +53,7 @@ import { resolveBuiltinModules, runtimeRootsForReachableGlobalNeeds } from './bu import { displayPathFrom } from './path-display.mjs' import { compilerRoot as resolveCompilerRoot, compilerModuleUrl } from './resolve-compiler.mjs' import { compilerInputs, assertCompilerInputs } from './compiler-input.mjs' +import { brotliPrefix, windowsLibraries } from './native-libraries.mjs' const compilerRoot = resolveCompilerRoot() const { compile } = await import(compilerModuleUrl('.', compilerRoot)) @@ -1166,54 +1167,12 @@ function thinLtoLinkFlags() { return ['-fuse-ld=gold', `-Wl,-plugin-opt,jobs=${jobs}`, `-Wl,-plugin-opt,cache-dir=${cache}`] } -// The native libraries a Windows build links (zlib for node:zlib) come from -// vcpkg's static triplet -- static libraries over the static CRT, what a plain -// clang++ link uses -- found under VCPKG_ROOT, else C:\vcpkg, the same way the -// compiler's release checks find them (its scripts/host-toolchain.mjs). -// Install with `vcpkg install zlib --triplet x64-windows-static`. Null -// elsewhere: Linux and macOS link the system's libraries. -function windowsLibraries() { - if (process.platform !== 'win32') return null - for (const root of [process.env.VCPKG_ROOT, 'C:\\vcpkg'].filter(Boolean)) { - const installed = path.join(root, 'installed', 'x64-windows-static') - if (fs.existsSync(path.join(installed, 'lib'))) return installed - } - return null -} - // zlib is libz everywhere but vcpkg's static Windows build, which names it zs.lib. function zlibLinkFlags() { const windows = windowsLibraries() return windows ? [`-L${path.join(windows, 'lib')}`, '-lzs'] : ['-lz'] } -function brotliPrefix() { - if (brotliPrefix.cache !== undefined) return brotliPrefix.cache - const candidates = [] - if (process.env.BROTLI_DIR) candidates.push(path.resolve(process.env.BROTLI_DIR)) - else if (process.platform === 'win32') { - if (windowsLibraries()) candidates.push(windowsLibraries()) - } else if (process.platform === 'darwin') { - try { - candidates.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) - } catch {} - candidates.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') - } - brotliPrefix.cache = candidates.find((prefix) => fs.existsSync(path.join(prefix, 'include', 'brotli', 'decode.h'))) ?? null - if (brotliPrefix.cache === null && (process.platform !== 'linux' || process.env.BROTLI_DIR)) { - const install = - process.platform === 'darwin' - ? 'Install with `brew install brotli`' - : process.platform === 'win32' - ? 'Install with `vcpkg install brotli --triplet x64-windows-static`' - : 'Install libbrotli-dev (Debian/Ubuntu) or brotli-devel (Fedora)' - throw new Error( - `brotli (libbrotlidec) not found: node:zlib's brotli decoder needs its headers and library. ${install}, or set BROTLI_DIR to a prefix holding include/brotli/decode.h and lib/.` - ) - } - return brotliPrefix.cache -} - function brotliVersionIsPatched(version) { const major = version >>> 24 const minor = (version >>> 12) & 0xfff diff --git a/scripts/native-libraries.mjs b/scripts/native-libraries.mjs new file mode 100644 index 0000000..530c979 --- /dev/null +++ b/scripts/native-libraries.mjs @@ -0,0 +1,45 @@ +import fs from 'node:fs' +import path from 'node:path' +import { execFileSync } from 'node:child_process' + +// The native libraries a Windows build links (zlib for node:zlib) come from +// vcpkg's static triplet -- static libraries over the static CRT, what a plain +// clang++ link uses -- found under VCPKG_ROOT, else C:\vcpkg, the same way the +// compiler's release checks find them (its scripts/host-toolchain.mjs). +// Install with `vcpkg install zlib --triplet x64-windows-static`. Null +// elsewhere: Linux and macOS link the system's libraries. +export function windowsLibraries() { + if (process.platform !== 'win32') return null + for (const root of [process.env.VCPKG_ROOT, 'C:\\vcpkg'].filter(Boolean)) { + const installed = path.join(root, 'installed', 'x64-windows-static') + if (fs.existsSync(path.join(installed, 'lib'))) return installed + } + return null +} + +export function brotliPrefix() { + if (brotliPrefix.cache !== undefined) return brotliPrefix.cache + const candidates = [] + if (process.env.BROTLI_DIR) candidates.push(path.resolve(process.env.BROTLI_DIR)) + else if (process.platform === 'win32') { + if (windowsLibraries()) candidates.push(windowsLibraries()) + } else if (process.platform === 'darwin') { + try { + candidates.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) + } catch {} + candidates.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') + } + brotliPrefix.cache = candidates.find((prefix) => fs.existsSync(path.join(prefix, 'include', 'brotli', 'decode.h'))) ?? null + if (brotliPrefix.cache === null && (process.platform !== 'linux' || process.env.BROTLI_DIR)) { + const install = + process.platform === 'darwin' + ? 'Install with `brew install brotli`' + : process.platform === 'win32' + ? 'Install with `vcpkg install brotli --triplet x64-windows-static`' + : 'Install libbrotli-dev (Debian/Ubuntu) or brotli-devel (Fedora)' + throw new Error( + `brotli (libbrotlidec) not found: node:zlib's brotli decoder needs its headers and library. ${install}, or set BROTLI_DIR to a prefix holding include/brotli/decode.h and lib/.` + ) + } + return brotliPrefix.cache +} diff --git a/scripts/test-native-brotli.mjs b/scripts/test-native-brotli.mjs index 79e32b2..16c75d7 100644 --- a/scripts/test-native-brotli.mjs +++ b/scripts/test-native-brotli.mjs @@ -4,6 +4,7 @@ import { execFileSync } from 'node:child_process' import { existsSync, mkdirSync } from 'node:fs' import { dirname, resolve } from 'node:path' +import { brotliPrefix } from './native-libraries.mjs' import { compilerRuntimeInclude } from './resolve-compiler.mjs' const root = resolve(import.meta.dirname, '..') @@ -20,16 +21,7 @@ const operations = inputs.map((input) => `std::cout << hex(gea::node::brotli::de const trailing = Buffer.concat([brotliCompressSync(inputs[1]), Buffer.from('xyz')]) operations.push(`std::cout << hex(gea::node::brotli::decompress(${bytes(trailing)})) << '\\n';`) const truncated = brotliCompressSync(inputs[3]).subarray(0, 40) -const prefixes = [] -if (process.env.BROTLI_DIR) prefixes.push(resolve(process.env.BROTLI_DIR)) -else if (process.platform === 'win32') prefixes.push(...[process.env.VCPKG_ROOT, 'C:\\vcpkg'].filter(Boolean).map((r) => resolve(r, 'installed', 'x64-windows-static'))) -else if (process.platform === 'darwin') { - try { - prefixes.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) - } catch {} - prefixes.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') -} -const prefix = prefixes.find((p) => existsSync(resolve(p, 'include', 'brotli', 'decode.h'))) +const prefix = brotliPrefix() const brotliFlags = [ ...(prefix ? [`-I${resolve(prefix, 'include')}`, `-L${resolve(prefix, 'lib')}`] : []), ...(prefix && process.platform !== 'win32' ? [`-Wl,-rpath,${resolve(prefix, 'lib')}`] : []),