From 98b3eec35c732657821a5d65e98093f789e453dd Mon Sep 17 00:00:00 2001 From: Nathaniel Chestnut Date: Mon, 27 Jul 2026 13:27:25 -0700 Subject: [PATCH] feat(bgcode): heatshrink decompression (DD-011 phase 3) (#188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds heatshrink (LZSS) decompression — windows 11 & 12, lookahead 4 — to the .bgcode decoder. With this, ALL four spec compression codecs (None/DEFLATE/heatshrink-11/heatshrink-12) and all encodings (None/MeatPack/MeatPack-comments) are decoded; a heatshrink-compressed GCode block now decodes end-to-end to plain G-code. The decoder is a TS port of the LZSS decoder from the ISC atomicobject/heatshrink (© 2013-2015 Scott Vokes), attribution preserved — no AGPL libbgcode (RR-003 §8). Wire format: MSB-first bit stream; tag bit (1=literal 8 bits; 0=backref of windowBits index + lookaheadBits count, each value-1); a backref copies count+1 bytes from index+1 back in a circular window, re-emitting each (self-referential runs supported). Validated against vectors built by an INDEPENDENT MSB-first bit-packer from the spec (literals, single + multi-byte backrefs, offset-2 interleaving, self-referential runs, window 11 & 12, a realistic repeated g-code fragment) + block-level integration through openBgcode. Output bounded (bomb defense). The stale phase-1 "heatshrink unsupported" test → an unknown-compression-id rejection. Real PrusaSlicer heatshrink files = phase-4 golden-equivalence. Co-Authored-By: Claude Opus 4.8 --- .changeset/bgcode-heatshrink-phase3.md | 15 +++ packages/gcode-bgcode/README.md | 7 +- .../gcode-bgcode/src/__tests__/assemble.ts | 15 ++- .../gcode-bgcode/src/__tests__/bgcode.test.ts | 9 +- .../src/__tests__/heatshrink.test.ts | 91 +++++++++++++++++++ packages/gcode-bgcode/src/bgcode.ts | 11 ++- packages/gcode-bgcode/src/heatshrink.ts | 79 ++++++++++++++++ packages/gcode-bgcode/src/index.ts | 5 +- tools/pack-check-snapshots.json | 3 + 9 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 .changeset/bgcode-heatshrink-phase3.md create mode 100644 packages/gcode-bgcode/src/__tests__/heatshrink.test.ts create mode 100644 packages/gcode-bgcode/src/heatshrink.ts diff --git a/.changeset/bgcode-heatshrink-phase3.md b/.changeset/bgcode-heatshrink-phase3.md new file mode 100644 index 00000000..4856ac25 --- /dev/null +++ b/.changeset/bgcode-heatshrink-phase3.md @@ -0,0 +1,15 @@ +--- +'@chestnutlabs/gcode-bgcode': minor +--- + +Binary G-code decode **phase 3** (DD-011, #188): **heatshrink** decompression (windows 11 & 12, +lookahead 4). With this, **all four `.bgcode` compression codecs and all encodings the spec defines +are decoded** — a `.bgcode` GCode block compressed with heatshrink now decodes end-to-end to plain +G-code. + +The decoder is a TypeScript port of the LZSS decoder from the **ISC** `atomicobject/heatshrink` +(© 2013–2015 Scott Vokes) — attribution preserved, no AGPL `libbgcode` (RR-003 §8). It is validated +against vectors built by an **independent MSB-first bit-packer** from the wire format (literal, single +and multi-byte back-references, self-referential runs, window 11 & 12, a realistic repeated G-code +fragment) plus block-level integration through `openBgcode`. Output is bounded (decompression-bomb +defense). Container-adapter integration + real PrusaSlicer-file golden-equivalence follow in phase 4. diff --git a/packages/gcode-bgcode/README.md b/packages/gcode-bgcode/README.md index 6cf3027a..b9c8481b 100644 --- a/packages/gcode-bgcode/README.md +++ b/packages/gcode-bgcode/README.md @@ -14,8 +14,9 @@ network), zero runtime dependencies beyond `@chestnutlabs/toolpath-core` and The reference `libbgcode` and OctoPrint-MeatPack are **AGPL-3.0**; none of their code is used here — the block walker is written clean-room from the published format spec ([RR-003](https://github.com/ChestnutLabs/gcode-preview/blob/dev/docs/research/RR-003-bgcode-licensing-and-format-audit.md)). The **MeatPack** decoder is a faithful port of the **MIT** [`jamesgopsill/meatpack`](https://github.com/jamesgopsill/meatpack) -(© 2025 James Gopsill), attribution preserved; the heatshrink codec (phase 3) is a license-clean port of -the **ISC** heatshrink library. This package is MIT. +(© 2025 James Gopsill), and the **heatshrink** decoder is a port of the **ISC** +[`atomicobject/heatshrink`](https://github.com/atomicobject/heatshrink) (© 2013–2015 Scott Vokes) — +both attributions preserved in their source files. This package is MIT. ## Status — phased (DD-011 §14) @@ -25,7 +26,7 @@ the **ISC** heatshrink library. This package is MIT. | Compression: None, DEFLATE | ✅ phase 1 | | Encoding: None | ✅ phase 1 | | Encoding: MeatPack (both variants) | ✅ phase 2 | -| Compression: heatshrink 11/12 | ⏳ phase 3 (honest `E_BGCODE_UNSUPPORTED_COMPRESSION` until then) | +| Compression: heatshrink 11/12 | ✅ phase 3 | | Container-adapter + worker registration + metadata/thumbnails + golden-equivalence | ⏳ phase 4 | ## Usage diff --git a/packages/gcode-bgcode/src/__tests__/assemble.ts b/packages/gcode-bgcode/src/__tests__/assemble.ts index 6a454b1f..6207045d 100644 --- a/packages/gcode-bgcode/src/__tests__/assemble.ts +++ b/packages/gcode-bgcode/src/__tests__/assemble.ts @@ -16,6 +16,12 @@ export interface AssembleBlock { thumbnail?: { format: number; width: number; height: number }; /** The uncompressed block payload. */ data: Uint8Array; + /** + * Pre-compressed stored bytes + the declared uncompressed size, for codecs the assembler can't + * produce (heatshrink) — the test supplies a hand-computed stream and its decoded length. When set, + * `data` is ignored for the payload and `compression` is written as declared (not re-compressed). + */ + preCompressed?: { stored: Uint8Array; uncompressedSize: number }; } export interface AssembleOptions { @@ -70,11 +76,16 @@ export async function assembleBgcode(blocks: AssembleBlock[], opts: AssembleOpti pushU16(out, checksum === 'crc32' ? 1 : 0); for (const b of blocks) { - const stored = b.compression === BgcodeCompression.Deflate ? await deflateRaw(b.data) : b.data; + const stored = b.preCompressed + ? b.preCompressed.stored + : b.compression === BgcodeCompression.Deflate + ? await deflateRaw(b.data) + : b.data; + const uncompressedSize = b.preCompressed ? b.preCompressed.uncompressedSize : b.data.length; const block: number[] = []; pushU16(block, b.type); pushU16(block, b.compression); - pushU32(block, b.data.length); // uncompressed size + pushU32(block, uncompressedSize); // uncompressed size if (b.compression !== BgcodeCompression.None) pushU32(block, stored.length); // compressed size // Parameters. if (b.type === BgcodeBlockType.Thumbnail) { diff --git a/packages/gcode-bgcode/src/__tests__/bgcode.test.ts b/packages/gcode-bgcode/src/__tests__/bgcode.test.ts index d3a9db18..675306ff 100644 --- a/packages/gcode-bgcode/src/__tests__/bgcode.test.ts +++ b/packages/gcode-bgcode/src/__tests__/bgcode.test.ts @@ -114,11 +114,10 @@ describe('openBgcode — honest rejection & bounds', () => { await expect(openBgcode(buf.subarray(0, 14))).rejects.toMatchObject({ code: 'E_BGCODE_TRUNCATED' }); }); - it('honestly reports heatshrink compression (phase 3) as not-yet-supported', async () => { - const hs = await assembleBgcode([ - { type: 1, compression: BgcodeCompression.Heatshrink12, encoding: 0, data: enc(GCODE) } - ]); - await expect(openBgcode(hs)).rejects.toMatchObject({ code: 'E_BGCODE_UNSUPPORTED_COMPRESSION' }); + it('honestly rejects an unknown compression id', async () => { + // compression id 7 is outside the spec enum (0..3) — bounded structured error, not a guess. + const bad = await assembleBgcode([{ type: 1, compression: 7, encoding: 0, data: enc(GCODE) }]); + await expect(openBgcode(bad)).rejects.toMatchObject({ code: 'E_BGCODE_UNSUPPORTED_COMPRESSION' }); }); it('enforces the output-bytes cap (decompression-bomb defense)', async () => { diff --git a/packages/gcode-bgcode/src/__tests__/heatshrink.test.ts b/packages/gcode-bgcode/src/__tests__/heatshrink.test.ts new file mode 100644 index 00000000..0f4f903d --- /dev/null +++ b/packages/gcode-bgcode/src/__tests__/heatshrink.test.ts @@ -0,0 +1,91 @@ +/** + * heatshrink decoder — phase 3 (DD-011, #188). Validated against vectors built by an INDEPENDENT + * MSB-first bit-packer (`packBits`) from the wire format described in the spec: a tag bit (1=literal + * 8 bits; 0=backref of `windowBits` index + `lookaheadBits` count, both value−1). The packer is a + * separate code path from the decoder's bit reader, so `decode(pack(tokens)) === expected` validates + * the decoder against the spec, not against itself. Real PrusaSlicer heatshrink blocks → phase 4. + */ +import { describe, expect, it } from 'vitest'; +import { heatshrinkDecode, openBgcode, BgcodeBlockType, BgcodeCompression, BgcodeEncoding } from '../index.js'; +import { assembleBgcode } from './assemble.js'; + +const dec = (b: Uint8Array): string => new TextDecoder().decode(b); + +/** Pack [value, bitCount] fields MSB-first into bytes (independent of the decoder's reader). */ +function packBits(fields: Array<[number, number]>): Uint8Array { + const bits: number[] = []; + for (const [val, n] of fields) for (let i = n - 1; i >= 0; i--) bits.push((val >> i) & 1); + const bytes = new Uint8Array(Math.ceil(bits.length / 8)); + for (let i = 0; i < bits.length; i++) if (bits[i]) bytes[i >> 3] |= 0x80 >> (i & 7); + return bytes; +} + +const LIT = (c: string): Array<[number, number]> => [ + [1, 1], + [c.charCodeAt(0), 8] +]; +/** A back-reference token: copy `count` bytes from `offset` back, in a `windowBits` window. */ +const BACKREF = (offset: number, count: number, windowBits: number): Array<[number, number]> => [ + [0, 1], + [offset - 1, windowBits], + [count - 1, 4] +]; + +describe('heatshrinkDecode — spec vectors (independent bit-packer)', () => { + it('literals decode to their bytes', () => { + expect(dec(heatshrinkDecode(packBits([...LIT('A'), ...LIT('B')]), 11, 4, 1024))).toBe('AB'); + }); + + it('a back-reference copies from the window (window 11)', () => { + // "A", then copy 3 bytes from 1 back → "AAAA". + const v = packBits([...LIT('A'), ...BACKREF(1, 3, 11)]); + expect(dec(heatshrinkDecode(v, 11, 4, 1024))).toBe('AAAA'); + }); + + it('a back-reference with a 12-bit window index', () => { + const v = packBits([...LIT('A'), ...BACKREF(1, 3, 12)]); + expect(dec(heatshrinkDecode(v, 12, 4, 1024))).toBe('AAAA'); + }); + + it('a self-referential run (copy length exceeds the offset) expands correctly', () => { + // "A", then copy 5 from 1 back → "AAAAAA". + const v = packBits([...LIT('A'), ...BACKREF(1, 5, 11)]); + expect(dec(heatshrinkDecode(v, 11, 4, 1024))).toBe('AAAAAA'); + }); + + it('a multi-byte back-reference at offset 2 interleaves correctly', () => { + // "AB", then copy 4 from 2 back → "ABAB" → total "ABABAB". + const v = packBits([...LIT('A'), ...LIT('B'), ...BACKREF(2, 4, 11)]); + expect(dec(heatshrinkDecode(v, 11, 4, 1024))).toBe('ABABAB'); + }); + + it('decodes a realistic repeated G-code fragment', () => { + // "G1 X" then a back-reference to repeat "G1 X" → "G1 XG1 X". + const v = packBits([...LIT('G'), ...LIT('1'), ...LIT(' '), ...LIT('X'), ...BACKREF(4, 4, 11)]); + expect(dec(heatshrinkDecode(v, 11, 4, 1024))).toBe('G1 XG1 X'); + }); + + it('enforces the output cap (decompression-bomb defense)', () => { + // count 16 is the 4-bit-lookahead maximum; 'A' + a 16-byte copy = 17 bytes > the 4-byte cap. + const v = packBits([...LIT('A'), ...BACKREF(1, 16, 11)]); + expect(() => heatshrinkDecode(v, 11, 4, 4)).toThrow(/limit/); + }); +}); + +describe('openBgcode — heatshrink-compressed block (phase 3 integration)', () => { + it('decodes a Heatshrink12 GCode block end-to-end', async () => { + const stored = packBits([...LIT('A'), ...BACKREF(1, 3, 12)]); // → "AAAA" + const buf = await assembleBgcode([ + { + type: BgcodeBlockType.GCode, + compression: BgcodeCompression.Heatshrink12, + encoding: BgcodeEncoding.None, + data: new Uint8Array(), + preCompressed: { stored, uncompressedSize: 4 } + } + ]); + const { gcode, blocks } = await openBgcode(buf); + expect(dec(gcode)).toBe('AAAA'); + expect(blocks[0].compression).toBe(BgcodeCompression.Heatshrink12); + }); +}); diff --git a/packages/gcode-bgcode/src/bgcode.ts b/packages/gcode-bgcode/src/bgcode.ts index cb0fdc89..2c2fa757 100644 --- a/packages/gcode-bgcode/src/bgcode.ts +++ b/packages/gcode-bgcode/src/bgcode.ts @@ -14,6 +14,7 @@ */ import { ContainerError, crc32, crc32Final } from '@chestnutlabs/gcode-containers'; import { meatpackDecode } from './meatpack.js'; +import { heatshrinkDecode } from './heatshrink.js'; /** ASCII magic at the start of every `.bgcode` file: "GCDE". */ export const BGCODE_MAGIC = 'GCDE'; @@ -157,10 +158,12 @@ async function decompress( return out; } if (compression === BgcodeCompression.Heatshrink11 || compression === BgcodeCompression.Heatshrink12) { - throw new ContainerError( - 'E_BGCODE_UNSUPPORTED_COMPRESSION', - 'heatshrink compression is not yet supported (DD-011 phase 3)' - ); + const windowBits = compression === BgcodeCompression.Heatshrink11 ? 11 : 12; + const out = heatshrinkDecode(data, windowBits, 4, limit); + if (out.length !== uncompressed) { + throw new ContainerError('E_BGCODE_SIZE', `heatshrink size disagreement (${out.length} vs ${uncompressed})`); + } + return out; } throw new ContainerError('E_BGCODE_UNSUPPORTED_COMPRESSION', `unknown compression id ${compression}`); } diff --git a/packages/gcode-bgcode/src/heatshrink.ts b/packages/gcode-bgcode/src/heatshrink.ts new file mode 100644 index 00000000..5f1c5199 --- /dev/null +++ b/packages/gcode-bgcode/src/heatshrink.ts @@ -0,0 +1,79 @@ +/** + * heatshrink LZSS decoder (DD-011 phase 3, #188). + * + * A TypeScript port of the decoder from **atomicobject/heatshrink** (ISC, © 2013–2015 Scott Vokes — + * https://github.com/atomicobject/heatshrink). ISC is permissive and MIT-compatible; porting with the + * attribution preserved is the sanctioned path (RR-003 §8) — the AGPL `libbgcode` is never copied. + * + * Wire format (whole-buffer form of the reference's streaming state machine): a big-endian (MSB-first) + * bit stream of tokens. Each token starts with a tag bit — `1` = an 8-bit literal byte; `0` = a + * back-reference: `windowBits` of index then `lookaheadBits` of count (both stored as value−1). A + * back-reference copies `count+1` bytes from `index+1` bytes back in a circular window, re-emitting + * each into the window (so runs can reference themselves). bgcode uses window 11 or 12, lookahead 4. + */ +import { ContainerError } from '@chestnutlabs/gcode-containers'; + +/** Decode a heatshrink stream. `windowBits`/`lookaheadBits` come from the bgcode compression id. */ +export function heatshrinkDecode( + data: Uint8Array, + windowBits: number, + lookaheadBits: number, + limit: number +): Uint8Array { + const windowSize = 1 << windowBits; + const mask = windowSize - 1; + const window = new Uint8Array(windowSize); + let head = 0; // write cursor into the circular window (monotonic; masked on access) + + let out = new Uint8Array(Math.max(64, Math.min(limit, data.length * 4))); + let pos = 0; + const emit = (c: number): void => { + if (pos >= limit) throw new ContainerError('E_BGCODE_BOMB', `heatshrink output exceeds the limit (${pos})`); + if (pos >= out.length) { + const bigger = new Uint8Array(Math.min(limit, Math.max(out.length * 2, pos + 1))); + bigger.set(out.subarray(0, pos)); + out = bigger; + } + out[pos++] = c; + window[head++ & mask] = c; + }; + + // MSB-first bit reader over `data`. Returns null when fewer than `count` bits remain (end of input). + let bytePos = 0; + let bitIndex = 0; // 0 means "load next byte"; otherwise a single-bit mask (0x80..0x01) + let currentByte = 0; + const getBits = (count: number): number | null => { + let acc = 0; + for (let i = 0; i < count; i++) { + if (bitIndex === 0) { + if (bytePos >= data.length) return null; + currentByte = data[bytePos++]; + bitIndex = 0x80; + } + acc = (acc << 1) | (currentByte & bitIndex ? 1 : 0); + bitIndex >>= 1; + } + return acc; + }; + + for (;;) { + const tag = getBits(1); + if (tag === null) break; // end of input (trailing byte-padding can never form a full token) + if (tag === 1) { + const byte = getBits(8); + if (byte === null) break; + emit(byte & 0xff); + } else { + const index = getBits(windowBits); + if (index === null) break; + const count = getBits(lookaheadBits); + if (count === null) break; + const negOffset = index + 1; + const copyCount = count + 1; + for (let i = 0; i < copyCount; i++) { + emit(window[(head - negOffset) & mask]); + } + } + } + return out.subarray(0, pos); +} diff --git a/packages/gcode-bgcode/src/index.ts b/packages/gcode-bgcode/src/index.ts index 977d8267..7761b41f 100644 --- a/packages/gcode-bgcode/src/index.ts +++ b/packages/gcode-bgcode/src/index.ts @@ -10,8 +10,8 @@ * no `three`, framework, filesystem, or network. * * Phase 1: block walker + CRC32 + None/DEFLATE compression + None encoding. Phase 2: **MeatPack** - * encoding (both variants). heatshrink compression (phase 3) still decodes to an honest, structured - * "not yet supported" error. + * encoding (both variants). Phase 3: **heatshrink** compression (windows 11 & 12). All spec codecs are + * now decoded; container-adapter integration + real-file golden-equivalence follow in phase 4. */ export { BGCODE_MAGIC, @@ -24,3 +24,4 @@ export { } from './bgcode.js'; export type { BgcodeBlockInfo, BgcodeDecodeResult, BgcodeDecodeOptions } from './bgcode.js'; export { meatpackDecode } from './meatpack.js'; +export { heatshrinkDecode } from './heatshrink.js'; diff --git a/tools/pack-check-snapshots.json b/tools/pack-check-snapshots.json index d49ce21f..41d4ddd5 100644 --- a/tools/pack-check-snapshots.json +++ b/tools/pack-check-snapshots.json @@ -89,6 +89,9 @@ "dist/bgcode.d.ts", "dist/bgcode.d.ts.map", "dist/bgcode.js", + "dist/heatshrink.d.ts", + "dist/heatshrink.d.ts.map", + "dist/heatshrink.js", "dist/index.d.ts", "dist/index.d.ts.map", "dist/index.js",