experiment.py

#!/usr/bin/env python3
"""Controlled allocation experiment for .hyper files.

Builds families of files where exactly one variable changes, so we can attribute
byte-level differences to that variable.
"""
import os, struct, random, string
from tableauhyperapi import (HyperProcess, Connection, Telemetry, CreateMode,
                             TableDefinition, TableName, SqlType, Inserter, NOT_NULLABLE)
from lz4probe import lz4_block_decode

PAGE = 0x1000
random.seed(1)


def live_sb(d):
    b = max((0, PAGE), key=lambda b: struct.unpack_from("<Q", d, b + 0x30)[0])
    g = lambda o: struct.unpack_from("<Q", d, b + o)[0]
    return dict(page=b // PAGE, txn=g(0x30), size=g(0x38), hwm=g(0x40),
                p48=g(0x48), p50=g(0x50))


def build(path, coltype, values):
    tbl = TableDefinition(TableName("Extract", "T"),
                          [TableDefinition.Column("c", coltype, NOT_NULLABLE)])
    if os.path.exists(path):
        os.remove(path)
    with HyperProcess(telemetry=Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU) as h:
        with Connection(h.endpoint, path, CreateMode.CREATE_AND_REPLACE) as c:
            c.catalog.create_schema("Extract")
            c.catalog.create_table(tbl)
            with Inserter(c, tbl) as ins:
                ins.add_rows([[v] for v in values])
                ins.execute()
    return open(path, "rb").read()


def find_rowcount_blocks(d, expect, lo=0x4000):
    """Locate lz4 blocks whose decoded prefix is the u8 row count."""
    found = []
    for s in range(lo, len(d) - 8):
        out, end = lz4_block_decode(d, s, limit=64)
        if len(out) >= 8 and struct.unpack_from("<Q", out, 0)[0] == expect and end - s > 16:
            found.append(s)
    return found


def row(label, d, extra=""):
    sb = live_sb(d)
    print(f"  {label:<26} size={len(d):>9,} pages={len(d)//PAGE:>4} "
          f"hwm={sb['hwm']:#09x} txn={sb['txn']} p48={sb['p48']} p50={sb['p50']} {extra}")


print("=" * 96)
print("A. INT column, values 0..N-1  (row count varies, everything else fixed)")
print("=" * 96)
prev = None
for n in (1, 10, 100, 1_000, 10_000, 100_000, 1_000_000):
    d = build(f"exp_int_{n}.hyper", SqlType.big_int(), range(n))
    hits = find_rowcount_blocks(d, n)
    delta = f"(+{len(d)-prev:,})" if prev else ""
    row(f"N={n:<9,}", d, f"rowcount-blocks@{[hex(x) for x in hits[:3]]} {delta}")
    prev = len(d)

print()
print("=" * 96)
print("B. 100k-row VARCHAR, fixed row count, cardinality varies  (dictionary size)")
print("=" * 96)
for k in (1, 2, 16, 256, 4096, 100_000):
    vals = [f"val{i%k:07d}" for i in range(100_000)]
    d = build(f"exp_card_{k}.hyper", SqlType.text(), vals)
    row(f"cardinality={k:<8,}", d)

print()
print("=" * 96)
print("C. 100k-row TEXT, fixed cardinality, compressibility varies")
print("=" * 96)
cases = {
    "all-identical": ["A" * 32] * 100_000,
    "low-entropy": [("AB" * 16)[:32] for _ in range(100_000)],
    "random-ascii": ["".join(random.choices(string.ascii_letters, k=32)) for _ in range(100_000)],
}
for name, vals in cases.items():
    d = build(f"exp_ent_{name}.hyper", SqlType.text(), vals)
    row(name, d)

print()
print("=" * 96)
print("D. INT range width, fixed 100k rows  (bit-packing / FOR encoding)")
print("=" * 96)
for width, hi in (("2 values", 2), ("8-bit", 256), ("16-bit", 65536), ("32-bit", 2**31), ("64-bit", 2**62)):
    vals = [random.randrange(hi) for _ in range(100_000)]
    d = build(f"exp_bits_{hi}.hyper", SqlType.big_int(), vals)
    row(f"{width:<12} (0..{hi})", d)