probe9.py

#!/usr/bin/env python3
"""probe9.py - the wide (8-byte) string dictionary entry and its trigger.

`more-samples`' Japanese chart file has a `Varchar` column on scheme 14, which
the scheme table calls a *native* dictionary - a family that was supposed to be
integer-only. It is not a mis-tabulated scheme: Hyper really does have a second
string dictionary entry, twice the width of the documented one.

  narrow, 4 B   u1 length, then the text inline when length <= 3, else a u3
                heap offset
  wide,   8 B   u4 length, then the text inline when length <= 4, else a u4
                heap offset

The `u1` in the narrow entry caps a value at 255 bytes, and that is exactly the
trigger. This probe walks the longest value in a column across the boundary and
asserts where the flip happens, rather than inferring it from the one real file
that happens to contain long strings - the same mistake as corrections 1-4.

It also drives the cases the chart file does not have: NULLs, Bytea rather than
text, and an all-distinct column where a dictionary buys nothing, so the wide
entry is exercised under several scheme codes.

  /usr/bin/python3 probe9.py
"""
import os
import random
import shutil
import sys
from collections import Counter

import hyperparse as hp

try:
    from tableauhyperapi import (HyperProcess, Connection, Telemetry, CreateMode,
                                 TableDefinition, SqlType, NULLABLE, TableName,
                                 Inserter)
except ImportError:
    sys.exit("needs tableauhyperapi; try /usr/bin/python3")

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, "probes9")
ROWS = 4000
NARROW_CAP = 255          # the u1 length field in the 4-byte entry


def build(proc, name, sqltype, rows):
    path = os.path.join(OUT, f"{name}.hyper")
    tbl = TableName("E", "T")
    cols = [TableDefinition.Column("c", sqltype, NULLABLE)]
    with Connection(proc.endpoint, path, CreateMode.CREATE_AND_REPLACE) as c:
        c.catalog.create_schema("E")
        c.catalog.create_table(TableDefinition(tbl, cols))
        with Inserter(c, TableDefinition(tbl, cols)) as ins:
            ins.add_rows(rows)
            ins.execute()
    return path


def inspect(path):
    """(scheme, entry family, bytes per dictionary entry) for the first block."""
    d = open(path, "rb").read()
    f = hp.parse(path)
    ct = hp.ColType.parse(hp.relations(f.live_catalog)[0]["attributes"][0]["type"])
    e = hp.column_objects(f, 0, 1)[0]
    b = hp.read_object(d, e)
    ix = hp.parse_block_index(b)
    dom = hp._u("<I", b, hp.SMA_OFF + hp.domain_offset(ct))
    spec = hp.SCHEMES.get(ix.scheme)
    return ix.scheme, (spec[3] if spec else "?"), (ix.dict_size / dom if dom else 0)


def oracle_match(path, proc):
    d = open(path, "rb").read()
    f = hp.parse(path)
    ct = hp.ColType.parse(hp.relations(f.live_catalog)[0]["attributes"][0]["type"])
    mine, note = hp.decode_relation_column(d, f, 0, 1, ct)
    if note and not mine:
        return False, note
    with Connection(proc.endpoint, path) as conn:
        truth = [r[0] for r in conn.execute_list_query('SELECT "c" FROM "E"."T"')]
    return Counter(map(repr, mine)) == Counter(map(repr, truth)), ""


def main():
    shutil.rmtree(OUT, ignore_errors=True)
    os.makedirs(OUT, exist_ok=True)
    random.seed(7)
    ok = bad = 0
    with HyperProcess(telemetry=Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU,
                      parameters={"default_database_version": "4"}) as proc:
        print("longest value in the column vs the entry family chosen:")
        for L in (10, 200, 254, 255, 256, 300, 1000):
            # exactly one row carries the long value; every other row is short,
            # so nothing but the maximum length changes across the sweep.
            rows = [[("v%04d" % i).ljust(L if i == 0 else 8, "x")]
                    for i in range(ROWS)]
            p = build(proc, f"max{L}", SqlType.text(), rows)
            scheme, fam, per = inspect(p)
            want = "narrow" if L <= NARROW_CAP else "native"
            good = fam == want and round(per) == (4 if want == "narrow" else 8)
            m, note = oracle_match(p, proc)
            print(f"  longest={L:<5} scheme={scheme:<3} {hp.scheme_name(scheme):<22} family={fam:<7} "
                  f"{per:.2f} B/entry  expect {want}  "
                  f"{'ok' if good else 'UNEXPECTED'}  oracle={'ok' if m else 'FAIL ' + note}")
            ok += good and m
            bad += not (good and m)

        print("\nthe wide entry under other schemes:")
        def longstr(i, L):
            return "v%06d" % i + "".join(random.choice("abcdef") for _ in range(L))

        cases = {
            # all-distinct, so a dictionary buys nothing and the engine is free
            # to pick a different scheme entirely
            "alldistinct": (SqlType.text(), [[longstr(i, 300)] for i in range(ROWS)]),
            "withnulls": (SqlType.text(),
                          [[None if i % 3 == 0 else longstr(i, 300)]
                           for i in range(ROWS)]),
            # raw octets, not text - the inline path must not decode as UTF-8
            "bytea": (SqlType.bytes(),
                      [[bytes([(i + j) % 256 for j in range(300)])]
                       for i in range(ROWS)]),
            # values straddling the 4-byte inline cap of the wide entry
            "inlinecap": (SqlType.text(),
                          [[("abcde"[:1 + i % 5] if i % 2 else longstr(i, 300))]
                           for i in range(ROWS)]),
        }
        for nm, (t, rows) in cases.items():
            p = build(proc, nm, t, rows)
            scheme, fam, per = inspect(p)
            m, note = oracle_match(p, proc)
            print(f"  {nm:<12} scheme={scheme:<3} {hp.scheme_name(scheme):<22} family={fam:<7} {per:.2f} B/entry  "
                  f"oracle={'ok' if m else 'FAIL ' + note}")
            ok += m
            bad += not m

        # Enough distinct values that no dictionary is worth building, so the
        # engine writes the cell itself once per row. The cell is a dictionary
        # entry of the same family - 4 bytes narrow, 8 bytes wide - NOT the
        # type's own width, which a Varchar does not have. Reading `bits // 8 or
        # ct.width` here gives 0 and the block does not decode at all.
        #
        # This also shows why Dict4/SmallDict4 are dead codes: the uncompressed
        # heap is deduplicated too, so at 4-byte codes a dictionary costs the
        # same per row *plus* the dictionary. Nothing can select them.
        print("\nuncompressed string blocks (no dictionary at all):")
        big, distinct = 131_000, 70_000
        raw_cases = {
            "raw_short": lambda i: "s%07d" % (i % distinct),
            "raw_long": lambda i: "L%07d" % (i % distinct) + "y" * 300,
            "raw_shortnull": lambda i: None if i % 11 == 0 else "s%07d" % (i % distinct),
            "raw_longnull": lambda i: (None if i % 11 == 0
                                       else "L%07d" % (i % distinct) + "y" * 300),
        }
        for nm, fn in raw_cases.items():
            p = build(proc, nm, SqlType.text(), [[fn(i)] for i in range(big)])
            d = open(p, "rb").read()
            f = hp.parse(p)
            ix = hp.parse_block_index(hp.read_object(d, hp.column_objects(f, 0, 1)[0]))
            m, note = oracle_match(p, proc)
            print(f"  {nm:<14} scheme={ix.scheme:<3} {hp.scheme_name(ix.scheme):<22} "
                  f"dict={ix.dict_size} {ix.data_size * 8 / ix.tuple_count:.0f} bits/row  "
                  f"oracle={'ok' if m else 'FAIL ' + note}")
            ok += m
            bad += not m
    print(f"\nTOTAL: {ok} pass, {bad} fail -> {OUT}")
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())