[WIP][POC][BenchmarkingOnly] PFOR interleaved and fastlanes numbers

.gitignore

@@ -94,3 +94,6 @@ rat.txt
 
 # for ODBC DLL
 *.rc
+
+# Local out-of-tree benchmark build dir
+cpp/build-bench/

ab_compare.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+#
+# A/B comparison protocol for pfor_comparison_benchmark.
+#
+# Why this exists
+#   This benchmark has two deterministic confounds, both larger than most
+#   effects being measured. Neither is noise, and neither shrinks with more
+#   repetitions. See INTEL_RESULTS.md section 10.
+#
+#   (1) ARROW_USER_SIMD_LEVEL caps Arrow's runtime dispatch (bpacking.cc:29-45).
+#       It reaches BM_PforDecode only -- the FastLanes kernel has no dispatch.
+#       Capping to AVX2 makes Arrow's sequential decoder 3.21x faster than
+#       leaving it uncapped on Granite Rapids, because the AVX-512 family takes
+#       a second pass for the frame bias where the xsimd kernels fold it in.
+#       unset means the maximum, i.e. AVX512, so the default selects the slow path.
+#
+#   (2) The interleaved decoder reads either ~36 or ~48 GiB/s (1.33x) at identical
+#       binary, cap, filter and reps. It is stable to 1.00-1.03x across
+#       consecutive processes but flips between batches minutes apart, and it
+#       also moves with how many columns the filter admits (more columns ->
+#       more corpora allocated before the hot buffer). fl_order is immune.
+#
+# The protocol
+#   Run every variant alternating inside one batch, several processes each, with
+#   the cap pinned. Ratios are then taken within a batch, where confound (2) is
+#   constant. Report the cross-process spread, because the within-process stddev
+#   is 0.2-1% here and hides both confounds completely.
+#
+#   The benchmarks and their groups come from bench_groups.sh, which states
+#   which of them may be divided by which. This script used to name the production decoder against an
+#   exception-free standalone one and call the ratio a layout result.
+#
+# Usage
+#   ./ab_compare.sh <binA>:<cap> <binB>:<cap> [...]      # binaries under bin/
+#   ROUNDS=3 REPS=3 CORE=2 ./ab_compare.sh bench_O2_256:AVX2 fix_O2_256:AVX2
+#   GROUPS='GROUP_LAYOUT GROUP_DEST' ./ab_compare.sh bench_O2_256:AVX2
+#
+set -uo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+. ./bench_groups.sh
+
+BIN_DIR=${BIN_DIR:-$HOME/Projects/pfor_x86_handoff/width_matrix_v2/bin}
+OUT=${OUT:-/tmp/ab}
+ROUNDS=${ROUNDS:-3}
+REPS=${REPS:-3}
+CORE=${CORE:-2}
+
+# Which groups to time. Several in one process is fine and preferred, since
+# benchmarks timed together share the machine state; dividing across groups is
+# not -- see bench_groups.sh.
+GROUPS=${GROUPS:-'GROUP_LAYOUT GROUP_ORDER'}
+SETS=()
+for NAME in $GROUPS; do SETS+=("${!NAME}"); done
+FILTER=$(bench_filter "${SETS[@]}")
+
+[ $# -ge 1 ] || { sed -n '2,40p' "$0"; exit 1; }
+
+mkdir -p "$OUT"
+LOAD=$(awk '{print int($1)}' /proc/loadavg)
+if [ "$LOAD" -gt 2 ] && [ "${FORCE:-0}" != 1 ]; then
+  echo "!! loadavg is $LOAD -- too busy. Set FORCE=1 to override."; exit 1
+fi
+
+for i in $(seq 1 "$ROUNDS"); do
+  for SPEC in "$@"; do
+    IFS=: read -r B CAP <<<"$SPEC"
+    [ -x "$BIN_DIR/$B" ] || { echo "-- no binary $BIN_DIR/$B, skipping"; continue; }
+    # Cap pinned per variant. Stated in the output filename so the analysis
+    # cannot silently mix caps -- that is confound (1).
+    ARROW_USER_SIMD_LEVEL="${CAP:-MAX}" taskset -c "$CORE" "$BIN_DIR/$B" \
+      --benchmark_filter="$FILTER" \
+      --benchmark_repetitions="$REPS" \
+      --benchmark_report_aggregates_only=true \
+      --benchmark_out="$OUT/${B}__${CAP:-MAX}__$i.json" \
+      --benchmark_out_format=json >/dev/null 2>&1 \
+      && echo "  $B cap=${CAP:-MAX} round $i" \
+      || echo "  !! $B cap=${CAP:-MAX} round $i FAILED"
+  done
+done
+echo
+echo "analyze with: python3 ~/Projects/pfor_x86_handoff/ab_analyze.py $OUT"

bench_groups.sh

@@ -0,0 +1,71 @@
+# Shared benchmark groups and column set for pfor_comparison_benchmark drivers.
+# Sourced, not executed:  . "$(dirname "$0")/bench_groups.sh"
+#
+# Why this file exists
+#   Four drivers each carried their own copy of the column list and the
+#   benchmark regex, and all four copies named the same wrong pair: BM_PforDecode (the
+#   production decoder, which patches exceptions) against BM_InterleavedPforDecode
+#   and BM_InterleavedPforFlOrderDecode (a standalone path with no exception
+#   handling at all). A ratio between those is not a layout result -- the
+#   interleaved side is doing less work. The correct pair had already landed in
+#   the same benchmark file when those drivers were written.
+#
+#   Keeping the sets here means the rule below is stated once and cannot drift
+#   between drivers.
+#
+# Which benchmarks may be compared with which
+#   Only within a group. Every benchmark in a group runs the same code with the same
+#   exception handling and the same destination policy, so the named variable is
+#   the only difference.
+#
+#   GROUP_LAYOUT  sequential vs interleaved container, both through the
+#                production encoder and decoder with delta declined on both
+#                sides. This is the pair the layout question is read from.
+#
+#   GROUP_ORDER   file order vs the paper's lane assignment inside the
+#                interleaved container. These run arrow/util/fastlanes/
+#                interleaved_pfor.h, which has no exception handling anywhere in
+#                it, so the file-order benchmark in this group is their only
+#                valid baseline. Never pair one of these against a production
+#                benchmark.
+#                The production format has no lane-assignment mode, which is why
+#                this group exists outside it.
+#
+#   GROUP_DEST    whole-output materialization vs a reused block-sized
+#                destination, at each layout. Answers what the output buffer
+#                costs, not what the layout costs.
+#
+#                These benchmarks arrive with the output-buffer-reuse change
+#                and are absent from a tree without it; a filter naming them then
+#                matches nothing, which google-benchmark reports as no
+#                benchmarks to run.
+#
+#   GROUP_SHIPPED the shipping default, planner free to difference a vector. Not
+#                a layout comparison: on a sorted or correlated column it deltas, and
+#                the decode then also pays a serial prefix sum.
+#
+# Column set
+#   Changing it changes the numbers, because the count of columns a filter
+#   admits decides how many corpora are allocated before the timed buffer. Hold
+#   it fixed across anything meant to be compared.
+
+COLS=${COLS:-'TpcdsSoldDateSk|TpcdsStoreSk|TpcdsItemSk|TpcdsQuantity|EventDate|ClientIP|CounterID|SortedKeys|MonotoneRowId|RandomWalk'}
+
+GROUP_LAYOUT='BM_(PforPlainSeqDecode|PforPlainInterleavedDecode)'
+GROUP_ORDER='BM_(InterleavedPforDecode|InterleavedPforFlOrderRawDecode|InterleavedPforFlOrderDecode)'
+GROUP_DEST='BM_Pfor(Whole|Reuse)(Seq|Interleaved)Decode'
+GROUP_SHIPPED='BM_Pfor(64)?Decode'
+
+# Build a google-benchmark filter from one or more groups.
+#   FILTER=$(bench_filter "$GROUP_LAYOUT" "$GROUP_ORDER")
+# Groups are unioned so one process can time several of them -- that is fine and
+# is in fact preferred, since benchmarks timed in one process share the machine
+# state.
+# What is not fine is dividing a number from one group by a number from another.
+bench_filter() {
+  local joined="" a
+  for a in "$@"; do
+    joined="${joined:+$joined|}${a}"
+  done
+  printf '(%s)/(%s)/' "$joined" "$COLS"
+}

build_transpose_ab.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Build the AVX2 Transpose32x32 A/B points.
+#
+# Treatment: transposed_delta.h gained Transpose32x32Avx2 (sixteen in-register
+# 8x8 transposes) behind #if defined(__AVX2__). Before this, x86 fell through to
+# Transpose32x32Scalar -- 1024 four-byte loads at a 128-byte stride plus 1024
+# scalar stores per block, because only NEON had a hand-written path. Isolated,
+# the permutation goes 28.67 -> 73.56 GiB/s (2.57x).
+#
+# Only the kFlOrder decoder calls the transpose, so BM_InterleavedPforDecode (file
+# order) and BM_PforDecode are controls this change cannot reach. If either
+# moves by more than the cross-process spread, the comparison is contaminated --
+# see INTEL_RESULTS.md section 10.
+#
+# Correctness is not left to inspection: pfor_comparison_benchmark.cc
+# round-trips every column through encode+decode and ARROW_CHECKs the result
+# before timing starts, for both orders. A wrong transpose aborts the binary.
+# Transpose32x32 is also its own inverse on a square grid, so encode and decode
+# share the path and a broken permutation cannot cancel itself out.
+#
+# Build only. Time with ab_compare.sh.
+set -uo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+export PATH="$HOME/.local/bin:$PATH"   # cmake must be <4
+
+BUILD=build-x86-sweep
+OUT="$HOME/Projects/pfor_x86_handoff/width_matrix_v2"
+mkdir -p "$OUT/bin"
+
+# Matched to the existing fix_* binaries so the only difference is the transpose.
+POINTS=(
+  "tr_O3_256:AVX2:-O3:256"
+  "tr_O2_256:AVX2:-O2:256"
+)
+
+for P in "${POINTS[@]}"; do
+  IFS=: read -r NAME LEVEL OPT PVW <<<"$P"
+  echo "== $NAME : ARROW_SIMD_LEVEL=$LEVEL $OPT -mprefer-vector-width=$PVW"
+  # Explicit every time: CMAKE_CXX_FLAGS_RELEASE is a CACHE variable and would
+  # otherwise inherit the previous point's value.
+  if ! cmake -S cpp -B "$BUILD" -GNinja \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DARROW_SIMD_LEVEL="$LEVEL" \
+      -DCMAKE_CXX_FLAGS_RELEASE="$OPT -DNDEBUG -mprefer-vector-width=$PVW" \
+      -DCMAKE_C_FLAGS_RELEASE="$OPT -DNDEBUG -mprefer-vector-width=$PVW" \
+      -DARROW_PARQUET=ON -DARROW_BUILD_BENCHMARKS=ON \
+      -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON \
+      -DARROW_BUILD_TESTS=OFF > "$OUT/configure_$NAME.log" 2>&1; then
+    echo "!! configure FAILED -- $OUT/configure_$NAME.log"; continue
+  fi
+  if ! cmake --build "$BUILD" --target parquet-pfor-comparison-benchmark -j 6 \
+      > "$OUT/build_$NAME.log" 2>&1; then
+    echo "!! build FAILED -- tail:"; tail -30 "$OUT/build_$NAME.log"; continue
+  fi
+  cp "$BUILD/release/parquet-pfor-comparison-benchmark" "$OUT/bin/bench_$NAME"
+  echo "   -> $OUT/bin/bench_$NAME"
+  # Prove the AVX2 transpose is actually in the kFlOrder kernel.
+  objdump -d --demangle "$OUT/bin/bench_$NAME" 2>/dev/null | awk '
+    /^[0-9a-f]+ <.*InterleavedPforDecode<.*>:$/ {
+      inside=1
+      name=($0 ~ /InterleavedPforOrder\)1/) ? "kFlOrder(1)" : "kFileOrder(0)"
+      n=0; y=0; z=0; perm=0; next
+    }
+    /^[0-9a-f]+ </ { if (inside) printf "      %-14s insns=%-7d ymm=%-6d zmm=%-6d vperm2x128=%d\n", name,n,y,z,perm; inside=0; next }
+    inside { nf=split($0,p,"\t"); if (nf<3) next; n++
+             y+=gsub(/%ymm[0-9]+/,"&",p[3]); z+=gsub(/%zmm[0-9]+/,"&",p[3])
+             if (p[3] ~ /vperm2[fi]128/) perm++ }
+    END { if (inside) printf "      %-14s insns=%-7d ymm=%-6d zmm=%-6d vperm2x128=%d\n", name,n,y,z,perm }'
+done
+echo "TRANSPOSE_BUILDS_DONE"

build_width_matrix.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+#
+# Register-width x optimizer matrix: one benchmark binary per point, each
+# preserved under $OUT/bin/ so timing can be replayed without rebuilding and so
+# builds and timing runs never overlap.
+#
+# The optimizer level is set explicitly at every point and never inherited.
+# CMAKE_BUILD_TYPE=Release does not mean -O2 here: cpp/cmake_modules/
+# SetupCxxFlags.cmake:637 carries a local override that keeps CMake's default
+# -O3 -DNDEBUG for the pfor benchmark, so a point that lets Release supply its
+# own flags comes out -O3 whatever it was meant to be. Width scaling read off a
+# run that inherited its level mixes levels across the points and is not a width
+# axis.
+#
+# CMAKE_CXX_FLAGS_RELEASE is a CACHE variable, so a value set at one point
+# persists into the next configure of the same build dir. It is passed at every
+# point here, so no point can inherit a neighbour's flags.
+#
+# -mprefer-vector-width is set at every point too. Arrow's
+# ARROW_SIMD_LEVEL=AVX512 uses -march=skylake-avx512, and GCC defaults that
+# target to -mprefer-vector-width=256, so a nominally-512 build emits zero zmm
+# unless asked for.
+#
+# Width is verified inside the kernel function, not across the whole binary.
+# Whole-binary counts are dominated by Arrow's own explicit-vector code and say
+# nothing about what the autovectorizer did to the FastLanes kernel.
+#
+# No 512-bit leg
+#   Arrow's bit-unpack dispatch is capped at 256 bits, because the AVX-512
+#   kernels assemble their input register from scalar loads and measure 0.67x of
+#   the scalar kernel. Asking for the 512-bit level therefore hands the
+#   sequential decoder the 256-bit kernel back while the interleaved kernel really
+#   does widen, so such a point compares two widths on one side and one on the
+#   other. It comes back when a 512-bit unpack kernel exists that beats the
+#   256-bit one.
+#
+# Build only. Time the binaries with ab_compare.sh, which alternates the
+# variants inside one batch -- these numbers come from a shared virtualized
+# container and a point timed minutes after its neighbour is not comparable to
+# it.
+#
+set -uo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+export PATH="$HOME/.local/bin:$PATH"   # cmake must be <4; 3.31.10 lives here
+
+BUILD=build-x86-sweep                  # reused so bundled deps are not rebuilt
+# Output directory keeps its name so the binaries and verification logs already
+# on the x86 box are not orphaned.
+OUT="$HOME/Projects/pfor_x86_handoff/width_matrix_v2"
+mkdir -p "$OUT/bin"
+
+# The autovectorized FastLanes kernel under test. Order 0 = plain interleaved,
+# Order 1 = the paper's lane assignment. Order 0 is the one the width claim is
+# about. This kernel has no exception handling, so its timings belong to the
+# order group in bench_groups.sh and never against a production decoder.
+KERNEL='arrow::util::fastlanes::InterleavedPforDecode<(arrow::util::fastlanes::InterleavedPforOrder)0>'
+
+# name | ARROW_SIMD_LEVEL | optimizer | prefer-vector-width
+POINTS=(
+  "O2_128:SSE4_2:-O2:128"
+  "O2_256:AVX2:-O2:256"
+  "O3_128:SSE4_2:-O3:128"
+  "O3_256:AVX2:-O3:256"
+)
+
+for P in "${POINTS[@]}"; do
+  IFS=: read -r NAME LEVEL OPT PVW <<<"$P"
+  echo "=================================================================="
+  echo "== $NAME : ARROW_SIMD_LEVEL=$LEVEL  $OPT  -mprefer-vector-width=$PVW"
+  echo "=================================================================="
+
+  # Explicit every time. Nothing inherited, nothing assumed about Release.
+  if ! cmake -S cpp -B "$BUILD" -GNinja \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DARROW_SIMD_LEVEL="$LEVEL" \
+      -DCMAKE_CXX_FLAGS_RELEASE="$OPT -DNDEBUG -mprefer-vector-width=$PVW" \
+      -DCMAKE_C_FLAGS_RELEASE="$OPT -DNDEBUG -mprefer-vector-width=$PVW" \
+      -DARROW_PARQUET=ON -DARROW_BUILD_BENCHMARKS=ON \
+      -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON \
+      -DARROW_BUILD_TESTS=OFF > "$OUT/configure_$NAME.log" 2>&1; then
+    echo "!! configure FAILED -- $OUT/configure_$NAME.log"; continue
+  fi
+  # Record the flags cmake actually resolved, so the log proves the level.
+  grep -E "CMAKE_CXX_FLAGS(_RELEASE)?:" "$OUT/configure_$NAME.log" | sed 's/^/   /'
+
+  if ! cmake --build "$BUILD" --target parquet-pfor-comparison-benchmark -j 6 \
+      > "$OUT/build_$NAME.log" 2>&1; then
+    echo "!! build FAILED -- $OUT/build_$NAME.log"; continue
+  fi
+
+  SRC="$BUILD/release/parquet-pfor-comparison-benchmark"
+  BIN="$OUT/bin/bench_$NAME"
+  cp "$SRC" "$BIN"
+
+  # ---- verify the width the autovectorizer gave this kernel ----------------
+  # Disassemble only the kernel's address range: from its symbol to the next.
+  ADDR=$(nm -C "$BIN" | grep -F "$KERNEL" | awk '{print $1}' | head -1)
+  {
+    echo "point=$NAME level=$LEVEL opt=$OPT prefer=$PVW"
+    echo "sha256: $(sha256sum "$BIN" | cut -d' ' -f1)"
+    echo "kernel: $KERNEL"
+    echo "kernel addr: 0x$ADDR"
+    if [ -n "$ADDR" ]; then
+      # Boundaries come from objdump's own '<addr> <symbol>:' labels, not from
+      # sorting nm addresses. The nm approach silently produced an empty range
+      # for O2_256 -- kernel_O2_256.asm came out 0 bytes and the point then
+      # reported "zmm: 0 ymm: 0", which reads as a failed vectorization rather
+      # than a failed extraction. nm lists symbols from the symbol table in an
+      # order that need not bracket the function body (aliases, local symbols
+      # and ifunc resolvers all land in between), so "next address after this
+      # one" is not the function's end. objdump cannot disagree with itself.
+      objdump -dC --no-show-raw-insn "$BIN" 2>/dev/null \
+        | awk -v s="$KERNEL" '
+            /^[0-9a-f]+ </ { inside = index($0, s) > 0 }
+            inside' > "$OUT/kernel_$NAME.asm"
+      echo "kernel insns: $(grep -cE '^\s+[0-9a-f]+:' "$OUT/kernel_$NAME.asm")"
+      echo "IN-KERNEL vector register usage:"
+      for R in xmm ymm zmm; do
+        echo "    $R: $(grep -c "%$R" "$OUT/kernel_$NAME.asm")"
+      done
+      echo "IN-KERNEL notable ops:"
+      # v?-prefixed: a 128-bit SSE build emits non-VEX psrld/movdqa, so
+      # VEX-only patterns reported zero shifts and zero stack traffic for every
+      # 128-bit point -- which made the narrow builds look spill-free when
+      # O3_128 in fact had the most stack traffic in the matrix.
+      for OP in 'v?psrld' 'v?pslld' 'vpsrlvd' 'vpsllvd' 'v?pand' 'v?por' \
+                'v?movdqu' 'v?movdqa' 'vpgatherdd' 'vpermd' 'v?pshufb' \
+                'vpbroadcastd' 'vzeroupper'; do
+        C=$(grep -cE "^\s+[0-9a-f]+:\s+$OP" "$OUT/kernel_$NAME.asm")
+        [ "$C" -gt 0 ] && echo "    $OP: $C"
+      done
+      # Vector moves with a stack-relative operand. Reported as traffic, not
+      # spills: a genuine stack scratch buffer matches this too, which is
+      # exactly what the pre-fix kFileOrder path was (a 4 KiB grid, ~1984
+      # write-only stores with zero matching reloads).
+      echo "    stack vector traffic: $(grep -cE '^\s+[0-9a-f]+:\s+v?mov(dqu|dqa|ups|aps).*\((%rsp|%rbp)' "$OUT/kernel_$NAME.asm")"
+    else
+      echo "!! kernel symbol not found -- cannot verify width"
+    fi
+  } > "$OUT/verify_$NAME.txt" 2>&1
+  cat "$OUT/verify_$NAME.txt"
+
+  # A width that did not materialize is reported, never silently averaged in.
+  # Only the 256-bit points can miss: they are built at AVX2, where the compiler
+  # may still choose xmm. The 128-bit points are built at SSE4_2, where __AVX2__
+  # is undefined and ymm cannot be emitted at all.
+  case "$PVW" in
+    256) grep -qE '^    ymm: [1-9]' "$OUT/verify_$NAME.txt" \
+           || echo "!! WARNING $NAME: no ymm IN KERNEL -- this is NOT a 256-bit point" ;;
+  esac
+  echo
+done
+
+echo "=== builds complete. binaries in $OUT/bin/ ==="
+sha256sum "$OUT"/bin/* 2>/dev/null
+echo
+echo "Distinct binaries (if two points share a sha256, they are the SAME build"
+echo "and any difference between their timings is noise):"
+sha256sum "$OUT"/bin/* 2>/dev/null | awk '{print $1}' | sort -u | wc -l
+echo
+echo "Now time them, when the box is quiet:"
+echo "  BIN_DIR=$OUT/bin ./ab_compare.sh \\"
+echo "    bench_O2_128:SSE4_2 bench_O2_256:AVX2 \\"
+echo "    bench_O3_128:SSE4_2 bench_O3_256:AVX2"