diff --git a/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt b/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt index eb4c9853..0e047041 100644 --- a/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt +++ b/cpp/monoprop/detail/evolution/layer_build/CMakeLists.txt @@ -7,6 +7,8 @@ target_sources( "Common.h" "Engine.h" "FusedApply.h" + "PartnerMerge.h" + "QueryWire.h" "Resolve.h" "Scan.h" ) diff --git a/cpp/monoprop/detail/evolution/layer_build/Common.h b/cpp/monoprop/detail/evolution/layer_build/Common.h index e3c203d3..53ebd758 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Common.h +++ b/cpp/monoprop/detail/evolution/layer_build/Common.h @@ -104,25 +104,6 @@ struct FusedContract { std::vector cross_half; // R>1: one half per cross-rank query (resolver +φ, querier −φ) }; -// Queries ride flat VecZ buffers: kQueryWords elements per query (W monomial words + one ±1 phase word). -// The source index is not in the payload — the resolver answers by position; the querier holds src_idx_r[r][q]. -template -inline constexpr size_t kQueryWords = mpi_detail::kWords + 1; - -// Fused query+value record width (R>1): the plain query record plus one trailing word holding the source's -// pre-cos coeff (v_src, bit-cast from double), so query + value ride a single alltoallv instead of two. -template -inline constexpr size_t kQueryWordsFused = kQueryWords + 1; - -// The unsigned-int intermediate normalizes the ±1 sign bit into a fixed 32-bit pattern so the round-trip -// is exact for any VecZ element width. Edit encode/decode as a pair. -inline auto encode_phase(int phase) -> size_t { - return static_cast(static_cast(phase)); -} -inline auto decode_phase(size_t word) -> int { - return static_cast(static_cast(word)); -} - // bit_cast, not a conversion, so v_src arrives over the wire bit-identical. static_assert(sizeof(size_t) == sizeof(double), "fused query value word assumes 64-bit VecZ element"); inline auto encode_value(double v) -> size_t { @@ -132,45 +113,4 @@ inline auto decode_value(size_t word) -> double { return std::bit_cast(word); } -template -inline auto query_push(VecZ &buf, const Monomial &mono, int phase) -> void { - mpi_detail::append_monomial_words(mono, buf); - buf.push_back(encode_phase(phase)); -} - -// The mono + phase words occupy the same leading offsets in the plain and fused record, so readers differ -// only in the per-record stride QW (defaulted to the plain width). -template > -inline auto query_read(const VecZ &buf, size_t q, Monomial &mono_out, int &phase_out) -> void { - const size_t base = q * QW; - mono_out = mpi_detail::read_monomial_from_words(buf, base); - phase_out = decode_phase(buf[base + mpi_detail::kWords]); -} - -// No monomial reconstruction: process_responses needs only the phase. -template > -inline auto query_phase(const VecZ &buf, size_t q) -> int { - return decode_phase(buf[q * QW + mpi_detail::kWords]); -} - -template -inline auto query_value(const VecZ &buf, size_t q) -> double { - return decode_value(buf[q * kQueryWordsFused + mpi_detail::kWords + 1]); -} - -// Requires v.size() == q.size()/kQueryWords: exactly one value per query record. -template -inline auto build_fused_query_value(const VecZ &q, const std::vector &v, VecZ &out) -> void { - constexpr size_t W = kQueryWords; - const size_t nq = q.empty() ? 0 : q.size() / W; - out.clear(); - out.reserve(nq * kQueryWordsFused); - for (size_t i = 0; i < nq; ++i) { - out.insert(out.end(), - q.begin() + static_cast(i * W), - q.begin() + static_cast((i + 1) * W)); - out.push_back(encode_value(v[i])); - } -} - } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index d073f00c..e2dfbc7f 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,8 @@ #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/PartnerMerge.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" #include "monoprop/detail/evolution/layer_build/Resolve.h" #include "monoprop/detail/evolution/layer_build/Scan.h" #include "monoprop/detail/graph_encoding/MPGraphEncodingStorage.h" @@ -70,7 +73,8 @@ inline auto append_inserted_endpoints(CosMask &cos_all, size_t combined_size, co template struct GraphSink { static constexpr bool wants_values = false; - static constexpr size_t kStride = kQueryWords; + [[nodiscard]] auto incoming_form() const -> QueryForm { return QueryForm::Plain; } + [[nodiscard]] auto querier_form() const -> QueryForm { return QueryForm::Plain; } using Response = TermIndex; static auto init_response() -> Response { return std::numeric_limits::max(); } @@ -134,10 +138,13 @@ struct GraphSink { auto &out = acc[r].out_entries; const size_t base = out.size(); const size_t nq = resp.size(); + const QueryForm form = querier_form(); out.resize(base + nq); + size_t off = 0; for (size_t q = 0; q < nq; ++q) { assert(resp[q] != std::numeric_limits::max() && "resolver must insert absent cross-rank terms"); - out[base + q] = {srcs[q], query_phase(qbuf, q)}; + out[base + q] = {srcs[q], QueryWire::phase_at(qbuf, off)}; + off = QueryWire::next_off(qbuf, form, off); } } @@ -186,7 +193,10 @@ struct GraphSink { template struct ContractSink { static constexpr bool wants_values = true; - static constexpr size_t kStride = kQueryWordsFused; + // This rank receives fused records, but on_response_block is handed its own plain queries_r; + // reading the wrong form there decodes a neighbouring record's phase, i.e. a coefficient sign flip. + [[nodiscard]] auto incoming_form() const -> QueryForm { return QueryForm::Fused; } + [[nodiscard]] auto querier_form() const -> QueryForm { return QueryForm::Plain; } using Response = double; static auto init_response() -> Response { return 0.0; } @@ -205,8 +215,7 @@ struct ContractSink { // No constructor on purpose: as an aggregate the call site names each field, so the two adjacent // bools cannot be swapped silently. GraphSink keeps its ctor because it sizes `acc` from R. - // Self-resolve hit (both endpoints local). always_inline: called once per surviving rotation in the - // R=1 hot loop, where a real call is a measurable regression on the Pauli benches. + // Self-resolve hit: both endpoints are local. [[gnu::always_inline]] auto self_hit(size_t src, size_t found, int phase, double v_src) -> void { const double v_tgt = fused_scale ? op_coeffs[found] * inv_cos : op_coeffs[found]; fc.hits.push_back(RotationRec{src, found, v_src, v_tgt, static_cast(phase)}); @@ -226,7 +235,7 @@ struct ContractSink { -> std::vector & { scratch.resize(queries.size()); for (size_t r = 0; r < queries.size(); ++r) { - build_fused_query_value(queries[r], vals[r], scratch[r]); + QueryWire::build_fused(queries[r], vals[r], scratch[r]); } return scratch; } @@ -240,7 +249,7 @@ struct ContractSink { } auto on_resolved(size_t g, size_t s, - size_t q, + size_t /*q*/, size_t ip, const IncomingProbe &pr, const std::vector &incoming) -> Response { @@ -249,14 +258,13 @@ struct ContractSink { v_tgt = fused_scale ? op_coeffs[ip] * inv_cos : op_coeffs[ip]; } else if (schrodinger) { - v_tgt = - is_paired(pr.mono[g]) ? algebra_state_phase(basis, pr.mono[g], state_mask_) : 0.0; + v_tgt = pr.is_paired_at(g) ? algebra_state_phase(basis, pr.mono_at(g), state_mask_) : 0.0; } else { v_tgt = 0.0; // Heisenberg fresh insert } fc.cross_half[cross_base_ + g] = HalfRotationRec{ip, - query_value(incoming[s], q), + QueryWire::value_at(incoming[s], pr.off_of[g]), static_cast(pr.phase_of[g]), /*is_insert=*/ip >= pr.base}; return v_tgt; @@ -276,9 +284,12 @@ struct ContractSink { const std::vector &srcs, const VecZ &qbuf) -> void { const size_t nq = rval.size(); + const QueryForm form = querier_form(); + size_t off = 0; for (size_t q = 0; q < nq; ++q) { - const auto nphase = static_cast(-query_phase(qbuf, q)); + const auto nphase = static_cast(-QueryWire::phase_at(qbuf, off)); fc.cross_half.push_back(HalfRotationRec{srcs[q], rval[q], nphase, /*is_insert=*/false}); + off = QueryWire::next_off(qbuf, form, off); } } @@ -297,8 +308,13 @@ struct ContractSink { // Owns build_layer's machinery over a compile-time Sink policy. combined_size = the pre-layer operator size. template struct LayerBuildEngine { + using RowPosT = typename OperatorIndex::PosT; + + // A miss keeps its decoded positions (pos_at indexes deferred_pos_flat_) and the probe's hash. struct DeferredSelfMiss { - Monomial mono; + size_t pos_at; + uint32_t k; + uint32_t hash; size_t src; int phase; double v_src = 0.0; // ContractSink only: op_pre[src] captured at scan emit; 0 for GraphSink @@ -314,6 +330,11 @@ struct LayerBuildEngine { std::vector queries_r; std::vector> src_idx_r; std::vector deferred_self_misses; + // Deferred-miss positions, concatenated in miss order; parallel to deferred_self_misses. + std::vector deferred_pos_flat_; + // This pass's self-owned queries as positions, straight from the scan: never encoded, so the resolve + // below has nothing to decode. Parallel to src_idx_r[my_rank]. + SelfQueryStage self_stage_; // Scan-captured v_src per query (ContractSink only via Sink::wants_values; empty for GraphSink). std::vector> src_val_r; // Fused query+value send scratch (ContractSink, R>1): shared by a gate's two exchange passes. @@ -341,15 +362,14 @@ struct LayerBuildEngine { // Resolve this rank's own query stream inline, then clear it so the alltoallv never sends to self. auto resolve_self_queries(bool is_leader_pass) -> void { - VecZ &lq = queries_r[my_rank]; std::vector &ls = src_idx_r[my_rank]; std::vector *lv = nullptr; if constexpr (Sink::wants_values) { lv = &src_val_r[my_rank]; } - const size_t nq = lq.empty() ? 0 : lq.size() / kQueryWords; - resolve_range_(lq, ls, lv, 0, nq, is_leader_pass); - lq.clear(); + // The scan routes a self-owned partner to the stage, never to the wire buffer. + resolve_range_(ls, lv, is_leader_pass); + self_stage_.clear(); ls.clear(); if constexpr (Sink::wants_values) { src_val_r[my_rank].clear(); @@ -365,9 +385,11 @@ struct LayerBuildEngine { auto run_exchange(bool is_leader_pass, std::vector &&queries, std::vector> &&src_idx, - std::vector> &&src_val) -> void { + std::vector> &&src_val, + SelfQueryStage &&self_stage) -> void { queries_r = std::move(queries); src_idx_r = std::move(src_idx); + self_stage_ = std::move(self_stage); // src_val is empty unless Sink::wants_values, so the move is a no-op under GraphSink. src_val_r = std::move(src_val); if (!is_leader_pass && R > 1) { @@ -389,7 +411,8 @@ struct LayerBuildEngine { // Followers a leader already matched must not be re-resolved over the wire, so compact them out. auto drop_matched_cross_rank_followers() -> void { - constexpr size_t W = kQueryWords; + using QW = QueryWire; + const QueryForm form = sink.querier_form(); for (size_t r = 0; r < R; ++r) { if (r == my_rank) { continue; @@ -403,22 +426,22 @@ struct LayerBuildEngine { } const size_t nq = s.size(); size_t kept = 0; + // Two cursors, since a dropped query has no fixed width; order is the accumulation order. + size_t src_off = 0; + size_t dst_off = 0; for (size_t k = 0; k < nq; ++k) { - if (matched.is_marked(s[k])) { - continue; - } - if (kept != k) { - std::copy(q.begin() + static_cast(k * W), - q.begin() + static_cast((k + 1) * W), - q.begin() + static_cast(kept * W)); - } - s[kept] = s[k]; - if (v != nullptr) { - (*v)[kept] = (*v)[k]; + const size_t next = QW::next_off(q, form, src_off); + if (!matched.is_marked(s[k])) { + dst_off += QW::move_query(q, form, src_off, dst_off); + s[kept] = s[k]; + if (v != nullptr) { + (*v)[kept] = (*v)[k]; + } + ++kept; } - ++kept; + src_off = next; } - q.resize(kept * W); + q.resize(dst_off); s.resize(kept); if (v != nullptr) { v->resize(kept); @@ -426,22 +449,26 @@ struct LayerBuildEngine { } } - // Sub-step of finish() — do not call directly. Precondition: call only after both resolve passes - // complete, else the base+k ↔ record-slot assignment and per-miss distinctness break. Deferred self - // misses are pairwise-distinct (mono = source⊕G, ⊕G injective) and still absent, so miss k gets - // base+k in leader-then-follower order. See insert_absent_terms. + // Sub-step of finish() — call only after both resolve passes complete, or the base+k assignment and + // per-miss distinctness break. Misses are pairwise-distinct (mono = source⊕G, ⊕G injective), so miss + // k gets base+k in leader-then-follower order. auto insert_deferred_self_misses() -> void { const size_t n_miss = deferred_self_misses.size(); if (n_miss == 0) { return; } - auto key_at = [&](size_t k) -> const Monomial & { return deferred_self_misses[k].mono; }; sink.prepare_deferred(n_miss); - insert_absent_terms(local_op, n_miss, key_at, [&](size_t k, size_t base) { + // Grow the rows, write each miss's positions, insert; same base+k ordering as insert_absent_terms. + // Kept as the dense reference sparse_resolve_tests.cpp differentially tests this path against. + const size_t base = local_op.store->grow_rows_geometric(n_miss); + for (size_t k = 0; k < n_miss; ++k) { const auto &m = deferred_self_misses[k]; - assign_row(*local_op.store, base + k, m.mono); + local_op.store->set_positions(base + k, + std::span(deferred_pos_flat_).subspan(m.pos_at, m.k)); sink.emit_deferred(k, base + k, m.src, m.phase, m.v_src); - }); + } + local_op.store->bulk_insert_hashed(n_miss, base, [&](size_t j) { return deferred_self_misses[j].hash; }); + local_op.reindex_after_growth(base, n_miss); } auto finish(CosMask &&cos_all, CosMask *out_cos = nullptr) -> std::shared_ptr { @@ -455,7 +482,8 @@ struct LayerBuildEngine { auto response_recv_counts() const -> std::vector { std::vector counts(R); for (size_t r = 0; r < R; ++r) { - counts[r] = static_cast(queries_r[r].size() / kQueryWords); + // One response per query, and src_idx_r[r] holds one source per query: no walk, no division. + counts[r] = static_cast(src_idx_r[r].size()); } return counts; } @@ -463,19 +491,19 @@ struct LayerBuildEngine { // Batched self-resolve over the index's group-prefetch find_batch; hits/misses are emitted to the sink // in query order. `lv` is the per-query v_src array parallel to `ls` (read only when Sink::wants_values). static constexpr size_t kResolveBatch = 64; - auto resolve_range_(VecZ &lq, - std::vector &ls, - [[maybe_unused]] std::vector *lv, - size_t lo, - size_t hi, - bool is_leader_pass) -> void { + auto resolve_range_(std::vector &ls, [[maybe_unused]] std::vector *lv, bool is_leader_pass) + -> void { const size_t op_size = local_op.store->size(); - std::array, kResolveBatch> keys; + // Gathered per batch because a matched follower is skipped; offsets stay absolute into pos_flat. + std::array pos_off; + std::array k_of; + std::array hashes; std::array phases; std::array srcs; std::array vals; std::array found; - size_t q = lo; + const size_t hi = self_stage_.size(); + size_t q = 0; while (q < hi) { size_t m = 0; for (; q < hi && m < kResolveBatch; ++q) { @@ -483,7 +511,9 @@ struct LayerBuildEngine { if (!is_leader_pass && matched.is_marked(src)) { continue; // follower already matched by a leader → not an independent rotation } - query_read(lq, q, keys[m], phases[m]); + pos_off[m] = self_stage_.pos_off[q]; + k_of[m] = self_stage_.k_of[q]; + phases[m] = self_stage_.phase_of[q]; srcs[m] = src; if constexpr (Sink::wants_values) { vals[m] = (*lv)[q]; @@ -493,7 +523,12 @@ struct LayerBuildEngine { if (m == 0) { break; } - local_op.store->find_batch(keys.data(), m, found.data()); + // The hashes come back because a miss needs one at insert, folded from these same positions. + local_op.store->find_batch_positions(std::span(self_stage_.pos_flat), + std::span(pos_off).first(m), + std::span(k_of).first(m), + std::span(found).first(m), + std::span(hashes).first(m)); for (size_t j = 0; j < m; ++j) { double v_src = 0.0; if constexpr (Sink::wants_values) { @@ -508,7 +543,11 @@ struct LayerBuildEngine { sink.self_hit(srcs[j], found[j], phases[j], v_src); } else { - deferred_self_misses.push_back({keys[j], srcs[j], phases[j], v_src}); + // The stage dies with this pass and the misses are flushed after both, so copy now. + const size_t at = deferred_pos_flat_.size(); + const auto *const first = self_stage_.pos_flat.data() + pos_off[j]; + deferred_pos_flat_.insert(deferred_pos_flat_.end(), first, first + k_of[j]); + deferred_self_misses.push_back({at, k_of[j], hashes[j], srcs[j], phases[j], v_src}); } } } @@ -560,7 +599,7 @@ auto build_layer(MPOperator &local_op, } assert(fused_scale_coeffs == nullptr || (local_coeffs && &local_coeffs->get() == fused_scale_coeffs)); - FusedScanResult fused = [&] { + FusedScanResult fused = [&] { double *const sweep_ptr = fused_scale ? fused_scale_coeffs->data() : nullptr; return with_algebra(basis, [&]() { return fused_find_and_collect(local_op, @@ -602,11 +641,13 @@ auto build_layer(MPOperator &local_op, eng.run_exchange(/*is_leader_pass=*/true, std::move(fused.leader_queries), std::move(fused.leader_src), - std::move(fused.leader_val)); + std::move(fused.leader_val), + std::move(fused.leader_self)); eng.run_exchange(/*is_leader_pass=*/false, std::move(fused.follower_queries), std::move(fused.follower_src), - std::move(fused.follower_val)); + std::move(fused.follower_val), + std::move(fused.follower_self)); return eng.finish(std::move(cos_all), out_cos); }; diff --git a/cpp/monoprop/detail/evolution/layer_build/PartnerMerge.h b/cpp/monoprop/detail/evolution/layer_build/PartnerMerge.h new file mode 100644 index 00000000..2ab49d98 --- /dev/null +++ b/cpp/monoprop/detail/evolution/layer_build/PartnerMerge.h @@ -0,0 +1,148 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// M⊕G as ascending positions: a slot in both M and G cancels, so the partner is the symmetric +// difference of two ascending position lists, and one merge yields the positions and overlap together. + +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/operator/OperatorIndex.h" + +namespace monoprop::detail { + +/*! @brief The symmetric difference's length together with the count of cancelled positions. */ +struct MergedPartner { + size_t count; //!< positions written to out + size_t overlap; //!< positions present in both inputs, which therefore cancelled +}; + +/*! @brief Writes the symmetric difference of `a` and `b` to `out`. + * + * Both inputs must be ascending, or the result is silently wrong; `out` needs room for + * `a.size() + b.size()`. One pass yields the positions and the overlap together. + */ +template +[[gnu::always_inline]] inline auto merge_partner_positions(const Row &a, const Gen &b, Out &&out) noexcept + -> MergedPartner { + using PosT = std::ranges::range_value_t; + const size_t ka = std::ranges::size(a); + const size_t kb = std::ranges::size(b); + size_t i = 0; + size_t j = 0; + size_t n = 0; + size_t overlap = 0; + while (i < ka && j < kb) { + const size_t pa = static_cast(a[i]); + const size_t pb = static_cast(b[j]); + if (pa == pb) { + ++overlap; + ++i; + ++j; + continue; + } + const size_t p = pa < pb ? pa : pb; + i += static_cast(pa < pb); + j += static_cast(pb < pa); + out[n++] = static_cast(p); + } + for (; i < ka; ++i) { + out[n++] = static_cast(a[i]); + } + for (; j < kb; ++j) { + out[n++] = static_cast(b[j]); + } + return {n, overlap}; +} + +/*! @brief Stages self-owned query positions for direct use by OperatorIndex's + * find_batch_positions and set_positions, with no encoding step. + */ +template +struct SelfQueryStage { + using PosT = typename OperatorIndex::PosT; + + //! Sized to capacity, not filled: the logical length is size()/positions(), not the vectors' own size(). + DefaultInitVector pos_flat; //!< ascending positions, concatenated in push order + DefaultInitVector pos_off; //!< query -> absolute offset into pos_flat + DefaultInitVector k_of; //!< positions per query + DefaultInitVector phase_of; //!< emit_phase is ternary, so a byte is the whole range + + [[nodiscard]] auto size() const -> size_t { return n_; } + [[nodiscard]] auto positions() const -> size_t { return pos_n_; } + + auto clear() -> void { + n_ = 0; + pos_n_ = 0; + } + + auto reserve(size_t n_queries, size_t positions_per_query) -> void { + if (pos_off.size() < n_queries) { + pos_off.resize(n_queries); + k_of.resize(n_queries); + phase_of.resize(n_queries); + } + if (pos_flat.size() < n_queries * positions_per_query) { + pos_flat.resize(n_queries * positions_per_query); + } + } + + //! Appends one query's positions and its (offset, k, phase) record; grows only when capacity runs out. + template + auto push(const Pos &pos, int phase) -> void { + assert(phase >= -1 && phase <= 1 && "emit_phase is ternary: rotation_sign, or REAL_PARTS entry"); + const size_t k = std::ranges::size(pos); + const size_t n = n_; + const size_t at = pos_n_; + if (n == pos_off.size() || at + k > pos_flat.size()) { + grow_(k); + } + pos_off[n] = at; + PosT *dst = pos_flat.data() + at; + for (size_t j = 0; j < k; ++j) { + dst[j] = pos[j]; + } + k_of[n] = static_cast(k); + phase_of[n] = static_cast(phase); + n_ = n + 1; + pos_n_ = at + k; + } + +private: + size_t n_ = 0; //!< queries pushed + size_t pos_n_ = 0; //!< positions pushed + + //! Doubles capacity, but grows pos_flat by at least what this push needs, so a wide term can't leave it short. + [[gnu::noinline]] auto grow_(size_t k) -> void { + if (n_ == pos_off.size()) { + const size_t want = (pos_off.size() * 2) + 64; + pos_off.resize(want); + k_of.resize(want); + phase_of.resize(want); + } + if (pos_n_ + k > pos_flat.size()) { + pos_flat.resize(std::max((pos_flat.size() * 2) + 256, pos_n_ + k)); + } + } +}; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/layer_build/QueryWire.h b/cpp/monoprop/detail/evolution/layer_build/QueryWire.h new file mode 100644 index 00000000..7f296d8a --- /dev/null +++ b/cpp/monoprop/detail/evolution/layer_build/QueryWire.h @@ -0,0 +1,336 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/evolution/layer_build/Common.h" + +namespace monoprop::detail { + +/*! @brief `Fused` records carry a trailing value word after the positions; `Plain` ones do not. */ +enum class QueryForm { Plain, Fused }; + +/*! @brief One term's wire record for a cross-rank query: its ascending set-bit positions, + * gap-coded, plus a phase. + * + * It replaces a fixed dense stride, which would spend one word per 64 modes however few bits a + * term actually sets. Fields pack LSB-first from word 0: a 2-bit phase biased to unsigned, a + * 5-bit popcount @p k whose all-ones value escapes to a wider field, a 4-bit gap width @p gw, + * then the first position in `kPosBits` bits and the remaining k-1 positions as gaps of @p gw + * bits to their predecessor. Both k and gw are per-record, so a two-bit term and a full-support + * one each cost what they are. + * + * At `NumModes = 128` (`kBits = 256`, so `kPosBits = 8`) a term at positions {3, 7, 8, 40} with + * phase +1 has k = 4 and gaps {3, 0, 31}, hence `gw = bit_width(31) = 5`: + * + * @code + * bits 0..1 phase 2 +1, biased by 1 + * bits 2..6 k 4 below kKEscape, so no wide-k field follows + * bits 7..10 gw 5 + * bits 11..18 first 3 + * bits 19..23 gap 3 7 - 3 - 1 + * bits 24..28 gap 0 8 - 7 - 1 + * bits 29..33 gap 31 40 - 8 - 1 + * @endcode + * + * 34 bits, so one word, against the four a 256-bit dense stride spends on the same term. + */ +template +struct QueryWire { + using PosT = uint16_t; + using WireView = std::span; //!< a serialized record stream, read-only + + static constexpr size_t kBits = 2 * NumModes; + static_assert(kBits <= 65535, "a physical bit position and the popcount must both fit a uint16_t"); + + //! Bits for one raw position in [0, 2*NumModes); compile-time, so the lane width is free. + static constexpr size_t kPosBits = static_cast(std::bit_width(kBits - 1)); + + static constexpr size_t kPhaseBits = 2; + static constexpr size_t kKBits = 5; + //! k is a popcount of a kBits bitset, so the escape field never needs more bits than this. + static constexpr size_t kLongKBits = static_cast(std::bit_width(kBits)); + static constexpr size_t kGwBits = 4; + static constexpr size_t kKEscape = (1U << kKBits) - 1U; + static constexpr size_t kHeaderBits = kPhaseBits + kKBits + kGwBits; + static_assert(kPosBits <= (1U << kGwBits) - 1U, "gw <= kPosBits must fit the header's gap-width field"); + static_assert(kHeaderBits + kLongKBits <= 64, "the widest header must be readable from word 0 alone"); + + static constexpr size_t kMaxPositions = kBits; + + //! Reserve hint only, for a caller batching many records into one flat position buffer. + static constexpr size_t kReservePositionsPerQuery = 6; + + /*! @brief What one decode call read and consumed. */ + struct Decoded { + size_t next; //!< word offset just past what this call consumed + int phase; //!< the record's ternary phase + }; + + /*! @brief Bit-packs variable-width fields into `buf`, one 64-bit word at a time. */ + struct Writer { + VecZ &buf; + uint64_t cur = 0; + size_t nbits = 0; //!< bits held in cur, always less than 64 + size_t words = 0; + + [[gnu::always_inline]] auto put(uint64_t v, size_t width) noexcept -> void { + if (width == 0) { + return; + } + // Assert before masking: masking alone turns an overflow into a different well-formed record. + assert((v >> width) == 0 && "field value does not fit its width"); + v &= (uint64_t{1} << width) - 1U; + cur |= v << nbits; + if (nbits + width < 64) { + nbits += width; + return; + } + buf.push_back(static_cast(cur)); + ++words; + cur = v >> (64U - nbits); + nbits = nbits + width - 64U; + } + + auto flush() noexcept -> void { + if (nbits != 0) { + buf.push_back(static_cast(cur)); + ++words; + cur = 0; + nbits = 0; + } + } + }; + + /*! @brief Unpacks variable-width fields out of `buf`, starting at word `base`. */ + struct Reader { + WireView buf; + size_t base; //!< word offset of the record start + size_t nbits = 0; + + [[nodiscard]] auto get(size_t width) noexcept -> uint64_t { + if (width == 0) { + return 0; + } + const size_t word = nbits >> 6U; + const size_t off = nbits & 63U; + uint64_t v = static_cast(buf[base + word]) >> off; + if (off + width > 64) { + v |= static_cast(buf[base + word + 1]) << (64U - off); + } + nbits += width; + return (width < 64) ? (v & ((uint64_t{1} << width) - 1U)) : v; + } + }; + + /*! @brief A record's decoded header fields. */ + struct Header { + int phase = 0; + size_t k = 0; + size_t gw = 0; + size_t bits = 0; //!< header width, i.e. where the payload begins + }; + + [[nodiscard]] static auto header_at(WireView buf, size_t off) noexcept -> Header { + const auto w0 = static_cast(buf[off]); + Header h; + h.phase = static_cast(w0 & 0x3U) - 1; + h.k = static_cast((w0 >> kPhaseBits) & kKEscape); + h.bits = kPhaseBits + kKBits; + if (h.k == kKEscape) { + h.k = static_cast((w0 >> h.bits) & ((uint64_t{1} << kLongKBits) - 1U)); + h.bits += kLongKBits; + } + h.gw = static_cast((w0 >> h.bits) & ((uint64_t{1} << kGwBits) - 1U)); + h.bits += kGwBits; + return h; + } + + [[nodiscard]] static constexpr auto header_bits_for(size_t k) noexcept -> size_t { + return kHeaderBits + ((k >= kKEscape) ? kLongKBits : 0U); + } + + [[nodiscard]] static constexpr auto gap_bits(size_t k, size_t gw) noexcept -> size_t { + return header_bits_for(k) + ((k == 0) ? 0U : kPosBits + (k - 1U) * gw); + } + [[nodiscard]] static constexpr auto words_of(size_t bits) noexcept -> size_t { return (bits + 63U) / 64U; } + + //! The record's word count, derived from the header alone: k does not determine it since gw varies too. + [[nodiscard]] static constexpr auto words_of_header(const Header &h) noexcept -> size_t { + return words_of(gap_bits(h.k, h.gw)); + } + + [[nodiscard]] static auto words_at(WireView buf, size_t off) noexcept -> size_t { + return words_of_header(header_at(buf, off)); + } + + [[nodiscard]] static auto k_at(WireView buf, size_t off) noexcept -> size_t { return header_at(buf, off).k; } + [[nodiscard]] static auto phase_at(WireView buf, size_t off) noexcept -> int { return header_at(buf, off).phase; } + + //! gw = bit_width(max gap), folded into push()'s own pass over the positions. + template + [[nodiscard]] static auto gap_width(const Pos &pos) noexcept -> size_t { + const size_t k = std::ranges::size(pos); + size_t g = 0; + for (size_t j = 1; j < k; ++j) { + const size_t d = static_cast(pos[j] - pos[j - 1] - 1U); + const auto b = static_cast(std::bit_width(d)); + g = (b > g) ? b : g; + } + return g; + } + + /*! @brief Appends one record for `pos` and `phase`, and returns the words written. + * + * `pos` must be strictly ascending with every position below `kBits`; a violation encodes a + * different, still well-formed record rather than failing. The element type is deduced because + * the store's position width is narrower than the wire's below 129 modes. + */ + template + static auto push(VecZ &buf, const Pos &pos, int phase) -> size_t { + const size_t k = std::ranges::size(pos); + assert(k <= kMaxPositions && "term has more positions than the record's width admits"); + assert(phase >= -1 && phase <= 1 && "emit_phase is ternary: rotation_sign, or REAL_PARTS entry"); + + const size_t gw = gap_width(pos); + Writer w{buf}; + w.put(static_cast(phase + 1), kPhaseBits); + if (k >= kKEscape) { + w.put(kKEscape, kKBits); + w.put(static_cast(k), kLongKBits); + } + else { + w.put(static_cast(k), kKBits); + } + w.put(static_cast(gw), kGwBits); + if (k != 0) { + w.put(static_cast(pos[0]), kPosBits); + for (size_t j = 1; j < k; ++j) { + w.put(static_cast(pos[j] - pos[j - 1] - 1U), gw); + } + } + w.flush(); + return w.words; + } + + /*! @brief Decodes the record at `off` into `out`, whose element type is deduced so the resolve + * path can decode straight into the store's narrower position width. + * + * `Decoded::next` names the word just past the positions, excluding any value word. + */ + template + static auto read_positions(WireView buf, size_t off, Out &&out) -> Decoded { + using OutT = std::ranges::range_value_t; + const Header h = header_at(buf, off); + Reader r{buf, off, h.bits}; + if (h.k != 0) { + auto prev = static_cast(r.get(kPosBits)); + out[0] = static_cast(prev); + for (size_t j = 1; j < h.k; ++j) { + prev += static_cast(r.get(h.gw)) + 1U; + out[j] = static_cast(prev); + } + } + return {off + words_of_header(h), h.phase}; + } + + //! d, recomputed rather than carried: in ascending order a pair is an even position then its successor. + template + [[nodiscard]] static auto pair_count(const Pos &pos) noexcept -> size_t { + const size_t k = std::ranges::size(pos); + size_t d = 0; + for (size_t j = 0; j + 1 < k; ++j) { + if ((pos[j] % 2 == 0) && (pos[j + 1] == pos[j] + 1)) { + ++d; + } + } + return d; + } + + //! Offset of the next record in a stream; `off` always names the start of one. + [[nodiscard]] static auto next_off(WireView buf, QueryForm form, size_t off) -> size_t { + return off + words_at(buf, off) + (form == QueryForm::Fused ? 1U : 0U); + } + + /*! @brief Decodes one record from a query stream, whose `form` says whether a value word + * follows. `Decoded::next` names the following record. + */ + template + static auto read_query(WireView buf, QueryForm form, size_t off, Out &&out) -> Decoded { + Decoded d = read_positions(buf, off, std::forward(out)); + d.next += (form == QueryForm::Fused ? 1U : 0U); + return d; + } + + //! The fused value word, a bit_cast that follows a record's positions. + [[nodiscard]] static auto value_at(WireView buf, size_t off) -> double { + return decode_value(buf[off + words_at(buf, off)]); + } + + static auto push_value(VecZ &buf, double v) -> void { buf.push_back(encode_value(v)); } + + //! Number of records in the stream: widths vary, so this walks rather than divides. + [[nodiscard]] static auto count_queries(WireView buf, QueryForm form) -> size_t { + size_t off = 0; + size_t n = 0; + while (off < buf.size()) { + off = next_off(buf, form, off); + ++n; + } + return n; + } + + //! Interleaves a plain query stream with its parallel value array into one fused stream. + static auto build_fused(WireView queries, std::span vals, VecZ &out) -> void { + out.clear(); + out.reserve(queries.size() + vals.size()); + size_t off = 0; + size_t i = 0; + while (off < queries.size()) { + const size_t n = words_at(queries, off); + out.insert(out.end(), + queries.begin() + static_cast(off), + queries.begin() + static_cast(off + n)); + assert(i < vals.size() && "fused build needs exactly one value per query"); + out.push_back(encode_value(vals[i])); + off += n; + ++i; + } + } + + //! Copies the record at src_off (and its value word, if fused) to dst_off; returns the words moved. + static auto move_query(VecZ &buf, QueryForm form, size_t src_off, size_t dst_off) -> size_t { + const size_t n = words_at(buf, src_off) + (form == QueryForm::Fused ? 1U : 0U); + if (src_off != dst_off) { + assert(dst_off < src_off && "compaction only ever moves a query earlier"); + std::copy(buf.begin() + static_cast(src_off), + buf.begin() + static_cast(src_off + n), + buf.begin() + static_cast(dst_off)); + } + return n; + } +}; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/layer_build/Resolve.h b/cpp/monoprop/detail/evolution/layer_build/Resolve.h index ef90d91f..0b1787a0 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Resolve.h +++ b/cpp/monoprop/detail/evolution/layer_build/Resolve.h @@ -16,12 +16,14 @@ #include #include +#include #include #include "monoprop/TypeAliases.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" @@ -33,28 +35,61 @@ namespace monoprop::detail { // pairwise distinct ⇒ misses distinct and absent. template struct IncomingProbe { - std::vector goff; // rank_count+1 flat offsets: g = goff[s] + q - DefaultInitVector sender_of; // g → sender rank - DefaultInitVector> mono; // g → deserialized query monomial - DefaultInitVector phase_of; // g → query phase - DefaultInitVector idx_of; // g → resolved index (hit: < base; miss: base+j) - std::vector miss_g; // j → the g that became miss j (Phase 4 reads mono[miss_g[j]]) - size_t base = 0; // op size before the miss inserts (the miss-index base) + // The operator store's position width, not the wire's: these positions exist to become rows. + using PosT = typename OperatorIndex::PosT; + + std::vector goff; // rank_count+1 flat offsets: g = goff[s] + q + DefaultInitVector sender_of; // g → sender rank + DefaultInitVector phase_of; // g → query phase + // g → word offset of that query inside incoming[sender_of[g]]; a query ordinal names no position. + DefaultInitVector off_of; + DefaultInitVector idx_of; // g → resolved index (hit: < base; miss: base+j) + std::vector miss_g; // j → the g that became miss j (Phase 4 reads the key of miss_g[j]) + size_t base = 0; // op size before the miss inserts (the miss-index base) size_t nq_total = 0; + + // The queries as they arrived, flat: query g owns pos_flat[pos_off[g] .. pos_off[g] + k_of[g]). + DefaultInitVector pos_flat; + DefaultInitVector pos_off; + DefaultInitVector k_of; + // g → fold_hash of the query key, folded by the probe and reused by the insert. + DefaultInitVector hash_of; + + //! Query g's ascending positions, as a view into pos_flat. + [[nodiscard]] auto positions_at(size_t g) const -> std::span { + return std::span(pos_flat).subspan(pos_off[g], k_of[g]); + } + + //! Builds a dense bitset; only the fully paired minority of callers needs one. + [[nodiscard]] auto mono_at(size_t g) const -> Monomial { + Monomial m; + for (const PosT q : positions_at(g)) { + m.set(static_cast(q)); + } + return m; + } + + //! Every mode of query g carries both Majoranas, read off the positions. + [[nodiscard]] auto is_paired_at(size_t g) const -> bool { + const auto pos = positions_at(g); + return pos.size() == 2 * QueryWire::pair_count(pos); + } }; -// Phases 1-2, read-only w.r.t. operator contents. QW = per-record stride: the plain query width, or -// kQueryWordsFused for the fused resolver. The caller runs Phase 3, then insert_incoming_misses. -template > +// Read-only phases 1-2 of the exchange: `form` says whether the incoming records are fused +// (ContractSink) or plain (GraphSink). The caller runs phase 3, then insert_incoming_misses. +template auto probe_incoming_queries(const std::vector &incoming, // serialized, one VecZ per sender MPOperator &op, - size_t rank_count) -> IncomingProbe { - constexpr size_t W = QW; + size_t rank_count, + QueryForm form) -> IncomingProbe { + using QW = QueryWire; + using PosT = typename IncomingProbe::PosT; IncomingProbe pr; pr.goff.assign(rank_count + 1, 0); for (size_t s = 0; s < rank_count; ++s) { - const size_t nq = incoming[s].empty() ? 0 : incoming[s].size() / W; + const size_t nq = QW::count_queries(incoming[s], form); pr.goff[s + 1] = pr.goff[s] + nq; } pr.nq_total = pr.goff[rank_count]; @@ -69,22 +104,38 @@ auto probe_incoming_queries(const std::vector &incoming, // serialized, on static_cast(s)); } - // Phase 1 (read-only): deserialize, then probe with the group-prefetch batch find. - pr.mono.resize(pr.nq_total); + // Phase 1 (read-only): deserialize, then probe with the group-prefetch batch find. One walk per sender. pr.phase_of.resize(pr.nq_total); + pr.off_of.resize(pr.nq_total); pr.idx_of.resize(pr.nq_total); - for (size_t g = 0; g < pr.nq_total; ++g) { - const size_t s = pr.sender_of[g]; - const size_t q = g - pr.goff[s]; - Monomial m; - int ph = 0; - query_read(incoming[s], q, m, ph); - pr.mono[g] = m; - pr.phase_of[g] = ph; + pr.pos_off.resize(pr.nq_total); + pr.k_of.resize(pr.nq_total); + pr.hash_of.resize(pr.nq_total); + pr.pos_flat.clear(); + // A hint only, so this stays one allocation for the common case. + pr.pos_flat.reserve(pr.nq_total * QW::kReservePositionsPerQuery); + for (size_t s = 0; s < rank_count; ++s) { + size_t off = 0; + for (size_t g = pr.goff[s]; g < pr.goff[s + 1]; ++g) { + const size_t k = QW::k_at(incoming[s], off); + const size_t at = pr.pos_flat.size(); + pr.pos_flat.resize(at + k); // default-init grow: read_query writes every element + pr.pos_off[g] = at; + pr.k_of[g] = static_cast(k); + pr.off_of[g] = off; + const auto d = QW::read_query(incoming[s], form, off, std::span(pr.pos_flat).subspan(at, k)); + pr.phase_of[g] = d.phase; + off = d.next; + } } { const size_t op_size = op.store->size(); - op.store->find_batch(pr.mono.data(), pr.nq_total, pr.idx_of.data()); + // The vectors are sized to capacity, not to nq_total, so every span is trimmed explicitly. + op.store->find_batch_positions(std::span(pr.pos_flat), + std::span(pr.pos_off).first(pr.nq_total), + std::span(pr.k_of).first(pr.nq_total), + std::span(pr.idx_of).first(pr.nq_total), + std::span(pr.hash_of).first(pr.nq_total)); for (size_t g = 0; g < pr.nq_total; ++g) { if (pr.idx_of[g] >= op_size) { // kNotFound is size_t max → also lands here pr.idx_of[g] = kMissingIndex; @@ -111,11 +162,15 @@ auto insert_incoming_misses(MPOperator &op, const IncomingProbe( - op, - n_miss, - [&](size_t j) -> const Monomial & { return pr.mono[pr.miss_g[j]]; }, - [&](size_t j, size_t base) { assign_row(*op.store, base + j, pr.mono[pr.miss_g[j]]); }); + // insert_absent_terms' three steps without its two dense round-trips, and on the same ordering + // contract, which is what matters: slot j lands at base+j, in miss order = (sender, record) order. + const size_t base = op.store->grow_rows_geometric(n_miss); + for (size_t j = 0; j < n_miss; ++j) { + const size_t g = pr.miss_g[j]; + op.store->set_positions(base + j, pr.positions_at(g)); + } + op.store->bulk_insert_hashed(n_miss, base, [&](size_t j) { return pr.hash_of[pr.miss_g[j]]; }); + op.reindex_after_growth(base, n_miss); } // resolve_incoming / process_responses are the picture-independent cross-rank exchange skeletons; what @@ -134,7 +189,7 @@ auto resolve_incoming(const std::vector &incoming, // serialized, one VecZ size_t combined_size, // pre-layer op size: bounds the matched set Sink &sink) -> std::vector> { using Resp = typename Sink::Response; - const IncomingProbe pr = probe_incoming_queries(incoming, op, rank_count); + const IncomingProbe pr = probe_incoming_queries(incoming, op, rank_count, sink.incoming_form()); std::vector> responses(rank_count); for (size_t s = 0; s < rank_count; ++s) { responses[s].assign(pr.goff[s + 1] - pr.goff[s], Sink::init_response()); diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index d5b8a77e..23fe3a4e 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "monoprop/TypeAliases.h" @@ -29,6 +30,8 @@ #include "monoprop/core/Monomial.h" #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/PartnerMerge.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" #include "monoprop/detail/graph_encoding/MPGraphEncodingTypes.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/operator/InvertedIndex.h" @@ -157,23 +160,52 @@ inline auto rotation_dynamic_gate(std::optional only_rotate_len_k, return true; } +// The dense form is unavoidable: the owner hash folds every word and the basis sign reads the source +// bitset, so it is built regardless, and the merge below runs beside it. +template +struct PartnerProduct { + Monomial new_mono; + size_t k = 0; // popcount(M⊕G) + size_t overlap = 0; // slots in both M and G, which cancel + int phase_factor = 0; +}; + // phase_factor is the basis-specific sign only: Majorana interleave_phase, still to be folded with -// hermitian_phase at emit; Pauli pauli_rotation_sign, already rotation-ready. -template +// hermitian_phase at emit; Pauli pauli_rotation_sign, already rotation-ready. `out_pos` receives the +// partner's ascending positions; a spilled source row has no position array, so that case walks the +// dense partner to fill it instead. +template [[gnu::always_inline]] inline auto emit_term_products(const OperatorIndex &ham, size_t i, const typename A::GenContext &ctx, - Monomial &new_mono, - size_t &overlap, - int &phase_factor) -> void { - Monomial mono; - ham.for_each_position(i, [&](size_t pos) { mono.set(pos); }); + std::span gen_pos, + std::span out_pos) -> PartnerProduct { + const size_t gen_pop = gen_pos.size(); const Monomial &gen = A::generator(ctx); - new_mono = mono ^ gen; - overlap = mono.count_and(gen); - phase_factor = A::rotation_sign(ctx, mono, new_mono); + PartnerProduct out; + Monomial mono; + if (const auto src = ham.row_positions(i); src.inlined()) { + const auto merged = merge_partner_positions(src.pos, gen_pos, out_pos); + out.k = merged.count; + out.overlap = merged.overlap; + for (const PosT q : src.pos) { + mono.set(static_cast(q)); + } + out.new_mono = mono ^ gen; + } + else { + ham.for_each_position(i, [&](size_t pos) { mono.set(pos); }); + out.new_mono = mono ^ gen; + out.overlap = mono.count_and(gen); + for (size_t b = out.new_mono.find_first(); b < out.new_mono.size(); b = out.new_mono.find_next(b)) { + out_pos[out.k++] = static_cast(b); + } + } + out.phase_factor = A::rotation_sign(ctx, mono, out.new_mono); + return out; } +template struct FusedScanResult { std::vector cos_blocks; // ascending, disjoint, chunk order std::vector leader_queries; // size R: serialized leader queries per owner rank @@ -184,6 +216,10 @@ struct FusedScanResult { // leader_src / follower_src. Empty when capture_values is false. std::vector> leader_val; std::vector> follower_val; + // Self-owned queries, staged as positions instead of queued to the wire and resolved inline. + // Order must match leader_src[my_rank] / follower_src[my_rank], or resolution attributes the wrong source. + SelfQueryStage leader_self; + SelfQueryStage follower_self; }; // Classify, cut off and emit in one pass over the anticommuting terms. Queries go to the owner of @@ -202,12 +238,12 @@ auto fused_find_and_collect(const MPOperator &op, size_t my_rank, bool capture_values = false, double *fused_scale_coeffs = nullptr, - double fused_scale_cos = 1.0) -> FusedScanResult { + double fused_scale_cos = 1.0) -> FusedScanResult { validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); const size_t gen_pop = gen.count(); const auto ectx = A::make_gen_context(gen); - FusedScanResult res; + FusedScanResult res; res.leader_queries.assign(rank_count, VecZ{}); res.leader_src.assign(rank_count, std::vector{}); res.follower_queries.assign(rank_count, VecZ{}); @@ -268,40 +304,61 @@ auto fused_find_and_collect(const MPOperator &op, auto &fs = res.follower_src; auto &fv = res.follower_val; + const OperatorIndex &ham = *op.store; + using RowPosT = typename OperatorIndex::PosT; + + // The generator's positions, once per gate: the merge's second input. + std::vector gen_pos; + gen_pos.reserve(gen_pop); + for (size_t b = gen.find_first(); b < gen.size(); b = gen.find_next(b)) { + gen_pos.push_back(static_cast(b)); + } + // pbuf capacity is 2*NumModes: the partner's positions are distinct, so this always suffices. + std::vector pbuf(2 * NumModes); + + auto push = [&](const Monomial &dense, + std::span pos, + int phase, + size_t i, + double v_src, + bool is_follower) { + // Single rank: every partner is self-owned, skip the O(W) hash; multi-rank routes by owner. + // Must be the same function find_rank computes (MPIUtils.h) or a term is placed and queried + // on different ranks, which duplicates a row silently; mpi_utils_tests.cpp asserts it. + size_t r_prime = my_rank; + if (rank_count != 1) { + r_prime = monomial_hash(dense) % rank_count; + } + if (r_prime == my_rank) { + (is_follower ? res.follower_self : res.leader_self).push(pos, phase); + } + else { + QueryWire::push(is_follower ? fq[r_prime] : lq[r_prime], pos, phase); + } + (is_follower ? fs[r_prime] : ls[r_prime]).push_back(i); + if (capture_values) { + (is_follower ? fv[r_prime] : lv[r_prime]).push_back(v_src); + } + }; + // The dynamic gate runs before emit_term_products, so a gate-rejected term computes no products. // abs_c/v_src come from the caller's coeff read, not re-read. auto emit = [&](size_t mono_pop, size_t i, double abs_c, double v_src, bool is_follower) { if (!rotation_dynamic_gate(only_rotate_len_k, mono_pop, cut_st, abs_c)) { return; } - Monomial new_mono; - size_t overlap = 0; - int phase_factor = 0; - emit_term_products(*op.store, i, ectx, new_mono, overlap, phase_factor); + const auto p = emit_term_products(ham, + i, + ectx, + std::span(gen_pos), + std::span(pbuf)); // Structural cutoff on the partner M⊕G, unless upper_atol rescues it (CutoffContext::is_above_upper). - const size_t new_pop = mono_pop + gen_pop - 2 * overlap; - const bool struct_pass = cutoff_eval.passes_with_popcount(new_mono, new_pop); + const bool struct_pass = cutoff_eval.passes_with_popcount(p.new_mono, p.k); if (!struct_pass && !cut_st.is_above_upper(abs_c)) { return; } - const int phase = A::emit_phase(phase_factor, mono_pop, gen_pop, overlap); - // Single rank: every partner is self-owned, skip the O(W) hash; multi-rank routes by owner. - const size_t r_prime = (rank_count == 1) ? my_rank : (monomial_hash(new_mono) % rank_count); - const size_t source = i; - if (is_follower) { - query_push(fq[r_prime], new_mono, phase); - fs[r_prime].push_back(source); - if (capture_values) { - fv[r_prime].push_back(v_src); - } - } - else { - query_push(lq[r_prime], new_mono, phase); - ls[r_prime].push_back(source); - if (capture_values) { - lv[r_prime].push_back(v_src); - } - } + const int phase = A::emit_phase(p.phase_factor, mono_pop, gen_pop, p.overlap); + push(p.new_mono, std::span(pbuf).first(p.k), phase, i, v_src, is_follower); }; // Pass 1 and pass 2 stay fused over `nz`: splitting them regressed measurably, as `nz` spills L1 @@ -327,9 +384,11 @@ auto fused_find_and_collect(const MPOperator &op, n_foll); } if (rank_count == 1) { - lq[my_rank].reserve((n_anti - n_foll) * kQueryWords); + // A hint only; wider terms grow the buffer as needed. + const size_t pq = QueryWire::kReservePositionsPerQuery; + res.leader_self.reserve(n_anti - n_foll, pq); ls[my_rank].reserve(n_anti - n_foll); - fq[my_rank].reserve(n_foll * kQueryWords); + res.follower_self.reserve(n_foll, pq); fs[my_rank].reserve(n_foll); } auto derive_coeff = [&](size_t i) -> std::pair { diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index 8a3c29b2..f817c4cf 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -17,11 +17,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -96,6 +98,9 @@ class OperatorIndex { [[nodiscard]] auto size() const -> size_t { return size_; } + // Rows that exceeded inline_width_ and spilled; observable so a test can compare the two insert paths. + [[nodiscard]] auto overflow_size() const -> size_t { return overflow_.size(); } + auto reserve(size_t n) -> void { reserve_rows(n); reserve_index(n); @@ -137,6 +142,32 @@ class OperatorIndex { } } + // set() from the row's own form: a row is an ascending position list. Same postcondition as set(), + // including the dropped stale overflow entry. + // + // Precondition: `pos` strictly ascending, every entry < 2*NumModes. A violation is silent in release + // -- an unsorted row simply never matches, and an out-of-range one decodes to a different term. + auto set_positions(size_t i, std::span pos) -> void { + const size_t count = pos.size(); + assert((count == 0 || static_cast(pos[count - 1]) < 2 * NumModes) && "row position out of range"); + PosT *row = &rows_[i * stride_]; + if (count > inline_width_) { + // The spill path has no position array, so build the dense form -- only here. + row[0] = kOverflowMarker; + value_type mono; + for (size_t j = 0; j < count; ++j) { + mono.set(pos[j]); + } + overflow_[i] = mono; + return; + } + if (!overflow_.empty()) { + overflow_.erase(i); + } + row[0] = static_cast(count); + std::copy_n(pos.data(), count, row + 1); + } + [[nodiscard]] auto row(size_t i) const -> value_type { const PosT c = rows_[i * stride_]; if (c == kOverflowMarker) { @@ -170,6 +201,19 @@ class OperatorIndex { } return overflow_.at(i).count(); } + /*! @brief The row's stored ascending positions, empty for a spilled row. Invalidated by any insert. */ + struct RowPositions { + std::span pos; + //! A spilled row has no position array at all, which an empty inline row still does. + [[nodiscard]] auto inlined() const -> bool { return pos.data() != nullptr; } + }; + [[nodiscard]] auto row_positions(size_t i) const -> RowPositions { + const PosT c = rows_[i * stride_]; + if (c == kOverflowMarker) { + return {}; + } + return {std::span(&rows_[(i * stride_) + 1], static_cast(c))}; + } [[nodiscard]] auto memory_bytes() const -> size_t { size_t total = rows_.capacity() * sizeof(PosT); total += overflow_.size() * (sizeof(value_type) + sizeof(size_t) + 24); @@ -233,6 +277,64 @@ class OperatorIndex { } } + // find_batch over ascending position lists: query q is pos_flat[pos_off[q] .. pos_off[q] + k_of[q]). + // Identical results to find_batch on the monomials those positions describe. + auto find_batch_positions(std::span pos_flat, + std::span pos_off, + std::span k_of, + std::span out, + std::span hash_out = {}) const -> void { + const size_t n = pos_off.size(); + static constexpr size_t G = 16; + std::array hh; + std::array sp; + std::array cand; + for (size_t base = 0; base < n; base += G) { + const size_t g = std::min(G, n - base); + for (size_t j = 0; j < g; ++j) { + hh[j] = fold_hash_positions(pos_flat.subspan(pos_off[base + j], k_of[base + j])); + sp[j] = spread(hh[j]); + __builtin_prefetch(&table_.slots[sp[j] & table_.mask], 0, 0); + } + if (!hash_out.empty()) { + std::copy_n(hh.begin(), g, hash_out.begin() + static_cast(base)); + } + for (size_t j = 0; j < g; ++j) { + cand[j] = kEmptySlot; + if (table_.count == 0) { + continue; + } + cand[j] = probe_hash_match_(hh[j], sp[j] & table_.mask); + if (cand[j] != kEmptySlot) { + __builtin_prefetch(&rows_[static_cast(cand[j]) * stride_], 0, 0); + } + } + for (size_t j = 0; j < g; ++j) { + const size_t q = base + j; + const std::span qpos = pos_flat.subspan(pos_off[q], k_of[q]); + if (cand[j] == kEmptySlot) { + out[q] = kNotFound; + } + else if (row_eq_positions(static_cast(cand[j]), qpos)) { + out[q] = static_cast(cand[j]); + } + else { + // A 32-bit collision: rare enough to walk the chain from the top rather than resume it. + out[q] = find_positions_(hh[j], qpos); + } + } + } + } + + // fold_hash of the monomial `pos` describes, through the same fold, so it is equal by construction. + [[nodiscard]] static auto fold_hash_positions(std::span pos) noexcept -> uint32_t { + key_type mono; + for (size_t j = 0; j < pos.size(); ++j) { + mono.set(pos[j]); + } + return fold_hash(mono); + } + // Insert-or-no-op. Row at `value` must already be written (the confirm reads dense rows). auto emplace(const key_type &key, mapped_type value) -> void { check_index_fits(value); @@ -251,12 +353,31 @@ class OperatorIndex { // Insert n distinct rows with consecutive indices [base, base+n). Rows must already be written. template auto bulk_insert(size_t n, mapped_type base, KeyFn &&key_at) -> void { + if (n == 0) { + return; + } + bulk_insert_hashed(n, base, [this, &key_at](size_t k) { return fold_hash(key_at(k)); }); + } + // bulk_insert with the hashes already in hand: same precondition (n distinct rows, already written, + // at consecutive indices) and the same slot assignment. `hashes[k]` must be fold_hash of the key of + // row base+k -- a wrong one leaves the row unfindable, which surfaces later as a duplicate insert. + template + auto bulk_insert_hashed(size_t n, mapped_type base, HashFn &&hash_at) -> void { if (n == 0) { return; } check_index_fits(base + n - 1); - for (size_t k = 0; k < n; ++k) { - insert_slot_(static_cast(base + k), fold_hash(key_at(k))); + static constexpr size_t G = 16; + std::array hh; + for (size_t b = 0; b < n; b += G) { + const size_t g = std::min(G, n - b); + for (size_t j = 0; j < g; ++j) { + hh[j] = hash_at(b + j); + __builtin_prefetch(&table_.slots[spread(hh[j]) & table_.mask], /*rw=*/1, /*locality=*/0); + } + for (size_t j = 0; j < g; ++j) { + insert_slot_(static_cast(base + b + j), hh[j]); + } } } template @@ -386,6 +507,38 @@ class OperatorIndex { return true; } + // Compare row i against an ascending position list; a spilled row falls back to a dense compare. + [[nodiscard]] auto row_eq_positions(size_t i, std::span q) const -> bool { + const PosT c = rows_[i * stride_]; + if (c == kOverflowMarker) { + key_type mono; + for (size_t j = 0; j < q.size(); ++j) { + mono.set(q[j]); + } + return overflow_.at(i) == mono; + } + if (q.size() != static_cast(c)) { + return false; + } + return std::equal(q.begin(), q.end(), &rows_[(i * stride_) + 1]); + } + + // find()'s chain walk for a position-list key, hash already folded; only the collision arm reaches it. + [[nodiscard]] auto find_positions_(uint32_t h, std::span q) const -> size_t { + if (table_.count == 0) { + return kNotFound; + } + for (size_t s = spread(h) & table_.mask;; s = (s + 1) & table_.mask) { + const Slot &e = table_.slots[s]; + if (e.idx == kEmptySlot) { + return kNotFound; + } + if (e.h == h && row_eq_positions(static_cast(e.idx), q)) { + return static_cast(e.idx); + } + } + } + static auto check_index_fits(size_t value) -> void { if (value >= kIndexCeiling) { throw TermIndexCeilingReached("OperatorIndex: operator index reached the TermIndex ceiling; rebuild with " diff --git a/cpp/tests/README.md b/cpp/tests/README.md index 8263082a..78a2844a 100644 --- a/cpp/tests/README.md +++ b/cpp/tests/README.md @@ -73,6 +73,9 @@ name and cannot address suite-nested cases, tests use flat - **`ExchangeLayoutOracle.h`**: `build_layer_exchange_layout` — the independent reference for a layer's exchange counts and displacements, which `derive_exchange_layout` in the library is checked against. +- **`dense_query_reference.h`**: the retired dense query record, frozen as the + independent oracle for `sparse_query_tests.cpp`. Test-only, and not kept in + sync with the wire format. - **`TestData.{h,cpp}`**: the `CaseData` struct and msgpack fixture loader. - **`boost-test.cmake` / `boostAddTests.cmake`**: CMake test discovery. @@ -82,18 +85,25 @@ name and cannot address suite-nested cases, tests use flat - **Containers / algebra / utilities**: `bitset_tests.cpp` (the Bitset container vs a std::bitset oracle), `mpfunctions.cpp` (MP utilities + bit-flip helpers), `pauli_algebra_tests.cpp`, `majorana_cutoff_tests.cpp` (length/support cutoff, - CutoffEvaluator, interleave phase, coeff encode/decode), `validation_tests.cpp` - (parameter validators), `mpi_utils_tests.cpp` (find_rank + word serialization), - `evolution_detail_tests.cpp` (MatchedEpochSet + CutoffContext), + CutoffEvaluator, interleave phase, coeff encode/decode, cutoff_sums vs a + bitwise reference), `validation_tests.cpp` + (parameter validators), `mpi_utils_tests.cpp` (find_rank, word serialization, + scan routing agreement), `evolution_detail_tests.cpp` (MatchedEpochSet + + CutoffContext), `row_accessor_tests.cpp` (dense vs OperatorIndex row accessors). - **Operator store**: `operator_index_tests.cpp`, `inverted_index_tests.cpp`, `mp_operator_tests.cpp` (MPOperator get_state Pauli/Majorana scoring, get_operator init-map drain, update_initial_operator picture branches, - insert_absent_terms, inverted-index sync, memory estimate, deep copy). + insert_absent_terms, inverted-index sync, memory estimate, deep copy), + `bulk_insert_tests.cpp` (the grouped-prefetch insert vs a one-key-at-a-time + reference: table state and enumeration order). - **Layer build / evolution**: `build_graph_tests.cpp`, `pauli_build_layer_tests.cpp`, `fused_cos_sweep_tests.cpp`, - `fused_query_codec_tests.cpp`, `combined_recompute_equivalence.cpp` - (recompute equivalence + snapshot invariance), `exact_upper_atol_rescue.cpp`, + `sparse_query_tests.cpp` (the QueryWire wire record against the frozen dense + oracle, plus the fused value channel), `sparse_resolve_tests.cpp` (probe and + insert from wire positions vs the dense Monomial-keyed path), + `combined_recompute_equivalence.cpp` (recompute equivalence + + snapshot invariance), `exact_upper_atol_rescue.cpp`, `large_cosine_storage_tests.cpp`, `gate_boundaries.cpp`. - **Graph encoding / packing**: `graph_encoding_tests.cpp` (CosineWordBuilder coalescer, checked_* overflow guards, packed-phase storage + int8 read, diff --git a/cpp/tests/bulk_insert_tests.cpp b/cpp/tests/bulk_insert_tests.cpp new file mode 100644 index 00000000..68f4ec15 --- /dev/null +++ b/cpp/tests/bulk_insert_tests.cpp @@ -0,0 +1,188 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// bulk_insert prefetches 16 slot addresses; being a pure hint it must leave the table in exactly the +// state an unpipelined loop leaves it in, slot order included -- for_each makes it Python-visible. + +#include + +#include +#include +#include +#include +#include +#include + +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/operator/OperatorIndex.h" + +using namespace monoprop; + +namespace { + +constexpr size_t kN = 250; +using Index = detail::OperatorIndex; + +auto draw_distinct(std::mt19937_64 &rng, size_t n) -> std::vector> { + std::vector> out; + std::set> seen; + std::uniform_int_distribution bit(0, Monomial::size() - 1); + std::uniform_int_distribution pop(0, 12); + while (out.size() < n) { + Monomial m; + const size_t k = pop(rng); + for (size_t placed = 0; placed < k;) { + const size_t b = bit(rng); + if (!m.test(b)) { + m.set(b); + ++placed; + } + } + std::vector key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + if (seen.insert(key).second) { + out.push_back(m); + } + } + return out; +} + +// Deliberately not reserved: rehash_if_needed firing mid-group frees the table already-issued +// addresses point into, the interesting case for a prefetch. +auto build(const std::vector> &terms) -> std::unique_ptr { + auto idx = std::make_unique(); + const size_t base = idx->grow_rows_geometric(terms.size()); + for (size_t k = 0; k < terms.size(); ++k) { + idx->set(base + k, terms[k]); + } + idx->bulk_insert(terms.size(), base, [&](size_t k) -> const Monomial & { return terms[k]; }); + return idx; +} + +// The oracle: N one-key calls, so every group is of size one and none of the grouped loop's boundary +// arithmetic runs. It shares insert_slot_ but not the grouping, which is the thing under test. +auto build_reference(const std::vector> &terms) -> std::unique_ptr { + auto idx = std::make_unique(); + const size_t base = idx->grow_rows_geometric(terms.size()); + for (size_t k = 0; k < terms.size(); ++k) { + idx->set(base + k, terms[k]); + idx->bulk_insert(1, base + k, [&](size_t) -> const Monomial & { return terms[k]; }); + } + return idx; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(bulk_insert_finds_every_key) { + std::mt19937_64 rng(20260814); + const auto terms = draw_distinct(rng, 4000); + const auto idx = build(terms); + BOOST_REQUIRE_EQUAL(idx->size(), terms.size()); + for (size_t i = 0; i < terms.size(); ++i) { + const auto found = idx->find(terms[i]); + BOOST_REQUIRE(found.has_value()); + BOOST_TEST(*found == i); + } +} + +BOOST_AUTO_TEST_CASE(bulk_insert_batch_find_agrees_with_one_key_at_a_time) { + std::mt19937_64 rng(20260815); + const auto terms = draw_distinct(rng, 4000); + const auto absent = draw_distinct(rng, 500); + const auto grouped = build(terms); + const auto one_by_one = build_reference(terms); + + std::vector a(terms.size(), 0); + std::vector b(terms.size(), 0); + one_by_one->find_batch(terms.data(), terms.size(), a.data()); + grouped->find_batch(terms.data(), terms.size(), b.data()); + for (size_t i = 0; i < terms.size(); ++i) { + BOOST_TEST(a[i] == b[i]); + } + + // A table that answered everything would satisfy the loop above and prove nothing. + std::vector ma(absent.size(), 0); + std::vector mb(absent.size(), 0); + one_by_one->find_batch(absent.data(), absent.size(), ma.data()); + grouped->find_batch(absent.data(), absent.size(), mb.data()); + size_t genuinely_absent = 0; + for (size_t i = 0; i < absent.size(); ++i) { + BOOST_TEST(ma[i] == mb[i]); + if (!one_by_one->find(absent[i]).has_value()) { + ++genuinely_absent; + } + } + BOOST_TEST(genuinely_absent > 0U); +} + +BOOST_AUTO_TEST_CASE(bulk_insert_preserves_the_enumeration_order) { + // Identical, not merely equal as a set: this sequence is what the Python API returns terms in. + std::mt19937_64 rng(20260816); + const auto terms = draw_distinct(rng, 4000); + const auto grouped = build(terms); + const auto one_by_one = build_reference(terms); + + std::vector order_ref; + std::vector order_grouped; + one_by_one->for_each([&](const Monomial &, size_t i) { order_ref.push_back(i); }); + grouped->for_each([&](const Monomial &, size_t i) { order_grouped.push_back(i); }); + + BOOST_REQUIRE_EQUAL(order_grouped.size(), order_ref.size()); + BOOST_REQUIRE_EQUAL(order_ref.size(), terms.size()); + // Both coming out in index order would make the comparison blind to a reshuffle preserving it. + bool is_sorted_by_index = true; + for (size_t i = 1; i < order_ref.size(); ++i) { + if (order_ref[i] < order_ref[i - 1]) { + is_sorted_by_index = false; + break; + } + } + BOOST_TEST(!is_sorted_by_index); + for (size_t i = 0; i < order_ref.size(); ++i) { + BOOST_TEST(order_grouped[i] == order_ref[i]); + } +} + +BOOST_AUTO_TEST_CASE(bulk_insert_handles_a_partial_final_group) { + // The group width is 16, so these sizes straddle the boundary, including fewer than one group. + std::mt19937_64 rng(20260817); + for (const size_t n : {size_t{1}, size_t{15}, size_t{16}, size_t{17}, size_t{31}, size_t{33}}) { + const auto terms = draw_distinct(rng, n); + const auto grouped = build(terms); + const auto one_by_one = build_reference(terms); + BOOST_REQUIRE_EQUAL(grouped->size(), n); + for (size_t i = 0; i < n; ++i) { + const auto fa = one_by_one->find(terms[i]); + const auto fb = grouped->find(terms[i]); + BOOST_REQUIRE(fa.has_value()); + BOOST_REQUIRE(fb.has_value()); + BOOST_TEST(*fb == *fa); + } + } +} + +BOOST_AUTO_TEST_CASE(bulk_insert_of_nothing_is_a_no_op) { + std::mt19937_64 rng(20260818); + const auto terms = draw_distinct(rng, 100); + auto idx = build(terms); + const size_t before = idx->size(); + const auto key_at = [&](size_t k) -> const Monomial & { return terms[k]; }; + idx->bulk_insert(0, 0, key_at); + BOOST_TEST(idx->size() == before); + for (size_t i = 0; i < terms.size(); ++i) { + BOOST_REQUIRE(idx->find(terms[i]).has_value()); + } +} diff --git a/cpp/tests/dense_query_reference.h b/cpp/tests/dense_query_reference.h new file mode 100644 index 00000000..5f820dfa --- /dev/null +++ b/cpp/tests/dense_query_reference.h @@ -0,0 +1,106 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The retired dense query record, frozen here as a deliberately independent oracle for +// sparse_query_tests.cpp's differential: test-only, and it does not follow the wire format. + +#pragma once + +#include +#include +#include +#include +#include + +#include "monoprop/TypeAliases.h" +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" +#include "monoprop/detail/mpi/MPIUtils.h" + +namespace monoprop::test_ref { + +using monoprop::VecZ; + +// W monomial words + one ±1 phase word. +template +inline constexpr size_t kQueryWords = mpi_detail::kWords + 1; + +template +inline constexpr size_t kQueryWordsFused = kQueryWords + 1; + +template +inline auto query_push(VecZ &buf, const Monomial &mono, int phase) -> void { + mpi_detail::append_monomial_words(mono, buf); + buf.push_back(static_cast(static_cast(phase))); +} + +template > +inline auto query_read(const VecZ &buf, size_t q, Monomial &mono_out, int &phase_out) -> void { + const size_t base = q * QW; + mono_out = mpi_detail::read_monomial_from_words(buf, base); + phase_out = static_cast(static_cast(buf[base + mpi_detail::kWords])); +} + +template +inline auto query_value(const VecZ &buf, size_t q) -> double { + return detail::decode_value(buf[q * kQueryWordsFused + mpi_detail::kWords + 1]); +} + +// Requires v.size() == q.size()/kQueryWords: exactly one value per query record. +template +inline auto build_fused_query_value(const VecZ &q, const std::vector &v, VecZ &out) -> void { + constexpr size_t W = kQueryWords; + const size_t nq = q.empty() ? 0 : q.size() / W; + out.clear(); + out.reserve(nq * kQueryWordsFused); + for (size_t i = 0; i < nq; ++i) { + out.insert(out.end(), + q.begin() + static_cast(i * W), + q.begin() + static_cast((i + 1) * W)); + out.push_back(detail::encode_value(v[i])); + } +} + +// Every field a wire header carries must be recoverable from the record's own positions, or it rots +// unnoticed. Checks the phase in range, k against the positions handed back, the positions strictly +// ascending and in bounds, and gw exactly the widest gap. Recomputed here rather than in the library, +// so it runs in the Release build the suite actually uses. +template +[[nodiscard]] inline auto wire_header_is_consistent(std::span buf, size_t off, const Pos &pos) -> bool { + const auto h = detail::QueryWire::header_at(buf, off); + if (h.phase < -1 || h.phase > 1) { + return false; + } + if (h.k != std::ranges::size(pos)) { + return false; + } + for (size_t j = 0; j + 1 < h.k; ++j) { + if (static_cast(pos[j]) >= static_cast(pos[j + 1])) { + return false; // positions must arrive strictly ascending + } + } + if (h.k != 0 && static_cast(pos[h.k - 1]) >= 2 * NumModes) { + return false; + } + // gw is the maximum gap width: too small truncates a gap silently, too large wastes bits. + size_t g = 0; + for (size_t j = 1; j < h.k; ++j) { + const auto b = static_cast(std::bit_width(static_cast(pos[j] - pos[j - 1] - 1))); + g = (b > g) ? b : g; + } + return g == h.gw; +} + +} // namespace monoprop::test_ref diff --git a/cpp/tests/evolution_detail_tests.cpp b/cpp/tests/evolution_detail_tests.cpp index 91f2662b..d15c17aa 100644 --- a/cpp/tests/evolution_detail_tests.cpp +++ b/cpp/tests/evolution_detail_tests.cpp @@ -154,8 +154,17 @@ BOOST_AUTO_TEST_CASE(self_resolve_mark_bounded_by_combined_size) { matched, combined_size, RecordingSink{}); - detail::query_push<8>(eng.queries_r[0], terms[1], 1); - detail::query_push<8>(eng.queries_r[0], terms[5], -1); + // The self leg is staged as positions, never encoded, so this feeds the stage the scan would fill. + using Eng = detail::LayerBuildEngine<8, RecordingSink>; + const auto stage_self = [&eng](const Monomial<8> &m, int phase) { + std::vector pos; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + pos.push_back(static_cast(b)); + } + eng.self_stage_.push(pos, phase); + }; + stage_self(terms[1], 1); + stage_self(terms[5], -1); eng.src_idx_r[0] = {0, 2}; eng.resolve_self_queries(/*is_leader_pass=*/true); diff --git a/cpp/tests/fused_query_codec_tests.cpp b/cpp/tests/fused_query_codec_tests.cpp deleted file mode 100644 index 72f76763..00000000 --- a/cpp/tests/fused_query_codec_tests.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include - -#include "monoprop/TypeAliases.h" -#include "monoprop/core/Monomial.h" -#include "monoprop/detail/evolution/layer_build/Common.h" - -// Query+value fusion codec: the fused R>1 exchange rides the source coefficient (v_src) on each query -// record as a trailing bit-cast word so one alltoallv carries both streams. Round-tripping must be -// byte-for-byte, including the FP corner cases a lossy value channel would mangle. - -namespace { - -using namespace monoprop; -using monoprop::detail::build_fused_query_value; -using monoprop::detail::kQueryWords; -using monoprop::detail::kQueryWordsFused; -using monoprop::detail::query_push; -using monoprop::detail::query_read; -using monoprop::detail::query_value; - -constexpr size_t kModes = 8; // 2*kModes = 16 majorana bits, one 64-bit word - -// A deterministic, distinct majorana bit pattern per record index. -auto make_mono(size_t r) -> Monomial { - Monomial m; - for (size_t b = 0; b < 2 * kModes; ++b) { - if (((r * 2654435761u + b * 40503u) & 3u) == 0u) { - m.set(b); - } - } - return m; -} - -BOOST_AUTO_TEST_CASE(fused_record_roundtrip_exact) { - const std::vector phases = {1, -1, 1, -1, 1, 1, -1}; - const std::vector values = { - 0.0, - -0.0, - 1.0, - -1.0, - 3.141592653589793, - -2.718281828459045e-300, // near-denormal magnitude - std::numeric_limits::min(), // smallest normal - }; - const size_t nq = values.size(); - - VecZ plain; - std::vector> monos(nq); - for (size_t r = 0; r < nq; ++r) { - monos[r] = make_mono(r); - query_push(plain, monos[r], phases[r]); - } - BOOST_REQUIRE_EQUAL(plain.size(), nq * kQueryWords); - - VecZ fused; - build_fused_query_value(plain, values, fused); - BOOST_REQUIRE_EQUAL(fused.size(), nq * kQueryWordsFused); - - for (size_t q = 0; q < nq; ++q) { - Monomial m_out; - int ph_out = 0; - query_read>(fused, q, m_out, ph_out); - BOOST_CHECK(m_out == monos[q]); - BOOST_CHECK_EQUAL(ph_out, phases[q]); - // Compare the raw payload, so -0.0 and denormals stay distinguished from 0.0. - const double v_out = query_value(fused, q); - BOOST_CHECK(std::memcmp(&v_out, &values[q], sizeof(double)) == 0); - } -} - -// Reusing `out` across calls must leak no stale words: capacity is a high-water mark, size is exact. -// This is the reuse pattern LayerBuildEngine::combined_qv_ relies on gate to gate. -BOOST_AUTO_TEST_CASE(fused_buffer_reuse_shrinks_logical_size) { - VecZ plain_big; - std::vector vbig; - for (size_t r = 0; r < 32; ++r) { - query_push(plain_big, make_mono(r), (r % 2 == 0) ? 1 : -1); - vbig.push_back(static_cast(r) * 1.5 - 7.0); - } - VecZ out; - build_fused_query_value(plain_big, vbig, out); - const size_t cap_after_big = out.capacity(); - - VecZ plain_small; - std::vector vsmall = {42.0, -42.0, 0.25}; - for (size_t r = 0; r < vsmall.size(); ++r) { - query_push(plain_small, make_mono(100 + r), 1); - } - build_fused_query_value(plain_small, vsmall, out); - BOOST_CHECK_EQUAL(out.size(), vsmall.size() * kQueryWordsFused); - BOOST_CHECK_GE(out.capacity(), cap_after_big); - for (size_t q = 0; q < vsmall.size(); ++q) { - const double v_out = query_value(out, q); - BOOST_CHECK(std::memcmp(&v_out, &vsmall[q], sizeof(double)) == 0); - } -} - -// Empty input arises for the self slot, which resolve_self_queries clears before the exchange. -BOOST_AUTO_TEST_CASE(fused_empty_input) { - VecZ empty; - std::vector no_values; - VecZ out{1, 2, 3}; // pre-dirtied; build must clear it - build_fused_query_value(empty, no_values, out); - BOOST_CHECK(out.empty()); -} - -} // namespace diff --git a/cpp/tests/majorana_cutoff_tests.cpp b/cpp/tests/majorana_cutoff_tests.cpp index 50f8632f..bc236422 100644 --- a/cpp/tests/majorana_cutoff_tests.cpp +++ b/cpp/tests/majorana_cutoff_tests.cpp @@ -17,7 +17,9 @@ #include +#include #include +#include #include #include @@ -28,6 +30,60 @@ using namespace monoprop; using cd = std::complex; +namespace { + +// Bit-by-bit reference for cutoff_sums(const Monomial&, size_t): sums over the active window only, mode +// m owning raw bits (active_bit_offset + 2m, active_bit_offset + 2m + 1). +template +auto reference_cutoff_sums(const Monomial &mono, size_t logical_num_modes) -> CutoffSums { + const size_t active_bit_offset = 2 * (N - logical_num_modes); + size_t xor_sum = 0; + size_t popcount_sum = 0; + size_t or_sum = 0; + for (size_t m = 0; m < logical_num_modes; ++m) { + const size_t bit0 = active_bit_offset + (2 * m); + const bool a = mono.test(bit0); + const bool b = mono.test(bit0 + 1); + xor_sum += static_cast(a != b); + popcount_sum += static_cast(a) + static_cast(b); + or_sum += static_cast(a || b); + } + return {xor_sum, popcount_sum, or_sum}; +} + +template +auto check_cutoff_sums_width(std::mt19937_64 &rng, size_t logical) -> void { + std::uniform_int_distribution bit(2 * (N - logical), (2 * N) - 1); + for (size_t weight = 1; weight <= std::min(2 * logical, 20); ++weight) { + for (int rep = 0; rep < 20; ++rep) { + Monomial mono; + for (size_t k = 0; k < weight; ++k) { + mono.set(bit(rng)); + } + const auto got = cutoff_sums(mono, logical); + const auto want = reference_cutoff_sums(mono, logical); + BOOST_REQUIRE_EQUAL(got.xor_sum, want.xor_sum); + BOOST_REQUIRE_EQUAL(got.popcount_sum, want.popcount_sum); + BOOST_REQUIRE_EQUAL(got.or_sum, want.or_sum); + } + } +} + +} // namespace + +// cutoff_sums(const Monomial&, size_t) directly, differentially against a bit-by-bit reference, across +// widths spanning the single-word (with and without an active offset), multi-word-not-a-multiple-of-64, +// exactly-two-word and production-scale paths. +BOOST_AUTO_TEST_CASE(majorana_cutoff_sums_matches_bitwise_reference_across_widths) { + std::mt19937_64 rng(0xD16E57U); + check_cutoff_sums_width<32>(rng, 32); // W = 64, one word, no active offset + check_cutoff_sums_width<32>(rng, 30); // W = 64, active_bit_offset = 4 + check_cutoff_sums_width<48>(rng, 45); // W = 96 -- not a multiple of 64 + check_cutoff_sums_width<64>(rng, 64); // W = 128, exactly two words + check_cutoff_sums_width<128>(rng, 120); + check_cutoff_sums_width<256>(rng, 250); // the production shape +} + // Raw bits {0,1} and {4,5} are two complete pairs. BOOST_AUTO_TEST_CASE(majorana_cutoff_paired_kept_unconditionally) { constexpr size_t N = 32; diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index 8372e88f..e400fe84 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -12,18 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -// The pure MPIUtils.h primitives (term->owner mapping, wire word packing), driven without a comm. +// The pure MPIUtils.h primitives (term->owner mapping, wire word packing), driven without a comm -- +// plus the routing agreement between find_rank and the scan, a property of neither call site alone. #include +#include +#include +#include +#include #include #include #include "monoprop/algebra/MajoranaAlgebra.h" +#include "monoprop/detail/evolution/CutoffContext.h" +#include "monoprop/detail/evolution/layer_build/Scan.h" #include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/operator/OperatorIndex.h" using namespace monoprop; +// find_rank is splitmix over the dense words modulo the rank count, and nothing else, so the oracle +// is asserted unconditionally rather than as one of several permitted hashes. BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_range_and_hash_mod) { constexpr size_t N = 32; std::mt19937_64 rng(0x9E3779B9ULL); @@ -36,8 +47,8 @@ BOOST_AUTO_TEST_CASE(mpi_utils_find_rank_range_and_hash_mod) { const auto mono = indices_to_bitset(inds); for (size_t n_ranks : {size_t{1}, size_t{2}, size_t{3}, size_t{7}}) { const size_t r = find_rank(mono, n_ranks); - BOOST_TEST(r < n_ranks); BOOST_TEST(r == monomial_hash(mono) % n_ranks); + BOOST_TEST(r < n_ranks); BOOST_TEST(r == find_rank(mono, n_ranks)); // deterministic } } @@ -73,3 +84,122 @@ BOOST_AUTO_TEST_CASE(mpi_utils_monomial_words_roundtrip) { BOOST_REQUIRE(sbuf.size() == mpi_detail::kWords); BOOST_TEST((mpi_detail::read_monomial_from_words(sbuf, 0) == d)); } + +namespace { + +template +auto draw_well_formed(std::mt19937_64 &rng, size_t logical, size_t weight) -> Monomial { + VecZ idx; + std::uniform_int_distribution dist(0, (2 * logical) - 1); + while (idx.size() < weight) { + const size_t v = dist(rng); + if (std::find(idx.begin(), idx.end(), v) == idx.end()) { + idx.push_back(v); + } + } + return indices_to_bitset(idx); +} + +auto build_op(const std::vector> &terms) -> detail::MPOperator<32> { + detail::MPOperator<32> op; + op.basis = Basis::Majorana; + detail::insert_absent_terms<32>( + op, + terms.size(), + [&](size_t k) -> const Monomial<32> & { return terms[k]; }, + [&](size_t k, size_t base) { assign_row<32>(*op.store, base + k, terms[k]); }); + return op; +} + +auto check_bucket_ownership(const std::vector &buckets, size_t ranks, size_t &checked) -> void { + // Every offset comes from the record walk: widths vary, so a hardcoded stride would compare a + // monomial decoded at the wrong offset against the wrong rank. + using QW = detail::QueryWire<32>; + const detail::QueryForm form = detail::QueryForm::Plain; + for (size_t r = 0; r < buckets.size(); ++r) { + size_t off = 0; + while (off < buckets[r].size()) { + const size_t k = QW::k_at(buckets[r], off); + std::vector pos(k); + (void)QW::read_positions(buckets[r], off, pos); + Monomial<32> mono; + for (size_t j = 0; j < k; ++j) { + mono.set(static_cast(pos[j])); + } + BOOST_REQUIRE_EQUAL(find_rank<32>(mono, ranks), r); + off = QW::next_off(buckets[r], form, off); + ++checked; + } + BOOST_REQUIRE_EQUAL(off, buckets[r].size()); + } +} + +// The self-owned bucket is staged as positions, not encoded, so it is invisible to the walk above -- +// without this the r == my_rank arm of the routing decision goes unchecked. +auto check_self_ownership(const detail::SelfQueryStage<32> &stage, size_t ranks, size_t my_rank, size_t &checked) + -> void { + for (size_t q = 0; q < stage.size(); ++q) { + Monomial<32> mono; + for (size_t j = 0; j < stage.k_of[q]; ++j) { + mono.set(static_cast(stage.pos_flat[stage.pos_off[q] + j])); + } + BOOST_REQUIRE_EQUAL(find_rank<32>(mono, ranks), my_rank); + ++checked; + } +} + +} // namespace + +// The scan hashes the partner it just built; find_rank hashes what the resolve side decoded off the +// wire. Nothing downstream notices if they diverge -- the term simply exists twice. +BOOST_AUTO_TEST_CASE(mpi_utils_scan_routing_agrees_with_find_rank) { + constexpr size_t kN = 32; + constexpr size_t kLogical = 30; + std::mt19937_64 rng(0xB0B1E5U); + + std::vector> terms; + for (size_t i = 0; i < 2000; ++i) { + terms.push_back(draw_well_formed(rng, kLogical, 1 + (rng() % 6))); + } + auto op = build_op(terms); + const Monomial gen = draw_well_formed(rng, kLogical, 4); + VecD coeffs(op.store->size(), 1.0); + + const CutoffFn fn = detail::LengthCutoff{10, kLogical}; + const detail::CutoffEvaluator eval(fn); + const auto cut = detail::build_majorana_evolution_cutoff_state(std::nullopt, + std::cref(coeffs), + std::nullopt, + std::optional{0.3}); + + size_t checked = 0; + size_t self_checked = 0; + for (const size_t ranks : {2U, 4U, 8U}) { + const auto res = detail::fused_find_and_collect>(op, + gen, + eval, + cut, + coeffs, + std::nullopt, + ranks, + 0, + false, + nullptr, + 1.0); + BOOST_REQUIRE_EQUAL(res.leader_queries.size(), ranks); + // The scan routes a self-owned partner to the stage, so bucket 0 must be empty here. + BOOST_REQUIRE(res.leader_queries[0].empty()); + BOOST_REQUIRE(res.follower_queries[0].empty()); + check_bucket_ownership(res.leader_queries, ranks, checked); + check_bucket_ownership(res.follower_queries, ranks, checked); + check_self_ownership(res.leader_self, ranks, /*my_rank=*/0, self_checked); + check_self_ownership(res.follower_self, ranks, /*my_rank=*/0, self_checked); + } + // Without this the loop above passes trivially if the scan emitted nothing. The floor is on the total + // because that is what is invariant across the split; each arm also keeps its own floor so a routing + // bug that sends everything one way still fails. + BOOST_TEST_MESSAGE("encoded=" << checked << " staged=" << self_checked); + BOOST_TEST(checked + self_checked > 1000U); + BOOST_TEST(checked > 500U); + BOOST_TEST(self_checked > 200U); +} diff --git a/cpp/tests/partner_merge_tests.cpp b/cpp/tests/partner_merge_tests.cpp new file mode 100644 index 00000000..a833fed6 --- /dev/null +++ b/cpp/tests/partner_merge_tests.cpp @@ -0,0 +1,165 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The partner merge against the dense XOR it replaces. Random pairs alone do not reach the cases that +// decide it: the merge's whole surface is how many of G's slots the source already holds, so every +// overlap in [0, gen_pop] is drawn on purpose, and the paired case is drawn separately because a +// fully-paired monomial is 94 in 20.9M on production models. + +#include + +#include +#include +#include +#include +#include + +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/evolution/layer_build/PartnerMerge.h" + +using namespace monoprop; + +namespace { + +constexpr size_t kN = 64; +constexpr size_t kBits = 2 * kN; + +// The dense reference: positions of M^G, ascending, straight off the bitset. +auto dense_partner(const Monomial &mono, const Monomial &gen) -> std::vector { + const Monomial nm = mono ^ gen; + std::vector out; + for (size_t b = nm.find_first(); b < nm.size(); b = nm.find_next(b)) { + out.push_back(b); + } + return out; +} + +auto positions_of(const Monomial &m) -> std::vector { + std::vector out; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + out.push_back(static_cast(b)); + } + return out; +} + +// Checks the merge against the dense form on one pair, and returns the merged count so a caller can +// assert it saw work. Every field the emit site consumes is compared, not just the positions. +auto check_pair(const Monomial &mono, const Monomial &gen) -> size_t { + const auto src = positions_of(mono); + const auto gpos = positions_of(gen); + std::vector out(kBits); + const auto merged = detail::merge_partner_positions(src, gpos, out); + const size_t k = merged.count; + const size_t overlap = merged.overlap; + + const auto expect = dense_partner(mono, gen); + BOOST_REQUIRE_EQUAL(k, expect.size()); + for (size_t j = 0; j < k; ++j) { + BOOST_REQUIRE_EQUAL(static_cast(out[j]), expect[j]); + } + BOOST_REQUIRE_EQUAL(overlap, mono.count_and(gen)); + // The popcount identity the emit site relies on. + BOOST_REQUIRE_EQUAL(k, mono.count() + gen.count() - (2 * overlap)); + return k; +} + +// A generator of `gen_pop` slots, and a source holding exactly `overlap` of them plus `extra` others. +auto build_case(std::mt19937_64 &rng, size_t gen_pop, size_t overlap, size_t extra) + -> std::pair, Monomial> { + std::vector all(kBits); + for (size_t i = 0; i < kBits; ++i) { + all[i] = i; + } + std::shuffle(all.begin(), all.end(), rng); + Monomial gen; + for (size_t i = 0; i < gen_pop; ++i) { + gen.set(all[i]); + } + Monomial mono; + for (size_t i = 0; i < overlap; ++i) { + mono.set(all[i]); // a slot G also holds: it cancels + } + for (size_t i = 0; i < extra; ++i) { + mono.set(all[gen_pop + i]); // disjoint from G: it survives + } + return {mono, gen}; +} + +} // namespace + +// Every overlap between the source and the generator, which is the branch the merge exists to take. +BOOST_AUTO_TEST_CASE(partner_merge_matches_dense_at_every_overlap) { + std::mt19937_64 rng(0xA11CEU); + size_t cases = 0; + size_t nonempty = 0; + for (size_t gen_pop = 1; gen_pop <= 6; ++gen_pop) { + for (size_t overlap = 0; overlap <= gen_pop; ++overlap) { + for (const size_t extra : {size_t{0}, size_t{1}, size_t{5}, size_t{20}}) { + for (size_t rep = 0; rep < 8; ++rep) { + const auto [mono, gen] = build_case(rng, gen_pop, overlap, extra); + BOOST_REQUIRE_EQUAL(mono.count_and(gen), overlap); // the case is the one intended + nonempty += (check_pair(mono, gen) != 0) ? 1 : 0; + ++cases; + } + } + } + } + BOOST_REQUIRE_EQUAL(cases, 6U * 4U * 8U + (1U + 2U + 3U + 4U + 5U + 6U) * 4U * 8U); + // Total cancellation (overlap == gen_pop, extra == 0) is the only empty partner, so most must not be. + BOOST_TEST(nonempty > 600U); +} + +// The paired population separately: a uniform draw almost never produces a fully paired monomial, so +// this case is drawn on purpose rather than left to chance in the sweep above. +BOOST_AUTO_TEST_CASE(partner_merge_matches_dense_on_paired_monomials) { + std::mt19937_64 rng(0xBEEFU); + size_t paired_seen = 0; + for (size_t rep = 0; rep < 400; ++rep) { + // Whole modes only, so both slots of each are set and the result is fully paired. + std::vector modes(kN); + for (size_t i = 0; i < kN; ++i) { + modes[i] = i; + } + std::shuffle(modes.begin(), modes.end(), rng); + Monomial mono; + const size_t n_modes = 1 + (rng() % 8); + for (size_t i = 0; i < n_modes; ++i) { + mono.set(2 * modes[i]); + mono.set((2 * modes[i]) + 1); + } + Monomial gen; + const size_t g_modes = 1 + (rng() % 3); + for (size_t i = 0; i < g_modes; ++i) { + gen.set(2 * modes[kN - 1 - i]); + gen.set((2 * modes[kN - 1 - i]) + 1); + } + check_pair(mono, gen); + paired_seen += is_paired(mono ^ gen) ? 1 : 0; + } + // The draw is meant to land on the paired branch every time; a zero here means it stopped doing so. + BOOST_REQUIRE_EQUAL(paired_seen, 400U); +} + +// An empty generator and an empty source are both reachable (a truncated gate, a fresh row). +BOOST_AUTO_TEST_CASE(partner_merge_handles_empty_inputs) { + Monomial mono; + mono.set(4); + mono.set(5); + mono.set(70); + const Monomial empty; + BOOST_REQUIRE_EQUAL(check_pair(mono, empty), 3U); + BOOST_REQUIRE_EQUAL(check_pair(empty, mono), 3U); + BOOST_REQUIRE_EQUAL(check_pair(empty, empty), 0U); +} diff --git a/cpp/tests/sparse_query_tests.cpp b/cpp/tests/sparse_query_tests.cpp new file mode 100644 index 00000000..c978d47a --- /dev/null +++ b/cpp/tests/sparse_query_tests.cpp @@ -0,0 +1,572 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Differential against the frozen dense oracle in dense_query_reference.h; cases chosen, not sampled. + +#include + +#include +#include +#include +#include +#include + +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" + +#include "dense_query_reference.h" + +using namespace monoprop; +using namespace monoprop::detail; + +namespace { + +template +auto differential(const std::vector &pos, int phase) -> size_t { + using QW = QueryWire; + const size_t k = pos.size(); + + Monomial want; + for (const auto p : pos) { + want.set(p); + } + BOOST_REQUIRE_EQUAL(want.count(), k); // the caller must not hand us duplicates + VecZ dbuf; + test_ref::query_push(dbuf, want, phase); + Monomial dmono; + int dphase = 99; + test_ref::query_read(dbuf, 0, dmono, dphase); + BOOST_REQUIRE((dmono == want)); + BOOST_REQUIRE_EQUAL(dphase, phase); + + VecZ sbuf; + const size_t sw = QW::push(sbuf, pos, phase); + BOOST_REQUIRE_EQUAL(sbuf.size(), sw); + + BOOST_TEST(QW::words_at(sbuf, 0) == sw); + BOOST_TEST(QW::k_at(sbuf, 0) == k); + BOOST_TEST(QW::phase_at(sbuf, 0) == phase); + + std::vector sout(k == 0 ? 1 : k); + const auto sread = QW::read_positions(sbuf, 0, sout); + BOOST_TEST(sread.next == sw); + BOOST_TEST(sread.phase == phase); + sout.resize(k); + BOOST_TEST(sout == pos, boost::test_tools::per_element()); + // The header must be recoverable from the positions it describes; see dense_query_reference.h. + BOOST_TEST(test_ref::wire_header_is_consistent(sbuf, 0, sout)); + + // Round-trips the decoded positions through a Monomial, cross-checked against the dense oracle. + Monomial sm; + for (size_t j = 0; j < k; ++j) { + sm.set(sout[j]); + } + BOOST_TEST(sm.count() == k); + BOOST_TEST((sm == dmono)); + + // Encoding from positions taken off `want` itself must match encoding from `pos` directly: production + // always builds queries from an ascending position vector, never from a bitset. + VecZ mbuf; + std::vector from_mono; + for (size_t b = want.find_first(); b < want.size(); b = want.find_next(b)) { + from_mono.push_back(static_cast(b)); + } + const size_t mw = QW::push(mbuf, from_mono, phase); + BOOST_TEST(mw == sw); + BOOST_TEST(mbuf == sbuf, boost::test_tools::per_element()); + + return sw; +} + +auto strided(size_t k, size_t start, size_t step, size_t universe) -> std::vector { + std::vector v; + for (size_t j = 0; j < k; ++j) { + const size_t p = start + j * step; + if (p >= universe) { + break; + } + v.push_back(static_cast(p)); + } + return v; +} + +// Uniform draws are the widest gap widths, which is the case a regular pattern never reaches; see +// sparse_record_reaches_the_widest_gap_width for why a narrow universe needs a run plus one outlier. +auto scattered(size_t k, size_t universe, std::mt19937_64 &rng) -> std::vector { + std::vector pool(universe); + for (size_t j = 0; j < universe; ++j) { + pool[j] = static_cast(j); + } + std::shuffle(pool.begin(), pool.end(), rng); + pool.resize(std::min(k, universe)); + std::sort(pool.begin(), pool.end()); + return pool; +} + +// The reference d, written against the definition (mode m owns bits 2m and 2m+1), not the wire record. +auto reference_pair_count(const std::vector &pos) -> size_t { + size_t d = 0; + for (size_t j = 0; j + 1 < pos.size(); ++j) { + if ((pos[j] % 2 == 0) && (pos[j + 1] == pos[j] + 1)) { + ++d; + } + } + return d; +} + +} // namespace + +// Flat names: boostAddTests.cmake strips the indentation encoding suite nesting, so a suite-wrapped +// case errors at setup having asserted nothing. + +BOOST_AUTO_TEST_CASE(sparse_record_agrees_with_the_dense_oracle_across_widths) { + for (const int phase : {-1, 0, 1}) { + for (const size_t k : {size_t{0}, size_t{1}, size_t{2}, size_t{5}, size_t{6}, size_t{7}, size_t{15}}) { + differential<32>(strided(k, 0, 2, 64), phase); + differential<128>(strided(k, 3, 7, 256), phase); + differential<250>(strided(k, 11, 23, 500), phase); + differential<512>(strided(k, 1, 41, 1024), phase); + differential<1024>(strided(k, 5, 97, 2048), phase); + } + } +} + +BOOST_AUTO_TEST_CASE(sparse_record_handles_widths_that_are_not_whole_words) { + // Widths with no whole word: 12 modes (LiH) is kBits=24, so the record's payload is a partial word. + std::mt19937_64 rng(0xB17U); + for (const int phase : {-1, 0, 1}) { + for (const size_t k : {size_t{0}, size_t{1}, size_t{5}, size_t{9}, size_t{16}, size_t{24}}) { + differential<12>(strided(k, 0, 1, 24), phase); // kBits=24: no whole word at all + differential<12>(strided(k, 0, 2, 24), phase); + differential<12>(scattered(k, 24, rng), phase); + } + for (const size_t k : {size_t{1}, size_t{7}, size_t{20}, size_t{40}, size_t{70}}) { + differential<50>(strided(k, 0, 1, 100), phase); // kBits=100: one whole word plus 36 bits + differential<33>(strided(k, 0, 1, 66), phase); // kBits=66: one whole word plus 2 bits + differential<250>(strided(k, 0, 1, 500), phase); // kBits=500: seven words plus 52 bits + differential<50>(scattered(k, 100, rng), phase); + differential<33>(scattered(k, 66, rng), phase); + differential<250>(scattered(k, 500, rng), phase); + } + } + for (const size_t bits : {size_t{24}, size_t{100}, size_t{66}}) { + std::vector all(bits); + for (size_t j = 0; j < bits; ++j) { + all[j] = static_cast(j); + } + if (bits == 24) { + differential<12>(all, 1); + } + else if (bits == 100) { + differential<50>(all, 1); + } + else { + differential<33>(all, 1); + } + } +} + +BOOST_AUTO_TEST_CASE(sparse_record_reaches_the_widest_gap_width) { + // gw == kPosBits is the record's worst case, and uniform draws cannot reach it at kBits=24: it needs + // one gap of at least half the universe, which only a dense run plus one far outlier forces. + const auto count_widest = [](auto tag, size_t universe) { + using QW = QueryWire; + size_t used = 0; + size_t bad = 0; + const auto tally = [&](const std::vector &pos) { + if (pos.size() < 2 || pos.size() > QW::kMaxPositions) { + return; + } + VecZ buf; + const size_t w = QW::push(buf, pos, 1); + const size_t gw = QW::gap_width(pos); + if (gw != QW::kPosBits) { + return; + } + ++used; + std::vector back(pos.size()); + (void)QW::read_positions(buf, 0, back); + bad += static_cast(back != pos || QW::k_at(buf, 0) != pos.size() + || w != QW::words_of(QW::gap_bits(pos.size(), gw))); + }; + std::mt19937_64 rng(0xB1747U ^ universe); + for (size_t trial = 0; trial < 600; ++trial) { + tally(scattered(1 + (rng() % universe), universe, rng)); + } + for (size_t run = 1; run < universe; ++run) { + for (size_t outlier = run; outlier < universe; ++outlier) { + std::vector pos; + pos.reserve(run + 1); + for (size_t j = 0; j < run; ++j) { + pos.push_back(static_cast(j)); + } + pos.push_back(static_cast(outlier)); + tally(pos); + } + } + return std::pair{used, bad}; + }; + const auto narrow = count_widest(std::integral_constant{}, 24); + const auto partial = count_widest(std::integral_constant{}, 500); + const auto bucket = count_widest(std::integral_constant{}, 256); + BOOST_TEST(narrow.first > 0U); + BOOST_TEST(partial.first > 0U); + BOOST_TEST(bucket.first > 0U); + BOOST_TEST(narrow.second == 0U); + BOOST_TEST(partial.second == 0U); + BOOST_TEST(bucket.second == 0U); +} + +BOOST_AUTO_TEST_CASE(sparse_record_survives_the_five_bit_k_escape) { + // The escape is at k = 31; both boundaries are exercised here so a field-width change is caught. + for (const size_t k : {size_t{30}, size_t{31}, size_t{32}, size_t{62}, size_t{63}, size_t{64}, size_t{200}}) { + const auto pos = strided(k, 0, 3, 2048); + BOOST_REQUIRE_EQUAL(pos.size(), k); + differential<1024>(pos, 1); + } +} + +BOOST_AUTO_TEST_CASE(sparse_record_bounds_the_fully_paired_term) { + // Every bit set means every gap is 0, so gw is 0 and the header plus one raw position fit one word. + std::vector all(2048); + for (size_t j = 0; j < all.size(); ++j) { + all[j] = static_cast(j); + } + const size_t sw = differential<1024>(all, 1); + BOOST_TEST(sw == 1U); +} + +BOOST_AUTO_TEST_CASE(sparse_record_never_exceeds_its_own_raw_lanes) { + // Exhaustive rather than sampled: gw = bit_width(max gap) <= kPosBits, so kPosBits + (k-1)*gw <= + // k*kPosBits at every k. + const auto check = [](auto tag) { + using QW = QueryWire; + size_t cells = 0; + size_t bad = 0; + for (size_t k = 0; k <= QW::kMaxPositions; ++k) { + for (size_t gw = 0; gw <= QW::kPosBits; ++gw) { + const size_t lanes = QW::words_of(QW::header_bits_for(k) + (k * QW::kPosBits)); + bad += static_cast(QW::words_of(QW::gap_bits(k, gw)) > lanes); + ++cells; + } + } + return std::pair{cells, bad}; + }; + const auto narrow = check(std::integral_constant{}); + const auto bucket = check(std::integral_constant{}); + const auto wide = check(std::integral_constant{}); + BOOST_TEST(narrow.second == 0U); + BOOST_TEST(bucket.second == 0U); + BOOST_TEST(wide.second == 0U); + // A guarded loop that asserted nothing would pass the three above; these are the cell counts. + BOOST_TEST(narrow.first == 25U * 6U); + BOOST_TEST(bucket.first == 257U * 9U); + BOOST_TEST(wide.first == 501U * 10U); +} + +BOOST_AUTO_TEST_CASE(sparse_record_walks_a_multi_query_buffer_exactly) { + // Mixed width, which is the case a hardcoded stride gets wrong. + using QW = QueryWire<128>; + const QueryForm form = QueryForm::Plain; + VecZ buf; + std::vector offs; + size_t off = 0; + const std::vector> terms = { + strided(3, 0, 1, 256), // consecutive -> gap width 0 + strided(6, 10, 40, 256), // wide gaps -> gap width near the raw position width + {}, // empty + strided(40, 0, 6, 256), // wide, evenly spread positions + strided(1, 255, 1, 256), // single position at the very top + strided(20, 7, 2, 256), // uniform stride 2 + }; + for (const auto &t : terms) { + offs.push_back(off); + off += QW::push(buf, t, 1); + } + BOOST_TEST(QW::count_queries(buf, form) == terms.size()); + + off = 0; + for (size_t i = 0; i < terms.size(); ++i) { + BOOST_TEST(off == offs[i]); + BOOST_TEST(QW::k_at(buf, off) == terms[i].size()); + std::vector out(terms[i].size() + 1); + (void)QW::read_positions(buf, off, out); + out.resize(terms[i].size()); + BOOST_TEST(out == terms[i], boost::test_tools::per_element()); + off = QW::next_off(buf, form, off); + } + BOOST_TEST(off == buf.size()); +} + +BOOST_AUTO_TEST_CASE(sparse_record_is_exactly_the_gap_code_it_costed) { + // The encoder's width and the costing function must agree on every draw, uniform ones included. + using QW = QueryWire<128>; + std::mt19937_64 rng(12345); + for (size_t trial = 0; trial < 400; ++trial) { + const size_t k = rng() % 60; + const auto pos = scattered(k, 256, rng); + VecZ buf; + const size_t w = QW::push(buf, pos, 1); + const size_t gwid = QW::gap_width(pos); + BOOST_TEST(w == QW::words_of(QW::gap_bits(pos.size(), gwid)), + "k=" << k << " wrote " << w << " words, costed " << QW::words_of(QW::gap_bits(pos.size(), gwid))); + BOOST_TEST(w <= QW::words_of(QW::header_bits_for(pos.size()) + (pos.size() * QW::kPosBits))); + } +} + +BOOST_AUTO_TEST_CASE(sparse_record_position_width_is_the_compile_time_bucket) { + BOOST_TEST(QueryWire<32>::kPosBits == 6U); // U=64 + BOOST_TEST(QueryWire<128>::kPosBits == 8U); // U=256, both lattice models + BOOST_TEST(QueryWire<250>::kPosBits == 9U); // U=500 + BOOST_TEST(QueryWire<512>::kPosBits == 10U); // U=1024 + BOOST_TEST(QueryWire<1024>::kPosBits == 11U); // U=2048 +} + +BOOST_AUTO_TEST_CASE(sparse_record_carries_the_extreme_bit_positions) { + // MSb0 ordering puts logical index 0 at the top, so bit 2N-1 is the common case, not a rare one. + differential<32>({0}, 1); + differential<32>({63}, 1); + differential<32>({0, 63}, -1); + differential<128>({0, 255}, 1); + differential<250>({0, 499}, 1); + differential<1024>({0, 2047}, -1); + differential<12>({0, 23}, 1); // the non-word-multiple width, at both extremes +} + +BOOST_AUTO_TEST_CASE(sparse_record_encoding_is_deterministic) { + std::mt19937_64 rng(0xDE7ULL); + for (size_t trial = 0; trial < 200; ++trial) { + const auto pos = scattered(rng() % 40, 256, rng); + VecZ a; + VecZ b; + const size_t wa = QueryWire<128>::push(a, pos, 1); + const size_t wb = QueryWire<128>::push(b, pos, 1); + BOOST_TEST(wa == wb); + BOOST_TEST(a == b, boost::test_tools::per_element()); + } +} + +BOOST_AUTO_TEST_CASE(sparse_record_pair_count_recomputes_d_from_positions) { + // d is recomputed from positions, so it must agree with the definition: mode m owns bits 2m, 2m+1. + std::mt19937_64 rng(0xD1D1ULL); + for (size_t trial = 0; trial < 100; ++trial) { + const auto pos = scattered(rng() % 30, 256, rng); + BOOST_TEST(QueryWire<128>::pair_count(pos) == reference_pair_count(pos)); + } + const std::vector straddle{1, 2, 5, 6}; + BOOST_TEST(QueryWire<128>::pair_count(straddle) == 0U); + const std::vector real{2, 3, 6, 7}; + BOOST_TEST(QueryWire<128>::pair_count(real) == 2U); + std::vector paired; + for (uint16_t m = 0; m < 16; ++m) { + paired.push_back(static_cast(2 * m)); + paired.push_back(static_cast(2 * m + 1)); + } + BOOST_TEST(QueryWire<128>::pair_count(paired) == paired.size() / 2); +} + +namespace { +// Extracts an ascending position vector from a Monomial, mirroring what a real query source (the scan, +// the partner merge) already holds; production never encodes straight from a bitset. +template +auto positions_of(const Monomial &m) -> std::vector { + std::vector pos; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + pos.push_back(static_cast(b)); + } + return pos; +} +} // namespace + +BOOST_AUTO_TEST_CASE(sparse_record_fused_stream_interleaves_values_and_stays_walkable) { + using QW = QueryWire<128>; + std::mt19937_64 rng(0xF5EDULL); + std::vector> terms; + std::vector vals; + VecZ plain; + for (size_t i = 0; i < 24; ++i) { + terms.push_back(scattered(rng() % 45, 256, rng)); + vals.push_back(static_cast(i) * 0.5 - 3.25); + Monomial<128> m; + for (const auto p : terms.back()) { + m.set(p); + } + const auto pos = positions_of<128>(m); + (void)QW::push(plain, pos, 1); + } + VecZ fused; + QW::build_fused(plain, vals, fused); + + const QueryForm form = QueryForm::Fused; + BOOST_TEST(QW::count_queries(fused, form) == terms.size()); + size_t off = 0; + for (size_t i = 0; i < terms.size(); ++i) { + BOOST_TEST(QW::k_at(fused, off) == terms[i].size()); + BOOST_TEST(QW::value_at(fused, off) == vals[i]); + std::vector out(terms[i].size() + 1); + const auto rq = QW::read_query(fused, form, off, out); + BOOST_TEST(rq.phase == 1); + out.resize(terms[i].size()); + BOOST_TEST(out == terms[i], boost::test_tools::per_element()); + off = QW::next_off(fused, form, off); + BOOST_TEST(rq.next == off); + } + BOOST_TEST(off == fused.size()); +} + +// A separate case because `-0.0 == 0.0` evaluates true, so bit-exactness needs its own assertion. +BOOST_AUTO_TEST_CASE(sparse_record_fused_value_channel_is_bit_exact_and_reusable) { + using QW = QueryWire<128>; + const QueryForm form = QueryForm::Fused; + + auto push_terms = [](VecZ &buf, const std::vector> &terms) { + for (const auto &t : terms) { + Monomial<128> m; + for (const auto p : t) { + m.set(p); + } + const auto pos = positions_of<128>(m); + (void)QW::push(buf, pos, 1); + } + }; + + // Bit-exactness via memcmp, so -0.0 stays distinguished from 0.0; widths differ per term. + const std::vector values = { + 0.0, + -0.0, + 1.0, + -1.0, + 3.141592653589793, + -2.718281828459045e-300, // near-denormal magnitude + std::numeric_limits::min(), // smallest normal + std::numeric_limits::denorm_min(), + }; + const std::vector> terms = { + {3}, + {0, 255}, + {1, 2, 3, 4, 5, 6, 7}, + {9, 40}, + {2, 3}, + {5, 60, 61, 200}, + {17}, + {0, 1, 2}, + }; + BOOST_REQUIRE_EQUAL(terms.size(), values.size()); + + VecZ plain; + push_terms(plain, terms); + VecZ fused; + QW::build_fused(plain, values, fused); + + size_t off = 0; + for (size_t i = 0; i < values.size(); ++i) { + const double v_out = QW::value_at(fused, off); + BOOST_CHECK(std::memcmp(&v_out, &values[i], sizeof(double)) == 0); + off = QW::next_off(fused, form, off); + } + BOOST_TEST(off == fused.size()); + + // Buffer reuse: sizes must be exact, or a shorter gate reads the previous gate's trailing words. + VecZ plain_big; + std::vector vbig; + std::vector> big; + for (size_t r = 0; r < 32; ++r) { + big.push_back({static_cast(r), static_cast(r + 60)}); + vbig.push_back(static_cast(r) * 1.5 - 7.0); + } + push_terms(plain_big, big); + VecZ out; + QW::build_fused(plain_big, vbig, out); + const size_t cap_after_big = out.capacity(); + + VecZ plain_small; + const std::vector vsmall = {42.0, -42.0, 0.25}; + const std::vector> small = {{1}, {2, 3}, {4, 5, 6}}; + push_terms(plain_small, small); + QW::build_fused(plain_small, vsmall, out); + BOOST_TEST(QW::count_queries(out, form) == vsmall.size()); + BOOST_CHECK_GE(out.capacity(), cap_after_big); + off = 0; + for (size_t i = 0; i < vsmall.size(); ++i) { + const double v_out = QW::value_at(out, off); + BOOST_CHECK(std::memcmp(&v_out, &vsmall[i], sizeof(double)) == 0); + off = QW::next_off(out, form, off); + } + BOOST_TEST(off == out.size()); + + // Empty input: the self slot is cleared before the exchange, into a buffer holding stale words. + VecZ empty; + VecZ dirty{1, 2, 3}; + QW::build_fused(empty, {}, dirty); + BOOST_TEST(dirty.empty()); +} + +// Positions with exactly k entries whose widest gap is exactly `gw`: one gap of 2^(gw-1) -- the smallest +// value of that bit width -- then a contiguous run. Empty if the shape does not fit the universe. +namespace { +auto gap_shaped(size_t k, size_t gw, size_t universe) -> std::vector { + std::vector v; + if (k == 0) { + return v; + } + if (gw == 0 || k == 1) { + if (gw != 0 || k > universe) { + return v; + } + for (size_t j = 0; j < k; ++j) { + v.push_back(static_cast(j)); + } + return v; + } + const size_t second = (size_t{1} << (gw - 1)) + 1U; // gap value 2^(gw-1), i.e. bit_width == gw + if (second + (k - 2U) >= universe) { + return v; + } + v.push_back(0); + for (size_t j = 0; j + 1 < k; ++j) { + v.push_back(static_cast(second + j)); + } + return v; +} +} // namespace + +BOOST_AUTO_TEST_CASE(sparse_record_round_trips_every_reachable_k_and_gap_width) { + // The whole (k, gw) surface, constructed rather than drawn: a random draw reaches neither + // gw = kPosBits nor the escape boundary. `cells` guards against a shape silently failing to fit. + using QW = QueryWire<128>; + size_t cells = 0; + for (size_t k = 0; k <= 40U; ++k) { + for (size_t gw = 0; gw <= QW::kPosBits; ++gw) { + if (k < 2U && gw > 0U) { + continue; // one gap width is reachable below k = 2, so the other rows are the same cell + } + const auto pos = gap_shaped(k, gw, QW::kBits); + if (pos.size() != k) { + continue; + } + BOOST_REQUIRE_EQUAL(QW::gap_width(pos), k < 2 ? 0U : gw); + const size_t w = differential<128>(pos, (k % 3U) == 0U ? 0 : ((k % 3U) == 1U ? 1 : -1)); + BOOST_TEST(w == QW::words_of(QW::gap_bits(k, k < 2 ? 0U : gw))); + ++cells; + } + } + BOOST_TEST(cells == 353U); + + // The width boundary: k = kBits is a fully paired term, one word because every gap is 0. + for (const size_t k : {size_t{254}, size_t{255}, size_t{256}}) { + const auto pos = gap_shaped(k, 0, QW::kBits); + BOOST_REQUIRE_EQUAL(pos.size(), k); + BOOST_TEST(differential<128>(pos, 1) == 1U); + } +} diff --git a/cpp/tests/sparse_resolve_tests.cpp b/cpp/tests/sparse_resolve_tests.cpp new file mode 100644 index 00000000..3ff45eac --- /dev/null +++ b/cpp/tests/sparse_resolve_tests.cpp @@ -0,0 +1,386 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The position-form resolve path, differentially against the queries the caller built and the dense +// Monomial-keyed insert path, neither of which is the code under test. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" +#include "monoprop/detail/evolution/layer_build/Resolve.h" +#include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/operator/OperatorIndex.h" + +using namespace monoprop; + +namespace { + +template +auto random_monomial(std::mt19937_64 &rng, size_t k) -> Monomial { + Monomial m; + std::uniform_int_distribution bit(0, Monomial::size() - 1); + size_t placed = 0; + while (placed < k) { + const size_t b = bit(rng); + if (!m.test(b)) { + m.set(b); + ++placed; + } + } + return m; +} + +// Fully paired terms are the only source of wide records in production, so drawn here explicitly. +template +auto random_paired_monomial(std::mt19937_64 &rng, size_t d) -> Monomial { + Monomial m; + std::uniform_int_distribution mode(0, NumModes - 1); + size_t placed = 0; + while (placed < d) { + const size_t mo = mode(rng); + if (!m.test(2 * mo)) { + m.set(2 * mo); + m.set((2 * mo) + 1); + ++placed; + } + } + return m; +} + +// 0 and 1 for the degenerate records, up to 20 for multi-word ones, 14 for the overflow spill. +const std::vector kPopcounts = {0, 1, 2, 4, 5, 6, 7, 8, 11, 12, 14, 20}; + +template +auto make_op(const std::vector> &terms) -> detail::MPOperator { + detail::MPOperator op; + op.basis = Basis::Majorana; + if (terms.empty()) { + return op; + } + detail::insert_absent_terms( + op, + terms.size(), + [&](size_t k) -> const Monomial & { return terms[k]; }, + [&](size_t k, size_t base) { assign_row(*op.store, base + k, terms[k]); }); + return op; +} + +template +auto draw_distinct(std::mt19937_64 &rng, size_t n) -> std::vector> { + std::vector> out; + std::set> seen; + std::uniform_int_distribution pick(0, kPopcounts.size() - 1); + while (out.size() < n) { + const size_t k = kPopcounts[pick(rng)]; + const auto m = + ((rng() & 1U) != 0U) ? random_paired_monomial(rng, k / 2) : random_monomial(rng, k); + std::vector key; + key.reserve(Monomial::num_words()); + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + if (seen.insert(key).second) { + out.push_back(m); + } + } + return out; +} + +// Extracts an ascending position vector from a Monomial: the wire record is built from positions, and +// production never encodes straight from a bitset. +template +auto positions_of(const Monomial &m) -> std::vector { + std::vector pos; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + pos.push_back(static_cast(b)); + } + return pos; +} + +template +auto serialize(const std::vector>> &queries, bool fused) -> std::vector { + std::vector incoming(queries.size()); + for (size_t s = 0; s < queries.size(); ++s) { + for (size_t q = 0; q < queries[s].size(); ++q) { + const int phase = ((q % 2) == 0) ? 1 : -1; + const auto pos = positions_of(queries[s][q]); + detail::QueryWire::push(incoming[s], pos, phase); + if (fused) { + detail::QueryWire::push_value(incoming[s], 0.5 + static_cast(q)); + } + } + } + return incoming; +} + +template +auto check_probe_matches_the_queries(std::mt19937_64 &rng, size_t n_seed, size_t n_query, size_t rank_count, bool fused) + -> void { + const auto seed_terms = draw_distinct(rng, n_seed); + const auto fresh_terms = draw_distinct(rng, n_query); + + // Hits matter even though the production hit rate is ~0: only they exercise the confirm. + std::vector>> queries(rank_count); + std::set> queried; + size_t hits_planned = 0; + size_t misses_planned = 0; + for (size_t i = 0; i < n_query; ++i) { + const bool want_hit = (i % 3) == 0 && !seed_terms.empty(); + const auto m = want_hit ? seed_terms[i % seed_terms.size()] : fresh_terms[i]; + std::vector key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + // A repeat would violate bulk_insert's precondition; the engine gets distinctness from ^G. + if (!queried.insert(key).second) { + continue; + } + (want_hit ? hits_planned : misses_planned) += 1; + queries[i % rank_count].push_back(m); + } + BOOST_REQUIRE(hits_planned > 0); + BOOST_REQUIRE(misses_planned > 0); + + std::vector> expect_mono; + std::vector expect_phase; + std::vector expect_sender; + for (size_t s = 0; s < rank_count; ++s) { + for (size_t q = 0; q < queries[s].size(); ++q) { + expect_mono.push_back(queries[s][q]); + expect_phase.push_back(((q % 2) == 0) ? 1 : -1); + expect_sender.push_back(s); + } + } + + const auto incoming = serialize(queries, fused); + const detail::QueryForm form = fused ? detail::QueryForm::Fused : detail::QueryForm::Plain; + + auto op = make_op(seed_terms); + const auto pr = detail::probe_incoming_queries(incoming, op, rank_count, form); + + BOOST_REQUIRE_EQUAL(pr.nq_total, expect_mono.size()); + BOOST_REQUIRE(pr.nq_total > 0); + BOOST_REQUIRE_EQUAL(pr.pos_off.size(), pr.nq_total); + + std::set> seeded; + for (const auto &m : seed_terms) { + std::vector key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + seeded.insert(key); + } + + size_t hits_seen = 0; + size_t wide_seen = 0; + std::vector> expected_misses; + for (size_t g = 0; g < pr.nq_total; ++g) { + const Monomial &want = expect_mono[g]; + BOOST_TEST((pr.mono_at(g) == want)); + BOOST_TEST(pr.k_of[g] == want.count()); + BOOST_TEST(pr.phase_of[g] == expect_phase[g]); + BOOST_TEST(pr.sender_of[g] == expect_sender[g]); + BOOST_TEST(pr.is_paired_at(g) == monoprop::is_paired(want)); + + std::vector key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(want.word(w)); + } + const bool want_hit = seeded.count(key) != 0; + BOOST_TEST((pr.idx_of[g] < pr.base) == want_hit); + if (want_hit) { + ++hits_seen; + } + else { + expected_misses.push_back(want); + } + VecZ scratch; + const auto want_pos = positions_of(want); + if (detail::QueryWire::push(scratch, want_pos, expect_phase[g]) > 1U) { + ++wide_seen; + } + } + // Vacuous-pass guards: no hit means the confirm never ran, no wide record means the cursor didn't. + BOOST_TEST(hits_seen > 0); + BOOST_TEST(wide_seen > 0); + + BOOST_REQUIRE_EQUAL(pr.miss_g.size(), expected_misses.size()); + for (size_t j = 0; j < pr.miss_g.size(); ++j) { + BOOST_TEST((expect_mono[pr.miss_g[j]] == expected_misses[j])); + BOOST_TEST(pr.idx_of[pr.miss_g[j]] == pr.base + j); + } + + detail::insert_incoming_misses(op, pr); + + // The second implementation: the dense Monomial-keyed path, sharing no code with set_positions. + auto ref = make_op(seed_terms); + detail::insert_absent_terms( + ref, + expected_misses.size(), + [&](size_t j) -> const Monomial & { return expected_misses[j]; }, + [&](size_t j, size_t base) { assign_row(*ref.store, base + j, expected_misses[j]); }); + + BOOST_REQUIRE_EQUAL(op.store->size(), ref.store->size()); + BOOST_TEST(op.store->size() > pr.base); + size_t overflow_seen = 0; + for (size_t i = 0; i < ref.store->size(); ++i) { + BOOST_TEST((op.store->row(i) == ref.store->row(i))); + BOOST_TEST(op.store->popcount(i) == ref.store->popcount(i)); + if (!ref.store->row_positions(i).inlined()) { + ++overflow_seen; + } + } + BOOST_TEST(overflow_seen > 0); + + // The index, not just the rows: a wrong hash leaves the row correct and unfindable. + for (size_t i = 0; i < ref.store->size(); ++i) { + const auto key = ref.store->row(i); + const auto in_op = op.store->find(key); + const auto in_ref = ref.store->find(key); + BOOST_REQUIRE(in_ref.has_value()); + BOOST_REQUIRE(in_op.has_value()); + BOOST_TEST(*in_op == *in_ref); + BOOST_TEST(*in_ref == i); + } +} + +} // namespace + +/* ── The check, across both position widths and both buffer layouts ── */ + +BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_narrow_positions) { + std::mt19937_64 rng(20260814); + static_assert(sizeof(detail::OperatorIndex<32>::PosT) == 1, "this case exists to cover the narrowing decode"); + check_probe_matches_the_queries<32>(rng, /*n_seed=*/40, /*n_query=*/90, /*rank_count=*/3, /*fused=*/false); +} + +BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_wide_positions) { + std::mt19937_64 rng(20260815); + static_assert(sizeof(detail::OperatorIndex<250>::PosT) == 2, "this case exists to cover the wide store"); + check_probe_matches_the_queries<250>(rng, /*n_seed=*/60, /*n_query=*/140, /*rank_count=*/4, /*fused=*/false); +} + +BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_fused_layout) { + std::mt19937_64 rng(20260816); + check_probe_matches_the_queries<250>(rng, /*n_seed=*/50, /*n_query=*/120, /*rank_count=*/2, /*fused=*/true); +} + +BOOST_AUTO_TEST_CASE(sparse_resolve_probe_matches_single_sender) { + std::mt19937_64 rng(20260817); + check_probe_matches_the_queries<32>(rng, /*n_seed=*/25, /*n_query=*/60, /*rank_count=*/1, /*fused=*/true); +} + +/* ── The pieces, pinned individually ──────────────────────────────────────── */ + +BOOST_AUTO_TEST_CASE(sparse_resolve_set_positions_matches_set) { + constexpr size_t kN = 250; + constexpr size_t kInlineWidth = 11; + std::mt19937_64 rng(20260818); + const auto terms = draw_distinct(rng, 200); + + detail::OperatorIndex from_mono(kInlineWidth); + detail::OperatorIndex from_pos(kInlineWidth); + from_mono.grow_rows_geometric(terms.size()); + from_pos.grow_rows_geometric(terms.size()); + + size_t spilled = 0; + for (size_t i = 0; i < terms.size(); ++i) { + from_mono.set(i, terms[i]); + std::vector::PosT> pos; + for (size_t b = terms[i].find_first(); b < terms[i].size(); b = terms[i].find_next(b)) { + pos.push_back(static_cast::PosT>(b)); + } + from_pos.set_positions(i, pos); + if (pos.size() > kInlineWidth) { + ++spilled; + } + } + BOOST_TEST(spilled > 0); + for (size_t i = 0; i < terms.size(); ++i) { + BOOST_TEST((from_pos.row(i) == from_mono.row(i))); + BOOST_TEST((from_pos.row(i) == terms[i])); + BOOST_TEST(from_pos.popcount(i) == from_mono.popcount(i)); + BOOST_TEST(from_pos.row_positions(i).inlined() == from_mono.row_positions(i).inlined()); + } + BOOST_TEST(from_pos.overflow_size() == from_mono.overflow_size()); +} + +BOOST_AUTO_TEST_CASE(sparse_resolve_finds_dense_inserted_keys) { + // The hash identity, isolated: fold_hash_positions differing from fold_hash misses, and legally. + constexpr size_t kN = 250; + std::mt19937_64 rng(20260819); + const auto terms = draw_distinct(rng, 300); + auto op = make_op(terms); + + std::vector::PosT> flat; + std::vector off; + std::vector kk; + for (const auto &m : terms) { + off.push_back(flat.size()); + size_t k = 0; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + flat.push_back(static_cast::PosT>(b)); + ++k; + } + kk.push_back(static_cast(k)); + } + std::vector out(terms.size(), 0); + std::vector hashes(terms.size(), 0); + op.store->find_batch_positions(flat, off, kk, out, hashes); + + std::vector out_dense(terms.size(), 0); + op.store->find_batch(terms.data(), terms.size(), out_dense.data()); + for (size_t i = 0; i < terms.size(); ++i) { + BOOST_REQUIRE(out[i] != detail::OperatorIndex::kNotFound); + BOOST_TEST(out[i] == i); + BOOST_TEST(out[i] == out_dense[i]); + BOOST_TEST(hashes[i] + == detail::OperatorIndex::fold_hash_positions( + std::span::PosT>(flat).subspan(off[i], kk[i]))); + } + + const auto absent = draw_distinct(rng, 50); + std::vector::PosT> aflat; + std::vector aoff; + std::vector akk; + for (const auto &m : absent) { + aoff.push_back(aflat.size()); + size_t k = 0; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + aflat.push_back(static_cast::PosT>(b)); + ++k; + } + akk.push_back(static_cast(k)); + } + std::vector aout(absent.size(), 0); + op.store->find_batch_positions(aflat, aoff, akk, aout); + size_t genuinely_absent = 0; + for (size_t i = 0; i < absent.size(); ++i) { + // draw_distinct may re-draw a seeded term; only genuinely absent ones are evidence. + if (!op.store->find(absent[i]).has_value()) { + BOOST_TEST(aout[i] == detail::OperatorIndex::kNotFound); + ++genuinely_absent; + } + } + BOOST_TEST(genuinely_absent > 0); +}