From 653bb9ed9f17c7ecc465ac10cc581c1ec774abcc Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 2 Nov 2026 19:54:16 +0100 Subject: [PATCH 1/2] fix(callgrind): model REP copies with calibrated bulk costs Replace per-element REP MOVS/STOS cycle estimates with measured hot-copy curves on amd64. Charge the calibrated marginal through existing Ct/Cl counters and report only REP data misses, retaining cache simulation, instruction counters, and all miss penalties. Enable the model with cache and cycle estimation by default; --rep-bulk-cost=no preserves legacy accounting. Keep the output format unchanged and document mixed-counter, overlap, cache-residency, and vector-comparison limitations. Add default and legacy runtime regressions for zero-length copies, hot/cold counters, backward overlap, stores, and self-copy. --- CODSPEED-CHANGELOG.md | 24 + callgrind/Makefile.am | 10 +- callgrind/clo.c | 7 + callgrind/cycledecode.c | 96 + callgrind/cycledecode.h | 28 + callgrind/docs/cl-manual.xml | 41 + callgrind/global.h | 26 + callgrind/main.c | 106 + callgrind/rep_bulk.c | 208 ++ callgrind/rep_bulk.h | 87 + callgrind/rep_bulk_table.inc | 3578 ++++++++++++++++++++ callgrind/sim.c | 145 +- callgrind/tests/Makefile.am | 11 +- callgrind/tests/rep_bulk.c | 128 + callgrind/tests/rep_bulk.post.exp | 11 + callgrind/tests/rep_bulk.stderr.exp | 0 callgrind/tests/rep_bulk.vgtest | 5 + callgrind/tests/rep_bulk_check.pl | 125 + callgrind/tests/rep_bulk_legacy.post.exp | 8 + callgrind/tests/rep_bulk_legacy.stderr.exp | 0 callgrind/tests/rep_bulk_legacy.vgtest | 5 + callgrind/threads.c | 15 + 22 files changed, 4651 insertions(+), 13 deletions(-) create mode 100644 callgrind/rep_bulk.c create mode 100644 callgrind/rep_bulk.h create mode 100644 callgrind/rep_bulk_table.inc create mode 100644 callgrind/tests/rep_bulk.c create mode 100644 callgrind/tests/rep_bulk.post.exp create mode 100644 callgrind/tests/rep_bulk.stderr.exp create mode 100644 callgrind/tests/rep_bulk.vgtest create mode 100755 callgrind/tests/rep_bulk_check.pl create mode 100644 callgrind/tests/rep_bulk_legacy.post.exp create mode 100644 callgrind/tests/rep_bulk_legacy.stderr.exp create mode 100644 callgrind/tests/rep_bulk_legacy.vgtest diff --git a/CODSPEED-CHANGELOG.md b/CODSPEED-CHANGELOG.md index 61808930c4..eff5b339c0 100644 --- a/CODSPEED-CHANGELOG.md +++ b/CODSPEED-CHANGELOG.md @@ -23,6 +23,30 @@ matching Linux on AArch64. ## Features +### REP bulk cost model for `rep movs` / `rep stos` + +amd64 `rep movs{b,w,d,q}` and `rep stos{b,w,d,q}` are now priced from a +calibrated hot-copy bulk curve instead of the per-element instruction/access +model. It is enabled by default when both `--cache-sim=yes` and +`--cycle-estimation=yes` are active; `--rep-bulk-cost=no` restores the previous +behaviour. No event names or counters are added, and disabled cache or cycle +modes are unchanged. + +For a modeled REP instruction, `Ct` and `Cl` carry the calibrated cost. +Instruction-side counters (`Ir`/`I1mr`/`ILmr`) are unchanged. On the data side, +only accesses that miss in L1 are reported as `Dr`/`Dw`, together with their +`D1mr`/`D1mw` and `DLmr`/`DLmw` misses: per-element data L1 hit charges +disappear while every real miss is still counted. The cache simulation, the +counters of all non-REP accesses and instructions, and the output format are +unchanged. + +The table is calibrated for disjoint-buffer MOVS/STOS with cache-resident +working sets: MOVS up to 16 KiB and STOS up to 32 KiB per element width, with +the last measured marginal slope used beyond that. Overlapping MOVS and +L2/L3-resident bulk copies are outside the calibrated domain. +This changes REP pricing only, not the existing scalar/vector cost model. +Relative REP-versus-vector scores are not calibrated native speedup estimates. + ### Fixed Linux startup stack layout Reduce simulated cache-miss variation from argument and environment sizes diff --git a/callgrind/Makefile.am b/callgrind/Makefile.am index de92e16b89..0b90003128 100644 --- a/callgrind/Makefile.am +++ b/callgrind/Makefile.am @@ -22,11 +22,14 @@ noinst_HEADERS = \ cycledecode_capstone.h \ sigkey.h \ events.h \ - global.h + global.h \ + rep_bulk.h # Generated cost tables (#included by cycledecode.c under CLG_WITH_CAPSTONE; -# the arch is selected at compile time). -EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc +# the arch is selected at compile time) and the calibrated REP bulk curve +# table included by rep_bulk.c (generated by the instruction cost calibration +# pipeline). +EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc rep_bulk_table.inc #---------------------------------------------------------------------------- # callgrind- @@ -52,6 +55,7 @@ CALLGRIND_SOURCES_COMMON = \ fn.c \ jumps.c \ main.c \ + rep_bulk.c \ sim.c \ subprocess.c \ threads.c diff --git a/callgrind/clo.c b/callgrind/clo.c index 911051a27d..5675b99de4 100644 --- a/callgrind/clo.c +++ b/callgrind/clo.c @@ -568,6 +568,7 @@ Bool CLG_(process_cmd_line_option)(const HChar* arg) else if VG_BOOL_CLO(arg, "--branch-sim", CLG_(clo).simulate_branch) {} else if VG_BOOL_CLO(arg, "--cycle-estimation", CLG_(clo).cycle_estimation) {} + else if VG_BOOL_CLO(arg, "--rep-bulk-cost", CLG_(clo).rep_bulk_cost) {} else { Bool isCachesimOption = (*CLG_(cachesim).parse_opt)(arg); @@ -617,6 +618,11 @@ void CLG_(print_usage)(void) " --collect-jumps=no|yes Collect jumps? [no]\n" " --collect-bus=no|yes Collect global bus events? [no]\n" " --cycle-estimation=no|yes Estimate per-instruction cycles (Ct/Cl)? [no]\n" +" --rep-bulk-cost=no|yes Model supported rep movs/stos by a calibrated\n" +" bulk curve (opt-out for A/B) [yes]\n" +" Active only with --cache-sim=yes and --cycle-estimation=yes; the\n" +" raw L1-hit refs of a modeled rep are replaced by its bulk cost while\n" +" misses stay reported. See docs/cl-manual.xml.\n" #if CLG_EXPERIMENTAL " --collect-alloc=no|yes Collect memory allocation info? [no]\n" #endif @@ -704,6 +710,7 @@ void CLG_(set_clo_defaults)(void) CLG_(clo).simulate_cache = False; CLG_(clo).simulate_branch = False; CLG_(clo).cycle_estimation = False; + CLG_(clo).rep_bulk_cost = True; /* Call graph */ CLG_(clo).pop_on_jump = False; diff --git a/callgrind/cycledecode.c b/callgrind/cycledecode.c index c3ba0720ca..d17259f3f2 100644 --- a/callgrind/cycledecode.c +++ b/callgrind/cycledecode.c @@ -351,3 +351,99 @@ void clg_cycledecode_stats(ClgCdStats* out) "Callgrind cycle estimation requires Capstone; configure with --with-capstone=PATH (or set CAPSTONE_DIR)." #endif + +#if defined(VGA_amd64) || defined(VGA_x86) + +/*------------------------------------------------------------*/ +/*--- REP string instruction classification ---*/ +/*------------------------------------------------------------*/ + +/* CodSpeed extension, see cycledecode.h. Plain x86/amd64 encoding check: the + * IMark bytes of one instruction, prefixes followed by A4/A5/AA/AB. F2 + * (REPNE) is rejected so only forward-countable bulk copies are classified; + * this never has to agree with VEX's decoder on anything else. */ +void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, + ClgRepBulkInfo* out) +{ + unsigned i = 0; + int rep = 0, opsz16 = 0, addr32p = 0; + int rex_w = 0, rex_seen = 0; + unsigned char opc; + unsigned width; + + out->op = CLG_REP_OP_NONE; + out->elem_width = 0; + out->addr32 = 0; + if (!bytes || len < 2 || len > 15) + return; + + while (i < len) { + unsigned char c = bytes[i]; + if (c == 0xF3 && !rep) { + rep = 1; + i++; + continue; + } + if (c == 0xF2) + return; /* REPNE: never a supported bulk copy */ + if (c == 0x66 && !opsz16) { + opsz16 = 1; + i++; + continue; + } + if (c == 0x67 && !addr32p) { + addr32p = 1; + i++; + continue; + } + if (c >= 0x40 && c <= 0x4F && !rex_seen) { + rex_seen = 1; + rex_w = (c & 0x08) != 0; + i++; + continue; + } + break; + } + + /* Exactly the opcode must follow the prefix run: no operands, no + * trailing bytes (the IMark length always covers the whole insn). */ + if (!rep || i + 1 != len) + return; + + opc = bytes[i]; + switch (opc) { + case 0xA4: /* movsb */ + width = 1; + out->op = CLG_REP_OP_MOVS; + break; + case 0xA5: /* movsw/movsd/movsq */ + width = rex_w ? 8 : (opsz16 ? 2 : 4); + out->op = CLG_REP_OP_MOVS; + break; + case 0xAA: /* stosb */ + width = 1; + out->op = CLG_REP_OP_STOS; + break; + case 0xAB: /* stosw/stosd/stosq */ + width = rex_w ? 8 : (opsz16 ? 2 : 4); + out->op = CLG_REP_OP_STOS; + break; + default: + return; + } + + out->elem_width = (unsigned char)width; + out->addr32 = addr32p ? 1 : 0; +} + +#else + +void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, + ClgRepBulkInfo* out) +{ + out->op = CLG_REP_OP_NONE; + out->elem_width = 0; + out->addr32 = 0; +} + +#endif diff --git a/callgrind/cycledecode.h b/callgrind/cycledecode.h index de0e53815b..bbd3517224 100644 --- a/callgrind/cycledecode.h +++ b/callgrind/cycledecode.h @@ -53,4 +53,32 @@ void clg_cycle_cost(const unsigned char* bytes, /* Per-tier diagnostic counters since init. */ void clg_cycledecode_stats(ClgCdStats* out); +/*------------------------------------------------------------*/ +/*--- REP string instruction classification ---*/ +/*------------------------------------------------------------*/ + +/* CodSpeed extension, used by the REP bulk cost model. This is a plain + * x86/amd64 encoding check, independent of Capstone and of the cycle table: + * only rep-prefixed MOVS and STOS are classified, because those are the only + * string forms whose per-element cost can be replaced by a bulk copy curve. + * CMPS/SCAS/LODS/INS/OUTS, F2 (REPNE) prefixed insns and non-rep MOVS/STOS + * are reported as CLG_REP_OP_NONE so the legacy model is kept for them. */ + +typedef enum { + CLG_REP_OP_NONE = 0, + CLG_REP_OP_MOVS = 1, /* rep movs{b,w,d,q}: one read + one write per element */ + CLG_REP_OP_STOS = 2 /* rep stos{b,w,d,q}: one write per element */ +} ClgRepBulkOp; + +typedef struct { + unsigned char op; /* ClgRepBulkOp */ + unsigned char elem_width; /* element size in bytes (1,2,4,8), 0 if none */ + unsigned char addr32; /* 67 prefix: the repeat count is 32-bit */ +} ClgRepBulkInfo; + +/* Classify the guest instruction in [bytes, bytes+len). Always initialises + * *out; *out->op == CLG_REP_OP_NONE when not a supported bulk form. */ +void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len, + ClgRepBulkInfo* out); + #endif /* CLG_CYCLEDECODE_H */ diff --git a/callgrind/docs/cl-manual.xml b/callgrind/docs/cl-manual.xml index 8b4e50d853..e24b03dbe3 100644 --- a/callgrind/docs/cl-manual.xml +++ b/callgrind/docs/cl-manual.xml @@ -995,6 +995,47 @@ Also see . + + + + + + Model amd64 rep movs and + rep stos string instructions with a + calibrated bulk cost curve instead of the per-element model. The option + is active only when both and + are given; otherwise it is + inert and the output is unchanged. It is also disabled for + , and + non-amd64 guests. + For a modeled instruction the estimated cycles ("Ct"/"Cl") come + from a calibration table measured on hot, cache-resident data, and both + counters carry the same value so a geometric-mean consumer recovers it. + Instruction-side counters are unchanged: a modeled REP still reports one + instruction reference per element (the VEX self-loop) and its real + instruction misses. On the data side, only accesses that miss in L1 are + reported, as one "Dr"/"Dw" reference together with its "D1mr"/"D1mw" and, + where applicable, "DLmr"/"DLmw" miss; data L1 hits are not reported + because the bulk curve already covers hot data movement. Thus Dr equals + D1mr and Dw equals D1mw for modeled REP instructions: these are modeled + data references, not raw per-element accesses. The cache simulation + itself is unchanged. No event names are added, so + reproduces the previous per-element + model exactly. + The calibration covers disjoint-buffer MOVS/STOS with + cache-resident working sets up to a per-width calibrated maximum + (16 KiB for MOVS, 32 KiB for STOS); larger copies are extrapolated with + the last measured marginal slope, and overlapping MOVS may take a + slower hardware path that the table does not model. Accuracy for + cache-miss-heavy or L2/L3-resident bulk copies is therefore limited. On + lines or functions that mix modeled REP instructions with other + instructions the consumer's geometric mean of the summed Ct/Cl is an + approximation, not an exact decomposition. Scalar and vector instruction + pricing is unchanged, so relative REP-versus-vector scores are not + calibrated native speedup estimates. + + + diff --git a/callgrind/global.h b/callgrind/global.h index 97a22bf252..e323d6e523 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -128,6 +128,7 @@ struct _CommandLineOptions { Bool simulate_cache; /* Call into cache simulator ? */ Bool simulate_branch; /* Call into branch prediction simulator ? */ Bool cycle_estimation; /* Estimate per-instruction cycles (Ct/Cl) ? */ + Bool rep_bulk_cost; /* Replace per-element REP cost by bulk curve ? */ /* Call graph generation */ Bool pop_on_jump; /* Handle a jump between functions as ret+call */ @@ -279,6 +280,12 @@ struct _InstrInfo { UInt ct_incl; UInt cl_cost; UInt cl_incl; + /* REP bulk cost model classification (see cycledecode.h). Only filled on + * first translation and only when the model is active; rep_bulk_op == 0 + * means the instruction is not a supported rep-prefixed bulk copy. */ + UChar rep_bulk_op; + UChar rep_elem_width; + UChar rep_addr32; }; @@ -528,6 +535,16 @@ struct _exec_state { BBCC* nonskipped; Int call_stack_bottom; /* Index into fn_stack */ + + /* REP bulk cost model: alignment class and direction are latched once per + * REP operation from its first element, and the expected remaining count + * detects the next operation. Saved/restored on thread and signal + * switches so an interrupted REP keeps its class. */ + InstrInfo* rep_ii; /* instruction site of the running operation */ + ULong rep_next_rem; /* expected remaining count after last element */ + UChar rep_dir; /* CLG_REP_DIR_* latched at operation start */ + UChar rep_align; /* CLG_REP_ALIGN_* latched at operation start */ + ULong rep_tail_carry; /* remainder of the tail slope, per operation */ }; /* Global state structures */ @@ -687,10 +704,19 @@ struct cachesim_if void (*log_0I1Dr)(InstrInfo*, Addr, Word) VG_REGPARM(3); void (*log_0I1Dw)(InstrInfo*, Addr, Word) VG_REGPARM(3); + /* REP bulk cost model: called once per executed element of a supported + * rep-prefixed bulk copy. Args: inode, repeat count after the decrement + * of this element, source address (0 if none), destination address, + * packed op|elem_width<<8, guest DF flag. Zero when the model is off. + * Uses the ordinary amd64 calling convention (regparms=0), not a + * register-parm ABI. */ + void (*log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word, Word); + // function names of helpers (for debugging generated code) const HChar *log_1I0D_name, *log_2I0D_name, *log_3I0D_name; const HChar *log_1I1Dr_name, *log_1I1Dw_name; const HChar *log_0I1Dr_name, *log_0I1Dw_name; + const HChar *log_rep_bulk_name; }; // Event groups diff --git a/callgrind/main.c b/callgrind/main.c index 69af08255f..1592947ded 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -41,6 +41,17 @@ #include "cycledecode.h" +#if defined(VGA_amd64) +/* REP bulk cost model: read the repeat counter and DF from the guest state. + * VEX translates rep movs/stos on amd64 as one element per BB execution with + * an unconditional jump back to the instruction, and uses RCX (or ECX with a + * 67 prefix) as the repeat counter, decremented before the element access. */ +#include "libvex_guest_amd64.h" +#define CLG_REP_COUNT_OFF offsetof(VexGuestAMD64State, guest_RCX) +#define CLG_REP_COUNT_TY Ity_I64 +#define CLG_REP_DFLAG_OFF offsetof(VexGuestAMD64State, guest_DFLAG) +#endif + /*------------------------------------------------------------*/ /*--- Global variables ---*/ /*------------------------------------------------------------*/ @@ -427,6 +438,14 @@ static void flushEvents ( ClgState* clgs ) default: tl_assert(0); } + + /* REP bulk model: REP instructions carry per-instruction Ct/Cl + * slots, filled at runtime from the calibrated bulk curve instead + * of the static per-instruction cycle estimate. */ + if (ev->inode->rep_bulk_op) { + ev->inode->eventset = + CLG_(add_event_group)(ev->inode->eventset, EG_CYCLES); + } } } @@ -594,6 +613,72 @@ static void flushEvents ( ClgState* clgs ) addStmtToIRSB( clgs->sbOut, IRStmt_Dirty(di) ); } + /* REP bulk cost model: charge one calibrated bulk delta per executed + * element. Emitted after the normal memory helpers; the cache simulation + * is unchanged. `Get(RCX)` is the repeat count + * *after* VEX's decrement for this element (VEX decrements before the + * access), so the helper adds 1 back. */ + if (CLG_(cachesim).log_rep_bulk && clgs->events_used > 0) { + InstrInfo* rii = 0; + IRExpr* src = 0; + IRExpr* dst = 0; + Bool has_data = False; + Int j; + + for (j = clgs->events_used - 1; j >= 0; j--) { + Event* e = &clgs->events[j]; + if (e->tag != Ev_Dr && e->tag != Ev_Dw && e->tag != Ev_Dm) + continue; + if (!rii) + rii = e->inode; + if (e->inode != rii) + continue; + has_data = True; + if (e->tag == Ev_Dr) { + if (!src) + src = e->Ev.Dr.ea; + } else { + if (!dst) + dst = get_Event_dea(e); + } + } + + if (rii && rii->rep_bulk_op && has_data && dst) { + /* The two addresses are only used to classify alignment once per + * operation; STOS and merged read-modify events have no source. */ + IRExpr* packed = mkIRExpr_HWord( + (HWord)rii->rep_bulk_op | ((HWord)rii->rep_elem_width << 8)); +#if defined(VGA_amd64) + IRTemp count_tmp = newIRTemp(clgs->sbOut->tyenv, CLG_REP_COUNT_TY); + IRTemp dflag_tmp = newIRTemp(clgs->sbOut->tyenv, Ity_I64); + IRExpr* cnt = IRExpr_RdTmp(count_tmp); + IRExpr* dfl = IRExpr_RdTmp(dflag_tmp); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( + count_tmp, IRExpr_Get(CLG_REP_COUNT_OFF, CLG_REP_COUNT_TY))); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( + dflag_tmp, IRExpr_Get(CLG_REP_DFLAG_OFF, Ity_I64))); + + if (rii->rep_addr32) { + IRTemp masked_tmp = newIRTemp(clgs->sbOut->tyenv, Ity_I64); + addStmtToIRSB(clgs->sbOut, IRStmt_WrTmp( + masked_tmp, IRExpr_Binop(Iop_And64, cnt, + IRExpr_Const(IRConst_U64(0xFFFFFFFFULL))))); + cnt = IRExpr_RdTmp(masked_tmp); + } + + di = unsafeIRDirty_0_N( + 0, CLG_(cachesim).log_rep_bulk_name, + VG_(fnptr_to_fnentry)(CLG_(cachesim).log_rep_bulk), + mkIRExprVec_6( mkIRExpr_HWord((HWord)rii), cnt, + src ? src : mkIRExpr_HWord(0), + dst, packed, dfl ) ); + addStmtToIRSB( clgs->sbOut, IRStmt_Dirty(di) ); +#else + (void)src; (void)dst; (void)packed; +#endif + } + } + clgs->events_used = 0; } @@ -795,6 +880,9 @@ InstrInfo* next_InstrInfo ( ClgState* clgs, UInt instr_size ) ii->ct_incl = 0; ii->cl_cost = 0; ii->cl_incl = 0; + ii->rep_bulk_op = 0; + ii->rep_elem_width = 0; + ii->rep_addr32 = 0; } clgs->ii_index++; @@ -1055,6 +1143,24 @@ IRSB* CLG_(instrument)( VgCallbackClosure* closure, curr_inode->ct_cost = ct; curr_inode->cl_cost = cl; } + + // Classify supported rep-prefixed bulk copies (rep movs/rep stos) for + // the REP bulk cost model. The decision is static per instruction. + if (CLG_(clo).rep_bulk_cost && !clgs.seen_before) { + ClgRepBulkInfo ri; + unsigned len = st->Ist.IMark.len ? st->Ist.IMark.len : isize; + clg_rep_bulk_classify((const unsigned char*)cia, len, &ri); + curr_inode->rep_bulk_op = ri.op; + curr_inode->rep_elem_width = ri.elem_width; + curr_inode->rep_addr32 = ri.addr32; + if (ri.op != CLG_REP_OP_NONE) { + /* The calibrated bulk curve replaces this instruction's + * static per-element cycle estimate completely: drop it from + * both the dump-time charge and the inclusive prefix sums. */ + curr_inode->ct_cost = 0; + curr_inode->cl_cost = 0; + } + } addEvent_Ir( &clgs, curr_inode ); break; } diff --git a/callgrind/rep_bulk.c b/callgrind/rep_bulk.c new file mode 100644 index 0000000000..4de335d815 --- /dev/null +++ b/callgrind/rep_bulk.c @@ -0,0 +1,208 @@ +/*--------------------------------------------------------------------*/ +/*--- REP bulk cost model: calibrated hot-copy curve. ---*/ +/*--- rep_bulk.c ---*/ +/*--------------------------------------------------------------------*/ + +/* CodSpeed extension. See rep_bulk.h. + * + * The calibrated table is generated by the instruction cost calibration + * pipeline; it is committed as `rep_bulk_table.inc` and included here. This + * file only does lookup, monotone-safe integer interpolation and the exact + * linear tail extrapolation. + */ + +#include "rep_bulk.h" + +/* Generated by the calibration pipeline. Defines + * clg_rep_bulk_provenance[], clg_rep_bulk_pts_[] and + * clg_rep_bulk_curves[] (types from rep_bulk.h). */ +#include "rep_bulk_table.inc" + +#define N_CURVES ((UInt)(sizeof(clg_rep_bulk_curves) / \ + sizeof(clg_rep_bulk_curves[0]))) + +#define N_WIDTHS 4 +#define N_DIRS 2 +#define N_ALIGN 3 + +/* Exact-key index, resolved once at startup. Only valid keys are stored; the + * model is enabled only if every required key is present. */ +static const ClgRepBulkCurve* curve_idx[CLG_REP_OP_STOS + 1][N_WIDTHS + 1] + [N_DIRS][N_ALIGN]; +static Bool curve_idx_done = False; + +static Int width_index(unsigned elem_width) +{ + switch (elem_width) { + case 1: return 0; + case 2: return 1; + case 4: return 2; + case 8: return 3; + default: return -1; + } +} + +static Bool key_valid(const ClgRepBulkKey* k) +{ + return (k->op == CLG_REP_OP_MOVS || k->op == CLG_REP_OP_STOS) + && width_index(k->elem_width) >= 0 + && k->direction < N_DIRS + && k->align_class < N_ALIGN; +} + +/* Interpolate linearly between the neighbouring sampled byte sizes. Integer + * floor rounding keeps the result monotone, so marginal costs never become + * negative (consecutive element deltas telescope exactly for the same F). */ +static ULong cost_at(const ClgRepBulkCurve* c, ULong bytes) +{ + const ClgRepBulkPoint* p = c->pts; + UInt n = c->npts; + + if (n == 0) + return 0; + if (bytes <= p[0].bytes) + return p[0].centicycles; + + /* Last point <= bytes. */ + UInt lo = 0, hi = n - 1; + while (lo < hi) { + UInt mid = lo + (hi - lo + 1) / 2; + if (p[mid].bytes <= bytes) + lo = mid; + else + hi = mid - 1; + } + + if (p[lo].bytes == bytes) + return p[lo].centicycles; + + if (lo + 1 < n) { + ULong dbytes = (ULong)(p[lo + 1].bytes - p[lo].bytes); + ULong dcost = p[lo + 1].centicycles - p[lo].centicycles; + ULong off = bytes - p[lo].bytes; + return p[lo].centicycles + (off * dcost) / dbytes; + } + + /* Beyond the last sampled size: extrapolate with its marginal slope. */ + { + ULong dbytes = (ULong)(p[lo].bytes - p[lo - 1].bytes); + ULong dcost = p[lo].centicycles - p[lo - 1].centicycles; + ULong off = bytes - p[lo].bytes; + return p[lo].centicycles + (off * dcost) / dbytes; + } +} + +UInt clg_rep_bulk_init(void) +{ + UInt bad = 0, i, w, d, a; + + for (w = 0; w <= N_WIDTHS; w++) + for (d = 0; d < N_DIRS; d++) + for (a = 0; a < N_ALIGN; a++) { + curve_idx[CLG_REP_OP_MOVS][w][d][a] = 0; + curve_idx[CLG_REP_OP_STOS][w][d][a] = 0; + } + + /* Table validation: keys, F(0)==0, ascending sizes, monotone costs. */ + for (i = 0; i < N_CURVES; i++) { + const ClgRepBulkCurve* c = &clg_rep_bulk_curves[i]; + UInt j; + + if (!key_valid(&c->key) || c->npts < 2) { + bad++; + continue; + } + if (c->pts[0].bytes != 0 || c->pts[0].centicycles != 0) { + bad++; + continue; + } + for (j = 1; j < c->npts; j++) { + if (c->pts[j].bytes <= c->pts[j - 1].bytes + || c->pts[j].centicycles < c->pts[j - 1].centicycles) { + bad++; + break; + } + } + if (j != c->npts) + continue; + + curve_idx[c->key.op][width_index(c->key.elem_width)] + [c->key.direction][c->key.align_class] = c; + } + + /* Every key is required: MOVS and STOS, all four widths, forward and + * backward, all three alignment classes. There is no cross-direction or + * cross-class substitution; the caller disables the model if any key is + * missing so the legacy per-element model stays in effect. */ + for (w = 0; w < N_WIDTHS; w++) + for (d = 0; d < N_DIRS; d++) + for (a = 0; a < N_ALIGN; a++) { + if (!curve_idx[CLG_REP_OP_MOVS][w][d][a]) + bad++; + if (!curve_idx[CLG_REP_OP_STOS][w][d][a]) + bad++; + } + + /* Keep the provenance string referenced (it documents the artifact). */ + if (clg_rep_bulk_provenance[0] == 0) + bad++; + + curve_idx_done = True; + return bad; +} + +ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class, + ULong remaining_bytes, ULong elem_bytes, + ULong* tail_carry) +{ + const ClgRepBulkCurve* c; + const ClgRepBulkPoint* p; + UInt n; + Int wi; + ULong hi, lo; + + if (!curve_idx_done) + clg_rep_bulk_init(); + + wi = width_index(elem_width); + if (wi < 0 || op <= CLG_REP_OP_NONE || op > CLG_REP_OP_STOS + || direction >= N_DIRS || align_class >= N_ALIGN) + return 0; + c = curve_idx[op][wi][direction][align_class]; + if (!c || c->npts < 2 || elem_bytes == 0 || remaining_bytes < elem_bytes) + return 0; + + p = c->pts; + n = c->npts; + + /* Inside the calibrated range: exact telescoping difference of F. */ + if (remaining_bytes <= p[n - 1].bytes) { + *tail_carry = 0; + lo = cost_at(c, remaining_bytes - elem_bytes); + hi = cost_at(c, remaining_bytes); + return hi > lo ? hi - lo : 0; + } + + /* Entirely beyond the last calibrated point: F is linear there, so the + * exact marginal is elem_bytes * (dcost/dbytes). Keep the fractional + * remainder in *tail_carry so any run of tail elements sums to the exact + * linear tail (rounding never invents a zero-marginal plateau). */ + if (remaining_bytes - elem_bytes >= p[n - 1].bytes) { + ULong dbytes = (ULong)(p[n - 1].bytes - p[n - 2].bytes); + ULong dcost = p[n - 1].centicycles - p[n - 2].centicycles; + /* Overflow-safe split of elem_bytes*dcost/dbytes. */ + ULong q = dcost / dbytes; + ULong r = dcost % dbytes; + ULong num = elem_bytes * r + *tail_carry; + ULong delta = elem_bytes * q + num / dbytes; + *tail_carry = num % dbytes; + return delta; + } + + /* Straddling the last calibrated point: use the endpoint values. */ + *tail_carry = 0; + lo = cost_at(c, remaining_bytes - elem_bytes); + hi = cost_at(c, remaining_bytes); + return hi > lo ? hi - lo : 0; +} diff --git a/callgrind/rep_bulk.h b/callgrind/rep_bulk.h new file mode 100644 index 0000000000..182a4971b1 --- /dev/null +++ b/callgrind/rep_bulk.h @@ -0,0 +1,87 @@ +/*--------------------------------------------------------------------*/ +/*--- REP bulk cost model: calibrated hot-copy curve. ---*/ +/*--- rep_bulk.h ---*/ +/*--------------------------------------------------------------------*/ + +/* CodSpeed extension. + * + * Replaces the per-element (per-byte) cost that the cache simulator charges + * for rep-prefixed string instructions with a calibrated bulk cost curve. + * + * The curve F is the measured cost of one rep-prefixed instruction executing + * a given number of element bytes with hot cache-resident data, in + * centi-cycles. Per executed element the model adds + * + * F(remaining_bytes) - F(remaining_bytes - elem_bytes) + * + * which telescopes to F(total_bytes) for one instruction instance and stays + * nonnegative and exact across signal interruption, because "remaining" is + * read from the guest repeat counter at every element. + * + * The table itself is generated by the calibration pipeline and included by + * rep_bulk.c as `rep_bulk_table.inc`; it defines + * clg_rep_bulk_provenance[] + * clg_rep_bulk_pts_[] + * clg_rep_bulk_curves[] + * with the types below and no typedefs or includes. + */ + +#ifndef CLG_REP_BULK_H +#define CLG_REP_BULK_H + +#include "pub_tool_basics.h" +#include "cycledecode.h" /* ClgRepBulkOp, ClgRepBulkInfo */ + +/* Direction of the string operation, from the guest DF flag at the first + * element (DF does not change while an instruction executes). */ +#define CLG_REP_DIR_FORWARD 0 +#define CLG_REP_DIR_BACKWARD 1 + +/* Alignment class, keyed ONCE per operation from its initial pointers and + * kept for the whole operation, so address changes between elements cannot + * change the curve mid-instruction: + * 0 aligned: MOVS src%64==0 && dst%64==0; STOS dst%64==0 + * 1 same_unaligned: MOVS src%64==dst%64 (nonzero); STOS dst%64==8 + * 2 other/split: everything else (conservative envelope) + */ +#define CLG_REP_ALIGN_ALIGNED 0 +#define CLG_REP_ALIGN_SAME_UNALIGN 1 +#define CLG_REP_ALIGN_OTHER 2 + +typedef struct { + unsigned char op; /* ClgRepBulkOp */ + unsigned char elem_width; /* 1, 2, 4 or 8 bytes */ + unsigned char direction; /* CLG_REP_DIR_* */ + unsigned char align_class; /* CLG_REP_ALIGN_* */ +} ClgRepBulkKey; + +typedef struct { + UInt bytes; /* cumulative element bytes, strictly ascending */ + ULong centicycles; /* total hot-resident cost, centi-cycles */ +} ClgRepBulkPoint; + +typedef struct { + ClgRepBulkKey key; + const ClgRepBulkPoint* pts; + UInt npts; +} ClgRepBulkCurve; + +/* Build the lookup index and check the table. Returns the number of table + * problems (0 == clean); the caller reports them. Safe to call once at + * startup. */ +UInt clg_rep_bulk_init(void); + +/* Marginal bulk cost of the element at `remaining_bytes` (bytes still to + * execute before this element, always a multiple of elem_bytes), in + * centi-cycles. Uses exactly the (op, width, direction, align_class) curve; + * the table must cover that key. Beyond the last calibrated point the exact + * linear tail is used, with the fractional remainder carried in + * *tail_carry (in/out; must be 0 at the start of an operation) so that the + * summed tail deltas equal the exact tail cost. Returns 0 if the key is not + * present (the caller treats that as "model disabled"). */ +ULong clg_rep_bulk_marginal(unsigned op, unsigned elem_width, + unsigned direction, unsigned align_class, + ULong remaining_bytes, ULong elem_bytes, + ULong* tail_carry); + +#endif /* CLG_REP_BULK_H */ diff --git a/callgrind/rep_bulk_table.inc b/callgrind/rep_bulk_table.inc new file mode 100644 index 0000000000..6698d8adea --- /dev/null +++ b/callgrind/rep_bulk_table.inc @@ -0,0 +1,3578 @@ +/* Generated by the off-tree REP bulk calibration pipeline -- do not edit. + * + * Schema (from callgrind/rep_bulk.h): + * ClgRepBulkPoint { unsigned bytes; unsigned long long centicycles; } + * ClgRepBulkCurve { ClgRepBulkKey key; const ClgRepBulkPoint *pts; + * unsigned npts; } + * + * F(bytes) is the cumulative hot L1-resident total cost of ONE rep-prefixed + * instruction, in centi-cycles (1 cycle = 100), from dispatch through + * completion. It excludes surrounding code and every L1/LL/RAM miss penalty. + * Per executed element (width w): F(rem*w) - F((rem-1)*w), telescoping. + * Between points the consumer uses integer-linear floor interpolation. + * Beyond the last point the consumer carry-extends the last segment slope; + * the summed tail equals that segment's exact linear extension. + * align 0 = both 64B aligned, 1 = both offset 8, 2 = other (conservative + * envelope of odd/split measurements; see provenance). + * + * raw sha256: d2c9c6d724ebbec2330ba57d1f51962b56d662f0f9c80256060b0a894c8742b4, f7739cdcdf5cc1524e3daa376415a8e6f7e60c7c116449b6aa624152ae15b95d + * Calibration: AMD Ryzen 9 9950X (Zen5), measured fenced completion cost. + * This is NOT the static Zen4 uops.info LUT. + * calibrated max: MOVS 16384 bytes, STOS 32768 bytes + */ +static const char clg_rep_bulk_provenance[] = "repl-bulk-v1 host=AMD Ryzen 9 9950X 16-Core Processor date=2026-11-02T18:37:57.359329+00:00 raw_sha256=d2c9c6d724ebbec2330ba57d1f51962b56d662f0f9c80256060b0a894c8742b4,f7739cdcdf5cc1524e3daa376415a8e6f7e60c7c116449b6aa624152ae15b95d toolchain=gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 calibrated_max=MOVS:16384,STOS:32768 units=centi-cycles(1cycle=100) regime=hot-L1-resident align2=conservative-envelope >max=carry-linear-tail heldout=64KiB..1MiB(L2/LL-evidence-only,not-fitted) domain=disjoint-buffers(overlap-not-claimed) cache-miss-penalties-excluded"; + +static const ClgRepBulkPoint clg_rep_bulk_pts_0[] = { + { 0u, 0ull }, + { 1u, 114ull }, + { 2u, 114ull }, + { 3u, 114ull }, + { 4u, 114ull }, + { 5u, 114ull }, + { 6u, 114ull }, + { 7u, 114ull }, + { 8u, 114ull }, + { 9u, 114ull }, + { 10u, 114ull }, + { 11u, 114ull }, + { 12u, 114ull }, + { 13u, 114ull }, + { 14u, 114ull }, + { 15u, 114ull }, + { 16u, 114ull }, + { 17u, 114ull }, + { 18u, 114ull }, + { 19u, 114ull }, + { 20u, 114ull }, + { 21u, 114ull }, + { 22u, 114ull }, + { 23u, 114ull }, + { 24u, 114ull }, + { 25u, 114ull }, + { 26u, 114ull }, + { 27u, 114ull }, + { 28u, 114ull }, + { 30u, 114ull }, + { 31u, 114ull }, + { 32u, 114ull }, + { 33u, 114ull }, + { 36u, 114ull }, + { 40u, 114ull }, + { 48u, 114ull }, + { 56u, 114ull }, + { 63u, 114ull }, + { 64u, 114ull }, + { 65u, 114ull }, + { 80u, 114ull }, + { 96u, 114ull }, + { 112u, 114ull }, + { 127u, 114ull }, + { 128u, 2867ull }, + { 129u, 2873ull }, + { 160u, 2873ull }, + { 192u, 2874ull }, + { 224u, 2874ull }, + { 256u, 2874ull }, + { 320u, 2874ull }, + { 384u, 2874ull }, + { 448u, 2892ull }, + { 512u, 2892ull }, + { 640u, 2892ull }, + { 768u, 3057ull }, + { 896u, 3208ull }, + { 1024u, 3508ull }, + { 1280u, 3871ull }, + { 1536u, 4213ull }, + { 1792u, 4665ull }, + { 2048u, 5077ull }, + { 2560u, 5984ull }, + { 3072u, 6789ull }, + { 3584u, 7623ull }, + { 4096u, 8395ull }, + { 5120u, 10025ull }, + { 6144u, 11679ull }, + { 7168u, 13241ull }, + { 8192u, 14886ull }, + { 10240u, 18075ull }, + { 12288u, 22206ull }, + { 14336u, 25433ull }, + { 16384u, 29327ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_1[] = { + { 0u, 0ull }, + { 1u, 72ull }, + { 2u, 72ull }, + { 3u, 72ull }, + { 4u, 72ull }, + { 5u, 72ull }, + { 6u, 72ull }, + { 7u, 72ull }, + { 8u, 72ull }, + { 9u, 72ull }, + { 10u, 72ull }, + { 11u, 72ull }, + { 12u, 72ull }, + { 13u, 72ull }, + { 14u, 72ull }, + { 15u, 72ull }, + { 16u, 72ull }, + { 17u, 72ull }, + { 18u, 72ull }, + { 19u, 72ull }, + { 20u, 72ull }, + { 21u, 72ull }, + { 22u, 72ull }, + { 23u, 72ull }, + { 24u, 72ull }, + { 25u, 72ull }, + { 26u, 72ull }, + { 27u, 72ull }, + { 28u, 72ull }, + { 30u, 72ull }, + { 31u, 72ull }, + { 32u, 72ull }, + { 33u, 72ull }, + { 36u, 72ull }, + { 40u, 72ull }, + { 48u, 72ull }, + { 56u, 72ull }, + { 63u, 72ull }, + { 64u, 72ull }, + { 65u, 91ull }, + { 80u, 91ull }, + { 96u, 91ull }, + { 112u, 91ull }, + { 127u, 91ull }, + { 128u, 3127ull }, + { 129u, 3142ull }, + { 160u, 3142ull }, + { 192u, 3237ull }, + { 224u, 3237ull }, + { 256u, 3411ull }, + { 320u, 3481ull }, + { 384u, 3707ull }, + { 448u, 3840ull }, + { 512u, 3989ull }, + { 640u, 4213ull }, + { 768u, 4403ull }, + { 896u, 4606ull }, + { 1024u, 4803ull }, + { 1280u, 5198ull }, + { 1536u, 5593ull }, + { 1792u, 5995ull }, + { 2048u, 6383ull }, + { 2560u, 6990ull }, + { 3072u, 7930ull }, + { 3584u, 8574ull }, + { 4096u, 9365ull }, + { 5120u, 10961ull }, + { 6144u, 12566ull }, + { 7168u, 14249ull }, + { 8192u, 15887ull }, + { 10240u, 18949ull }, + { 12288u, 22285ull }, + { 14336u, 25371ull }, + { 16384u, 29163ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_2[] = { + { 0u, 0ull }, + { 1u, 30ull }, + { 2u, 30ull }, + { 3u, 61ull }, + { 4u, 61ull }, + { 5u, 103ull }, + { 6u, 103ull }, + { 7u, 116ull }, + { 8u, 116ull }, + { 9u, 116ull }, + { 10u, 116ull }, + { 11u, 156ull }, + { 12u, 156ull }, + { 13u, 167ull }, + { 14u, 167ull }, + { 15u, 167ull }, + { 16u, 167ull }, + { 17u, 191ull }, + { 18u, 191ull }, + { 19u, 193ull }, + { 20u, 193ull }, + { 21u, 224ull }, + { 22u, 224ull }, + { 23u, 224ull }, + { 24u, 224ull }, + { 25u, 224ull }, + { 26u, 224ull }, + { 27u, 224ull }, + { 28u, 224ull }, + { 30u, 224ull }, + { 31u, 224ull }, + { 32u, 224ull }, + { 33u, 226ull }, + { 36u, 226ull }, + { 40u, 226ull }, + { 48u, 226ull }, + { 56u, 479ull }, + { 63u, 627ull }, + { 64u, 854ull }, + { 65u, 884ull }, + { 80u, 884ull }, + { 96u, 884ull }, + { 112u, 884ull }, + { 127u, 884ull }, + { 128u, 3105ull }, + { 129u, 3118ull }, + { 160u, 3118ull }, + { 192u, 3217ull }, + { 224u, 3217ull }, + { 256u, 3423ull }, + { 320u, 3448ull }, + { 384u, 3660ull }, + { 448u, 3827ull }, + { 512u, 3974ull }, + { 640u, 4173ull }, + { 768u, 4353ull }, + { 896u, 4568ull }, + { 1024u, 4765ull }, + { 1280u, 5161ull }, + { 1536u, 5557ull }, + { 1792u, 5956ull }, + { 2048u, 7019ull }, + { 2560u, 8346ull }, + { 3072u, 9736ull }, + { 3584u, 11398ull }, + { 4096u, 15059ull }, + { 5120u, 18234ull }, + { 6144u, 19884ull }, + { 7168u, 22561ull }, + { 8192u, 26774ull }, + { 10240u, 32316ull }, + { 12288u, 38177ull }, + { 14336u, 42978ull }, + { 16384u, 50100ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_3[] = { + { 0u, 0ull }, + { 1u, 727ull }, + { 2u, 830ull }, + { 3u, 929ull }, + { 4u, 1024ull }, + { 5u, 1123ull }, + { 6u, 1221ull }, + { 7u, 1314ull }, + { 8u, 1421ull }, + { 9u, 1517ull }, + { 10u, 1613ull }, + { 11u, 1713ull }, + { 12u, 1816ull }, + { 13u, 1912ull }, + { 14u, 2011ull }, + { 15u, 2306ull }, + { 16u, 2401ull }, + { 17u, 2508ull }, + { 18u, 2602ull }, + { 19u, 2701ull }, + { 20u, 2802ull }, + { 21u, 2903ull }, + { 22u, 2998ull }, + { 23u, 3096ull }, + { 24u, 3194ull }, + { 25u, 3296ull }, + { 26u, 3400ull }, + { 27u, 3492ull }, + { 28u, 3593ull }, + { 30u, 3791ull }, + { 31u, 3881ull }, + { 32u, 3989ull }, + { 33u, 4086ull }, + { 36u, 4384ull }, + { 40u, 4776ull }, + { 48u, 5568ull }, + { 56u, 6361ull }, + { 63u, 7071ull }, + { 64u, 7150ull }, + { 65u, 7250ull }, + { 80u, 8746ull }, + { 96u, 10336ull }, + { 112u, 11908ull }, + { 127u, 13387ull }, + { 128u, 13497ull }, + { 129u, 13588ull }, + { 160u, 16656ull }, + { 192u, 19822ull }, + { 224u, 22981ull }, + { 256u, 26133ull }, + { 320u, 33193ull }, + { 384u, 39493ull }, + { 448u, 45884ull }, + { 512u, 52220ull }, + { 640u, 65584ull }, + { 768u, 78243ull }, + { 896u, 91666ull }, + { 1024u, 104378ull }, + { 1280u, 130513ull }, + { 1536u, 156623ull }, + { 1792u, 182726ull }, + { 2048u, 208873ull }, + { 2560u, 261093ull }, + { 3072u, 313435ull }, + { 3584u, 365841ull }, + { 4096u, 418064ull }, + { 5120u, 522512ull }, + { 6144u, 626934ull }, + { 7168u, 732061ull }, + { 8192u, 835842ull }, + { 10240u, 1045605ull }, + { 12288u, 1255028ull }, + { 14336u, 1463910ull }, + { 16384u, 1673139ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_4[] = { + { 0u, 0ull }, + { 1u, 826ull }, + { 2u, 924ull }, + { 3u, 1028ull }, + { 4u, 1122ull }, + { 5u, 1227ull }, + { 6u, 1322ull }, + { 7u, 1425ull }, + { 8u, 1536ull }, + { 9u, 1620ull }, + { 10u, 1723ull }, + { 11u, 1821ull }, + { 12u, 1920ull }, + { 13u, 2023ull }, + { 14u, 2123ull }, + { 15u, 2418ull }, + { 16u, 2520ull }, + { 17u, 2624ull }, + { 18u, 2730ull }, + { 19u, 2814ull }, + { 20u, 2916ull }, + { 21u, 3023ull }, + { 22u, 3115ull }, + { 23u, 3219ull }, + { 24u, 3313ull }, + { 25u, 3411ull }, + { 26u, 3518ull }, + { 27u, 3611ull }, + { 28u, 3714ull }, + { 30u, 3915ull }, + { 31u, 4014ull }, + { 32u, 4113ull }, + { 33u, 4210ull }, + { 36u, 4508ull }, + { 40u, 4907ull }, + { 48u, 5707ull }, + { 56u, 6515ull }, + { 63u, 7223ull }, + { 64u, 7330ull }, + { 65u, 7428ull }, + { 80u, 8924ull }, + { 96u, 10519ull }, + { 112u, 12111ull }, + { 127u, 13607ull }, + { 128u, 13706ull }, + { 129u, 13823ull }, + { 160u, 16899ull }, + { 192u, 20086ull }, + { 224u, 23277ull }, + { 256u, 26470ull }, + { 320u, 33611ull }, + { 384u, 39974ull }, + { 448u, 46380ull }, + { 512u, 52755ull }, + { 640u, 66236ull }, + { 768u, 79001ull }, + { 896u, 92563ull }, + { 1024u, 105410ull }, + { 1280u, 131724ull }, + { 1536u, 158046ull }, + { 1792u, 184360ull }, + { 2048u, 210840ull }, + { 2560u, 263383ull }, + { 3072u, 316162ull }, + { 3584u, 368923ull }, + { 4096u, 421676ull }, + { 5120u, 527352ull }, + { 6144u, 632904ull }, + { 7168u, 738678ull }, + { 8192u, 843771ull }, + { 10240u, 1054438ull }, + { 12288u, 1265775ull }, + { 14336u, 1475945ull }, + { 16384u, 1687623ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_5[] = { + { 0u, 0ull }, + { 1u, 841ull }, + { 2u, 927ull }, + { 3u, 1030ull }, + { 4u, 1133ull }, + { 5u, 1246ull }, + { 6u, 1346ull }, + { 7u, 1436ull }, + { 8u, 1523ull }, + { 9u, 1625ull }, + { 10u, 1720ull }, + { 11u, 1825ull }, + { 12u, 1936ull }, + { 13u, 2042ull }, + { 14u, 2130ull }, + { 15u, 2430ull }, + { 16u, 2535ull }, + { 17u, 2625ull }, + { 18u, 2739ull }, + { 19u, 2831ull }, + { 20u, 2937ull }, + { 21u, 3032ull }, + { 22u, 3135ull }, + { 23u, 3234ull }, + { 24u, 3332ull }, + { 25u, 3434ull }, + { 26u, 3529ull }, + { 27u, 3629ull }, + { 28u, 3722ull }, + { 30u, 3929ull }, + { 31u, 4024ull }, + { 32u, 4119ull }, + { 33u, 4222ull }, + { 36u, 4526ull }, + { 40u, 4922ull }, + { 48u, 5722ull }, + { 56u, 6518ull }, + { 63u, 7215ull }, + { 64u, 7313ull }, + { 65u, 7415ull }, + { 80u, 8908ull }, + { 96u, 10505ull }, + { 112u, 12093ull }, + { 127u, 13587ull }, + { 128u, 13691ull }, + { 129u, 13789ull }, + { 160u, 16875ull }, + { 192u, 20077ull }, + { 224u, 23283ull }, + { 256u, 26469ull }, + { 320u, 33571ull }, + { 384u, 39957ull }, + { 448u, 46340ull }, + { 512u, 52719ull }, + { 640u, 66184ull }, + { 768u, 78974ull }, + { 896u, 92536ull }, + { 1024u, 105335ull }, + { 1280u, 131653ull }, + { 1536u, 157944ull }, + { 1792u, 184310ull }, + { 2048u, 210753ull }, + { 2560u, 263501ull }, + { 3072u, 316203ull }, + { 3584u, 368960ull }, + { 4096u, 421755ull }, + { 5120u, 527288ull }, + { 6144u, 632425ull }, + { 7168u, 737620ull }, + { 8192u, 843491ull }, + { 10240u, 1054228ull }, + { 12288u, 1264896ull }, + { 14336u, 1475809ull }, + { 16384u, 1686477ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_6[] = { + { 0u, 0ull }, + { 2u, 0ull }, + { 4u, 0ull }, + { 6u, 0ull }, + { 8u, 0ull }, + { 10u, 0ull }, + { 12u, 0ull }, + { 14u, 0ull }, + { 16u, 0ull }, + { 18u, 0ull }, + { 20u, 0ull }, + { 22u, 0ull }, + { 24u, 0ull }, + { 26u, 0ull }, + { 28u, 0ull }, + { 30u, 0ull }, + { 32u, 0ull }, + { 34u, 0ull }, + { 36u, 0ull }, + { 38u, 0ull }, + { 40u, 0ull }, + { 42u, 0ull }, + { 44u, 0ull }, + { 46u, 0ull }, + { 48u, 0ull }, + { 50u, 0ull }, + { 52u, 0ull }, + { 54u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 62u, 0ull }, + { 64u, 0ull }, + { 66u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 96u, 0ull }, + { 112u, 0ull }, + { 126u, 0ull }, + { 128u, 0ull }, + { 130u, 0ull }, + { 160u, 0ull }, + { 192u, 0ull }, + { 224u, 0ull }, + { 254u, 0ull }, + { 256u, 313ull }, + { 258u, 579ull }, + { 320u, 579ull }, + { 384u, 579ull }, + { 448u, 579ull }, + { 512u, 630ull }, + { 640u, 925ull }, + { 768u, 1127ull }, + { 896u, 1229ull }, + { 1024u, 1425ull }, + { 1280u, 1830ull }, + { 1536u, 2229ull }, + { 1792u, 2627ull }, + { 2048u, 3026ull }, + { 2560u, 3925ull }, + { 3072u, 4724ull }, + { 3584u, 5516ull }, + { 4096u, 6315ull }, + { 5120u, 7936ull }, + { 6144u, 9523ull }, + { 7168u, 11137ull }, + { 8192u, 12734ull }, + { 10240u, 15952ull }, + { 12288u, 19156ull }, + { 14336u, 22359ull }, + { 16384u, 26149ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_7[] = { + { 0u, 0ull }, + { 2u, 0ull }, + { 4u, 0ull }, + { 6u, 0ull }, + { 8u, 0ull }, + { 10u, 0ull }, + { 12u, 0ull }, + { 14u, 0ull }, + { 16u, 0ull }, + { 18u, 0ull }, + { 20u, 0ull }, + { 22u, 0ull }, + { 24u, 0ull }, + { 26u, 0ull }, + { 28u, 0ull }, + { 30u, 0ull }, + { 32u, 0ull }, + { 34u, 0ull }, + { 36u, 0ull }, + { 38u, 0ull }, + { 40u, 0ull }, + { 42u, 0ull }, + { 44u, 0ull }, + { 46u, 0ull }, + { 48u, 0ull }, + { 50u, 0ull }, + { 52u, 0ull }, + { 54u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 62u, 0ull }, + { 64u, 0ull }, + { 66u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 96u, 0ull }, + { 112u, 0ull }, + { 126u, 0ull }, + { 128u, 0ull }, + { 130u, 0ull }, + { 160u, 0ull }, + { 192u, 0ull }, + { 224u, 0ull }, + { 254u, 0ull }, + { 256u, 1367ull }, + { 258u, 1367ull }, + { 320u, 1472ull }, + { 384u, 1575ull }, + { 448u, 1676ull }, + { 512u, 1943ull }, + { 640u, 2141ull }, + { 768u, 2339ull }, + { 896u, 2535ull }, + { 1024u, 2738ull }, + { 1280u, 3132ull }, + { 1536u, 3527ull }, + { 1792u, 3925ull }, + { 2048u, 4321ull }, + { 2560u, 5061ull }, + { 3072u, 5859ull }, + { 3584u, 6648ull }, + { 4096u, 7444ull }, + { 5120u, 9025ull }, + { 6144u, 10632ull }, + { 7168u, 12214ull }, + { 8192u, 13822ull }, + { 10240u, 16983ull }, + { 12288u, 20221ull }, + { 14336u, 23421ull }, + { 16384u, 27141ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_8[] = { + { 0u, 0ull }, + { 2u, 1732ull }, + { 4u, 1732ull }, + { 6u, 1844ull }, + { 8u, 1844ull }, + { 10u, 1969ull }, + { 12u, 1978ull }, + { 14u, 2052ull }, + { 16u, 2052ull }, + { 18u, 2124ull }, + { 20u, 2124ull }, + { 22u, 2124ull }, + { 24u, 2124ull }, + { 26u, 2124ull }, + { 28u, 2124ull }, + { 30u, 2143ull }, + { 32u, 2143ull }, + { 34u, 2169ull }, + { 36u, 2169ull }, + { 38u, 2169ull }, + { 40u, 2169ull }, + { 42u, 2193ull }, + { 44u, 2193ull }, + { 46u, 2193ull }, + { 48u, 2193ull }, + { 50u, 2198ull }, + { 52u, 2198ull }, + { 54u, 2222ull }, + { 56u, 2388ull }, + { 60u, 2584ull }, + { 62u, 2747ull }, + { 64u, 2747ull }, + { 66u, 2834ull }, + { 72u, 2834ull }, + { 80u, 2834ull }, + { 96u, 2834ull }, + { 112u, 2834ull }, + { 126u, 2953ull }, + { 128u, 4692ull }, + { 130u, 4692ull }, + { 160u, 4693ull }, + { 192u, 4791ull }, + { 224u, 4791ull }, + { 254u, 4791ull }, + { 256u, 4890ull }, + { 258u, 4890ull }, + { 320u, 4992ull }, + { 384u, 5095ull }, + { 448u, 5188ull }, + { 512u, 5298ull }, + { 640u, 5466ull }, + { 768u, 5662ull }, + { 896u, 5857ull }, + { 1024u, 6059ull }, + { 1280u, 6455ull }, + { 1536u, 7244ull }, + { 1792u, 7697ull }, + { 2048u, 8558ull }, + { 2560u, 10329ull }, + { 3072u, 11628ull }, + { 3584u, 13365ull }, + { 4096u, 17075ull }, + { 5120u, 20183ull }, + { 6144u, 21844ull }, + { 7168u, 24518ull }, + { 8192u, 28833ull }, + { 10240u, 34030ull }, + { 12288u, 39889ull }, + { 14336u, 45240ull }, + { 16384u, 52440ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_9[] = { + { 0u, 0ull }, + { 2u, 732ull }, + { 4u, 821ull }, + { 6u, 922ull }, + { 8u, 1012ull }, + { 10u, 1112ull }, + { 12u, 1215ull }, + { 14u, 1317ull }, + { 16u, 1409ull }, + { 18u, 1522ull }, + { 20u, 1610ull }, + { 22u, 1708ull }, + { 24u, 1820ull }, + { 26u, 1911ull }, + { 28u, 2010ull }, + { 30u, 2305ull }, + { 32u, 2382ull }, + { 34u, 2507ull }, + { 36u, 2601ull }, + { 38u, 2694ull }, + { 40u, 2803ull }, + { 42u, 2906ull }, + { 44u, 3001ull }, + { 46u, 3088ull }, + { 48u, 3198ull }, + { 50u, 3288ull }, + { 52u, 3393ull }, + { 54u, 3486ull }, + { 56u, 3596ull }, + { 60u, 3782ull }, + { 62u, 3893ull }, + { 64u, 3989ull }, + { 66u, 4083ull }, + { 72u, 4385ull }, + { 80u, 4778ull }, + { 96u, 5575ull }, + { 112u, 6362ull }, + { 126u, 7052ull }, + { 128u, 7152ull }, + { 130u, 7249ull }, + { 160u, 8734ull }, + { 192u, 10310ull }, + { 224u, 11893ull }, + { 254u, 13375ull }, + { 256u, 13487ull }, + { 258u, 13572ull }, + { 320u, 16663ull }, + { 384u, 19834ull }, + { 448u, 23004ull }, + { 512u, 26184ull }, + { 640u, 33189ull }, + { 768u, 39536ull }, + { 896u, 45860ull }, + { 1024u, 52190ull }, + { 1280u, 65580ull }, + { 1536u, 78285ull }, + { 1792u, 91669ull }, + { 2048u, 104396ull }, + { 2560u, 130452ull }, + { 3072u, 156612ull }, + { 3584u, 182689ull }, + { 4096u, 209007ull }, + { 5120u, 261222ull }, + { 6144u, 313313ull }, + { 7168u, 366022ull }, + { 8192u, 418225ull }, + { 10240u, 522764ull }, + { 12288u, 627306ull }, + { 14336u, 731912ull }, + { 16384u, 836672ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_10[] = { + { 0u, 0ull }, + { 2u, 778ull }, + { 4u, 875ull }, + { 6u, 974ull }, + { 8u, 1065ull }, + { 10u, 1164ull }, + { 12u, 1277ull }, + { 14u, 1367ull }, + { 16u, 1470ull }, + { 18u, 1575ull }, + { 20u, 1676ull }, + { 22u, 1777ull }, + { 24u, 1874ull }, + { 26u, 1963ull }, + { 28u, 2071ull }, + { 30u, 2355ull }, + { 32u, 2464ull }, + { 34u, 2569ull }, + { 36u, 2668ull }, + { 38u, 2766ull }, + { 40u, 2867ull }, + { 42u, 2966ull }, + { 44u, 3050ull }, + { 46u, 3159ull }, + { 48u, 3263ull }, + { 50u, 3359ull }, + { 52u, 3463ull }, + { 54u, 3562ull }, + { 56u, 3663ull }, + { 60u, 3851ull }, + { 62u, 3967ull }, + { 64u, 4063ull }, + { 66u, 4158ull }, + { 72u, 4456ull }, + { 80u, 4854ull }, + { 96u, 5656ull }, + { 112u, 6448ull }, + { 126u, 7148ull }, + { 128u, 7246ull }, + { 130u, 7349ull }, + { 160u, 8839ull }, + { 192u, 10428ull }, + { 224u, 12020ull }, + { 254u, 13527ull }, + { 256u, 13619ull }, + { 258u, 13719ull }, + { 320u, 16812ull }, + { 384u, 20043ull }, + { 448u, 23218ull }, + { 512u, 26416ull }, + { 640u, 33483ull }, + { 768u, 39857ull }, + { 896u, 46244ull }, + { 1024u, 52645ull }, + { 1280u, 66141ull }, + { 1536u, 78945ull }, + { 1792u, 92440ull }, + { 2048u, 105223ull }, + { 2560u, 131499ull }, + { 3072u, 157828ull }, + { 3584u, 184125ull }, + { 4096u, 210630ull }, + { 5120u, 263162ull }, + { 6144u, 316196ull }, + { 7168u, 368787ull }, + { 8192u, 421330ull }, + { 10240u, 526744ull }, + { 12288u, 632106ull }, + { 14336u, 737515ull }, + { 16384u, 843202ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_11[] = { + { 0u, 0ull }, + { 2u, 810ull }, + { 4u, 896ull }, + { 6u, 1013ull }, + { 8u, 1102ull }, + { 10u, 1182ull }, + { 12u, 1294ull }, + { 14u, 1397ull }, + { 16u, 1483ull }, + { 18u, 1582ull }, + { 20u, 1702ull }, + { 22u, 1807ull }, + { 24u, 1902ull }, + { 26u, 1987ull }, + { 28u, 2091ull }, + { 30u, 2389ull }, + { 32u, 2492ull }, + { 34u, 2594ull }, + { 36u, 2699ull }, + { 38u, 2799ull }, + { 40u, 2905ull }, + { 42u, 2995ull }, + { 44u, 3088ull }, + { 46u, 3201ull }, + { 48u, 3300ull }, + { 50u, 3401ull }, + { 52u, 3497ull }, + { 54u, 3598ull }, + { 56u, 3701ull }, + { 60u, 3897ull }, + { 62u, 3999ull }, + { 64u, 4099ull }, + { 66u, 4199ull }, + { 72u, 4593ull }, + { 80u, 4990ull }, + { 96u, 5791ull }, + { 112u, 6579ull }, + { 126u, 7277ull }, + { 128u, 7379ull }, + { 130u, 7478ull }, + { 160u, 8970ull }, + { 192u, 10561ull }, + { 224u, 12149ull }, + { 254u, 13954ull }, + { 256u, 14064ull }, + { 258u, 14161ull }, + { 320u, 16975ull }, + { 384u, 20177ull }, + { 448u, 23357ull }, + { 512u, 26538ull }, + { 640u, 33625ull }, + { 768u, 40018ull }, + { 896u, 46445ull }, + { 1024u, 52822ull }, + { 1280u, 66323ull }, + { 1536u, 79169ull }, + { 1792u, 92773ull }, + { 2048u, 105590ull }, + { 2560u, 131845ull }, + { 3072u, 158380ull }, + { 3584u, 184703ull }, + { 4096u, 211038ull }, + { 5120u, 263843ull }, + { 6144u, 316516ull }, + { 7168u, 369174ull }, + { 8192u, 421870ull }, + { 10240u, 527418ull }, + { 12288u, 632697ull }, + { 14336u, 738150ull }, + { 16384u, 843726ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_12[] = { + { 0u, 0ull }, + { 4u, 0ull }, + { 8u, 0ull }, + { 12u, 0ull }, + { 16u, 0ull }, + { 20u, 0ull }, + { 24u, 0ull }, + { 28u, 0ull }, + { 32u, 0ull }, + { 36u, 0ull }, + { 40u, 0ull }, + { 44u, 0ull }, + { 48u, 0ull }, + { 52u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 64u, 0ull }, + { 68u, 0ull }, + { 72u, 0ull }, + { 76u, 0ull }, + { 80u, 0ull }, + { 84u, 0ull }, + { 88u, 0ull }, + { 92u, 0ull }, + { 96u, 0ull }, + { 100u, 0ull }, + { 104u, 0ull }, + { 108u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 124u, 0ull }, + { 128u, 0ull }, + { 132u, 0ull }, + { 144u, 0ull }, + { 160u, 0ull }, + { 192u, 0ull }, + { 224u, 0ull }, + { 252u, 0ull }, + { 256u, 302ull }, + { 260u, 533ull }, + { 320u, 533ull }, + { 384u, 533ull }, + { 448u, 533ull }, + { 508u, 814ull }, + { 512u, 814ull }, + { 516u, 1020ull }, + { 640u, 1020ull }, + { 768u, 1087ull }, + { 896u, 1211ull }, + { 1024u, 1410ull }, + { 1280u, 1808ull }, + { 1536u, 2207ull }, + { 1792u, 2590ull }, + { 2048u, 3004ull }, + { 2560u, 3899ull }, + { 3072u, 4691ull }, + { 3584u, 5488ull }, + { 4096u, 6285ull }, + { 5120u, 7898ull }, + { 6144u, 9493ull }, + { 7168u, 11094ull }, + { 8192u, 12709ull }, + { 10240u, 15938ull }, + { 12288u, 19942ull }, + { 14336u, 23143ull }, + { 16384u, 26992ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_13[] = { + { 0u, 0ull }, + { 4u, 0ull }, + { 8u, 0ull }, + { 12u, 0ull }, + { 16u, 0ull }, + { 20u, 0ull }, + { 24u, 0ull }, + { 28u, 0ull }, + { 32u, 0ull }, + { 36u, 0ull }, + { 40u, 0ull }, + { 44u, 0ull }, + { 48u, 0ull }, + { 52u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 64u, 0ull }, + { 68u, 0ull }, + { 72u, 0ull }, + { 76u, 0ull }, + { 80u, 0ull }, + { 84u, 0ull }, + { 88u, 0ull }, + { 92u, 0ull }, + { 96u, 0ull }, + { 100u, 0ull }, + { 104u, 0ull }, + { 108u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 124u, 0ull }, + { 128u, 0ull }, + { 132u, 0ull }, + { 144u, 0ull }, + { 160u, 0ull }, + { 192u, 0ull }, + { 224u, 0ull }, + { 252u, 0ull }, + { 256u, 1252ull }, + { 260u, 1265ull }, + { 320u, 1390ull }, + { 384u, 1483ull }, + { 448u, 1560ull }, + { 508u, 1560ull }, + { 512u, 1649ull }, + { 516u, 1649ull }, + { 640u, 1828ull }, + { 768u, 2033ull }, + { 896u, 2229ull }, + { 1024u, 2436ull }, + { 1280u, 2831ull }, + { 1536u, 3229ull }, + { 1792u, 3624ull }, + { 2048u, 4020ull }, + { 2560u, 4713ull }, + { 3072u, 5511ull }, + { 3584u, 6302ull }, + { 4096u, 7094ull }, + { 5120u, 8677ull }, + { 6144u, 10284ull }, + { 7168u, 11875ull }, + { 8192u, 13484ull }, + { 10240u, 16676ull }, + { 12288u, 20647ull }, + { 14336u, 23919ull }, + { 16384u, 27677ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_14[] = { + { 0u, 0ull }, + { 4u, 1738ull }, + { 8u, 1738ull }, + { 12u, 1983ull }, + { 16u, 1984ull }, + { 20u, 2132ull }, + { 24u, 2132ull }, + { 28u, 2132ull }, + { 32u, 2132ull }, + { 36u, 2132ull }, + { 40u, 2132ull }, + { 44u, 2157ull }, + { 48u, 2157ull }, + { 52u, 2182ull }, + { 56u, 2388ull }, + { 60u, 2657ull }, + { 64u, 2663ull }, + { 68u, 2881ull }, + { 72u, 2881ull }, + { 76u, 2881ull }, + { 80u, 2881ull }, + { 84u, 2899ull }, + { 88u, 2899ull }, + { 92u, 2899ull }, + { 96u, 2899ull }, + { 100u, 2899ull }, + { 104u, 2899ull }, + { 108u, 2899ull }, + { 112u, 2899ull }, + { 120u, 2899ull }, + { 124u, 2965ull }, + { 128u, 4694ull }, + { 132u, 4697ull }, + { 144u, 4700ull }, + { 160u, 4700ull }, + { 192u, 4798ull }, + { 224u, 4799ull }, + { 252u, 4799ull }, + { 256u, 4895ull }, + { 260u, 4895ull }, + { 320u, 5001ull }, + { 384u, 5104ull }, + { 448u, 5201ull }, + { 508u, 5201ull }, + { 512u, 5278ull }, + { 516u, 5278ull }, + { 640u, 5480ull }, + { 768u, 5672ull }, + { 896u, 5868ull }, + { 1024u, 6065ull }, + { 1280u, 6465ull }, + { 1536u, 7267ull }, + { 1792u, 7691ull }, + { 2048u, 8567ull }, + { 2560u, 10353ull }, + { 3072u, 11655ull }, + { 3584u, 13338ull }, + { 4096u, 17098ull }, + { 5120u, 20209ull }, + { 6144u, 21858ull }, + { 7168u, 24564ull }, + { 8192u, 28919ull }, + { 10240u, 34030ull }, + { 12288u, 39823ull }, + { 14336u, 45350ull }, + { 16384u, 52490ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_15[] = { + { 0u, 0ull }, + { 4u, 726ull }, + { 8u, 818ull }, + { 12u, 920ull }, + { 16u, 1018ull }, + { 20u, 1121ull }, + { 24u, 1212ull }, + { 28u, 1305ull }, + { 32u, 1422ull }, + { 36u, 1517ull }, + { 40u, 1596ull }, + { 44u, 1698ull }, + { 48u, 1801ull }, + { 52u, 1902ull }, + { 56u, 1997ull }, + { 60u, 2285ull }, + { 64u, 2404ull }, + { 68u, 2501ull }, + { 72u, 2600ull }, + { 76u, 2691ull }, + { 80u, 2798ull }, + { 84u, 2893ull }, + { 88u, 2993ull }, + { 92u, 3096ull }, + { 96u, 3190ull }, + { 100u, 3290ull }, + { 104u, 3381ull }, + { 108u, 3478ull }, + { 112u, 3592ull }, + { 120u, 3784ull }, + { 124u, 3875ull }, + { 128u, 3976ull }, + { 132u, 4082ull }, + { 144u, 4386ull }, + { 160u, 4779ull }, + { 192u, 5559ull }, + { 224u, 6356ull }, + { 252u, 7050ull }, + { 256u, 7136ull }, + { 260u, 7243ull }, + { 320u, 8726ull }, + { 384u, 10304ull }, + { 448u, 11883ull }, + { 508u, 13368ull }, + { 512u, 13460ull }, + { 516u, 13562ull }, + { 640u, 16646ull }, + { 768u, 19820ull }, + { 896u, 23018ull }, + { 1024u, 26194ull }, + { 1280u, 33225ull }, + { 1536u, 39548ull }, + { 1792u, 45903ull }, + { 2048u, 52212ull }, + { 2560u, 65635ull }, + { 3072u, 78323ull }, + { 3584u, 91556ull }, + { 4096u, 104341ull }, + { 5120u, 130285ull }, + { 6144u, 156388ull }, + { 7168u, 182785ull }, + { 8192u, 208775ull }, + { 10240u, 260825ull }, + { 12288u, 313386ull }, + { 14336u, 365578ull }, + { 16384u, 417636ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_16[] = { + { 0u, 0ull }, + { 4u, 748ull }, + { 8u, 833ull }, + { 12u, 944ull }, + { 16u, 1038ull }, + { 20u, 1146ull }, + { 24u, 1231ull }, + { 28u, 1335ull }, + { 32u, 1436ull }, + { 36u, 1539ull }, + { 40u, 1634ull }, + { 44u, 1736ull }, + { 48u, 1829ull }, + { 52u, 1923ull }, + { 56u, 2028ull }, + { 60u, 2322ull }, + { 64u, 2429ull }, + { 68u, 2533ull }, + { 72u, 2632ull }, + { 76u, 2734ull }, + { 80u, 2818ull }, + { 84u, 2926ull }, + { 88u, 3020ull }, + { 92u, 3114ull }, + { 96u, 3224ull }, + { 100u, 3324ull }, + { 104u, 3420ull }, + { 108u, 3527ull }, + { 112u, 3620ull }, + { 120u, 3825ull }, + { 124u, 3925ull }, + { 128u, 4015ull }, + { 132u, 4121ull }, + { 144u, 4423ull }, + { 160u, 4822ull }, + { 192u, 5615ull }, + { 224u, 6408ull }, + { 252u, 7104ull }, + { 256u, 7207ull }, + { 260u, 7302ull }, + { 320u, 8795ull }, + { 384u, 10380ull }, + { 448u, 11977ull }, + { 508u, 13466ull }, + { 512u, 13560ull }, + { 516u, 13665ull }, + { 640u, 16788ull }, + { 768u, 19975ull }, + { 896u, 23161ull }, + { 1024u, 26348ull }, + { 1280u, 33443ull }, + { 1536u, 39811ull }, + { 1792u, 46202ull }, + { 2048u, 52582ull }, + { 2560u, 66034ull }, + { 3072u, 78627ull }, + { 3584u, 92115ull }, + { 4096u, 104859ull }, + { 5120u, 131057ull }, + { 6144u, 157367ull }, + { 7168u, 183751ull }, + { 8192u, 209908ull }, + { 10240u, 262610ull }, + { 12288u, 315184ull }, + { 14336u, 367654ull }, + { 16384u, 420124ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_17[] = { + { 0u, 0ull }, + { 4u, 767ull }, + { 8u, 859ull }, + { 12u, 964ull }, + { 16u, 1062ull }, + { 20u, 1159ull }, + { 24u, 1249ull }, + { 28u, 1348ull }, + { 32u, 1457ull }, + { 36u, 1553ull }, + { 40u, 1650ull }, + { 44u, 1754ull }, + { 48u, 1849ull }, + { 52u, 1951ull }, + { 56u, 2041ull }, + { 60u, 2342ull }, + { 64u, 2436ull }, + { 68u, 2544ull }, + { 72u, 2737ull }, + { 76u, 2833ull }, + { 80u, 2929ull }, + { 84u, 3029ull }, + { 88u, 3127ull }, + { 92u, 3229ull }, + { 96u, 3323ull }, + { 100u, 3429ull }, + { 104u, 3527ull }, + { 108u, 3633ull }, + { 112u, 3731ull }, + { 120u, 3930ull }, + { 124u, 4028ull }, + { 128u, 4124ull }, + { 132u, 4223ull }, + { 144u, 4518ull }, + { 160u, 4916ull }, + { 192u, 5711ull }, + { 224u, 6807ull }, + { 252u, 7262ull }, + { 256u, 7305ull }, + { 260u, 7402ull }, + { 320u, 8891ull }, + { 384u, 10472ull }, + { 448u, 12096ull }, + { 508u, 13585ull }, + { 512u, 13673ull }, + { 516u, 13783ull }, + { 640u, 17150ull }, + { 768u, 20050ull }, + { 896u, 23255ull }, + { 1024u, 26439ull }, + { 1280u, 33495ull }, + { 1536u, 39908ull }, + { 1792u, 46322ull }, + { 2048u, 52723ull }, + { 2560u, 66198ull }, + { 3072u, 79011ull }, + { 3584u, 92516ull }, + { 4096u, 105379ull }, + { 5120u, 131520ull }, + { 6144u, 157903ull }, + { 7168u, 184214ull }, + { 8192u, 210393ull }, + { 10240u, 263373ull }, + { 12288u, 315578ull }, + { 14336u, 368123ull }, + { 16384u, 420627ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_18[] = { + { 0u, 0ull }, + { 8u, 0ull }, + { 16u, 0ull }, + { 24u, 0ull }, + { 32u, 0ull }, + { 40u, 0ull }, + { 48u, 0ull }, + { 56u, 0ull }, + { 64u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 88u, 0ull }, + { 96u, 0ull }, + { 104u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 128u, 0ull }, + { 136u, 0ull }, + { 144u, 0ull }, + { 152u, 0ull }, + { 160u, 0ull }, + { 168u, 0ull }, + { 176u, 0ull }, + { 184u, 0ull }, + { 192u, 0ull }, + { 200u, 0ull }, + { 208u, 0ull }, + { 216u, 0ull }, + { 224u, 0ull }, + { 240u, 0ull }, + { 248u, 0ull }, + { 256u, 216ull }, + { 264u, 512ull }, + { 288u, 512ull }, + { 320u, 512ull }, + { 384u, 512ull }, + { 448u, 512ull }, + { 504u, 739ull }, + { 512u, 739ull }, + { 520u, 912ull }, + { 640u, 912ull }, + { 768u, 985ull }, + { 896u, 1184ull }, + { 1016u, 1557ull }, + { 1024u, 1557ull }, + { 1032u, 1781ull }, + { 1280u, 1781ull }, + { 1536u, 2186ull }, + { 1792u, 2581ull }, + { 2048u, 2978ull }, + { 2560u, 3878ull }, + { 3072u, 4676ull }, + { 3584u, 5493ull }, + { 4096u, 6288ull }, + { 5120u, 7886ull }, + { 6144u, 9475ull }, + { 7168u, 11083ull }, + { 8192u, 12710ull }, + { 10240u, 15912ull }, + { 12288u, 19973ull }, + { 14336u, 23175ull }, + { 16384u, 26977ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_19[] = { + { 0u, 0ull }, + { 8u, 0ull }, + { 16u, 0ull }, + { 24u, 0ull }, + { 32u, 0ull }, + { 40u, 0ull }, + { 48u, 0ull }, + { 56u, 0ull }, + { 64u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 88u, 0ull }, + { 96u, 0ull }, + { 104u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 128u, 0ull }, + { 136u, 0ull }, + { 144u, 0ull }, + { 152u, 0ull }, + { 160u, 0ull }, + { 168u, 0ull }, + { 176u, 0ull }, + { 184u, 0ull }, + { 192u, 0ull }, + { 200u, 0ull }, + { 208u, 0ull }, + { 216u, 0ull }, + { 224u, 0ull }, + { 240u, 0ull }, + { 248u, 0ull }, + { 256u, 1156ull }, + { 264u, 1167ull }, + { 288u, 1167ull }, + { 320u, 1266ull }, + { 384u, 1312ull }, + { 448u, 1312ull }, + { 504u, 1312ull }, + { 512u, 1547ull }, + { 520u, 1547ull }, + { 640u, 1750ull }, + { 768u, 1946ull }, + { 896u, 2013ull }, + { 1016u, 2013ull }, + { 1024u, 2342ull }, + { 1032u, 2342ull }, + { 1280u, 2740ull }, + { 1536u, 3132ull }, + { 1792u, 3531ull }, + { 2048u, 3927ull }, + { 2560u, 4622ull }, + { 3072u, 5415ull }, + { 3584u, 6224ull }, + { 4096u, 7019ull }, + { 5120u, 8613ull }, + { 6144u, 10202ull }, + { 7168u, 11802ull }, + { 8192u, 13406ull }, + { 10240u, 16577ull }, + { 12288u, 20634ull }, + { 14336u, 23818ull }, + { 16384u, 27572ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_20[] = { + { 0u, 0ull }, + { 8u, 1737ull }, + { 16u, 1973ull }, + { 24u, 2122ull }, + { 32u, 2122ull }, + { 40u, 2122ull }, + { 48u, 2157ull }, + { 56u, 2393ull }, + { 64u, 2666ull }, + { 72u, 2738ull }, + { 80u, 2738ull }, + { 88u, 2781ull }, + { 96u, 2781ull }, + { 104u, 2781ull }, + { 112u, 2781ull }, + { 120u, 2781ull }, + { 128u, 4682ull }, + { 136u, 4686ull }, + { 144u, 4686ull }, + { 152u, 4686ull }, + { 160u, 4686ull }, + { 168u, 4686ull }, + { 176u, 4686ull }, + { 184u, 4686ull }, + { 192u, 4787ull }, + { 200u, 4787ull }, + { 208u, 4787ull }, + { 216u, 4787ull }, + { 224u, 4787ull }, + { 240u, 4787ull }, + { 248u, 4787ull }, + { 256u, 4886ull }, + { 264u, 4887ull }, + { 288u, 4890ull }, + { 320u, 4977ull }, + { 384u, 5092ull }, + { 448u, 5178ull }, + { 504u, 5197ull }, + { 512u, 5292ull }, + { 520u, 5292ull }, + { 640u, 5469ull }, + { 768u, 5666ull }, + { 896u, 5864ull }, + { 1016u, 6069ull }, + { 1024u, 6071ull }, + { 1032u, 6267ull }, + { 1280u, 6466ull }, + { 1536u, 7262ull }, + { 1792u, 7708ull }, + { 2048u, 8561ull }, + { 2560u, 10370ull }, + { 3072u, 11677ull }, + { 3584u, 13558ull }, + { 4096u, 17111ull }, + { 5120u, 20210ull }, + { 6144u, 21871ull }, + { 7168u, 24647ull }, + { 8192u, 28896ull }, + { 10240u, 33998ull }, + { 12288u, 39815ull }, + { 14336u, 45423ull }, + { 16384u, 52662ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_21[] = { + { 0u, 0ull }, + { 8u, 715ull }, + { 16u, 811ull }, + { 24u, 906ull }, + { 32u, 1016ull }, + { 40u, 1102ull }, + { 48u, 1206ull }, + { 56u, 1308ull }, + { 64u, 1401ull }, + { 72u, 1496ull }, + { 80u, 1596ull }, + { 88u, 1701ull }, + { 96u, 1799ull }, + { 104u, 1897ull }, + { 112u, 2003ull }, + { 120u, 2300ull }, + { 128u, 2399ull }, + { 136u, 2485ull }, + { 144u, 2583ull }, + { 152u, 2686ull }, + { 160u, 2785ull }, + { 168u, 2888ull }, + { 176u, 2977ull }, + { 184u, 3082ull }, + { 192u, 3183ull }, + { 200u, 3274ull }, + { 208u, 3380ull }, + { 216u, 3480ull }, + { 224u, 3583ull }, + { 240u, 3776ull }, + { 248u, 3881ull }, + { 256u, 3981ull }, + { 264u, 4072ull }, + { 288u, 4375ull }, + { 320u, 4767ull }, + { 384u, 5557ull }, + { 448u, 6355ull }, + { 504u, 7045ull }, + { 512u, 7130ull }, + { 520u, 7240ull }, + { 640u, 8727ull }, + { 768u, 10308ull }, + { 896u, 11896ull }, + { 1016u, 13385ull }, + { 1024u, 13490ull }, + { 1032u, 13584ull }, + { 1280u, 16626ull }, + { 1536u, 19793ull }, + { 1792u, 22950ull }, + { 2048u, 26106ull }, + { 2560u, 33122ull }, + { 3072u, 39432ull }, + { 3584u, 45771ull }, + { 4096u, 52108ull }, + { 5120u, 65473ull }, + { 6144u, 78210ull }, + { 7168u, 91591ull }, + { 8192u, 104269ull }, + { 10240u, 130329ull }, + { 12288u, 156358ull }, + { 14336u, 182669ull }, + { 16384u, 208560ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_22[] = { + { 0u, 0ull }, + { 8u, 679ull }, + { 16u, 773ull }, + { 24u, 871ull }, + { 32u, 978ull }, + { 40u, 1074ull }, + { 48u, 1166ull }, + { 56u, 1273ull }, + { 64u, 1372ull }, + { 72u, 1467ull }, + { 80u, 1561ull }, + { 88u, 1663ull }, + { 96u, 1762ull }, + { 104u, 1863ull }, + { 112u, 1961ull }, + { 120u, 2255ull }, + { 128u, 2364ull }, + { 136u, 2451ull }, + { 144u, 2549ull }, + { 152u, 2651ull }, + { 160u, 2750ull }, + { 168u, 2851ull }, + { 176u, 2942ull }, + { 184u, 3046ull }, + { 192u, 3156ull }, + { 200u, 3248ull }, + { 208u, 3347ull }, + { 216u, 3447ull }, + { 224u, 3536ull }, + { 240u, 3735ull }, + { 248u, 3837ull }, + { 256u, 3936ull }, + { 264u, 4044ull }, + { 288u, 4339ull }, + { 320u, 4735ull }, + { 384u, 5515ull }, + { 448u, 6312ull }, + { 504u, 7014ull }, + { 512u, 7108ull }, + { 520u, 7211ull }, + { 640u, 8695ull }, + { 768u, 10282ull }, + { 896u, 11870ull }, + { 1016u, 13363ull }, + { 1024u, 13464ull }, + { 1032u, 13564ull }, + { 1280u, 16585ull }, + { 1536u, 19786ull }, + { 1792u, 22956ull }, + { 2048u, 26117ull }, + { 2560u, 33137ull }, + { 3072u, 39474ull }, + { 3584u, 45843ull }, + { 4096u, 52158ull }, + { 5120u, 65529ull }, + { 6144u, 78330ull }, + { 7168u, 91752ull }, + { 8192u, 104442ull }, + { 10240u, 130501ull }, + { 12288u, 156833ull }, + { 14336u, 182927ull }, + { 16384u, 208969ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_23[] = { + { 0u, 0ull }, + { 8u, 689ull }, + { 16u, 785ull }, + { 24u, 890ull }, + { 32u, 985ull }, + { 40u, 1087ull }, + { 48u, 1173ull }, + { 56u, 1283ull }, + { 64u, 1384ull }, + { 72u, 1487ull }, + { 80u, 1584ull }, + { 88u, 1679ull }, + { 96u, 1775ull }, + { 104u, 1877ull }, + { 112u, 1963ull }, + { 120u, 2358ull }, + { 128u, 2372ull }, + { 136u, 2471ull }, + { 144u, 2573ull }, + { 152u, 2670ull }, + { 160u, 2764ull }, + { 168u, 2867ull }, + { 176u, 2967ull }, + { 184u, 3063ull }, + { 192u, 3164ull }, + { 200u, 3262ull }, + { 208u, 3362ull }, + { 216u, 3461ull }, + { 224u, 3557ull }, + { 240u, 3759ull }, + { 248u, 3862ull }, + { 256u, 3958ull }, + { 264u, 4065ull }, + { 288u, 4356ull }, + { 320u, 4750ull }, + { 384u, 5534ull }, + { 448u, 6327ull }, + { 504u, 7024ull }, + { 512u, 7123ull }, + { 520u, 7231ull }, + { 640u, 8719ull }, + { 768u, 10311ull }, + { 896u, 11905ull }, + { 1016u, 13395ull }, + { 1024u, 13524ull }, + { 1032u, 13640ull }, + { 1280u, 16645ull }, + { 1536u, 19812ull }, + { 1792u, 22976ull }, + { 2048u, 26150ull }, + { 2560u, 33214ull }, + { 3072u, 39626ull }, + { 3584u, 45992ull }, + { 4096u, 52436ull }, + { 5120u, 65866ull }, + { 6144u, 78694ull }, + { 7168u, 92102ull }, + { 8192u, 104965ull }, + { 10240u, 131162ull }, + { 12288u, 157398ull }, + { 14336u, 183671ull }, + { 16384u, 209937ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_24[] = { + { 0u, 0ull }, + { 1u, 4ull }, + { 2u, 4ull }, + { 3u, 4ull }, + { 4u, 4ull }, + { 5u, 4ull }, + { 6u, 4ull }, + { 7u, 4ull }, + { 8u, 4ull }, + { 9u, 4ull }, + { 10u, 4ull }, + { 11u, 4ull }, + { 12u, 4ull }, + { 13u, 4ull }, + { 14u, 4ull }, + { 15u, 4ull }, + { 16u, 4ull }, + { 17u, 4ull }, + { 18u, 4ull }, + { 19u, 4ull }, + { 20u, 4ull }, + { 21u, 4ull }, + { 22u, 4ull }, + { 23u, 4ull }, + { 24u, 4ull }, + { 25u, 4ull }, + { 26u, 4ull }, + { 27u, 4ull }, + { 28u, 4ull }, + { 30u, 4ull }, + { 31u, 4ull }, + { 32u, 4ull }, + { 33u, 4ull }, + { 36u, 4ull }, + { 40u, 4ull }, + { 48u, 4ull }, + { 56u, 4ull }, + { 63u, 4ull }, + { 64u, 4ull }, + { 65u, 4ull }, + { 80u, 4ull }, + { 96u, 4ull }, + { 112u, 4ull }, + { 127u, 4ull }, + { 128u, 2950ull }, + { 129u, 2950ull }, + { 160u, 2953ull }, + { 192u, 2953ull }, + { 224u, 2953ull }, + { 256u, 2953ull }, + { 320u, 2953ull }, + { 384u, 2953ull }, + { 448u, 2953ull }, + { 512u, 2953ull }, + { 640u, 2953ull }, + { 768u, 3145ull }, + { 896u, 3340ull }, + { 1024u, 3543ull }, + { 1280u, 4138ull }, + { 1536u, 4333ull }, + { 1792u, 4734ull }, + { 2048u, 5018ull }, + { 2560u, 6010ull }, + { 3072u, 6999ull }, + { 3584u, 7594ull }, + { 4096u, 8603ull }, + { 5120u, 10089ull }, + { 6144u, 11782ull }, + { 7168u, 13344ull }, + { 8192u, 14870ull }, + { 10240u, 18033ull }, + { 12288u, 22045ull }, + { 14336u, 25258ull }, + { 16384u, 28098ull }, + { 20480u, 34442ull }, + { 24576u, 40800ull }, + { 28672u, 47153ull }, + { 32768u, 54326ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_25[] = { + { 0u, 0ull }, + { 1u, 0ull }, + { 2u, 0ull }, + { 3u, 0ull }, + { 4u, 0ull }, + { 5u, 0ull }, + { 6u, 0ull }, + { 7u, 0ull }, + { 8u, 0ull }, + { 9u, 0ull }, + { 10u, 0ull }, + { 11u, 0ull }, + { 12u, 0ull }, + { 13u, 0ull }, + { 14u, 0ull }, + { 15u, 0ull }, + { 16u, 0ull }, + { 17u, 0ull }, + { 18u, 0ull }, + { 19u, 0ull }, + { 20u, 0ull }, + { 21u, 0ull }, + { 22u, 0ull }, + { 23u, 0ull }, + { 24u, 0ull }, + { 25u, 0ull }, + { 26u, 0ull }, + { 27u, 0ull }, + { 28u, 0ull }, + { 30u, 0ull }, + { 31u, 0ull }, + { 32u, 0ull }, + { 33u, 0ull }, + { 36u, 0ull }, + { 40u, 0ull }, + { 48u, 0ull }, + { 56u, 0ull }, + { 63u, 0ull }, + { 64u, 0ull }, + { 65u, 0ull }, + { 80u, 0ull }, + { 96u, 0ull }, + { 112u, 0ull }, + { 127u, 0ull }, + { 128u, 3067ull }, + { 129u, 3067ull }, + { 160u, 3067ull }, + { 192u, 3159ull }, + { 224u, 3166ull }, + { 256u, 3359ull }, + { 320u, 3459ull }, + { 384u, 3459ull }, + { 448u, 3560ull }, + { 512u, 3560ull }, + { 640u, 3754ull }, + { 768u, 4348ull }, + { 896u, 4446ull }, + { 1024u, 4542ull }, + { 1280u, 4943ull }, + { 1536u, 5129ull }, + { 1792u, 5926ull }, + { 2048u, 6121ull }, + { 2560u, 6834ull }, + { 3072u, 7820ull }, + { 3584u, 8409ull }, + { 4096u, 9301ull }, + { 5120u, 11012ull }, + { 6144u, 12582ull }, + { 7168u, 14170ull }, + { 8192u, 15766ull }, + { 10240u, 18946ull }, + { 12288u, 23006ull }, + { 14336u, 26185ull }, + { 16384u, 28973ull }, + { 20480u, 35343ull }, + { 24576u, 41703ull }, + { 28672u, 48051ull }, + { 32768u, 54996ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_26[] = { + { 0u, 0ull }, + { 1u, 0ull }, + { 2u, 0ull }, + { 3u, 0ull }, + { 4u, 0ull }, + { 5u, 0ull }, + { 6u, 0ull }, + { 7u, 0ull }, + { 8u, 0ull }, + { 9u, 0ull }, + { 10u, 0ull }, + { 11u, 0ull }, + { 12u, 0ull }, + { 13u, 2ull }, + { 14u, 2ull }, + { 15u, 2ull }, + { 16u, 2ull }, + { 17u, 2ull }, + { 18u, 2ull }, + { 19u, 2ull }, + { 20u, 2ull }, + { 21u, 2ull }, + { 22u, 2ull }, + { 23u, 2ull }, + { 24u, 2ull }, + { 25u, 2ull }, + { 26u, 2ull }, + { 27u, 2ull }, + { 28u, 2ull }, + { 30u, 2ull }, + { 31u, 2ull }, + { 32u, 2ull }, + { 33u, 2ull }, + { 36u, 3ull }, + { 40u, 3ull }, + { 48u, 7ull }, + { 56u, 7ull }, + { 63u, 7ull }, + { 64u, 7ull }, + { 65u, 29ull }, + { 80u, 29ull }, + { 96u, 29ull }, + { 112u, 29ull }, + { 127u, 29ull }, + { 128u, 3018ull }, + { 129u, 3018ull }, + { 160u, 3018ull }, + { 192u, 3117ull }, + { 224u, 3120ull }, + { 256u, 3386ull }, + { 320u, 3477ull }, + { 384u, 3477ull }, + { 448u, 3567ull }, + { 512u, 3567ull }, + { 640u, 3765ull }, + { 768u, 4359ull }, + { 896u, 4460ull }, + { 1024u, 4561ull }, + { 1280u, 4957ull }, + { 1536u, 5157ull }, + { 1792u, 5948ull }, + { 2048u, 6150ull }, + { 2560u, 6833ull }, + { 3072u, 7839ull }, + { 3584u, 8439ull }, + { 4096u, 9333ull }, + { 5120u, 11020ull }, + { 6144u, 12614ull }, + { 7168u, 14199ull }, + { 8192u, 15787ull }, + { 10240u, 18941ull }, + { 12288u, 22988ull }, + { 14336u, 26171ull }, + { 16384u, 28869ull }, + { 20480u, 35225ull }, + { 24576u, 41588ull }, + { 28672u, 47952ull }, + { 32768u, 55011ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_27[] = { + { 0u, 0ull }, + { 1u, 494ull }, + { 2u, 594ull }, + { 3u, 704ull }, + { 4u, 793ull }, + { 5u, 885ull }, + { 6u, 983ull }, + { 7u, 1087ull }, + { 8u, 1191ull }, + { 9u, 1286ull }, + { 10u, 1383ull }, + { 11u, 1562ull }, + { 12u, 1645ull }, + { 13u, 1770ull }, + { 14u, 1773ull }, + { 15u, 2096ull }, + { 16u, 2179ull }, + { 17u, 2290ull }, + { 18u, 2379ull }, + { 19u, 2480ull }, + { 20u, 2577ull }, + { 21u, 2677ull }, + { 22u, 2776ull }, + { 23u, 2864ull }, + { 24u, 2968ull }, + { 25u, 3073ull }, + { 26u, 3169ull }, + { 27u, 3293ull }, + { 28u, 3422ull }, + { 30u, 3611ull }, + { 31u, 3675ull }, + { 32u, 3766ull }, + { 33u, 3867ull }, + { 36u, 4162ull }, + { 40u, 4552ull }, + { 48u, 5353ull }, + { 56u, 6187ull }, + { 63u, 6857ull }, + { 64u, 6941ull }, + { 65u, 7043ull }, + { 80u, 8532ull }, + { 96u, 10113ull }, + { 112u, 11698ull }, + { 127u, 13200ull }, + { 128u, 13290ull }, + { 129u, 13390ull }, + { 160u, 16462ull }, + { 192u, 19636ull }, + { 224u, 22811ull }, + { 256u, 25990ull }, + { 320u, 32567ull }, + { 384u, 38916ull }, + { 448u, 45258ull }, + { 512u, 51614ull }, + { 640u, 64523ull }, + { 768u, 77246ull }, + { 896u, 90159ull }, + { 1024u, 102917ull }, + { 1280u, 128451ull }, + { 1536u, 154130ull }, + { 1792u, 179734ull }, + { 2048u, 205335ull }, + { 2560u, 256718ull }, + { 3072u, 308040ull }, + { 3584u, 359265ull }, + { 4096u, 410463ull }, + { 5120u, 512907ull }, + { 6144u, 615663ull }, + { 7168u, 717890ull }, + { 8192u, 820600ull }, + { 10240u, 1025324ull }, + { 12288u, 1230594ull }, + { 14336u, 1435938ull }, + { 16384u, 1640760ull }, + { 20480u, 2050838ull }, + { 24576u, 2460905ull }, + { 28672u, 2871541ull }, + { 32768u, 3282176ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_28[] = { + { 0u, 0ull }, + { 1u, 700ull }, + { 2u, 719ull }, + { 3u, 822ull }, + { 4u, 921ull }, + { 5u, 1025ull }, + { 6u, 1125ull }, + { 7u, 1220ull }, + { 8u, 1328ull }, + { 9u, 1424ull }, + { 10u, 1580ull }, + { 11u, 1598ull }, + { 12u, 1726ull }, + { 13u, 1899ull }, + { 14u, 1982ull }, + { 15u, 2211ull }, + { 16u, 2312ull }, + { 17u, 2417ull }, + { 18u, 2515ull }, + { 19u, 2598ull }, + { 20u, 2712ull }, + { 21u, 2815ull }, + { 22u, 2912ull }, + { 23u, 3013ull }, + { 24u, 3181ull }, + { 25u, 3253ull }, + { 26u, 3328ull }, + { 27u, 3417ull }, + { 28u, 3511ull }, + { 30u, 3698ull }, + { 31u, 3805ull }, + { 32u, 3910ull }, + { 33u, 4009ull }, + { 36u, 4310ull }, + { 40u, 4730ull }, + { 48u, 5521ull }, + { 56u, 6319ull }, + { 63u, 7017ull }, + { 64u, 7114ull }, + { 65u, 7214ull }, + { 80u, 8718ull }, + { 96u, 10318ull }, + { 112u, 11914ull }, + { 127u, 13421ull }, + { 128u, 13509ull }, + { 129u, 13610ull }, + { 160u, 16709ull }, + { 192u, 19912ull }, + { 224u, 23100ull }, + { 256u, 26309ull }, + { 320u, 32911ull }, + { 384u, 39364ull }, + { 448u, 45755ull }, + { 512u, 52148ull }, + { 640u, 65184ull }, + { 768u, 77950ull }, + { 896u, 91006ull }, + { 1024u, 103877ull }, + { 1280u, 129609ull }, + { 1536u, 155411ull }, + { 1792u, 181279ull }, + { 2048u, 207130ull }, + { 2560u, 258753ull }, + { 3072u, 310496ull }, + { 3584u, 362054ull }, + { 4096u, 413834ull }, + { 5120u, 516956ull }, + { 6144u, 620303ull }, + { 7168u, 723538ull }, + { 8192u, 826815ull }, + { 10240u, 1033373ull }, + { 12288u, 1240042ull }, + { 14336u, 1447746ull }, + { 16384u, 1653940ull }, + { 20480u, 2067008ull }, + { 24576u, 2480120ull }, + { 28672u, 2893866ull }, + { 32768u, 3309228ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_29[] = { + { 0u, 0ull }, + { 1u, 638ull }, + { 2u, 722ull }, + { 3u, 829ull }, + { 4u, 932ull }, + { 5u, 1028ull }, + { 6u, 1133ull }, + { 7u, 1232ull }, + { 8u, 1329ull }, + { 9u, 1430ull }, + { 10u, 1588ull }, + { 11u, 1691ull }, + { 12u, 1760ull }, + { 13u, 1900ull }, + { 14u, 1963ull }, + { 15u, 2221ull }, + { 16u, 2320ull }, + { 17u, 2422ull }, + { 18u, 2514ull }, + { 19u, 2616ull }, + { 20u, 2729ull }, + { 21u, 2832ull }, + { 22u, 2917ull }, + { 23u, 3014ull }, + { 24u, 3113ull }, + { 25u, 3256ull }, + { 26u, 3319ull }, + { 27u, 3495ull }, + { 28u, 3588ull }, + { 30u, 3726ull }, + { 31u, 3868ull }, + { 32u, 3924ull }, + { 33u, 4012ull }, + { 36u, 4329ull }, + { 40u, 4740ull }, + { 48u, 5521ull }, + { 56u, 6334ull }, + { 63u, 7080ull }, + { 64u, 7126ull }, + { 65u, 7229ull }, + { 80u, 8724ull }, + { 96u, 10319ull }, + { 112u, 11917ull }, + { 127u, 13434ull }, + { 128u, 13522ull }, + { 129u, 13621ull }, + { 160u, 16723ull }, + { 192u, 19923ull }, + { 224u, 23119ull }, + { 256u, 26312ull }, + { 320u, 32942ull }, + { 384u, 39359ull }, + { 448u, 45736ull }, + { 512u, 52151ull }, + { 640u, 65198ull }, + { 768u, 78018ull }, + { 896u, 91034ull }, + { 1024u, 103891ull }, + { 1280u, 129681ull }, + { 1536u, 155455ull }, + { 1792u, 181338ull }, + { 2048u, 207198ull }, + { 2560u, 258802ull }, + { 3072u, 310501ull }, + { 3584u, 362140ull }, + { 4096u, 413852ull }, + { 5120u, 517080ull }, + { 6144u, 620316ull }, + { 7168u, 723796ull }, + { 8192u, 827360ull }, + { 10240u, 1033751ull }, + { 12288u, 1240653ull }, + { 14336u, 1448103ull }, + { 16384u, 1654649ull }, + { 20480u, 2067751ull }, + { 24576u, 2480901ull }, + { 28672u, 2895256ull }, + { 32768u, 3309150ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_30[] = { + { 0u, 0ull }, + { 2u, 0ull }, + { 4u, 0ull }, + { 6u, 0ull }, + { 8u, 0ull }, + { 10u, 0ull }, + { 12u, 0ull }, + { 14u, 0ull }, + { 16u, 0ull }, + { 18u, 0ull }, + { 20u, 0ull }, + { 22u, 0ull }, + { 24u, 0ull }, + { 26u, 0ull }, + { 28u, 0ull }, + { 30u, 0ull }, + { 32u, 0ull }, + { 34u, 0ull }, + { 36u, 0ull }, + { 38u, 0ull }, + { 40u, 0ull }, + { 42u, 0ull }, + { 44u, 0ull }, + { 46u, 0ull }, + { 48u, 0ull }, + { 50u, 0ull }, + { 52u, 0ull }, + { 54u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 62u, 0ull }, + { 64u, 0ull }, + { 66u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 96u, 0ull }, + { 112u, 0ull }, + { 126u, 0ull }, + { 128u, 881ull }, + { 130u, 1060ull }, + { 160u, 1060ull }, + { 192u, 1060ull }, + { 224u, 1130ull }, + { 254u, 1130ull }, + { 256u, 1130ull }, + { 258u, 1202ull }, + { 320u, 1202ull }, + { 384u, 1202ull }, + { 448u, 1202ull }, + { 512u, 1202ull }, + { 640u, 1279ull }, + { 768u, 1580ull }, + { 896u, 1679ull }, + { 1024u, 1879ull }, + { 1280u, 2572ull }, + { 1536u, 2780ull }, + { 1792u, 3131ull }, + { 2048u, 3483ull }, + { 2560u, 4482ull }, + { 3072u, 5483ull }, + { 3584u, 6083ull }, + { 4096u, 6968ull }, + { 5120u, 8588ull }, + { 6144u, 10196ull }, + { 7168u, 11799ull }, + { 8192u, 13398ull }, + { 10240u, 16600ull }, + { 12288u, 20691ull }, + { 14336u, 23916ull }, + { 16384u, 26822ull }, + { 20480u, 33232ull }, + { 24576u, 39672ull }, + { 28672u, 46123ull }, + { 32768u, 53252ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_31[] = { + { 0u, 0ull }, + { 2u, 0ull }, + { 4u, 0ull }, + { 6u, 0ull }, + { 8u, 0ull }, + { 10u, 0ull }, + { 12u, 0ull }, + { 14u, 0ull }, + { 16u, 0ull }, + { 18u, 0ull }, + { 20u, 0ull }, + { 22u, 0ull }, + { 24u, 0ull }, + { 26u, 0ull }, + { 28u, 0ull }, + { 30u, 0ull }, + { 32u, 0ull }, + { 34u, 0ull }, + { 36u, 0ull }, + { 38u, 0ull }, + { 40u, 0ull }, + { 42u, 0ull }, + { 44u, 0ull }, + { 46u, 0ull }, + { 48u, 0ull }, + { 50u, 0ull }, + { 52u, 0ull }, + { 54u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 62u, 0ull }, + { 64u, 0ull }, + { 66u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 96u, 0ull }, + { 112u, 0ull }, + { 126u, 0ull }, + { 128u, 1484ull }, + { 130u, 1491ull }, + { 160u, 1493ull }, + { 192u, 1591ull }, + { 224u, 1592ull }, + { 254u, 1593ull }, + { 256u, 1790ull }, + { 258u, 1791ull }, + { 320u, 1893ull }, + { 384u, 1899ull }, + { 448u, 1970ull }, + { 512u, 1971ull }, + { 640u, 2170ull }, + { 768u, 2764ull }, + { 896u, 2856ull }, + { 1024u, 2964ull }, + { 1280u, 3360ull }, + { 1536u, 3659ull }, + { 1792u, 4354ull }, + { 2048u, 4556ull }, + { 2560u, 5234ull }, + { 3072u, 6245ull }, + { 3584u, 6942ull }, + { 4096u, 7735ull }, + { 5120u, 9475ull }, + { 6144u, 11019ull }, + { 7168u, 12615ull }, + { 8192u, 14241ull }, + { 10240u, 17372ull }, + { 12288u, 21509ull }, + { 14336u, 24677ull }, + { 16384u, 27415ull }, + { 20480u, 33825ull }, + { 24576u, 40190ull }, + { 28672u, 46555ull }, + { 32768u, 53449ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_32[] = { + { 0u, 0ull }, + { 2u, 0ull }, + { 4u, 0ull }, + { 6u, 0ull }, + { 8u, 0ull }, + { 10u, 0ull }, + { 12u, 0ull }, + { 14u, 0ull }, + { 16u, 0ull }, + { 18u, 0ull }, + { 20u, 0ull }, + { 22u, 0ull }, + { 24u, 0ull }, + { 26u, 0ull }, + { 28u, 0ull }, + { 30u, 0ull }, + { 32u, 0ull }, + { 34u, 0ull }, + { 36u, 0ull }, + { 38u, 0ull }, + { 40u, 0ull }, + { 42u, 0ull }, + { 44u, 0ull }, + { 46u, 0ull }, + { 48u, 0ull }, + { 50u, 0ull }, + { 52u, 0ull }, + { 54u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 62u, 0ull }, + { 64u, 0ull }, + { 66u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 96u, 0ull }, + { 112u, 0ull }, + { 126u, 0ull }, + { 128u, 1410ull }, + { 130u, 1410ull }, + { 160u, 1410ull }, + { 192u, 1477ull }, + { 224u, 1477ull }, + { 254u, 1577ull }, + { 256u, 1775ull }, + { 258u, 1775ull }, + { 320u, 1864ull }, + { 384u, 1864ull }, + { 448u, 1956ull }, + { 512u, 1956ull }, + { 640u, 2202ull }, + { 768u, 2750ull }, + { 896u, 2846ull }, + { 1024u, 2943ull }, + { 1280u, 3332ull }, + { 1536u, 3641ull }, + { 1792u, 4339ull }, + { 2048u, 4537ull }, + { 2560u, 5233ull }, + { 3072u, 6227ull }, + { 3584u, 6922ull }, + { 4096u, 9445ull }, + { 5120u, 11041ull }, + { 6144u, 12728ull }, + { 7168u, 14287ull }, + { 8192u, 17963ull }, + { 10240u, 21973ull }, + { 12288u, 27392ull }, + { 14336u, 30588ull }, + { 16384u, 35655ull }, + { 20480u, 44290ull }, + { 24576u, 53036ull }, + { 28672u, 61577ull }, + { 32768u, 70215ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_33[] = { + { 0u, 0ull }, + { 2u, 5091ull }, + { 4u, 5188ull }, + { 6u, 5293ull }, + { 8u, 5393ull }, + { 10u, 5493ull }, + { 12u, 5595ull }, + { 14u, 5688ull }, + { 16u, 5787ull }, + { 18u, 5891ull }, + { 20u, 5976ull }, + { 22u, 6078ull }, + { 24u, 6182ull }, + { 26u, 6282ull }, + { 28u, 6377ull }, + { 30u, 6686ull }, + { 32u, 6777ull }, + { 34u, 6876ull }, + { 36u, 6964ull }, + { 38u, 7069ull }, + { 40u, 7164ull }, + { 42u, 7273ull }, + { 44u, 7362ull }, + { 46u, 7468ull }, + { 48u, 7571ull }, + { 50u, 7666ull }, + { 52u, 7771ull }, + { 54u, 7853ull }, + { 56u, 7964ull }, + { 60u, 8167ull }, + { 62u, 8270ull }, + { 64u, 8361ull }, + { 66u, 8472ull }, + { 72u, 8762ull }, + { 80u, 9151ull }, + { 96u, 9943ull }, + { 112u, 10741ull }, + { 126u, 11438ull }, + { 128u, 11543ull }, + { 130u, 11633ull }, + { 160u, 13116ull }, + { 192u, 14699ull }, + { 224u, 16288ull }, + { 254u, 17774ull }, + { 256u, 17879ull }, + { 258u, 17973ull }, + { 320u, 21048ull }, + { 384u, 24221ull }, + { 448u, 27395ull }, + { 512u, 30567ull }, + { 640u, 37095ull }, + { 768u, 43451ull }, + { 896u, 49820ull }, + { 1024u, 56151ull }, + { 1280u, 69120ull }, + { 1536u, 81820ull }, + { 1792u, 94721ull }, + { 2048u, 107409ull }, + { 2560u, 133055ull }, + { 3072u, 158658ull }, + { 3584u, 184255ull }, + { 4096u, 209868ull }, + { 5120u, 261092ull }, + { 6144u, 312236ull }, + { 7168u, 363681ull }, + { 8192u, 414674ull }, + { 10240u, 517059ull }, + { 12288u, 619502ull }, + { 14336u, 722003ull }, + { 16384u, 825217ull }, + { 20480u, 1030022ull }, + { 24576u, 1235048ull }, + { 28672u, 1440330ull }, + { 32768u, 1644894ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_34[] = { + { 0u, 0ull }, + { 2u, 5263ull }, + { 4u, 5350ull }, + { 6u, 5460ull }, + { 8u, 5563ull }, + { 10u, 5661ull }, + { 12u, 5752ull }, + { 14u, 5862ull }, + { 16u, 5945ull }, + { 18u, 6058ull }, + { 20u, 6142ull }, + { 22u, 6242ull }, + { 24u, 6356ull }, + { 26u, 6454ull }, + { 28u, 6550ull }, + { 30u, 6850ull }, + { 32u, 6966ull }, + { 34u, 7049ull }, + { 36u, 7151ull }, + { 38u, 7248ull }, + { 40u, 7354ull }, + { 42u, 7445ull }, + { 44u, 7548ull }, + { 46u, 7653ull }, + { 48u, 7759ull }, + { 50u, 7856ull }, + { 52u, 7959ull }, + { 54u, 8046ull }, + { 56u, 8138ull }, + { 60u, 8358ull }, + { 62u, 8457ull }, + { 64u, 8548ull }, + { 66u, 8655ull }, + { 72u, 8948ull }, + { 80u, 9349ull }, + { 96u, 10157ull }, + { 112u, 10953ull }, + { 126u, 11657ull }, + { 128u, 11755ull }, + { 130u, 11861ull }, + { 160u, 13361ull }, + { 192u, 14956ull }, + { 224u, 16553ull }, + { 254u, 18051ull }, + { 256u, 18151ull }, + { 258u, 18246ull }, + { 320u, 21354ull }, + { 384u, 24555ull }, + { 448u, 27751ull }, + { 512u, 30966ull }, + { 640u, 37536ull }, + { 768u, 43964ull }, + { 896u, 50363ull }, + { 1024u, 56780ull }, + { 1280u, 69837ull }, + { 1536u, 82572ull }, + { 1792u, 95640ull }, + { 2048u, 108450ull }, + { 2560u, 134305ull }, + { 3072u, 160133ull }, + { 3584u, 185941ull }, + { 4096u, 211730ull }, + { 5120u, 263479ull }, + { 6144u, 315079ull }, + { 7168u, 366751ull }, + { 8192u, 418408ull }, + { 10240u, 521608ull }, + { 12288u, 624894ull }, + { 14336u, 728228ull }, + { 16384u, 831560ull }, + { 20480u, 1038603ull }, + { 24576u, 1245032ull }, + { 28672u, 1452470ull }, + { 32768u, 1659199ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_35[] = { + { 0u, 0ull }, + { 2u, 5263ull }, + { 4u, 5348ull }, + { 6u, 5451ull }, + { 8u, 5546ull }, + { 10u, 5647ull }, + { 12u, 5759ull }, + { 14u, 5848ull }, + { 16u, 5945ull }, + { 18u, 6049ull }, + { 20u, 6151ull }, + { 22u, 6253ull }, + { 24u, 6346ull }, + { 26u, 6451ull }, + { 28u, 6555ull }, + { 30u, 6853ull }, + { 32u, 6963ull }, + { 34u, 7054ull }, + { 36u, 7161ull }, + { 38u, 7249ull }, + { 40u, 7360ull }, + { 42u, 7457ull }, + { 44u, 7545ull }, + { 46u, 7645ull }, + { 48u, 7749ull }, + { 50u, 7854ull }, + { 52u, 7944ull }, + { 54u, 8055ull }, + { 56u, 8155ull }, + { 60u, 8340ull }, + { 62u, 8442ull }, + { 64u, 8545ull }, + { 66u, 8648ull }, + { 72u, 8936ull }, + { 80u, 9351ull }, + { 96u, 10146ull }, + { 112u, 10948ull }, + { 126u, 11651ull }, + { 128u, 11753ull }, + { 130u, 11855ull }, + { 160u, 13355ull }, + { 192u, 14951ull }, + { 224u, 16556ull }, + { 254u, 18048ull }, + { 256u, 18147ull }, + { 258u, 18249ull }, + { 320u, 21349ull }, + { 384u, 24566ull }, + { 448u, 27753ull }, + { 512u, 30956ull }, + { 640u, 37540ull }, + { 768u, 43985ull }, + { 896u, 50407ull }, + { 1024u, 56823ull }, + { 1280u, 69835ull }, + { 1536u, 82619ull }, + { 1792u, 95628ull }, + { 2048u, 108462ull }, + { 2560u, 134298ull }, + { 3072u, 160095ull }, + { 3584u, 185933ull }, + { 4096u, 211768ull }, + { 5120u, 263431ull }, + { 6144u, 315096ull }, + { 7168u, 366786ull }, + { 8192u, 418383ull }, + { 10240u, 521851ull }, + { 12288u, 625097ull }, + { 14336u, 728216ull }, + { 16384u, 831524ull }, + { 20480u, 1038135ull }, + { 24576u, 1245587ull }, + { 28672u, 1452543ull }, + { 32768u, 1658846ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_36[] = { + { 0u, 0ull }, + { 4u, 0ull }, + { 8u, 0ull }, + { 12u, 0ull }, + { 16u, 0ull }, + { 20u, 0ull }, + { 24u, 0ull }, + { 28u, 0ull }, + { 32u, 0ull }, + { 36u, 0ull }, + { 40u, 0ull }, + { 44u, 0ull }, + { 48u, 0ull }, + { 52u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 64u, 0ull }, + { 68u, 0ull }, + { 72u, 0ull }, + { 76u, 0ull }, + { 80u, 0ull }, + { 84u, 0ull }, + { 88u, 0ull }, + { 92u, 0ull }, + { 96u, 0ull }, + { 100u, 0ull }, + { 104u, 0ull }, + { 108u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 124u, 0ull }, + { 128u, 871ull }, + { 132u, 1050ull }, + { 144u, 1050ull }, + { 160u, 1050ull }, + { 192u, 1050ull }, + { 224u, 1119ull }, + { 252u, 1119ull }, + { 256u, 1119ull }, + { 260u, 1195ull }, + { 320u, 1195ull }, + { 384u, 1195ull }, + { 448u, 1195ull }, + { 508u, 1369ull }, + { 512u, 1369ull }, + { 516u, 1435ull }, + { 640u, 1435ull }, + { 768u, 1567ull }, + { 896u, 1669ull }, + { 1024u, 1869ull }, + { 1280u, 2553ull }, + { 1536u, 2771ull }, + { 1792u, 3120ull }, + { 2048u, 3469ull }, + { 2560u, 4459ull }, + { 3072u, 5468ull }, + { 3584u, 6070ull }, + { 4096u, 6975ull }, + { 5120u, 8578ull }, + { 6144u, 10183ull }, + { 7168u, 11781ull }, + { 8192u, 13386ull }, + { 10240u, 16586ull }, + { 12288u, 19801ull }, + { 14336u, 23003ull }, + { 16384u, 25864ull }, + { 20480u, 32309ull }, + { 24576u, 38724ull }, + { 28672u, 45142ull }, + { 32768u, 52338ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_37[] = { + { 0u, 0ull }, + { 4u, 0ull }, + { 8u, 0ull }, + { 12u, 0ull }, + { 16u, 0ull }, + { 20u, 0ull }, + { 24u, 0ull }, + { 28u, 0ull }, + { 32u, 0ull }, + { 36u, 0ull }, + { 40u, 0ull }, + { 44u, 0ull }, + { 48u, 0ull }, + { 52u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 64u, 0ull }, + { 68u, 0ull }, + { 72u, 0ull }, + { 76u, 0ull }, + { 80u, 0ull }, + { 84u, 0ull }, + { 88u, 0ull }, + { 92u, 0ull }, + { 96u, 0ull }, + { 100u, 0ull }, + { 104u, 0ull }, + { 108u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 124u, 0ull }, + { 128u, 1476ull }, + { 132u, 1488ull }, + { 144u, 1488ull }, + { 160u, 1488ull }, + { 192u, 1584ull }, + { 224u, 1584ull }, + { 252u, 1587ull }, + { 256u, 1785ull }, + { 260u, 1785ull }, + { 320u, 1881ull }, + { 384u, 1894ull }, + { 448u, 1968ull }, + { 508u, 1968ull }, + { 512u, 1968ull }, + { 516u, 1968ull }, + { 640u, 2166ull }, + { 768u, 2763ull }, + { 896u, 2862ull }, + { 1024u, 2960ull }, + { 1280u, 3359ull }, + { 1536u, 3656ull }, + { 1792u, 4350ull }, + { 2048u, 4549ull }, + { 2560u, 5245ull }, + { 3072u, 6238ull }, + { 3584u, 6823ull }, + { 4096u, 7927ull }, + { 5120u, 9461ull }, + { 6144u, 11054ull }, + { 7168u, 12593ull }, + { 8192u, 14229ull }, + { 10240u, 17365ull }, + { 12288u, 20639ull }, + { 14336u, 23692ull }, + { 16384u, 26493ull }, + { 20480u, 32860ull }, + { 24576u, 39223ull }, + { 28672u, 45588ull }, + { 32768u, 52505ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_38[] = { + { 0u, 0ull }, + { 4u, 0ull }, + { 8u, 0ull }, + { 12u, 0ull }, + { 16u, 0ull }, + { 20u, 0ull }, + { 24u, 0ull }, + { 28u, 0ull }, + { 32u, 0ull }, + { 36u, 0ull }, + { 40u, 0ull }, + { 44u, 0ull }, + { 48u, 0ull }, + { 52u, 0ull }, + { 56u, 0ull }, + { 60u, 0ull }, + { 64u, 0ull }, + { 68u, 0ull }, + { 72u, 0ull }, + { 76u, 0ull }, + { 80u, 0ull }, + { 84u, 0ull }, + { 88u, 0ull }, + { 92u, 0ull }, + { 96u, 0ull }, + { 100u, 0ull }, + { 104u, 0ull }, + { 108u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 124u, 0ull }, + { 128u, 1416ull }, + { 132u, 1416ull }, + { 144u, 1416ull }, + { 160u, 1416ull }, + { 192u, 1462ull }, + { 224u, 1462ull }, + { 252u, 1557ull }, + { 256u, 1761ull }, + { 260u, 1768ull }, + { 320u, 1863ull }, + { 384u, 1863ull }, + { 448u, 1951ull }, + { 508u, 1951ull }, + { 512u, 1951ull }, + { 516u, 1951ull }, + { 640u, 2147ull }, + { 768u, 2734ull }, + { 896u, 2843ull }, + { 1024u, 2940ull }, + { 1280u, 3340ull }, + { 1536u, 3636ull }, + { 1792u, 4331ull }, + { 2048u, 4522ull }, + { 2560u, 5226ull }, + { 3072u, 6217ull }, + { 3584u, 6816ull }, + { 4096u, 9429ull }, + { 5120u, 11243ull }, + { 6144u, 12814ull }, + { 7168u, 14290ull }, + { 8192u, 17958ull }, + { 10240u, 21124ull }, + { 12288u, 26505ull }, + { 14336u, 29675ull }, + { 16384u, 34739ull }, + { 20480u, 43386ull }, + { 24576u, 52027ull }, + { 28672u, 60653ull }, + { 32768u, 69494ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_39[] = { + { 0u, 0ull }, + { 4u, 5063ull }, + { 8u, 5165ull }, + { 12u, 5269ull }, + { 16u, 5367ull }, + { 20u, 5458ull }, + { 24u, 5560ull }, + { 28u, 5671ull }, + { 32u, 5782ull }, + { 36u, 5864ull }, + { 40u, 5963ull }, + { 44u, 6068ull }, + { 48u, 6168ull }, + { 52u, 6265ull }, + { 56u, 6355ull }, + { 60u, 6673ull }, + { 64u, 6768ull }, + { 68u, 6857ull }, + { 72u, 6951ull }, + { 76u, 7054ull }, + { 80u, 7156ull }, + { 84u, 7255ull }, + { 88u, 7342ull }, + { 92u, 7439ull }, + { 96u, 7541ull }, + { 100u, 7641ull }, + { 104u, 7735ull }, + { 108u, 7839ull }, + { 112u, 7936ull }, + { 120u, 8134ull }, + { 124u, 8239ull }, + { 128u, 8340ull }, + { 132u, 8431ull }, + { 144u, 8724ull }, + { 160u, 9122ull }, + { 192u, 9932ull }, + { 224u, 10726ull }, + { 252u, 11422ull }, + { 256u, 11525ull }, + { 260u, 11633ull }, + { 320u, 13103ull }, + { 384u, 14699ull }, + { 448u, 16290ull }, + { 508u, 17786ull }, + { 512u, 17885ull }, + { 516u, 17971ull }, + { 640u, 21064ull }, + { 768u, 24240ull }, + { 896u, 27428ull }, + { 1024u, 30604ull }, + { 1280u, 37167ull }, + { 1536u, 43517ull }, + { 1792u, 49872ull }, + { 2048u, 56238ull }, + { 2560u, 69161ull }, + { 3072u, 81842ull }, + { 3584u, 94755ull }, + { 4096u, 107408ull }, + { 5120u, 132991ull }, + { 6144u, 158585ull }, + { 7168u, 184187ull }, + { 8192u, 209911ull }, + { 10240u, 260982ull }, + { 12288u, 312244ull }, + { 14336u, 363787ull }, + { 16384u, 414570ull }, + { 20480u, 516943ull }, + { 24576u, 619345ull }, + { 28672u, 721704ull }, + { 32768u, 824832ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_40[] = { + { 0u, 0ull }, + { 4u, 5229ull }, + { 8u, 5322ull }, + { 12u, 5425ull }, + { 16u, 5524ull }, + { 20u, 5625ull }, + { 24u, 5723ull }, + { 28u, 5825ull }, + { 32u, 5922ull }, + { 36u, 6028ull }, + { 40u, 6124ull }, + { 44u, 6241ull }, + { 48u, 6327ull }, + { 52u, 6423ull }, + { 56u, 6524ull }, + { 60u, 6827ull }, + { 64u, 6923ull }, + { 68u, 7022ull }, + { 72u, 7128ull }, + { 76u, 7225ull }, + { 80u, 7320ull }, + { 84u, 7419ull }, + { 88u, 7518ull }, + { 92u, 7617ull }, + { 96u, 7717ull }, + { 100u, 7831ull }, + { 104u, 7941ull }, + { 108u, 8019ull }, + { 112u, 8137ull }, + { 120u, 8331ull }, + { 124u, 8417ull }, + { 128u, 8523ull }, + { 132u, 8630ull }, + { 144u, 8926ull }, + { 160u, 9316ull }, + { 192u, 10129ull }, + { 224u, 10920ull }, + { 252u, 11639ull }, + { 256u, 11711ull }, + { 260u, 11808ull }, + { 320u, 13337ull }, + { 384u, 14932ull }, + { 448u, 16529ull }, + { 508u, 18029ull }, + { 512u, 18129ull }, + { 516u, 18236ull }, + { 640u, 21337ull }, + { 768u, 24537ull }, + { 896u, 27746ull }, + { 1024u, 30948ull }, + { 1280u, 37522ull }, + { 1536u, 43947ull }, + { 1792u, 50356ull }, + { 2048u, 56774ull }, + { 2560u, 69743ull }, + { 3072u, 82575ull }, + { 3584u, 95667ull }, + { 4096u, 108276ull }, + { 5120u, 134028ull }, + { 6144u, 159773ull }, + { 7168u, 185492ull }, + { 8192u, 211347ull }, + { 10240u, 262845ull }, + { 12288u, 314603ull }, + { 14336u, 366122ull }, + { 16384u, 417507ull }, + { 20480u, 520574ull }, + { 24576u, 623735ull }, + { 28672u, 726740ull }, + { 32768u, 829411ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_41[] = { + { 0u, 0ull }, + { 4u, 5235ull }, + { 8u, 5339ull }, + { 12u, 5439ull }, + { 16u, 5536ull }, + { 20u, 5634ull }, + { 24u, 5741ull }, + { 28u, 5836ull }, + { 32u, 5944ull }, + { 36u, 6036ull }, + { 40u, 6138ull }, + { 44u, 6237ull }, + { 48u, 6339ull }, + { 52u, 6436ull }, + { 56u, 6538ull }, + { 60u, 6846ull }, + { 64u, 6937ull }, + { 68u, 7037ull }, + { 72u, 7134ull }, + { 76u, 7242ull }, + { 80u, 7344ull }, + { 84u, 7432ull }, + { 88u, 7545ull }, + { 92u, 7640ull }, + { 96u, 7733ull }, + { 100u, 7838ull }, + { 104u, 7936ull }, + { 108u, 8037ull }, + { 112u, 8146ull }, + { 120u, 8338ull }, + { 124u, 8431ull }, + { 128u, 8534ull }, + { 132u, 8636ull }, + { 144u, 8941ull }, + { 160u, 9346ull }, + { 192u, 10134ull }, + { 224u, 10940ull }, + { 252u, 11633ull }, + { 256u, 11752ull }, + { 260u, 11830ull }, + { 320u, 13360ull }, + { 384u, 14960ull }, + { 448u, 16547ull }, + { 508u, 18064ull }, + { 512u, 18160ull }, + { 516u, 18256ull }, + { 640u, 21364ull }, + { 768u, 24572ull }, + { 896u, 27783ull }, + { 1024u, 30980ull }, + { 1280u, 37608ull }, + { 1536u, 44005ull }, + { 1792u, 50433ull }, + { 2048u, 56835ull }, + { 2560u, 69846ull }, + { 3072u, 82637ull }, + { 3584u, 95647ull }, + { 4096u, 108464ull }, + { 5120u, 134251ull }, + { 6144u, 160079ull }, + { 7168u, 185853ull }, + { 8192u, 211934ull }, + { 10240u, 263596ull }, + { 12288u, 314969ull }, + { 14336u, 366595ull }, + { 16384u, 418221ull }, + { 20480u, 521464ull }, + { 24576u, 624769ull }, + { 28672u, 728482ull }, + { 32768u, 831092ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_42[] = { + { 0u, 0ull }, + { 8u, 0ull }, + { 16u, 0ull }, + { 24u, 0ull }, + { 32u, 0ull }, + { 40u, 0ull }, + { 48u, 0ull }, + { 56u, 0ull }, + { 64u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 88u, 0ull }, + { 96u, 0ull }, + { 104u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 128u, 872ull }, + { 136u, 1058ull }, + { 144u, 1058ull }, + { 152u, 1058ull }, + { 160u, 1058ull }, + { 168u, 1058ull }, + { 176u, 1058ull }, + { 184u, 1058ull }, + { 192u, 1058ull }, + { 200u, 1143ull }, + { 208u, 1143ull }, + { 216u, 1143ull }, + { 224u, 1143ull }, + { 240u, 1143ull }, + { 248u, 1143ull }, + { 256u, 1143ull }, + { 264u, 1228ull }, + { 288u, 1228ull }, + { 320u, 1228ull }, + { 384u, 1228ull }, + { 448u, 1228ull }, + { 504u, 1361ull }, + { 512u, 1361ull }, + { 520u, 1417ull }, + { 640u, 1417ull }, + { 768u, 1551ull }, + { 896u, 1656ull }, + { 1016u, 2212ull }, + { 1024u, 2212ull }, + { 1032u, 2567ull }, + { 1280u, 2567ull }, + { 1536u, 2769ull }, + { 1792u, 3120ull }, + { 2048u, 3454ull }, + { 2560u, 4469ull }, + { 3072u, 5455ull }, + { 3584u, 6075ull }, + { 4096u, 6977ull }, + { 5120u, 8579ull }, + { 6144u, 10162ull }, + { 7168u, 11784ull }, + { 8192u, 13391ull }, + { 10240u, 16578ull }, + { 12288u, 19800ull }, + { 14336u, 23001ull }, + { 16384u, 25901ull }, + { 20480u, 32287ull }, + { 24576u, 38723ull }, + { 28672u, 45150ull }, + { 32768u, 52354ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_43[] = { + { 0u, 0ull }, + { 8u, 0ull }, + { 16u, 0ull }, + { 24u, 0ull }, + { 32u, 0ull }, + { 40u, 0ull }, + { 48u, 0ull }, + { 56u, 0ull }, + { 64u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 88u, 0ull }, + { 96u, 0ull }, + { 104u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 128u, 1451ull }, + { 136u, 1451ull }, + { 144u, 1451ull }, + { 152u, 1451ull }, + { 160u, 1451ull }, + { 168u, 1451ull }, + { 176u, 1451ull }, + { 184u, 1451ull }, + { 192u, 1544ull }, + { 200u, 1544ull }, + { 208u, 1544ull }, + { 216u, 1544ull }, + { 224u, 1544ull }, + { 240u, 1544ull }, + { 248u, 1544ull }, + { 256u, 1772ull }, + { 264u, 1777ull }, + { 288u, 1777ull }, + { 320u, 1837ull }, + { 384u, 1837ull }, + { 448u, 1837ull }, + { 504u, 1837ull }, + { 512u, 1973ull }, + { 520u, 1973ull }, + { 640u, 2168ull }, + { 768u, 2654ull }, + { 896u, 2654ull }, + { 1016u, 2654ull }, + { 1024u, 2951ull }, + { 1032u, 2964ull }, + { 1280u, 3362ull }, + { 1536u, 3659ull }, + { 1792u, 4357ull }, + { 2048u, 4533ull }, + { 2560u, 5253ull }, + { 3072u, 6222ull }, + { 3584u, 6847ull }, + { 4096u, 7940ull }, + { 5120u, 9482ull }, + { 6144u, 11076ull }, + { 7168u, 12622ull }, + { 8192u, 14269ull }, + { 10240u, 17385ull }, + { 12288u, 20666ull }, + { 14336u, 23758ull }, + { 16384u, 26564ull }, + { 20480u, 32907ull }, + { 24576u, 39289ull }, + { 28672u, 45672ull }, + { 32768u, 52653ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_44[] = { + { 0u, 0ull }, + { 8u, 0ull }, + { 16u, 0ull }, + { 24u, 0ull }, + { 32u, 0ull }, + { 40u, 0ull }, + { 48u, 0ull }, + { 56u, 0ull }, + { 64u, 0ull }, + { 72u, 0ull }, + { 80u, 0ull }, + { 88u, 0ull }, + { 96u, 0ull }, + { 104u, 0ull }, + { 112u, 0ull }, + { 120u, 0ull }, + { 128u, 1431ull }, + { 136u, 1431ull }, + { 144u, 1431ull }, + { 152u, 1431ull }, + { 160u, 1431ull }, + { 168u, 1471ull }, + { 176u, 1476ull }, + { 184u, 1476ull }, + { 192u, 1530ull }, + { 200u, 1530ull }, + { 208u, 1530ull }, + { 216u, 1530ull }, + { 224u, 1530ull }, + { 240u, 1569ull }, + { 248u, 1569ull }, + { 256u, 1644ull }, + { 264u, 1644ull }, + { 288u, 1644ull }, + { 320u, 1864ull }, + { 384u, 1870ull }, + { 448u, 1954ull }, + { 504u, 1954ull }, + { 512u, 1964ull }, + { 520u, 1964ull }, + { 640u, 2204ull }, + { 768u, 2753ull }, + { 896u, 2854ull }, + { 1016u, 2938ull }, + { 1024u, 2946ull }, + { 1032u, 2946ull }, + { 1280u, 3348ull }, + { 1536u, 3634ull }, + { 1792u, 4323ull }, + { 2048u, 4541ull }, + { 2560u, 5236ull }, + { 3072u, 6227ull }, + { 3584u, 6826ull }, + { 4096u, 9453ull }, + { 5120u, 11248ull }, + { 6144u, 12727ull }, + { 7168u, 14296ull }, + { 8192u, 17978ull }, + { 10240u, 21167ull }, + { 12288u, 26535ull }, + { 14336u, 29708ull }, + { 16384u, 34773ull }, + { 20480u, 43428ull }, + { 24576u, 52038ull }, + { 28672u, 60827ull }, + { 32768u, 69352ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_45[] = { + { 0u, 0ull }, + { 8u, 5071ull }, + { 16u, 5174ull }, + { 24u, 5271ull }, + { 32u, 5366ull }, + { 40u, 5465ull }, + { 48u, 5564ull }, + { 56u, 5659ull }, + { 64u, 5759ull }, + { 72u, 5860ull }, + { 80u, 5954ull }, + { 88u, 6055ull }, + { 96u, 6163ull }, + { 104u, 6258ull }, + { 112u, 6354ull }, + { 120u, 6651ull }, + { 128u, 6746ull }, + { 136u, 6851ull }, + { 144u, 6953ull }, + { 152u, 7047ull }, + { 160u, 7149ull }, + { 168u, 7247ull }, + { 176u, 7350ull }, + { 184u, 7445ull }, + { 192u, 7541ull }, + { 200u, 7647ull }, + { 208u, 7743ull }, + { 216u, 7844ull }, + { 224u, 7937ull }, + { 240u, 8133ull }, + { 248u, 8232ull }, + { 256u, 8330ull }, + { 264u, 8437ull }, + { 288u, 8729ull }, + { 320u, 9128ull }, + { 384u, 9928ull }, + { 448u, 10710ull }, + { 504u, 11397ull }, + { 512u, 11500ull }, + { 520u, 11599ull }, + { 640u, 13133ull }, + { 768u, 14713ull }, + { 896u, 16310ull }, + { 1016u, 17768ull }, + { 1024u, 17876ull }, + { 1032u, 17981ull }, + { 1280u, 21015ull }, + { 1536u, 24191ull }, + { 1792u, 27363ull }, + { 2048u, 30520ull }, + { 2560u, 37057ull }, + { 3072u, 43407ull }, + { 3584u, 49725ull }, + { 4096u, 56020ull }, + { 5120u, 68950ull }, + { 6144u, 81714ull }, + { 7168u, 94541ull }, + { 8192u, 107282ull }, + { 10240u, 132745ull }, + { 12288u, 158466ull }, + { 14336u, 184025ull }, + { 16384u, 209612ull }, + { 20480u, 260627ull }, + { 24576u, 311736ull }, + { 28672u, 363026ull }, + { 32768u, 414217ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_46[] = { + { 0u, 0ull }, + { 8u, 5205ull }, + { 16u, 5302ull }, + { 24u, 5407ull }, + { 32u, 5502ull }, + { 40u, 5599ull }, + { 48u, 5704ull }, + { 56u, 5799ull }, + { 64u, 5894ull }, + { 72u, 5996ull }, + { 80u, 6098ull }, + { 88u, 6199ull }, + { 96u, 6298ull }, + { 104u, 6401ull }, + { 112u, 6498ull }, + { 120u, 6794ull }, + { 128u, 6894ull }, + { 136u, 6996ull }, + { 144u, 7098ull }, + { 152u, 7192ull }, + { 160u, 7293ull }, + { 168u, 7395ull }, + { 176u, 7490ull }, + { 184u, 7593ull }, + { 192u, 7691ull }, + { 200u, 7794ull }, + { 208u, 7891ull }, + { 216u, 7993ull }, + { 224u, 8091ull }, + { 240u, 8290ull }, + { 248u, 8392ull }, + { 256u, 8490ull }, + { 264u, 8593ull }, + { 288u, 8895ull }, + { 320u, 9300ull }, + { 384u, 10074ull }, + { 448u, 10879ull }, + { 504u, 11602ull }, + { 512u, 11678ull }, + { 520u, 11785ull }, + { 640u, 13315ull }, + { 768u, 14925ull }, + { 896u, 16505ull }, + { 1016u, 18025ull }, + { 1024u, 18094ull }, + { 1032u, 18191ull }, + { 1280u, 21324ull }, + { 1536u, 24446ull }, + { 1792u, 27641ull }, + { 2048u, 30830ull }, + { 2560u, 37409ull }, + { 3072u, 43789ull }, + { 3584u, 50163ull }, + { 4096u, 56541ull }, + { 5120u, 69521ull }, + { 6144u, 82282ull }, + { 7168u, 95167ull }, + { 8192u, 108120ull }, + { 10240u, 133888ull }, + { 12288u, 159631ull }, + { 14336u, 185372ull }, + { 16384u, 211142ull }, + { 20480u, 262656ull }, + { 24576u, 314160ull }, + { 28672u, 365412ull }, + { 32768u, 417164ull }, +}; + +static const ClgRepBulkPoint clg_rep_bulk_pts_47[] = { + { 0u, 0ull }, + { 8u, 5217ull }, + { 16u, 5321ull }, + { 24u, 5420ull }, + { 32u, 5519ull }, + { 40u, 5622ull }, + { 48u, 5721ull }, + { 56u, 5821ull }, + { 64u, 5918ull }, + { 72u, 6018ull }, + { 80u, 6119ull }, + { 88u, 6220ull }, + { 96u, 6319ull }, + { 104u, 6415ull }, + { 112u, 6519ull }, + { 120u, 6815ull }, + { 128u, 6915ull }, + { 136u, 7014ull }, + { 144u, 7116ull }, + { 152u, 7216ull }, + { 160u, 7321ull }, + { 168u, 7418ull }, + { 176u, 7517ull }, + { 184u, 7611ull }, + { 192u, 7715ull }, + { 200u, 7822ull }, + { 208u, 7915ull }, + { 216u, 8016ull }, + { 224u, 8123ull }, + { 240u, 8314ull }, + { 248u, 8417ull }, + { 256u, 8512ull }, + { 264u, 8614ull }, + { 288u, 8917ull }, + { 320u, 9307ull }, + { 384u, 10109ull }, + { 448u, 10906ull }, + { 504u, 11608ull }, + { 512u, 11706ull }, + { 520u, 11809ull }, + { 640u, 13313ull }, + { 768u, 14937ull }, + { 896u, 16534ull }, + { 1016u, 18077ull }, + { 1024u, 18120ull }, + { 1032u, 18273ull }, + { 1280u, 21370ull }, + { 1536u, 24512ull }, + { 1792u, 27703ull }, + { 2048u, 30896ull }, + { 2560u, 37473ull }, + { 3072u, 43871ull }, + { 3584u, 50275ull }, + { 4096u, 56740ull }, + { 5120u, 69734ull }, + { 6144u, 82539ull }, + { 7168u, 95535ull }, + { 8192u, 108336ull }, + { 10240u, 134126ull }, + { 12288u, 159872ull }, + { 14336u, 185631ull }, + { 16384u, 211446ull }, + { 20480u, 263072ull }, + { 24576u, 314771ull }, + { 28672u, 366428ull }, + { 32768u, 417842ull }, +}; + +static const ClgRepBulkCurve clg_rep_bulk_curves[] = { + { { CLG_REP_OP_MOVS, 1u, 0u, 0u }, clg_rep_bulk_pts_0, 74u }, + { { CLG_REP_OP_MOVS, 1u, 0u, 1u }, clg_rep_bulk_pts_1, 74u }, + { { CLG_REP_OP_MOVS, 1u, 0u, 2u }, clg_rep_bulk_pts_2, 74u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 0u }, clg_rep_bulk_pts_3, 74u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 1u }, clg_rep_bulk_pts_4, 74u }, + { { CLG_REP_OP_MOVS, 1u, 1u, 2u }, clg_rep_bulk_pts_5, 74u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 0u }, clg_rep_bulk_pts_6, 70u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 1u }, clg_rep_bulk_pts_7, 70u }, + { { CLG_REP_OP_MOVS, 2u, 0u, 2u }, clg_rep_bulk_pts_8, 70u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 0u }, clg_rep_bulk_pts_9, 70u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 1u }, clg_rep_bulk_pts_10, 70u }, + { { CLG_REP_OP_MOVS, 2u, 1u, 2u }, clg_rep_bulk_pts_11, 70u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 0u }, clg_rep_bulk_pts_12, 66u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 1u }, clg_rep_bulk_pts_13, 66u }, + { { CLG_REP_OP_MOVS, 4u, 0u, 2u }, clg_rep_bulk_pts_14, 66u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 0u }, clg_rep_bulk_pts_15, 66u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 1u }, clg_rep_bulk_pts_16, 66u }, + { { CLG_REP_OP_MOVS, 4u, 1u, 2u }, clg_rep_bulk_pts_17, 66u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 0u }, clg_rep_bulk_pts_18, 62u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 1u }, clg_rep_bulk_pts_19, 62u }, + { { CLG_REP_OP_MOVS, 8u, 0u, 2u }, clg_rep_bulk_pts_20, 62u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 0u }, clg_rep_bulk_pts_21, 62u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 1u }, clg_rep_bulk_pts_22, 62u }, + { { CLG_REP_OP_MOVS, 8u, 1u, 2u }, clg_rep_bulk_pts_23, 62u }, + { { CLG_REP_OP_STOS, 1u, 0u, 0u }, clg_rep_bulk_pts_24, 78u }, + { { CLG_REP_OP_STOS, 1u, 0u, 1u }, clg_rep_bulk_pts_25, 78u }, + { { CLG_REP_OP_STOS, 1u, 0u, 2u }, clg_rep_bulk_pts_26, 78u }, + { { CLG_REP_OP_STOS, 1u, 1u, 0u }, clg_rep_bulk_pts_27, 78u }, + { { CLG_REP_OP_STOS, 1u, 1u, 1u }, clg_rep_bulk_pts_28, 78u }, + { { CLG_REP_OP_STOS, 1u, 1u, 2u }, clg_rep_bulk_pts_29, 78u }, + { { CLG_REP_OP_STOS, 2u, 0u, 0u }, clg_rep_bulk_pts_30, 74u }, + { { CLG_REP_OP_STOS, 2u, 0u, 1u }, clg_rep_bulk_pts_31, 74u }, + { { CLG_REP_OP_STOS, 2u, 0u, 2u }, clg_rep_bulk_pts_32, 74u }, + { { CLG_REP_OP_STOS, 2u, 1u, 0u }, clg_rep_bulk_pts_33, 74u }, + { { CLG_REP_OP_STOS, 2u, 1u, 1u }, clg_rep_bulk_pts_34, 74u }, + { { CLG_REP_OP_STOS, 2u, 1u, 2u }, clg_rep_bulk_pts_35, 74u }, + { { CLG_REP_OP_STOS, 4u, 0u, 0u }, clg_rep_bulk_pts_36, 70u }, + { { CLG_REP_OP_STOS, 4u, 0u, 1u }, clg_rep_bulk_pts_37, 70u }, + { { CLG_REP_OP_STOS, 4u, 0u, 2u }, clg_rep_bulk_pts_38, 70u }, + { { CLG_REP_OP_STOS, 4u, 1u, 0u }, clg_rep_bulk_pts_39, 70u }, + { { CLG_REP_OP_STOS, 4u, 1u, 1u }, clg_rep_bulk_pts_40, 70u }, + { { CLG_REP_OP_STOS, 4u, 1u, 2u }, clg_rep_bulk_pts_41, 70u }, + { { CLG_REP_OP_STOS, 8u, 0u, 0u }, clg_rep_bulk_pts_42, 66u }, + { { CLG_REP_OP_STOS, 8u, 0u, 1u }, clg_rep_bulk_pts_43, 66u }, + { { CLG_REP_OP_STOS, 8u, 0u, 2u }, clg_rep_bulk_pts_44, 66u }, + { { CLG_REP_OP_STOS, 8u, 1u, 0u }, clg_rep_bulk_pts_45, 66u }, + { { CLG_REP_OP_STOS, 8u, 1u, 1u }, clg_rep_bulk_pts_46, 66u }, + { { CLG_REP_OP_STOS, 8u, 1u, 2u }, clg_rep_bulk_pts_47, 66u }, +}; diff --git a/callgrind/sim.c b/callgrind/sim.c index db3f9bef18..ff3ddb9720 100644 --- a/callgrind/sim.c +++ b/callgrind/sim.c @@ -29,6 +29,7 @@ */ #include "global.h" +#include "rep_bulk.h" /* Notes: @@ -1181,8 +1182,14 @@ static void log_1I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - inc_costs(DrRes, cost_Dr, - CLG_(current_state).cost + fullOffset(EG_DR) ); + /* REP bulk model: data refs are reported only for L1-miss results + * (with the matching D1/LL miss refs), so REP data L1 hits are carried + * by the bulk curve instead of the consumer's per-L1-hit term. Every + * miss, including repeated same-line misses, is still counted. + * Instruction-side counters are unchanged. */ + if (!ii->rep_bulk_op || DrRes != L1_Hit) + inc_costs(DrRes, cost_Dr, + CLG_(current_state).cost + fullOffset(EG_DR) ); } } @@ -1209,8 +1216,9 @@ static void log_0I1Dr(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dr = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DR]; - inc_costs(DrRes, cost_Dr, - CLG_(current_state).cost + fullOffset(EG_DR) ); + if (!ii->rep_bulk_op || DrRes != L1_Hit) + inc_costs(DrRes, cost_Dr, + CLG_(current_state).cost + fullOffset(EG_DR) ); } } @@ -1244,8 +1252,9 @@ static void log_1I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) inc_costs(IrRes, cost_Ir, CLG_(current_state).cost + fullOffset(EG_IR) ); - inc_costs(DwRes, cost_Dw, - CLG_(current_state).cost + fullOffset(EG_DW) ); + if (!ii->rep_bulk_op || DwRes != L1_Hit) + inc_costs(DwRes, cost_Dw, + CLG_(current_state).cost + fullOffset(EG_DW) ); } } @@ -1269,13 +1278,100 @@ static void log_0I1Dw(InstrInfo* ii, Addr data_addr, Word data_size) else cost_Dw = CLG_(cost_base) + ii->cost_offset + ii->eventset->offset[EG_DW]; - inc_costs(DwRes, cost_Dw, - CLG_(current_state).cost + fullOffset(EG_DW) ); + if (!ii->rep_bulk_op || DwRes != L1_Hit) + inc_costs(DwRes, cost_Dw, + CLG_(current_state).cost + fullOffset(EG_DW) ); } } +/*------------------------------------------------------------*/ +/*--- REP bulk cost model helper ---*/ +/*------------------------------------------------------------*/ + +/* Alignment class, computed from the pointers of the FIRST element of the + * operation (see rep_bulk.h). STOS has no source; a merged read-modify event + * (self copy) passes src == 0 and uses the destination offset for both. */ +static unsigned rep_align_class(unsigned op, Addr src, Addr dst) +{ + UInt so; + UInt dofs = (UInt)(dst & 63); + + /* A merged read-modify event (self copy) has no separate source address; + * both pointers are the same, so use the destination offset for both. */ + if (src == 0) + src = dst; + so = (UInt)(src & 63); + + if (op == CLG_REP_OP_MOVS) { + if (so == 0 && dofs == 0) + return CLG_REP_ALIGN_ALIGNED; + if (so == 8 && dofs == 8) + return CLG_REP_ALIGN_SAME_UNALIGN; + return CLG_REP_ALIGN_OTHER; + } + if (dofs == 0) + return CLG_REP_ALIGN_ALIGNED; + if (dofs == 8) + return CLG_REP_ALIGN_SAME_UNALIGN; + return CLG_REP_ALIGN_OTHER; +} + +/* Called once per executed element of a supported rep-prefixed bulk copy. + * `rem_after` is the repeat count after VEX decremented it for this element. + * The element is priced with the calibrated hot bulk curve; the delta is + * charged to BOTH Ct and Cl (self and global inclusive), which is what the + * consumer's sqrt(Ct*Cl)/100 combines back to the calibrated cost for a + * REP-only sum. Only REP *data* L1-hit refs are suppressed in the data log + * helpers; instruction-side refs and every L1/LL data miss stay reported. */ +static void log_rep_bulk(InstrInfo* ii, ULong rem_after, Addr src, Addr dst, + Word packed, Word dflag) +{ + exec_state* es = &CLG_(current_state); + unsigned op = (unsigned)(packed & 0xFF); + unsigned width = (unsigned)((packed >> 8) & 0xFF); + ULong rem_before = rem_after + 1; + unsigned dir, align; + ULong delta; + ULong * cost_C, *global_C; + + /* Alignment and direction are latched once per operation from its first + * element and kept for the whole operation, so a mid-operation address + * change cannot switch curves. The next element continues iff it reports + * exactly the count we expect after the previous one. */ + if (es->rep_ii == ii && es->rep_next_rem == rem_before) { + dir = es->rep_dir; + align = es->rep_align; + } else { + dir = ((Long)dflag < 0) ? CLG_REP_DIR_BACKWARD : CLG_REP_DIR_FORWARD; + align = rep_align_class(op, src, dst); + es->rep_tail_carry = 0; /* new curve run: restart tail remainder */ + } + es->rep_ii = ii; + es->rep_next_rem = rem_before - 1; + + if (!es->collect || width == 0 || op == CLG_REP_OP_NONE) + return; + + delta = clg_rep_bulk_marginal(op, width, dir, align, + rem_before * (ULong)width, (ULong)width, + &es->rep_tail_carry); + + if (es->nonskipped) + cost_C = es->nonskipped->skipped + fullOffset(EG_CYCLES); + else + cost_C = CLG_(cost_base) + ii->cost_offset + + ii->eventset->offset[EG_CYCLES]; + global_C = es->cost + fullOffset(EG_CYCLES); + + cost_C[0] += delta; + cost_C[1] += delta; + global_C[0] += delta; + global_C[1] += delta; +} + + /*------------------------------------------------------------*/ /*--- Cache configuration ---*/ /*------------------------------------------------------------*/ @@ -1290,6 +1386,30 @@ static void cachesim_post_clo_init(void) /* Cache configurations. */ cache_t I1c, D1c, LLc; + /* REP bulk cost model (on by default). It needs real cache simulation and + * the per-instruction cycle model it replaces, and is currently amd64-only + * because it reads the repeat counter and DF from the guest state. In any + * other mode it stays inert, so prior behavior is unchanged. */ + if (CLG_(clo).rep_bulk_cost) { + Bool usable = CLG_(clo).simulate_cache && CLG_(clo).cycle_estimation + && !clo_simulate_writeback && !clo_collect_cacheuse; +#if !defined(VGA_amd64) + usable = False; +#endif + if (!usable) { + CLG_(clo).rep_bulk_cost = False; + } else { + UInt bad = clg_rep_bulk_init(); + if (bad > 0) { + VG_(message)(Vg_UserMsg, + "--rep-bulk-cost: calibrated REP bulk table rejected " + "(%u problem(s)); keeping the per-element model.\n", + bad); + CLG_(clo).rep_bulk_cost = False; + } + } + } + /* Initialize access handlers */ if (!CLG_(clo).simulate_cache) { CLG_(cachesim).log_1I0D = 0; @@ -1367,6 +1487,11 @@ static void cachesim_post_clo_init(void) CLG_(cachesim).log_0I1Dr_name = "log_0I1Dr"; CLG_(cachesim).log_0I1Dw_name = "log_0I1Dw"; + if (CLG_(clo).rep_bulk_cost) { + CLG_(cachesim).log_rep_bulk = log_rep_bulk; + CLG_(cachesim).log_rep_bulk_name = "log_rep_bulk"; + } + if (clo_collect_cacheuse) { /* Output warning for not supported option combinations */ @@ -1733,6 +1858,8 @@ struct cachesim_if CLG_(cachesim) = { .log_0I1Dr = 0, .log_0I1Dw = 0, + .log_rep_bulk = 0, + .log_1I0D_name = "(no function)", .log_2I0D_name = "(no function)", .log_3I0D_name = "(no function)", @@ -1742,6 +1869,8 @@ struct cachesim_if CLG_(cachesim) = { .log_0I1Dr_name = "(no function)", .log_0I1Dw_name = "(no function)", + + .log_rep_bulk_name = "(no function)", }; diff --git a/callgrind/tests/Makefile.am b/callgrind/tests/Makefile.am index 0b7f9965db..105c3eb15a 100644 --- a/callgrind/tests/Makefile.am +++ b/callgrind/tests/Makefile.am @@ -44,9 +44,12 @@ EXTRA_DIST = \ find-source.vgtest find-source.stderr.exp find-source.post.exp \ inline-samefile.vgtest inline-samefile.stderr.exp inline-samefile.stdout.exp inline-samefile.post.exp \ inline-crossfile.vgtest inline-crossfile.stderr.exp inline-crossfile.stdout.exp inline-crossfile.post.exp \ - inline-crossfile-helper1.h inline-crossfile-helper2.h filter_inline + inline-crossfile-helper1.h inline-crossfile-helper2.h filter_inline \ + rep_bulk.c rep_bulk.vgtest rep_bulk_legacy.vgtest \ + rep_bulk.post.exp rep_bulk_legacy.post.exp rep_bulk_check.pl \ + rep_bulk.stderr.exp rep_bulk_legacy.stderr.exp -check_PROGRAMS = clreq register_desc register_desc_threads find_debuginfo simwork threads inline-samefile inline-crossfile runtime_obj_skip_c runtime_obj_skip_underflow deterministic_rng +check_PROGRAMS = clreq register_desc register_desc_threads find_debuginfo simwork threads inline-samefile inline-crossfile runtime_obj_skip_c runtime_obj_skip_underflow deterministic_rng rep_bulk if HAVE_PR_GET_AUXV check_PROGRAMS += prctl_get_auxv endif @@ -58,6 +61,10 @@ AM_CXXFLAGS += $(AM_FLAG_M3264_PRI) inline_samefile_CFLAGS = $(AM_CFLAGS) -O2 -g inline_crossfile_CFLAGS = $(AM_CFLAGS) -O2 -g +# Keep the measured part free of avoidable stack traffic: inline asm cases +# with the pointers already in registers. +rep_bulk_CFLAGS = $(AM_CFLAGS) -O2 + threads_LDADD = -lpthread deterministic_rng_LDADD = -lpthread register_desc_threads_LDADD = -lpthread diff --git a/callgrind/tests/rep_bulk.c b/callgrind/tests/rep_bulk.c new file mode 100644 index 0000000000..d94ae24c75 --- /dev/null +++ b/callgrind/tests/rep_bulk.c @@ -0,0 +1,128 @@ +/*--------------------------------------------------------------------*/ +/*--- Regression test for the CodSpeed REP bulk cost model. ---*/ +/*--------------------------------------------------------------------*/ + +/* Each case is one callgrind part: ZERO_STATS, one rep instruction, + * DUMP_STATS. The post-processing script checks the emitted counters and + * that no new event names appear, with the cache geometry pinned by the + * .vgtest file (64 byte lines). */ + + +#if defined(__x86_64__) + +#include "../callgrind.h" + +#define COPY_SZ 10240u /* 160 x 64B lines */ +#define BACK_SZ 4096u +#define STOS_SZ 4096u +#define SELF_SZ 4096u +#define FLUSH_SZ (256u * 1024u) + +static unsigned char srcbuf[COPY_SZ + 128] __attribute__((aligned(64))); +static unsigned char dstbuf[COPY_SZ + 128] __attribute__((aligned(64))); +static volatile unsigned char flushbuf[FLUSH_SZ] __attribute__((aligned(64))); + +/* Evict the source/destination lines from D1. */ +static void flush_d1(void) +{ + unsigned i; + for (i = 0; i < FLUSH_SZ; i += 64) + flushbuf[i] = (unsigned char)i; +} + +static int verify_copy(void) +{ + unsigned i; + for (i = 0; i < COPY_SZ; i++) + if (dstbuf[i] != srcbuf[i]) + return 0; + return 1; +} + +__attribute__((noinline)) +static void measure_copy(unsigned char *d, const unsigned char *s, unsigned long n) +{ + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("rep movsb" + : "+D"(d), "+S"(s), "+c"(n) + : "a"(0u) + : "memory"); + CALLGRIND_DUMP_STATS; +} + +int main(void) +{ + unsigned char *d; + const unsigned char *s; + unsigned long n; + unsigned i; + + /* Make both copy buffers D1-resident before the first measured part so + * the "hot" cases really have no misses. */ + for (i = 0; i < COPY_SZ; i++) { + srcbuf[i] = (unsigned char)(i * 7u); + dstbuf[i] = 0; + } + + /* part 0: one-byte forward copy (single element) */ + measure_copy(dstbuf, srcbuf, 1); + + /* part 1: hot 10 KiB forward copy, both buffers 64B-aligned */ + measure_copy(dstbuf, srcbuf, COPY_SZ); + if (!verify_copy()) + return 2; + + /* part 2: zero-length rep (no element may be charged) */ + measure_copy(dstbuf, srcbuf, 0); + if (!verify_copy()) + return 4; + + /* part 3: backward overlapping move (memmove pattern) */ + for (i = 0; i < BACK_SZ; i++) + dstbuf[i] = (unsigned char)(i * 13u + 1u); + d = dstbuf + 8 + BACK_SZ - 1; s = dstbuf + BACK_SZ - 1; n = BACK_SZ; + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("std\n\trep movsb\n\tcld" + : "+D"(d), "+S"(s), "+c"(n) + : "a"(0u) + : "memory", "cc"); + CALLGRIND_DUMP_STATS; + for (i = 0; i < BACK_SZ; i++) + if (dstbuf[i + 8] != (unsigned char)(i * 13u + 1u)) + return 5; + + /* part 4: hot rep stosb */ + d = dstbuf; s = srcbuf; n = STOS_SZ; + CALLGRIND_ZERO_STATS; + __asm__ __volatile__("rep stosb" + : "+D"(d), "+S"(s), "+c"(n) + : "a"((unsigned char)0x5a) + : "memory"); + CALLGRIND_DUMP_STATS; + for (i = 0; i < STOS_SZ; i++) + if (dstbuf[i] != 0x5a) + return 6; + + /* part 5: self copy (source and destination are the same buffer) */ + measure_copy(dstbuf, dstbuf, SELF_SZ); + for (i = 0; i < SELF_SZ; i++) + if (dstbuf[i] != 0x5a) + return 7; + + /* part 6: cold 10 KiB forward copy after flushing D1 */ + flush_d1(); + measure_copy(dstbuf, srcbuf, COPY_SZ); + if (!verify_copy()) + return 3; + + return 0; +} + +#else /* !__x86_64__ */ + +int main(void) +{ + return 0; +} + +#endif diff --git a/callgrind/tests/rep_bulk.post.exp b/callgrind/tests/rep_bulk.post.exp new file mode 100644 index 0000000000..b7d55c1b12 --- /dev/null +++ b/callgrind/tests/rep_bulk.post.exp @@ -0,0 +1,11 @@ +events: ok +one_byte_bulk_cost: ok +hot_lines_suppressed: ok +instruction_refs_kept: ok +bulk_cycles: ok +zero_no_startup: ok +backward: ok +stos: ok +self: ok +cold_miss_refs: ok +cache_independent_cycles: ok diff --git a/callgrind/tests/rep_bulk.stderr.exp b/callgrind/tests/rep_bulk.stderr.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/callgrind/tests/rep_bulk.vgtest b/callgrind/tests/rep_bulk.vgtest new file mode 100644 index 0000000000..0445cc0825 --- /dev/null +++ b/callgrind/tests/rep_bulk.vgtest @@ -0,0 +1,5 @@ +prereq: rm -f callgrind.rep_bulk.out* && test "$(uname -m)" = x86_64 +prog: rep_bulk +vgopts: -q --combine-dumps=yes --cache-sim=yes --cycle-estimation=yes --I1=32768,8,64 --D1=32768,8,64 --LL=8388608,16,64 --callgrind-out-file=callgrind.rep_bulk.out +post: perl ./rep_bulk_check.pl callgrind.rep_bulk.out on +cleanup: rm -f callgrind.rep_bulk.out callgrind.rep_bulk.out.* diff --git a/callgrind/tests/rep_bulk_check.pl b/callgrind/tests/rep_bulk_check.pl new file mode 100755 index 0000000000..2774023da9 --- /dev/null +++ b/callgrind/tests/rep_bulk_check.pl @@ -0,0 +1,125 @@ +#!/usr/bin/perl +# Check the callgrind output of the rep_bulk test. +# Usage: rep_bulk_check.pl on|off +use strict; +use warnings; + +my ($file, $mode) = @ARGV; +die "usage: rep_bulk_check.pl on|off\n" + unless defined $file && defined $mode && ($mode eq 'on' || $mode eq 'off'); + +open(my $fh, '<', $file) or die "cannot open $file: $!\n"; +my @events; +my @parts; +while (my $line = <$fh>) { + chomp $line; + if ($line =~ /^events:\s+(.*)$/) { + @events = split(' ', $1) unless @events; + } elsif ($line =~ /^summary:\s+(.*)$/) { + my @v = split(' ', $1); + my %p; + for my $i (0 .. $#events) { + $p{$events[$i]} = $v[$i]; + } + push @parts, \%p; + } +} +close($fh); + +my @fail; +sub check { + my ($name, $ok) = @_; + if ($ok) { + print "$name: ok\n"; + } else { + print "$name: MISMATCH\n"; + push @fail, $name; + } +} +sub val { + my ($part, $name) = @_; + die "part $part has no event $name\n" unless defined $parts[$part]{$name}; + return $parts[$part]{$name} + 0; +} + +my $expected_events = 'Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw Ct Cl'; +die "expected at least 7 dump parts, got ".scalar(@parts)."\n" if @parts < 7; + +check('events', join(' ', @events) eq $expected_events); + +# Deltas of a part against part 2 (zero-length rep: pure wrapper cost). +sub delta { my ($part, $name) = @_; return val($part, $name) - val(2, $name); } + +if ($mode eq 'on') { + check('one_byte_bulk_cost', val(0, 'Ct') > val(2, 'Ct')); + + check('hot_lines_suppressed', + delta(1, 'Dr') == 0 && delta(1, 'Dw') == 0 + && delta(1, 'D1mr') == 0 && delta(1, 'D1mw') == 0); + + # Instruction-side counters are never modeled: one element executes one + # more IMark than the zero-length dispatch (1 vs 2 dispatches). + check('instruction_refs_kept', + delta(0, 'Ir') == 1 && delta(1, 'Ir') == 10240); + + my $ct = delta(1, 'Ct'); + check('bulk_cycles', $ct > 0 && $ct < 300000 && delta(1, 'Cl') > 0); + + check('zero_no_startup', + val(2, 'Ct') < val(0, 'Ct') + && val(2, 'Ct') <= val(1, 'Ct') + && val(2, 'Ct') <= val(3, 'Ct') + && val(2, 'Ct') <= val(4, 'Ct') + && val(2, 'Ct') <= val(5, 'Ct')); + + check('backward', + val(3, 'Ct') > val(2, 'Ct') + && delta(3, 'Dr') == 0 && delta(3, 'Dw') == 0); + + check('stos', + val(4, 'Ct') > val(2, 'Ct') + && delta(4, 'Dr') == 0 && delta(4, 'Dw') == 0); + + check('self', + val(5, 'Ct') > val(2, 'Ct') + && delta(5, 'Dr') == 0 && delta(5, 'Dw') == 0); + + my $dr = delta(6, 'Dr'); + my $dw = delta(6, 'Dw'); + my $d1r = delta(6, 'D1mr'); + my $d1w = delta(6, 'D1mw'); + check('cold_miss_refs', + $dr >= 160 && $dr < 1024 && + $dw >= 160 && $dw < 1024 && + $d1r >= 160 && $d1r < 1024 && + $d1w >= 160 && $d1w < 1024); + + check('cache_independent_cycles', + val(1, 'Ct') == val(6, 'Ct') && val(1, 'Cl') == val(6, 'Cl')); +} else { + check('element_refs', delta(0, 'Dr') == 1 && delta(0, 'Dw') == 1); + + check('legacy_hot_refs', delta(1, 'Dr') == 10240 && delta(1, 'Dw') == 10240); + + check('legacy_cycles', delta(1, 'Ct') > 500000); + + check('legacy_zero_dispatch', delta(0, 'Ct') >= 100); + + check('legacy_backward_refs', delta(3, 'Dr') == 4096 && delta(3, 'Dw') == 4096); + + check('legacy_stos_refs', delta(4, 'Dr') == 0 && delta(4, 'Dw') == 4096); + + + my $dr = delta(6, 'Dr'); + my $d1r = delta(6, 'D1mr'); + my $d1w = delta(6, 'D1mw'); + check('legacy_cold_refs', + $dr >= 10240 && $d1r >= 160 && $d1r < 1024 && + $d1w >= 160 && $d1w < 1024); +} + +if (@fail) { + print "FAILED: ", join(', ', @fail), "\n"; + exit 1; +} +exit 0; diff --git a/callgrind/tests/rep_bulk_legacy.post.exp b/callgrind/tests/rep_bulk_legacy.post.exp new file mode 100644 index 0000000000..192e7303e5 --- /dev/null +++ b/callgrind/tests/rep_bulk_legacy.post.exp @@ -0,0 +1,8 @@ +events: ok +element_refs: ok +legacy_hot_refs: ok +legacy_cycles: ok +legacy_zero_dispatch: ok +legacy_backward_refs: ok +legacy_stos_refs: ok +legacy_cold_refs: ok diff --git a/callgrind/tests/rep_bulk_legacy.stderr.exp b/callgrind/tests/rep_bulk_legacy.stderr.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/callgrind/tests/rep_bulk_legacy.vgtest b/callgrind/tests/rep_bulk_legacy.vgtest new file mode 100644 index 0000000000..b052754ca8 --- /dev/null +++ b/callgrind/tests/rep_bulk_legacy.vgtest @@ -0,0 +1,5 @@ +prereq: rm -f callgrind.rep_bulk.out* && test "$(uname -m)" = x86_64 +prog: rep_bulk +vgopts: -q --combine-dumps=yes --cache-sim=yes --cycle-estimation=yes --rep-bulk-cost=no --I1=32768,8,64 --D1=32768,8,64 --LL=8388608,16,64 --callgrind-out-file=callgrind.rep_bulk.out +post: perl ./rep_bulk_check.pl callgrind.rep_bulk.out off +cleanup: rm -f callgrind.rep_bulk.out callgrind.rep_bulk.out.* diff --git a/callgrind/threads.c b/callgrind/threads.c index 642813e4cb..36c96ae8cd 100644 --- a/callgrind/threads.c +++ b/callgrind/threads.c @@ -441,6 +441,11 @@ void CLG_(init_exec_state)(exec_state* es) es->jmps_passed = 0; es->bbcc = 0; es->nonskipped = 0; + es->rep_ii = 0; + es->rep_next_rem = 0; + es->rep_dir = 0; + es->rep_align = 0; + es->rep_tail_carry = 0; } @@ -540,6 +545,11 @@ exec_state* exec_state_save(void) es->jmps_passed = CLG_(current_state).jmps_passed; es->bbcc = CLG_(current_state).bbcc; es->nonskipped = CLG_(current_state).nonskipped; + es->rep_ii = CLG_(current_state).rep_ii; + es->rep_next_rem = CLG_(current_state).rep_next_rem; + es->rep_dir = CLG_(current_state).rep_dir; + es->rep_align = CLG_(current_state).rep_align; + es->rep_tail_carry = CLG_(current_state).rep_tail_carry; CLG_ASSERT(es->cost == CLG_(current_state).cost); CLG_DEBUGIF(1) { @@ -565,6 +575,11 @@ exec_state* exec_state_restore(void) CLG_(current_state).jmps_passed = es->jmps_passed; CLG_(current_state).bbcc = es->bbcc; CLG_(current_state).nonskipped = es->nonskipped; + CLG_(current_state).rep_ii = es->rep_ii; + CLG_(current_state).rep_next_rem = es->rep_next_rem; + CLG_(current_state).rep_dir = es->rep_dir; + CLG_(current_state).rep_align = es->rep_align; + CLG_(current_state).rep_tail_carry = es->rep_tail_carry; CLG_(current_state).cost = es->cost; CLG_(current_state).sig = es->sig; From a315af8dd41d31d77958297ca65549a91c4ab873 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 2 Nov 2026 19:56:15 +0100 Subject: [PATCH 2/2] docs(callgrind): clarify calibrated alignment class Describe the measured offset-eight class precisely rather than implying every matching unaligned pointer pair selects that curve. --- callgrind/rep_bulk.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/callgrind/rep_bulk.h b/callgrind/rep_bulk.h index 182a4971b1..2468c4a6d7 100644 --- a/callgrind/rep_bulk.h +++ b/callgrind/rep_bulk.h @@ -41,7 +41,7 @@ * kept for the whole operation, so address changes between elements cannot * change the curve mid-instruction: * 0 aligned: MOVS src%64==0 && dst%64==0; STOS dst%64==0 - * 1 same_unaligned: MOVS src%64==dst%64 (nonzero); STOS dst%64==8 + * 1 same_unaligned: MOVS src%64==8 && dst%64==8; STOS dst%64==8 * 2 other/split: everything else (conservative envelope) */ #define CLG_REP_ALIGN_ALIGNED 0