fix(callgrind): model REP copies with calibrated bulk costs

CODSPEED-CHANGELOG.md

@@ -23,6 +23,30 @@ matching Linux on AArch64.
 
 ## Features
 
+### REP bulk cost model for `rep movs` / `rep stos`
+
+amd64 `rep movs{b,w,d,q}` and `rep stos{b,w,d,q}` are now priced from a
+calibrated hot-copy bulk curve instead of the per-element instruction/access
+model. It is enabled by default when both `--cache-sim=yes` and
+`--cycle-estimation=yes` are active; `--rep-bulk-cost=no` restores the previous
+behaviour. No event names or counters are added, and disabled cache or cycle
+modes are unchanged.
+
+For a modeled REP instruction, `Ct` and `Cl` carry the calibrated cost.
+Instruction-side counters (`Ir`/`I1mr`/`ILmr`) are unchanged. On the data side,
+only accesses that miss in L1 are reported as `Dr`/`Dw`, together with their
+`D1mr`/`D1mw` and `DLmr`/`DLmw` misses: per-element data L1 hit charges
+disappear while every real miss is still counted. The cache simulation, the
+counters of all non-REP accesses and instructions, and the output format are
+unchanged.
+
+The table is calibrated for disjoint-buffer MOVS/STOS with cache-resident
+working sets: MOVS up to 16 KiB and STOS up to 32 KiB per element width, with
+the last measured marginal slope used beyond that. Overlapping MOVS and
+L2/L3-resident bulk copies are outside the calibrated domain.
+This changes REP pricing only, not the existing scalar/vector cost model.
+Relative REP-versus-vector scores are not calibrated native speedup estimates.
+
 ### Fixed Linux startup stack layout
 
 Reduce simulated cache-miss variation from argument and environment sizes

callgrind/Makefile.am

@@ -22,11 +22,14 @@ noinst_HEADERS = \
 	cycledecode_capstone.h \
 	sigkey.h \
 	events.h \
-	global.h
+	global.h \
+	rep_bulk.h
 
 # Generated cost tables (#included by cycledecode.c under CLG_WITH_CAPSTONE;
-# the arch is selected at compile time).
-EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc
+# the arch is selected at compile time) and the calibrated REP bulk curve
+# table included by rep_bulk.c (generated by the instruction cost calibration
+# pipeline).
+EXTRA_DIST += x86_caps_lut.inc arm64_caps_lut.inc rep_bulk_table.inc
 
 #----------------------------------------------------------------------------
 # callgrind-<platform>
@@ -52,6 +55,7 @@ CALLGRIND_SOURCES_COMMON = \
 	fn.c \
 	jumps.c \
 	main.c \
+	rep_bulk.c \
 	sim.c \
 	subprocess.c \
 	threads.c

callgrind/clo.c

@@ -568,6 +568,7 @@ Bool CLG_(process_cmd_line_option)(const HChar* arg)
    else if VG_BOOL_CLO(arg, "--branch-sim",      CLG_(clo).simulate_branch) {}
 
    else if VG_BOOL_CLO(arg, "--cycle-estimation", CLG_(clo).cycle_estimation) {}
+   else if VG_BOOL_CLO(arg, "--rep-bulk-cost",    CLG_(clo).rep_bulk_cost) {}
    else {
        Bool isCachesimOption = (*CLG_(cachesim).parse_opt)(arg);
 
@@ -617,6 +618,11 @@ void CLG_(print_usage)(void)
 "    --collect-jumps=no|yes    Collect jumps? [no]\n"
 "    --collect-bus=no|yes      Collect global bus events? [no]\n"
 "    --cycle-estimation=no|yes Estimate per-instruction cycles (Ct/Cl)? [no]\n"
+"    --rep-bulk-cost=no|yes    Model supported rep movs/stos by a calibrated\n"
+"                              bulk curve (opt-out for A/B) [yes]\n"
+"        Active only with --cache-sim=yes and --cycle-estimation=yes; the\n"
+"        raw L1-hit refs of a modeled rep are replaced by its bulk cost while\n"
+"        misses stay reported. See docs/cl-manual.xml.\n"
 #if CLG_EXPERIMENTAL
 "    --collect-alloc=no|yes    Collect memory allocation info? [no]\n"
 #endif
@@ -704,6 +710,7 @@ void CLG_(set_clo_defaults)(void)
   CLG_(clo).simulate_cache = False;
   CLG_(clo).simulate_branch = False;
   CLG_(clo).cycle_estimation = False;
+  CLG_(clo).rep_bulk_cost = True;
 
   /* Call graph */
   CLG_(clo).pop_on_jump = False;

callgrind/cycledecode.c

@@ -351,3 +351,99 @@ void clg_cycledecode_stats(ClgCdStats* out)
    "Callgrind cycle estimation requires Capstone; configure with --with-capstone=PATH (or set CAPSTONE_DIR)."
 
 #endif
+
+#if defined(VGA_amd64) || defined(VGA_x86)
+
+/*------------------------------------------------------------*/
+/*--- REP string instruction classification                 ---*/
+/*------------------------------------------------------------*/
+
+/* CodSpeed extension, see cycledecode.h. Plain x86/amd64 encoding check: the
+ * IMark bytes of one instruction, prefixes followed by A4/A5/AA/AB. F2
+ * (REPNE) is rejected so only forward-countable bulk copies are classified;
+ * this never has to agree with VEX's decoder on anything else. */
+void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len,
+                           ClgRepBulkInfo* out)
+{
+   unsigned i = 0;
+   int      rep = 0, opsz16 = 0, addr32p = 0;
+   int      rex_w = 0, rex_seen = 0;
+   unsigned char opc;
+   unsigned width;
+
+   out->op = CLG_REP_OP_NONE;
+   out->elem_width = 0;
+   out->addr32 = 0;
+   if (!bytes || len < 2 || len > 15)
+      return;
+
+   while (i < len) {
+      unsigned char c = bytes[i];
+      if (c == 0xF3 && !rep) {
+         rep = 1;
+         i++;
+         continue;
+      }
+      if (c == 0xF2)
+         return; /* REPNE: never a supported bulk copy */
+      if (c == 0x66 && !opsz16) {
+         opsz16 = 1;
+         i++;
+         continue;
+      }
+      if (c == 0x67 && !addr32p) {
+         addr32p = 1;
+         i++;
+         continue;
+      }
+      if (c >= 0x40 && c <= 0x4F && !rex_seen) {
+         rex_seen = 1;
+         rex_w = (c & 0x08) != 0;
+         i++;
+         continue;
+      }
+      break;
+   }
+
+   /* Exactly the opcode must follow the prefix run: no operands, no
+    * trailing bytes (the IMark length always covers the whole insn). */
+   if (!rep || i + 1 != len)
+      return;
+
+   opc = bytes[i];
+   switch (opc) {
+   case 0xA4: /* movsb */
+      width = 1;
+      out->op = CLG_REP_OP_MOVS;
+      break;
+   case 0xA5: /* movsw/movsd/movsq */
+      width = rex_w ? 8 : (opsz16 ? 2 : 4);
+      out->op = CLG_REP_OP_MOVS;
+      break;
+   case 0xAA: /* stosb */
+      width = 1;
+      out->op = CLG_REP_OP_STOS;
+      break;
+   case 0xAB: /* stosw/stosd/stosq */
+      width = rex_w ? 8 : (opsz16 ? 2 : 4);
+      out->op = CLG_REP_OP_STOS;
+      break;
+   default:
+      return;
+   }
+
+   out->elem_width = (unsigned char)width;
+   out->addr32 = addr32p ? 1 : 0;
+}
+
+#else
+
+void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len,
+                           ClgRepBulkInfo* out)
+{
+   out->op = CLG_REP_OP_NONE;
+   out->elem_width = 0;
+   out->addr32 = 0;
+}
+
+#endif

callgrind/cycledecode.h

@@ -53,4 +53,32 @@ void clg_cycle_cost(const unsigned char* bytes,
 /* Per-tier diagnostic counters since init. */
 void clg_cycledecode_stats(ClgCdStats* out);
 
+/*------------------------------------------------------------*/
+/*--- REP string instruction classification                 ---*/
+/*------------------------------------------------------------*/
+
+/* CodSpeed extension, used by the REP bulk cost model. This is a plain
+ * x86/amd64 encoding check, independent of Capstone and of the cycle table:
+ * only rep-prefixed MOVS and STOS are classified, because those are the only
+ * string forms whose per-element cost can be replaced by a bulk copy curve.
+ * CMPS/SCAS/LODS/INS/OUTS, F2 (REPNE) prefixed insns and non-rep MOVS/STOS
+ * are reported as CLG_REP_OP_NONE so the legacy model is kept for them. */
+
+typedef enum {
+   CLG_REP_OP_NONE = 0,
+   CLG_REP_OP_MOVS = 1,  /* rep movs{b,w,d,q}: one read + one write per element */
+   CLG_REP_OP_STOS = 2   /* rep stos{b,w,d,q}: one write per element */
+} ClgRepBulkOp;
+
+typedef struct {
+   unsigned char op;         /* ClgRepBulkOp                            */
+   unsigned char elem_width; /* element size in bytes (1,2,4,8), 0 if none */
+   unsigned char addr32;     /* 67 prefix: the repeat count is 32-bit   */
+} ClgRepBulkInfo;
+
+/* Classify the guest instruction in [bytes, bytes+len). Always initialises
+ * *out; *out->op == CLG_REP_OP_NONE when not a supported bulk form. */
+void clg_rep_bulk_classify(const unsigned char* bytes, unsigned len,
+                           ClgRepBulkInfo* out);
+
 #endif /* CLG_CYCLEDECODE_H */

callgrind/docs/cl-manual.xml

@@ -995,6 +995,47 @@ Also see <xref linkend="cl-manual.cycles"/>.</para>
     </listitem>
   </varlistentry>
 
+  <varlistentry id="clopt.rep-bulk-cost" xreflabel="--rep-bulk-cost">
+    <term>
+      <option><![CDATA[--rep-bulk-cost=<yes|no> [default: yes] ]]></option>
+    </term>
+    <listitem>
+      <para>Model amd64 <computeroutput>rep movs</computeroutput> and
+      <computeroutput>rep stos</computeroutput> string instructions with a
+      calibrated bulk cost curve instead of the per-element model. The option
+      is active only when both <option>--cache-sim=yes</option> and
+      <option>--cycle-estimation=yes</option> are given; otherwise it is
+      inert and the output is unchanged. It is also disabled for
+      <option>--simulate-wb=yes</option>, <option>--cacheuse=yes</option> and
+      non-amd64 guests.</para>
+      <para>For a modeled instruction the estimated cycles ("Ct"/"Cl") come
+      from a calibration table measured on hot, cache-resident data, and both
+      counters carry the same value so a geometric-mean consumer recovers it.
+      Instruction-side counters are unchanged: a modeled REP still reports one
+      instruction reference per element (the VEX self-loop) and its real
+      instruction misses. On the data side, only accesses that miss in L1 are
+      reported, as one "Dr"/"Dw" reference together with its "D1mr"/"D1mw" and,
+      where applicable, "DLmr"/"DLmw" miss; data L1 hits are not reported
+      because the bulk curve already covers hot data movement. Thus Dr equals
+      D1mr and Dw equals D1mw for modeled REP instructions: these are modeled
+      data references, not raw per-element accesses. The cache simulation
+      itself is unchanged. No event names are added, so
+      <option>--rep-bulk-cost=no</option> reproduces the previous per-element
+      model exactly.</para>
+      <para>The calibration covers disjoint-buffer MOVS/STOS with
+      cache-resident working sets up to a per-width calibrated maximum
+      (16 KiB for MOVS, 32 KiB for STOS); larger copies are extrapolated with
+      the last measured marginal slope, and overlapping MOVS may take a
+      slower hardware path that the table does not model. Accuracy for
+      cache-miss-heavy or L2/L3-resident bulk copies is therefore limited. On
+      lines or functions that mix modeled REP instructions with other
+      instructions the consumer's geometric mean of the summed Ct/Cl is an
+      approximation, not an exact decomposition. Scalar and vector instruction
+      pricing is unchanged, so relative REP-versus-vector scores are not
+      calibrated native speedup estimates.</para>
+    </listitem>
+  </varlistentry>
+
   <varlistentry id="clopt.branch-sim" xreflabel="--branch-sim">
     <term>
       <option><![CDATA[--branch-sim=<yes|no> [default: no] ]]></option>

callgrind/global.h

@@ -128,6 +128,7 @@ struct _CommandLineOptions {
   Bool simulate_cache;      /* Call into cache simulator ? */
   Bool simulate_branch;     /* Call into branch prediction simulator ? */
   Bool cycle_estimation;    /* Estimate per-instruction cycles (Ct/Cl) ? */
+  Bool rep_bulk_cost;       /* Replace per-element REP cost by bulk curve ? */
 
   /* Call graph generation */
   Bool pop_on_jump;       /* Handle a jump between functions as ret+call */
@@ -279,6 +280,12 @@ struct _InstrInfo {
   UInt ct_incl;
   UInt cl_cost;
   UInt cl_incl;
+  /* REP bulk cost model classification (see cycledecode.h). Only filled on
+   * first translation and only when the model is active; rep_bulk_op == 0
+   * means the instruction is not a supported rep-prefixed bulk copy. */
+  UChar rep_bulk_op;
+  UChar rep_elem_width;
+  UChar rep_addr32;
 };
 
 
@@ -528,6 +535,16 @@ struct _exec_state {
   BBCC* nonskipped;
 
   Int call_stack_bottom; /* Index into fn_stack */
+
+  /* REP bulk cost model: alignment class and direction are latched once per
+   * REP operation from its first element, and the expected remaining count
+   * detects the next operation. Saved/restored on thread and signal
+   * switches so an interrupted REP keeps its class. */
+  InstrInfo* rep_ii;       /* instruction site of the running operation */
+  ULong      rep_next_rem; /* expected remaining count after last element */
+  UChar      rep_dir;      /* CLG_REP_DIR_* latched at operation start */
+  UChar      rep_align;    /* CLG_REP_ALIGN_* latched at operation start */
+  ULong      rep_tail_carry; /* remainder of the tail slope, per operation */
 };
 
 /* Global state structures */
@@ -687,10 +704,19 @@ struct cachesim_if
     void (*log_0I1Dr)(InstrInfo*, Addr, Word) VG_REGPARM(3);
     void (*log_0I1Dw)(InstrInfo*, Addr, Word) VG_REGPARM(3);
 
+    /* REP bulk cost model: called once per executed element of a supported
+     * rep-prefixed bulk copy. Args: inode, repeat count after the decrement
+     * of this element, source address (0 if none), destination address,
+     * packed op|elem_width<<8, guest DF flag. Zero when the model is off.
+     * Uses the ordinary amd64 calling convention (regparms=0), not a
+     * register-parm ABI. */
+    void (*log_rep_bulk)(InstrInfo*, ULong, Addr, Addr, Word, Word);
+
     // function names of helpers (for debugging generated code)
     const HChar *log_1I0D_name, *log_2I0D_name, *log_3I0D_name;
     const HChar *log_1I1Dr_name, *log_1I1Dw_name;
     const HChar *log_0I1Dr_name, *log_0I1Dw_name;
+    const HChar *log_rep_bulk_name;
 };
 
 // Event groups