Randomize per-thread recording order to avoid concurrency hotspots

sdk/all/src/jmh/java/io/opentelemetry/sdk/MetricRecordBenchmark.java

@@ -30,6 +30,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Fork;
 import org.openjdk.jmh.annotations.Group;
@@ -73,6 +74,13 @@
  * record activities over more distinct series. The highest contention scenario is cardinality=1,
  * threads=4. Any scenario with threads=1 has zero contention.
  *
+ * <p>To make the cardinality dimension meaningful under contention, each thread traverses the
+ * series in its own independent (per-thread shuffled) order — see {@link ThreadState}. This models
+ * independent threads recording to arbitrary series. A naive shared {@code i % cardinality} index
+ * would instead march all threads through the same series in lockstep (contention self-synchronizes
+ * them), collapsing high-cardinality multi-thread runs into a single rotating hotspot that behaves
+ * like cardinality=1.
+ *
  * <p>It's useful to characterize the performance of the metrics system under contention, as some
  * high-performance applications may have many threads trying to record to the same series. It's
  * also useful to characterize the performance of the metrics system under low contention, as some
@@ -125,6 +133,9 @@ public static class BenchmarkState {
     List<Attributes> attributesList;
     Span span;
     io.opentelemetry.context.Scope contextScope;
+    // Hands out a distinct seed to each recording thread's ThreadState so threads traverse the
+    // series in independent orders (see ThreadState).
+    final AtomicInteger threadSeedSequence = new AtomicInteger();
 
     @Setup
     @SuppressWarnings("MustBeClosedChecker")
@@ -184,15 +195,47 @@ public void tearDown() {
     }
   }
 
+  /**
+   * Per-thread series traversal order. Each recording thread shuffles {@code [0, cardinality)} with
+   * a distinct seed, so that at any given record the threads are recording to <em>different</em>
+   * series rather than marching through the same series in lockstep. Without this, the shared
+   * sequential {@code i % cardinality} index plus contention's self-synchronizing effect collapses
+   * the high-cardinality, multi-thread cases into a single rotating hotspot (effectively
+   * cardinality=1 contention), which does not reflect real-world recording where independent
+   * threads touch arbitrary series.
+   */
+  @State(Scope.Thread)
+  public static class ThreadState {
+    int[] order;
+
+    @Setup
+    public void setup(BenchmarkState benchmarkState) {
+      int cardinality = benchmarkState.cardinality;
+      order = new int[cardinality];
+      for (int i = 0; i < cardinality; i++) {
+        order[i] = i;
+      }
+      // Distinct seed per thread => independent permutations => no cross-thread lockstep.
+      Random random =
+          new Random(INITIAL_SEED + benchmarkState.threadSeedSequence.getAndIncrement());
+      for (int i = cardinality - 1; i > 0; i--) {
+        int j = random.nextInt(i + 1);
+        int tmp = order[i];
+        order[i] = order[j];
+        order[j] = tmp;
+      }
+    }
+  }
+
   @Benchmark
   @Group("threads1")
   @GroupThreads(1)
   @Fork(3)
   @Warmup(iterations = 3, time = 1)
   @Measurement(iterations = 10, time = 1)
   @OperationsPerInvocation(RECORDS_PER_INVOCATION)
-  public void record_SingleThread(BenchmarkState benchmarkState) {
-    record(benchmarkState);
+  public void record_SingleThread(BenchmarkState benchmarkState, ThreadState threadState) {
+    record(benchmarkState, threadState);
   }
 
   @Benchmark
@@ -202,14 +245,15 @@ public void record_SingleThread(BenchmarkState benchmarkState) {
   @Warmup(iterations = 3, time = 1)
   @Measurement(iterations = 10, time = 1)
   @OperationsPerInvocation(RECORDS_PER_INVOCATION)
-  public void record_MultipleThreads(BenchmarkState benchmarkState) {
-    record(benchmarkState);
+  public void record_MultipleThreads(BenchmarkState benchmarkState, ThreadState threadState) {
+    record(benchmarkState, threadState);
   }
 
-  private static void record(BenchmarkState benchmarkState) {
+  private static void record(BenchmarkState benchmarkState, ThreadState threadState) {
+    // Per-thread series order: at a given i, different threads hit different series (no lockstep).
+    int[] order = threadState.order;
     for (int i = 0; i < RECORDS_PER_INVOCATION; i++) {
-      Attributes attributes =
-          benchmarkState.attributesList.get(i % benchmarkState.attributesList.size());
+      Attributes attributes = benchmarkState.attributesList.get(order[i % order.length]);
       long value = benchmarkState.measurements.get(i % benchmarkState.measurements.size());
       benchmarkState.instrument.record(value, attributes);
     }