Bring the 9.1.0 changes to main and start 9.2.0-SNAPSHOT

NOTICE

@@ -1,5 +1,5 @@
 Apache DataSketches Java
-Copyright 2025 The Apache Software Foundation
+Copyright 2019-2026 The Apache Software Foundation
 
 Copyright 2015-2018 Yahoo Inc.
 Copyright 2019-2020 Verizon Media

pom.xml

@@ -27,7 +27,7 @@ under the License.
 
   <groupId>org.apache.datasketches</groupId>
   <artifactId>datasketches-java</artifactId>
-  <version>9.1.0-SNAPSHOT</version>
+  <version>9.2.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <name>${project.artifactId}</name>
@@ -53,9 +53,9 @@ under the License.
     </mailingList>
     <mailingList>
       <name>DataSketches Users</name>
-      <subscribe>user-subscribe@datasketches.apache.org</subscribe>
-      <unsubscribe>user-unsubscribe@datasketches.apache.org</unsubscribe>
-      <post>user@datasketches.apache.org</post>
+      <subscribe>users-subscribe@datasketches.apache.org</subscribe>
+      <unsubscribe>users-unsubscribe@datasketches.apache.org</unsubscribe>
+      <post>users@datasketches.apache.org</post>
       <archive>https://mail-archives.apache.org/mod_mbox/datasketches-user</archive>
     </mailingList>
   </mailingLists>
@@ -574,6 +574,8 @@ under the License.
             </execution>
           </executions>
           <configuration>
+            <!-- Maven commands are sometimes run on the release artifact directly, which is not a Git repository -->
+            <failOnNoGitDirectory>false</failOnNoGitDirectory>
             <generateGitPropertiesFile>true</generateGitPropertiesFile>
             <generateGitPropertiesFilename>${project.basedir}/git.properties</generateGitPropertiesFilename>
             <includeOnlyProperties>
@@ -606,7 +608,7 @@ under the License.
               <inputExclude>**/*.yml</inputExclude>
               <inputExclude>**/.*</inputExclude>
               <inputExclude>**/test-output/**/*</inputExclude>
-              <!-- <inputExclude>**/git.properties</inputExclude> -->
+              <inputExclude>**/git.properties</inputExclude>
               <inputExclude>**/doc/**</inputExclude>
               <inputExclude>**/docs/**</inputExclude>
               <inputExclude>**/*.sk</inputExclude>

src/main/java/org/apache/datasketches/common/ArrayOfNumbersSerDe.java

@@ -127,6 +127,7 @@ public Number[] deserializeFromMemorySegment(final MemorySegment seg, final int
   public Number[] deserializeFromMemorySegment(final MemorySegment seg, final long offsetBytes, final int numItems) {
     Objects.requireNonNull(seg, "MemorySegment must not be null");
     if (numItems <= 0) { return new Number[0]; }
+    Util.checkBounds(offsetBytes, numItems, seg.byteSize()); //each item has at least a 1-byte type code
     final Number[] array = new Number[numItems];
     long offset = offsetBytes;
     for (int i = 0; i < numItems; i++) {

src/main/java/org/apache/datasketches/count/CountMinSketch.java

@@ -441,6 +441,8 @@ public byte[] toByteArray() {
    * @return The deserialized CountMinSketch.
    */
   public static CountMinSketch deserialize(final byte[] b, final long seed) {
+    final int preambleBytes = Family.COUNTMIN.getMinPreLongs() * Long.BYTES;
+    Util.checkBounds(0, preambleBytes, b.length);
     final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(b));
 
     final byte preambleLongs = posSeg.getByte();
@@ -476,8 +478,12 @@ public static CountMinSketch deserialize(final byte[] b, final long seed) {
           + Util.computeSeedHash(seed));
     }
 
-    final CountMinSketch cms = new CountMinSketch(numHashes, numBuckets, seed);
     final boolean empty = (flagsByte & Flag.IS_EMPTY.mask()) > 0;
+    if (!empty) { //the image must hold the total weight and the whole sketch array
+      final long arrayBytes = (long) numHashes * numBuckets * Long.BYTES;
+      Util.checkBounds(preambleBytes, Long.BYTES + Math.max(arrayBytes, 0), b.length);
+    }
+    final CountMinSketch cms = new CountMinSketch(numHashes, numBuckets, seed);
     if (empty) {
       return cms;
     }

src/main/java/org/apache/datasketches/cpc/CompressedState.java

@@ -49,6 +49,7 @@
 
 import java.lang.foreign.MemorySegment;
 
+import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.common.Util;
 
 /**
@@ -115,7 +116,7 @@ int getWindowOffset() {
   long getRequiredSerializedBytes() {
     final Format format = getFormat();
     final int preInts = getDefinedPreInts(format);
-    return 4L * (preInts + csvLengthInts + cwLengthInts);
+    return 4L * ((long) preInts + csvLengthInts + cwLengthInts); //long arithmetic, so it cannot overflow
   }
 
   static CompressedState importFromSegment(final MemorySegment seg) {
@@ -216,10 +217,28 @@ static CompressedState importFromSegment(final MemorySegment seg) {
       }
     }
     checkCapacity(seg.byteSize(),
-        4L * (getPreInts(seg) + state.csvLengthInts + state.cwLengthInts));
+        4L * ((long) getPreInts(seg) + state.csvLengthInts + state.cwLengthInts));
+    checkCounts(state);
     return state;
   }
 
+  /**
+   * Checks the coupon and surprising-value counts read from an image, before they are used to size
+   * the arrays that hold the uncompressed values. The matrix has k rows of 64 columns, so there are
+   * at most 64k coupons. The surprising values are a subset of the coupons, and each one takes at
+   * least one bit of the compressed stream.
+   * @param state the imported state
+   */
+  private static void checkCounts(final CompressedState state) {
+    final long k = 1L << state.lgK;
+    if ((state.numCoupons < 0) || (state.numCoupons > (64L * k))
+        || (state.numCsv < 0) || (state.numCsv > state.numCoupons)
+        || (state.numCsv > (32L * state.csvLengthInts))) {
+      throw new SketchesArgumentException("Possible corruption: numCoupons: " + state.numCoupons
+          + ", numSv: " + state.numCsv + ", svLengthInts: " + state.csvLengthInts);
+    }
+  }
+
   void exportToSegment(final MemorySegment wseg) {
     final Format format = getFormat();
     switch (format) {

src/main/java/org/apache/datasketches/cpc/PreambleUtil.java

@@ -375,6 +375,7 @@ static long getWStreamOffset(final MemorySegment seg) {
   static int[] getSvStream(final MemorySegment seg) {
     final long offset = getSvStreamOffset(seg);
     final int svLengthInts = getSvLengthInts(seg);
+    checkBounds(offset, (long) svLengthInts * Integer.BYTES, seg.byteSize()); //before allocating
     final int[] svStream = new int[svLengthInts];
     MemorySegment.copy(seg, JAVA_INT_UNALIGNED, offset, svStream, 0, svLengthInts);
     return svStream;
@@ -383,6 +384,7 @@ static int[] getSvStream(final MemorySegment seg) {
   static int[] getWStream(final MemorySegment seg) {
     final long offset = getWStreamOffset(seg);
     final int wLength = getWLengthInts(seg);
+    checkBounds(offset, (long) wLength * Integer.BYTES, seg.byteSize()); //before allocating
     final int[] wStream = new int[wLength];
     MemorySegment.copy(seg, JAVA_INT_UNALIGNED, offset, wStream, 0, wLength);
     return wStream;

src/main/java/org/apache/datasketches/filters/bloomfilter/HeapBitArray.java

@@ -78,6 +78,10 @@ static HeapBitArray heapify(final PositionalSegment posSeg, final boolean isEmpt
     // will be -1 if dirty
     final long numBitsSet = posSeg.getLong();
 
+    if (posSeg.getRemaining() < ((long) numLongs * Long.BYTES)) {
+      throw new SketchesArgumentException("Possible corruption: bit array of " + numLongs
+          + " longs does not fit in the remaining " + posSeg.getRemaining() + " bytes");
+    }
     final long[] data = new long[numLongs];
     posSeg.getLongArray(data, 0, numLongs);
     return new HeapBitArray(numBitsSet, data);

src/main/java/org/apache/datasketches/frequencies/FrequentItemsSketch.java

@@ -43,6 +43,7 @@
 import static org.apache.datasketches.frequencies.PreambleUtil.insertSerVer;
 import static org.apache.datasketches.frequencies.Util.LG_MIN_MAP_SIZE;
 import static org.apache.datasketches.frequencies.Util.SAMPLE_SIZE;
+import static org.apache.datasketches.frequencies.Util.checkMapSizes;
 import static org.apache.datasketches.frequencies.Util.checkStreamWeight;
 
 import java.lang.foreign.MemorySegment;
@@ -275,17 +276,19 @@ public static <T> FrequentItemsSketch<T> getInstance(final MemorySegment srcSeg,
     MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, 0, preArr, 0, preLongs);
     checkStreamWeight(preArr[2]);
 
-    final FrequentItemsSketch<T> fis = new FrequentItemsSketch<>(lgMaxMapSize, lgCurMapSize);
-    fis.streamWeight = 0; //update after
-    fis.offset = preArr[3];
-
     final int preBytes = preLongs << 3;
     final int activeItems = extractActiveItems(preArr[1]);
+    final int lgStartMapSize =
+        checkMapSizes(lgMaxMapSize, lgCurMapSize, activeItems, ReversePurgeItemHashMap.getLoadFactor());
+    final long reqBytes = (preBytes + ((long)activeItems * Long.BYTES)); //count Arr only
+    checkBounds(0, reqBytes, srcSeg.byteSize()); //check MemorySegment capacity before allocating
+
+    final FrequentItemsSketch<T> fis = new FrequentItemsSketch<>(lgMaxMapSize, lgStartMapSize);
+    fis.streamWeight = 0; //update after
+    fis.offset = preArr[3];
 
     //Get countArray
     final long[] countArray = new long[activeItems];
-    final long reqBytes = (preBytes + ((long)activeItems * Long.BYTES)); //count Arr only
-    checkBounds(0, reqBytes, srcSeg.byteSize()); //check MemorySegment capacity
 
     MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preBytes, countArray, 0, activeItems);
 

src/main/java/org/apache/datasketches/frequencies/FrequentLongsSketch.java

@@ -42,6 +42,7 @@
 import static org.apache.datasketches.frequencies.PreambleUtil.insertSerVer;
 import static org.apache.datasketches.frequencies.Util.LG_MIN_MAP_SIZE;
 import static org.apache.datasketches.frequencies.Util.SAMPLE_SIZE;
+import static org.apache.datasketches.frequencies.Util.checkMapSizes;
 import static org.apache.datasketches.frequencies.Util.checkStreamWeight;
 
 import java.lang.foreign.MemorySegment;
@@ -278,17 +279,19 @@ public static FrequentLongsSketch getInstance(final MemorySegment srcSeg) {
     MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, 0, preArr, 0, preLongs);
     checkStreamWeight(preArr[2]);
 
-    final FrequentLongsSketch fls = new FrequentLongsSketch(lgMaxMapSize, lgCurMapSize);
-    fls.streamWeight = 0; //update after
-    fls.offset = preArr[3];
-
     final int preBytes = preLongs << 3;
     final int activeItems = extractActiveItems(preArr[1]);
+    final int lgStartMapSize =
+        checkMapSizes(lgMaxMapSize, lgCurMapSize, activeItems, ReversePurgeLongHashMap.getLoadFactor());
+    final long reqBytes = preBytes + (2L * activeItems * Long.BYTES); //count Arr + Items Arr
+    checkBounds(0, reqBytes, srcSeg.byteSize()); //check MemorySegment capacity before allocating
+
+    final FrequentLongsSketch fls = new FrequentLongsSketch(lgMaxMapSize, lgStartMapSize);
+    fls.streamWeight = 0; //update after
+    fls.offset = preArr[3];
 
     //Get countArray
     final long[] countArray = new long[activeItems];
-    final int reqBytes = preBytes + (2 * activeItems * Long.BYTES); //count Arr + Items Arr
-    checkBounds(0, reqBytes, srcSeg.byteSize()); //check MemorySegment capacity
     MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preBytes, countArray, 0, activeItems);
 
     //Get itemArray

src/main/java/org/apache/datasketches/frequencies/Util.java

@@ -53,6 +53,42 @@ static long hash(long key) {
     return key;
   }
 
+  /**
+   * Checks the map sizes and the number of active items read from the serialized image of a
+   * non-empty sketch, and returns the log2 size of the map to rebuild the sketch into.
+   *
+   * <p>The returned size is just large enough to hold the active items, and no larger than
+   * <i>lgCurMapSize</i>. The map then grows as needed while the items are inserted, so the memory
+   * allocated is bounded by the size of the image, not by the declared map size.</p>
+   *
+   * @param lgMaxMapSize log2 of the maximum map size
+   * @param lgCurMapSize log2 of the current map size
+   * @param activeItems the number of active items
+   * @param loadFactor the load factor of the hash map
+   * @return log2 of the map size to rebuild into
+   * @throws SketchesArgumentException if the values are not consistent
+   */
+  static int checkMapSizes(final int lgMaxMapSize, final int lgCurMapSize, final int activeItems,
+      final double loadFactor) {
+    if (lgCurMapSize < LG_MIN_MAP_SIZE) {
+      throw new SketchesArgumentException(
+          "Possible Corruption: LgCurMapSize must be at least " + LG_MIN_MAP_SIZE + ": " + lgCurMapSize);
+    }
+    if (lgCurMapSize > lgMaxMapSize) {
+      throw new SketchesArgumentException(
+          "Possible Corruption: LgCurMapSize must not exceed LgMaxMapSize: "
+            + lgCurMapSize + " > " + lgMaxMapSize);
+    }
+    final long maxActive = (long) ((1L << Math.min(lgCurMapSize, 62)) * loadFactor);
+    if ((activeItems < 0) || (activeItems > maxActive)) {
+      throw new SketchesArgumentException(
+          "Possible Corruption: ActiveItems exceeds map capacity: " + activeItems);
+    }
+    int lgStart = LG_MIN_MAP_SIZE;
+    while ((lgStart < lgCurMapSize) && (((1L << lgStart) * loadFactor) < activeItems)) { lgStart++; }
+    return lgStart;
+  }
+
   /**
    * Checks the stream weight read from the serialized image of a non-empty sketch.
    * @param streamWeight the stream weight

src/main/java/org/apache/datasketches/hll/HllSketch.java

@@ -254,12 +254,12 @@ static final HllSketch writableWrap( final MemorySegment srcWseg, final boolean
           "Cannot perform a writableWrap of a sketch image that is in compact form. "
           + "Compact sketches are by definition immutable.");
     }
+    final CurMode curMode = checkPreamble(srcWseg); //checks lgK before it is used
     final int lgConfigK = extractLgK(srcWseg);
     final TgtHllType tgtHllType = extractTgtHllType(srcWseg);
     final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
     final long capBytes = srcWseg.byteSize();
     HllUtil.checkSegSize(minBytes, capBytes);
-    final CurMode curMode = checkPreamble(srcWseg);
     final HllSketch directSketch;
     if (curMode == CurMode.HLL) {
       if (tgtHllType == TgtHllType.HLL_4) {

src/main/java/org/apache/datasketches/hll/HllUtil.java

@@ -23,8 +23,13 @@
 import static org.apache.datasketches.hll.PreambleUtil.HASH_SET_PREINTS;
 import static org.apache.datasketches.hll.PreambleUtil.HLL_PREINTS;
 import static org.apache.datasketches.hll.PreambleUtil.LIST_PREINTS;
+import static org.apache.datasketches.hll.PreambleUtil.extractAuxCount;
 import static org.apache.datasketches.hll.PreambleUtil.extractCurMode;
 import static org.apache.datasketches.hll.PreambleUtil.extractFamilyId;
+import static org.apache.datasketches.hll.PreambleUtil.extractHashSetCount;
+import static org.apache.datasketches.hll.PreambleUtil.extractLgArr;
+import static org.apache.datasketches.hll.PreambleUtil.extractLgK;
+import static org.apache.datasketches.hll.PreambleUtil.extractListCount;
 import static org.apache.datasketches.hll.PreambleUtil.extractPreInts;
 import static org.apache.datasketches.hll.PreambleUtil.extractSerVer;
 
@@ -111,9 +116,40 @@ static CurMode checkPreamble(final MemorySegment seg) {
       throw new SketchesArgumentException("Possible Corruption, Invalid Preamble:"
           + PreambleUtil.toString(seg));
     }
+    checkSizes(seg, curMode);
     return curMode;
   }
 
+  /**
+   * Checks lgK and the sizes declared by the image.
+   * LgK must be in the documented range. The coupon hash set and the auxiliary (exception) hash map
+   * are never larger than K, and a coupon list never holds more than its initial capacity.
+   * @param seg the image
+   * @param curMode the current mode of the image
+   */
+  private static void checkSizes(final MemorySegment seg, final CurMode curMode) {
+    final int lgK = extractLgK(seg);
+    checkLgK(lgK);
+    if (curMode == CurMode.LIST) {
+      final int count = extractListCount(seg);
+      if (count > (1 << LG_INIT_LIST_SIZE)) {
+        throw new SketchesArgumentException("Possible Corruption: list count " + count);
+      }
+    } else if (curMode == CurMode.SET) {
+      final int count = extractHashSetCount(seg);
+      final int lgArr = extractLgArr(seg);
+      if ((count < 0) || (count > (1 << lgK)) || (lgArr > lgK)) {
+        throw new SketchesArgumentException("Possible Corruption: set count " + count + ", lgArr " + lgArr);
+      }
+    } else { //HLL
+      final int auxCount = extractAuxCount(seg);
+      final int lgArr = extractLgArr(seg);
+      if ((auxCount < 0) || (auxCount > (1 << lgK)) || ((auxCount > 0) && (lgArr > lgK))) {
+        throw new SketchesArgumentException("Possible Corruption: aux count " + auxCount + ", lgArr " + lgArr);
+      }
+    }
+  }
+
   //Exceptions
   static void noWriteAccess() {
     throw new SketchesReadOnlyException(

src/main/java/org/apache/datasketches/quantiles/QuantilesItemsSketch.java

@@ -214,9 +214,16 @@ public static <T> QuantilesItemsSketch<T> heapify(
     //Not empty, must have valid preamble + min, max
     final long n = extractN(srcSeg);
 
-    //can't check MemorySegment capacity here, not enough information
+    //can't check the exact MemorySegment capacity here, not enough information
     final int extra = 2; //for min, max
     final int numSegItems = computeRetainedItems(k, n) + extra;
+    //No serialized item takes less than one bit (booleans are packed 8 per byte),
+    //so the image bounds the number of items before the buffer is allocated.
+    final long itemsBytes = segCapBytes - ((long) preambleLongs * Long.BYTES);
+    if ((n < 0) || ((long) numSegItems > (8L * itemsBytes))) {
+      throw new SketchesArgumentException("Possible corruption: n = " + n + " requires " + numSegItems
+          + " items, but only " + itemsBytes + " bytes remain");
+    }
 
     //set class members
     sk.n_ = n;