diff --git a/docs/appendices/release-notes/6.5.2.rst b/docs/appendices/release-notes/6.5.2.rst index 90898d53c319..f3c6a21f9437 100644 --- a/docs/appendices/release-notes/6.5.2.rst +++ b/docs/appendices/release-notes/6.5.2.rst @@ -46,4 +46,7 @@ series. Fixes ===== -None +- Fixed an issue causing inaccurate estimates of distinct counts of values in a + column (e.g. by executing ``SELECT n_distinct FROM pg_catalog.pg_stats``). As + a consequence, some optimization rules will be applied more accurately. + diff --git a/server/src/main/java/io/crate/statistics/ColumnSketchBuilder.java b/server/src/main/java/io/crate/statistics/ColumnSketchBuilder.java index b318eb4b3bd1..10ffe311aa8c 100644 --- a/server/src/main/java/io/crate/statistics/ColumnSketchBuilder.java +++ b/server/src/main/java/io/crate/statistics/ColumnSketchBuilder.java @@ -28,16 +28,16 @@ import java.util.Objects; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.jspecify.annotations.Nullable; import io.crate.Streamer; import io.crate.types.DataType; -/** - * Constructs a {@link ColumnStats} from a set of samples - */ +/// Constructs [ColumnStats] from a set of samples. public abstract class ColumnSketchBuilder { protected final DataType dataType; @@ -45,44 +45,77 @@ public abstract class ColumnSketchBuilder { protected long sampleCount; protected long nullCount; protected long totalBytes; + + // The NDV is estimated either using DistinctValuesSketch or using DistinctSample. + // DistinctSample is meant to be a replacement for DistinctValuesSketch, due to its + // inaccuracy (see: https://github.com/crate/crate/issues/20210 for details). + // Both exist at the same time to support rolling upgrades, given that they cannot be "mixed". + // If all nodes in the cluster are on a version higher than 6.5.1, then DistinctSample is used. + + @Deprecated(since = "6.6.0") + @Nullable protected DistinctValuesSketch distinctSketch; + /// Only filled if [#distinctSketch] is `null`. + @Nullable + protected final DistinctSample distinctCounts; + /** * Creates a new ColumnSketchBuilder for the given DataType + * + * @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE */ - public ColumnSketchBuilder(DataType dataType) { + public ColumnSketchBuilder(DataType dataType, Version minNodeVersion) { this.dataType = dataType; - this.distinctSketch = DistinctValuesSketch.newSketch(); + boolean useSketch = minNodeVersion.onOrBefore(Version.V_6_5_1); + this.distinctSketch = useSketch ? DistinctValuesSketch.newSketch() : null; + this.distinctCounts = useSketch ? null : new DistinctSample(); } - public ColumnSketchBuilder(DataType dataType, StreamInput in) throws IOException { + /** + * @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE, + * as used to create the streamed builder + */ + public ColumnSketchBuilder(DataType dataType, Version minNodeVersion, StreamInput in) throws IOException { this.dataType = dataType; + boolean useSketch = minNodeVersion.onOrBefore(Version.V_6_5_1); this.sampleCount = in.readLong(); this.nullCount = in.readLong(); this.totalBytes = in.readLong(); - this.distinctSketch = DistinctValuesSketch.fromStream(in); + if (useSketch) { + this.distinctSketch = DistinctValuesSketch.fromStream(in); + this.distinctCounts = null; + } else { + this.distinctSketch = null; + this.distinctCounts = new DistinctSample(in); + } } public final void writeTo(StreamOutput out) throws IOException { out.writeLong(sampleCount); out.writeLong(nullCount); out.writeLong(totalBytes); - out.writeByteArray(distinctSketch.getSketch().toByteArray()); + if (distinctSketch != null) { + out.writeByteArray(distinctSketch.getSketch().toByteArray()); + } else { + distinctCounts.writeTo(out); + } writeSketches(out); } protected abstract void writeSketches(StreamOutput out) throws IOException; - /** - * Add a sample to the sketch - */ public void add(T value) { sampleCount++; if (value == null) { nullCount++; } else { totalBytes += dataType.valueBytes(value); - distinctSketch.update(value.toString()); + if (distinctSketch != null) { + distinctSketch.update(value.toString()); + } else { + distinctCounts.add(value); + } updateSketches(value); } } @@ -101,9 +134,12 @@ public final void addAll(Collection values) { public abstract ColumnSketchBuilder merge(ColumnSketchBuilder other); /** - * Produce a streamable and merge-able representation of the sketch + * Produce a streamable and merge-able representation of the sketch. + * + * @param totalDocs the total (non-sampled) row count of the table, used to extrapolate + * the sample's distinct count to the full population */ - public abstract ColumnStats toStats(); + public abstract ColumnStats toStats(long totalDocs); double nullFraction() { if (nullCount == 0 || sampleCount == 0) { @@ -112,6 +148,75 @@ public final void addAll(Collection values) { return (double) nullCount / (double) sampleCount; } + /// Estimates the NDV using either the [DistinctValuesSketch] or the Haas-Stokes estimator. + final double estimateDistinct(long totalDocs) { + if (distinctSketch != null) { + return distinctSketch.getSketch().getEstimate(); + } + + return estimateWithHaasStokes(totalDocs); + } + + /// Estimates the NDV of the sample, not of the table. + final double sampleDistinct() { + if (distinctSketch != null) { + return distinctSketch.getSketch().getEstimate(); + } + return Math.min(distinctCounts.distinctValues(), sampleCount - nullCount); + } + + private double estimateWithHaasStokes(long totalRows) { + // `d` and `f1` are estimated from the hashes kept by the DistinctSample. + double d = distinctCounts.distinctValues(); + double f1 = distinctCounts.singletons(); + double n = sampleCount - nullCount; + double nonSingletonRows = distinctCounts.nonSingletonRows(); + + if (nonSingletonRows <= 0) { + // All sample values are unique. We assume it is a unique column, but we need to discount for null values. + return (1.0 - nullFraction()) * totalRows; + } + if (f1 == 0) { + // Every value appeared more than once. Assume the column has just these values. + // This should address cases like boolean or enum columns, where there is a small fixed set of possible values. + return d; + } + + // Estimate the number of distinct values using the estimator + // proposed by Haas and Stokes in IBM Research Report RJ 10025: + // ``` + // n*d / (n - f1 + f1*n/N) + // ``` + // where: + // * `n` is the (non-null) sample size + // * `d` the distinct count in the sample + // * `f1` the count of values seen exactly once in the sample + // * `N` the total count of non-null values + // + // This is their Duj1 estimator; the other estimators they + // recommend are considerably more complex, and are numerically + // very unstable when n is much smaller than N. + // + // Also see PostgreSQL's [compute_distinct_stats](https://github.com/postgres/postgres/blob/9f4bd91a1960f0263c497a9a1ddf121cbcefbb98/src/backend/commands/analyze.c#L2121). + double N = totalRows * (1.0 - nullFraction()); + double approxDistinct; + if (N > 0) { + // For near-unique columns, the exact `n` minus the estimated `f1` would be dominated + // by the error of `f1`, so `n - f1` is estimated as a whole. + approxDistinct = (n * d) / (nonSingletonRows + f1 * n / N); + } else { + approxDistinct = 0; + } + // Clamp to sane range in case of round-off error + if (approxDistinct < d) { + approxDistinct = d; + } + if (approxDistinct > N) { + approxDistinct = N; + } + return Math.floor(approxDistinct + 0.5); + } + /** * An implementation for single-valued data types */ @@ -121,14 +226,14 @@ public static class SingleValued extends ColumnSketchBuilder { private MostCommonValuesSketch mostCommonValuesSketch; private HistogramSketch histogramSketch; - public SingleValued(Class clazz, DataType dataType) { - super(dataType); + public SingleValued(Class clazz, DataType dataType, Version minNodeVersion) { + super(dataType, minNodeVersion); this.mostCommonValuesSketch = new MostCommonValuesSketch<>(dataType.streamer()); this.histogramSketch = new HistogramSketch<>(clazz, dataType); } - public SingleValued(Class clazz, DataType dataType, StreamInput in) throws IOException { - super(dataType, in); + public SingleValued(Class clazz, DataType dataType, Version minNodeVersion, StreamInput in) throws IOException { + super(dataType, minNodeVersion, in); this.mostCommonValuesSketch = new MostCommonValuesSketch<>(dataType.streamer(), in); this.histogramSketch = new HistogramSketch<>(clazz, dataType, in); } @@ -156,7 +261,11 @@ public ColumnSketchBuilder merge(ColumnSketchBuilder other) { this.sampleCount += other.sampleCount; this.nullCount += other.nullCount; this.totalBytes += other.totalBytes; - this.distinctSketch = this.distinctSketch.merge(other.distinctSketch); + if (this.distinctSketch != null) { + this.distinctSketch = this.distinctSketch.merge(other.distinctSketch); + } else { + this.distinctCounts.merge(typedOther.distinctCounts); + } this.mostCommonValuesSketch = this.mostCommonValuesSketch.merge(typedOther.mostCommonValuesSketch); this.histogramSketch = this.histogramSketch.merge(typedOther.histogramSketch); @@ -164,11 +273,11 @@ public ColumnSketchBuilder merge(ColumnSketchBuilder other) { } @Override - public ColumnStats toStats() { + public ColumnStats toStats(long totalDocs) { double nullFraction = nullFraction(); double avgSizeInBytes = (double) totalBytes / ((double) sampleCount - nullCount); - double approxDistinct = this.distinctSketch.getSketch().getEstimate(); - MostCommonValues mcv = this.mostCommonValuesSketch.toMostCommonValues(sampleCount, approxDistinct); + double approxDistinct = estimateDistinct(totalDocs); + MostCommonValues mcv = this.mostCommonValuesSketch.toMostCommonValues(sampleCount, sampleDistinct()); return new ColumnStats<>( nullFraction, avgSizeInBytes, @@ -190,13 +299,13 @@ public static class Composite extends ColumnSketchBuilder { private MostCommonValuesSketch mostCommonValuesSketch; private final BytesStreamOutput scratch = new BytesStreamOutput(); - public Composite(DataType dataType) { - super(dataType); + public Composite(DataType dataType, Version minNodeVersion) { + super(dataType, minNodeVersion); this.mostCommonValuesSketch = new MostCommonValuesSketch<>(BYTE_ARRAY_STREAMER); } - public Composite(DataType dataType, StreamInput in) throws IOException { - super(dataType, in); + public Composite(DataType dataType, Version minNodeVersion, StreamInput in) throws IOException { + super(dataType, minNodeVersion, in); this.mostCommonValuesSketch = new MostCommonValuesSketch<>(BYTE_ARRAY_STREAMER, in); } @@ -229,19 +338,23 @@ public ColumnSketchBuilder merge(ColumnSketchBuilder other) { this.sampleCount += other.sampleCount; this.nullCount += other.nullCount; this.totalBytes += other.totalBytes; - this.distinctSketch = this.distinctSketch.merge(other.distinctSketch); + if (this.distinctSketch != null) { + this.distinctSketch = this.distinctSketch.merge(other.distinctSketch); + } else { + this.distinctCounts.merge(typedOther.distinctCounts); + } this.mostCommonValuesSketch = this.mostCommonValuesSketch.merge(typedOther.mostCommonValuesSketch); return this; } @Override - public ColumnStats toStats() { + public ColumnStats toStats(long totalDocs) { double nullFraction = nullFraction(); double avgSizeInBytes = (double) totalBytes / ((double) sampleCount - nullCount); - double approxDistinct = this.distinctSketch.getSketch().getEstimate(); + double approxDistinct = estimateDistinct(totalDocs); MostCommonValues mcv = - this.mostCommonValuesSketch.toMostCommonValues(sampleCount, approxDistinct, this::fromByteStream); + this.mostCommonValuesSketch.toMostCommonValues(sampleCount, sampleDistinct(), this::fromByteStream); return new ColumnStats<>( nullFraction, avgSizeInBytes, diff --git a/server/src/main/java/io/crate/statistics/ColumnStatsSupport.java b/server/src/main/java/io/crate/statistics/ColumnStatsSupport.java index bb2779527533..a2732ac51c41 100644 --- a/server/src/main/java/io/crate/statistics/ColumnStatsSupport.java +++ b/server/src/main/java/io/crate/statistics/ColumnStatsSupport.java @@ -23,6 +23,7 @@ import java.io.IOException; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import io.crate.types.DataType; @@ -35,27 +36,31 @@ public interface ColumnStatsSupport { /** * Creates a column sketch builder for a column + * + * @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE */ - ColumnSketchBuilder sketchBuilder(); + ColumnSketchBuilder sketchBuilder(Version minNodeVersion); /** * Reads a column sketch from a stream + * + * @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE */ - ColumnSketchBuilder readSketchFrom(StreamInput in) throws IOException; + ColumnSketchBuilder readSketchFrom(Version minNodeVersion, StreamInput in) throws IOException; /** * Creates a support instance for a single-valued datatype */ static ColumnStatsSupport singleValued(Class clazz, DataType dataType) { - return new ColumnStatsSupport() { + return new ColumnStatsSupport<>() { @Override - public ColumnSketchBuilder sketchBuilder() { - return new ColumnSketchBuilder.SingleValued<>(clazz, dataType); + public ColumnSketchBuilder sketchBuilder(Version minNodeVersion) { + return new ColumnSketchBuilder.SingleValued<>(clazz, dataType, minNodeVersion); } @Override - public ColumnSketchBuilder readSketchFrom(StreamInput in) throws IOException { - return new ColumnSketchBuilder.SingleValued<>(clazz, dataType, in); + public ColumnSketchBuilder readSketchFrom(Version minNodeVersion, StreamInput in) throws IOException { + return new ColumnSketchBuilder.SingleValued<>(clazz, dataType, minNodeVersion, in); } }; } @@ -64,15 +69,15 @@ public ColumnSketchBuilder readSketchFrom(StreamInput in) throws IOException * Creates a support instance for a composite data type, e.g. an array */ static ColumnStatsSupport composite(DataType dataType) { - return new ColumnStatsSupport() { + return new ColumnStatsSupport<>() { @Override - public ColumnSketchBuilder sketchBuilder() { - return new ColumnSketchBuilder.Composite<>(dataType); + public ColumnSketchBuilder sketchBuilder(Version minNodeVersion) { + return new ColumnSketchBuilder.Composite<>(dataType, minNodeVersion); } @Override - public ColumnSketchBuilder readSketchFrom(StreamInput in) throws IOException { - return new ColumnSketchBuilder.Composite<>(dataType, in); + public ColumnSketchBuilder readSketchFrom(Version minNodeVersion, StreamInput in) throws IOException { + return new ColumnSketchBuilder.Composite<>(dataType, minNodeVersion, in); } }; } diff --git a/server/src/main/java/io/crate/statistics/DistinctSample.java b/server/src/main/java/io/crate/statistics/DistinctSample.java new file mode 100644 index 000000000000..5fe81aade213 --- /dev/null +++ b/server/src/main/java/io/crate/statistics/DistinctSample.java @@ -0,0 +1,234 @@ +/* + * Licensed to Crate.io GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.statistics; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import com.carrotsearch.hppc.BitMixer; +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.cursors.LongCursor; + +/// A simplified version of Gibbons' "Distinct Sample" described in his paper +/// "Distinct Sampling for Highly-Accurate Answers to Distinct Values Queries and Event Reports". +/// At a high level: +/// - `DistinctSample` is a set of hashes of column values. It has an upper bound on its size. +/// - The last bit of a hash is the duplicate flag (0 for singleton value, 1 for a duplicate). +/// - Every hash is assigned a "level": the number of leading 0s. +/// - `DistinctSample` has an associated level. All the hashes in a `DistinctSample` have a level +/// that is greater or equal to `DistinctSample`'s level. +/// - Once `DistinctSample` reaches its max. size, the level is increased. Hashes that have a +/// smaller level are removed. +/// - The number of values is estimated as: `distinct_sample_size * 2^level`. +/// +/// The simplification in this implementation is in following: +/// - We don't store rows per value. We only store a duplicate flag. +/// - We store the number of rows per level. This gives a better estimate of the number of non-singleton rows. +public final class DistinctSample { + + // For hashes that represent duplicate values, the last bit is set to 1. + // NB: This increases the chances of hash collisions by a bit, but it's + // not significant. The possibility is: number_of_pairs / number_of_possible_hash_values. + // Given that we sample 30k values per node, number_of_pairs = (30k * num_nodes)^2 / 2. + // number_of_possible_hash_values = 2^63 ≈ 10^19. + // Even if we assume 100 nodes are used, and all nodes return completely distinct samples, + // the chances of collision are less than 1/10^6. + private static final long DUPLICATE_FLAG = 1L; + // For hashes that represent singletons, the last bit is set to 0. + private static final long KEY_MASK = ~DUPLICATE_FLAG; + + // We balance between: + // * size of the LongHashSet (increases memory footprint), + // * number of retained hashes (increases accuracy), and + // * load factor (increases lookup time). + // The values below keep the performance and memory footprint + // at similar level as the DistinctValuesSketch that was used previously. + private static final double LOAD_FACTOR = 15.0 / 16.0; + // This keeps LongHashSet limited to 8192 slots. At this size, the memory footprint + // is the same as prior to introducing this class. + private static final int MAX_RETAINED = (int) (8192 * LOAD_FACTOR); + private static final int INITIAL_EXPECTED_ELEMENTS = 50; + // A hash can have between 0 and 64 leading zeros. + private static final int NUM_LEVELS = Long.SIZE + 1; + // value ^ HASH_SEED: this is done to make sure 0 is properly removed from a "level". + // 0 would be level 64, i.e. it would never be dropped from the hash. + // Whatever level we finish at, 0 would be in it. At that level, + // the number of distinct values would be estimated as 2^level * num_values_at_level, + // which would likely be far from the actual NDV. + private static final long HASH_SEED = 0x9E3779B97F4A7C15L; + + private final LongHashSet hashes; + private int level; + // The number of added rows by the level of their hash, including the rows of dropped hashes. + private final long[] rowsByLevel; + + DistinctSample() { + this.hashes = new LongHashSet(INITIAL_EXPECTED_ELEMENTS, LOAD_FACTOR); + this.rowsByLevel = new long[NUM_LEVELS]; + this.level = 0; + } + + DistinctSample(StreamInput in) throws IOException { + this.level = in.readVInt(); + this.rowsByLevel = in.readVLongArray(); + int size = in.readVInt(); + this.hashes = new LongHashSet(size, LOAD_FACTOR); + for (int i = 0; i < size; i++) { + hashes.add(in.readLong()); + } + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(level); + out.writeVLongArray(rowsByLevel); + out.writeVInt(hashes.size()); + for (LongCursor cursor : hashes) { + out.writeLong(cursor.value); + } + } + + public void add(Object val) { + long v; + switch (val) { + case Boolean b -> v = b ? 1L : 0L; + case Long l -> v = l; + case Integer i -> v = i; + case Short s -> v = s; + case Byte b -> v = b; + case Double d -> v = Double.doubleToLongBits(d); + case Float f -> v = Float.floatToIntBits(f); + default -> { + add(val.toString().getBytes(StandardCharsets.UTF_8)); + return; + } + } + // LongHashSet uses BitMixer as well. It's possible to override the hash function + // in LongHashSet, but benchmarks didn't show a significant difference, so we're + // keeping it simple. + addHashedValue(BitMixer.mix64(v ^ HASH_SEED)); + } + + private void add(byte[] bytes) { + long key = org.elasticsearch.common.hash.MurmurHash3.hash64(bytes, 0, bytes.length); + addHashedValue(key); + } + + private void addHashedValue(long hash) { + long key = hash & KEY_MASK; + int keyLevel = levelOf(key); + rowsByLevel[keyLevel]++; + if (keyLevel >= level) { + addKey(key); + } + } + + private static int levelOf(long entry) { + return Long.numberOfLeadingZeros(entry & KEY_MASK); + } + + private void addKey(long key) { + long duplicate = key | DUPLICATE_FLAG; + if (hashes.contains(duplicate)) { + return; + } + if (hashes.remove(key)) { + hashes.add(duplicate); + } else { + hashes.add(key); + shrinkIfFull(); + } + } + + private void shrinkIfFull() { + while (hashes.size() >= MAX_RETAINED) { + raiseLevel(level + 1); + } + } + + private void raiseLevel(int newLevel) { + level = newLevel; + hashes.removeAll(entry -> levelOf(entry) < newLevel); + } + + public DistinctSample merge(DistinctSample other) { + for (int i = 0; i < NUM_LEVELS; i++) { + rowsByLevel[i] += other.rowsByLevel[i]; + } + if (other.level > level) { + raiseLevel(other.level); + } + for (LongCursor cursor : other.hashes) { + // `level` may have been raised by an earlier entry. + if (levelOf(cursor.value) >= level) { + addMergeEntry(cursor.value); + } + } + return this; + } + + private void addMergeEntry(long entry) { + // If `entry` was a duplicate in the original set, + // then it will also be duplicate in the union of that set and this set. + // Because of this, a duplicate is added as-is. + boolean isDuplicate = (entry & DUPLICATE_FLAG) != 0; + if (isDuplicate) { + long nonDupe = entry & KEY_MASK; + hashes.remove(nonDupe); + hashes.add(entry); + shrinkIfFull(); + } else { + addKey(entry); + } + } + + // Estimated number of distinct values. + public double distinctValues() { + return Math.scalb((double) hashes.size(), level); + } + + // Estimated number of distinct values that appear exactly once. + public double singletons() { + return Math.scalb((double) keptSingletons(), level); + } + + // Estimated number of rows whose value appears more than once (`n - f1` in Haas-Stokes). + public double nonSingletonRows() { + long keptRows = 0; + for (int i = level; i < NUM_LEVELS; i++) { + keptRows += rowsByLevel[i]; + } + return Math.scalb((double) (keptRows - keptSingletons()), level); + } + + private int keptSingletons() { + int singletons = 0; + for (LongCursor cursor : hashes) { + if ((cursor.value & DUPLICATE_FLAG) == 0) { + singletons++; + } + } + return singletons; + } +} diff --git a/server/src/main/java/io/crate/statistics/FetchSampleRequest.java b/server/src/main/java/io/crate/statistics/FetchSampleRequest.java index f06734d8f42b..b37e7295bf37 100644 --- a/server/src/main/java/io/crate/statistics/FetchSampleRequest.java +++ b/server/src/main/java/io/crate/statistics/FetchSampleRequest.java @@ -41,14 +41,18 @@ public final class FetchSampleRequest extends TransportRequest { private final RelationName relationName; private final int tableOid; private final List columns; + // the lowest version of the nodes taking part in the ANALYZE + private final Version minNodeVersion; public FetchSampleRequest(RelationName relationName, int tableOid, List columns, + Version minNodeVersion, Version nodeVersion) { this.relationName = relationName; this.tableOid = tableOid; this.columns = columns; + this.minNodeVersion = minNodeVersion; if (nodeVersion.before(Version.V_5_7_0)) { throw new UnsupportedOperationException("Cannot run ANALYZE request in a mixed cluster with nodes older than 5.7.0"); } @@ -73,6 +77,12 @@ public FetchSampleRequest(StreamInput in) throws IOException { for (int i = 0; i < numColumns; i++) { columns.add(Reference.fromStream(in)); } + if (in.getVersion().after(Version.V_6_5_1)) { + this.minNodeVersion = Version.readVersion(in); + } else { + // Sent by a handler node on 6.5.1 or older, so that is the lowest version taking part. + this.minNodeVersion = in.getVersion(); + } } @Override @@ -89,6 +99,9 @@ public void writeTo(StreamOutput out) throws IOException { for (Reference column : columns) { Reference.toStream(out, column); } + if (out.getVersion().after(Version.V_6_5_1)) { + Version.writeVersion(minNodeVersion, out); + } } public RelationName relation() { @@ -103,9 +116,13 @@ public List columns() { return columns; } + public Version minNodeVersion() { + return minNodeVersion; + } + @Override public int hashCode() { - return Objects.hash(relationName, tableOid, columns); + return Objects.hash(relationName, tableOid, columns, minNodeVersion); } @Override @@ -121,6 +138,7 @@ public boolean equals(Object obj) { } return relationName.equals(other.relation()) && tableOid == other.tableOid() && - columns.equals(other.columns()); + columns.equals(other.columns()) && + minNodeVersion.equals(other.minNodeVersion()); } } diff --git a/server/src/main/java/io/crate/statistics/FetchSampleResponse.java b/server/src/main/java/io/crate/statistics/FetchSampleResponse.java index 9831cfe80ecb..0f6c11b4902f 100644 --- a/server/src/main/java/io/crate/statistics/FetchSampleResponse.java +++ b/server/src/main/java/io/crate/statistics/FetchSampleResponse.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.List; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.transport.TransportResponse; @@ -38,8 +39,8 @@ public final class FetchSampleResponse extends TransportResponse { this.samples = samples; } - public FetchSampleResponse(List references, StreamInput in) throws IOException { - this.samples = new Samples(references, in); + public FetchSampleResponse(List references, Version minNodeVersion, StreamInput in) throws IOException { + this.samples = new Samples(references, minNodeVersion, in); } Samples samples() { diff --git a/server/src/main/java/io/crate/statistics/ReservoirSampler.java b/server/src/main/java/io/crate/statistics/ReservoirSampler.java index dc1310789683..d8825af1a536 100644 --- a/server/src/main/java/io/crate/statistics/ReservoirSampler.java +++ b/server/src/main/java/io/crate/statistics/ReservoirSampler.java @@ -132,7 +132,10 @@ private void updateReadLimit(ByteSizeValue newReadLimit) { rateLimiter.setMBPerSec(newReadLimit.getMbFrac()); // mbPerSec is volatile in SimpleRateLimiter, one volatile write } - Samples getSamples(RelationName relationName, int tableOid, List columns) { + Samples getSamples(RelationName relationName, + int tableOid, + List columns, + Version minNodeVersion) { TableInfo table; try { table = tableOid == Metadata.OID_UNASSIGNED ? @@ -153,6 +156,7 @@ Samples getSamples(RelationName relationName, int tableOid, List colu try (SketchRamAccounting rla = new SketchRamAccounting(ramAccounting, rateLimiter)) { return getSamples( columns, + minNodeVersion, docTable, rla, random, @@ -164,6 +168,7 @@ Samples getSamples(RelationName relationName, int tableOid, List colu } private Samples getSamples(List columns, + Version minNodeVersion, DocTableInfo docTable, SketchRamAccounting ramAccounting, Random random, @@ -181,7 +186,7 @@ private Samples getSamples(List columns, List> columnCollectors = new ArrayList<>(); for (int i = 0; i < columns.size(); i++) { - columnCollectors.add(new ColumnCollector<>(ramAccounting, columns.get(i).valueType())); + columnCollectors.add(new ColumnCollector<>(ramAccounting, columns.get(i).valueType(), minNodeVersion)); } List searchersToRelease = new ArrayList<>(); @@ -272,8 +277,8 @@ private static class ColumnCollector { final SketchRamAccounting ramAccounting; final DataType dataType; - private ColumnCollector(SketchRamAccounting ramAccounting, DataType dataType) { - this.statsBuilder = dataType.columnStatsSupport().sketchBuilder(); + private ColumnCollector(SketchRamAccounting ramAccounting, DataType dataType, Version minNodeVersion) { + this.statsBuilder = dataType.columnStatsSupport().sketchBuilder(minNodeVersion); this.dataType = dataType; this.ramAccounting = ramAccounting; } diff --git a/server/src/main/java/io/crate/statistics/Samples.java b/server/src/main/java/io/crate/statistics/Samples.java index a7ebbcbdf711..3214076474e4 100644 --- a/server/src/main/java/io/crate/statistics/Samples.java +++ b/server/src/main/java/io/crate/statistics/Samples.java @@ -49,7 +49,7 @@ class Samples implements Writeable { this.numTotalSizeInBytes = numTotalSizeInBytes; } - public Samples(List references, StreamInput in) throws IOException { + public Samples(List references, Version minNodeVersion, StreamInput in) throws IOException { if (in.getVersion().before(Version.V_5_7_0)) { throw new UnsupportedOperationException("Cannot run ANALYZE in a mixed version cluster"); } @@ -63,7 +63,7 @@ public Samples(List references, StreamInput in) throws IOException { this.columnSketches = new ArrayList<>(numRecords); for (int i = 0; i < numRecords; i++) { Reference ref = references.get(i); - this.columnSketches.add(ref.valueType().columnStatsSupport().readSketchFrom(in)); + this.columnSketches.add(ref.valueType().columnStatsSupport().readSketchFrom(minNodeVersion, in)); } } @@ -105,7 +105,7 @@ public Stats createTableStats(List primitiveColumns) { Map> statsByColumn = HashMap.newHashMap(primitiveColumns.size()); for (int i = 0; i < primitiveColumns.size(); i++) { Reference primitiveColumn = primitiveColumns.get(i); - statsByColumn.put(primitiveColumn.column(), columnSketches.get(i).toStats()); + statsByColumn.put(primitiveColumn.column(), columnSketches.get(i).toStats(numTotalDocs)); } return new Stats(numTotalDocs, numTotalSizeInBytes, statsByColumn); } diff --git a/server/src/main/java/io/crate/statistics/TransportAnalyzeAction.java b/server/src/main/java/io/crate/statistics/TransportAnalyzeAction.java index 7e6104f36617..9a238dd1e11f 100644 --- a/server/src/main/java/io/crate/statistics/TransportAnalyzeAction.java +++ b/server/src/main/java/io/crate/statistics/TransportAnalyzeAction.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -107,7 +108,16 @@ public TransportAnalyzeAction(TransportService transportService, if (previous == null) { newSamples.completeAsync( - () -> reservoirSampler.getSamples(req.relation(), req.tableOid(), req.columns()), + () -> reservoirSampler.getSamples( + req.relation(), + req.tableOid(), + req.columns(), + // Min. node version is needed to determine how the stats will be calculated. + // We take the version from the original request, because the min. node version + // might have changed between when the handler node sent `FetchSampleRequest`s, + // and when they are handled here. + req.minNodeVersion() + ), executor ); return newSamples @@ -214,6 +224,7 @@ private CompletableFuture fetchSamples(RelationName relationName, List columns) { FutureActionListener listener = new FutureActionListener<>(); DiscoveryNodes discoveryNodes = clusterService.state().nodes(); + Version minNodeVersion = discoveryNodes.getMinNodeVersion(); MultiActionListener multiListener = new MultiActionListener<>( discoveryNodes.getSize(), Collectors.reducing( @@ -224,14 +235,14 @@ private CompletableFuture fetchSamples(RelationName relationName, ActionListenerResponseHandler responseHandler = new ActionListenerResponseHandler<>( FETCH_SAMPLES, multiListener, - in -> new FetchSampleResponse(columns, in), + in -> new FetchSampleResponse(columns, minNodeVersion, in), ThreadPool.Names.SAME ); for (DiscoveryNode node : discoveryNodes) { transportService.sendRequest( node, FETCH_SAMPLES, - new FetchSampleRequest(relationName, tableOid, columns, node.getVersion()), + new FetchSampleRequest(relationName, tableOid, columns, minNodeVersion, node.getVersion()), responseHandler ); } diff --git a/server/src/test/java/io/crate/planner/selectivity/SelectivityFunctionsTest.java b/server/src/test/java/io/crate/planner/selectivity/SelectivityFunctionsTest.java index 24c01c2b86fe..acf39654ddd6 100644 --- a/server/src/test/java/io/crate/planner/selectivity/SelectivityFunctionsTest.java +++ b/server/src/test/java/io/crate/planner/selectivity/SelectivityFunctionsTest.java @@ -134,7 +134,7 @@ public void test_not_reverses_selectivity_of_inner_function() { .collect(Collectors.toList()); var columnStats = StatsUtils.statsFromValues(DataTypes.INTEGER, numbers); Stats stats = new Stats(20_000, 16, Map.of(ColumnIdent.of("x"), columnStats)); - assertThat(estimate(stats, query)).isEqualTo(19998L); + assertThat(estimate(stats, query)).isEqualTo(19999L); } @Test diff --git a/server/src/test/java/io/crate/statistics/ColumnSketchBuilderTest.java b/server/src/test/java/io/crate/statistics/ColumnSketchBuilderTest.java index 4e4e65ddcf7b..45b721514fa9 100644 --- a/server/src/test/java/io/crate/statistics/ColumnSketchBuilderTest.java +++ b/server/src/test/java/io/crate/statistics/ColumnSketchBuilderTest.java @@ -22,6 +22,7 @@ package io.crate.statistics; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.withinPercentage; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -29,11 +30,13 @@ import java.util.List; import java.util.Map; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.InputStreamStreamInput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.test.ESTestCase; +import org.junit.Test; import com.carrotsearch.randomizedtesting.generators.RandomStrings; @@ -46,17 +49,18 @@ public class ColumnSketchBuilderTest extends ESTestCase { + @Test public void test_array_stats() { - var builder = new ColumnSketchBuilder.Composite<>( - new ArrayType<>(ObjectType.of(ColumnPolicy.DYNAMIC).setInnerType("id", IntegerType.INSTANCE).build()) + new ArrayType<>(ObjectType.of(ColumnPolicy.DYNAMIC).setInnerType("id", IntegerType.INSTANCE).build()), + Version.CURRENT ); builder.add(List.of(Map.of("id", 1))); builder.add(List.of(Map.of("id", 2))); builder.add(List.of(Map.of("id", 1))); - var stats = builder.toStats(); + var stats = builder.toStats(builder.sampleCount); assertThat(stats.mostCommonValues().values()).containsExactly( List.of(Map.of("id", 1)), @@ -64,6 +68,7 @@ public void test_array_stats() { ); } + @Test public void test_empty_sketch() { var stats = StatsUtils.statsFromValues(LongType.INSTANCE, List.of()); assertThat(stats.approxDistinct()).isEqualTo(0); @@ -71,22 +76,111 @@ public void test_empty_sketch() { assertThat(arrayStats.approxDistinct()).isEqualTo(0); } + @Test public void test_streaming() throws IOException { var support = StringType.INSTANCE.columnStatsSupport(); - var sketch = support.sketchBuilder(); + var sketch = support.sketchBuilder(Version.CURRENT); for (int i = 0; i < 200; i++) { sketch.add(RandomStrings.randomUnicodeOfLength(random(), 10)); } + var serializedSketch = streamOutIn(sketch, support, Version.CURRENT); + + assertThat(sketch.toStats(sketch.sampleCount)) + .isEqualTo(serializedSketch.toStats(serializedSketch.sampleCount)); + assertThat(serializedSketch.distinctCounts).isNotNull(); + } + + @Test + public void test_streaming_pre_6_5_1() throws IOException { + var support = StringType.INSTANCE.columnStatsSupport(); + var sketch = support.sketchBuilder(Version.V_6_5_0); + + for (int i = 0; i < 200; i++) { + sketch.add(RandomStrings.randomUnicodeOfLength(random(), 10)); + } + + var serializedSketch = streamOutIn(sketch, support, Version.V_6_5_0); + + assertThat(sketch.toStats(sketch.sampleCount)) + .isEqualTo(serializedSketch.toStats(serializedSketch.sampleCount)); + // We stopped relying on `distinctSketch` in 6.5.1. + assertThat(serializedSketch.distinctSketch).isNotNull(); + } + + @Test + public void test_streaming_distinct_sample_set() throws IOException { + var support = StringType.INSTANCE.columnStatsSupport(); + var sketch = support.sketchBuilder(Version.CURRENT); + for (int i = 0; i < 200; i++) { + String value = RandomStrings.randomUnicodeOfLength(random(), 10); + sketch.add(value); + if (i % 2 == 0) { + sketch.add(value); + } + } + + var serializedSketch = streamOutIn(sketch, support, Version.CURRENT); + + assertThat(serializedSketch.distinctSketch).isNull(); + assertThat(serializedSketch.toStats(10_000)).isEqualTo(sketch.toStats(10_000)); + } + + @Test + public void test_merge() { + for (Version version : List.of(Version.V_6_5_0, Version.CURRENT)) { + var support = IntegerType.INSTANCE.columnStatsSupport(); + var builder1 = support.sketchBuilder(version); + var builder2 = support.sketchBuilder(version); + + for (int j = 1; j <= 100; j++) { + builder1.add(j); + } + for (int j = 51; j <= 150; j++) { + builder2.add(j); + } + + var merged = builder1.merge(builder2); + + assertThat(merged.toStats(merged.sampleCount).approxDistinct()) + .as("version: " + version) + .isEqualTo(150.0); + } + } + + @Test + public void test_approx_distinct_sample_larger_than_reservoir_cap() { + // Sample (n = 40,000) exceeds Reservoir.NUM_SAMPLES (30,000). + // 30,000 values occur once (f1 = 30,000) + // 5,000 values occur twice each + // Number of distinct values: 35,000. + var builder = IntegerType.INSTANCE.columnStatsSupport().sketchBuilder(Version.CURRENT); + for (int i = 0; i < 30_000; i++) { + builder.add(i); + } + for (int i = 30_000; i < 35_000; i++) { + builder.add(i); + builder.add(i); + } + + // Haas-Stokes: n*d / (n - f1 + f1*n/N) + // = 40000*35000 / (10000 + 30000*40000/400000) + // = 1,400,000,000 / 13,000 = 107,692.307692... + assertThat(builder.toStats(400_000).approxDistinct()) + .isCloseTo(107_692, withinPercentage(5)); + } + + private static ColumnSketchBuilder streamOutIn(ColumnSketchBuilder sketch, + ColumnStatsSupport support, + Version version) throws IOException { ByteArrayOutputStream os = new ByteArrayOutputStream(); StreamOutput out = new OutputStreamStreamOutput(os); + out.setVersion(version); sketch.writeTo(out); StreamInput in = new InputStreamStreamInput(new ByteArrayInputStream(os.toByteArray())); - var serializedSketch = support.readSketchFrom(in); - - assertThat(sketch.toStats()).isEqualTo(serializedSketch.toStats()); + in.setVersion(version); + return support.readSketchFrom(version, in); } - } diff --git a/server/src/test/java/io/crate/statistics/ColumnStatsTest.java b/server/src/test/java/io/crate/statistics/ColumnStatsTest.java index fe04040c082f..ca989ad5304b 100644 --- a/server/src/test/java/io/crate/statistics/ColumnStatsTest.java +++ b/server/src/test/java/io/crate/statistics/ColumnStatsTest.java @@ -50,6 +50,35 @@ public void test_column_stats_generation() { assertThat(mostCommonValues.values()).hasSize(3); } + @Test + public void test_approx_distinct_sample_covers_whole_table() { + List values = IntStream.range(0, 500).boxed().toList(); + ColumnStats columnStats = StatsUtils.statsFromValues(DataTypes.INTEGER, values, 0, 500); + assertThat(columnStats.approxDistinct()).isEqualTo(500.0); + } + + @Test + public void test_approx_distinct_sample_is_all_unique() { + // Regression test + // Earlier, ColumnStats.approxDistinct() used to report the distinct values from the sample, + // without extrapolating to the full table. + // This meant that, for high-cardinality columns, the reports num. of distinct values was + // always 30k. + List values = IntStream.range(0, 30_000).boxed().toList(); + ColumnStats columnStats = StatsUtils.statsFromValues(DataTypes.INTEGER, values, 0, 5_000_000); + assertThat(columnStats.approxDistinct()).isEqualTo(5_000_000.0); + } + + @Test + public void test_no_most_common_values_for_unique_sample() { + // todo clarify logic in MCV + for (int offset : new int[] {0, 30_000}) { + List values = IntStream.range(offset, offset + 30_000).boxed().toList(); + ColumnStats stats = StatsUtils.statsFromValues(DataTypes.INTEGER, values, 0, 10_000_000); + assertThat(stats.mostCommonValues().values()).as("offset " + offset).isEmpty(); + } + } + @Property public void test_null_fraction_is_between_incl_0_and_incl_1(ArrayList numbers, @IntRange(min = 0) int nullCount, diff --git a/server/src/test/java/io/crate/statistics/DistinctSampleTest.java b/server/src/test/java/io/crate/statistics/DistinctSampleTest.java new file mode 100644 index 000000000000..d8895b5ff20b --- /dev/null +++ b/server/src/test/java/io/crate/statistics/DistinctSampleTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to Crate.io GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.statistics; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.junit.Test; + +public class DistinctSampleTest extends ESTestCase { + private DistinctSample set1; + private DistinctSample set2; + + @Before + public void setUp() throws Exception { + super.setUp(); + set1 = new DistinctSample(); + set2 = new DistinctSample(); + } + + @Test + public void test_all_unique_values() { + set1.add("a"); + set1.add("b"); + assertThat(set1.distinctValues()).isEqualTo(2); + assertThat(set1.singletons()).isEqualTo(2); + } + + @Test + public void test_duplicate_values_are_not_singletons() { + set1.add("a"); + set1.add("a"); + assertThat(set1.distinctValues()).isEqualTo(1); + assertThat(set1.singletons()).isEqualTo(0); + + set1.add("b"); + assertThat(set1.distinctValues()).isEqualTo(2); + assertThat(set1.singletons()).isEqualTo(1); + + set1.add("a"); + assertThat(set1.distinctValues()).isEqualTo(2); + assertThat(set1.singletons()).isEqualTo(1); + } + + @Test + public void test_empty_strings_are_counted() { + set1.add(""); + set1.add(""); + assertThat(set1.distinctValues()).isEqualTo(1); + assertThat(set1.singletons()).isEqualTo(0); + } + + @Test + public void test_merge_non_overlapping() { + set1.add("a"); + set2.add("b"); + + set1.merge(set2); + assertThat(set1.distinctValues()).isEqualTo(2); + assertThat(set1.singletons()).isEqualTo(2); + } + + @Test + public void test_merge_value_seen_once_in_each_set() { + set1.add("a"); + set2.add("a"); + + set1.merge(set2); + assertThat(set1.distinctValues()).isEqualTo(1); + assertThat(set1.singletons()).isEqualTo(0); + } + + @Test + public void test_merge_value_seen_multiple_times() { + set1.add("a"); + set2.add("a"); + set2.add("a"); + + set1.merge(set2); + assertThat(set1.distinctValues()).isEqualTo(1); + assertThat(set1.singletons()).isEqualTo(0); + } + + @Test + public void test_merge_keeps_duplicates() { + set1.add("a"); + set1.add("a"); + set2.add("b"); + + set1.merge(set2); + assertThat(set1.distinctValues()).isEqualTo(2); + assertThat(set1.singletons()).isEqualTo(1); + } + + @Test + public void test_numeric_values() { + set1.add(1); + set1.add(1); + set1.add(2); + set1.add(3); + + set1.add(2); + set1.add(4); + set1.add(4); + + set1.merge(set2); + assertThat(set1.distinctValues()).isEqualTo(4); + assertThat(set1.singletons()).isEqualTo(1); + } + + @Test + public void test_double_values() { + set1.add(Double.NaN); + set1.add(Double.NaN); + + assertThat(set1.distinctValues()).isEqualTo(1); + assertThat(set1.singletons()).isEqualTo(0); + + set1.add(Double.POSITIVE_INFINITY); + set1.add(Double.NEGATIVE_INFINITY); + + assertThat(set1.distinctValues()).isEqualTo(3); + assertThat(set1.singletons()).isEqualTo(2); + + set1.add(-0.0d); + set1.add(0.0d); + + assertThat(set1.distinctValues()).isEqualTo(5); + assertThat(set1.singletons()).isEqualTo(4); + } + + @Test + public void test_streaming() throws IOException { + set1.add("a"); + set1.add("a"); + set1.add("b"); + + BytesStreamOutput out = new BytesStreamOutput(); + set1.writeTo(out); + DistinctSample deserialized = new DistinctSample(out.bytes().streamInput()); + + assertThat(deserialized.distinctValues()).isEqualTo(2); + assertThat(deserialized.singletons()).isEqualTo(1); + } + + @Test + public void test_non_singleton_values() { + set1.add("a"); + set1.add("b"); + set1.add("b"); + set1.add("c"); + set1.add("c"); + set1.add("c"); + + assertThat(set1.nonSingletonRows()).isEqualTo(5); + assertThat(set1.distinctValues()).isEqualTo(3); + assertThat(set1.singletons()).isEqualTo(1); + } +} diff --git a/server/src/test/java/io/crate/statistics/FetchSampleRequestTest.java b/server/src/test/java/io/crate/statistics/FetchSampleRequestTest.java index 81222097d227..b1fe1c0e29b3 100644 --- a/server/src/test/java/io/crate/statistics/FetchSampleRequestTest.java +++ b/server/src/test/java/io/crate/statistics/FetchSampleRequestTest.java @@ -40,7 +40,7 @@ public void test_streaming_table_oids() throws Exception { for (Version version : List.of(Version.V_6_3_7, Version.V_6_4_2, Version.CURRENT)) { RelationName relation = new RelationName("doc", "tbl"); int tableOid = 1234; - FetchSampleRequest request = new FetchSampleRequest(relation, tableOid, List.of(), version); + FetchSampleRequest request = new FetchSampleRequest(relation, tableOid, List.of(), version, version); var out = new BytesStreamOutput(); out.setVersion(version); @@ -56,7 +56,7 @@ public void test_streaming_table_oids() throws Exception { // streaming to nodes with unsupported versions for (Version version : List.of(Version.V_6_3_6, Version.V_6_4_0, Version.V_6_4_1)) { RelationName relation = new RelationName("doc", "tbl"); - FetchSampleRequest request = new FetchSampleRequest(relation, 1234, List.of(), version); + FetchSampleRequest request = new FetchSampleRequest(relation, 1234, List.of(), version, version); var out = new BytesStreamOutput(); out.setVersion(version); diff --git a/server/src/test/java/io/crate/statistics/TransportAnalyzeActionTest.java b/server/src/test/java/io/crate/statistics/TransportAnalyzeActionTest.java index a4d3a2dc8f55..5d89452f98f4 100644 --- a/server/src/test/java/io/crate/statistics/TransportAnalyzeActionTest.java +++ b/server/src/test/java/io/crate/statistics/TransportAnalyzeActionTest.java @@ -25,6 +25,7 @@ import java.util.List; +import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.junit.Test; @@ -44,8 +45,8 @@ public class TransportAnalyzeActionTest extends ESTestCase { public void test_create_stats_for_tables_with_array_columns_with_nulls() { ArrayType type = DataTypes.STRING_ARRAY; - var col1 = type.columnStatsSupport().sketchBuilder(); - var col2 = type.columnStatsSupport().sketchBuilder(); + var col1 = type.columnStatsSupport().sketchBuilder(Version.CURRENT); + var col2 = type.columnStatsSupport().sketchBuilder(Version.CURRENT); col1.add(null); col2.add(null); var samples = new Samples( diff --git a/server/src/testFixtures/java/io/crate/statistics/StatsUtils.java b/server/src/testFixtures/java/io/crate/statistics/StatsUtils.java index ac957e6af1c9..1daa04e934ad 100644 --- a/server/src/testFixtures/java/io/crate/statistics/StatsUtils.java +++ b/server/src/testFixtures/java/io/crate/statistics/StatsUtils.java @@ -23,6 +23,8 @@ import java.util.Collection; +import org.elasticsearch.Version; + import io.crate.types.DataType; public final class StatsUtils { @@ -30,17 +32,31 @@ public final class StatsUtils { private StatsUtils() {} public static ColumnStats statsFromValues(DataType dataType, Collection values) { - ColumnSketchBuilder builder = dataType.columnStatsSupport().sketchBuilder(); + ColumnSketchBuilder builder = dataType.columnStatsSupport().sketchBuilder(Version.CURRENT); builder.addAll(values); - return builder.toStats(); + // Assume the sample is the whole table: totalDocs == number of rows just added. + return builder.toStats(builder.sampleCount); } public static ColumnStats statsFromValues(DataType dataType, Collection values, int nullCount) { - ColumnSketchBuilder builder = dataType.columnStatsSupport().sketchBuilder(); + ColumnSketchBuilder builder = dataType.columnStatsSupport().sketchBuilder(Version.CURRENT); + builder.addAll(values); + for (int i = 0; i < nullCount; i++) { + builder.add(null); + } + // Assume the sample is the whole table: totalDocs == number of rows just added. + return builder.toStats(builder.sampleCount); + } + + public static ColumnStats statsFromValues(DataType dataType, + Collection values, + int nullCount, + long totalDocs) { + ColumnSketchBuilder builder = dataType.columnStatsSupport().sketchBuilder(Version.CURRENT); builder.addAll(values); for (int i = 0; i < nullCount; i++) { builder.add(null); } - return builder.toStats(); + return builder.toStats(totalDocs); } }