Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/appendices/release-notes/6.5.2.rst
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,7 @@ series.
Fixes
=====

None
- Fixed an issue causing inaccurate estimates of distinct counts of values in a
column (e.g. by executing ``SELECT n_distinct FROM pg_catalog.pg_stats``). As
a consequence, some optimization rules will be applied more accurately.

173 changes: 143 additions & 30 deletions server/src/main/java/io/crate/statistics/ColumnSketchBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,61 +28,94 @@
import java.util.Objects;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.jspecify.annotations.Nullable;

import io.crate.Streamer;
import io.crate.types.DataType;

/**
* Constructs a {@link ColumnStats} from a set of samples
*/
/// Constructs [ColumnStats] from a set of samples.
public abstract class ColumnSketchBuilder<T> {

protected final DataType<T> dataType;

protected long sampleCount;
protected long nullCount;
protected long totalBytes;

// The NDV is estimated either using DistinctValuesSketch or using DistinctSample.
// DistinctSample is meant to be a replacement for DistinctValuesSketch, due to its
// inaccuracy (see: https://github.com/crate/crate/issues/20210 for details).
// Both exist at the same time to support rolling upgrades, given that they cannot be "mixed".
// If all nodes in the cluster are on a version higher than 6.5.1, then DistinctSample is used.

@Deprecated(since = "6.6.0")
@Nullable
protected DistinctValuesSketch distinctSketch;

/// Only filled if [#distinctSketch] is `null`.
@Nullable
protected final DistinctSample distinctCounts;

/**
* Creates a new ColumnSketchBuilder for the given DataType
*
* @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE
*/
public ColumnSketchBuilder(DataType<T> dataType) {
public ColumnSketchBuilder(DataType<T> dataType, Version minNodeVersion) {
this.dataType = dataType;
this.distinctSketch = DistinctValuesSketch.newSketch();
boolean useSketch = minNodeVersion.onOrBefore(Version.V_6_5_1);
this.distinctSketch = useSketch ? DistinctValuesSketch.newSketch() : null;
this.distinctCounts = useSketch ? null : new DistinctSample();
}

public ColumnSketchBuilder(DataType<T> dataType, StreamInput in) throws IOException {
/**
* @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE,
* as used to create the streamed builder
*/
public ColumnSketchBuilder(DataType<T> dataType, Version minNodeVersion, StreamInput in) throws IOException {
this.dataType = dataType;
boolean useSketch = minNodeVersion.onOrBefore(Version.V_6_5_1);
this.sampleCount = in.readLong();
this.nullCount = in.readLong();
this.totalBytes = in.readLong();
this.distinctSketch = DistinctValuesSketch.fromStream(in);
if (useSketch) {
this.distinctSketch = DistinctValuesSketch.fromStream(in);
this.distinctCounts = null;
} else {
this.distinctSketch = null;
this.distinctCounts = new DistinctSample(in);
}
}

public final void writeTo(StreamOutput out) throws IOException {
out.writeLong(sampleCount);
out.writeLong(nullCount);
out.writeLong(totalBytes);
out.writeByteArray(distinctSketch.getSketch().toByteArray());
if (distinctSketch != null) {
out.writeByteArray(distinctSketch.getSketch().toByteArray());
} else {
distinctCounts.writeTo(out);
}
writeSketches(out);
}

protected abstract void writeSketches(StreamOutput out) throws IOException;

/**
* Add a sample to the sketch
*/
public void add(T value) {
sampleCount++;
if (value == null) {
nullCount++;
} else {
totalBytes += dataType.valueBytes(value);
distinctSketch.update(value.toString());
if (distinctSketch != null) {
distinctSketch.update(value.toString());
} else {
distinctCounts.add(value);
}
updateSketches(value);
}
}
Expand All @@ -101,9 +134,12 @@ public final void addAll(Collection<T> values) {
public abstract ColumnSketchBuilder<T> merge(ColumnSketchBuilder<?> other);

/**
* Produce a streamable and merge-able representation of the sketch
* Produce a streamable and merge-able representation of the sketch.
*
* @param totalDocs the total (non-sampled) row count of the table, used to extrapolate
* the sample's distinct count to the full population
*/
public abstract ColumnStats<T> toStats();
public abstract ColumnStats<T> toStats(long totalDocs);

double nullFraction() {
if (nullCount == 0 || sampleCount == 0) {
Expand All @@ -112,6 +148,75 @@ public final void addAll(Collection<T> values) {
return (double) nullCount / (double) sampleCount;
}

/// Estimates the NDV using either the [DistinctValuesSketch] or the Haas-Stokes estimator.
final double estimateDistinct(long totalDocs) {
if (distinctSketch != null) {
return distinctSketch.getSketch().getEstimate();
}

return estimateWithHaasStokes(totalDocs);
}

/// Estimates the NDV of the sample, not of the table.
final double sampleDistinct() {
if (distinctSketch != null) {
return distinctSketch.getSketch().getEstimate();
}
return Math.min(distinctCounts.distinctValues(), sampleCount - nullCount);
}

private double estimateWithHaasStokes(long totalRows) {
// `d` and `f1` are estimated from the hashes kept by the DistinctSample.
double d = distinctCounts.distinctValues();
double f1 = distinctCounts.singletons();
double n = sampleCount - nullCount;
double nonSingletonRows = distinctCounts.nonSingletonRows();

if (nonSingletonRows <= 0) {
// All sample values are unique. We assume it is a unique column, but we need to discount for null values.
return (1.0 - nullFraction()) * totalRows;
}
if (f1 == 0) {
// Every value appeared more than once. Assume the column has just these values.
// This should address cases like boolean or enum columns, where there is a small fixed set of possible values.
return d;
}

// Estimate the number of distinct values using the estimator
// proposed by Haas and Stokes in IBM Research Report RJ 10025:
// ```
// n*d / (n - f1 + f1*n/N)
// ```
// where:
// * `n` is the (non-null) sample size
// * `d` the distinct count in the sample
// * `f1` the count of values seen exactly once in the sample
// * `N` the total count of non-null values
//
// This is their Duj1 estimator; the other estimators they
// recommend are considerably more complex, and are numerically
// very unstable when n is much smaller than N.
//
// Also see PostgreSQL's [compute_distinct_stats](https://github.com/postgres/postgres/blob/9f4bd91a1960f0263c497a9a1ddf121cbcefbb98/src/backend/commands/analyze.c#L2121).
double N = totalRows * (1.0 - nullFraction());
double approxDistinct;
if (N > 0) {
// For near-unique columns, the exact `n` minus the estimated `f1` would be dominated
// by the error of `f1`, so `n - f1` is estimated as a whole.
approxDistinct = (n * d) / (nonSingletonRows + f1 * n / N);
} else {
approxDistinct = 0;
}
// Clamp to sane range in case of round-off error
if (approxDistinct < d) {
approxDistinct = d;
}
if (approxDistinct > N) {
approxDistinct = N;
}
return Math.floor(approxDistinct + 0.5);
}

/**
* An implementation for single-valued data types
*/
Expand All @@ -121,14 +226,14 @@ public static class SingleValued<T> extends ColumnSketchBuilder<T> {
private MostCommonValuesSketch<T> mostCommonValuesSketch;
private HistogramSketch<T> histogramSketch;

public SingleValued(Class<T> clazz, DataType<T> dataType) {
super(dataType);
public SingleValued(Class<T> clazz, DataType<T> dataType, Version minNodeVersion) {
super(dataType, minNodeVersion);
this.mostCommonValuesSketch = new MostCommonValuesSketch<>(dataType.streamer());
this.histogramSketch = new HistogramSketch<>(clazz, dataType);
}

public SingleValued(Class<T> clazz, DataType<T> dataType, StreamInput in) throws IOException {
super(dataType, in);
public SingleValued(Class<T> clazz, DataType<T> dataType, Version minNodeVersion, StreamInput in) throws IOException {
super(dataType, minNodeVersion, in);
this.mostCommonValuesSketch = new MostCommonValuesSketch<>(dataType.streamer(), in);
this.histogramSketch = new HistogramSketch<>(clazz, dataType, in);
}
Expand Down Expand Up @@ -156,19 +261,23 @@ public ColumnSketchBuilder<T> merge(ColumnSketchBuilder<?> other) {
this.sampleCount += other.sampleCount;
this.nullCount += other.nullCount;
this.totalBytes += other.totalBytes;
this.distinctSketch = this.distinctSketch.merge(other.distinctSketch);
if (this.distinctSketch != null) {
this.distinctSketch = this.distinctSketch.merge(other.distinctSketch);
} else {
this.distinctCounts.merge(typedOther.distinctCounts);
}
this.mostCommonValuesSketch = this.mostCommonValuesSketch.merge(typedOther.mostCommonValuesSketch);
this.histogramSketch = this.histogramSketch.merge(typedOther.histogramSketch);

return this;
}

@Override
public ColumnStats<T> toStats() {
public ColumnStats<T> toStats(long totalDocs) {
double nullFraction = nullFraction();
double avgSizeInBytes = (double) totalBytes / ((double) sampleCount - nullCount);
double approxDistinct = this.distinctSketch.getSketch().getEstimate();
MostCommonValues<T> mcv = this.mostCommonValuesSketch.toMostCommonValues(sampleCount, approxDistinct);
double approxDistinct = estimateDistinct(totalDocs);
MostCommonValues<T> mcv = this.mostCommonValuesSketch.toMostCommonValues(sampleCount, sampleDistinct());
return new ColumnStats<>(
nullFraction,
avgSizeInBytes,
Expand All @@ -190,13 +299,13 @@ public static class Composite<T> extends ColumnSketchBuilder<T> {
private MostCommonValuesSketch<BytesRef> mostCommonValuesSketch;
private final BytesStreamOutput scratch = new BytesStreamOutput();

public Composite(DataType<T> dataType) {
super(dataType);
public Composite(DataType<T> dataType, Version minNodeVersion) {
super(dataType, minNodeVersion);
this.mostCommonValuesSketch = new MostCommonValuesSketch<>(BYTE_ARRAY_STREAMER);
}

public Composite(DataType<T> dataType, StreamInput in) throws IOException {
super(dataType, in);
public Composite(DataType<T> dataType, Version minNodeVersion, StreamInput in) throws IOException {
super(dataType, minNodeVersion, in);
this.mostCommonValuesSketch = new MostCommonValuesSketch<>(BYTE_ARRAY_STREAMER, in);
}

Expand Down Expand Up @@ -229,19 +338,23 @@ public ColumnSketchBuilder<T> merge(ColumnSketchBuilder<?> other) {
this.sampleCount += other.sampleCount;
this.nullCount += other.nullCount;
this.totalBytes += other.totalBytes;
this.distinctSketch = this.distinctSketch.merge(other.distinctSketch);
if (this.distinctSketch != null) {
this.distinctSketch = this.distinctSketch.merge(other.distinctSketch);
} else {
this.distinctCounts.merge(typedOther.distinctCounts);
}
this.mostCommonValuesSketch = this.mostCommonValuesSketch.merge(typedOther.mostCommonValuesSketch);

return this;
}

@Override
public ColumnStats<T> toStats() {
public ColumnStats<T> toStats(long totalDocs) {
double nullFraction = nullFraction();
double avgSizeInBytes = (double) totalBytes / ((double) sampleCount - nullCount);
double approxDistinct = this.distinctSketch.getSketch().getEstimate();
double approxDistinct = estimateDistinct(totalDocs);
MostCommonValues<T> mcv =
this.mostCommonValuesSketch.toMostCommonValues(sampleCount, approxDistinct, this::fromByteStream);
this.mostCommonValuesSketch.toMostCommonValues(sampleCount, sampleDistinct(), this::fromByteStream);
return new ColumnStats<>(
nullFraction,
avgSizeInBytes,
Expand Down
29 changes: 17 additions & 12 deletions server/src/main/java/io/crate/statistics/ColumnStatsSupport.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import java.io.IOException;

import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;

import io.crate.types.DataType;
Expand All @@ -35,27 +36,31 @@ public interface ColumnStatsSupport<T> {

/**
* Creates a column sketch builder for a column
*
* @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE
*/
ColumnSketchBuilder<T> sketchBuilder();
ColumnSketchBuilder<T> sketchBuilder(Version minNodeVersion);

/**
* Reads a column sketch from a stream
*
* @param minNodeVersion the lowest version of the nodes taking part in the ANALYZE
*/
ColumnSketchBuilder<T> readSketchFrom(StreamInput in) throws IOException;
ColumnSketchBuilder<T> readSketchFrom(Version minNodeVersion, StreamInput in) throws IOException;

/**
* Creates a support instance for a single-valued datatype
*/
static <T> ColumnStatsSupport<T> singleValued(Class<T> clazz, DataType<T> dataType) {
return new ColumnStatsSupport<T>() {
return new ColumnStatsSupport<>() {
@Override
public ColumnSketchBuilder<T> sketchBuilder() {
return new ColumnSketchBuilder.SingleValued<>(clazz, dataType);
public ColumnSketchBuilder<T> sketchBuilder(Version minNodeVersion) {
return new ColumnSketchBuilder.SingleValued<>(clazz, dataType, minNodeVersion);
}

@Override
public ColumnSketchBuilder<T> readSketchFrom(StreamInput in) throws IOException {
return new ColumnSketchBuilder.SingleValued<>(clazz, dataType, in);
public ColumnSketchBuilder<T> readSketchFrom(Version minNodeVersion, StreamInput in) throws IOException {
return new ColumnSketchBuilder.SingleValued<>(clazz, dataType, minNodeVersion, in);
}
};
}
Expand All @@ -64,15 +69,15 @@ public ColumnSketchBuilder<T> readSketchFrom(StreamInput in) throws IOException
* Creates a support instance for a composite data type, e.g. an array
*/
static <T> ColumnStatsSupport<T> composite(DataType<T> dataType) {
return new ColumnStatsSupport<T>() {
return new ColumnStatsSupport<>() {
@Override
public ColumnSketchBuilder<T> sketchBuilder() {
return new ColumnSketchBuilder.Composite<>(dataType);
public ColumnSketchBuilder<T> sketchBuilder(Version minNodeVersion) {
return new ColumnSketchBuilder.Composite<>(dataType, minNodeVersion);
}

@Override
public ColumnSketchBuilder<T> readSketchFrom(StreamInput in) throws IOException {
return new ColumnSketchBuilder.Composite<>(dataType, in);
public ColumnSketchBuilder<T> readSketchFrom(Version minNodeVersion, StreamInput in) throws IOException {
return new ColumnSketchBuilder.Composite<>(dataType, minNodeVersion, in);
}
};
}
Expand Down
Loading
Loading