From 64b01b58ca5ae55e3723ac8859dde32400052f62 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 22 Oct 2025 15:03:16 +0300 Subject: [PATCH 01/19] Switch interval validation to accept `DateHistogramInterval` --- .../action/downsample/DownsampleConfig.java | 17 ++++++++++------- .../downsample/TransportDownsampleAction.java | 11 ++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java index 444785d55e3d4..5f3b4f71d753d 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java @@ -144,25 +144,28 @@ public DownsampleConfig(final StreamInput in) throws IOException { * - The target interval needs to be a multiple of the source interval * throws an IllegalArgumentException to signal that the target interval is not acceptable */ - public static void validateSourceAndTargetIntervals(DownsampleConfig source, DownsampleConfig target) { - long sourceMillis = source.fixedInterval.estimateMillis(); - long targetMillis = target.fixedInterval.estimateMillis(); + public static void validateSourceAndTargetIntervals( + DateHistogramInterval sourceFxedInterval, + DateHistogramInterval targetFixedInterval + ) { + long sourceMillis = sourceFxedInterval.estimateMillis(); + long targetMillis = targetFixedInterval.estimateMillis(); if (sourceMillis >= targetMillis) { // Downsampling interval must be greater than source interval throw new IllegalArgumentException( "Downsampling interval [" - + target.fixedInterval + + targetFixedInterval + "] must be greater than the source index interval [" - + source.fixedInterval + + sourceFxedInterval + "]." ); } else if (targetMillis % sourceMillis != 0) { // Downsampling interval must be a multiple of the source interval throw new IllegalArgumentException( "Downsampling interval [" - + target.fixedInterval + + targetFixedInterval + "] must be a multiple of the source index interval [" - + source.fixedInterval + + sourceFxedInterval + "]." ); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 8a3758cd57249..afb38c99aa06a 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -863,15 +863,16 @@ private static void validateDownsamplingConfiguration( Map meta = timestampFieldType.meta(); if (meta.isEmpty() == false) { - String interval = meta.get(config.getIntervalType()); - DownsampleConfig.SamplingMethod sourceSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); - if (interval != null) { + String sourceInterval = meta.get(config.getIntervalType()); + if (sourceInterval != null) { try { - DownsampleConfig sourceConfig = new DownsampleConfig(new DateHistogramInterval(interval), sourceSamplingMethod); - DownsampleConfig.validateSourceAndTargetIntervals(sourceConfig, config); + DownsampleConfig.validateSourceAndTargetIntervals(new DateHistogramInterval(sourceInterval), config.getFixedInterval()); } catch (IllegalArgumentException exception) { e.addValidationError("Source index is a downsampled index. " + exception.getMessage()); } + DownsampleConfig.SamplingMethod sourceSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata( + sourceIndexMetadata + ); if (Objects.equals(sourceSamplingMethod, config.getSamplingMethodOrDefault()) == false) { e.addValidationError( "Source index is a downsampled index. Downsampling method [" From 345346dc006e78249dc8a8787dced631174e63c3 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 22 Oct 2025 15:04:18 +0300 Subject: [PATCH 02/19] Switch interval validation to accept `DateHistogramInterval` --- .../elasticsearch/cluster/metadata/DataStreamLifecycle.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index 7e60ddf0818d1..068946f3d6065 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -576,7 +576,10 @@ static void validateRounds(List rounds) { + "." ); } - DownsampleConfig.validateSourceAndTargetIntervals(previous.config(), round.config()); + DownsampleConfig.validateSourceAndTargetIntervals( + previous.config().getFixedInterval(), + round.config().getFixedInterval() + ); } } } From 097115d7a1a279130ad3f187f24ca0592e50bf34 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 22 Oct 2025 18:13:22 +0300 Subject: [PATCH 03/19] Do not use DownsampleConfig in DataStreamLifecycle --- .../lifecycle/DataStreamLifecycleService.java | 4 +- .../MetadataIndexTemplateServiceTests.java | 7 +-- .../DataStreamLifecycleFixtures.java | 7 +-- .../DataStreamLifecycleServiceTests.java | 12 ++--- .../cluster/metadata/DataStreamLifecycle.java | 32 +++++++------ .../add_last_value_downsample_dlm.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../DataStreamLifecycleTemplateTests.java | 40 ++++------------ .../metadata/DataStreamLifecycleTests.java | 46 ++++--------------- .../cluster/metadata/DataStreamTests.java | 28 +++-------- .../MetadataIndexTemplateServiceTests.java | 10 +--- .../TimeSeriesUsageTransportActionIT.java | 8 +--- ...StreamLifecycleDownsampleDisruptionIT.java | 10 +--- .../DataStreamLifecycleDownsampleIT.java | 38 +++------------ ...StreamLifecycleDownsamplingSecurityIT.java | 21 ++------- 15 files changed, 68 insertions(+), 198 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 776ee03928b27..06f311b0177c0 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -556,7 +556,7 @@ private Set waitForInProgressOrTriggerDownsampling( String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName( DOWNSAMPLED_INDEX_PREFIX, backingIndex, - round.config().getFixedInterval() + round.fixedInterval() ); IndexMetadata targetDownsampleIndexMeta = project.index(downsampleIndexName); boolean targetDownsampleIndexExists = targetDownsampleIndexMeta != null; @@ -601,7 +601,7 @@ private void downsampleIndexOnce( sourceIndex, downsampleIndexName, null, - round.config() + new DownsampleConfig(round.fixedInterval(), null) ); transportActionsDeduplicator.executeOnce( Tuple.tuple(projectId, request), diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index fd29a2087db86..f6df2f7d6dd1c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.datastreams; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; @@ -255,7 +254,7 @@ private static List randomRounds() { List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(randomIntBetween(1, 365)), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -268,9 +267,7 @@ private static List randomRounds() { private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 163bff3dfd2cb..0eed2d3f66373 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStore; @@ -164,7 +163,7 @@ private static List randomDownsamplingRou List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(randomIntBetween(1, 365)), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -177,9 +176,7 @@ private static List randomDownsamplingRou private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 0ce32cf1f94ab..faf800b79dc00 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1228,9 +1228,7 @@ public void testDownsampling() throws Exception { .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1377,9 +1375,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1662,9 +1658,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName, settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.timeValueMillis(1)) .build(), now diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index 068946f3d6065..fab6b067d7212 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -58,6 +58,7 @@ public class DataStreamLifecycle implements SimpleDiffable, // Versions over the wire public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X; + public static final TransportVersion ADD_LAST_VALUE_DOWNSAMPLE_DLM = TransportVersion.fromName("add_last_value_downsample_dlm"); public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention"; public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode"; @@ -522,9 +523,9 @@ public String displayName() { /** * A round represents the configuration for when and how elasticsearch will downsample a backing index. * @param after is a TimeValue configuring how old (based on generation age) should a backing index be before downsampling - * @param config contains the interval that the backing index is going to be downsampled. + * @param fixedInterval contains the interval that the backing index is going to be downsampled. */ - public record DownsamplingRound(TimeValue after, DownsampleConfig config) implements Writeable, ToXContentObject { + public record DownsamplingRound(TimeValue after, DateHistogramInterval fixedInterval) implements Writeable, ToXContentObject { public static final ParseField AFTER_FIELD = new ParseField("after"); public static final ParseField FIXED_INTERVAL_FIELD = new ParseField("fixed_interval"); @@ -533,7 +534,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "downsampling_round", false, - (args, unused) -> new DownsamplingRound((TimeValue) args[0], new DownsampleConfig((DateHistogramInterval) args[1])) + (args, unused) -> new DownsamplingRound((TimeValue) args[0], (DateHistogramInterval) args[1]) ); static { @@ -545,7 +546,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem PARSER.declareField( constructorArg(), p -> new DateHistogramInterval(p.text()), - new ParseField(FIXED_INTERVAL_FIELD.getPreferredName()), + FIXED_INTERVAL_FIELD, ObjectParser.ValueType.STRING ); } @@ -576,22 +577,23 @@ static void validateRounds(List rounds) { + "." ); } - DownsampleConfig.validateSourceAndTargetIntervals( - previous.config().getFixedInterval(), - round.config().getFixedInterval() - ); + DownsampleConfig.validateSourceAndTargetIntervals(previous.fixedInterval(), round.fixedInterval()); } } } public static DownsamplingRound read(StreamInput in) throws IOException { - return new DownsamplingRound(in.readTimeValue(), new DownsampleConfig(in)); + TimeValue after = in.readTimeValue(); + DateHistogramInterval fixedInterval = in.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM) + ? new DateHistogramInterval(in) + : new DownsampleConfig(in).getFixedInterval(); + return new DownsamplingRound(after, fixedInterval); } public DownsamplingRound { - if (config.getFixedInterval().estimateMillis() < FIVE_MINUTES_MILLIS) { + if (fixedInterval.estimateMillis() < FIVE_MINUTES_MILLIS) { throw new IllegalArgumentException( - "A downsampling round must have a fixed interval of at least five minutes but found: " + config.getFixedInterval() + "A downsampling round must have a fixed interval of at least five minutes but found: " + fixedInterval ); } } @@ -599,14 +601,18 @@ public static DownsamplingRound read(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeTimeValue(after); - out.writeWriteable(config); + if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) { + out.writeWriteable(fixedInterval); + } else { + out.writeWriteable(new DownsampleConfig(fixedInterval, null)); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(AFTER_FIELD.getPreferredName(), after.getStringRep()); - config.toXContentFragment(builder); + builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval().toString()); builder.endObject(); return builder; } diff --git a/server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv b/server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv new file mode 100644 index 0000000000000..d8997da1b2882 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv @@ -0,0 +1 @@ +9201000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 311c14ca764ac..8b8a312ad47d7 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -inference_cached_tokens,9200000 +add_last_value_downsample_dlm,9201000 diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 1d1c352788346..731d258af0243 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.cluster.metadata; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; @@ -106,14 +105,8 @@ public void testInvalidDownsamplingConfiguration() { () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(3), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) ) ) .buildTemplate() @@ -129,14 +122,8 @@ public void testInvalidDownsamplingConfiguration() { () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) ) ) .buildTemplate() @@ -149,14 +136,8 @@ public void testInvalidDownsamplingConfiguration() { () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) ) ) .buildTemplate() @@ -180,7 +161,7 @@ public void testInvalidDownsamplingConfiguration() { .map( i -> new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(i), - new DownsampleConfig(new DateHistogramInterval(i + "h")) + new DateHistogramInterval(i + "h") ) ) .toList() @@ -195,12 +176,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2m")) - ) - ) + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .buildTemplate() ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 198163645c83b..a231f5695a428 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.ClusterSettings; @@ -217,14 +216,8 @@ public void testInvalidDownsamplingConfiguration() { () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(3), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) ) ) .build() @@ -240,14 +233,8 @@ public void testInvalidDownsamplingConfiguration() { () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) ) ) .build() @@ -260,14 +247,8 @@ public void testInvalidDownsamplingConfiguration() { () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) ) ) .build() @@ -291,7 +272,7 @@ public void testInvalidDownsamplingConfiguration() { .map( i -> new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(i), - new DownsampleConfig(new DateHistogramInterval(i + "h")) + new DateHistogramInterval(i + "h") ) ) .toList() @@ -306,12 +287,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2m")) - ) - ) + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .build() ); @@ -533,7 +509,7 @@ static List randomDownsampling() { List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( randomTimeValue(1, 365, TimeUnit.DAYS), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -546,9 +522,7 @@ static List randomDownsampling() { private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index c822b066629f8..d520578f65c32 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -1592,17 +1591,11 @@ public void testGetDownsampleRounds() { DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(2000), - new DownsampleConfig(new DateHistogramInterval("10m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3200), - new DownsampleConfig(new DateHistogramInterval("100m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueMillis(3500), - new DownsampleConfig(new DateHistogramInterval("1000m")) + new DateHistogramInterval("1000m") ) ) @@ -1650,18 +1643,9 @@ public void testGetDownsampleRounds() { DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(2000), - new DownsampleConfig(new DateHistogramInterval("10m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3200), - new DownsampleConfig(new DateHistogramInterval("100m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3500), - new DownsampleConfig(new DateHistogramInterval("1000m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3500), new DateHistogramInterval("1000m")) ) ) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index a94b92527bc53..52f2a5b77deb1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.PutRequest; @@ -1104,14 +1103,7 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.Template lifecycle45d = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(TimeValue.timeValueDays(45)) - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) - ) - ) + .downsampling(List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")))) .buildTemplate(); String ct45d = "ct_45d"; project = addComponentTemplate(service, project, ct45d, lifecycle45d); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java index 8630accb5c3a7..36ce37b125d59 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.action; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -471,12 +470,7 @@ private List randomDownsamplingRounds() { int minutes = 5; int days = 1; for (int i = 0; i < randomIntBetween(1, 10); i++) { - rounds.add( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(days), - new DownsampleConfig(new DateHistogramInterval(minutes + "m")) - ) - ); + rounds.add(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(days), new DateHistogramInterval(minutes + "m"))); minutes *= randomIntBetween(2, 5); days += randomIntBetween(1, 5); } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index b07a00c13903c..2807bf9be5b79 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -52,14 +51,7 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { final String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ) - ) - ) + .downsampling(List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .buildTemplate(); setupTSDBDataStreamAndIngestDocs( dataStreamName, diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index 114bdeeff7e98..0e1ea25f561d1 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -45,14 +44,8 @@ public void testDownsampling() throws Exception { DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -117,16 +110,10 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at // least 2 seconds since rollover. only the 10 seconds round should be executed. - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -184,16 +171,10 @@ public void testUpdateDownsampleRound() throws Exception { DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at // least 2 seconds since rollover. only the 10 seconds round should be executed. - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -246,12 +227,7 @@ public void testUpdateDownsampleRound() throws Exception { // `10s` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10s` one. DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("20m")) - ) - ) + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("20m"))) ) .build(); assertAcked( diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 9def00c964a7d..adb85208b9d1d 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.lifecycle.ErrorEntry; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; @@ -123,14 +122,8 @@ public void testDownsamplingAuthorized() throws Exception { DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -413,14 +406,8 @@ public static class SystemDataStreamWithDownsamplingConfigurationPlugin extends public static final DataStreamLifecycle.Template LIFECYCLE = DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); From 3195327edc9ed5f00459d509daab2234e4224f45 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 23 Oct 2025 11:31:32 +0300 Subject: [PATCH 04/19] Rename downsampling to downsampling rounds --- .../MetadataIndexTemplateServiceTests.java | 6 +- .../DataStreamLifecycleServiceTests.java | 6 +- .../PutDataStreamLifecycleAction.java | 4 +- .../cluster/metadata/DataStream.java | 6 +- .../cluster/metadata/DataStreamLifecycle.java | 72 +++++++++---------- .../DataStreamFailureStoreTemplateTests.java | 2 +- .../DataStreamLifecycleTemplateTests.java | 30 ++++---- .../metadata/DataStreamLifecycleTests.java | 36 +++++----- ...amLifecycleWithRetentionWarningsTests.java | 14 ++-- .../cluster/metadata/DataStreamTests.java | 4 +- .../MetadataIndexTemplateServiceTests.java | 10 +-- .../TimeSeriesUsageTransportActionIT.java | 4 +- .../TimeSeriesUsageTransportAction.java | 4 +- ...StreamLifecycleDownsampleDisruptionIT.java | 4 +- .../DataStreamLifecycleDownsampleIT.java | 8 +-- ...StreamLifecycleDownsamplingSecurityIT.java | 4 +- 16 files changed, 111 insertions(+), 103 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index f6df2f7d6dd1c..8570613073fbf 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -157,7 +157,7 @@ public void testLifecycleComposition() { // Defaults to true assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); } // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false @@ -171,7 +171,7 @@ public void testLifecycleComposition() { DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); } // If both lifecycle have all properties, then the latest one overwrites all the others { @@ -189,7 +189,7 @@ public void testLifecycleComposition() { DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(lifecycle2.enabled())); assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle2.downsamplingRounds().get())); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index faf800b79dc00..35c709462d5d9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1228,7 +1228,7 @@ public void testDownsampling() throws Exception { .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1375,7 +1375,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1658,7 +1658,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName, settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.timeValueMillis(1)) .build(), now diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index 45b71fd63cb16..e0c6bbc79bd3e 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -145,14 +145,14 @@ public Request( String[] names, @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsampling + @Nullable List downsamplingRounds ) { super(masterNodeTimeout, ackTimeout); this.names = names; this.lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataRetention) .enabled(enabled == null || enabled) - .downsampling(downsampling) + .downsamplingRounds(downsamplingRounds) .build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 9e7fcb1e60f16..93be5259e29f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -1247,7 +1247,7 @@ public List getDownsamplingRoundsFor( LongSupplier nowSupplier ) { assert backingIndices.indices.contains(index) : "the provided index must be a backing index for this datastream"; - if (lifecycle == null || lifecycle.downsampling() == null) { + if (lifecycle == null || lifecycle.downsamplingRounds() == null) { return List.of(); } @@ -1260,8 +1260,8 @@ public List getDownsamplingRoundsFor( if (indexGenerationTime != null) { long nowMillis = nowSupplier.getAsLong(); long indexGenerationTimeMillis = indexGenerationTime.millis(); - List orderedRoundsForIndex = new ArrayList<>(lifecycle.downsampling().size()); - for (DownsamplingRound round : lifecycle.downsampling()) { + List orderedRoundsForIndex = new ArrayList<>(lifecycle.downsamplingRounds().size()); + for (DownsamplingRound round : lifecycle.downsamplingRounds()) { if (nowMillis >= indexGenerationTimeMillis + round.after().getMillis()) { orderedRoundsForIndex.add(round); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index fab6b067d7212..ebdc267e5c0e1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -143,7 +143,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { @Nullable private final TimeValue dataRetention; @Nullable - private final List downsampling; + private final List downsamplingRounds; /** * This constructor is visible for testing, please use {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List)} or @@ -153,16 +153,16 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { LifecycleType lifecycleType, @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsampling + @Nullable List downsamplingRounds ) { this.lifecycleType = lifecycleType; this.enabled = enabled == null || enabled; this.dataRetention = dataRetention; - if (lifecycleType == LifecycleType.FAILURES && downsampling != null) { + if (lifecycleType == LifecycleType.FAILURES && downsamplingRounds != null) { throw new IllegalArgumentException(DOWNSAMPLING_NOT_SUPPORTED_ERROR_MESSAGE); } - DownsamplingRound.validateRounds(downsampling); - this.downsampling = downsampling; + DownsamplingRound.validateRounds(downsamplingRounds); + this.downsamplingRounds = downsamplingRounds; } /** @@ -308,8 +308,8 @@ public void addWarningHeaderIfDataRetentionNotEffective( * not configured then it returns null. */ @Nullable - public List downsampling() { - return downsampling; + public List downsamplingRounds() { + return downsamplingRounds; } @Override @@ -320,13 +320,13 @@ public boolean equals(Object o) { final DataStreamLifecycle that = (DataStreamLifecycle) o; return lifecycleType == that.lifecycleType && Objects.equals(dataRetention, that.dataRetention) - && Objects.equals(downsampling, that.downsampling) + && Objects.equals(downsamplingRounds, that.downsamplingRounds) && enabled == that.enabled; } @Override public int hashCode() { - return Objects.hash(lifecycleType, enabled, dataRetention, downsampling); + return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds); } @Override @@ -341,9 +341,9 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (out.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - out.writeOptionalCollection(downsampling); + out.writeOptionalCollection(downsamplingRounds); } else { - writeLegacyOptionalValue(downsampling, out, StreamOutput::writeCollection); + writeLegacyOptionalValue(downsamplingRounds, out, StreamOutput::writeCollection); } out.writeBoolean(enabled()); } @@ -364,13 +364,13 @@ public DataStreamLifecycle(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (in.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - downsampling = in.readOptionalCollectionAsList(DownsamplingRound::read); + downsamplingRounds = in.readOptionalCollectionAsList(DownsamplingRound::read); } else { - downsampling = readLegacyOptionalValue(in, is -> is.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = readLegacyOptionalValue(in, is -> is.readCollectionAsList(DownsamplingRound::read)); } enabled = in.readBoolean(); } else { - downsampling = null; + downsamplingRounds = null; enabled = true; } lifecycleType = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; @@ -426,8 +426,8 @@ public String toString() { + enabled + ", dataRetention=" + dataRetention - + ", downsampling=" - + downsampling + + ", downsamplingRounds=" + + downsamplingRounds + '}'; } @@ -466,8 +466,8 @@ public XContentBuilder toXContent( } } - if (downsampling != null) { - builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsampling); + if (downsamplingRounds != null) { + builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsamplingRounds); } if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); @@ -667,7 +667,7 @@ public record Template( LifecycleType lifecycleType, boolean enabled, ResettableValue dataRetention, - ResettableValue> downsampling + ResettableValue> downsamplingRounds ) implements ToXContentObject, Writeable { Template( @@ -680,11 +680,11 @@ public record Template( } public Template { - if (lifecycleType == LifecycleType.FAILURES && downsampling.get() != null) { + if (lifecycleType == LifecycleType.FAILURES && downsamplingRounds.get() != null) { throw new IllegalArgumentException(DOWNSAMPLING_NOT_SUPPORTED_ERROR_MESSAGE); } - if (downsampling.isDefined() && downsampling.get() != null) { - DownsamplingRound.validateRounds(downsampling.get()); + if (downsamplingRounds.isDefined() && downsamplingRounds.get() != null) { + DownsamplingRound.validateRounds(downsamplingRounds.get()); } } @@ -736,9 +736,9 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (out.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - ResettableValue.write(out, downsampling, StreamOutput::writeCollection); + ResettableValue.write(out, downsamplingRounds, StreamOutput::writeCollection); } else { - writeLegacyValue(out, downsampling, StreamOutput::writeCollection); + writeLegacyValue(out, downsamplingRounds, StreamOutput::writeCollection); } out.writeBoolean(enabled); } @@ -839,7 +839,7 @@ public XContentBuilder toXContent( builder.startObject(); builder.field(ENABLED_FIELD.getPreferredName(), enabled); dataRetention.toXContent(builder, params, DATA_RETENTION_FIELD.getPreferredName(), TimeValue::getStringRep); - downsampling.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); + downsamplingRounds.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); rolloverConfiguration.evaluateAndConvertToXContent( @@ -853,7 +853,7 @@ public XContentBuilder toXContent( } public DataStreamLifecycle toDataStreamLifecycle() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsampling.get()); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get()); } } @@ -890,7 +890,7 @@ public static class Builder { @Nullable private TimeValue dataRetention = null; @Nullable - private List downsampling = null; + private List downsamplingRounds = null; private Builder(LifecycleType lifecycleType) { this.lifecycleType = lifecycleType; @@ -900,21 +900,21 @@ private Builder(DataStreamLifecycle.Template template) { lifecycleType = template.lifecycleType(); enabled = template.enabled(); dataRetention = template.dataRetention().get(); - downsampling = template.downsampling().get(); + downsamplingRounds = template.downsamplingRounds().get(); } private Builder(DataStreamLifecycle lifecycle) { lifecycleType = lifecycle.lifecycleType; enabled = lifecycle.enabled(); dataRetention = lifecycle.dataRetention(); - downsampling = lifecycle.downsampling(); + downsamplingRounds = lifecycle.downsamplingRounds(); } public Builder composeTemplate(DataStreamLifecycle.Template template) { assert lifecycleType == template.lifecycleType() : "Trying to compose templates with different lifecycle types"; enabled(template.enabled()); dataRetention(template.dataRetention()); - downsampling(template.downsampling()); + downsamplingRounds(template.downsamplingRounds()); return this; } @@ -935,24 +935,24 @@ public Builder dataRetention(@Nullable TimeValue dataRetention) { return this; } - public Builder downsampling(ResettableValue> downsampling) { + public Builder downsamplingRounds(ResettableValue> downsampling) { if (downsampling.isDefined()) { - this.downsampling = downsampling.get(); + this.downsamplingRounds = downsampling.get(); } return this; } - public Builder downsampling(@Nullable List downsampling) { - this.downsampling = downsampling; + public Builder downsamplingRounds(@Nullable List downsampling) { + this.downsamplingRounds = downsampling; return this; } public DataStreamLifecycle build() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsampling); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsamplingRounds); } public Template buildTemplate() { - return new Template(lifecycleType, enabled, dataRetention, downsampling); + return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java index 3b7834b67d7a1..f07ff27961424 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java @@ -146,7 +146,7 @@ private static DataStreamFailureStore.Template normalise(DataStreamFailureStore. template.lifecycleType(), template.enabled(), template.dataRetention().get(), - template.downsampling().get() + template.downsamplingRounds().get() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 731d258af0243..11d34fc9744f6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -43,27 +43,27 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa var lifecycleTarget = instance.lifecycleType(); var enabled = instance.enabled(); var retention = instance.dataRetention(); - var downsampling = instance.downsampling(); + var downsamplingRounds = instance.downsamplingRounds(); switch (randomInt(3)) { case 0 -> { lifecycleTarget = lifecycleTarget == DataStreamLifecycle.LifecycleType.DATA ? DataStreamLifecycle.LifecycleType.FAILURES : DataStreamLifecycle.LifecycleType.DATA; if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { - downsampling = ResettableValue.undefined(); + downsamplingRounds = ResettableValue.undefined(); } } case 1 -> enabled = enabled == false; case 2 -> retention = randomValueOtherThan(retention, DataStreamLifecycleTemplateTests::randomRetention); case 3 -> { - downsampling = randomValueOtherThan(downsampling, DataStreamLifecycleTemplateTests::randomDownsampling); - if (downsampling.get() != null) { + downsamplingRounds = randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTemplateTests::randomDownsamplingRounds); + if (downsamplingRounds.get() != null) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } default -> throw new AssertionError("Illegal randomisation branch"); } - return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsampling); + return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsamplingRounds); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -103,7 +103,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) @@ -120,7 +120,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) @@ -134,7 +134,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) @@ -147,7 +147,7 @@ public void testInvalidDownsamplingConfiguration() { { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> DataStreamLifecycle.dataLifecycleBuilder().downsampling((List.of())).buildTemplate() + () -> DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds((List.of())).buildTemplate() ); assertThat(exception.getMessage(), equalTo("Downsampling configuration should have at least one round configured.")); } @@ -155,7 +155,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( Stream.iterate(1, i -> i * 2) .limit(12) .map( @@ -175,7 +175,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .buildTemplate() @@ -188,7 +188,7 @@ public void testInvalidDownsamplingConfiguration() { } public static DataStreamLifecycle.Template randomDataLifecycleTemplate() { - return DataStreamLifecycle.createDataLifecycleTemplate(randomBoolean(), randomRetention(), randomDownsampling()); + return DataStreamLifecycle.createDataLifecycleTemplate(randomBoolean(), randomRetention(), randomDownsamplingRounds()); } public void testInvalidLifecycleConfiguration() { @@ -198,7 +198,7 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), - DataStreamLifecycleTests.randomDownsampling() + DataStreamLifecycleTests.randomDownsamplingRounds() ) ); assertThat( @@ -229,10 +229,10 @@ private static ResettableValue randomRetention() { }; } - private static ResettableValue> randomDownsampling() { + private static ResettableValue> randomDownsamplingRounds() { return switch (randomIntBetween(0, 1)) { case 0 -> ResettableValue.reset(); - case 1 -> ResettableValue.create(DataStreamLifecycleTests.randomDownsampling()); + case 1 -> ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); default -> throw new IllegalStateException("Unknown randomisation path"); }; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index a231f5695a428..33466674ed23a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -63,14 +63,14 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw : DataStreamLifecycle.LifecycleType.DATA; var enabled = instance.enabled(); var retention = instance.dataRetention(); - var downsampling = instance.downsampling(); + var downsamplingRounds = instance.downsamplingRounds(); switch (randomInt(3)) { case 0 -> { if (instance.targetsFailureStore()) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } else { lifecycleTarget = DataStreamLifecycle.LifecycleType.FAILURES; - downsampling = null; + downsamplingRounds = null; } } case 1 -> { @@ -81,20 +81,20 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw } } case 2 -> { - if (downsampling == null) { - downsampling = randomDownsampling(); + if (downsamplingRounds == null) { + downsamplingRounds = randomDownsamplingRounds(); if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } else { - downsampling = randomBoolean() + downsamplingRounds = randomBoolean() ? null - : randomValueOtherThan(downsampling, DataStreamLifecycleTests::randomDownsampling); + : randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTests::randomDownsamplingRounds); } } default -> enabled = enabled == false; } - return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsampling); + return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsamplingRounds); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -214,7 +214,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) @@ -231,7 +231,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) @@ -245,7 +245,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) @@ -258,7 +258,7 @@ public void testInvalidDownsamplingConfiguration() { { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> DataStreamLifecycle.dataLifecycleBuilder().downsampling((List.of())).build() + () -> DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds((List.of())).build() ); assertThat(exception.getMessage(), equalTo("Downsampling configuration should have at least one round configured.")); } @@ -266,7 +266,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( Stream.iterate(1, i -> i * 2) .limit(12) .map( @@ -286,7 +286,7 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .build() @@ -302,7 +302,7 @@ public void testEffectiveRetention() { // No retention in the data stream lifecycle { DataStreamLifecycle noDataRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamLifecycle noFailuresRetentionLifecycle = DataStreamLifecycle.failuresLifecycleBuilder().build(); TimeValue maxRetention = TimeValue.timeValueDays(randomIntBetween(50, 100)); @@ -357,7 +357,7 @@ public void testEffectiveRetention() { TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); DataStreamLifecycle dataLifecycleRetention = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataStreamRetention) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamLifecycle failuresLifecycleRetention = DataStreamLifecycle.failuresLifecycleBuilder() .dataRetention(dataStreamRetention) @@ -488,7 +488,7 @@ public void testEffectiveRetentionParams() { public static DataStreamLifecycle randomDataLifecycle() { return DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : randomTimeValue(1, 365, TimeUnit.DAYS)) - .downsampling(randomBoolean() ? null : randomDownsampling()) + .downsamplingRounds(randomBoolean() ? null : randomDownsamplingRounds()) .enabled(randomBoolean()) .build(); } @@ -504,7 +504,7 @@ public static DataStreamLifecycle randomFailuresLifecycle() { .build(); } - static List randomDownsampling() { + static List randomDownsamplingRounds() { var count = randomIntBetween(0, 9); List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( @@ -535,7 +535,7 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, null, null, - DataStreamLifecycleTests.randomDownsampling() + DataStreamLifecycleTests.randomDownsamplingRounds() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index 3b7a28a894156..9e1a06126fa96 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsampling; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsamplingRounds; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.hamcrest.Matchers.containsString; @@ -60,7 +60,9 @@ public void testNoHeaderWarning() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); HeaderWarning.setThreadContext(threadContext); - DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder().downsampling(randomDownsampling()).build(); + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingRounds(randomDownsamplingRounds()) + .build(); noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null, randomBoolean()); Map> responseHeaders = threadContext.getResponseHeaders(); assertThat(responseHeaders.isEmpty(), is(true)); @@ -68,7 +70,7 @@ public void testNoHeaderWarning() { TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); DataStreamLifecycle lifecycleWithRetention = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataStreamRetention) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( TimeValue.timeValueDays(2), @@ -83,7 +85,9 @@ public void testDefaultRetentionHeaderWarning() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); HeaderWarning.setThreadContext(threadContext); - DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder().downsampling(randomDownsampling()).build(); + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingRounds(randomDownsamplingRounds()) + .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( randomTimeValue(2, 10, TimeUnit.DAYS), randomBoolean() ? null : TimeValue.timeValueDays(20) @@ -107,7 +111,7 @@ public void testMaxRetentionHeaderWarning() { TimeValue maxRetention = randomTimeValue(2, 100, TimeUnit.DAYS); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : TimeValue.timeValueDays(maxRetention.days() + 1)) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, maxRetention); lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index d520578f65c32..e3a2147991ac4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -1589,7 +1589,7 @@ public void testGetDownsampleRounds() { settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), @@ -1641,7 +1641,7 @@ public void testGetDownsampleRounds() { // no TSDB settings settings(IndexVersion.current()), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 52f2a5b77deb1..2949ff2660cc3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1103,7 +1103,9 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.Template lifecycle45d = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(TimeValue.timeValueDays(45)) - .downsampling(List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")))) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h"))) + ) .buildTemplate(); String ct45d = "ct_45d"; project = addComponentTemplate(service, project, ct45d, lifecycle45d); @@ -1159,7 +1161,7 @@ public void testResolveLifecycle() throws Exception { lifecycle30d, DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(lifecycle30d.dataRetention()) - .downsampling(lifecycle45d.downsampling()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) .buildTemplate() ); @@ -1181,7 +1183,7 @@ public void testResolveLifecycle() throws Exception { project, List.of(ctEmptyLifecycle, ct45d), lifecycleNullRetention, - DataStreamLifecycle.dataLifecycleBuilder().downsampling(lifecycle45d.downsampling()).buildTemplate() + DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds(lifecycle45d.downsamplingRounds()).buildTemplate() ); // Component A: "lifecycle": {"retention": "30d"} @@ -1195,7 +1197,7 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate(), DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(lifecycle45d.dataRetention()) - .downsampling(lifecycle45d.downsampling()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) .enabled(false) .buildTemplate() ); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java index 36ce37b125d59..f668956d88056 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java @@ -163,7 +163,7 @@ public void testAction() throws Exception { timeSeriesDataStreamCount.incrementAndGet(); if (downsamplingConfiguredBy == DownsampledBy.DLM) { dlmDownsampledDataStreamCount.incrementAndGet(); - updateRounds(lifecycle.downsampling().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); + updateRounds(lifecycle.downsamplingRounds().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); } else if (downsamplingConfiguredBy == DownsampledBy.ILM) { ilmDownsampledDataStreamCount.incrementAndGet(); } @@ -409,7 +409,7 @@ private DataStreamLifecycle maybeCreateLifecycle(boolean isDownsampled, boolean } var builder = DataStreamLifecycle.dataLifecycleBuilder(); if (isDownsampled) { - builder.downsampling(randomDownsamplingRounds()); + builder.downsamplingRounds(randomDownsamplingRounds()); } return builder.build(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java index 77245a396e5db..5e980387bc4f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java @@ -89,9 +89,9 @@ protected void localClusterStateOperation( continue; } tsDataStreamCount++; - Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsampling() == null + Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsamplingRounds() == null ? null - : ds.getDataLifecycle().downsampling().size(); + : ds.getDataLifecycle().downsamplingRounds().size(); for (Index backingIndex : ds.getIndices()) { IndexMetadata indexMetadata = projectMetadata.index(backingIndex); diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index 2807bf9be5b79..8fff5bb52a75f 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -51,7 +51,9 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { final String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m"))) + ) .buildTemplate(); setupTSDBDataStreamAndIngestDocs( dataStreamName, diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index 0e1ea25f561d1..adb72ceb47ce0 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -42,7 +42,7 @@ public void testDownsampling() throws Exception { String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) @@ -108,7 +108,7 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception String dataStreamName = "metrics-bar"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at @@ -169,7 +169,7 @@ public void testUpdateDownsampleRound() throws Exception { String dataStreamName = "metrics-baz"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at @@ -226,7 +226,7 @@ public void testUpdateDownsampleRound() throws Exception { // the different interval should yield a different downsample index name so we expect the data stream lifecycle to get the previous // `10s` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10s` one. DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("20m"))) ) .build(); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index adb85208b9d1d..87b9b482d428e 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -120,7 +120,7 @@ public void testDownsamplingAuthorized() throws Exception { String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) @@ -404,7 +404,7 @@ private void bulkIndex(Client client, String dataStreamName, Supplier Date: Thu, 23 Oct 2025 12:00:39 +0300 Subject: [PATCH 05/19] Add downsampling method to the data stream lifecycle config --- .../MetadataIndexTemplateServiceTests.java | 13 +- .../action/GetDataStreamsResponseTests.java | 2 +- .../DataStreamLifecycleFixtures.java | 14 +- .../cluster/metadata/DataStreamLifecycle.java | 156 +++++++++++++++--- .../datastreams/GetDataStreamActionTests.java | 2 +- .../ExplainIndexDataStreamLifecycleTests.java | 2 +- .../GetDataStreamLifecycleActionTests.java | 2 +- .../downsample/DownsampleConfigTests.java | 2 +- .../DataStreamFailureStoreTemplateTests.java | 3 +- .../DataStreamLifecycleTemplateTests.java | 64 ++++++- .../metadata/DataStreamLifecycleTests.java | 31 +++- .../MetadataIndexTemplateServiceTests.java | 1 + ...taStreamLifecycleFeatureSetUsageTests.java | 6 +- 13 files changed, 250 insertions(+), 48 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index 8570613073fbf..563cb86b79cd0 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.EmptySystemIndices; @@ -150,7 +151,8 @@ public void testLifecycleComposition() { DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( true, randomRetention(), - randomDownsampling() + randomDownsampling(), + ResettableValue.create(DataStreamLifecycleFixtures.randomSamplingMethod()) ); List lifecycles = List.of(lifecycle); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); @@ -165,7 +167,8 @@ public void testLifecycleComposition() { DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + DataStreamLifecycleFixtures.randomSamplingMethod() ); List lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); @@ -178,12 +181,14 @@ public void testLifecycleComposition() { DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + DataStreamLifecycleFixtures.randomSamplingMethod() ); DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate( true, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + DataStreamLifecycleFixtures.randomSamplingMethod() ); List lifecycles = List.of(lifecycle1, lifecycle2); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index 8f6605cefae9f..10b772249e436 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -154,7 +154,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti .setGeneration(3) .setAllowCustomRouting(true) .setIndexMode(IndexMode.STANDARD) - .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null)) + .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null, null)) .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) .build(); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 0eed2d3f66373..1411f2f4e99fd 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStore; @@ -37,6 +38,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.test.ESIntegTestCase.client; +import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.frequently; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.junit.Assert.assertTrue; @@ -145,7 +147,8 @@ static DataStreamLifecycle.Template randomDataLifecycleTemplate() { return DataStreamLifecycle.createDataLifecycleTemplate( frequently(), randomResettable(ESTestCase::randomTimeValue), - randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds) + randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds), + randomResettable(DataStreamLifecycleFixtures::randomSamplingMethod) ); } @@ -179,4 +182,13 @@ private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecyc var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } + + public static DownsampleConfig.SamplingMethod randomSamplingMethod() { + return switch (between(0, 2)) { + case 0 -> null; + case 1 -> DownsampleConfig.SamplingMethod.AGGREGATE; + case 2 -> DownsampleConfig.SamplingMethod.LAST_VALUE; + default -> throw new IllegalStateException("Unknown randomisation path"); + }; + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index ebdc267e5c0e1..cbc913758bc5e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -52,7 +52,7 @@ * Lifecycle supports the following configurations: * - enabled, applicable to data and failures * - data retention, applicable to data and failures - * - downsampling, applicable only to data + * - downsampling and downsampling method, applicable only to data */ public class DataStreamLifecycle implements SimpleDiffable, ToXContentObject { @@ -75,6 +75,8 @@ public class DataStreamLifecycle implements SimpleDiffable, "Failure store lifecycle does not support downsampling, please remove the downsampling configuration."; private static final TransportVersion INTRODUCE_LIFECYCLE_TEMPLATE = TransportVersion.fromName("introduce_lifecycle_template"); + public static final String DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR = + "Downsampling method can only be set when there is at least one downsampling round."; /** * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that @@ -95,7 +97,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { Setting.Property.NodeScope ); - public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null); + public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null, null); public static final DataStreamLifecycle DEFAULT_FAILURE_LIFECYCLE = DataStreamLifecycle.createFailuresLifecycle(null, null); public static final String DATA_STREAM_LIFECYCLE_ORIGIN = "data_stream_lifecycle"; @@ -105,13 +107,20 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static final ParseField EFFECTIVE_RETENTION_FIELD = new ParseField("effective_retention"); public static final ParseField RETENTION_SOURCE_FIELD = new ParseField("retention_determined_by"); public static final ParseField DOWNSAMPLING_FIELD = new ParseField("downsampling"); + public static final ParseField DOWNSAMPLING_METHOD_FIELD = new ParseField("downsampling_method"); private static final ParseField ROLLOVER_FIELD = new ParseField("rollover"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "lifecycle", false, - (args, lt) -> new DataStreamLifecycle(lt, (Boolean) args[0], (TimeValue) args[1], (List) args[2]) + (args, lt) -> new DataStreamLifecycle( + lt, + (Boolean) args[0], + (TimeValue) args[1], + (List) args[2], + (DownsampleConfig.SamplingMethod) args[3] + ) ); static { @@ -134,6 +143,12 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { return AbstractObjectParser.parseArray(p, null, DownsamplingRound::fromXContent); } }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()), + DOWNSAMPLING_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); } private static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = TransportVersion.fromName("introduce_failures_lifecycle"); @@ -144,16 +159,20 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { private final TimeValue dataRetention; @Nullable private final List downsamplingRounds; + @Nullable + private final DownsampleConfig.SamplingMethod downsamplingMethod; /** - * This constructor is visible for testing, please use {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List)} or + * This constructor is visible for testing, please use + * {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List, DownsampleConfig.SamplingMethod)} or * {@link DataStreamLifecycle#createFailuresLifecycle(Boolean, TimeValue)}. */ DataStreamLifecycle( LifecycleType lifecycleType, @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsamplingRounds + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { this.lifecycleType = lifecycleType; this.enabled = enabled == null || enabled; @@ -163,6 +182,10 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { } DownsamplingRound.validateRounds(downsamplingRounds); this.downsamplingRounds = downsamplingRounds; + if (downsamplingMethod != null && downsamplingRounds == null) { + throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR); + } + this.downsamplingMethod = downsamplingMethod; } /** @@ -172,9 +195,10 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static DataStreamLifecycle createDataLifecycle( @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { - return new DataStreamLifecycle(LifecycleType.DATA, enabled, dataRetention, downsampling); + return new DataStreamLifecycle(LifecycleType.DATA, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } /** @@ -182,7 +206,7 @@ public static DataStreamLifecycle createDataLifecycle( * means it supports only enabling and retention. */ public static DataStreamLifecycle createFailuresLifecycle(@Nullable Boolean enabled, @Nullable TimeValue dataRetention) { - return new DataStreamLifecycle(LifecycleType.FAILURES, enabled, dataRetention, null); + return new DataStreamLifecycle(LifecycleType.FAILURES, enabled, dataRetention, null, null); } /** @@ -305,13 +329,21 @@ public void addWarningHeaderIfDataRetentionNotEffective( /** * The configured downsampling rounds with the `after` and the `fixed_interval` per round. If downsampling is - * not configured then it returns null. + * not configured, then it returns null. */ @Nullable public List downsamplingRounds() { return downsamplingRounds; } + /** + * The configured downsampling method. If downsampling is not configured, then it returns null. + */ + @Nullable + public DownsampleConfig.SamplingMethod downsamplingMethod() { + return downsamplingMethod; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -321,12 +353,13 @@ public boolean equals(Object o) { return lifecycleType == that.lifecycleType && Objects.equals(dataRetention, that.dataRetention) && Objects.equals(downsamplingRounds, that.downsamplingRounds) + && Objects.equals(downsamplingMethod, that.downsamplingMethod) && enabled == that.enabled; } @Override public int hashCode() { - return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds); + return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } @Override @@ -350,6 +383,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } + if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) { + out.writeOptionalWriteable(downsamplingMethod); + } } public DataStreamLifecycle(StreamInput in) throws IOException { @@ -374,6 +410,9 @@ public DataStreamLifecycle(StreamInput in) throws IOException { enabled = true; } lifecycleType = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; + downsamplingMethod = in.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM) + ? in.readOptionalWriteable(DownsampleConfig.SamplingMethod::read) + : null; } /** @@ -428,6 +467,8 @@ public String toString() { + dataRetention + ", downsamplingRounds=" + downsamplingRounds + + ", downsamplingMethod=" + + downsamplingMethod + '}'; } @@ -469,6 +510,9 @@ public XContentBuilder toXContent( if (downsamplingRounds != null) { builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsamplingRounds); } + if (downsamplingMethod != null) { + builder.field(DOWNSAMPLING_METHOD_FIELD.getPreferredName(), downsamplingMethod.toString()); + } if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); rolloverConfiguration.evaluateAndConvertToXContent(builder, params, effectiveDataRetentionWithSource.v1()); @@ -634,9 +678,16 @@ public String toString() { public static Template createDataLifecycleTemplate( boolean enabled, TimeValue dataRetention, - List downsampling + List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod ) { - return new Template(LifecycleType.DATA, enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + return new Template( + LifecycleType.DATA, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.create(downsamplingRounds), + ResettableValue.create(downsamplingMethod) + ); } /** @@ -646,9 +697,10 @@ public static Template createDataLifecycleTemplate( public static Template createDataLifecycleTemplate( boolean enabled, ResettableValue dataRetention, - ResettableValue> downsampling + ResettableValue> downsamplingRounds, + ResettableValue downsamplingMethod ) { - return new Template(LifecycleType.DATA, enabled, dataRetention, downsampling); + return new Template(LifecycleType.DATA, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } /** @@ -656,7 +708,13 @@ public static Template createDataLifecycleTemplate( * means it supports only setting the enabled and the retention. */ public static Template createFailuresLifecycleTemplate(boolean enabled, TimeValue dataRetention) { - return new Template(LifecycleType.FAILURES, enabled, ResettableValue.create(dataRetention), ResettableValue.undefined()); + return new Template( + LifecycleType.FAILURES, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.undefined(), + ResettableValue.undefined() + ); } /** @@ -667,16 +725,24 @@ public record Template( LifecycleType lifecycleType, boolean enabled, ResettableValue dataRetention, - ResettableValue> downsamplingRounds + ResettableValue> downsamplingRounds, + ResettableValue downsamplingMethod ) implements ToXContentObject, Writeable { Template( LifecycleType lifecycleType, boolean enabled, TimeValue dataRetention, - List downsampling + List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod ) { - this(lifecycleType, enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + this( + lifecycleType, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.create(downsamplingRounds), + ResettableValue.create(downsamplingMethod) + ); } public Template { @@ -685,6 +751,8 @@ public record Template( } if (downsamplingRounds.isDefined() && downsamplingRounds.get() != null) { DownsamplingRound.validateRounds(downsamplingRounds.get()); + } else if (downsamplingMethod.isDefined() && downsamplingMethod.get() != null) { + throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR); } } @@ -692,6 +760,7 @@ public record Template( LifecycleType.DATA, true, ResettableValue.undefined(), + ResettableValue.undefined(), ResettableValue.undefined() ); @@ -703,7 +772,8 @@ public record Template( lt, args[0] == null || (boolean) args[0], args[1] == null ? ResettableValue.undefined() : (ResettableValue) args[1], - args[2] == null ? ResettableValue.undefined() : (ResettableValue>) args[2] + args[2] == null ? ResettableValue.undefined() : (ResettableValue>) args[2], + args[3] == null ? ResettableValue.undefined() : (ResettableValue) args[3] ) ); @@ -722,6 +792,10 @@ public record Template( return ResettableValue.create(AbstractObjectParser.parseArray(p, null, DownsamplingRound::fromXContent)); } }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + String value = p.textOrNull(); + return value == null ? ResettableValue.reset() : ResettableValue.create(DownsampleConfig.SamplingMethod.fromString(value)); + }, DOWNSAMPLING_METHOD_FIELD, ObjectParser.ValueType.STRING_OR_NULL); } @Override @@ -745,6 +819,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } + if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) { + ResettableValue.write(out, downsamplingMethod, StreamOutput::writeWriteable); + } } /** @@ -784,7 +861,7 @@ static ResettableValue readLegacyValues(StreamInput in, Writeable.Reader< public static Template read(StreamInput in) throws IOException { boolean enabled = true; ResettableValue dataRetention = ResettableValue.undefined(); - ResettableValue> downsampling = ResettableValue.undefined(); + ResettableValue> downsamplingRounds = ResettableValue.undefined(); // The order of the fields is like this for bwc reasons if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { @@ -796,16 +873,20 @@ public static Template read(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (in.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - downsampling = ResettableValue.read(in, i -> i.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = ResettableValue.read(in, i -> i.readCollectionAsList(DownsamplingRound::read)); } else { - downsampling = readLegacyValues(in, i -> i.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = readLegacyValues(in, i -> i.readCollectionAsList(DownsamplingRound::read)); } enabled = in.readBoolean(); } var lifecycleTarget = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; - return new Template(lifecycleTarget, enabled, dataRetention, downsampling); + ResettableValue downsamplingMethod = in.getTransportVersion() + .supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM) + ? ResettableValue.read(in, DownsampleConfig.SamplingMethod::read) + : ResettableValue.undefined(); + return new Template(lifecycleTarget, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } public static Template dataLifecycleTemplatefromXContent(XContentParser parser) throws IOException { @@ -840,6 +921,12 @@ public XContentBuilder toXContent( builder.field(ENABLED_FIELD.getPreferredName(), enabled); dataRetention.toXContent(builder, params, DATA_RETENTION_FIELD.getPreferredName(), TimeValue::getStringRep); downsamplingRounds.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); + downsamplingMethod.toXContent( + builder, + params, + DOWNSAMPLING_METHOD_FIELD.getPreferredName(), + DownsampleConfig.SamplingMethod::toString + ); if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); rolloverConfiguration.evaluateAndConvertToXContent( @@ -853,7 +940,7 @@ public XContentBuilder toXContent( } public DataStreamLifecycle toDataStreamLifecycle() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get()); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get(), downsamplingMethod.get()); } } @@ -891,6 +978,8 @@ public static class Builder { private TimeValue dataRetention = null; @Nullable private List downsamplingRounds = null; + @Nullable + private DownsampleConfig.SamplingMethod downsamplingMethod = null; private Builder(LifecycleType lifecycleType) { this.lifecycleType = lifecycleType; @@ -901,6 +990,7 @@ private Builder(DataStreamLifecycle.Template template) { enabled = template.enabled(); dataRetention = template.dataRetention().get(); downsamplingRounds = template.downsamplingRounds().get(); + downsamplingMethod = template.downsamplingMethod().get(); } private Builder(DataStreamLifecycle lifecycle) { @@ -908,6 +998,7 @@ private Builder(DataStreamLifecycle lifecycle) { enabled = lifecycle.enabled(); dataRetention = lifecycle.dataRetention(); downsamplingRounds = lifecycle.downsamplingRounds(); + downsamplingMethod = lifecycle.downsamplingMethod(); } public Builder composeTemplate(DataStreamLifecycle.Template template) { @@ -915,6 +1006,7 @@ public Builder composeTemplate(DataStreamLifecycle.Template template) { enabled(template.enabled()); dataRetention(template.dataRetention()); downsamplingRounds(template.downsamplingRounds()); + downsamplingMethod(template.downsamplingMethod()); return this; } @@ -947,12 +1039,24 @@ public Builder downsamplingRounds(@Nullable List downsampling return this; } + public Builder downsamplingMethod(ResettableValue downsamplingMethod) { + if (downsamplingMethod.isDefined()) { + this.downsamplingMethod = downsamplingMethod.get(); + } + return this; + } + + public Builder downsamplingMethod(@Nullable DownsampleConfig.SamplingMethod downsamplingMethod) { + this.downsamplingMethod = downsamplingMethod; + return this; + } + public DataStreamLifecycle build() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsamplingRounds); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } public Template buildTemplate() { - return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds); + return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } } diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index c90669650a949..c9027799a57c6 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -107,7 +107,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo private static DataStream newDataStreamInstance(boolean isSystem, TimeValue retention) { List indices = List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))); - DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null); + DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null, null); Settings settings = randomSettings(); CompressedXContent mappings = randomMappings(); return DataStream.builder(randomAlphaOfLength(50), indices) diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java index 43a39f762ce69..2fb4a1c0ced4f 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java @@ -200,7 +200,7 @@ public void testToXContent() throws Exception { TimeValue configuredRetention = TimeValue.timeValueDays(100); TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); - DataStreamLifecycle dataStreamLifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null); + DataStreamLifecycle dataStreamLifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null, null); { boolean isSystemDataStream = true; ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = createManagedIndexDataStreamLifecycleExplanation( diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java index 32269d0bc5113..7c72764901505 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java @@ -96,7 +96,7 @@ public void testDataStreamLifecycleToXContent() throws Exception { TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); - DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null); + DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null, null); { boolean isInternalDataStream = true; GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle( diff --git a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java index adbf2a8888ae6..4ca89da4c9413 100644 --- a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java +++ b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java @@ -49,7 +49,7 @@ public static DownsampleConfig randomConfig() { return new DownsampleConfig(randomInterval(), randomSamplingMethod()); } - private static DownsampleConfig.SamplingMethod randomSamplingMethod() { + public static DownsampleConfig.SamplingMethod randomSamplingMethod() { return switch (between(0, 2)) { case 0 -> null; case 1 -> DownsampleConfig.SamplingMethod.AGGREGATE; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java index f07ff27961424..950eece0793b7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java @@ -146,7 +146,8 @@ private static DataStreamFailureStore.Template normalise(DataStreamFailureStore. template.lifecycleType(), template.enabled(), template.dataRetention().get(), - template.downsamplingRounds().get() + template.downsamplingRounds().get(), + template.downsamplingMethod().get() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 11d34fc9744f6..95c6431d17b5f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -9,6 +9,8 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; @@ -44,7 +46,8 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa var enabled = instance.enabled(); var retention = instance.dataRetention(); var downsamplingRounds = instance.downsamplingRounds(); - switch (randomInt(3)) { + var downsamplingMethod = instance.downsamplingMethod(); + switch (randomInt(4)) { case 0 -> { lifecycleTarget = lifecycleTarget == DataStreamLifecycle.LifecycleType.DATA ? DataStreamLifecycle.LifecycleType.FAILURES @@ -61,9 +64,16 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } + case 4 -> { + if (downsamplingRounds.get() == null) { + downsamplingRounds = ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); + lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; + } + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DataStreamLifecycleTemplateTests::randomDownsamplingMethod); + } default -> throw new AssertionError("Illegal randomisation branch"); } - return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsamplingRounds); + return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsamplingRounds, downsamplingMethod); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -185,10 +195,31 @@ public void testInvalidDownsamplingConfiguration() { equalTo("A downsampling round must have a fixed interval of at least five minutes but found: 2m") ); } + + { + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(randomFrom(DownsampleConfig.SamplingMethod.values())) + .buildTemplate() + ); + assertThat( + exception.getMessage(), + equalTo("Downsampling method can only be set when there is at least one downsampling round.") + ); + } } public static DataStreamLifecycle.Template randomDataLifecycleTemplate() { - return DataStreamLifecycle.createDataLifecycleTemplate(randomBoolean(), randomRetention(), randomDownsamplingRounds()); + ResettableValue> downsamplingRounds = randomDownsamplingRounds(); + return DataStreamLifecycle.createDataLifecycleTemplate( + randomBoolean(), + randomRetention(), + downsamplingRounds, + downsamplingRounds.get() == null + ? randomBoolean() ? ResettableValue.undefined() : ResettableValue.reset() + : ResettableValue.undefined() + ); } public void testInvalidLifecycleConfiguration() { @@ -198,13 +229,29 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), - DataStreamLifecycleTests.randomDownsamplingRounds() + DataStreamLifecycleTests.randomDownsamplingRounds(), + null ) ); assertThat( exception.getMessage(), containsString("Failure store lifecycle does not support downsampling, please remove the downsampling configuration.") ); + + exception = expectThrows( + IllegalArgumentException.class, + () -> new DataStreamLifecycle.Template( + DataStreamLifecycle.LifecycleType.DATA, + randomBoolean(), + randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), + null, + randomFrom(DownsampleConfig.SamplingMethod.values()) + ) + ); + assertThat( + exception.getMessage(), + containsString("Downsampling method can only be set when there is at least one downsampling round.") + ); } /** @@ -216,6 +263,7 @@ public static DataStreamLifecycle.Template randomFailuresLifecycleTemplate() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomRetention(), + ResettableValue.undefined(), ResettableValue.undefined() ); } @@ -236,4 +284,12 @@ private static ResettableValue> rand default -> throw new IllegalStateException("Unknown randomisation path"); }; } + + private static ResettableValue randomDownsamplingMethod() { + return switch (randomIntBetween(0, 1)) { + case 0 -> ResettableValue.reset(); + case 1 -> ResettableValue.create(DownsampleConfigTests.randomSamplingMethod()); + default -> throw new IllegalStateException("Unknown randomisation path"); + }; + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 33466674ed23a..eed9be4b9d2c7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.ClusterSettings; @@ -64,7 +66,8 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw var enabled = instance.enabled(); var retention = instance.dataRetention(); var downsamplingRounds = instance.downsamplingRounds(); - switch (randomInt(3)) { + var downsamplingMethod = instance.downsamplingMethod(); + switch (randomInt(4)) { case 0 -> { if (instance.targetsFailureStore()) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; @@ -90,11 +93,28 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw downsamplingRounds = randomBoolean() ? null : randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTests::randomDownsamplingRounds); + if (downsamplingRounds == null) { + downsamplingMethod = null; + } + } + } + case 3 -> { + // We need to enable downsampling in order to add a non value downsampling method + if (downsamplingRounds == null) { + downsamplingRounds = randomDownsamplingRounds(); + lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; + } + if (downsamplingMethod == null) { + downsamplingMethod = randomFrom(DownsampleConfig.SamplingMethod.values()); + } else if (downsamplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE) { + downsamplingMethod = randomBoolean() ? null : DownsampleConfig.SamplingMethod.LAST_VALUE; + } else { + downsamplingMethod = randomBoolean() ? null : DownsampleConfig.SamplingMethod.AGGREGATE; } } default -> enabled = enabled == false; } - return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsamplingRounds); + return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsamplingRounds, downsamplingMethod); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -486,9 +506,11 @@ public void testEffectiveRetentionParams() { } public static DataStreamLifecycle randomDataLifecycle() { + List downsamplingRounds = randomBoolean() ? null : randomDownsamplingRounds(); return DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : randomTimeValue(1, 365, TimeUnit.DAYS)) - .downsamplingRounds(randomBoolean() ? null : randomDownsamplingRounds()) + .downsamplingRounds(downsamplingRounds) + .downsamplingMethod(downsamplingRounds == null ? null : DownsampleConfigTests.randomSamplingMethod()) .enabled(randomBoolean()) .build(); } @@ -535,7 +557,8 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, null, null, - DataStreamLifecycleTests.randomDownsamplingRounds() + DataStreamLifecycleTests.randomDownsamplingRounds(), + DownsampleConfigTests.randomSamplingMethod() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 2949ff2660cc3..a29aa3fa3b221 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1113,6 +1113,7 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.Template lifecycleNullRetention = DataStreamLifecycle.createDataLifecycleTemplate( true, ResettableValue.reset(), + ResettableValue.undefined(), ResettableValue.undefined() ); String ctNullRetention = "ct_null_retention"; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java index fa4e7ead7eaf9..1516353731b56 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java @@ -117,7 +117,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueSeconds(50), null) + DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueSeconds(50), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), @@ -125,7 +125,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueMillis(150), null) + DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueMillis(150), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), @@ -133,7 +133,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(false, TimeValue.timeValueSeconds(5), null) + DataStreamLifecycle.createDataLifecycle(false, TimeValue.timeValueSeconds(5), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), From 9107d7628cf7a8963af3bd6d5cfe79fa6c407b62 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 23 Oct 2025 14:26:08 +0300 Subject: [PATCH 06/19] Update rest endpoints --- .../RestPutDataStreamLifecycleAction.java | 14 ++++- .../test/data_stream/lifecycle/20_basic.yml | 52 +++++++++++++++++++ .../cluster.component_template/10_basic.yml | 32 ++++++++++++ .../indices.put_index_template/10_basic.yml | 34 ++++++++++++ .../PutDataStreamLifecycleAction.java | 26 ++++++++-- .../RestPutComponentTemplateAction.java | 4 +- .../RestPutComposableIndexTemplateAction.java | 5 +- 7 files changed, 158 insertions(+), 9 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java index 2d975845fcd27..0d1cd1be1d95e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getAckTimeout; @@ -29,6 +30,9 @@ @ServerlessScope(Scope.PUBLIC) public class RestPutDataStreamLifecycleAction extends BaseRestHandler { + private static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method"; + private static final Set CAPABILITIES = Set.of(SUPPORTS_DOWNSAMPLING_METHOD); + @Override public String getName() { return "put_data_lifecycles_action"; @@ -44,13 +48,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try (XContentParser parser = request.contentParser()) { PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest( parser, - (dataRetention, enabled, downsampling) -> new PutDataStreamLifecycleAction.Request( + (dataRetention, enabled, downsamplingRounds, downsamplingMethod) -> new PutDataStreamLifecycleAction.Request( getMasterNodeTimeout(request), getAckTimeout(request), Strings.splitStringByCommaToArray(request.param("name")), dataRetention, enabled, - downsampling + downsamplingRounds, + downsamplingMethod ) ); putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions())); @@ -61,4 +66,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli ); } } + + @Override + public Set supportedCapabilities() { + return CAPABILITIES; + } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 4bf6ccfbfa7ce..56c8f764a3d2d 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -171,6 +171,58 @@ teardown: - match: { data_streams.1.lifecycle.downsampling.1.after: '100d'} - match: { data_streams.1.lifecycle.downsampling.1.fixed_interval: '10h'} +--- +"Put data stream lifecycle with downsampling method": + - requires: + capabilities: + - method: PUT + path: /_data_stream/{name}/_lifecycle + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + indices.put_data_lifecycle: + name: "*" + body: > + { + "downsampling": [ + { + "after": "10d", + "fixed_interval": "1h" + }, + { + "after": "100d", + "fixed_interval": "10h" + } + ], + "downsampling_method": "aggregate", + "data_retention": "30d", + "enabled": false + } + + - is_true: acknowledged + + - do: + indices.get_data_lifecycle: + name: "*" + - length: { data_streams: 2 } + - match: { data_streams.0.name: data-stream-with-lifecycle } + - match: { data_streams.0.lifecycle.data_retention: "30d" } + - match: { data_streams.0.lifecycle.enabled: false} + - match: { data_streams.0.lifecycle.downsampling.0.after: '10d'} + - match: { data_streams.0.lifecycle.downsampling.0.fixed_interval: '1h'} + - match: { data_streams.0.lifecycle.downsampling.1.after: '100d'} + - match: { data_streams.0.lifecycle.downsampling.1.fixed_interval: '10h'} + - match: { data_streams.1.name: simple-data-stream1 } + - match: { data_streams.1.lifecycle.data_retention: "30d" } + - match: { data_streams.1.lifecycle.enabled: false} + - match: { data_streams.1.lifecycle.downsampling_method: 'aggregate'} + - match: { data_streams.1.lifecycle.downsampling.0.after: '10d'} + - match: { data_streams.1.lifecycle.downsampling.0.fixed_interval: '1h'} + - match: { data_streams.1.lifecycle.downsampling.1.after: '100d'} + - match: { data_streams.1.lifecycle.downsampling.1.fixed_interval: '10h'} + --- "Enable lifecycle": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml index 800dec2a795a4..c230371cc4003 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml @@ -142,6 +142,38 @@ - match: {component_templates.0.component_template.template.lifecycle.enabled: true} - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} +--- +"Add data stream lifecycle with downsampling": + - requires: + capabilities: + - method: PUT + path: /_component_template/{name} + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + cluster.put_component_template: + name: test-lifecycle + body: + template: + lifecycle: + data_retention: "10d" + downsampling_method: last_value + downsampling: + - {"after": "1d", "fixed_interval": "5m"} + + - do: + cluster.get_component_template: + name: test-lifecycle + + - match: {component_templates.0.name: test-lifecycle} + - match: {component_templates.0.component_template.template.lifecycle.enabled: true} + - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling_method: "last_value"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling.0.after: "1d"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling.0.fixed_interval: "5m"} + --- "Get data stream lifecycle with default rollover": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml index 81068f460b2b3..b538293d668ea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml @@ -197,3 +197,37 @@ name: baz - match: {index_templates.0.name: baz} + +--- +"Add data stream lifecycle with downsampling": + - requires: + capabilities: + - method: PUT + path: /_index_template/{name} + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + indices.put_index_template: + name: test-lifecycle + body: + index_patterns: downsampling-* + data_stream: {} + template: + lifecycle: + data_retention: "10d" + downsampling_method: last_value + downsampling: + - {"after": "1d", "fixed_interval": "5m"} + + - do: + indices.get_index_template: + name: test-lifecycle + + - match: {index_templates.0.name: test-lifecycle} + - match: {index_templates.0.index_template.template.lifecycle.enabled: true} + - match: {index_templates.0.index_template.template.lifecycle.data_retention: "10d"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling_method: "last_value"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling.0.after: "1d"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling.0.fixed_interval: "5m"} diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index e0c6bbc79bd3e..b293f195d1864 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -33,6 +34,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DATA_RETENTION_FIELD; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_FIELD; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_METHOD_FIELD; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.ENABLED_FIELD; /** @@ -50,7 +52,8 @@ public interface Factory { Request create( @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsampling + @Nullable List downsampling, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ); } @@ -58,7 +61,12 @@ Request create( public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "put_data_stream_lifecycle_request", false, - (args, factory) -> factory.create((TimeValue) args[0], (Boolean) args[1], (List) args[2]) + (args, factory) -> factory.create( + (TimeValue) args[0], + (Boolean) args[1], + (List) args[2], + (DownsampleConfig.SamplingMethod) args[3] + ) ); static { @@ -75,6 +83,12 @@ Request create( DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY ); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()), + DOWNSAMPLING_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); } public static Request parseRequest(XContentParser parser, Factory factory) { @@ -120,7 +134,7 @@ public void writeTo(StreamOutput out) throws IOException { } public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable TimeValue dataRetention) { - this(masterNodeTimeout, ackTimeout, names, dataRetention, null, null); + this(masterNodeTimeout, ackTimeout, names, dataRetention, null); } public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamLifecycle lifecycle) { @@ -136,7 +150,7 @@ public Request( @Nullable TimeValue dataRetention, @Nullable Boolean enabled ) { - this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null); + this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null, null); } public Request( @@ -145,7 +159,8 @@ public Request( String[] names, @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsamplingRounds + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { super(masterNodeTimeout, ackTimeout); this.names = names; @@ -153,6 +168,7 @@ public Request( .dataRetention(dataRetention) .enabled(enabled == null || enabled) .downsamplingRounds(downsamplingRounds) + .downsamplingMethod(downsamplingMethod) .build(); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java index aff4287ecbc58..d790b5ddb70db 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java @@ -32,10 +32,12 @@ public class RestPutComponentTemplateAction extends BaseRestHandler { public static final String SUPPORTS_FAILURE_STORE_LIFECYCLE = "data_stream_options.failure_store.lifecycle"; public static final String SUPPORTS_FAILURE_STORE = "data_stream_options.failure_store"; private static final String COMPONENT_TEMPLATE_TRACKING_INFO = "component_template_tracking_info"; + static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method"; private static final Set CAPABILITIES = Set.of( SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE, - COMPONENT_TEMPLATE_TRACKING_INFO + COMPONENT_TEMPLATE_TRACKING_INFO, + SUPPORTS_DOWNSAMPLING_METHOD ); @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java index 28f5430775004..15f713ea583d7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java @@ -25,6 +25,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; +import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_DOWNSAMPLING_METHOD; import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_FAILURE_STORE; import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_FAILURE_STORE_LIFECYCLE; @@ -32,10 +33,12 @@ public class RestPutComposableIndexTemplateAction extends BaseRestHandler { private static final String INDEX_TEMPLATE_TRACKING_INFO = "index_template_tracking_info"; + private static final Set CAPABILITIES = Set.of( SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE, - INDEX_TEMPLATE_TRACKING_INFO + INDEX_TEMPLATE_TRACKING_INFO, + SUPPORTS_DOWNSAMPLING_METHOD ); @Override From bf80af50db4d9d2fd6676b953ab893bd1df4b951 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 23 Oct 2025 15:29:42 +0300 Subject: [PATCH 07/19] Use the sampling method in the data stream lifecycle --- .../lifecycle/DataStreamLifecycleService.java | 32 ++-- .../DataStreamLifecycleDownsampleIT.java | 168 +++++++++++++++--- ...ampleShardPersistentTaskExecutorTests.java | 2 +- 3 files changed, 168 insertions(+), 34 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 06f311b0177c0..f606bec90be25 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -524,7 +524,10 @@ Set maybeExecuteDownsampling(ProjectState projectState, DataStream dataSt // - has matching downsample rounds // - is read-only // So let's wait for an in-progress downsampling operation to succeed or trigger the last matching round - affectedIndices.addAll(waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, project)); + var downsamplingMethod = dataStream.getDataLifecycle().downsamplingMethod(); + affectedIndices.addAll( + waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, downsamplingMethod, project) + ); } } @@ -541,6 +544,7 @@ private Set waitForInProgressOrTriggerDownsampling( DataStream dataStream, IndexMetadata backingIndex, List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod, ProjectMetadata project ) { assert dataStream.getIndices().contains(backingIndex.getIndex()) @@ -568,7 +572,8 @@ private Set waitForInProgressOrTriggerDownsampling( INDEX_DOWNSAMPLE_STATUS.get(targetDownsampleIndexMeta.getSettings()), round, lastRound, - index, + downsamplingMethod, + backingIndex, targetDownsampleIndexMeta.getIndex() ); if (downsamplingNotComplete.isEmpty() == false) { @@ -580,7 +585,7 @@ private Set waitForInProgressOrTriggerDownsampling( // no maintenance needed for previously started downsampling actions and we are on the last matching round so it's time // to kick off downsampling affectedIndices.add(index); - downsampleIndexOnce(round, project.id(), indexName, downsampleIndexName); + downsampleIndexOnce(round, downsamplingMethod, project.id(), backingIndex, downsampleIndexName); } } } @@ -592,16 +597,22 @@ private Set waitForInProgressOrTriggerDownsampling( */ private void downsampleIndexOnce( DataStreamLifecycle.DownsamplingRound round, + DownsampleConfig.SamplingMethod requestedDownsamplingMethod, ProjectId projectId, - String sourceIndex, + IndexMetadata sourceIndexMetadata, String downsampleIndexName ) { + var sourceIndexSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); + String sourceIndex = sourceIndexMetadata.getIndex().getName(); DownsampleAction.Request request = new DownsampleAction.Request( TimeValue.THIRTY_SECONDS /* TODO should this be longer/configurable? */, sourceIndex, downsampleIndexName, null, - new DownsampleConfig(round.fixedInterval(), null) + new DownsampleConfig( + round.fixedInterval(), + sourceIndexSamplingMethod == null ? requestedDownsamplingMethod : sourceIndexSamplingMethod + ) ); transportActionsDeduplicator.executeOnce( Tuple.tuple(projectId, request), @@ -632,11 +643,12 @@ private Set evaluateDownsampleStatus( IndexMetadata.DownsampleTaskStatus downsampleStatus, DataStreamLifecycle.DownsamplingRound currentRound, DataStreamLifecycle.DownsamplingRound lastRound, - Index backingIndex, + DownsampleConfig.SamplingMethod downsamplingMethod, + IndexMetadata backingIndex, Index downsampleIndex ) { Set affectedIndices = new HashSet<>(); - String indexName = backingIndex.getName(); + String indexName = backingIndex.getIndex().getName(); String downsampleIndexName = downsampleIndex.getName(); return switch (downsampleStatus) { case UNKNOWN -> { @@ -683,15 +695,15 @@ private Set evaluateDownsampleStatus( // NOTE that the downsample request is made through the deduplicator so it will only really be executed if // there isn't one already in-flight. This can happen if a previous request timed-out, failed, or there was a // master failover and data stream lifecycle needed to restart - downsampleIndexOnce(currentRound, projectId, indexName, downsampleIndexName); - affectedIndices.add(backingIndex); + downsampleIndexOnce(currentRound, downsamplingMethod, projectId, backingIndex, downsampleIndexName); + affectedIndices.add(backingIndex.getIndex()); yield affectedIndices; } case SUCCESS -> { if (dataStream.getIndices().contains(downsampleIndex) == false) { // at this point the source index is part of the data stream and the downsample index is complete but not // part of the data stream. we need to replace the source index with the downsample index in the data stream - affectedIndices.add(backingIndex); + affectedIndices.add(backingIndex.getIndex()); replaceBackingIndexWithDownsampleIndexOnce(projectId, dataStream, indexName, downsampleIndexName); } yield affectedIndices; diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index adb72ceb47ce0..ba3059a38c12b 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -9,12 +9,19 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.HashSet; @@ -25,6 +32,7 @@ import static org.elasticsearch.cluster.metadata.ClusterChangedEventUtils.indicesCreated; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class DataStreamLifecycleDownsampleIT extends DownsamplingIntegTestCase { @@ -41,7 +49,9 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testDownsampling() throws Exception { String dataStreamName = "metrics-foo"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(downsamplingMethod) .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), @@ -61,34 +71,34 @@ public void testDownsampling() throws Exception { List backingIndices = getDataStreamBackingIndexNames(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); - String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; - String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; + String fiveMinuteDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; + String tenMinuteDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; Set witnessedDownsamplingIndices = new HashSet<>(); clusterService().addListener(event -> { - if (indicesCreated(event).contains(oneSecondDownsampleIndex) - || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(oneSecondDownsampleIndex))) { - witnessedDownsamplingIndices.add(oneSecondDownsampleIndex); + if (indicesCreated(event).contains(fiveMinuteDownsampleIndex) + || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(fiveMinuteDownsampleIndex))) { + witnessedDownsamplingIndices.add(fiveMinuteDownsampleIndex); } - if (indicesCreated(event).contains(tenSecondsDownsampleIndex)) { - witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); + if (indicesCreated(event).contains(tenMinuteDownsampleIndex)) { + witnessedDownsamplingIndices.add(tenMinuteDownsampleIndex); } }); // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with - // testing so DSL doesn't have to wait for the end_time to lapse) + // testing, so DSL doesn't have to wait for the end_time to lapse) putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { // first downsampling round - assertThat(witnessedDownsamplingIndices.contains(oneSecondDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(fiveMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { assertThat(witnessedDownsamplingIndices.size(), is(2)); - assertThat(witnessedDownsamplingIndices.contains(oneSecondDownsampleIndex), is(true)); - assertThat(witnessedDownsamplingIndices.contains(tenSecondsDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(fiveMinuteDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(tenMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -98,16 +108,19 @@ public void testDownsampling() throws Exception { String writeIndex = dsBackingIndices.get(1); assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); // the last downsampling round must remain in the data stream - assertThat(dsBackingIndices.get(0), is(tenSecondsDownsampleIndex)); - assertThat(indexExists(oneSecondDownsampleIndex), is(false)); + assertThat(dsBackingIndices.get(0), is(tenMinuteDownsampleIndex)); + assertThat(indexExists(fiveMinuteDownsampleIndex), is(false)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, tenMinuteDownsampleIndex); } @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception { String dataStreamName = "metrics-bar"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(downsamplingMethod) .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), @@ -128,28 +141,28 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception List backingIndices = getDataStreamBackingIndexNames(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); - String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; - String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; + String fiveMinuteDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; + String tenMinuteDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; Set witnessedDownsamplingIndices = new HashSet<>(); clusterService().addListener(event -> { - if (indicesCreated(event).contains(oneSecondDownsampleIndex) - || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(oneSecondDownsampleIndex))) { - witnessedDownsamplingIndices.add(oneSecondDownsampleIndex); + if (indicesCreated(event).contains(fiveMinuteDownsampleIndex) + || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(fiveMinuteDownsampleIndex))) { + witnessedDownsamplingIndices.add(fiveMinuteDownsampleIndex); } - if (indicesCreated(event).contains(tenSecondsDownsampleIndex)) { - witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); + if (indicesCreated(event).contains(tenMinuteDownsampleIndex)) { + witnessedDownsamplingIndices.add(tenMinuteDownsampleIndex); } }); // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with - // testing so DSL doesn't have to wait for the end_time to lapse) + // testing, so DSL doesn't have to wait for the end_time to lapse) putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { assertThat(witnessedDownsamplingIndices.size(), is(1)); // only the ten seconds downsample round should've been executed - assertThat(witnessedDownsamplingIndices.contains(tenSecondsDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(tenMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -158,8 +171,9 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception assertThat(dsBackingIndices.size(), is(2)); String writeIndex = dsBackingIndices.get(1); assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); - assertThat(dsBackingIndices.get(0), is(tenSecondsDownsampleIndex)); + assertThat(dsBackingIndices.get(0), is(tenMinuteDownsampleIndex)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, tenMinuteDownsampleIndex); } @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") @@ -168,7 +182,9 @@ public void testUpdateDownsampleRound() throws Exception { // we expect the earlier round to be ignored String dataStreamName = "metrics-baz"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(downsamplingMethod) .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), @@ -253,5 +269,111 @@ public void testUpdateDownsampleRound() throws Exception { assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); assertThat(dsBackingIndices.get(0), is(thirtySecondsDownsampleIndex)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, thirtySecondsDownsampleIndex); + } + + /** + * This test ensures that when we change the sampling method, the already downsampled indices will use the original sampling method, + * while the raw data ones will be downsampled with the most recent configuration. + * To achieve that, we set the following test: + * 1. Create a data stream that is downsampled with the aggregate method. + * 2. Rollover and wait for the downsampling to occur + * 3. Double the downsample interval (so it can downsample the first index as well) and change the sampling method. + * 4. Rollover and wait for both indices to be downsampled with the new interval + * 5. Check that the indices have been downsampled with the correct method. + */ + public void testUpdateDownsampleSamplingMode() throws Exception { + String dataStreamName = "metrics-baz"; + DownsampleConfig.SamplingMethod initialSamplingMethod = randomBoolean() + ? null + : randomFrom(DownsampleConfig.SamplingMethod.values()); + DownsampleConfig.SamplingMethod updatedSamplingMethod = initialSamplingMethod == DownsampleConfig.SamplingMethod.LAST_VALUE + ? (randomBoolean() ? null : DownsampleConfig.SamplingMethod.AGGREGATE) + : DownsampleConfig.SamplingMethod.LAST_VALUE; + + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(initialSamplingMethod) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("5m"))) + ) + .buildTemplate(); + + // Start and end time there just to ease with testing, so DLM doesn't have to wait for the end_time to lapse + // First backing index. + setupTSDBDataStreamAndIngestDocs( + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); + + // before we roll over, we update the index template to have new start/end time boundaries + // Second backing index + putTSDBIndexTemplate(dataStreamName, "2022-01-08T23:40:53.384Z", "2023-01-08T23:40:53.384Z", lifecycle); + RolloverResponse rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); + assertTrue(rolloverResponse.isRolledOver()); + String firstBackingIndex = rolloverResponse.getOldIndex(); + String secondBackingIndex = rolloverResponse.getNewIndex(); + indexDocuments(dataStreamName, randomIntBetween(1, 1000), "2022-01-08T23:50:00"); + + // Ensure that the first backing index has been downsampled + final var waitForInitialDownsampling = ClusterServiceUtils.addMasterTemporaryStateListener(clusterState -> { + final var dataStream = clusterState.metadata().getProject().dataStreams().get(dataStreamName); + if (dataStream == null) { + return false; + } + return dataStream.getIndices().size() > 1 && dataStream.getIndices().getFirst().getName().startsWith("downsample-"); + }); + safeAwait(waitForInitialDownsampling); + + // update the lifecycle so that the sampling method is different. + DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(updatedSamplingMethod) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(5), new DateHistogramInterval("10m"))) + ) + .build(); + assertAcked( + client().execute( + PutDataStreamLifecycleAction.INSTANCE, + new PutDataStreamLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + new String[] { dataStreamName }, + updatedLifecycle + ) + ) + ); + + // Third backing index + putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); + rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); + assertTrue(rolloverResponse.isRolledOver()); + String downsampledPrefix = "downsample-10m-"; + final var waitForUpdatedDownsamplingRound = ClusterServiceUtils.addMasterTemporaryStateListener(clusterState -> { + ProjectMetadata projectMetadata = clusterState.metadata().getProject(); + final var dataStream = projectMetadata.dataStreams().get(dataStreamName); + if (dataStream == null) { + return false; + } + + return dataStream.getIndices().size() > 2 + && dataStream.getIndices().stream().filter(index -> index.getName().startsWith(downsampledPrefix)).count() == 2; + }); + safeAwait(waitForUpdatedDownsamplingRound); + assertDownsamplingMethod(initialSamplingMethod, downsampledPrefix + firstBackingIndex); + assertDownsamplingMethod(updatedSamplingMethod, downsampledPrefix + secondBackingIndex); + } + + private void assertDownsamplingMethod(DownsampleConfig.SamplingMethod downsamplingMethod, String... indexNames) { + String expected = DownsampleConfig.SamplingMethod.getOrDefault(downsamplingMethod).toString(); + GetSettingsResponse response = safeGet( + client().admin().indices().getSettings(new GetSettingsRequest(TimeValue.THIRTY_SECONDS).indices(indexNames)) + ); + for (String indexName : indexNames) { + assertThat(response.getSetting(indexName, IndexMetadata.INDEX_DOWNSAMPLE_METHOD_KEY), equalTo(expected)); + } } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java index 8f0254cdf586b..40bcfdac5b00a 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java @@ -157,7 +157,7 @@ public void testGetStatelessAssignment() { .build(); var params = new DownsampleShardTaskParams( - new DownsampleConfig(new DateHistogramInterval("1h")), + new DownsampleConfig(new DateHistogramInterval("1h"), randomSamplingMethod()), shardId.getIndexName(), 1, 1, From dd69548cb87b8417e5de3d6d18ffe7e882f2dde7 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Thu, 23 Oct 2025 15:34:18 +0300 Subject: [PATCH 08/19] Update docs/changelog/137023.yaml --- docs/changelog/137023.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/137023.yaml diff --git a/docs/changelog/137023.yaml b/docs/changelog/137023.yaml new file mode 100644 index 0000000000000..eafd4d1f2f4fd --- /dev/null +++ b/docs/changelog/137023.yaml @@ -0,0 +1,5 @@ +pr: 137023 +summary: Support choosing the downsampling method in data stream lifecycle +area: "Data streams, Downsampling" +type: enhancement +issues: [] From a328a4a39f06b991d82d20e344e89ec773186dec Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Thu, 23 Oct 2025 15:49:20 +0300 Subject: [PATCH 09/19] Update 137023.yaml --- docs/changelog/137023.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/137023.yaml b/docs/changelog/137023.yaml index eafd4d1f2f4fd..cab7620233562 100644 --- a/docs/changelog/137023.yaml +++ b/docs/changelog/137023.yaml @@ -1,5 +1,5 @@ pr: 137023 summary: Support choosing the downsampling method in data stream lifecycle -area: "Data streams, Downsampling" +area: "Data streams" type: enhancement issues: [] From ff2708c9204fe255aa66fe4a1ab339cb94cde9ab Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 23 Oct 2025 16:39:13 +0300 Subject: [PATCH 10/19] Fix test --- .../cluster/metadata/DataStreamLifecycleTemplateTests.java | 1 + .../elasticsearch/cluster/metadata/DataStreamLifecycleTests.java | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 95c6431d17b5f..2bcdfc7dda641 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -54,6 +54,7 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa : DataStreamLifecycle.LifecycleType.DATA; if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { downsamplingRounds = ResettableValue.undefined(); + downsamplingMethod = ResettableValue.undefined(); } } case 1 -> enabled = enabled == false; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index eed9be4b9d2c7..9cc27d99c350e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -74,6 +74,7 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw } else { lifecycleTarget = DataStreamLifecycle.LifecycleType.FAILURES; downsamplingRounds = null; + downsamplingMethod = null; } } case 1 -> { From 53e707997e4bafb1cab5d504d295721050d0731e Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 29 Oct 2025 13:35:22 +0200 Subject: [PATCH 11/19] Rename transport version --- .../cluster/metadata/DataStreamLifecycle.java | 14 +++++++------- ...lm.csv => add_sample_method_downsample_dlm.csv} | 0 .../main/resources/transport/upper_bounds/9.3.csv | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) rename server/src/main/resources/transport/definitions/referable/{add_last_value_downsample_dlm.csv => add_sample_method_downsample_dlm.csv} (100%) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index cbc913758bc5e..bb87ab1994ace 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -58,7 +58,7 @@ public class DataStreamLifecycle implements SimpleDiffable, // Versions over the wire public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X; - public static final TransportVersion ADD_LAST_VALUE_DOWNSAMPLE_DLM = TransportVersion.fromName("add_last_value_downsample_dlm"); + public static final TransportVersion ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM = TransportVersion.fromName("add_sample_method_downsample_dlm"); public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention"; public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode"; @@ -383,7 +383,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } - if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) { + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { out.writeOptionalWriteable(downsamplingMethod); } } @@ -410,7 +410,7 @@ public DataStreamLifecycle(StreamInput in) throws IOException { enabled = true; } lifecycleType = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; - downsamplingMethod = in.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM) + downsamplingMethod = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) ? in.readOptionalWriteable(DownsampleConfig.SamplingMethod::read) : null; } @@ -628,7 +628,7 @@ static void validateRounds(List rounds) { public static DownsamplingRound read(StreamInput in) throws IOException { TimeValue after = in.readTimeValue(); - DateHistogramInterval fixedInterval = in.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM) + DateHistogramInterval fixedInterval = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) ? new DateHistogramInterval(in) : new DownsampleConfig(in).getFixedInterval(); return new DownsamplingRound(after, fixedInterval); @@ -645,7 +645,7 @@ public static DownsamplingRound read(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeTimeValue(after); - if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) { + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { out.writeWriteable(fixedInterval); } else { out.writeWriteable(new DownsampleConfig(fixedInterval, null)); @@ -819,7 +819,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } - if (out.getTransportVersion().supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM)) { + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { ResettableValue.write(out, downsamplingMethod, StreamOutput::writeWriteable); } } @@ -883,7 +883,7 @@ public static Template read(StreamInput in) throws IOException { ? LifecycleType.read(in) : LifecycleType.DATA; ResettableValue downsamplingMethod = in.getTransportVersion() - .supports(ADD_LAST_VALUE_DOWNSAMPLE_DLM) + .supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) ? ResettableValue.read(in, DownsampleConfig.SamplingMethod::read) : ResettableValue.undefined(); return new Template(lifecycleTarget, enabled, dataRetention, downsamplingRounds, downsamplingMethod); diff --git a/server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv b/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv similarity index 100% rename from server/src/main/resources/transport/definitions/referable/add_last_value_downsample_dlm.csv rename to server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index b4e2c0d9d1878..19f05b14d0d95 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -add_last_value_downsample_dlm,9204000 +add_sample_method_downsample_dlm,9204000 From 67099d354600b553de7b972f26c38a46f1f8e1df Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 29 Oct 2025 16:50:46 +0200 Subject: [PATCH 12/19] Fix and simplify DataStreamLifecycleTests & DataStreamLifecycleTemplateTests --- .../metadata/DataStreamLifecycleTemplateTests.java | 8 +++++--- .../cluster/metadata/DataStreamLifecycleTests.java | 13 +++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 2bcdfc7dda641..e17d6891b71d6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -63,14 +63,16 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa downsamplingRounds = randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTemplateTests::randomDownsamplingRounds); if (downsamplingRounds.get() != null) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; + } else { + downsamplingMethod = ResettableValue.undefined(); } } case 4 -> { - if (downsamplingRounds.get() == null) { + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DataStreamLifecycleTemplateTests::randomDownsamplingMethod); + if (downsamplingMethod.get() != null && downsamplingRounds.get() == null) { downsamplingRounds = ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } - downsamplingMethod = randomValueOtherThan(downsamplingMethod, DataStreamLifecycleTemplateTests::randomDownsamplingMethod); } default -> throw new AssertionError("Illegal randomisation branch"); } @@ -219,7 +221,7 @@ public static DataStreamLifecycle.Template randomDataLifecycleTemplate() { downsamplingRounds, downsamplingRounds.get() == null ? randomBoolean() ? ResettableValue.undefined() : ResettableValue.reset() - : ResettableValue.undefined() + : randomDownsamplingMethod() ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 9cc27d99c350e..64bc13cb0dc98 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; @@ -100,18 +99,12 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw } } case 3 -> { - // We need to enable downsampling in order to add a non value downsampling method - if (downsamplingRounds == null) { + // We need to enable downsampling in order to add a non-value downsampling method + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DownsampleConfigTests::randomSamplingMethod); + if (downsamplingMethod != null && downsamplingRounds == null) { downsamplingRounds = randomDownsamplingRounds(); lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } - if (downsamplingMethod == null) { - downsamplingMethod = randomFrom(DownsampleConfig.SamplingMethod.values()); - } else if (downsamplingMethod == DownsampleConfig.SamplingMethod.AGGREGATE) { - downsamplingMethod = randomBoolean() ? null : DownsampleConfig.SamplingMethod.LAST_VALUE; - } else { - downsamplingMethod = randomBoolean() ? null : DownsampleConfig.SamplingMethod.AGGREGATE; - } } default -> enabled = enabled == false; } From 58a4994dfc373a33d93b4175c38ed6640394d9ed Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 29 Oct 2025 16:51:28 +0200 Subject: [PATCH 13/19] Make random sampling method more dynamic --- .../lifecycle/DataStreamLifecycleFixtures.java | 12 ++++++------ .../action/downsample/DownsampleConfigTests.java | 11 +++++------ .../xpack/downsample/DownsamplingIntegTestCase.java | 11 +++++------ .../downsample/DownsampleActionSingleNodeTests.java | 11 +++++------ .../DataStreamLifecycleDownsamplingSecurityIT.java | 11 +++++++++++ 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 1411f2f4e99fd..72be905f000e2 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -40,6 +40,7 @@ import static org.elasticsearch.test.ESIntegTestCase.client; import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.frequently; +import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.junit.Assert.assertTrue; @@ -184,11 +185,10 @@ private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecyc } public static DownsampleConfig.SamplingMethod randomSamplingMethod() { - return switch (between(0, 2)) { - case 0 -> null; - case 1 -> DownsampleConfig.SamplingMethod.AGGREGATE; - case 2 -> DownsampleConfig.SamplingMethod.LAST_VALUE; - default -> throw new IllegalStateException("Unknown randomisation path"); - }; + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } } } diff --git a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java index 4ca89da4c9413..3469564b28729 100644 --- a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java +++ b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java @@ -50,12 +50,11 @@ public static DownsampleConfig randomConfig() { } public static DownsampleConfig.SamplingMethod randomSamplingMethod() { - return switch (between(0, 2)) { - case 0 -> null; - case 1 -> DownsampleConfig.SamplingMethod.AGGREGATE; - case 2 -> DownsampleConfig.SamplingMethod.LAST_VALUE; - default -> throw new AssertionError("Illegal randomisation branch"); - }; + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } } @Override diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java index 90af97526ff34..9c1a38e1ac969 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java @@ -336,11 +336,10 @@ private static boolean hasTimeSeriesDimensionTrue(Map fieldMapping) { } public static DownsampleConfig.SamplingMethod randomSamplingMethod() { - return switch (between(0, 2)) { - case 0 -> null; - case 1 -> DownsampleConfig.SamplingMethod.AGGREGATE; - case 2 -> DownsampleConfig.SamplingMethod.LAST_VALUE; - default -> throw new IllegalStateException("Unexpected randomisation branch"); - }; + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index abdb2164d8bb8..19dcaa18ead2b 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1824,11 +1824,10 @@ public void testDuplicateDownsampleRequest() throws Exception { } static DownsampleConfig.SamplingMethod randomSamplingMethod() { - return switch (between(0, 2)) { - case 0 -> null; - case 1 -> DownsampleConfig.SamplingMethod.AGGREGATE; - case 2 -> DownsampleConfig.SamplingMethod.LAST_VALUE; - default -> throw new IllegalStateException("Unexpected randomisation branch"); - }; + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 87b9b482d428e..c37cd36dacb64 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.lifecycle.ErrorEntry; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; @@ -120,6 +121,7 @@ public void testDownsamplingAuthorized() throws Exception { String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(randomSamplingMethod()) .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), @@ -401,9 +403,18 @@ private void bulkIndex(Client client, String dataStreamName, Supplier Indexed [{}] documents. Dropped [{}] duplicates.", docsIndexed, duplicates); } + private static DownsampleConfig.SamplingMethod randomSamplingMethod() { + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } + } + public static class SystemDataStreamWithDownsamplingConfigurationPlugin extends Plugin implements SystemIndexPlugin { public static final DataStreamLifecycle.Template LIFECYCLE = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(randomSamplingMethod()) .downsamplingRounds( List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), From a1d989cf8def0935b8faaaeedc5a4417b0967d5b Mon Sep 17 00:00:00 2001 From: gmarouli Date: Wed, 29 Oct 2025 16:51:46 +0200 Subject: [PATCH 14/19] Improvements based on review --- .../DataStreamLifecycleDownsampleIT.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index ba3059a38c12b..7cfbdef90a2bd 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -189,7 +189,7 @@ public void testUpdateDownsampleRound() throws Exception { List.of( new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at - // least 2 seconds since rollover. only the 10 seconds round should be executed. + // least 2 seconds since rollover. Only the 10 seconds round should be executed. new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) @@ -240,7 +240,7 @@ public void testUpdateDownsampleRound() throws Exception { // update the lifecycle so that it only has one round, for the same `after` parameter as before, but a different interval // the different interval should yield a different downsample index name so we expect the data stream lifecycle to get the previous - // `10s` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10s` one. + // `10m` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10m` one. DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsamplingRounds( List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("20m"))) @@ -276,7 +276,7 @@ public void testUpdateDownsampleRound() throws Exception { * This test ensures that when we change the sampling method, the already downsampled indices will use the original sampling method, * while the raw data ones will be downsampled with the most recent configuration. * To achieve that, we set the following test: - * 1. Create a data stream that is downsampled with the aggregate method. + * 1. Create a data stream that is downsampled with a sampling method. * 2. Rollover and wait for the downsampling to occur * 3. Double the downsample interval (so it can downsample the first index as well) and change the sampling method. * 4. Rollover and wait for both indices to be downsampled with the new interval @@ -299,7 +299,7 @@ public void testUpdateDownsampleSamplingMode() throws Exception { .buildTemplate(); // Start and end time there just to ease with testing, so DLM doesn't have to wait for the end_time to lapse - // First backing index. + // Creating the first backing index. setupTSDBDataStreamAndIngestDocs( dataStreamName, "1986-01-08T23:40:53.384Z", @@ -310,7 +310,7 @@ public void testUpdateDownsampleSamplingMode() throws Exception { ); // before we roll over, we update the index template to have new start/end time boundaries - // Second backing index + // Creating the second backing index. putTSDBIndexTemplate(dataStreamName, "2022-01-08T23:40:53.384Z", "2023-01-08T23:40:53.384Z", lifecycle); RolloverResponse rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); assertTrue(rolloverResponse.isRolledOver()); @@ -319,16 +319,16 @@ public void testUpdateDownsampleSamplingMode() throws Exception { indexDocuments(dataStreamName, randomIntBetween(1, 1000), "2022-01-08T23:50:00"); // Ensure that the first backing index has been downsampled - final var waitForInitialDownsampling = ClusterServiceUtils.addMasterTemporaryStateListener(clusterState -> { + awaitClusterState(clusterState -> { final var dataStream = clusterState.metadata().getProject().dataStreams().get(dataStreamName); if (dataStream == null) { return false; } return dataStream.getIndices().size() > 1 && dataStream.getIndices().getFirst().getName().startsWith("downsample-"); }); - safeAwait(waitForInitialDownsampling); - - // update the lifecycle so that the sampling method is different. + assertDownsamplingMethod(initialSamplingMethod, "downsample-5m-" + firstBackingIndex); + // We change the sampling method, but also we double the downsampling interval. We expect the data stream lifecycle to get the + // previous `5m` interval downsampled index, downsample it to `10m` and replace it in the data stream with the `5m` one. DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() .downsamplingMethod(updatedSamplingMethod) .downsamplingRounds( @@ -347,7 +347,7 @@ public void testUpdateDownsampleSamplingMode() throws Exception { ) ); - // Third backing index + // We roll over one more time, so the second backing index will be eligible for downsampling putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); assertTrue(rolloverResponse.isRolledOver()); From 231506c29eac1f0adc8ead27828d200f01783ae7 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 31 Oct 2025 13:56:50 +0200 Subject: [PATCH 15/19] Add explanation about DLM using the source sampling method --- .../datastreams/lifecycle/DataStreamLifecycleService.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index f606bec90be25..5a998f43d0103 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -602,6 +602,14 @@ private void downsampleIndexOnce( IndexMetadata sourceIndexMetadata, String downsampleIndexName ) { + // When an index is already downsampled with a method, we require all later downsampling rounds to use the same method. + // This is necessary to preserve the relation of the downsampled index to the raw data. For example, if an index is already + // downsampled and downsampled it again to 1 hour; we know that a document represents either the aggregated raw data of an hour + // or the last value of the raw data within this hour. If we mix the methods, we cannot derive any meaning from them. + // Furthermore, data stream lifecycle is configured on the data stream level and not on the individual index level, meaning that + // when a user changes downsampling method, some indices would not be able to be downsampled anymore. + // For this reason, when we encounter an already downsampled index, we use the source downsampling method which might be different + // from the requested one. var sourceIndexSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); String sourceIndex = sourceIndexMetadata.getIndex().getName(); DownsampleAction.Request request = new DownsampleAction.Request( From 8d4fa3dbd127a4b1511efd73fafdda3f021136b6 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 31 Oct 2025 14:21:04 +0200 Subject: [PATCH 16/19] Test template composition of data stream lifecycle. --- .../MetadataIndexTemplateServiceTests.java | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index a29aa3fa3b221..4677ca1aed9b3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.PutRequest; @@ -1109,6 +1110,15 @@ public void testResolveLifecycle() throws Exception { .buildTemplate(); String ct45d = "ct_45d"; project = addComponentTemplate(service, project, ct45d, lifecycle45d); + DataStreamLifecycle.Template lifecycle60d = DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(TimeValue.timeValueDays(60)) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(7), new DateHistogramInterval("3h"))) + ) + .downsamplingMethod(DownsampleConfig.SamplingMethod.LAST_VALUE) + .buildTemplate(); + String ct60d = "ct_60d"; + project = addComponentTemplate(service, project, ct60d, lifecycle60d); DataStreamLifecycle.Template lifecycleNullRetention = DataStreamLifecycle.createDataLifecycleTemplate( true, @@ -1119,6 +1129,15 @@ public void testResolveLifecycle() throws Exception { String ctNullRetention = "ct_null_retention"; project = addComponentTemplate(service, project, ctNullRetention, lifecycleNullRetention); + DataStreamLifecycle.Template lifecycleNullDownsampling = DataStreamLifecycle.createDataLifecycleTemplate( + true, + ResettableValue.undefined(), + ResettableValue.reset(), + ResettableValue.reset() + ); + String ctNullDownsampling = "ct_null_downsampling"; + project = addComponentTemplate(service, project, ctNullDownsampling, lifecycleNullDownsampling); + String ctEmptyLifecycle = "ct_empty_lifecycle"; project = addComponentTemplate(service, project, ctEmptyLifecycle, emptyLifecycle); @@ -1220,6 +1239,72 @@ public void testResolveLifecycle() throws Exception { // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} // Result: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} assertLifecycleResolution(service, project, List.of(ct30d, ctDisabledLifecycle), lifecycle45d, lifecycle45d); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Result: "lifecycle": { + // "retention": "45d", + // "downsampling": [{"after": "30d", "fixed_interval": "3h"}], + // "downsampling_method": "last_value" + // } + assertLifecycleResolution( + service, + project, + List.of(ct60d), + lifecycle45d, + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(lifecycle45d.dataRetention()) + .downsamplingMethod(lifecycle60d.downsamplingMethod()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) + .buildTemplate() + ); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Component B: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Composable Z: "lifecycle": {"downsampling": null, "downsampling_method": null} + // Result: "lifecycle": {"retention": "45d"} + assertLifecycleResolution( + service, + project, + List.of(ct60d, ct45d), + lifecycleNullDownsampling, + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(lifecycle45d.dataRetention()).buildTemplate() + ); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Result: "lifecycle": { + // "retention": "45d", + // "downsampling": [{"after": "30d", "fixed_interval": "3h"}], + // "downsampling_method": "last_value" + // } + assertLifecycleResolution( + service, + project, + List.of(ct60d), + DataStreamLifecycle.createDataLifecycleTemplate( + true, + ResettableValue.undefined(), + ResettableValue.undefined(), + ResettableValue.reset() + ), + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(lifecycle60d.dataRetention()) + .downsamplingRounds(lifecycle60d.downsamplingRounds()) + .buildTemplate() + ); } public void testResolveFailureStore() throws Exception { From 3c10f8b1d0a1d9804ba8ed85e0ad3eba666a6733 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 31 Oct 2025 15:05:55 +0200 Subject: [PATCH 17/19] Fix test --- .../MetadataIndexTemplateServiceTests.java | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index 563cb86b79cd0..44854c5f0d5ba 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.datastreams; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; @@ -146,13 +147,14 @@ public void testLifecycleComposition() { List lifecycles = List.of(); assertThat(composeDataLifecycles(lifecycles), nullValue()); } - // One lifecycle results to this lifecycle as the final + // One lifecycle results in this lifecycle as the final { + ResettableValue> downsamplingRounds = randomDownsampling(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( true, randomRetention(), - randomDownsampling(), - ResettableValue.create(DataStreamLifecycleFixtures.randomSamplingMethod()) + downsamplingRounds, + randomSamplingMethod(downsamplingRounds) ); List lifecycles = List.of(lifecycle); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); @@ -160,6 +162,7 @@ public void testLifecycleComposition() { assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle.downsamplingMethod().get())); } // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false @@ -175,26 +178,31 @@ public void testLifecycleComposition() { assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle.downsamplingMethod().get())); } - // If both lifecycle have all properties, then the latest one overwrites all the others + // If both lifecycles have all properties, then the latest one overwrites all the others { + DownsampleConfig.SamplingMethod downsamplingMethod1 = randomFrom(DownsampleConfig.SamplingMethod.LAST_VALUE); DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), randomRounds(), - DataStreamLifecycleFixtures.randomSamplingMethod() + downsamplingMethod1 ); DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate( true, randomPositiveTimeValue(), randomRounds(), - DataStreamLifecycleFixtures.randomSamplingMethod() + downsamplingMethod1 == DownsampleConfig.SamplingMethod.LAST_VALUE + ? DownsampleConfig.SamplingMethod.AGGREGATE + : DownsampleConfig.SamplingMethod.LAST_VALUE ); List lifecycles = List.of(lifecycle1, lifecycle2); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(lifecycle2.enabled())); assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get())); assertThat(result.downsamplingRounds(), equalTo(lifecycle2.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle2.downsamplingMethod().get())); } } @@ -292,4 +300,18 @@ private static ResettableValue> rand default -> throw new IllegalStateException("Unknown randomisation path"); }; } + + private static ResettableValue randomSamplingMethod( + ResettableValue> rounds + ) { + if (rounds.get() == null) { + return randomBoolean() ? ResettableValue.undefined() : ResettableValue.reset(); + } else { + return randomBoolean() ? ResettableValue.create(DataStreamLifecycleFixtures.randomSamplingMethod()) : ResettableValue.reset(); + } + } + + private static DownsampleConfig.SamplingMethod randomSamplingMethod(List rounds) { + return rounds == null ? null : DataStreamLifecycleFixtures.randomSamplingMethod(); + } } From fcd8dd9e89af4d3970233d5f6635f10e482544bc Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 31 Oct 2025 15:12:04 +0200 Subject: [PATCH 18/19] Remove unused constructor --- .../action/downsample/DownsampleConfig.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java index 5f3b4f71d753d..ae08245ceb7c9 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java @@ -101,17 +101,6 @@ public class DownsampleConfig implements NamedWriteable, ToXContentObject { ); } - /** - * Create a new {@link DownsampleConfig} using the given configuration parameters. - * @param fixedInterval the fixed interval to use for computing the date histogram for the rolled up documents (required). - * @deprecated please use {@link DownsampleConfig#DownsampleConfig(DateHistogramInterval, SamplingMethod)}, this method is being kept - * until the sampling method is completely integrated with ILM and DLM. - */ - @Deprecated - public DownsampleConfig(final DateHistogramInterval fixedInterval) { - this(fixedInterval, null); - } - /** * Create a new {@link DownsampleConfig} using the given configuration parameters. * @param fixedInterval the fixed interval to use for computing the date histogram for the rolled up documents (required). From e23f767b2899bad27b47ad14d062cf14c2422f83 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Fri, 31 Oct 2025 21:53:10 +0200 Subject: [PATCH 19/19] Fix test adding sampling method when there are no rounds --- .../MetadataIndexTemplateServiceTests.java | 24 +++++-------------- .../DataStreamLifecycleFixtures.java | 17 +++++++++---- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index 44854c5f0d5ba..574c30376137c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.EmptySystemIndices; @@ -44,6 +43,8 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.composeDataLifecycles; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.MetadataDataStreamRolloverServiceTests.createSettingsProvider; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomResettable; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomSamplingMethod; import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -154,7 +155,7 @@ public void testLifecycleComposition() { true, randomRetention(), downsamplingRounds, - randomSamplingMethod(downsamplingRounds) + randomResettable(() -> randomSamplingMethod(downsamplingRounds.get())) ); List lifecycles = List.of(lifecycle); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); @@ -167,11 +168,12 @@ public void testLifecycleComposition() { // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false { + List downsamplingRounds = randomRounds(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds(), - DataStreamLifecycleFixtures.randomSamplingMethod() + downsamplingRounds, + randomSamplingMethod(downsamplingRounds) ); List lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); @@ -300,18 +302,4 @@ private static ResettableValue> rand default -> throw new IllegalStateException("Unknown randomisation path"); }; } - - private static ResettableValue randomSamplingMethod( - ResettableValue> rounds - ) { - if (rounds.get() == null) { - return randomBoolean() ? ResettableValue.undefined() : ResettableValue.reset(); - } else { - return randomBoolean() ? ResettableValue.create(DataStreamLifecycleFixtures.randomSamplingMethod()) : ResettableValue.reset(); - } - } - - private static DownsampleConfig.SamplingMethod randomSamplingMethod(List rounds) { - return rounds == null ? null : DataStreamLifecycleFixtures.randomSamplingMethod(); - } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 72be905f000e2..c0625e34f3153 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -145,15 +145,18 @@ static void putComposableIndexTemplate( } static DataStreamLifecycle.Template randomDataLifecycleTemplate() { + ResettableValue> downsampling = randomResettable( + DataStreamLifecycleFixtures::randomDownsamplingRounds + ); return DataStreamLifecycle.createDataLifecycleTemplate( frequently(), randomResettable(ESTestCase::randomTimeValue), - randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds), - randomResettable(DataStreamLifecycleFixtures::randomSamplingMethod) + downsampling, + randomResettable(() -> randomSamplingMethod(downsampling.get())) ); } - private static ResettableValue randomResettable(Supplier supplier) { + public static ResettableValue randomResettable(Supplier supplier) { return switch (randomIntBetween(0, 2)) { case 0 -> ResettableValue.undefined(); case 1 -> ResettableValue.reset(); @@ -184,8 +187,12 @@ private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecyc return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } - public static DownsampleConfig.SamplingMethod randomSamplingMethod() { - if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + /** + * In order to produce valid data stream lifecycle configurations, the sampling method can be defined only when + * the downsampling rounds are also defined. + */ + public static DownsampleConfig.SamplingMethod randomSamplingMethod(List downsamplingRounds) { + if (downsamplingRounds == null || between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { return null; } else { return randomFrom(DownsampleConfig.SamplingMethod.values());