From 571938b6c764b76a1316e0bf98b008d13bd820b6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:46 -0400 Subject: [PATCH 01/37] refactor(evp): centralize EVP proxy routing --- .../communication/BackendApiFactory.java | 54 +++-- .../java/datadog/communication/EvpProxy.java | 15 ++ .../datadog/communication/EvpProxyApi.java | 8 +- .../ddagent/DDAgentFeaturesDiscovery.java | 8 +- .../DDAgentFeaturesDiscoveryTest.groovy | 6 + .../communication/BackendApiFactoryTest.java | 191 ++++++++++++++++++ .../common/writer/ddintake/DDEvpProxyApi.java | 4 +- .../writer/ddintake/LLMObsSpanMapper.java | 3 +- .../writer/ddintake/DDEvpProxyApiTest.groovy | 5 +- 9 files changed, 270 insertions(+), 24 deletions(-) create mode 100644 communication/src/main/java/datadog/communication/EvpProxy.java create mode 100644 communication/src/test/java/datadog/communication/BackendApiFactoryTest.java diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 3ce78b88c22..ba5d68554ba 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -24,6 +24,11 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni } public @Nullable BackendApi createBackendApi(Intake intake) { + return createBackendApi(intake, null, true); + } + + public @Nullable BackendApi createBackendApi( + Intake intake, @Nullable String preferredEvpProxyEndpoint, boolean responseCompression) { HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true); if (intake.isAgentlessEnabled(config)) { @@ -46,23 +51,40 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); featuresDiscovery.discoverIfOutdated(); - if (featuresDiscovery.supportsEvpProxy()) { - String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); - String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); - HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); - String subdomain = intake.getUrlPrefix(); - return new EvpProxyApi( - traceId, - evpProxyUrl, - subdomain, - retryPolicyFactory, - sharedCommunicationObjects.agentHttpClient, - true); + String evpProxyEndpoint; + if (preferredEvpProxyEndpoint != null) { + if (!featuresDiscovery.supportsEvpProxyEndpoint(preferredEvpProxyEndpoint)) { + log.warn( + "Cannot create backend API client for {} since agent does not support requested EVP" + + " proxy endpoint {}", + intake, + preferredEvpProxyEndpoint); + return null; + } + evpProxyEndpoint = preferredEvpProxyEndpoint; + } else if (featuresDiscovery.supportsEvpProxy()) { + evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); + } else { + log.warn( + "Cannot create backend API client since agentless mode is disabled, " + + "and agent does not support EVP proxy"); + return null; } - log.warn( - "Cannot create backend API client since agentless mode is disabled, " - + "and agent does not support EVP proxy"); - return null; + String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); + log.debug( + "Creating EVP proxy client for {} using endpoint {} with responseCompression={}", + intake, + evpProxyEndpoint, + responseCompression); + HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); + String subdomain = intake.getUrlPrefix(); + return new EvpProxyApi( + traceId, + evpProxyUrl, + subdomain, + retryPolicyFactory, + sharedCommunicationObjects.agentHttpClient, + responseCompression); } } diff --git a/communication/src/main/java/datadog/communication/EvpProxy.java b/communication/src/main/java/datadog/communication/EvpProxy.java new file mode 100644 index 00000000000..c2453bccb25 --- /dev/null +++ b/communication/src/main/java/datadog/communication/EvpProxy.java @@ -0,0 +1,15 @@ +package datadog.communication; + +/** Shared EVP proxy constants. */ +public final class EvpProxy { + + public static final String SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; + + /** + * Default SDK-side target for uncompressed EVP request bodies. Writers may split batches at or + * below this size to keep Agent proxy requests comfortably bounded. + */ + public static final int PAYLOAD_SIZE_LIMIT_BYTES = 5 * 1024 * 1024; + + private EvpProxy() {} +} diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 83037ab9663..b00838c68f3 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -20,7 +20,6 @@ public class EvpProxyApi implements BackendApi { private static final Logger log = LoggerFactory.getLogger(EvpProxyApi.class); private static final String API_VERSION = "v2"; - private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id"; private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id"; private static final String ACCEPT_ENCODING_HEADER = "Accept-Encoding"; @@ -62,7 +61,7 @@ public T post( Request.Builder requestBuilder = new Request.Builder() .url(url) - .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, subdomain) + .addHeader(EvpProxy.SUBDOMAIN_HEADER, subdomain) .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); @@ -79,6 +78,11 @@ public T post( } final Request request = requestBuilder.post(requestBody).build(); + log.debug( + "Posting EVP request to {} with responseCompression={} requestCompression={}", + url, + responseCompression, + requestCompression); try (okhttp3.Response response = OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) { diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index 7cef3e02924..ba263a7f6e6 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -98,6 +98,7 @@ private static class State { String debuggerSnapshotEndpoint; String debuggerDiagnosticsEndpoint; String evpProxyEndpoint; + Set evpProxyEndpoints = emptySet(); String version; String telemetryProxyEndpoint; Set peerTags = emptySet(); @@ -140,7 +141,7 @@ protected long getFeaturesDiscoveryMinDelayMillis() { private synchronized void discoverIfOutdated(final long maxElapsedMs) { final long now = System.currentTimeMillis(); final long elapsed = now - discoveryState.lastTimeDiscovered; - if (elapsed > maxElapsedMs) { + if (elapsed >= maxElapsedMs) { final State newState = new State(); doDiscovery(newState); newState.lastTimeDiscovered = now; @@ -288,6 +289,7 @@ private boolean processInfoResponse(State newState, String response) { break; } } + newState.evpProxyEndpoints = unmodifiableSet(endpoints); for (String endpoint : telemetryProxyEndpoints) { if (containsEndpoint(endpoints, endpoint)) { @@ -422,6 +424,10 @@ public String getEvpProxyEndpoint() { return discoveryState.evpProxyEndpoint; } + public boolean supportsEvpProxyEndpoint(String endpoint) { + return containsEndpoint(discoveryState.evpProxyEndpoints, endpoint); + } + public HttpUrl buildUrl(String endpoint) { return agentBaseUrl.resolve(endpoint); } diff --git a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy index 8bcda6eb811..aab3c240d92 100644 --- a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy +++ b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy @@ -6,6 +6,8 @@ import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V05_ENDPOIN import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V06_METRICS_ENDPOINT import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V07_CONFIG_ENDPOINT import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V1_ENDPOINT +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_ID import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_TAGS_HASH import static datadog.trace.api.ProtocolVersion.V0_4 @@ -75,6 +77,8 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { features.supportsEvpProxy() features.supportsContentEncodingHeadersWithEvpProxy() features.getEvpProxyEndpoint() == "evp_proxy/v4/" + features.supportsEvpProxyEndpoint(V2_EVP_PROXY_ENDPOINT) + features.supportsEvpProxyEndpoint(V4_EVP_PROXY_ENDPOINT) features.getVersion() == "0.99.0" !features.supportsLongRunning() !features.supportsTelemetryProxy() @@ -489,6 +493,8 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { 1 * client.newCall(_) >> { Request request -> infoResponse(request, INFO_WITH_OLD_EVP_PROXY) } features.supportsEvpProxy() features.getEvpProxyEndpoint() == "evp_proxy/v2/" // v3 is advertised, but the tracer should ignore it + features.supportsEvpProxyEndpoint(V2_EVP_PROXY_ENDPOINT) + !features.supportsEvpProxyEndpoint(V4_EVP_PROXY_ENDPOINT) !features.supportsContentEncodingHeadersWithEvpProxy() features.supportsDebugger() features.getDebuggerSnapshotEndpoint() == "debugger/v1/diagnostics" diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java new file mode 100644 index 00000000000..c287790b4ae --- /dev/null +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -0,0 +1,191 @@ +package datadog.communication; + +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.metrics.api.Monitoring; +import datadog.trace.api.Config; +import datadog.trace.api.ProtocolVersion; +import datadog.trace.api.intake.Intake; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.Test; + +class BackendApiFactoryTest { + + private static final MediaType JSON = MediaType.parse("application/json"); + + @Test + void preferredEvpProxyEndpointMustBeAdvertisedByAgent() { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery( + V4_EVP_PROXY_ENDPOINT, Collections.singleton(V4_EVP_PROXY_ENDPOINT)); + final BackendApiFactory factory = + new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); + + assertNull(factory.createBackendApi(Intake.EVENT_PLATFORM, V2_EVP_PROXY_ENDPOINT, false)); + } + + @Test + void preferredEvpProxyEndpointUsesRequestedRouteWhenAdvertised() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery( + V4_EVP_PROXY_ENDPOINT, + new HashSet<>(Arrays.asList(V4_EVP_PROXY_ENDPOINT, V2_EVP_PROXY_ENDPOINT))); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = + factory.createBackendApi(Intake.EVENT_PLATFORM, V2_EVP_PROXY_ENDPOINT, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v2/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + @Test + void preferredEvpProxyEndpointDoesNotRequireLegacyDefaultRoute() throws Exception { + final String futureEndpoint = "evp_proxy/v9/"; + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery(null, Collections.singleton(futureEndpoint)); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = + factory.createBackendApi(Intake.EVENT_PLATFORM, futureEndpoint, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v9/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + @Test + void defaultEvpProxyEndpointSupportsDisabledResponseCompression() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = + new FakeFeaturesDiscovery( + V4_EVP_PROXY_ENDPOINT, + new HashSet<>(Arrays.asList(V4_EVP_PROXY_ENDPOINT, V2_EVP_PROXY_ENDPOINT))); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, null, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); + } finally { + agent.shutdown(); + } + } + + private static SharedCommunicationObjects sharedCommunicationObjects( + final DDAgentFeaturesDiscovery discovery, final HttpUrl agentUrl) { + final TestSharedCommunicationObjects sco = new TestSharedCommunicationObjects(discovery); + sco.agentUrl = agentUrl != null ? agentUrl : HttpUrl.get("http://localhost:8126/"); + sco.agentHttpClient = new OkHttpClient(); + return sco; + } + + private static final class TestSharedCommunicationObjects extends SharedCommunicationObjects { + private final DDAgentFeaturesDiscovery discovery; + + private TestSharedCommunicationObjects(final DDAgentFeaturesDiscovery discovery) { + this.discovery = discovery; + } + + @Override + public DDAgentFeaturesDiscovery featuresDiscovery(final Config config) { + return discovery; + } + } + + private static final class FakeFeaturesDiscovery extends DDAgentFeaturesDiscovery { + private final String evpProxyEndpoint; + private final Set evpProxyEndpoints; + + private FakeFeaturesDiscovery( + final String evpProxyEndpoint, final Set evpProxyEndpoints) { + super( + new OkHttpClient(), + Monitoring.DISABLED, + HttpUrl.get("http://localhost:8126/"), + ProtocolVersion.V0_5, + true, + false); + this.evpProxyEndpoint = evpProxyEndpoint; + this.evpProxyEndpoints = evpProxyEndpoints; + } + + @Override + public void discoverIfOutdated() {} + + @Override + public String getEvpProxyEndpoint() { + return evpProxyEndpoint; + } + + @Override + public boolean supportsEvpProxyEndpoint(final String endpoint) { + return evpProxyEndpoints.contains(endpoint) || evpProxyEndpoints.contains("/" + endpoint); + } + + @Override + public boolean supportsEvpProxy() { + return evpProxyEndpoint != null; + } + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java index e911a980c31..9ec4eca5f4e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddintake/DDEvpProxyApi.java @@ -3,6 +3,7 @@ import static datadog.trace.common.writer.DDIntakeWriter.DEFAULT_INTAKE_TIMEOUT; import static datadog.trace.common.writer.DDIntakeWriter.DEFAULT_INTAKE_VERSION; +import datadog.communication.EvpProxy; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.trace.api.civisibility.InstrumentationBridge; @@ -26,7 +27,6 @@ public class DDEvpProxyApi extends RemoteApi { private static final Logger log = LoggerFactory.getLogger(DDEvpProxyApi.class); - private static final String DD_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; private static final String CONTENT_ENCODING_HEADER = "Content-Encoding"; private static final String GZIP_CONTENT_TYPE = "gzip"; @@ -131,7 +131,7 @@ public Response sendSerializedTraces(Payload payload) { Request.Builder builder = new Request.Builder() .url(proxiedApiUrl) - .addHeader(DD_EVP_SUBDOMAIN_HEADER, subdomain) + .addHeader(EvpProxy.SUBDOMAIN_HEADER, subdomain) .tag(OkHttpUtils.CustomListener.class, telemetryListener); if (isCompressionEnabled()) { diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7849052b9d3..1f14502cc27 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -2,6 +2,7 @@ import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf; +import datadog.communication.EvpProxy; import datadog.communication.serialization.GrowableBuffer; import datadog.communication.serialization.Writable; import datadog.communication.serialization.msgpack.MsgPackWriter; @@ -99,7 +100,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private int spansWritten; public LLMObsSpanMapper() { - this(5 << 20); + this(EvpProxy.PAYLOAD_SIZE_LIMIT_BYTES); } private LLMObsSpanMapper(int size) { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy index fd797695d99..52ad90a9514 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/ddintake/DDEvpProxyApiTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.common.writer.ddintake import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper +import datadog.communication.EvpProxy import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter @@ -64,7 +65,7 @@ class DDEvpProxyApiTest extends DDCoreSpecification { clientResponse.status().present clientResponse.status().asInt == 200 agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(DDEvpProxyApi.DD_EVP_SUBDOMAIN_HEADER) == intakeSubdomain + agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain cleanup: agentEvpProxy.close() @@ -100,7 +101,7 @@ class DDEvpProxyApiTest extends DDCoreSpecification { clientResponse.status().present clientResponse.status().asInt == 200 agentEvpProxy.getLastRequest().path == path - agentEvpProxy.getLastRequest().getHeader(DDEvpProxyApi.DD_EVP_SUBDOMAIN_HEADER) == intakeSubdomain + agentEvpProxy.getLastRequest().getHeader(EvpProxy.SUBDOMAIN_HEADER) == intakeSubdomain cleanup: agentEvpProxy.close() From 4949ed7ee4b3c67b386a6b27c2fc214d905565cd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:55 -0400 Subject: [PATCH 02/37] refactor(feature-flagging): share EVP publishing --- .../featureflag/ExposureWriterImpl.java | 33 ++------ .../featureflag/FeatureFlagEvpContext.java | 22 ++++++ .../featureflag/FeatureFlagEvpPublisher.java | 79 +++++++++++++++++++ .../FeatureFlagEvpPublisherTest.java | 72 +++++++++++++++++ 4 files changed, 179 insertions(+), 27 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index eddcd520f27..1333fcead13 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -4,25 +4,19 @@ import static datadog.trace.util.AgentThreadFactory.newAgentThread; import static java.util.concurrent.TimeUnit.SECONDS; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.Moshi; import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; -import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.ExposuresRequest; -import datadog.trace.api.intake.Intake; import datadog.trace.api.internal.VisibleForTesting; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import okhttp3.RequestBody; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +26,7 @@ public class ExposureWriterImpl implements ExposureWriter { private static final int DEFAULT_CAPACITY = 1 << 16; // 65536 elements private static final int DEFAULT_FLUSH_INTERVAL_IN_SECONDS = 1; private static final int FLUSH_THRESHOLD = 100; + private static final String EXPOSURES_ROUTE = "exposures"; private final MessagePassingBlockingQueue queue; private final Thread serializerThread; @@ -47,21 +42,13 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con final SharedCommunicationObjects sco, final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); - final Map context = new HashMap<>(4); - context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); - if (config.getEnv() != null) { - context.put("env", config.getEnv()); - } - if (config.getVersion() != null) { - context.put("version", config.getVersion()); - } final ExposureSerializingHandler serializer = new ExposureSerializingHandler( new BackendApiFactory(config, sco), queue, flushInterval, timeUnit, - context, + FeatureFlagEvpContext.from(config), this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EXPOSURE_PROCESSOR, serializer); } @@ -100,10 +87,7 @@ private static class ExposureSerializingHandler implements Runnable { private final long ticksRequiredToFlush; private long lastTicks; - private final JsonAdapter jsonAdapter; - private final BackendApiFactory backendApiFactory; - private BackendApi evp; - + private final FeatureFlagEvpPublisher evpPublisher; private final Map context; private final ExposureCache cache; @@ -119,8 +103,7 @@ public ExposureSerializingHandler( final Runnable errorCallback) { this.queue = queue; this.cache = new LRUExposureCache(queue.capacity()); - this.jsonAdapter = new Moshi.Builder().build().adapter(ExposuresRequest.class); - this.backendApiFactory = backendApiFactory; + this.evpPublisher = new FeatureFlagEvpPublisher<>(backendApiFactory, ExposuresRequest.class); this.context = context; this.lastTicks = System.nanoTime(); @@ -133,8 +116,7 @@ public ExposureSerializingHandler( @Override public void run() { - evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM); - if (evp == null) { + if (!evpPublisher.start()) { errorCallback.run(); throw new IllegalArgumentException("EVP Proxy not available"); } @@ -180,10 +162,7 @@ protected void flushIfNecessary() { if (shouldFlush()) { try { final ExposuresRequest exposures = new ExposuresRequest(this.context, this.buffer); - final String reqBod = jsonAdapter.toJson(exposures); - final RequestBody requestBody = - RequestBody.create(okhttp3.MediaType.parse("application/json"), reqBod); - evp.post("exposures", requestBody, stream -> null, null, false); + evpPublisher.post(EXPOSURES_ROUTE, exposures); this.buffer.clear(); } catch (Exception e) { LOGGER.error("Could not submit exposures", e); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java new file mode 100644 index 00000000000..c964efa6c7f --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java @@ -0,0 +1,22 @@ +package com.datadog.featureflag; + +import datadog.trace.api.Config; +import java.util.HashMap; +import java.util.Map; + +final class FeatureFlagEvpContext { + + private FeatureFlagEvpContext() {} + + static Map from(final Config config) { + final Map context = new HashMap<>(4); + context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); + if (config.getEnv() != null) { + context.put("env", config.getEnv()); + } + if (config.getVersion() != null) { + context.put("version", config.getVersion()); + } + return context; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java new file mode 100644 index 00000000000..f0694d17aa4 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java @@ -0,0 +1,79 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.intake.Intake; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import okhttp3.MediaType; +import okhttp3.RequestBody; + +final class FeatureFlagEvpPublisher { + + private static final MediaType JSON = MediaType.parse("application/json"); + + private final BackendApiFactory backendApiFactory; + private final String preferredEvpProxyEndpoint; + private final boolean responseCompression; + private final JsonAdapter jsonAdapter; + private BackendApi evp; + + FeatureFlagEvpPublisher(final BackendApiFactory backendApiFactory, final Class requestType) { + this(backendApiFactory, requestType, null, true); + } + + FeatureFlagEvpPublisher( + final BackendApiFactory backendApiFactory, + final Class requestType, + final boolean responseCompression) { + this(backendApiFactory, requestType, null, responseCompression); + } + + FeatureFlagEvpPublisher( + final BackendApiFactory backendApiFactory, + final Class requestType, + final String preferredEvpProxyEndpoint, + final boolean responseCompression) { + this.backendApiFactory = backendApiFactory; + this.preferredEvpProxyEndpoint = preferredEvpProxyEndpoint; + this.responseCompression = responseCompression; + this.jsonAdapter = new Moshi.Builder().build().adapter(requestType); + } + + boolean start() { + if (evp == null) { + evp = + preferredEvpProxyEndpoint == null && responseCompression + ? backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM) + : backendApiFactory.createBackendApi( + Intake.EVENT_PLATFORM, preferredEvpProxyEndpoint, responseCompression); + } + return evp != null; + } + + void post(final String route, final T request) throws IOException { + post(route, serialize(request)); + } + + byte[] serialize(final T request) { + return utf8Bytes(jsonAdapter.toJson(request)); + } + + void post(final String route, final byte[] json) throws IOException { + if (!start()) { + throw new IllegalStateException("EVP Proxy not available"); + } + final RequestBody requestBody = RequestBody.create(JSON, json); + evp.post(route, requestBody, stream -> null, null, false); + } + + static byte[] utf8Bytes(final String json) { + try { + return json.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError("UTF-8 must be available", e); + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java new file mode 100644 index 00000000000..b5044f43c7a --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -0,0 +1,72 @@ +package com.datadog.featureflag; + +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.intake.Intake; +import okhttp3.RequestBody; +import org.junit.jupiter.api.Test; + +class FeatureFlagEvpPublisherTest { + + @Test + void defaultPublisherUsesDefaultBackendApiFactoryPath() { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM)).thenReturn(backendApi); + + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class); + + publisher.start(); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM); + verifyNoMoreInteractions(factory); + } + + @Test + void responseCompressionCanBeDisabledWithoutPinnedEndpoint() { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM, null, false)).thenReturn(backendApi); + + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class, false); + + publisher.start(); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM, null, false); + } + + @Test + void preferredEndpointAndRequestCompressionAreForwardedToBackendApi() throws Exception { + final BackendApi backendApi = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(Intake.EVENT_PLATFORM, V4_EVP_PROXY_ENDPOINT, false)) + .thenReturn(backendApi); + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class, V4_EVP_PROXY_ENDPOINT, false); + + publisher.post("flagevaluation", new TestRequest("value")); + + verify(factory).createBackendApi(Intake.EVENT_PLATFORM, V4_EVP_PROXY_ENDPOINT, false); + verify(backendApi) + .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); + } + + static class TestRequest { + public final String value; + + TestRequest(final String value) { + this.value = value; + } + } +} From 4aa06476b70d15ecf5298ccc5f16f6e1ce229cbd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:56 -0400 Subject: [PATCH 03/37] feat(openfeature): add flagevaluation contract --- .../api/config/FeatureFlaggingConfig.java | 8 ++ metadata/supported-configurations.json | 8 ++ .../featureflag/FeatureFlaggingGateway.java | 29 +++++ .../flagevaluation/FlagEvalEvent.java | 108 ++++++++++++++++++ .../flagevaluation/FlagEvaluationWriter.java | 27 +++++ 5 files changed, 180 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..5f567908254 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -3,4 +3,12 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + /** + * Killswitch for the EVP {@code flagevaluation} emission path. Default: enabled. Disabling it + * turns off EVP flag-evaluation counts while leaving the OTel {@code feature_flag.evaluations} + * metric path untouched. Maps to {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED}. + */ + public static final String FLAGGING_EVALUATION_COUNTS_ENABLED = + "flagging.evaluation.counts.enabled"; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index ae081138b71..7206f291599 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1489,6 +1489,14 @@ "aliases": [] } ], + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": [] + } + ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index b9d73ffa7ab..a0856fc0ab3 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -1,6 +1,7 @@ package datadog.trace.api.featureflag; import datadog.trace.api.featureflag.exposure.ExposureEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -19,6 +20,15 @@ public interface ExposureListener extends Consumer {} private static final AtomicReference CURRENT_CONFIG = new AtomicReference<>(); + /** + * The active EVP flagevaluation writer. Registered by {@code FlagEvaluationWriterImpl.start()} + * when the killswitch {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} is on (default). Read by + * {@code FlagEvalLoggingHook} to route evaluations into the two-tier aggregator. {@code null} + * when the EVP path is disabled. + */ + private static final AtomicReference FLAG_EVAL_WRITER = + new AtomicReference<>(); + private FeatureFlaggingGateway() {} public static void addConfigListener(final ConfigListener listener) { @@ -49,4 +59,23 @@ public static void removeExposureListener(final ExposureListener listener) { public static void dispatch(final ExposureEvent event) { EXPOSURE_LISTENERS.forEach(listener -> listener.accept(event)); } + + /** + * Registers the active EVP flagevaluation writer. Called by {@code + * FlagEvaluationWriterImpl.start()} when the feature is enabled. Replaces any previously + * registered writer. + * + * @param writer the writer to register, or {@code null} to deregister + */ + public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { + FLAG_EVAL_WRITER.set(writer); + } + + /** + * Returns the active EVP flagevaluation writer, or {@code null} when disabled (killswitch off or + * not yet started). + */ + public static FlagEvaluationWriter getFlagEvalWriter() { + return FLAG_EVAL_WRITER.get(); + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java new file mode 100644 index 00000000000..83397e5dccd --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -0,0 +1,108 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Lightweight data record capturing a single flag evaluation for EVP flagevaluation emission. + * + *

This is the currency passed from the {@code FlagEvalLoggingHook} (feature-flagging-api) to the + * {@code FlagEvaluationWriter} (feature-flagging-lib) via a non-blocking bounded queue. + * + *

Scalar fields are captured at hook-fire time on the evaluation thread. Context attributes can + * be supplied lazily so recursive flattening happens on the writer thread, not on the evaluation + * path. No aggregation happens here. + */ +public final class FlagEvalEvent { + + /** The feature flag key. Never null. */ + public final String flagKey; + + /** + * The OpenFeature variant key selected for the evaluation. {@code null} means the default value + * was returned (runtime default). + */ + public final String variant; + + /** The allocation key from flag metadata ("allocationKey"). May be null. */ + public final String allocationKey; + + /** The targeting key from the evaluation context. May be null. */ + public final String targetingKey; + + /** + * The evaluation error message when the evaluation failed, else {@code null}. Sourced from the + * OpenFeature evaluation details (error message, falling back to the error code). + */ + public final String errorMessage; + + /** + * Evaluation timestamp in milliseconds since epoch. Stamped at eval-entry time from flag metadata + * key {@code "dd.eval.timestamp_ms"}, or falls back to hook-fire time when absent. This ensures + * first/last_evaluation reflect evaluation time, not hook-fire time. + */ + public final long evalTimeMs; + + /** + * Flattened evaluation context attributes. Used for the full-tier canonical context key. May be + * empty but never null. + */ + public final Map attrs; + + private final Supplier> attrsSupplier; + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + this(flagKey, variant, allocationKey, targetingKey, null, evalTimeMs, attrs); + } + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final Map attrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.evalTimeMs = evalTimeMs; + this.attrs = attrs != null ? attrs : Collections.emptyMap(); + this.attrsSupplier = null; + } + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final Supplier> attrsSupplier) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.evalTimeMs = evalTimeMs; + this.attrs = Collections.emptyMap(); + this.attrsSupplier = attrsSupplier; + } + + public Map contextAttributes() { + if (attrsSupplier == null) { + return attrs; + } + final Map supplied = attrsSupplier.get(); + return supplied != null ? supplied : Collections.emptyMap(); + } +} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java new file mode 100644 index 00000000000..9bfe40d11ca --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java @@ -0,0 +1,27 @@ +package datadog.trace.api.featureflag.flagevaluation; + +/** + * Defines an EVP flagevaluation writer responsible for aggregating flag evaluation events and + * flushing them to the EVP proxy. + * + *

Implementations must use a background thread (serializing handler) for aggregation and + * transport. The {@link #enqueue(FlagEvalEvent)} method must be non-blocking and callable from the + * OpenFeature hook thread without backpressure. + */ +public interface FlagEvaluationWriter extends AutoCloseable { + + /** + * Non-blocking enqueue of a flag evaluation event. May silently drop the event if the internal + * bounded queue is full (best-effort, observable via drop counter). + * + * @param event the flag evaluation event captured at hook-fire time + */ + void enqueue(FlagEvalEvent event); + + /** Starts the background serializing thread. Must be called once after construction. */ + void start(); + + /** Stops the background thread and releases resources. */ + @Override + void close(); +} From ac126fb44a2a143612b3bda30ed605c42faef8f8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:56 -0400 Subject: [PATCH 04/37] refactor(openfeature): name metrics hook explicitly --- ...{FlagEvalHook.java => FlagEvalMetricsHook.java} | 4 ++-- .../datadog/trace/api/openfeature/Provider.java | 12 ++++++------ ...lHookTest.java => FlagEvalMetricsHookTest.java} | 14 +++++++------- .../trace/api/openfeature/ProviderTest.java | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) rename products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/{FlagEvalHook.java => FlagEvalMetricsHook.java} (91%) rename products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/{FlagEvalHookTest.java => FlagEvalMetricsHookTest.java} (89%) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java similarity index 91% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java rename to products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java index 1132602a53f..a3e0dbfc83c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetricsHook.java @@ -7,11 +7,11 @@ import dev.openfeature.sdk.ImmutableMetadata; import java.util.Map; -class FlagEvalHook implements Hook { +class FlagEvalMetricsHook implements Hook { private final FlagEvalMetrics metrics; - FlagEvalHook(FlagEvalMetrics metrics) { + FlagEvalMetricsHook(FlagEvalMetrics metrics) { this.metrics = metrics; } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index c5e8898fa4a..670d4d3642d 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -34,7 +34,7 @@ public class Provider extends EventProvider implements Metadata { private final AtomicReference initializationState = new AtomicReference<>(InitializationState.NOT_STARTED); private final FlagEvalMetrics flagEvalMetrics; - private final FlagEvalHook flagEvalHook; + private final FlagEvalMetricsHook flagEvalMetricsHook; public Provider() { this(DEFAULT_OPTIONS, null); @@ -48,16 +48,16 @@ public Provider(final Options options) { this.options = options; this.evaluator = evaluator; FlagEvalMetrics metrics = null; - FlagEvalHook hook = null; + FlagEvalMetricsHook hook = null; try { metrics = new FlagEvalMetrics(); - hook = new FlagEvalHook(metrics); + hook = new FlagEvalMetricsHook(metrics); } catch (LinkageError | Exception e) { // This outer catch fires when the metrics helper itself can't load (OTel API absent). log.warn("Evaluation metrics unavailable — OTel API classes not on classpath", e); } this.flagEvalMetrics = metrics; - this.flagEvalHook = hook; + this.flagEvalMetricsHook = hook; } @Override @@ -167,10 +167,10 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - if (flagEvalHook == null) { + if (flagEvalMetricsHook == null) { return Collections.emptyList(); } - return Collections.singletonList(flagEvalHook); + return Collections.singletonList(flagEvalMetricsHook); } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java similarity index 89% rename from products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java rename to products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java index 8ed17d91cbb..322a06d2de7 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalMetricsHookTest.java @@ -13,12 +13,12 @@ import java.util.Collections; import org.junit.jupiter.api.Test; -class FlagEvalHookTest { +class FlagEvalMetricsHookTest { @Test void finallyAfterRecordsBasicEvaluation() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -44,7 +44,7 @@ void finallyAfterRecordsBasicEvaluation() { @Test void finallyAfterRecordsErrorEvaluation() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -68,7 +68,7 @@ void finallyAfterRecordsErrorEvaluation() { @Test void finallyAfterHandlesNullFlagMetadata() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder() @@ -87,7 +87,7 @@ void finallyAfterHandlesNullFlagMetadata() { @Test void finallyAfterHandlesNullVariantAndReason() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); FlagEvaluationDetails details = FlagEvaluationDetails.builder().flagKey("my-flag").value("default").build(); @@ -100,7 +100,7 @@ void finallyAfterHandlesNullVariantAndReason() { @Test void finallyAfterNeverThrows() { FlagEvalMetrics metrics = mock(FlagEvalMetrics.class); - FlagEvalHook hook = new FlagEvalHook(metrics); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(metrics); // Should not throw even with completely null inputs hook.finallyAfter(null, null, null); @@ -110,7 +110,7 @@ void finallyAfterNeverThrows() { @Test void finallyAfterIsNoOpWhenMetricsIsNull() { - FlagEvalHook hook = new FlagEvalHook(null); + FlagEvalMetricsHook hook = new FlagEvalMetricsHook(null); FlagEvaluationDetails details = FlagEvaluationDetails.builder() diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 27d4dd5d2b5..06bbe7ef8d2 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -326,12 +326,12 @@ protected Class loadEvaluatorClass() throws ClassNotFoundException { } @Test - public void testGetProviderHooksReturnsFlagEvalHook() { + public void testGetProviderHooksReturnsFlagEvalMetricsHook() { Provider provider = new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)); List hooks = provider.getProviderHooks(); assertThat(hooks.size(), equalTo(1)); - assertThat(hooks.get(0) instanceof FlagEvalHook, equalTo(true)); + assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); } @Test From 7a43658da6bbf774948e73bb37af0506184fa5f0 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:57 -0400 Subject: [PATCH 05/37] feat(openfeature): add flagevaluation logging hook --- .../trace/api/openfeature/DDEvaluator.java | 6 +- .../api/openfeature/FlagEvalLoggingHook.java | 125 ++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 91c0aafdc7a..762ffe5997c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -387,11 +387,15 @@ private static ProviderEvaluation resolveVariant( + e.getMessage()); } + // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, + // not hook-fire time. Passed to the hook via provider metadata "dd.eval.timestamp_ms". + final long evalTimestampMs = System.currentTimeMillis(); final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = ImmutableMetadata.builder() .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) - .addString("allocationKey", allocation.key); + .addString("allocationKey", allocation.key) + .addLong("dd.eval.timestamp_ms", evalTimestampMs); final ProviderEvaluation result = ProviderEvaluation.builder() .value(mappedValue) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java new file mode 100644 index 00000000000..b6a08548fc0 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -0,0 +1,125 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * OpenFeature {@code Hook} that captures flag evaluation events for EVP {@code flagevaluation} + * emission. + * + *

Contract: {@code finallyAfter} does ONLY cheap scalar extraction + a non-blocking offer to the + * writer's bounded queue. No inline aggregation on the hook thread. + * + *

This hook is registered alongside the existing OTel {@link FlagEvalMetricsHook} - it does NOT + * replace it (the existing OTel metrics hook is left unchanged). + * + *

The writer is resolved lazily from {@link FeatureFlaggingGateway#getFlagEvalWriter()} on each + * call, so the hook is always safe to register - if the writer is absent (killswitch off or not yet + * started) it is a no-op. + */ +class FlagEvalLoggingHook implements Hook { + + /** + * Singleton instance: always registered when the provider is created; harmless when writer=null + * (killswitch off or not yet started). + */ + static final FlagEvalLoggingHook INSTANCE = new FlagEvalLoggingHook<>(); + + /** + * Writer resolver. Production instances resolve through {@link FeatureFlaggingGateway}; tests can + * inject a direct writer or a resolver that simulates old-bootstrap linkage failures. + */ + private final Supplier writerSupplier; + + /** Production constructor - resolves writer from gateway. */ + FlagEvalLoggingHook() { + this(FeatureFlaggingGateway::getFlagEvalWriter); + } + + /** Test-only constructor - injects a writer directly, bypassing the gateway. */ + FlagEvalLoggingHook(final FlagEvaluationWriter writer) { + this(() -> writer); + } + + /** Test-only constructor - injects a writer resolver directly, bypassing the gateway. */ + FlagEvalLoggingHook(final Supplier writerSupplier) { + this.writerSupplier = writerSupplier; + } + + /** + * Cheap capture + non-blocking enqueue only. Runs at the {@code finally} stage so it covers + * success, error, and default-value paths. + */ + @Override + public void finallyAfter( + final HookContext ctx, + final FlagEvaluationDetails details, + final Map hints) { + try { + final FlagEvaluationWriter w = writerSupplier.get(); + if (w == null || details == null) { + return; + } + + // Cheap scalar extraction - no JSON, no map lookups beyond metadata.asMap() + final String flagKey = details.getFlagKey(); + final ImmutableMetadata metadata = details.getFlagMetadata(); + + // allocationKey: "allocationKey" (camelCase) - consistent with FlagEvalMetricsHook.java + final String allocationKey = metadata != null ? metadata.getString("allocationKey") : null; + + // eval-time: from flag metadata "dd.eval.timestamp_ms" (Long), fallback to hook-fire time. + // ImmutableMetadata.getLong available since sdk 1.4+. + final Long evalTimeObj = metadata != null ? metadata.getLong("dd.eval.timestamp_ms") : null; + final long evalTimeMs = evalTimeObj != null ? evalTimeObj : System.currentTimeMillis(); + + // variant: the OpenFeature variant key (same source as the OTel FlagEvalMetricsHook), NOT the + // evaluated value. A null variant means no variant was selected (runtime default). + final String variant = details.getVariant(); + + // error message: prefer the human-readable message; fall back to the error code name when + // the message is empty (some providers populate only the code). null on success. + String errorMessage = details.getErrorMessage(); + if ((errorMessage == null || errorMessage.isEmpty()) && details.getErrorCode() != null) { + errorMessage = details.getErrorCode().name(); + } + if (errorMessage != null && errorMessage.isEmpty()) { + errorMessage = null; + } + + // targetingKey from evaluation context + final String targetingKey = + ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; + + w.enqueue( + new FlagEvalEvent( + flagKey, + variant, + allocationKey, + targetingKey, + errorMessage, + evalTimeMs, + () -> extractAttrs(ctx))); + } catch (LinkageError | Exception e) { + // Never let EVP recording break flag evaluation + } + } + + /** Extracts converted, flattened attributes from the evaluation context. */ + private Map extractAttrs(final HookContext ctx) { + if (ctx == null || ctx.getCtx() == null) { + return Collections.emptyMap(); + } + final Map attrs = DDEvaluator.flattenContext(ctx.getCtx()); + attrs.remove("targetingKey"); + return attrs; + } +} From 4ec77cae03b4a8aa4cda903ecaf860676630d359 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:57 -0400 Subject: [PATCH 06/37] test(openfeature): cover flagevaluation logging hook --- .../openfeature/FlagEvalLoggingHookTest.java | 384 ++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java new file mode 100644 index 00000000000..61643e6bb61 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -0,0 +1,384 @@ +package datadog.trace.api.openfeature; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.Reason; +import dev.openfeature.sdk.Value; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link FlagEvalLoggingHook}: cheap capture, non-blocking enqueue, eval-time + * metadata, absent-variant detection, and killswitch-via-writer-null behaviour. + */ +class FlagEvalLoggingHookTest { + + // ---- helpers ---- + + /** + * Creates a writer that captures the enqueued event for assertion. Uses an anonymous class since + * FlagEvaluationWriter has multiple abstract methods. + */ + private FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public void start() {} + + @Override + public void close() {} + }; + } + + private static FlagEvalLoggingHook hookWithWriter(final FlagEvaluationWriter writer) { + return new FlagEvalLoggingHook<>(writer); + } + + private static FlagEvaluationDetails details( + final String flagKey, + final Object value, + final String variant, + final String reason, + final ImmutableMetadata metadata) { + final FlagEvaluationDetails.FlagEvaluationDetailsBuilder builder = + FlagEvaluationDetails.builder().flagKey(flagKey).value(value).reason(reason); + if (variant != null) { + builder.variant(variant); + } + if (metadata != null) { + builder.flagMetadata(metadata); + } + return builder.build(); + } + + private static HookContext hookCtxWithTargetingKey( + final String flagKey, final String targetingKey) { + final MutableContext ctx = new MutableContext(targetingKey); + return HookContext.builder() + .flagKey(flagKey) + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + } + + // ---- test: hook calls writer.enqueue once with flagKey, variant, allocationKey ---- + + @Test + void finallyAfterEnqueuesEventWithAllBasicFields() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details( + "my-flag", + "on-value", + "on", + Reason.TARGETING_MATCH.name(), + ImmutableMetadata.builder().addString("allocationKey", "alloc-1").build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get(), "writer.enqueue must be called once"); + final FlagEvalEvent e = captured.get(); + assertEquals("my-flag", e.flagKey); + assertEquals("on", e.variant, "variant must be the OpenFeature variant key"); + assertEquals("alloc-1", e.allocationKey); + } + + // ---- variant comes from details.getVariant(), NOT details.getValue() ---- + + @Test + void variantIsTheVariantKeyNotTheEvaluatedValue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + // value and variant DIFFER, so a value-vs-variant mistake is detectable. + final FlagEvaluationDetails det = + details( + "g1-flag", + "the-evaluated-value", // value + "the-variant-key", // variant + Reason.TARGETING_MATCH.name(), + null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "the-variant-key", + captured.get().variant, + "variant must be sourced from details.getVariant(), not details.getValue()"); + } + + // ---- test: evalTimeMs from metadata "dd.eval.timestamp_ms" ---- + + @Test + void evalTimeMsComesFromMetadataWhenPresent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final long expectedTimestamp = 1_700_000_000_000L; + final FlagEvaluationDetails det = + details( + "ts-flag", + "v", + "v", + Reason.SPLIT.name(), + ImmutableMetadata.builder() + .addString("allocationKey", "a") + .addLong("dd.eval.timestamp_ms", expectedTimestamp) + .build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + expectedTimestamp, + captured.get().evalTimeMs, + "evalTimeMs must come from dd.eval.timestamp_ms metadata when present"); + } + + // ---- test: evalTimeMs falls back to System.currentTimeMillis() when absent ---- + + @Test + void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final long before = System.currentTimeMillis(); + final FlagEvaluationDetails det = + details("ts-flag", "v", "v", Reason.SPLIT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + final long after = System.currentTimeMillis(); + assertNotNull(captured.get()); + final long ts = captured.get().evalTimeMs; + assertTrue( + ts >= before && ts <= after, + "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); + } + + // ---- test: absent variant -> variant is null -> runtime default ---- + + @Test + void absentVariantProducesNullVariant() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + // A runtime default returns the default value but no variant. + final FlagEvaluationDetails det = + details("def-flag", "default-value", null, Reason.DEFAULT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().variant, "Absent variant must stay null (runtime default)"); + } + + // ---- test: error message captured from details (error object support) ---- + + @Test + void errorMessageCapturedFromDetails() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.TYPE_MISMATCH) + .errorMessage("value does not match declared type") + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "value does not match declared type", + captured.get().errorMessage, + "errorMessage must be captured from the evaluation details"); + } + + // ---- test: error code used as fallback message when error message is empty ---- + + @Test + void errorCodeUsedAsFallbackWhenMessageEmpty() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.FLAG_NOT_FOUND) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "FLAG_NOT_FOUND", + captured.get().errorMessage, + "error code name must be used when no error message is present"); + } + + // ---- test: success path has no error message ---- + + @Test + void successPathHasNullErrorMessage() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details("ok-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().errorMessage, "success path must have no error message"); + } + + // ---- test: hook does NO aggregation on the hook thread ---- + + @Test + void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + // Exactly one enqueue call, no start/close/aggregate + verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); + verify(writer, never()).close(); + verify(writer, never()).start(); + } + + // ---- test: writer=null -> no-op (killswitch off / not yet started) ---- + + @Test + void writerNullIsNoOp() { + final FlagEvalLoggingHook hook = hookWithWriter(null); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + // Must not throw; nothing is enqueued + hook.finallyAfter(null, det, Collections.emptyMap()); + } + + @Test + void writerLookupLinkageErrorIsNoOp() { + final FlagEvalLoggingHook hook = + new FlagEvalLoggingHook<>( + () -> { + throw new NoSuchMethodError("old bootstrap"); + }); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + assertDoesNotThrow(() -> hook.finallyAfter(null, det, Collections.emptyMap())); + } + + // ---- test: details=null -> no-op ---- + + @Test + void detailsNullIsNoOp() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + final FlagEvalLoggingHook hook = hookWithWriter(writer); + + // Should not throw + hook.finallyAfter(null, null, Collections.emptyMap()); + + verifyNoInteractions(writer); + } + + // ---- test: targetingKey extracted from evaluation context ---- + + @Test + void targetingKeyExtractedFromContext() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.SPLIT.name(), null); + + final HookContext hookCtx = hookCtxWithTargetingKey("ctx-flag", "user-42"); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "user-42", + captured.get().targetingKey, + "targetingKey must be extracted from the evaluation context"); + } + + @Test + void contextAttributesAreFlattenedAndConvertedAfterEnqueue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final Map profile = new HashMap<>(); + profile.put("tier", "gold"); + final Map attributes = new HashMap<>(); + attributes.put("score", 42); + attributes.put("profile", profile); + final MutableContext context = + new MutableContext(Value.objectToValue(attributes).asStructure().asMap()); + context.setTargetingKey("user-42"); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertTrue(captured.get().attrs.isEmpty(), "hook must not flatten context before enqueue"); + final Map attrs = captured.get().contextAttributes(); + assertEquals(42, attrs.get("score")); + assertEquals("gold", attrs.get("profile.tier")); + assertFalse(attrs.containsKey("targetingKey")); + assertTrue( + attrs.values().stream().noneMatch(Value.class::isInstance), + "context attrs must contain converted scalar values, not OpenFeature Value wrappers"); + } +} From 6fa7ade3e47b905b2aef4ff995f41205d6f0727a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 00:07:51 -0400 Subject: [PATCH 07/37] fix(openfeature): snapshot flagevaluation context --- .../trace/api/openfeature/DDEvaluator.java | 102 +++++++++++++++--- .../api/openfeature/FlagEvalLoggingHook.java | 23 ++-- .../api/openfeature/DDEvaluatorTest.java | 10 ++ .../openfeature/FlagEvalLoggingHookTest.java | 45 ++++++++ 4 files changed, 161 insertions(+), 19 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 762ffe5997c..0a137cc5669 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -19,6 +19,7 @@ import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.ImmutableStructure; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.Reason; import dev.openfeature.sdk.Structure; @@ -27,12 +28,16 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -504,38 +509,109 @@ private static String allocationKey(final ProviderEvaluation resolution) } static AbstractMap flattenContext(final EvaluationContext context) { - final Set keys = context.keySet(); + return flattenValues(snapshotValues(context)); + } + + static Map snapshotValues(final EvaluationContext context) { + final HashMap values = new HashMap<>(); + final Set seenContainers = Collections.newSetFromMap(new IdentityHashMap<>()); + for (final String key : context.keySet()) { + values.put(key, snapshotValue(context.getValue(key), seenContainers)); + } + return values; + } + + private static Value snapshotValue(final Value value, final Set seenContainers) { + if (value == null) { + return null; + } else if (value.isNull()) { + return new Value(); + } else if (value.isBoolean()) { + return new Value(value.asBoolean()); + } else if (value.isNumber()) { + final Object number = value.asObject(); + return number instanceof Integer + ? new Value((Integer) number) + : new Value(((Number) number).doubleValue()); + } else if (value.isString()) { + return new Value(value.asString()); + } else if (value.isInstant()) { + return new Value(value.asInstant()); + } else if (value.isList()) { + final List list = value.asList(); + if (!seenContainers.add(list)) { + return new Value(); + } + final List snapshot = new ArrayList<>(list.size()); + for (final Value item : list) { + snapshot.add(snapshotValue(item, seenContainers)); + } + seenContainers.remove(list); + return new Value(Collections.unmodifiableList(snapshot)); + } else if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (!seenContainers.add(structure)) { + return new Value(); + } + final Map snapshot = new HashMap<>(); + for (final String key : structure.keySet()) { + snapshot.put(key, snapshotValue(structure.getValue(key), seenContainers)); + } + seenContainers.remove(structure); + return new Value(new ImmutableStructure(snapshot)); + } + throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); + } + + static AbstractMap flattenValues(final Map values) { final HashMap result = new HashMap<>(); - final Set seen = new HashSet<>(); - for (final String key : keys) { + final Set seenContainers = Collections.newSetFromMap(new IdentityHashMap<>()); + for (final Map.Entry root : values.entrySet()) { final Deque deque = new LinkedList<>(); - deque.push(new FlattenEntry(key, context.getValue(key))); + deque.push(new FlattenEntry(root.getKey(), root.getValue())); while (!deque.isEmpty()) { final FlattenEntry entry = deque.pop(); final Value value = entry.value; - if (value == null || seen.add(value)) { - if (value == null) { - result.put(entry.key, null); - } else if (value.isList()) { - final List list = value.asList(); + if (value == null) { + result.put(entry.key, null); + } else if (value.isList()) { + final List list = value.asList(); + if (seenContainers.add(list)) { for (int i = 0; i < list.size(); i++) { deque.push(new FlattenEntry(entry.key + "[" + i + "]", list.get(i))); } - } else if (value.isStructure()) { - final Structure structure = value.asStructure(); + } + } else if (value.isStructure()) { + final Structure structure = value.asStructure(); + if (seenContainers.add(structure)) { for (final String property : structure.keySet()) { deque.push( new FlattenEntry(entry.key + "." + property, structure.getValue(property))); } - } else { - result.put(entry.key, context.convertValue(value)); } + } else { + result.put(entry.key, convertValue(value)); } } } return result; } + private static Object convertValue(final Value value) { + if (value == null || value.isNull()) { + return null; + } else if (value.isBoolean()) { + return value.asBoolean(); + } else if (value.isNumber()) { + return value.asObject(); + } else if (value.isString()) { + return value.asString(); + } else if (value.isInstant()) { + return value.asInstant(); + } + throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); + } + @FunctionalInterface private interface NumberComparator { boolean compare(double a, double b); diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java index b6a08548fc0..c394a154e01 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -3,10 +3,12 @@ import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.Value; import java.util.Collections; import java.util.Map; import java.util.function.Supplier; @@ -98,6 +100,7 @@ public void finallyAfter( // targetingKey from evaluation context final String targetingKey = ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; + final Map attrs = snapshotAttrs(ctx); w.enqueue( new FlagEvalEvent( @@ -107,19 +110,27 @@ public void finallyAfter( targetingKey, errorMessage, evalTimeMs, - () -> extractAttrs(ctx))); + () -> extractAttrs(attrs))); } catch (LinkageError | Exception e) { // Never let EVP recording break flag evaluation } } - /** Extracts converted, flattened attributes from the evaluation context. */ - private Map extractAttrs(final HookContext ctx) { + private Map snapshotAttrs(final HookContext ctx) { if (ctx == null || ctx.getCtx() == null) { return Collections.emptyMap(); } - final Map attrs = DDEvaluator.flattenContext(ctx.getCtx()); - attrs.remove("targetingKey"); - return attrs; + final EvaluationContext context = ctx.getCtx(); + final Map attrs = DDEvaluator.snapshotValues(context); + attrs.remove(EvaluationContext.TARGETING_KEY); + return attrs.isEmpty() ? Collections.emptyMap() : attrs; + } + + /** Extracts converted, flattened attributes from the evaluation context. */ + private Map extractAttrs(final Map attrs) { + if (attrs.isEmpty()) { + return Collections.emptyMap(); + } + return DDEvaluator.flattenValues(attrs); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index bb86c409bad..eee6661f024 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -15,6 +15,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasEntry; @@ -233,6 +234,10 @@ private static Arguments[] flatteningTestCases() { Arguments.of( mapOf("map", mapOf("key1", 1, "key2", 2, "key3", mapOf("key4", 4))), mapOf("map.key1", 1, "map.key2", 2, "map.key3.key4", 4))); + arguments.add( + Arguments.of( + mapOf("plan", "gold", "cohort", "gold"), + mapOf("plan", "gold", "cohort", "gold"))); return arguments.toArray(new Arguments[0]); } @@ -561,6 +566,11 @@ public void testEvaluate(final TestCase testCase) { if (expectedAllocation != null) { assertThat(details.getFlagMetadata().getString("allocationKey"), equalTo(expectedAllocation)); } + if (expected.errorCode == null && expected.variant != null) { + final Long evalTimestampMs = details.getFlagMetadata().getLong("dd.eval.timestamp_ms"); + assertThat(evalTimestampMs, notNullValue()); + assertThat(evalTimestampMs > 0, equalTo(true)); + } if (shouldDispatchExposure(expected)) { verify(exposureListener, times(1)).accept(exposureEventCaptor.capture()); final ExposureEvent capturedEvent = exposureEventCaptor.getValue(); diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java index 61643e6bb61..9fa50b03fdc 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -21,10 +21,13 @@ import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.MutableStructure; import dev.openfeature.sdk.Reason; import dev.openfeature.sdk.Value; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -381,4 +384,46 @@ void contextAttributesAreFlattenedAndConvertedAfterEnqueue() { attrs.values().stream().noneMatch(Value.class::isInstance), "context attrs must contain converted scalar values, not OpenFeature Value wrappers"); } + + @Test + void contextAttributesUseEnqueueTimeSnapshot() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalLoggingHook hook = hookWithWriter(capturingWriter(captured)); + + final MutableContext context = new MutableContext("user-42"); + context.add("region", "us-east-1"); + final MutableStructure profile = new MutableStructure(); + profile.add("tier", "gold"); + context.add("profile", profile); + final List cohorts = new ArrayList<>(); + cohorts.add(Value.objectToValue("beta")); + context.add("cohorts", cohorts); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + context.add("region", "eu-west-1"); + context.add("late", "ignored"); + profile.add("tier", "platinum"); + profile.add("late", "ignored"); + cohorts.set(0, Value.objectToValue("ga")); + cohorts.add(Value.objectToValue("late")); + + assertNotNull(captured.get()); + final Map attrs = captured.get().contextAttributes(); + assertEquals("us-east-1", attrs.get("region")); + assertEquals("gold", attrs.get("profile.tier")); + assertEquals("beta", attrs.get("cohorts[0]")); + assertFalse(attrs.containsKey("late")); + assertFalse(attrs.containsKey("profile.late")); + assertFalse(attrs.containsKey("cohorts[1]")); + } } From 986a9af6be6f86e240b54b7c905e24fa99bf56f9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:47:58 -0400 Subject: [PATCH 08/37] feat(openfeature): register flagevaluation hook --- .../trace/api/openfeature/Provider.java | 22 ++++- .../trace/api/openfeature/ProviderTest.java | 81 ++++++++++++++++++- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 670d4d3642d..e01a85f6fc5 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -16,6 +16,7 @@ import dev.openfeature.sdk.exceptions.OpenFeatureError; import dev.openfeature.sdk.exceptions.ProviderNotReadyError; import java.lang.reflect.Constructor; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -167,10 +168,25 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - if (flagEvalMetricsHook == null) { - return Collections.emptyList(); + final List hooks = new ArrayList<>(2); + if (flagEvalMetricsHook != null) { + hooks.add(flagEvalMetricsHook); } - return Collections.singletonList(flagEvalMetricsHook); + // EVP flagevaluation hook: always registered; no-op when writer is absent (killswitch off). + // Writer is resolved lazily from FeatureFlaggingGateway.getFlagEvalWriter() on each call. + try { + final Hook flagEvalLoggingHook = buildFlagEvalLoggingHook(); + if (flagEvalLoggingHook != null) { + hooks.add(flagEvalLoggingHook); + } + } catch (LinkageError | Exception e) { + // Keep older bootstrap/API combinations working: EVP recording is best-effort. + } + return Collections.unmodifiableList(hooks); + } + + Hook buildFlagEvalLoggingHook() { + return FlagEvalLoggingHook.INSTANCE; } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 06bbe7ef8d2..3fd4417d6f1 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -16,6 +16,8 @@ import static org.mockito.Mockito.when; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import datadog.trace.api.openfeature.Provider.Options; import dev.openfeature.sdk.Client; @@ -25,6 +27,8 @@ import dev.openfeature.sdk.Features; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.OpenFeatureAPI; import dev.openfeature.sdk.ProviderEvaluation; import dev.openfeature.sdk.ProviderEvent; @@ -67,6 +71,7 @@ public void tearDown() { executor.shutdownNow(); OpenFeatureAPI.getInstance().shutdown(); FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + FeatureFlaggingGateway.setFlagEvalWriter(null); } @Test @@ -330,10 +335,66 @@ public void testGetProviderHooksReturnsFlagEvalMetricsHook() { Provider provider = new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)); List hooks = provider.getProviderHooks(); + // Two hooks: OTel FlagEvalMetricsHook (index 0) + FlagEvalLoggingHook (index 1) + assertThat(hooks.size(), equalTo(2)); + assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); + assertThat(hooks.get(1) instanceof FlagEvalLoggingHook, equalTo(true)); + } + + @Test + public void testGetProviderHooksSkipsFlagEvalLoggingHookOnLinkageFailure() { + Provider provider = + new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)) { + @Override + Hook buildFlagEvalLoggingHook() { + throw new NoClassDefFoundError("old bootstrap"); + } + }; + + List hooks = provider.getProviderHooks(); + assertThat(hooks.size(), equalTo(1)); assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true)); } + @Test + public void testClientEvaluationRoutesThroughFlagEvalLoggingHook() throws Exception { + FeatureFlaggingGateway.dispatch(mock(ServerConfiguration.class)); + final AtomicReference captured = new AtomicReference<>(); + FeatureFlaggingGateway.setFlagEvalWriter(capturingWriter(captured)); + final Evaluator evaluator = mock(Evaluator.class); + when(evaluator.initialize(eq(10L), eq(SECONDS), any())).thenReturn(true); + when(evaluator.hasConfiguration()).thenReturn(true); + when(evaluator.evaluate(eq(String.class), eq("logged-flag"), eq("default"), any())) + .thenReturn( + ProviderEvaluation.builder() + .value("value") + .reason("STATIC") + .variant("variant-1") + .flagMetadata( + ImmutableMetadata.builder() + .addString("allocationKey", "allocation-1") + .addLong("dd.eval.timestamp_ms", 1_700_000_000_000L) + .build()) + .build()); + final OpenFeatureAPI api = OpenFeatureAPI.getInstance(); + api.setProviderAndWait(new Provider(new Options().initTimeout(10, SECONDS), evaluator)); + final MutableContext context = new MutableContext("user-1"); + context.add("region", "us-east-1"); + + final FlagEvaluationDetails details = + api.getClient().getStringDetails("logged-flag", "default", context); + + assertThat(details.getValue(), equalTo("value")); + final FlagEvalEvent event = captured.get(); + assertThat(event.flagKey, equalTo("logged-flag")); + assertThat(event.variant, equalTo("variant-1")); + assertThat(event.allocationKey, equalTo("allocation-1")); + assertThat(event.targetingKey, equalTo("user-1")); + assertThat(event.evalTimeMs, equalTo(1_700_000_000_000L)); + assertThat(event.contextAttributes().get("region"), equalTo("us-east-1")); + } + @Test public void testShutdownCleansUpMetrics() throws Exception { Evaluator evaluator = mock(Evaluator.class); @@ -343,9 +404,8 @@ public void testShutdownCleansUpMetrics() throws Exception { provider.initialize(null); provider.shutdown(); verify(evaluator).shutdown(); - // After shutdown, getProviderHooks still returns a list (hook is still present but metrics is - // shut down) - assertThat(provider.getProviderHooks().size(), equalTo(1)); + // After shutdown, getProviderHooks still returns a list with both OTel + logging hooks + assertThat(provider.getProviderHooks().size(), equalTo(2)); } public interface EvaluateMethod { @@ -394,4 +454,19 @@ private static String initializationState(final Provider provider) throws Except final AtomicReference state = (AtomicReference) stateField.get(provider); return state.get().toString(); } + + private static FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public void start() {} + + @Override + public void close() {} + }; + } } From f1c37aba169c8e82a2eb6da9381b6db7439d3341 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:08 -0400 Subject: [PATCH 09/37] feat(feature-flagging): canonicalize flagevaluation context --- .../featureflag/FlagEvaluationAggregator.java | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java new file mode 100644 index 00000000000..89b8104fa70 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -0,0 +1,194 @@ +package com.datadog.featureflag; + +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +final class FlagEvaluationAggregator { + + static final int MAX_CONTEXT_FIELDS = 256; + static final int MAX_FIELD_LENGTH = 256; + + private static final byte CTX_TAG_STRING = 's'; + private static final byte CTX_TAG_BOOL = 'b'; + private static final byte CTX_TAG_INT = 'i'; + private static final byte CTX_TAG_LONG = 'l'; + private static final byte CTX_TAG_FLOAT = 'f'; + private static final byte CTX_TAG_DOUBLE = 'd'; + private static final byte CTX_TAG_OTHER = 'o'; + + private FlagEvaluationAggregator() {} + + static Map pruneContext(final Map attrs) { + if (attrs == null || attrs.isEmpty()) { + return java.util.Collections.emptyMap(); + } + final TreeMap out = new TreeMap<>(); + final TreeMap sorted = new TreeMap<>(attrs); + int count = 0; + for (final Map.Entry entry : sorted.entrySet()) { + if (count >= MAX_CONTEXT_FIELDS) { + break; + } + final Object v = entry.getValue(); + if (v instanceof String && ((String) v).length() > MAX_FIELD_LENGTH) { + continue; + } + out.put(entry.getKey(), v); + count++; + } + return out; + } + + static String canonicalContextKey(final Map prunedAttrs) { + if (prunedAttrs == null || prunedAttrs.isEmpty()) { + return ""; + } + final Map sorted = + (prunedAttrs instanceof TreeMap) ? prunedAttrs : new TreeMap<>(prunedAttrs); + final StringBuilder sb = new StringBuilder(); + for (final Map.Entry entry : sorted.entrySet()) { + appendLengthDelimited(sb, entry.getKey()); + appendContextValue(sb, entry.getValue()); + } + return sb.toString(); + } + + private static void appendLengthDelimited(final StringBuilder sb, final String s) { + sb.append(String.format("%08x", (long) s.length())); + sb.append(s); + } + + private static void appendContextValue(final StringBuilder sb, final Object v) { + if (v instanceof Boolean) { + sb.append((char) CTX_TAG_BOOL); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Long) { + sb.append((char) CTX_TAG_LONG); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Integer) { + sb.append((char) CTX_TAG_INT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Float) { + sb.append((char) CTX_TAG_FLOAT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Double) { + sb.append((char) CTX_TAG_DOUBLE); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof String) { + sb.append((char) CTX_TAG_STRING); + appendLengthDelimited(sb, (String) v); + } else { + sb.append((char) CTX_TAG_OTHER); + appendLengthDelimited(sb, v == null ? "" : v.toString()); + } + } + + static class EvalBucket { + long count; + long firstEvalMs; + long lastEvalMs; + boolean runtimeDefaultUsed; + String flagKey; + String variant; + String allocationKey; + String targetingKey; + String errorMessage; + Map prunedAttrs; + + EvalBucket( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final boolean runtimeDefaultUsed, + final Map prunedAttrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.targetingKey = targetingKey; + this.errorMessage = errorMessage; + this.firstEvalMs = evalTimeMs; + this.lastEvalMs = evalTimeMs; + this.count = 1; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.prunedAttrs = prunedAttrs; + } + + int prunedContextFieldCount() { + return prunedAttrs == null ? 0 : prunedAttrs.size(); + } + + void merge(final long evalTimeMs, final boolean isDefault) { + count++; + if (evalTimeMs < firstEvalMs) { + firstEvalMs = evalTimeMs; + } + if (evalTimeMs > lastEvalMs) { + lastEvalMs = evalTimeMs; + } + if (isDefault) { + runtimeDefaultUsed = true; + } + } + } + + static final class FullKey { + private final String flagKey; + private final String variant; + private final String allocationKey; + private final boolean runtimeDefaultUsed; + private final String errorMessage; + private final String targetingKey; + private final String contextKey; + + FullKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final String targetingKey, + final String contextKey) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.errorMessage = errorMessage; + this.targetingKey = targetingKey; + this.contextKey = contextKey; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FullKey)) { + return false; + } + final FullKey fullKey = (FullKey) o; + return runtimeDefaultUsed == fullKey.runtimeDefaultUsed + && Objects.equals(flagKey, fullKey.flagKey) + && Objects.equals(variant, fullKey.variant) + && Objects.equals(allocationKey, fullKey.allocationKey) + && Objects.equals(errorMessage, fullKey.errorMessage) + && Objects.equals(targetingKey, fullKey.targetingKey) + && Objects.equals(contextKey, fullKey.contextKey); + } + + @Override + public int hashCode() { + return Objects.hash( + flagKey, + variant, + allocationKey, + runtimeDefaultUsed, + errorMessage, + targetingKey, + contextKey); + } + } +} From 115a407b76a5f57c48a23ae603fe65c20b98de05 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:08 -0400 Subject: [PATCH 10/37] feat(feature-flagging): aggregate flagevaluation rows --- .../featureflag/FlagEvaluationAggregator.java | 235 +++++++++++++++++- 1 file changed, 234 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java index 89b8104fa70..eb15eb884a5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -1,11 +1,29 @@ package com.datadog.featureflag; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; final class FlagEvaluationAggregator { + static final int EVAL_SCALE_TARGET_FLAGS = 2_500; + static final int EVAL_SCALE_FULL_BUCKETS_PER_FLAG = 50; + static final int EVAL_SCALE_USERS_PER_FLAG = 1_000; + static final int EVAL_SCALE_PER_FLAG_HEADROOM_MULTIPLIER = 10; + static final int EVAL_SCALE_DEGRADED_BUCKETS_PER_FLAG = 10; + static final int EVAL_SCALE_FULL_BUCKET_TARGET = + EVAL_SCALE_TARGET_FLAGS * EVAL_SCALE_FULL_BUCKETS_PER_FLAG; + static final int EVAL_SCALE_PER_FLAG_BUCKET_TARGET = + EVAL_SCALE_PER_FLAG_HEADROOM_MULTIPLIER * EVAL_SCALE_USERS_PER_FLAG; + static final int EVAL_SCALE_DEGRADED_BUCKET_TARGET = + EVAL_SCALE_TARGET_FLAGS * EVAL_SCALE_DEGRADED_BUCKETS_PER_FLAG; + static final int GLOBAL_CAP = 131_072; + static final int PER_FLAG_CAP = EVAL_SCALE_PER_FLAG_BUCKET_TARGET; + static final int DEGRADED_CAP = 32_768; + static final int MAX_CONTEXT_FIELDS = 256; static final int MAX_FIELD_LENGTH = 256; @@ -17,7 +35,165 @@ final class FlagEvaluationAggregator { private static final byte CTX_TAG_DOUBLE = 'd'; private static final byte CTX_TAG_OTHER = 'o'; - private FlagEvaluationAggregator() {} + final Map fullTier = new HashMap<>(); + final Map degradedTier = new HashMap<>(); + final Map perFlagCount = new HashMap<>(); + final AtomicLong droppedDegradedOverflow = new AtomicLong(0); + int globalFullCount = 0; + + void aggregate(final FlagEvalEvent event) { + final boolean isDefault = event.variant == null; + final Map prunedAttrs = pruneContext(event.contextAttributes()); + final String ctxKey = canonicalContextKey(prunedAttrs); + final FullKey fullKey = buildFullKey(event, ctxKey); + + EvalBucket bucket = fullTier.get(fullKey); + if (bucket != null) { + bucket.merge(event.evalTimeMs, isDefault); + return; + } + + final int flagCount = perFlagCount.getOrDefault(event.flagKey, 0); + if (globalFullCount < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { + fullTier.put( + fullKey, + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + event.targetingKey, + event.errorMessage, + event.evalTimeMs, + isDefault, + prunedAttrs)); + globalFullCount++; + perFlagCount.put(event.flagKey, flagCount + 1); + return; + } + + final DegradedKey degradedKey = buildDegradedKey(event); + bucket = degradedTier.get(degradedKey); + if (bucket != null) { + bucket.merge(event.evalTimeMs, isDefault); + return; + } + + if (degradedTier.size() < DEGRADED_CAP) { + degradedTier.put( + degradedKey, + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + null, + event.errorMessage, + event.evalTimeMs, + isDefault, + null)); + return; + } + + droppedDegradedOverflow.incrementAndGet(); + } + + boolean isEmpty() { + return fullTier.isEmpty() && degradedTier.isEmpty(); + } + + int fullTierSize() { + return fullTier.size(); + } + + long degradedEvaluationCount() { + long count = 0; + for (final EvalBucket bucket : degradedTier.values()) { + count += bucket.count; + } + return count; + } + + int bucketCount() { + return fullTier.size() + degradedTier.size(); + } + + Iterable fullBuckets() { + return fullTier.values(); + } + + Iterable degradedBuckets() { + return degradedTier.values(); + } + + void clear() { + fullTier.clear(); + degradedTier.clear(); + perFlagCount.clear(); + globalFullCount = 0; + } + + AggregatedState snapshot() { + return new AggregatedState( + new HashMap<>(fullTier), new HashMap<>(degradedTier), droppedDegradedOverflow.get()); + } + + void simulateFullTierAtCap() { + for (int i = globalFullCount; i < GLOBAL_CAP; i++) { + final String key = "synthetic-full-" + i; + fullTier.put( + new FullKey(key, "on", "alloc", false, null, null, ""), + new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); + globalFullCount++; + perFlagCount.merge(key, 1, Integer::sum); + } + } + + void simulateDegradedTierAtCap() { + for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { + final String key = "synthetic-dg-" + i; + degradedTier.put( + new DegradedKey(key, "on", "alloc", false, null), + new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); + } + } + + void addDegradedBucketForTest( + final String flagKey, + final String variant, + final String allocationKey, + final String errorMessage, + final long evalTimeMs) { + degradedTier.put( + new DegradedKey(flagKey, variant, allocationKey, variant == null, errorMessage), + new EvalBucket( + flagKey, + variant, + allocationKey, + null, + errorMessage, + evalTimeMs, + variant == null, + null)); + } + + private static FullKey buildFullKey(final FlagEvalEvent event, final String ctxKey) { + return new FullKey( + event.flagKey, + event.variant, + event.allocationKey, + event.variant == null, + event.errorMessage, + event.targetingKey, + ctxKey); + } + + private static DegradedKey buildDegradedKey(final FlagEvalEvent event) { + return new DegradedKey( + event.flagKey, + event.variant, + event.allocationKey, + event.variant == null, + event.errorMessage); + } static Map pruneContext(final Map attrs) { if (attrs == null || attrs.isEmpty()) { @@ -191,4 +367,61 @@ public int hashCode() { contextKey); } } + + static final class DegradedKey { + private final String flagKey; + private final String variant; + private final String allocationKey; + private final boolean runtimeDefaultUsed; + private final String errorMessage; + + DegradedKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.errorMessage = errorMessage; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DegradedKey)) { + return false; + } + final DegradedKey that = (DegradedKey) o; + return runtimeDefaultUsed == that.runtimeDefaultUsed + && Objects.equals(flagKey, that.flagKey) + && Objects.equals(variant, that.variant) + && Objects.equals(allocationKey, that.allocationKey) + && Objects.equals(errorMessage, that.errorMessage); + } + + @Override + public int hashCode() { + return Objects.hash(flagKey, variant, allocationKey, runtimeDefaultUsed, errorMessage); + } + } + + static class AggregatedState { + final Map fullTier; + final Map degradedTier; + final long droppedDegradedOverflow; + + AggregatedState( + final Map fullTier, + final Map degradedTier, + final long droppedDegradedOverflow) { + this.fullTier = fullTier; + this.degradedTier = degradedTier; + this.droppedDegradedOverflow = droppedDegradedOverflow; + } + } } From cc3b21c0c2c03aeaec09b82d0a12fb6b7e550e97 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:09 -0400 Subject: [PATCH 11/37] test(feature-flagging): cover flagevaluation aggregation --- .../FlagEvaluationAggregatorTest.java | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java new file mode 100644 index 00000000000..0adfcf35d3e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java @@ -0,0 +1,183 @@ +package com.datadog.featureflag; + +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvaluationAggregatorTest { + + @Test + void identicalEventsAggregateIntoOneBucketWithCount2() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("flag-a", "on", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("flag-a", "on", "alloc1", "user-1", 2000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(1, state.fullTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertEquals(1000L, bucket.firstEvalMs); + assertEquals(2000L, bucket.lastEvalMs); + assertTrue(bucket.firstEvalMs <= bucket.lastEvalMs); + } + + @Test + void differentValueTypesProduceDifferentBuckets() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map attrsInt = new HashMap<>(); + attrsInt.put("score", 1); + final Map attrsStr = new HashMap<>(); + attrsStr.put("score", "1"); + + aggregator.aggregate(event("flag-b", "on", "alloc1", "user-1", 1000L, attrsInt)); + aggregator.aggregate(event("flag-b", "on", "alloc1", "user-1", 1000L, attrsStr)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(2, state.fullTier.size()); + } + + @Test + void nulCharactersInKeyFieldsDoNotCollide() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("a\0b", "c", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("a", "b\0c", "alloc1", "user-1", 1000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(2, state.fullTier.size()); + } + + @Test + void globalCapOverflowRoutesToDegradedTier() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.simulateFullTierAtCap(); + aggregator.aggregate(simpleEvent("extra-flag", "on")); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertTrue(state.degradedTier.size() > 0); + assertEquals(0, state.droppedDegradedOverflow); + } + + @Test + void degradedCapOverflowIncrementsDroppedCounter() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.simulateFullTierAtCap(); + aggregator.simulateDegradedTierAtCap(); + aggregator.aggregate(simpleEvent("drop-flag", "on")); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertTrue(state.droppedDegradedOverflow > 0); + } + + @Test + void absentVariantSetsRuntimeDefaultUsed() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + aggregator.aggregate(event("flag-c", null, "alloc1", "user-1", 1000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(1, state.fullTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertTrue(bucket.runtimeDefaultUsed); + } + + @Test + void contextExceeding256FieldsIsPrunedToStoredPrunedAttrs() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map hugeAttrs = new HashMap<>(); + for (int i = 0; i < 300; i++) { + hugeAttrs.put("key" + i, "v" + i); + } + + aggregator.aggregate(event("flag-d", "on", "alloc1", "user-1", 1000L, hugeAttrs)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(256, bucket.prunedContextFieldCount()); + assertEquals(256, bucket.prunedAttrs.size()); + } + + @Test + void pruningIsDeterministicSortBeforeCut() { + final Map attrs = new HashMap<>(); + for (int i = 0; i < 300; i++) { + attrs.put(String.format("k%03d", i), "v" + i); + } + + final Map p1 = FlagEvaluationAggregator.pruneContext(attrs); + final Map p2 = FlagEvaluationAggregator.pruneContext(new HashMap<>(attrs)); + + assertEquals(256, p1.size()); + assertEquals(p1.keySet(), p2.keySet()); + assertTrue(p1.containsKey("k000")); + assertTrue(p1.containsKey("k255")); + assertFalse(p1.containsKey("k256")); + assertFalse(p1.containsKey("k299")); + } + + @Test + void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + final Map attrs = new HashMap<>(); + attrs.put("long-val", repeat('x', 300)); + attrs.put("short-val", "ok"); + + aggregator.aggregate(event("flag-e", "on", "alloc1", "user-1", 1000L, attrs)); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + final FlagEvaluationAggregator.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertFalse(bucket.prunedAttrs.containsKey("long-val")); + assertTrue(bucket.prunedAttrs.containsKey("short-val")); + } + + @Test + void capSizingUsesNamedScaleConstants() { + assertEquals(125_000, FlagEvaluationAggregator.EVAL_SCALE_FULL_BUCKET_TARGET); + assertEquals(10_000, FlagEvaluationAggregator.EVAL_SCALE_PER_FLAG_BUCKET_TARGET); + assertEquals(25_000, FlagEvaluationAggregator.EVAL_SCALE_DEGRADED_BUCKET_TARGET); + assertEquals(131_072, FlagEvaluationAggregator.GLOBAL_CAP); + assertEquals(10_000, FlagEvaluationAggregator.PER_FLAG_CAP); + assertEquals(32_768, FlagEvaluationAggregator.DEGRADED_CAP); + } + + @Test + void flagEvalEventDoesNotCarryReason() { + final boolean hasReasonField = + Arrays.stream( + datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent.class + .getDeclaredFields()) + .anyMatch(field -> field.getName().equals("reason")); + + assertFalse(hasReasonField); + } + + private static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); + } + + private static FlagEvalEvent simpleEvent(final String flagKey, final String variant) { + return event(flagKey, variant, "alloc1", "user-1", 1000L, emptyMap()); + } + + private static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } +} From 31d7cb51169829fbebf79b6721db1c8efd97d85b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:09 -0400 Subject: [PATCH 12/37] feat(feature-flagging): encode flagevaluation payloads --- .../featureflag/FlagEvaluationPayloads.java | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java new file mode 100644 index 00000000000..c2b3cbfb517 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java @@ -0,0 +1,270 @@ +package com.datadog.featureflag; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +final class FlagEvaluationPayloads { + + private static final byte[] PAYLOAD_SUFFIX = FeatureFlagEvpPublisher.utf8Bytes("]}"); + private static final byte[] JSON_COMMA = FeatureFlagEvpPublisher.utf8Bytes(","); + private static final JsonAdapter EVENT_JSON_ADAPTER; + private static final JsonAdapter> CONTEXT_JSON_ADAPTER; + + static { + final Moshi moshi = new Moshi.Builder().build(); + EVENT_JSON_ADAPTER = moshi.adapter(FlagEvaluationEvent.class); + final Type contextType = Types.newParameterizedType(Map.class, String.class, String.class); + CONTEXT_JSON_ADAPTER = moshi.adapter(contextType); + } + + private FlagEvaluationPayloads() {} + + static class FlagEvaluationsRequest { + public final Map context; + public final List flagEvaluations; + + FlagEvaluationsRequest( + final Map context, final List flagEvaluations) { + this.context = context; + this.flagEvaluations = flagEvaluations; + } + } + + static EncodedPayloads buildPayloads( + final List events, + final Map context, + final int payloadSizeLimitBytes) { + final byte[] prefix = payloadPrefix(context); + EncodedPayloadBuilder current = new EncodedPayloadBuilder(prefix); + final List payloads = new ArrayList<>(); + long droppedPayloadLimit = 0; + long degradedPayloadLimit = 0; + + for (final FlagEvaluationEvent event : events) { + byte[] eventBytes = encodeEvent(event); + + if (!current.canAdd(eventBytes, payloadSizeLimitBytes) && !current.isEmpty()) { + payloads.add(current.toByteArray()); + current = new EncodedPayloadBuilder(prefix); + } + + if (current.canAdd(eventBytes, payloadSizeLimitBytes)) { + current.add(eventBytes); + continue; + } + + final FlagEvaluationEvent degraded = event.withoutTargetingKeyAndContext(); + if (degraded != null) { + eventBytes = encodeEvent(degraded); + if (!current.canAdd(eventBytes, payloadSizeLimitBytes) && !current.isEmpty()) { + payloads.add(current.toByteArray()); + current = new EncodedPayloadBuilder(prefix); + } + if (current.canAdd(eventBytes, payloadSizeLimitBytes)) { + current.add(eventBytes); + degradedPayloadLimit += event.evaluation_count; + continue; + } + } + + droppedPayloadLimit += event.evaluation_count; + } + + if (!current.isEmpty()) { + payloads.add(current.toByteArray()); + } + return new EncodedPayloads(payloads, droppedPayloadLimit, degradedPayloadLimit); + } + + private static byte[] payloadPrefix(final Map context) { + return FeatureFlagEvpPublisher.utf8Bytes( + "{\"context\":" + CONTEXT_JSON_ADAPTER.toJson(context) + ",\"flagEvaluations\":["); + } + + private static byte[] encodeEvent(final FlagEvaluationEvent event) { + return FeatureFlagEvpPublisher.utf8Bytes(EVENT_JSON_ADAPTER.toJson(event)); + } + + static final class EncodedPayloads { + final List bodies; + final long droppedPayloadLimit; + final long degradedPayloadLimit; + + private EncodedPayloads( + final List bodies, + final long droppedPayloadLimit, + final long degradedPayloadLimit) { + this.bodies = bodies; + this.droppedPayloadLimit = droppedPayloadLimit; + this.degradedPayloadLimit = degradedPayloadLimit; + } + } + + private static final class EncodedPayloadBuilder { + private final byte[] prefix; + private final List events = new ArrayList<>(); + private int eventBytes; + + private EncodedPayloadBuilder(final byte[] prefix) { + this.prefix = prefix; + } + + private boolean isEmpty() { + return events.isEmpty(); + } + + private boolean canAdd(final byte[] event, final int payloadSizeLimitBytes) { + return sizeWith(event) <= payloadSizeLimitBytes; + } + + private int sizeWith(final byte[] event) { + return prefix.length + PAYLOAD_SUFFIX.length + eventBytes + event.length + events.size(); + } + + private void add(final byte[] event) { + events.add(event); + eventBytes += event.length; + } + + private byte[] toByteArray() { + final int size = prefix.length + PAYLOAD_SUFFIX.length + eventBytes; + final ByteArrayOutputStream out = new ByteArrayOutputStream(size + events.size()); + out.write(prefix, 0, prefix.length); + for (int i = 0; i < events.size(); i++) { + if (i > 0) { + out.write(JSON_COMMA, 0, JSON_COMMA.length); + } + final byte[] event = events.get(i); + out.write(event, 0, event.length); + } + out.write(PAYLOAD_SUFFIX, 0, PAYLOAD_SUFFIX.length); + return out.toByteArray(); + } + } + + static class FlagEvaluationEvent { + public final long timestamp; + public final FlagKeyObject flag; + public final long first_evaluation; + public final long last_evaluation; + public final long evaluation_count; + public final KeyObject variant; + public final KeyObject allocation; + public final String targeting_key; + public final Boolean runtime_default_used; + public final EventContext context; + public final ErrorObject error; + + FlagEvaluationEvent( + final long timestamp, + final String flagKey, + final long firstEvalMs, + final long lastEvalMs, + final long count, + final String variant, + final String allocation, + final String targetingKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final Map evaluationAttrs) { + this.timestamp = timestamp; + this.flag = new FlagKeyObject(flagKey); + this.first_evaluation = firstEvalMs; + this.last_evaluation = lastEvalMs; + this.evaluation_count = count; + this.variant = (variant != null && !variant.isEmpty()) ? new KeyObject(variant) : null; + this.allocation = + (allocation != null && !allocation.isEmpty()) ? new KeyObject(allocation) : null; + this.targeting_key = targetingKey; + this.runtime_default_used = runtimeDefaultUsed ? Boolean.TRUE : null; + this.context = + (evaluationAttrs != null && !evaluationAttrs.isEmpty()) + ? new EventContext(evaluationAttrs) + : null; + this.error = + (errorMessage != null && !errorMessage.isEmpty()) ? new ErrorObject(errorMessage) : null; + } + + static FlagEvaluationEvent fromBucket( + final FlagEvaluationAggregator.EvalBucket bucket, + final boolean isFullTier, + final long flushTimeMs) { + return new FlagEvaluationEvent( + flushTimeMs, + bucket.flagKey, + bucket.firstEvalMs, + bucket.lastEvalMs, + bucket.count, + bucket.variant, + bucket.allocationKey, + isFullTier ? bucket.targetingKey : null, + bucket.runtimeDefaultUsed, + bucket.errorMessage, + isFullTier ? bucket.prunedAttrs : null); + } + + FlagEvaluationEvent withoutTargetingKeyAndContext() { + if (targeting_key == null && context == null) { + return null; + } + return new FlagEvaluationEvent( + timestamp, + flag.key, + first_evaluation, + last_evaluation, + evaluation_count, + keyOf(variant), + keyOf(allocation), + null, + Boolean.TRUE.equals(runtime_default_used), + messageOf(error), + null); + } + } + + private static String keyOf(final KeyObject object) { + return object == null ? null : object.key; + } + + private static String messageOf(final ErrorObject object) { + return object == null ? null : object.message; + } + + static class KeyObject { + public final String key; + + KeyObject(final String key) { + this.key = key; + } + } + + static class FlagKeyObject { + public final String key; + + FlagKeyObject(final String key) { + this.key = key; + } + } + + static class ErrorObject { + public final String message; + + ErrorObject(final String message) { + this.message = message; + } + } + + static class EventContext { + public final Map evaluation; + + EventContext(final Map evaluation) { + this.evaluation = evaluation; + } + } +} From e43f93acd0506c0b22dfef39467776d0e5a835fb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:10 -0400 Subject: [PATCH 13/37] test(feature-flagging): cover flagevaluation payload encoding --- .../FlagEvaluationPayloadsTest.java | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java new file mode 100644 index 00000000000..3ba2dc8fccf --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -0,0 +1,251 @@ +package com.datadog.featureflag; + +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FlagEvaluationPayloadsTest { + + private static final long EVAL_MS = 1_760_000_000_000L; + private static final Map CONTEXT = context(); + private static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + + @Test + void fullTierPayloadUsesWorkerWireShape() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event("my-flag", "on", "alloc-x", "user-1", 1, attrs)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertObjectWithKey(ev.get("variant"), "on"); + assertObjectWithKey(ev.get("allocation"), "alloc-x"); + assertObjectWithKey(ev.get("flag"), "my-flag"); + final Map ctx = (Map) ev.get("context"); + assertNotNull(ctx); + final Map evalAttrs = (Map) ctx.get("evaluation"); + assertNotNull(evalAttrs); + assertEquals("us-east-1", evalAttrs.get("region")); + assertFalse(ev.containsKey("reason")); + } + + @Test + void eventFromFullBucketUsesFlushTimeAndEvaluationBounds() throws Exception { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "ts-flag", "on", "alloc1", "user-1", null, EVAL_MS, false, emptyMap()); + bucket.merge(EVAL_MS + 10, false); + final long flushTimeMs = EVAL_MS + 5_000; + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket( + bucket, true, flushTimeMs)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertEquals((double) flushTimeMs, ((Number) ev.get("timestamp")).doubleValue()); + assertEquals((double) EVAL_MS, ((Number) ev.get("first_evaluation")).doubleValue()); + assertEquals((double) (EVAL_MS + 10), ((Number) ev.get("last_evaluation")).doubleValue()); + assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); + } + + @Test + void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "dg-flag", "on", "alloc1", null, null, EVAL_MS, false, null); + + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket(bucket, false, EVAL_MS)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void splitPayloadsByEncodedSize() throws Exception { + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 180)); + final java.util.ArrayList events = + new java.util.ArrayList<>(); + for (int i = 0; i < 4; i++) { + events.add(event("split-flag-" + i, "on", "alloc1", "user-" + i, 1, attrs)); + } + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(events, CONTEXT, 1_100); + + assertTrue(payloads.bodies.size() > 1); + int eventCount = 0; + for (final byte[] body : payloads.bodies) { + assertTrue(body.length <= 1_100); + eventCount += eventCount(parse(body)); + } + assertEquals(4, eventCount); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { + final Map attrs = new HashMap<>(); + for (int i = 0; i < 4; i++) { + attrs.put("payload-" + i, repeat('x', 200)); + } + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event("oversized-full", "on", "alloc1", "user-1", 2, attrs)), + CONTEXT, + 512); + + assertEquals(1, payloads.bodies.size()); + assertTrue(payloads.bodies.get(0).length <= 512); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(2, payloads.degradedPayloadLimit); + final Map ev = firstEvent(parse(payloads.bodies.get(0))); + assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void oversizedDegradedPayloadRowIsDropped() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event(repeat('f', 512), "on", "alloc1", null, 2, emptyMap())), + CONTEXT, + 128); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(2, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void errorPayloadSerializesErrorObject() throws Exception { + final Map json = + firstPayload( + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + "err-flag", + EVAL_MS, + EVAL_MS, + 1, + null, + null, + null, + true, + "type mismatch", + null)), + CONTEXT, + 1_000_000)); + + final Map ev = firstEvent(json); + final Map error = (Map) ev.get("error"); + assertNotNull(error); + assertEquals("type mismatch", error.get("message")); + assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); + } + + private static FlagEvaluationPayloads.FlagEvaluationEvent event( + final String flagKey, + final String variant, + final String allocation, + final String targetingKey, + final long count, + final Map attrs) { + return new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + flagKey, + EVAL_MS, + EVAL_MS, + count, + variant, + allocation, + targetingKey, + false, + null, + attrs); + } + + private static Map firstPayload( + final FlagEvaluationPayloads.EncodedPayloads payloads) throws Exception { + assertEquals(1, payloads.bodies.size()); + return parse(payloads.bodies.get(0)); + } + + private static Map parse(final byte[] body) throws Exception { + return JSON_MAP.fromJson(new String(body, java.nio.charset.StandardCharsets.UTF_8)); + } + + @SuppressWarnings("unchecked") + private static Map firstEvent(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events); + assertFalse(events.isEmpty()); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + private static int eventCount(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events); + return events.size(); + } + + private static void assertObjectWithKey(final Object object, final String expectedKey) { + assertTrue(object instanceof Map); + assertEquals(expectedKey, ((Map) object).get("key")); + } + + private static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + java.util.Arrays.fill(chars, c); + return new String(chars); + } + + private static Map context() { + final Map context = new HashMap<>(); + context.put("service", "test-service"); + return context; + } +} From 17d7785420fc08c081558b55a4161d179606bdb7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:48:10 -0400 Subject: [PATCH 14/37] feat(telemetry): support tagged core metric counts --- .../api/telemetry/CoreMetricCollector.java | 8 ++++++++ .../telemetry/CoreMetricCollectorTest.groovy | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java b/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java index d33fcf1529d..d09dc2f2ac0 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/CoreMetricCollector.java @@ -29,6 +29,14 @@ private CoreMetricCollector() { this.metricsQueue = new ArrayBlockingQueue<>(RAW_QUEUE_SIZE); } + public void count(String metricName, long value, String tag) { + if (value <= 0) { + return; + } + this.metricsQueue.offer( + new CoreMetric(METRIC_NAMESPACE, true, metricName, "count", value, tag)); + } + @Override public void prepareMetrics() { // Collect span metrics diff --git a/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy index 205da2bd0f5..5d0b920bfb9 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/telemetry/CoreMetricCollectorTest.groovy @@ -53,4 +53,24 @@ class CoreMetricCollectorTest extends DDSpecification { collector.prepareMetrics() collector.drain().size() == limit } + + def "direct count core metric"() { + setup: + def collector = CoreMetricCollector.getInstance() + collector.drain() + + when: + collector.count('flagevaluation.rows.dropped', 3, 'reason:queue_overflow') + def metrics = collector.drain() + + then: + metrics.size() == 1 + + def metric = metrics[0] + metric.type == 'count' + metric.value == 3 + metric.namespace == 'tracers' + metric.metricName == 'flagevaluation.rows.dropped' + metric.tags == ['reason:queue_overflow'] + } } From 5f88cadeb295fb2448905f735e93da6c6c1c4837 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:05 -0400 Subject: [PATCH 15/37] feat(feature-flagging): run flagevaluation writer lifecycle --- .../trace/util/AgentThreadFactory.java | 3 +- .../featureflag/FlagEvaluationWriterImpl.java | 294 ++++++++++++++++++ 2 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 752adb8899d..3269602a84c 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -66,7 +66,8 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), - FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"); + FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + FEATURE_FLAG_EVALUATION_PROCESSOR("dd-ffe-evaluation-processor"); public final String threadName; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java new file mode 100644 index 00000000000..68068d1ea4c --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -0,0 +1,294 @@ +package com.datadog.featureflag; + +import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_EVALUATION_PROCESSOR; +import static datadog.trace.util.AgentThreadFactory.newAgentThread; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import datadog.trace.api.telemetry.CoreMetricCollector; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * EVP {@code flagevaluation} writer for Java. + * + *

Owns the bounded hook hand-off queue, background aggregation loop, and writer lifecycle. The + * EVP payload/posting path is layered on top of this queue in the next review step. + */ +public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlagEvaluationWriterImpl.class); + + static final int DEFAULT_CAPACITY = 1 << 16; + static final int FLUSH_INTERVAL_SECONDS = 10; + static final int EVAL_SCALE_FULL_BUCKET_TARGET = + FlagEvaluationAggregator.EVAL_SCALE_FULL_BUCKET_TARGET; + static final int EVAL_SCALE_PER_FLAG_BUCKET_TARGET = + FlagEvaluationAggregator.EVAL_SCALE_PER_FLAG_BUCKET_TARGET; + static final int EVAL_SCALE_DEGRADED_BUCKET_TARGET = + FlagEvaluationAggregator.EVAL_SCALE_DEGRADED_BUCKET_TARGET; + static final int GLOBAL_CAP = FlagEvaluationAggregator.GLOBAL_CAP; + static final int PER_FLAG_CAP = FlagEvaluationAggregator.PER_FLAG_CAP; + static final int DEGRADED_CAP = FlagEvaluationAggregator.DEGRADED_CAP; + static final String FLAG_EVALUATION_DROPPED_METRIC = "flagevaluation.rows.dropped"; + static final String FLAG_EVALUATION_DEGRADED_METRIC = "flagevaluation.rows.degraded"; + static final String DROP_REASON_QUEUE_OVERFLOW = "queue_overflow"; + static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; + static final String DROP_REASON_CLOSED = "closed"; + static final String DROP_REASON_CONTEXT_ERROR = "context_error"; + static final String DEGRADED_REASON_CARDINALITY_CAP = "cardinality_cap"; + private static final CoreMetricCollector CORE_METRICS = CoreMetricCollector.getInstance(); + + static final int MAX_CONTEXT_FIELDS = FlagEvaluationAggregator.MAX_CONTEXT_FIELDS; + static final int MAX_FIELD_LENGTH = FlagEvaluationAggregator.MAX_FIELD_LENGTH; + + private final MessagePassingBlockingQueue queue; + private final FlagEvaluationSerializingHandler serializer; + private final Thread serializerThread; + private final Object lifecycleLock = new Object(); + private final AtomicBoolean closed = new AtomicBoolean(false); + + private final AtomicLong droppedQueueOverflow = new AtomicLong(0); + + public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { + this(DEFAULT_CAPACITY, FLUSH_INTERVAL_SECONDS, SECONDS, sco, config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final SharedCommunicationObjects sco, + final Config config) { + this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final BackendApiFactory backendApiFactory, + final Config config) { + this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); + this.serializer = + new FlagEvaluationSerializingHandler(queue, flushInterval, timeUnit, droppedQueueOverflow); + this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); + } + + @Override + public void start() { + synchronized (lifecycleLock) { + if (closed.get()) { + return; + } + FeatureFlaggingGateway.setFlagEvalWriter(this); + this.serializerThread.start(); + } + } + + void startForTest() { + synchronized (lifecycleLock) { + if (!closed.get()) { + this.serializerThread.start(); + } + } + } + + int aggregatorFullTierSizeForTest() { + return serializer.aggregator.fullTierSize(); + } + + @Override + public void close() { + synchronized (lifecycleLock) { + if (!closed.compareAndSet(false, true)) { + return; + } + FeatureFlaggingGateway.setFlagEvalWriter(null); + if (!this.serializerThread.isAlive()) { + return; + } + serializer.requestShutdown(); + this.serializerThread.interrupt(); + } + if (Thread.currentThread() == this.serializerThread) { + return; + } + try { + this.serializerThread.join(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void enqueue(final FlagEvalEvent event) { + if (event == null) { + return; + } + synchronized (lifecycleLock) { + if (closed.get()) { + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + return; + } + if (!queue.offer(event)) { + droppedQueueOverflow.incrementAndGet(); + } + } + } + + long droppedQueueOverflow() { + return droppedQueueOverflow.get(); + } + + FlagEvalEvent pollQueuedEventForTest() { + return queue.poll(); + } + + void flushForTest() { + serializer.flush(); + } + + static Map pruneContext(final Map attrs) { + return FlagEvaluationAggregator.pruneContext(attrs); + } + + static String canonicalContextKey(final Map prunedAttrs) { + return FlagEvaluationAggregator.canonicalContextKey(prunedAttrs); + } + + private static void countMetric(final String metricName, final long value, final String reason) { + if (value <= 0) { + return; + } + CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); + } + + static class FlagEvaluationSerializingHandler implements Runnable { + private final MessagePassingBlockingQueue queue; + private final long ticksRequiredToFlush; + + @SuppressFBWarnings( + value = "AT_NONATOMIC_64BIT_PRIMITIVE", + justification = "the field is confined to the single serializer thread") + private long lastTicks; + + private final AtomicLong droppedQueueOverflow; + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); + + FlagEvaluationSerializingHandler( + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final AtomicLong droppedQueueOverflow) { + this.queue = queue; + this.droppedQueueOverflow = droppedQueueOverflow; + this.lastTicks = System.nanoTime(); + this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + LOGGER.debug("starting flag evaluation serializer"); + } + + void requestShutdown() { + shutdownRequested.set(true); + } + + @Override + public void run() { + try { + runDutyCycle(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + drainAndFlush(); + } + LOGGER.debug("flag evaluation processor worker exited."); + } + + private void runDutyCycle() throws InterruptedException { + final Thread thread = Thread.currentThread(); + while (!thread.isInterrupted() && !shutdownRequested.get()) { + final FlagEvalEvent event = queue.poll(100, TimeUnit.MILLISECONDS); + if (event != null) { + aggregateEvent(event); + } + flushIfNecessary(); + } + } + + void drainAndFlush() { + FlagEvalEvent event; + while ((event = queue.poll()) != null) { + aggregateEvent(event); + } + flush(); + } + + void aggregateEvent(final FlagEvalEvent event) { + try { + aggregator.aggregate(event); + } catch (LinkageError | RuntimeException e) { + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CONTEXT_ERROR); + LOGGER.debug("Could not aggregate flag evaluation event", e); + } + } + + void flushIfNecessary() { + if (aggregator.isEmpty() && droppedQueueOverflow.get() == 0) { + return; + } + if (shouldFlush()) { + flush(); + } + } + + void flush() { + final long qDrops = droppedQueueOverflow.getAndSet(0); + countMetric(FLAG_EVALUATION_DROPPED_METRIC, qDrops, DROP_REASON_QUEUE_OVERFLOW); + if (qDrops > 0) { + LOGGER.warn( + "flag evaluation queue full - dropped {} evaluation(s) under backpressure" + + " (best-effort telemetry)", + qDrops); + } + final long dgDrops = aggregator.droppedDegradedOverflow.getAndSet(0); + countMetric(FLAG_EVALUATION_DROPPED_METRIC, dgDrops, DROP_REASON_DEGRADED_CAP); + if (dgDrops > 0) { + LOGGER.warn( + "degraded aggregation tier full - dropped {} evaluation(s); raise degraded cap" + + " (best-effort telemetry)", + dgDrops); + } + if (!aggregator.isEmpty()) { + countMetric( + FLAG_EVALUATION_DEGRADED_METRIC, + aggregator.degradedEvaluationCount(), + DEGRADED_REASON_CARDINALITY_CAP); + aggregator.clear(); + } + lastTicks = System.nanoTime(); + } + + private boolean shouldFlush() { + final long nanoTime = System.nanoTime(); + final long ticks = nanoTime - lastTicks; + if (ticks > ticksRequiredToFlush) { + lastTicks = nanoTime; + return true; + } + return false; + } + } +} From 597e97d10b2fe5ff0b851466147f72cf8650aa03 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:14 -0400 Subject: [PATCH 16/37] feat(feature-flagging): post flagevaluation payloads --- .../featureflag/FlagEvaluationWriterImpl.java | 313 +++++++++++++++++- 1 file changed, 296 insertions(+), 17 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 68068d1ea4c..18b04d5c99a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -7,6 +7,7 @@ import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; import datadog.communication.BackendApiFactory; +import datadog.communication.EvpProxy; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; @@ -14,7 +15,10 @@ import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.telemetry.CoreMetricCollector; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -24,14 +28,38 @@ /** * EVP {@code flagevaluation} writer for Java. * - *

Owns the bounded hook hand-off queue, background aggregation loop, and writer lifecycle. The - * EVP payload/posting path is layered on top of this queue in the next review step. + *

Uses the same EVP publisher path as {@link ExposureWriterImpl}, with two-tier aggregation + * replacing the single-exposure buffer. Routes to the Agent-advertised EVP proxy endpoint for + * {@code /api/v2/flagevaluation}. + * + *

Two-tier aggregation contract: + * + *

    + *
  • Two-tier aggregation: full -> degraded -> drop-counted. + *
  • Full key: (flagKey, variant, allocationKey, runtimeDefault, errorMessage, targetingKey, + * canonical-context-key). + *
  • Degraded key: (flagKey, variant, allocationKey, runtimeDefault, errorMessage) - no + * targetingKey/context. + *
  • Canonical context key: sorted entries, type-tagged length-delimited encoding - NOT a hash + * (collision-safe, comparable string identity). + *
  • Context pruning: deterministic (sort before cut), <=256 fields, string values <=256 chars; + * the pruned attributes are what gets aggregated and serialized. + *
  • Caps: globalCap=131072, perFlagCap=10000, degradedCap=32768. + *
  • Eval-time: min/max of firstEvalMs/lastEvalMs across events in the same bucket. + *
  • Runtime default: absent variant means {@code runtimeDefaultUsed=true}. + *
  • Flush interval: 10 seconds. + *
  • Queue: bounded MessagePassingBlockingQueue (capacity 2^16), non-blocking offer; on overflow + * the event is dropped and the {@code droppedQueueOverflow} counter is incremented and + * surfaced on flush. + *
  • Shutdown: {@link #close()} drains the queue and performs a final flush before the worker + * thread exits. + *
*/ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { private static final Logger LOGGER = LoggerFactory.getLogger(FlagEvaluationWriterImpl.class); - static final int DEFAULT_CAPACITY = 1 << 16; + static final int DEFAULT_CAPACITY = 1 << 16; // 65536 elements static final int FLUSH_INTERVAL_SECONDS = 10; static final int EVAL_SCALE_FULL_BUCKET_TARGET = FlagEvaluationAggregator.EVAL_SCALE_FULL_BUCKET_TARGET; @@ -42,15 +70,21 @@ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { static final int GLOBAL_CAP = FlagEvaluationAggregator.GLOBAL_CAP; static final int PER_FLAG_CAP = FlagEvaluationAggregator.PER_FLAG_CAP; static final int DEGRADED_CAP = FlagEvaluationAggregator.DEGRADED_CAP; + static final int FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES = EvpProxy.PAYLOAD_SIZE_LIMIT_BYTES; static final String FLAG_EVALUATION_DROPPED_METRIC = "flagevaluation.rows.dropped"; static final String FLAG_EVALUATION_DEGRADED_METRIC = "flagevaluation.rows.degraded"; + static final String FLAG_EVALUATION_SPLITS_METRIC = "flagevaluation.payload.splits"; static final String DROP_REASON_QUEUE_OVERFLOW = "queue_overflow"; - static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; static final String DROP_REASON_CLOSED = "closed"; static final String DROP_REASON_CONTEXT_ERROR = "context_error"; + static final String DROP_REASON_DEGRADED_CAP = "degraded_cap"; + static final String DROP_REASON_PAYLOAD_LIMIT = "payload_limit"; static final String DEGRADED_REASON_CARDINALITY_CAP = "cardinality_cap"; + static final String DEGRADED_REASON_PAYLOAD_LIMIT = "payload_limit"; + private static final String FLAG_EVALUATION_ROUTE = "flagevaluation"; private static final CoreMetricCollector CORE_METRICS = CoreMetricCollector.getInstance(); + // Context pruning limits: max fields and max string value length static final int MAX_CONTEXT_FIELDS = FlagEvaluationAggregator.MAX_CONTEXT_FIELDS; static final int MAX_FIELD_LENGTH = FlagEvaluationAggregator.MAX_FIELD_LENGTH; @@ -60,6 +94,17 @@ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { private final Object lifecycleLock = new Object(); private final AtomicBoolean closed = new AtomicBoolean(false); + private static void countMetric(final String metricName, final long value, final String reason) { + if (value <= 0) { + return; + } + CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); + } + + /** + * Observable counter for events dropped because the bounded hand-off queue was full when the hook + * tried to enqueue (backpressure). Incremented on the hook thread, surfaced on flush. + */ private final AtomicLong droppedQueueOverflow = new AtomicLong(0); public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { @@ -75,6 +120,7 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); } + /** Package-private constructor allowing a {@link BackendApiFactory} to be injected for tests. */ FlagEvaluationWriterImpl( final int capacity, final long flushInterval, @@ -83,7 +129,14 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); this.serializer = - new FlagEvaluationSerializingHandler(queue, flushInterval, timeUnit, droppedQueueOverflow); + new FlagEvaluationSerializingHandler( + backendApiFactory, + queue, + flushInterval, + timeUnit, + FeatureFlagEvpContext.from(config), + droppedQueueOverflow, + this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); } @@ -93,19 +146,23 @@ public void start() { if (closed.get()) { return; } + // Register with the gateway so FlagEvalLoggingHook can route evaluations to this writer FeatureFlaggingGateway.setFlagEvalWriter(this); this.serializerThread.start(); } } + /** Test seam: starts the worker thread WITHOUT registering with the global gateway. */ void startForTest() { synchronized (lifecycleLock) { - if (!closed.get()) { - this.serializerThread.start(); + if (closed.get()) { + return; } + this.serializerThread.start(); } } + /** Test seam: current full-tier bucket count in the worker's aggregator. */ int aggregatorFullTierSizeForTest() { return serializer.aggregator.fullTierSize(); } @@ -116,10 +173,12 @@ public void close() { if (!closed.compareAndSet(false, true)) { return; } + // Deregister from the gateway so no new events are enqueued. FeatureFlaggingGateway.setFlagEvalWriter(null); if (!this.serializerThread.isAlive()) { return; } + // Ask the worker to drain the queue and final-flush, then interrupt to wake it from poll(). serializer.requestShutdown(); this.serializerThread.interrupt(); } @@ -127,6 +186,7 @@ public void close() { return; } try { + // Bounded wait for the worker's final flush so queued events are not lost on shutdown. this.serializerThread.join(TimeUnit.SECONDS.toMillis(5)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -143,38 +203,58 @@ public void enqueue(final FlagEvalEvent event) { countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); return; } + // Non-blocking offer. Count overflow so loss is observable rather than silent; the count is + // surfaced on the next flush. if (!queue.offer(event)) { droppedQueueOverflow.incrementAndGet(); } } } + /** Returns the count of events dropped due to queue-overflow backpressure (observable). */ long droppedQueueOverflow() { return droppedQueueOverflow.get(); } + /** Test seam: returns one queued event without starting the worker. */ FlagEvalEvent pollQueuedEventForTest() { return queue.poll(); } + /** Test seam: flushes serializer state without starting the worker. */ void flushForTest() { serializer.flush(); } + // ---- Deterministic context pruning ---- + + /** + * Produces the deterministically-pruned context map used for both the canonical key and the + * serialized payload. Keys are sorted FIRST, then the first {@link #MAX_CONTEXT_FIELDS} + * non-oversized values are kept, so two logically-identical contexts always prune to the same + * subset (and therefore the same canonical key). Oversized string values (>{@link + * #MAX_FIELD_LENGTH} chars) are skipped. Returns an empty map for null/empty input. + */ static Map pruneContext(final Map attrs) { return FlagEvaluationAggregator.pruneContext(attrs); } + // ---- Canonical context key ---- + // Sorted entries, type-tagged length-delimited encoding. NOT a hash (collision-safe string key). + // Implementation mirrors dd-trace-go/openfeature/flagevaluation.go canonicalContextKey(). + + /** + * Builds the canonical context key from the already-pruned context map for the full-tier bucket + * identity. Expects a pruned, sorted map (e.g. produced by {@link #pruneContext(Map)}); iterating + * a {@link TreeMap} yields keys in sorted order, so the encoding is deterministic. + * + *

Returns an empty string for null/empty context. + */ static String canonicalContextKey(final Map prunedAttrs) { return FlagEvaluationAggregator.canonicalContextKey(prunedAttrs); } - private static void countMetric(final String metricName, final long value, final String reason) { - if (value <= 0) { - return; - } - CORE_METRICS.count(metricName, value, reason == null ? null : "reason:" + reason); - } + // ---- Serializing handler (background thread logic) ---- static class FlagEvaluationSerializingHandler implements Runnable { private final MessagePassingBlockingQueue queue; @@ -185,34 +265,80 @@ static class FlagEvaluationSerializingHandler implements Runnable { justification = "the field is confined to the single serializer thread") private long lastTicks; + private final FeatureFlagEvpPublisher + evpPublisher; + final Map context; private final AtomicLong droppedQueueOverflow; + private final Runnable errorCallback; + private final int payloadSizeLimitBytes; final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + + // Shutdown coordination: set by close(), drives a final drain+flush before the worker exits. private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); + private final CountDownLatch finalFlushDone = new CountDownLatch(1); + + FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final AtomicLong droppedQueueOverflow, + final Runnable errorCallback) { + this( + backendApiFactory, + queue, + flushInterval, + timeUnit, + context, + droppedQueueOverflow, + errorCallback, + FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, final MessagePassingBlockingQueue queue, final long flushInterval, final TimeUnit timeUnit, - final AtomicLong droppedQueueOverflow) { + final Map context, + final AtomicLong droppedQueueOverflow, + final Runnable errorCallback, + final int payloadSizeLimitBytes) { this.queue = queue; + this.evpPublisher = + new FeatureFlagEvpPublisher<>( + backendApiFactory, FlagEvaluationPayloads.FlagEvaluationsRequest.class, false); + this.context = context; this.droppedQueueOverflow = droppedQueueOverflow; + this.payloadSizeLimitBytes = payloadSizeLimitBytes; this.lastTicks = System.nanoTime(); this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + this.errorCallback = errorCallback; LOGGER.debug("starting flag evaluation serializer"); } + /** Signals the worker to drain the queue and perform a final flush before exiting. */ void requestShutdown() { shutdownRequested.set(true); } @Override public void run() { + if (!evpPublisher.start()) { + finalFlushDone.countDown(); + errorCallback.run(); + throw new IllegalArgumentException("EVP Proxy not available"); + } try { runDutyCycle(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } finally { + // On exit (interrupt or shutdown request), drain everything still buffered and flush it so + // queued events are not lost on shutdown. drainAndFlush(); + finalFlushDone.countDown(); } LOGGER.debug("flag evaluation processor worker exited."); } @@ -228,6 +354,7 @@ private void runDutyCycle() throws InterruptedException { } } + /** Drains all remaining queued events and performs a final flush. Used on shutdown. */ void drainAndFlush() { FlagEvalEvent event; while ((event = queue.poll()) != null) { @@ -236,6 +363,9 @@ void drainAndFlush() { flush(); } + // ---- Aggregation logic ---- + + /** Routes an event into the full tier or degraded tier, or drops and counts on overflow. */ void aggregateEvent(final FlagEvalEvent event) { try { aggregator.aggregate(event); @@ -245,6 +375,8 @@ void aggregateEvent(final FlagEvalEvent event) { } } + // ---- Flush logic ---- + void flushIfNecessary() { if (aggregator.isEmpty() && droppedQueueOverflow.get() == 0) { return; @@ -255,6 +387,8 @@ void flushIfNecessary() { } void flush() { + // Surface backpressure (queue-overflow) drops as an observable warning even when there is + // nothing else to flush. final long qDrops = droppedQueueOverflow.getAndSet(0); countMetric(FLAG_EVALUATION_DROPPED_METRIC, qDrops, DROP_REASON_QUEUE_OVERFLOW); if (qDrops > 0) { @@ -271,14 +405,68 @@ void flush() { + " (best-effort telemetry)", dgDrops); } - if (!aggregator.isEmpty()) { + + if (aggregator.isEmpty()) { + return; + } + boolean aggregatesWereEncoded = false; + try { countMetric( FLAG_EVALUATION_DEGRADED_METRIC, aggregator.degradedEvaluationCount(), DEGRADED_REASON_CARDINALITY_CAP); - aggregator.clear(); + final List events = buildEventList(); + if (events.isEmpty()) { + return; + } + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(events, context, payloadSizeLimitBytes); + aggregatesWereEncoded = true; + countMetric( + FLAG_EVALUATION_DROPPED_METRIC, + payloads.droppedPayloadLimit, + DROP_REASON_PAYLOAD_LIMIT); + countMetric( + FLAG_EVALUATION_DEGRADED_METRIC, + payloads.degradedPayloadLimit, + DEGRADED_REASON_PAYLOAD_LIMIT); + if (payloads.bodies.size() > 1) { + countMetric(FLAG_EVALUATION_SPLITS_METRIC, payloads.bodies.size() - 1, null); + } + if (payloads.droppedPayloadLimit > 0) { + LOGGER.warn( + "flag evaluation payload too large - dropped {} evaluation(s)" + + " (best-effort telemetry)", + payloads.droppedPayloadLimit); + } + for (final byte[] payload : payloads.bodies) { + evpPublisher.post(FLAG_EVALUATION_ROUTE, payload); + } + } catch (Exception e) { + LOGGER.error("Could not submit flag evaluations", e); + } finally { + if (aggregatesWereEncoded) { + // Once payload bytes are built, this writer is best-effort: clear encoded aggregates even + // if one split post fails so a later flush cannot duplicate already-sent rows. + aggregator.clear(); + lastTicks = System.nanoTime(); + } } - lastTicks = System.nanoTime(); + } + + private List buildEventList() { + final long flushTimeMs = System.currentTimeMillis(); + final List events = + new ArrayList<>(aggregator.bucketCount()); + for (final FlagEvaluationAggregator.EvalBucket bucket : aggregator.fullBuckets()) { + events.add( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket(bucket, true, flushTimeMs)); + } + for (final FlagEvaluationAggregator.EvalBucket bucket : aggregator.degradedBuckets()) { + events.add( + FlagEvaluationPayloads.FlagEvaluationEvent.fromBucket(bucket, false, flushTimeMs)); + } + return events; } private boolean shouldFlush() { @@ -291,4 +479,95 @@ private boolean shouldFlush() { return false; } } + + // ---- Test-seam inner class (package-private) ---- + + /** + * Test-accessible handler that exposes {@link #drainAndAggregate()} and {@link #flush()} without + * starting a real background thread. + */ + static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler { + + SerializingHandlerForTest(final BackendApiFactory factory, final Map context) { + this(factory, context, FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + SerializingHandlerForTest( + final BackendApiFactory factory, + final Map context, + final int payloadSizeLimitBytes) { + super( + factory, + Queues.mpscBlockingConsumerArrayQueue(DEFAULT_CAPACITY), + Long.MAX_VALUE, // effectively never auto-flush + TimeUnit.NANOSECONDS, + context, + new AtomicLong(0), + () -> {}, + payloadSizeLimitBytes); + } + + private final List staged = new ArrayList<>(); + + /** Adds an event to the staged list (simulates hook enqueue). */ + void add(final FlagEvalEvent event) { + staged.add(event); + } + + /** Aggregates all staged events and returns the current aggregation state. */ + FlagEvaluationAggregator.AggregatedState drainAndAggregate() { + for (final FlagEvalEvent e : staged) { + aggregateEvent(e); + } + staged.clear(); + return aggregator.snapshot(); + } + + /** Simulates filling the full tier to GLOBAL_CAP by injecting synthetic distinct buckets. */ + void simulateFullTierAtCap() { + aggregator.simulateFullTierAtCap(); + } + + /** + * Simulates filling the degraded tier to DEGRADED_CAP by injecting synthetic distinct buckets. + */ + void simulateDegradedTierAtCap() { + aggregator.simulateDegradedTierAtCap(); + } + + void addDroppedDegradedOverflowForTest(final long count) { + aggregator.droppedDegradedOverflow.addAndGet(count); + } + + void addDegradedBucketForTest( + final String flagKey, + final String variant, + final String allocationKey, + final String errorMessage, + final long evalTimeMs) { + aggregator.addDegradedBucketForTest( + flagKey, variant, allocationKey, errorMessage, evalTimeMs); + } + + void clearAggregationForTest() { + aggregator.clear(); + } + + int fullTierSizeForTest() { + return aggregator.fullTierSize(); + } + } + + /** Factory method for test use - creates a SerializingHandlerForTest. */ + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, final Map context) { + return new SerializingHandlerForTest(factory, context); + } + + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, + final Map context, + final int payloadSizeLimitBytes) { + return new SerializingHandlerForTest(factory, context, payloadSizeLimitBytes); + } } From 58df9361656a1d171c0312a506f57202034296ca Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:22 -0400 Subject: [PATCH 17/37] test(feature-flagging): add flagevaluation test support --- .../FlagEvaluationTestSupport.java | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java new file mode 100644 index 00000000000..11e5b4f407f --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -0,0 +1,212 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.telemetry.CoreMetricCollector; +import datadog.trace.api.telemetry.MetricCollector; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import okhttp3.RequestBody; +import okio.Buffer; + +final class FlagEvaluationTestSupport { + + static final long REALISTIC_EVAL_MS = 1_760_000_000_000L; + static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + + private FlagEvaluationTestSupport() {} + + static void clearCoreMetrics() { + CoreMetricCollector.getInstance().drain(); + } + + static FlagEvalEvent event( + final String flagKey, + final String variant, + final String allocationKey, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); + } + + static FlagEvalEvent errorEvent( + final String flagKey, final String errorMessage, final long evalTimeMs) { + return new FlagEvalEvent( + flagKey, null, null, null, errorMessage, evalTimeMs, java.util.Collections.emptyMap()); + } + + static FlagEvalEvent simpleEvent(final String flagKey, final String variant) { + return event(flagKey, variant, "alloc1", "user-1", 1000L, java.util.Collections.emptyMap()); + } + + static String repeat(final char c, final int count) { + final char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + static TestWriterSetup buildTestWriter(final BackendApi mockEvp) { + return buildTestWriter( + mockEvp, FlagEvaluationWriterImpl.FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + } + + static TestWriterSetup buildTestWriter( + final BackendApi mockEvp, final int payloadSizeLimitBytes) { + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + + final Map context = new HashMap<>(); + context.put("service", "test-service"); + + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context, payloadSizeLimitBytes); + + return new TestWriterSetup(handler, mockEvp, factory); + } + + static CapturedJson flushAndCapture(final TestWriterSetup setup) throws Exception { + final List captured = flushAndCaptureAll(setup); + assertEquals(1, captured.size(), "Expected exactly one posted payload"); + return captured.get(0); + } + + static List flushAndCaptureAll(final TestWriterSetup setup) throws Exception { + final List captured = new ArrayList<>(); + when(setup.mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured.add(inv.getArgument(1)); + return null; + }); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + final List json = new ArrayList<>(); + for (final RequestBody body : captured) { + json.add(readJson(body)); + } + return json; + } + + static CapturedJson readJson(final RequestBody body) throws Exception { + assertNotNull(body, "RequestBody must have been posted"); + final Buffer buf = new Buffer(); + body.writeTo(buf); + final String raw = buf.readUtf8(); + return new CapturedJson(raw, JSON_MAP.fromJson(raw)); + } + + static Map flushAndCaptureJson(final TestWriterSetup setup) throws Exception { + return flushAndCapture(setup).parsed; + } + + static long metricSum( + final Collection metrics, + final String metricName, + final String tag) { + long sum = 0; + for (final MetricCollector.Metric metric : metrics) { + if (!metricName.equals(metric.metricName)) { + continue; + } + if (tag == null) { + if (!metric.tags.isEmpty()) { + continue; + } + } else if (!metric.tags.contains(tag)) { + continue; + } + sum += metric.value.longValue(); + } + return sum; + } + + static datadog.trace.api.Config cfg() { + final datadog.trace.api.Config config = mock(datadog.trace.api.Config.class); + when(config.getServiceName()).thenReturn("test-service"); + return config; + } + + static void assertObjectWithKey(final Object o, final String expectedKey, final String msg) { + assertTrue(o instanceof Map, msg + " (must be a JSON object, not a bare string)"); + assertEquals(expectedKey, ((Map) o).get("key"), msg); + } + + @SuppressWarnings("unchecked") + static Map firstEvent(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + assertFalse(events.isEmpty(), "flagEvaluations must not be empty"); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + static int eventCount(final Map batch) { + final List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + return events.size(); + } + + @SuppressWarnings("unchecked") + static Map eventForFlag(final Map batch, final String flagKey) { + final List events = (List) batch.get("flagEvaluations"); + for (final Object o : events) { + final Map ev = (Map) o; + final Map flag = (Map) ev.get("flag"); + if (flag != null && flagKey.equals(flag.get("key"))) { + return ev; + } + } + return null; + } + + static class TestWriterSetup { + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + final BackendApi mockEvp; + final BackendApiFactory factory; + + TestWriterSetup( + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler, + final BackendApi mockEvp, + final BackendApiFactory factory) { + this.handler = handler; + this.mockEvp = mockEvp; + this.factory = factory; + } + } + + static class CapturedJson { + final String raw; + final Map parsed; + + CapturedJson(final String raw, final Map parsed) { + this.raw = raw; + this.parsed = parsed; + } + } +} From acc6e21940a1f1378dc2f6d298649e1a5d615da1 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:22 -0400 Subject: [PATCH 18/37] test(feature-flagging): cover flagevaluation writer --- .../FlagEvaluationWriterImplTest.java | 304 ++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java new file mode 100644 index 00000000000..bec4ed9eaee --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -0,0 +1,304 @@ +package com.datadog.featureflag; + +import static com.datadog.featureflag.FlagEvaluationTestSupport.JSON_MAP; +import static com.datadog.featureflag.FlagEvaluationTestSupport.buildTestWriter; +import static com.datadog.featureflag.FlagEvaluationTestSupport.cfg; +import static com.datadog.featureflag.FlagEvaluationTestSupport.clearCoreMetrics; +import static com.datadog.featureflag.FlagEvaluationTestSupport.event; +import static com.datadog.featureflag.FlagEvaluationTestSupport.eventForFlag; +import static com.datadog.featureflag.FlagEvaluationTestSupport.metricSum; +import static com.datadog.featureflag.FlagEvaluationTestSupport.repeat; +import static com.datadog.featureflag.FlagEvaluationTestSupport.simpleEvent; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.intake.Intake; +import datadog.trace.api.telemetry.CoreMetricCollector; +import datadog.trace.api.telemetry.MetricCollector; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.RequestBody; +import okio.Buffer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class FlagEvaluationWriterImplTest { + + @BeforeEach + void clearCoreMetricsBefore() { + clearCoreMetrics(); + } + + @AfterEach + void clearCoreMetricsAfter() { + clearCoreMetrics(); + } + + @Test + void degradedCapOverflowTelemetryIsEmittedOnFlush() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.addDroppedDegradedOverflowForTest(3); + setup.handler.flush(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 3, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_DEGRADED_CAP)); + } + + @Test + void queueOverflowIncrementsObservableDropCounter() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(2, 10L, TimeUnit.SECONDS, factory, cfg()); + + for (int i = 0; i < 100; i++) { + writer.enqueue(simpleEvent("of-flag", "on")); + } + + assertTrue(writer.droppedQueueOverflow() > 0); + final long queueDrops = writer.droppedQueueOverflow(); + writer.flushForTest(); + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + queueDrops, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_QUEUE_OVERFLOW)); + } + + @Test + void enqueueAfterCloseIsDroppedAndCounted() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.close(); + writer.enqueue(simpleEvent("closed-flag", "on")); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void enqueueDoesNotAggregateOnTheCallingThread() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.enqueue(simpleEvent("g2-flag", "on")); + writer.enqueue(simpleEvent("g2-flag", "on")); + + assertEquals(0, writer.aggregatorFullTierSizeForTest()); + assertEquals(0, writer.droppedQueueOverflow()); + } + + @Test + void enqueueDoesNotResolveContextBeforeBuffering() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + final AtomicInteger resolutions = new AtomicInteger(); + final Map attrs = new HashMap<>(); + attrs.put("tier", "gold"); + + writer.enqueue( + new FlagEvalEvent( + "lazy-flag", + "on", + "alloc1", + "user-1", + null, + 1000L, + () -> { + resolutions.incrementAndGet(); + return attrs; + })); + + final FlagEvalEvent queued = writer.pollQueuedEventForTest(); + assertNotNull(queued); + assertTrue(queued.attrs.isEmpty()); + assertEquals(0, resolutions.get()); + assertEquals("gold", queued.contextAttributes().get("tier")); + assertEquals(1, resolutions.get()); + } + + @Test + void contextMaterializationFailureDropsSingleEvent() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.add( + new FlagEvalEvent( + "bad-context", + "on", + "alloc1", + "user-1", + null, + 1000L, + () -> { + throw new IllegalArgumentException("bad context"); + })); + + setup.handler.drainAndAggregate(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CONTEXT_ERROR)); + assertEquals(0, setup.handler.fullTierSizeForTest()); + } + + @Test + void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { + final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); + final RequestBody[] captured = {null}; + final BackendApi mockEvp = mock(BackendApi.class); + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured[0] = inv.getArgument(1); + posted.countDown(); + return null; + }); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 64, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS, factory, cfg()); + + writer.startForTest(); + writer.enqueue(simpleEvent("shutdown-flag", "on")); + writer.close(); + + assertTrue(posted.await(5, TimeUnit.SECONDS)); + assertNotNull(captured[0]); + final Buffer buf = new Buffer(); + captured[0].writeTo(buf); + final Map json = JSON_MAP.fromJson(buf.readUtf8()); + assertNotNull(eventForFlag(json, "shutdown-flag")); + } + + @Test + void continuousTrafficFlushesWithoutWaitingForIdle() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(1 << 12, 1, TimeUnit.MILLISECONDS, factory, cfg()); + + writer.startForTest(); + boolean posted = false; + try { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (System.nanoTime() < deadline) { + writer.enqueue(simpleEvent("busy-flag", "on")); + try { + verify(mockEvp, atLeastOnce()) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + posted = true; + break; + } catch (AssertionError ignored) { + // Keep the worker busy until the deadline. + } + } + } finally { + writer.close(); + } + + assertTrue(posted); + } + + @Test + void flushPostsToFlagevaluationEndpoint() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.add(event("flag-f", "on", "alloc1", "user-1", 1000L, emptyMap())); + setup.handler.drainAndAggregate(); + setup.handler.flush(); + + verify(setup.factory).createBackendApi(eq(Intake.EVENT_PLATFORM), isNull(), eq(false)); + verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + } + + @Test + void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { + final int limit = 1_100; + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp, limit); + final AtomicInteger posts = new AtomicInteger(); + doAnswer( + invocation -> { + if (posts.incrementAndGet() == 2) { + throw new IOException("boom"); + } + return null; + }) + .when(mockEvp) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + + for (int i = 0; i < 4; i++) { + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 180)); + setup.handler.add(event("split-failure-" + i, "on", "alloc1", "user-" + i, 1000L, attrs)); + } + + setup.handler.drainAndAggregate(); + setup.handler.flush(); + assertEquals(2, posts.get()); + setup.handler.flush(); + assertEquals(2, posts.get()); + } +} From 96df04f1be66c6ebd237752e43c8f37b09ec8212 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:29 -0400 Subject: [PATCH 19/37] feat(feature-flagging): wire flagevaluation writer lifecycle --- .../featureflag/FeatureFlaggingSystem.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..c5be9b67f82 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -2,6 +2,8 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; +import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +13,7 @@ public class FeatureFlaggingSystem { private static volatile RemoteConfigService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; + private static volatile FlagEvaluationWriter FLAG_EVAL_WRITER; private FeatureFlaggingSystem() {} @@ -27,10 +30,29 @@ public static void start(final SharedCommunicationObjects sco) { EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); + final boolean evalCountsEnabled = + config + .configProvider() + .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); + if (evalCountsEnabled) { + final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); + evalWriter.start(); + FLAG_EVAL_WRITER = evalWriter; + LOGGER.debug("Flag evaluation EVP writer started"); + } else { + LOGGER.debug( + "Flag evaluation EVP writer disabled ({}=false)", + FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED); + } + LOGGER.debug("Feature Flagging system started"); } public static void stop() { + if (FLAG_EVAL_WRITER != null) { + FLAG_EVAL_WRITER.close(); + FLAG_EVAL_WRITER = null; + } if (EXPOSURE_WRITER != null) { EXPOSURE_WRITER.close(); EXPOSURE_WRITER = null; From edb264e7d8419dcaa161b4b8a077e61cbcc7ad29 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:30 -0400 Subject: [PATCH 20/37] test(feature-flagging): cover flagevaluation writer lifecycle --- .../FeatureFlaggingSystemTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 970c2cf16e3..82fae6ecdb0 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,6 +1,8 @@ package com.datadog.featureflag; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -15,6 +17,8 @@ import datadog.remoteconfig.ConfigurationPoller; import datadog.remoteconfig.Product; import datadog.trace.api.Config; +import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.junit.utils.config.WithConfig; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -23,6 +27,7 @@ class FeatureFlaggingSystemTest { @Test + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); @@ -39,14 +44,37 @@ void testFeatureFlagSystemInitialization() { verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + assertNotNull(FeatureFlaggingGateway.getFlagEvalWriter()); FeatureFlaggingSystem.stop(); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).removeListeners(Product.FFE_FLAGS); verify(poller).stop(); } + @Test + @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "false") + void testFlagEvaluationWriterCanBeDisabled() { + ConfigurationPoller poller = mock(ConfigurationPoller.class); + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + when(discovery.supportsEvpProxy()).thenReturn(true); + when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); + when(sharedCommunicationObjects.configurationPoller(any(Config.class))).thenReturn(poller); + when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); + sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); + sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } finally { + FeatureFlaggingSystem.stop(); + } + } + @Test @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { From 90ed6cb556c56be50657236faa366b7e0907d4e6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 23:49:30 -0400 Subject: [PATCH 21/37] perf(feature-flagging): benchmark flagevaluation hot path --- .../feature-flagging-lib/build.gradle.kts | 12 ++ .../FlagEvaluationHotPathBenchmark.java | 143 ++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 3291e239d40..73337d4676a 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -1,6 +1,7 @@ plugins { `java-library` id("dd-trace-java.version-file") + id("me.champeau.jmh") } apply(from = "$rootDir/gradle/java.gradle") @@ -28,3 +29,14 @@ dependencies { testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } + +jmh { + jmhVersion = libs.versions.jmh.get() + duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE + if (project.hasProperty("jmhIncludes")) { + includes = listOf(project.property("jmhIncludes").toString()) + } + if (project.hasProperty("jmhProf")) { + profilers = listOf(project.property("jmhProf").toString()) + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java new file mode 100644 index 00000000000..bd70b70d211 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -0,0 +1,143 @@ +package com.datadog.featureflag; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.communication.BackendApiFactory; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; +import java.util.Map; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Hot-path benchmark for EVP {@code flagevaluation} recording. + * + *

The OpenFeature {@code finally} hook runs synchronously on the caller's evaluation thread. The + * evaluation thread should only pay for scalar capture, lazy context supplier capture, and the + * non-blocking writer enqueue. Recursive context flattening, pruning, canonical-key construction, + * and aggregation are characterized separately as worker-thread cost. + * + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh -PjmhIncludes=FlagEvaluationHotPathBenchmark}. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class FlagEvaluationHotPathBenchmark { + + @Param({ + "typical/100flags_50users_10fields", + "stress/10flags_1000users_250fields", + "scale/2500flags_500users_20fields" + }) + public String profile; + + private Map attrs; + private String[] flagKeys; + private String[] targetingKeys; + private int cursor; + private FlagEvaluationWriterImpl writer; + private FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + + @Setup(Level.Iteration) + public void setUp() { + final Profile p = Profile.fromName(profile); + + attrs = new HashMap<>(); + for (int i = 0; i < p.numFields; i++) { + attrs.put("field" + i, "value"); + } + flagKeys = keys("bench-flag-", p.numFlags); + targetingKeys = keys("bench-user-", p.numUsers); + cursor = 0; + + final Config config = Config.get(); + final BackendApiFactory factory = new BackendApiFactory(config, null); + final Map ddContext = new HashMap<>(); + ddContext.put("service", "bench-service"); + handler = FlagEvaluationWriterImpl.createHandlerForTest(factory, ddContext); + + // Capacity large enough that the benchmark never overflows within a measurement window. + writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + } + + /** Evaluation-thread cost: lazy capture snapshot plus non-blocking enqueue. */ + @Benchmark + public void evalThreadCapture(final Blackhole blackhole) { + final FlagEvalEvent event = nextLazyEvent(); + writer.enqueue(event); + blackhole.consume(writer.pollQueuedEventForTest()); + blackhole.consume(event); + } + + /** Worker-thread cost: materialize context, prune, canonicalize, and aggregate. */ + @Benchmark + public void workerAggregate(final Blackhole blackhole) { + final FlagEvalEvent event = nextLazyEvent(); + handler.aggregateEvent(event); + if ((cursor % 10_000) == 0) { + handler.clearAggregationForTest(); + } + blackhole.consume(handler.fullTierSizeForTest()); + } + + private FlagEvalEvent nextLazyEvent() { + final int i = cursor++; + return new FlagEvalEvent( + flagKeys[Math.floorMod(i, flagKeys.length)], + "variant-" + Math.floorMod(i, 4), + "alloc-" + Math.floorMod(i, flagKeys.length), + targetingKeys[Math.floorMod(i, targetingKeys.length)], + null, + 1_700_000_000_000L + i, + () -> attrs); + } + + private static String[] keys(final String prefix, final int count) { + final String[] out = new String[count]; + for (int i = 0; i < count; i++) { + out[i] = prefix + i; + } + return out; + } + + private static final class Profile { + private final int numFlags; + private final int numUsers; + private final int numFields; + + private Profile(final int numFlags, final int numUsers, final int numFields) { + this.numFlags = numFlags; + this.numUsers = numUsers; + this.numFields = numFields; + } + + private static Profile fromName(final String name) { + if ("typical/100flags_50users_10fields".equals(name)) { + return new Profile(100, 50, 10); + } + if ("stress/10flags_1000users_250fields".equals(name)) { + return new Profile(10, 1_000, 250); + } + if ("scale/2500flags_500users_20fields".equals(name)) { + return new Profile(2_500, 500, 20); + } + throw new IllegalArgumentException("unknown benchmark profile: " + name); + } + } +} From c73cafdb46fc8fff39cb83f5f61f8665b24b6a4f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 10:28:16 -0400 Subject: [PATCH 22/37] chore: apply spotless formatting --- .../test/java/datadog/communication/BackendApiFactoryTest.java | 3 +-- .../java/datadog/trace/api/openfeature/DDEvaluatorTest.java | 3 +-- .../datadog/featureflag/FlagEvaluationHotPathBenchmark.java | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index c287790b4ae..38a75ddceb6 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -84,8 +84,7 @@ void preferredEvpProxyEndpointDoesNotRequireLegacyDefaultRoute() throws Exceptio final BackendApiFactory factory = new BackendApiFactory( Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); - final BackendApi api = - factory.createBackendApi(Intake.EVENT_PLATFORM, futureEndpoint, false); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, futureEndpoint, false); assertNotNull(api); api.post( diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index eee6661f024..e81b33bf4c5 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -236,8 +236,7 @@ private static Arguments[] flatteningTestCases() { mapOf("map.key1", 1, "map.key2", 2, "map.key3.key4", 4))); arguments.add( Arguments.of( - mapOf("plan", "gold", "cohort", "gold"), - mapOf("plan", "gold", "cohort", "gold"))); + mapOf("plan", "gold", "cohort", "gold"), mapOf("plan", "gold", "cohort", "gold"))); return arguments.toArray(new Arguments[0]); } diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java index bd70b70d211..f8bbc729f59 100644 --- a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -30,7 +30,8 @@ * non-blocking writer enqueue. Recursive context flattening, pruning, canonical-key construction, * and aggregation are characterized separately as worker-thread cost. * - *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh -PjmhIncludes=FlagEvaluationHotPathBenchmark}. + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh + * -PjmhIncludes=FlagEvaluationHotPathBenchmark}. */ @State(Scope.Benchmark) @Warmup(iterations = 3, time = 2, timeUnit = SECONDS) From 5b1456b3fbe467dd6b74ba919c85720e7f763921 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 11:36:17 -0400 Subject: [PATCH 23/37] test(feature-flagging): cover flag evaluation jacoco paths --- .../FeatureFlagEvpPublisherTest.java | 14 ++ .../FlagEvaluationAggregatorTest.java | 133 +++++++++++++ .../FlagEvaluationPayloadsTest.java | 123 ++++++++++++ .../FlagEvaluationTestSupport.java | 13 +- .../FlagEvaluationWriterImplTest.java | 184 ++++++++++++++++++ 5 files changed, 465 insertions(+), 2 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java index b5044f43c7a..ad66a148fc7 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -1,6 +1,8 @@ package com.datadog.featureflag; import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; @@ -62,6 +64,18 @@ void preferredEndpointAndRequestCompressionAreForwardedToBackendApi() throws Exc .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); } + @Test + void postThrowsWhenEvpBackendApiCannotBeCreated() { + final BackendApiFactory factory = mock(BackendApiFactory.class); + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(factory, TestRequest.class); + + assertFalse(publisher.start()); + assertThrows( + IllegalStateException.class, + () -> publisher.post("flagevaluation", FeatureFlagEvpPublisher.utf8Bytes("{}"))); + } + static class TestRequest { public final String value; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java index 0adfcf35d3e..57bc7251c1e 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationAggregatorTest.java @@ -3,6 +3,7 @@ import static java.util.Collections.emptyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; @@ -79,6 +80,24 @@ void degradedCapOverflowIncrementsDroppedCounter() { assertTrue(state.droppedDegradedOverflow > 0); } + @Test + void perFlagCapOverflowRoutesToDegradedTierAndMergesSameDegradedKey() { + final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); + aggregator.perFlagCount.put("hot-flag", FlagEvaluationAggregator.PER_FLAG_CAP); + + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-1", 1000L, emptyMap())); + aggregator.aggregate(event("hot-flag", "on", "alloc1", "user-2", 2000L, emptyMap())); + + final FlagEvaluationAggregator.AggregatedState state = aggregator.snapshot(); + assertEquals(0, state.fullTier.size()); + assertEquals(1, state.degradedTier.size()); + final FlagEvaluationAggregator.EvalBucket bucket = + state.degradedTier.values().iterator().next(); + assertEquals(2, bucket.count); + assertEquals(1000L, bucket.firstEvalMs); + assertEquals(2000L, bucket.lastEvalMs); + } + @Test void absentVariantSetsRuntimeDefaultUsed() { final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); @@ -125,6 +144,34 @@ void pruningIsDeterministicSortBeforeCut() { assertFalse(p1.containsKey("k299")); } + @Test + void emptyContextInputsProduceEmptyPrunedMapAndCanonicalKey() { + assertEquals(emptyMap(), FlagEvaluationAggregator.pruneContext(null)); + assertEquals(emptyMap(), FlagEvaluationAggregator.pruneContext(emptyMap())); + assertEquals("", FlagEvaluationAggregator.canonicalContextKey(null)); + assertEquals("", FlagEvaluationAggregator.canonicalContextKey(emptyMap())); + } + + @Test + void canonicalContextKeyEncodesSupportedValueTypes() { + final Map attrs = new HashMap<>(); + attrs.put("bool", true); + attrs.put("double", 1.5d); + attrs.put("float", 1.25f); + attrs.put("int", 1); + attrs.put("long", 2L); + attrs.put("null", null); + attrs.put("object", new StringBuilder("other")); + attrs.put("string", "value"); + + final String key = FlagEvaluationAggregator.canonicalContextKey(attrs); + + assertEquals(key, FlagEvaluationAggregator.canonicalContextKey(new HashMap<>(attrs))); + assertTrue(key.contains("bool")); + assertTrue(key.contains("string")); + assertTrue(key.contains("other")); + } + @Test void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { final FlagEvaluationAggregator aggregator = new FlagEvaluationAggregator(); @@ -161,6 +208,64 @@ void flagEvalEventDoesNotCarryReason() { assertFalse(hasReasonField); } + @Test + void evalBucketTracksBoundsDefaultStateAndNullContextFieldCount() { + final FlagEvaluationAggregator.EvalBucket bucket = + new FlagEvaluationAggregator.EvalBucket( + "bucket-flag", "on", "alloc1", "user-1", null, 1000L, false, null); + + assertEquals(0, bucket.prunedContextFieldCount()); + + bucket.merge(900L, true); + bucket.merge(1100L, false); + bucket.merge(1000L, false); + + assertEquals(4, bucket.count); + assertEquals(900L, bucket.firstEvalMs); + assertEquals(1100L, bucket.lastEvalMs); + assertTrue(bucket.runtimeDefaultUsed); + } + + @Test + void fullKeyEqualityUsesEveryDimension() { + final FlagEvaluationAggregator.FullKey base = + fullKey("flag", "on", "alloc", false, "error", "user", "ctx"); + final FlagEvaluationAggregator.FullKey same = + fullKey("flag", "on", "alloc", false, "error", "user", "ctx"); + + assertEquals(base, base); + assertEquals(base, same); + assertEquals(base.hashCode(), same.hashCode()); + assertNotEquals(base, null); + assertNotEquals(base, "not-a-key"); + assertNotEquals(base, fullKey("other", "on", "alloc", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "off", "alloc", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "other", false, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", true, "error", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "other", "user", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "error", "other", "ctx")); + assertNotEquals(base, fullKey("flag", "on", "alloc", false, "error", "user", "other")); + } + + @Test + void degradedKeyEqualityUsesEveryDimension() { + final FlagEvaluationAggregator.DegradedKey base = + degradedKey("flag", "on", "alloc", false, "error"); + final FlagEvaluationAggregator.DegradedKey same = + degradedKey("flag", "on", "alloc", false, "error"); + + assertEquals(base, base); + assertEquals(base, same); + assertEquals(base.hashCode(), same.hashCode()); + assertNotEquals(base, null); + assertNotEquals(base, "not-a-key"); + assertNotEquals(base, degradedKey("other", "on", "alloc", false, "error")); + assertNotEquals(base, degradedKey("flag", "off", "alloc", false, "error")); + assertNotEquals(base, degradedKey("flag", "on", "other", false, "error")); + assertNotEquals(base, degradedKey("flag", "on", "alloc", true, "error")); + assertNotEquals(base, degradedKey("flag", "on", "alloc", false, "other")); + } + private static FlagEvalEvent event( final String flagKey, final String variant, @@ -175,6 +280,34 @@ private static FlagEvalEvent simpleEvent(final String flagKey, final String vari return event(flagKey, variant, "alloc1", "user-1", 1000L, emptyMap()); } + private static FlagEvaluationAggregator.FullKey fullKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage, + final String targetingKey, + final String contextKey) { + return new FlagEvaluationAggregator.FullKey( + flagKey, + variant, + allocationKey, + runtimeDefaultUsed, + errorMessage, + targetingKey, + contextKey); + } + + private static FlagEvaluationAggregator.DegradedKey degradedKey( + final String flagKey, + final String variant, + final String allocationKey, + final boolean runtimeDefaultUsed, + final String errorMessage) { + return new FlagEvaluationAggregator.DegradedKey( + flagKey, variant, allocationKey, runtimeDefaultUsed, errorMessage); + } + private static String repeat(final char c, final int count) { final char[] chars = new char[count]; Arrays.fill(chars, c); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java index 3ba2dc8fccf..e3c2d120b00 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -11,6 +11,7 @@ import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; import java.lang.reflect.Type; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -144,6 +145,45 @@ void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { assertNull(ev.get("context")); } + @Test + void oversizedFullPayloadRowStartsNewPayloadWhenDegradedRowFitsByItself() throws Exception { + final FlagEvaluationPayloads.FlagEvaluationEvent first = + event("first-flag", "on", "alloc1", "user-1", 1, emptyMap()); + final Map attrs = new HashMap<>(); + attrs.put("payload", repeat('x', 400)); + final FlagEvaluationPayloads.FlagEvaluationEvent second = + event("second-flag", "on", "alloc1", "user-2", 3, attrs); + final FlagEvaluationPayloads.FlagEvaluationEvent degradedSecond = + second.withoutTargetingKeyAndContext(); + assertNotNull(degradedSecond); + + final int firstPayloadSize = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList(first), CONTEXT, 1_000_000) + .bodies + .get(0) + .length; + final int degradedPayloadSize = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList(degradedSecond), CONTEXT, 1_000_000) + .bodies + .get(0) + .length; + final int limit = Math.max(firstPayloadSize, degradedPayloadSize); + + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(Arrays.asList(first, second), CONTEXT, limit); + + assertEquals(2, payloads.bodies.size()); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(3, payloads.degradedPayloadLimit); + assertEquals(1, eventCount(parse(payloads.bodies.get(0)))); + final Map ev = firstEvent(parse(payloads.bodies.get(1))); + assertObjectWithKey(ev.get("flag"), "second-flag"); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + @Test void oversizedDegradedPayloadRowIsDropped() { final FlagEvaluationPayloads.EncodedPayloads payloads = @@ -158,6 +198,20 @@ void oversizedDegradedPayloadRowIsDropped() { assertEquals(0, payloads.degradedPayloadLimit); } + @Test + void oversizedFullPayloadRowIsDroppedWhenDegradedRowStillExceedsLimit() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads( + java.util.Collections.singletonList( + event(repeat('f', 512), "on", "alloc1", "user-1", 2, emptyMap())), + CONTEXT, + 128); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(2, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + @Test void errorPayloadSerializesErrorObject() throws Exception { final Map json = @@ -186,6 +240,75 @@ void errorPayloadSerializesErrorObject() throws Exception { assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); } + @Test + void emptyEventListProducesNoPayloads() { + final FlagEvaluationPayloads.EncodedPayloads payloads = + FlagEvaluationPayloads.buildPayloads(java.util.Collections.emptyList(), CONTEXT, 1_000_000); + + assertTrue(payloads.bodies.isEmpty()); + assertEquals(0, payloads.droppedPayloadLimit); + assertEquals(0, payloads.degradedPayloadLimit); + } + + @Test + void requestDtoStoresContextAndEvents() { + final List events = + java.util.Collections.singletonList( + event("dto-flag", "on", "alloc1", "user-1", 1, emptyMap())); + + final FlagEvaluationPayloads.FlagEvaluationsRequest request = + new FlagEvaluationPayloads.FlagEvaluationsRequest(CONTEXT, events); + + assertEquals(CONTEXT, request.context); + assertEquals(events, request.flagEvaluations); + } + + @Test + void degradingEventWithMissingOptionalFieldsKeepsOptionalObjectsAbsent() { + final FlagEvaluationPayloads.FlagEvaluationEvent degraded = + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, + "default-flag", + EVAL_MS, + EVAL_MS, + 1, + null, + null, + "user-1", + true, + null, + null) + .withoutTargetingKeyAndContext(); + + assertNotNull(degraded); + assertNull(degraded.variant); + assertNull(degraded.allocation); + assertNull(degraded.targeting_key); + assertNull(degraded.context); + assertNull(degraded.error); + assertEquals(Boolean.TRUE, degraded.runtime_default_used); + } + + @Test + void emptyOptionalStringsAreTreatedAsAbsentWhenContextIsPresent() { + final Map attrs = new HashMap<>(); + attrs.put("tier", "gold"); + final FlagEvaluationPayloads.FlagEvaluationEvent event = + new FlagEvaluationPayloads.FlagEvaluationEvent( + EVAL_MS, "empty-optionals", EVAL_MS, EVAL_MS, 1, "", "", null, false, "", attrs); + + assertNull(event.variant); + assertNull(event.allocation); + assertNull(event.error); + assertNotNull(event.context); + + final FlagEvaluationPayloads.FlagEvaluationEvent degraded = + event.withoutTargetingKeyAndContext(); + assertNotNull(degraded); + assertNull(degraded.targeting_key); + assertNull(degraded.context); + } + private static FlagEvaluationPayloads.FlagEvaluationEvent event( final String flagKey, final String variant, diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java index 11e5b4f407f..2ec8629dbfc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -71,8 +71,17 @@ static String repeat(final char c, final int count) { } static TestWriterSetup buildTestWriter(final BackendApi mockEvp) { - return buildTestWriter( - mockEvp, FlagEvaluationWriterImpl.FLAG_EVALUATION_PAYLOAD_SIZE_LIMIT_BYTES); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + + final Map context = new HashMap<>(); + context.put("service", "test-service"); + + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context); + + return new TestWriterSetup(handler, mockEvp, factory); } static TestWriterSetup buildTestWriter( diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index bec4ed9eaee..6737e5553d6 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -6,6 +6,7 @@ import static com.datadog.featureflag.FlagEvaluationTestSupport.clearCoreMetrics; import static com.datadog.featureflag.FlagEvaluationTestSupport.event; import static com.datadog.featureflag.FlagEvaluationTestSupport.eventForFlag; +import static com.datadog.featureflag.FlagEvaluationTestSupport.flushAndCaptureJson; import static com.datadog.featureflag.FlagEvaluationTestSupport.metricSum; import static com.datadog.featureflag.FlagEvaluationTestSupport.repeat; import static com.datadog.featureflag.FlagEvaluationTestSupport.simpleEvent; @@ -13,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -23,8 +25,12 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.intake.Intake; import datadog.trace.api.telemetry.CoreMetricCollector; @@ -35,6 +41,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import okhttp3.RequestBody; import okio.Buffer; import org.junit.jupiter.api.AfterEach; @@ -51,6 +58,7 @@ void clearCoreMetricsBefore() { @AfterEach void clearCoreMetricsAfter() { clearCoreMetrics(); + FeatureFlaggingGateway.setFlagEvalWriter(null); } @Test @@ -71,6 +79,45 @@ void degradedCapOverflowTelemetryIsEmittedOnFlush() { "reason:" + FlagEvaluationWriterImpl.DROP_REASON_DEGRADED_CAP)); } + @Test + void publicConstructorAndContextHelpersDelegateToSharedImplementations() { + final datadog.trace.api.Config config = cfg(); + when(config.getEnv()).thenReturn("prod"); + when(config.getVersion()).thenReturn("1.2.3"); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(new SharedCommunicationObjects(true), config); + final Map attrs = new HashMap<>(); + attrs.put("b", "2"); + attrs.put("a", "1"); + + final Map pruned = FlagEvaluationWriterImpl.pruneContext(attrs); + + assertEquals(2, pruned.size()); + assertEquals( + FlagEvaluationAggregator.canonicalContextKey(pruned), + FlagEvaluationWriterImpl.canonicalContextKey(pruned)); + writer.close(); + } + + @Test + void startRegistersWriterAndCloseDeregistersIt() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.start(); + assertEquals(writer, FeatureFlaggingGateway.getFlagEvalWriter()); + + writer.close(); + writer.close(); + writer.start(); + + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } + @Test void queueOverflowIncrementsObservableDropCounter() { final BackendApi mockEvp = mock(BackendApi.class); @@ -120,6 +167,21 @@ void enqueueAfterCloseIsDroppedAndCounted() { assertNull(writer.pollQueuedEventForTest()); } + @Test + void enqueueIgnoresNullEvent() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + writer.enqueue(null); + + assertNull(writer.pollQueuedEventForTest()); + assertEquals(0, writer.droppedQueueOverflow()); + } + @Test void enqueueDoesNotAggregateOnTheCallingThread() { final BackendApi mockEvp = mock(BackendApi.class); @@ -198,6 +260,122 @@ void contextMaterializationFailureDropsSingleEvent() { assertEquals(0, setup.handler.fullTierSizeForTest()); } + @Test + void handlerRunFailsFastWhenEvpProxyIsUnavailable() { + final BackendApiFactory factory = mock(BackendApiFactory.class); + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context()); + + assertThrows(IllegalArgumentException.class, handler::run); + } + + @Test + void flushIfNecessarySkipsEmptyStateAndWaitsForInterval() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.flushIfNecessary(); + setup.handler.add(simpleEvent("pending-flag", "on")); + setup.handler.drainAndAggregate(); + setup.handler.flushIfNecessary(); + + assertEquals(1, setup.handler.fullTierSizeForTest()); + } + + @Test + void flushIfNecessaryDoesNotReturnEarlyWhenOnlyQueueDropsArePending() { + final AtomicLong queueDrops = new AtomicLong(1); + final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = + new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( + mock(BackendApiFactory.class), + Queues.mpscBlockingConsumerArrayQueue(16), + Long.MAX_VALUE, + TimeUnit.NANOSECONDS, + context(), + queueDrops, + () -> {}); + + handler.flushIfNecessary(); + + assertEquals(1, queueDrops.get()); + } + + @Test + @SuppressWarnings("unchecked") + void workerHandlesEmptyPolls() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final MessagePassingBlockingQueue queue = + mock(MessagePassingBlockingQueue.class); + when(queue.poll(100, TimeUnit.MILLISECONDS)) + .thenAnswer( + invocation -> { + Thread.currentThread().interrupt(); + return null; + }); + final FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler handler = + new FlagEvaluationWriterImpl.FlagEvaluationSerializingHandler( + factory, + queue, + Long.MAX_VALUE, + TimeUnit.NANOSECONDS, + context(), + new AtomicLong(0), + () -> {}); + + handler.run(); + + assertTrue(Thread.interrupted()); + } + + @Test + void degradedBucketsAreSerializedWithoutTargetingKeyOrContext() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + setup.handler.addDegradedBucketForTest("degraded-flag", "on", "alloc1", null, 1000L); + + final Map json = flushAndCaptureJson(setup); + + final Map ev = eventForFlag(json, "degraded-flag"); + assertNotNull(ev); + assertNull(ev.get("targeting_key")); + assertNull(ev.get("context")); + } + + @Test + void testHandlerCanSimulateAndClearDegradedTierAtCap() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.simulateDegradedTierAtCap(); + setup.handler.clearAggregationForTest(); + setup.handler.add(simpleEvent("after-clear", "on")); + setup.handler.drainAndAggregate(); + + assertEquals(1, setup.handler.fullTierSizeForTest()); + } + + @Test + void payloadLimitDropsAreCountedOnFlush() { + final BackendApi mockEvp = mock(BackendApi.class); + final FlagEvaluationTestSupport.TestWriterSetup setup = buildTestWriter(mockEvp, 128); + setup.handler.add(event(repeat('f', 512), "on", "alloc1", "user-1", 1000L, emptyMap())); + + setup.handler.drainAndAggregate(); + setup.handler.flush(); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 1, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_PAYLOAD_LIMIT)); + } + @Test void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); @@ -301,4 +479,10 @@ void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { setup.handler.flush(); assertEquals(2, posts.get()); } + + private static Map context() { + final Map context = new HashMap<>(); + context.put("service", "test-service"); + return context; + } } From 6e1f81e9be0770e7f549d016120fc6c665b41016 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 12:54:39 -0400 Subject: [PATCH 24/37] fix(feature-flagging): gate flag evaluation enqueue during shutdown --- .../java/datadog/trace/bootstrap/Agent.java | 19 ++++++ .../featureflag/FeatureFlaggingSystem.java | 9 +++ .../FeatureFlaggingSystemTest.java | 33 ++++++++- .../api/openfeature/FlagEvalLoggingHook.java | 9 ++- .../openfeature/FlagEvalLoggingHookTest.java | 33 ++++++++- .../trace/api/openfeature/ProviderTest.java | 1 + .../featureflag/FeatureFlaggingGateway.java | 16 +++++ .../FeatureFlaggingGatewayTest.java | 2 + .../featureflag/FlagEvaluationWriterImpl.java | 21 +++++- .../FlagEvaluationWriterImplTest.java | 68 +++++++++++++++++++ 10 files changed, 203 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index aae283da55c..68b30bf9f07 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -521,6 +521,9 @@ public static void shutdown(final boolean sync) { if (profilingEnabled) { shutdownProfilingAgent(sync); } + if (featureFlaggingEnabled) { + shutdownFeatureFlagging(); + } if (telemetryEnabled) { stopTelemetry(); } @@ -1211,6 +1214,22 @@ private static void shutdownLogsIntake() { } } + private static void shutdownFeatureFlagging() { + if (AGENT_CLASSLOADER == null) { + // It wasn't started, so no need to shut it down + return; + } + try { + Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER); + final Class ffSysClass = + AGENT_CLASSLOADER.loadClass("com.datadog.featureflag.FeatureFlaggingSystem"); + final Method shutdownMethod = ffSysClass.getMethod("shutdown"); + shutdownMethod.invoke(null); + } catch (final Throwable ex) { + log.error("Throwable thrown while shutting down Feature Flagging", ex); + } + } + private static void startTelemetry(Instrumentation inst, Class scoClass, Object sco) { StaticEventLogger.begin("Telemetry"); diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index c5be9b67f82..91c03afa7ca 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -3,6 +3,7 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,12 +35,14 @@ public static void start(final SharedCommunicationObjects sco) { config .configProvider() .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(evalCountsEnabled); if (evalCountsEnabled) { final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); evalWriter.start(); FLAG_EVAL_WRITER = evalWriter; LOGGER.debug("Flag evaluation EVP writer started"); } else { + FeatureFlaggingGateway.setFlagEvalWriter(null); LOGGER.debug( "Flag evaluation EVP writer disabled ({}=false)", FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED); @@ -49,6 +52,8 @@ public static void start(final SharedCommunicationObjects sco) { } public static void stop() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + FeatureFlaggingGateway.setFlagEvalWriter(null); if (FLAG_EVAL_WRITER != null) { FLAG_EVAL_WRITER.close(); FLAG_EVAL_WRITER = null; @@ -63,4 +68,8 @@ public static void stop() { } LOGGER.debug("Feature Flagging system stopped"); } + + public static void shutdown() { + stop(); + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 82fae6ecdb0..ad18710c0dd 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,9 +1,11 @@ package com.datadog.featureflag; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -19,13 +21,21 @@ import datadog.trace.api.Config; import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.junit.utils.config.WithConfig; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class FeatureFlaggingSystemTest { + @AfterEach + void resetFlagEvaluationGateway() { + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + @Test @WithConfig(key = FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { @@ -38,15 +48,18 @@ void testFeatureFlagSystemInitialization() { when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); FeatureFlaggingSystem.start(sharedCommunicationObjects); verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + assertTrue(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); assertNotNull(FeatureFlaggingGateway.getFlagEvalWriter()); - FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.shutdown(); + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); @@ -66,15 +79,29 @@ void testFlagEvaluationWriterCanBeDisabled() { when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + FeatureFlaggingGateway.setFlagEvalWriter(mock(FlagEvaluationWriter.class)); try { FeatureFlaggingSystem.start(sharedCommunicationObjects); + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); } finally { - FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.shutdown(); } } + @Test + void testFeatureFlagSystemShutdownClearsGatewayState() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + FeatureFlaggingGateway.setFlagEvalWriter(mock(FlagEvaluationWriter.class)); + + FeatureFlaggingSystem.shutdown(); + + assertFalse(FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()); + assertNull(FeatureFlaggingGateway.getFlagEvalWriter()); + } + @Test @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { @@ -85,7 +112,7 @@ void testThatRemoteConfigIsRequired() { IllegalStateException.class, () -> FeatureFlaggingSystem.start(sharedCommunicationObjects)); } finally { - FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.shutdown(); } } } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java index c394a154e01..5a3c852a207 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalLoggingHook.java @@ -66,8 +66,15 @@ public void finallyAfter( final FlagEvaluationDetails details, final Map hints) { try { + if (details == null) { + return; + } + if (!FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled()) { + return; + } + final FlagEvaluationWriter w = writerSupplier.get(); - if (w == null || details == null) { + if (w == null) { return; } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java index 9fa50b03fdc..b498faf42bc 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalLoggingHookTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import dev.openfeature.sdk.ErrorCode; @@ -30,6 +31,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** @@ -38,6 +41,16 @@ */ class FlagEvalLoggingHookTest { + @BeforeEach + void enableFlagEvaluationEnqueue() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + @AfterEach + void resetFlagEvaluationEnqueue() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + // ---- helpers ---- /** @@ -283,12 +296,30 @@ void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { hook.finallyAfter(null, det, Collections.emptyMap()); - // Exactly one enqueue call, no start/close/aggregate + // Exactly one enqueue call, no start/close. verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); verify(writer, never()).close(); verify(writer, never()).start(); } + @Test + void enqueueDisabledIsNoOpBeforeWriterLookup() { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + final AtomicReference writerResolved = new AtomicReference<>(false); + final FlagEvalLoggingHook hook = + new FlagEvalLoggingHook<>( + () -> { + writerResolved.set(true); + throw new AssertionError("writer should not be resolved when enqueue is disabled"); + }); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtxWithTargetingKey("flag", "user-1"), det, Collections.emptyMap()); + + assertFalse(writerResolved.get()); + } + // ---- test: writer=null -> no-op (killswitch off / not yet started) ---- @Test diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 3fd4417d6f1..ef37cdb330b 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -72,6 +72,7 @@ public void tearDown() { OpenFeatureAPI.getInstance().shutdown(); FeatureFlaggingGateway.dispatch((ServerConfiguration) null); FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index a0856fc0ab3..61237e12760 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -29,6 +29,8 @@ public interface ExposureListener extends Consumer {} private static final AtomicReference FLAG_EVAL_WRITER = new AtomicReference<>(); + private static volatile boolean flagEvalEnqueueEnabled = true; + private FeatureFlaggingGateway() {} public static void addConfigListener(final ConfigListener listener) { @@ -71,6 +73,15 @@ public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { FLAG_EVAL_WRITER.set(writer); } + /** + * Enables or disables enqueueing EVP flagevaluation events on the OpenFeature hook path. This is + * populated from {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} at feature-flagging startup and + * cleared during shutdown before the writer drains. + */ + public static void setFlagEvaluationEnqueueEnabled(final boolean enabled) { + flagEvalEnqueueEnabled = enabled; + } + /** * Returns the active EVP flagevaluation writer, or {@code null} when disabled (killswitch off or * not yet started). @@ -78,4 +89,9 @@ public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { public static FlagEvaluationWriter getFlagEvalWriter() { return FLAG_EVAL_WRITER.get(); } + + /** Returns whether EVP flagevaluation hook events may be enqueued. */ + public static boolean isFlagEvaluationEnqueueEnabled() { + return flagEvalEnqueueEnabled; + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java index fc04cf2193b..da4a6c18891 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/FeatureFlaggingGatewayTest.java @@ -33,6 +33,8 @@ void setUp() { void tearDown() { FeatureFlaggingGateway.removeConfigListener(configListener); FeatureFlaggingGateway.removeExposureListener(exposureListener); + FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 18b04d5c99a..685d85fec79 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -173,7 +173,8 @@ public void close() { if (!closed.compareAndSet(false, true)) { return; } - // Deregister from the gateway so no new events are enqueued. + // Disable and deregister from the gateway so no new events are enqueued. + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); FeatureFlaggingGateway.setFlagEvalWriter(null); if (!this.serializerThread.isAlive()) { return; @@ -198,9 +199,13 @@ public void enqueue(final FlagEvalEvent event) { if (event == null) { return; } + if (isClosedOrEnqueueDisabled()) { + countClosedDropIfClosed(); + return; + } synchronized (lifecycleLock) { - if (closed.get()) { - countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + if (isClosedOrEnqueueDisabled()) { + countClosedDropIfClosed(); return; } // Non-blocking offer. Count overflow so loss is observable rather than silent; the count is @@ -211,6 +216,16 @@ public void enqueue(final FlagEvalEvent event) { } } + private boolean isClosedOrEnqueueDisabled() { + return closed.get() || !FeatureFlaggingGateway.isFlagEvaluationEnqueueEnabled(); + } + + private void countClosedDropIfClosed() { + if (closed.get()) { + countMetric(FLAG_EVALUATION_DROPPED_METRIC, 1, DROP_REASON_CLOSED); + } + } + /** Returns the count of events dropped due to queue-overflow backpressure (observable). */ long droppedQueueOverflow() { return droppedQueueOverflow.get(); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index 6737e5553d6..280b3028803 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -36,6 +36,7 @@ import datadog.trace.api.telemetry.CoreMetricCollector; import datadog.trace.api.telemetry.MetricCollector; import java.io.IOException; +import java.lang.reflect.Field; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -53,12 +54,14 @@ class FlagEvaluationWriterImplTest { @BeforeEach void clearCoreMetricsBefore() { clearCoreMetrics(); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @AfterEach void clearCoreMetricsAfter() { clearCoreMetrics(); FeatureFlaggingGateway.setFlagEvalWriter(null); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); } @Test @@ -167,6 +170,56 @@ void enqueueAfterCloseIsDroppedAndCounted() { assertNull(writer.pollQueuedEventForTest()); } + @Test + void enqueueDisabledDropsWithoutQueueingOrCountingClosedDrop() { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + writer.enqueue(simpleEvent("disabled-flag", "on")); + + final Collection metrics = + CoreMetricCollector.getInstance().drain(); + assertEquals( + 0, + metricSum( + metrics, + FlagEvaluationWriterImpl.FLAG_EVALUATION_DROPPED_METRIC, + "reason:" + FlagEvaluationWriterImpl.DROP_REASON_CLOSED)); + assertNull(writer.pollQueuedEventForTest()); + } + + @Test + void enqueueRechecksEnabledStateAfterTakingLifecycleLock() throws Exception { + final BackendApi mockEvp = mock(BackendApi.class); + final BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + when(factory.createBackendApi(any(), any(), eq(false))).thenReturn(mockEvp); + final FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(16, Long.MAX_VALUE, TimeUnit.NANOSECONDS, factory, cfg()); + final Thread enqueuer = new Thread(() -> writer.enqueue(simpleEvent("race-flag", "on"))); + + final Object lifecycleLock = lifecycleLock(writer); + try { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + synchronized (lifecycleLock) { + enqueuer.start(); + awaitThreadState(enqueuer, Thread.State.BLOCKED); + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(false); + } + enqueuer.join(TimeUnit.SECONDS.toMillis(5)); + assertTrue(!enqueuer.isAlive()); + } finally { + FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(true); + } + + assertNull(writer.pollQueuedEventForTest()); + } + @Test void enqueueIgnoresNullEvent() { final BackendApi mockEvp = mock(BackendApi.class); @@ -480,6 +533,21 @@ void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { assertEquals(2, posts.get()); } + private static Object lifecycleLock(final FlagEvaluationWriterImpl writer) throws Exception { + final Field field = FlagEvaluationWriterImpl.class.getDeclaredField("lifecycleLock"); + field.setAccessible(true); + return field.get(writer); + } + + private static void awaitThreadState(final Thread thread, final Thread.State state) + throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != state && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertEquals(state, thread.getState()); + } + private static Map context() { final Map context = new HashMap<>(); context.put("service", "test-service"); From 78d829789db405dfdb5d70f1eea5cecaab8a940f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 14:27:24 -0400 Subject: [PATCH 25/37] test(feature-flagging): cover flag eval event --- .../flagevaluation/FlagEvalEventTest.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java new file mode 100644 index 00000000000..7faa19c72b3 --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/test/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEventTest.java @@ -0,0 +1,73 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class FlagEvalEventTest { + + @Test + void storesFieldsWithEagerContextAttributes() { + final Map attrs = Collections.singletonMap("tier", "gold"); + + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", "on", "allocation-1", "target-1", 123L, attrs); + + assertEquals("my-flag", event.flagKey); + assertEquals("on", event.variant); + assertEquals("allocation-1", event.allocationKey); + assertEquals("target-1", event.targetingKey); + assertNull(event.errorMessage); + assertEquals(123L, event.evalTimeMs); + assertSame(attrs, event.attrs); + assertSame(attrs, event.contextAttributes()); + } + + @Test + void storesErrorMessageAndDefaultsNullEagerContextAttributes() { + final Map attrs = null; + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", null, null, null, "type mismatch", 456L, attrs); + + assertEquals("type mismatch", event.errorMessage); + assertTrue(event.attrs.isEmpty()); + assertTrue(event.contextAttributes().isEmpty()); + } + + @Test + void resolvesLazyContextAttributesOnDemand() { + final AtomicInteger resolutions = new AtomicInteger(); + final Map attrs = Collections.singletonMap("region", "us-east-1"); + final FlagEvalEvent event = + new FlagEvalEvent( + "my-flag", + "on", + "allocation-1", + "target-1", + null, + 789L, + () -> { + resolutions.incrementAndGet(); + return attrs; + }); + + assertTrue(event.attrs.isEmpty()); + assertEquals(0, resolutions.get()); + assertSame(attrs, event.contextAttributes()); + assertEquals(1, resolutions.get()); + } + + @Test + void defaultsNullLazyContextAttributes() { + final FlagEvalEvent event = + new FlagEvalEvent("my-flag", "on", null, null, null, 789L, () -> null); + + assertTrue(event.contextAttributes().isEmpty()); + } +} From 6acbb5efe1dea72033960343a41eb5717e8616ac Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 2 Jul 2026 22:31:33 -0400 Subject: [PATCH 26/37] fix(feature-flagging): make aggregator count atomic --- .../featureflag/FlagEvaluationAggregator.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java index eb15eb884a5..a146501f825 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationAggregator.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; final class FlagEvaluationAggregator { @@ -39,7 +40,7 @@ final class FlagEvaluationAggregator { final Map degradedTier = new HashMap<>(); final Map perFlagCount = new HashMap<>(); final AtomicLong droppedDegradedOverflow = new AtomicLong(0); - int globalFullCount = 0; + final AtomicInteger globalFullCount = new AtomicInteger(0); void aggregate(final FlagEvalEvent event) { final boolean isDefault = event.variant == null; @@ -54,7 +55,7 @@ void aggregate(final FlagEvalEvent event) { } final int flagCount = perFlagCount.getOrDefault(event.flagKey, 0); - if (globalFullCount < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { + if (globalFullCount.get() < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { fullTier.put( fullKey, new EvalBucket( @@ -66,7 +67,7 @@ void aggregate(final FlagEvalEvent event) { event.evalTimeMs, isDefault, prunedAttrs)); - globalFullCount++; + globalFullCount.incrementAndGet(); perFlagCount.put(event.flagKey, flagCount + 1); return; } @@ -128,7 +129,7 @@ void clear() { fullTier.clear(); degradedTier.clear(); perFlagCount.clear(); - globalFullCount = 0; + globalFullCount.set(0); } AggregatedState snapshot() { @@ -137,12 +138,12 @@ AggregatedState snapshot() { } void simulateFullTierAtCap() { - for (int i = globalFullCount; i < GLOBAL_CAP; i++) { + for (int i = globalFullCount.get(); i < GLOBAL_CAP; i++) { final String key = "synthetic-full-" + i; fullTier.put( new FullKey(key, "on", "alloc", false, null, null, ""), new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); - globalFullCount++; + globalFullCount.incrementAndGet(); perFlagCount.merge(key, 1, Integer::sum); } } From 370109dd45c3dc9c7983dbdcea55941394290e67 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 8 Jul 2026 13:31:09 -0400 Subject: [PATCH 27/37] Add agentless feature flag configuration source --- .../java/datadog/trace/bootstrap/Agent.java | 7 +- .../datadog/trace/api/ConfigDefaults.java | 4 + .../api/config/FeatureFlaggingConfig.java | 16 + .../main/java/datadog/trace/api/Config.java | 69 ++++ metadata/supported-configurations.json | 48 +++ .../featureflag/FeatureFlaggingSystem.java | 47 ++- .../FeatureFlaggingSystemTest.java | 79 +++++ .../datadog/featureflag/CdnConfigService.java | 284 ++++++++++++++++ .../featureflag/CdnConfigServiceTest.java | 303 ++++++++++++++++++ 9 files changed, 851 insertions(+), 6 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 876a07f1fd1..ea9793ce86d 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -135,7 +135,8 @@ private enum AgentFeature { APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); + FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_ENABLED, false), + DEPRECATED_FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); private final String configKey; private final String systemProp; @@ -282,7 +283,9 @@ public static void start( agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION); llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); - featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGGING); + featureFlaggingEnabled = + isFeatureEnabled(AgentFeature.FEATURE_FLAGGING) + || isFeatureEnabled(AgentFeature.DEPRECATED_FEATURE_FLAGGING); // setup writers when llmobs is enabled to accomodate apm and llmobs if (llmObsEnabled) { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 1aba2cb276b..42293cdea28 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -49,6 +49,10 @@ public final class ConfigDefaults { static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; + public static final String DEFAULT_FLAGGING_CONFIGURATION_SOURCE = "cdn"; + public static final double DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30.0D; + public static final double DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2.0D; + static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; static final boolean DEFAULT_TRACE_ENABLED = true; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..25faf9da12f 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -2,5 +2,21 @@ public class FeatureFlaggingConfig { + public static final String FLAGGING_ENABLED = "flagging.enabled"; + + /** Replaced by {@link #FLAGGING_ENABLED}. */ + @Deprecated public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + public static final String FLAGGING_CONFIGURATION_SOURCE = "flagging.configuration.source"; + public static final String FLAGGING_CONFIGURATION_SOURCE_BASE_URL = + "flagging.configuration.source.base.url"; + public static final String FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = + "flagging.configuration.source.poll.interval.seconds"; + public static final String FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = + "flagging.configuration.source.request.timeout.seconds"; + public static final String FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS = + "flagging.configuration.source.extra.headers"; + + private FeatureFlaggingConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 5ced61622ca..5a64fcf8aa6 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -86,6 +86,9 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_BODY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_PARAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_EXPERIMENTATAL_JEE_SPLIT_BY_DEPLOYMENT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_CLIENT_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_SERVER_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_HEALTH_METRICS_ENABLED; @@ -361,6 +364,11 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_EXCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_BASE_URL; +import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_QUEUE_SIZE; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL; @@ -1213,6 +1221,12 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; + private final String flaggingConfigurationSource; + private final String flaggingConfigurationSourceBaseUrl; + private final double flaggingConfigurationSourcePollIntervalSeconds; + private final double flaggingConfigurationSourceRequestTimeoutSeconds; + private final Map flaggingConfigurationSourceExtraHeaders; + private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; private final boolean dbmTracePreparedStatements; @@ -2835,6 +2849,23 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getInteger( REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES); + flaggingConfigurationSource = + normalizeFlaggingConfigurationSource( + configProvider.getString( + FLAGGING_CONFIGURATION_SOURCE, DEFAULT_FLAGGING_CONFIGURATION_SOURCE)); + flaggingConfigurationSourceBaseUrl = + configProvider.getStringNotEmpty(FLAGGING_CONFIGURATION_SOURCE_BASE_URL, null); + flaggingConfigurationSourcePollIntervalSeconds = + configProvider.getDouble( + FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS, + DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); + flaggingConfigurationSourceRequestTimeoutSeconds = + configProvider.getDouble( + FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS, + DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); + flaggingConfigurationSourceExtraHeaders = + configProvider.getMergedMap(FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS, '='); + dynamicInstrumentationEnabled = configProvider.getBoolean( DYNAMIC_INSTRUMENTATION_ENABLED, DEFAULT_DYNAMIC_INSTRUMENTATION_ENABLED); @@ -3747,6 +3778,14 @@ public boolean isInferredProxyPropagationEnabled() { return traceInferredProxyEnabled; } + private static String normalizeFlaggingConfigurationSource(final String source) { + if (source == null) { + return DEFAULT_FLAGGING_CONFIGURATION_SOURCE; + } + final String normalized = source.trim().toLowerCase(Locale.ROOT); + return normalized.isEmpty() ? DEFAULT_FLAGGING_CONFIGURATION_SOURCE : normalized; + } + public boolean isBaggageExtract() { return tracePropagationStylesToExtract.contains(TracePropagationStyle.BAGGAGE); } @@ -4654,6 +4693,26 @@ public int getRemoteConfigMaxExtraServices() { return remoteConfigMaxExtraServices; } + public String getFlaggingConfigurationSource() { + return flaggingConfigurationSource; + } + + public String getFlaggingConfigurationSourceBaseUrl() { + return flaggingConfigurationSourceBaseUrl; + } + + public double getFlaggingConfigurationSourcePollIntervalSeconds() { + return flaggingConfigurationSourcePollIntervalSeconds; + } + + public double getFlaggingConfigurationSourceRequestTimeoutSeconds() { + return flaggingConfigurationSourceRequestTimeoutSeconds; + } + + public Map getFlaggingConfigurationSourceExtraHeaders() { + return flaggingConfigurationSourceExtraHeaders; + } + public boolean isDynamicInstrumentationEnabled() { return dynamicInstrumentationEnabled; } @@ -6486,6 +6545,16 @@ public String toString() { + remoteConfigMaxPayloadSize + ", remoteConfigIntegrityCheckEnabled=" + remoteConfigIntegrityCheckEnabled + + ", flaggingConfigurationSource=" + + flaggingConfigurationSource + + ", flaggingConfigurationSourceBaseUrl=" + + flaggingConfigurationSourceBaseUrl + + ", flaggingConfigurationSourcePollIntervalSeconds=" + + flaggingConfigurationSourcePollIntervalSeconds + + ", flaggingConfigurationSourceRequestTimeoutSeconds=" + + flaggingConfigurationSourceRequestTimeoutSeconds + + ", flaggingConfigurationSourceExtraHeaderNames=" + + flaggingConfigurationSourceExtraHeaders.keySet() + ", debuggerEnabled=" + dynamicInstrumentationEnabled + ", debuggerUploadTimeout=" diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..4c99c583841 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1513,6 +1513,54 @@ "aliases": [] } ], + "DD_FLAGGING_CONFIGURATION_SOURCE": [ + { + "version": "A", + "type": "string", + "default": "cdn", + "aliases": [] + } + ], + "DD_FLAGGING_CONFIGURATION_SOURCE_BASE_URL": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], + "DD_FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS": [ + { + "version": "A", + "type": "map", + "default": null, + "aliases": [] + } + ], + "DD_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS": [ + { + "version": "A", + "type": "decimal", + "default": "30", + "aliases": [] + } + ], + "DD_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS": [ + { + "version": "A", + "type": "decimal", + "default": "2", + "aliases": [] + } + ], + "DD_FLAGGING_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": ["DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"] + } + ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..8d2ab24fb8f 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -8,21 +8,44 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); + private static final String SOURCE_CDN = "cdn"; + private static final String SOURCE_REMOTE_CONFIG = "remote_config"; + private static final String SOURCE_OFFLINE = "offline"; private static volatile RemoteConfigService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; + private static final ConfigServiceFactory DEFAULT_CDN_CONFIG_SERVICE_FACTORY = + (sco, config) -> new CdnConfigService(config); + private static final ConfigServiceFactory DEFAULT_REMOTE_CONFIG_SERVICE_FACTORY = + RemoteConfigServiceImpl::new; + private static volatile ConfigServiceFactory CDN_CONFIG_SERVICE_FACTORY = + DEFAULT_CDN_CONFIG_SERVICE_FACTORY; + private static volatile ConfigServiceFactory REMOTE_CONFIG_SERVICE_FACTORY = + DEFAULT_REMOTE_CONFIG_SERVICE_FACTORY; private FeatureFlaggingSystem() {} public static void start(final SharedCommunicationObjects sco) { LOGGER.debug("Feature Flagging system starting"); final Config config = Config.get(); + final String configurationSource = config.getFlaggingConfigurationSource(); - if (!config.isRemoteConfigEnabled()) { - throw new IllegalStateException("Feature Flagging system started without RC"); + if (SOURCE_REMOTE_CONFIG.equals(configurationSource)) { + if (!config.isRemoteConfigEnabled()) { + throw new IllegalStateException("Feature Flagging system started without RC"); + } + CONFIG_SERVICE = REMOTE_CONFIG_SERVICE_FACTORY.create(sco, config); + CONFIG_SERVICE.init(); + } else if (SOURCE_CDN.equals(configurationSource)) { + CONFIG_SERVICE = CDN_CONFIG_SERVICE_FACTORY.create(sco, config); + CONFIG_SERVICE.init(); + } else if (SOURCE_OFFLINE.equals(configurationSource)) { + LOGGER.debug( + "Feature Flagging offline configuration source selected; no config service started"); + } else { + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + configurationSource); } - CONFIG_SERVICE = new RemoteConfigServiceImpl(sco, config); - CONFIG_SERVICE.init(); EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); @@ -41,4 +64,20 @@ public static void stop() { } LOGGER.debug("Feature Flagging system stopped"); } + + public static void setServiceFactoriesForTest( + final ConfigServiceFactory cdnConfigServiceFactory, + final ConfigServiceFactory remoteConfigServiceFactory) { + CDN_CONFIG_SERVICE_FACTORY = cdnConfigServiceFactory; + REMOTE_CONFIG_SERVICE_FACTORY = remoteConfigServiceFactory; + } + + public static void resetServiceFactoriesForTest() { + CDN_CONFIG_SERVICE_FACTORY = DEFAULT_CDN_CONFIG_SERVICE_FACTORY; + REMOTE_CONFIG_SERVICE_FACTORY = DEFAULT_REMOTE_CONFIG_SERVICE_FACTORY; + } + + public interface ConfigServiceFactory { + RemoteConfigService create(SharedCommunicationObjects sco, Config config); + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 970c2cf16e3..c98617fd8f1 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,11 +1,13 @@ package com.datadog.featureflag; +import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; @@ -23,6 +25,8 @@ class FeatureFlaggingSystemTest { @Test + @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); @@ -48,6 +52,7 @@ void testFeatureFlagSystemInitialization() { } @Test + @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); @@ -60,4 +65,78 @@ void testThatRemoteConfigIsRequired() { FeatureFlaggingSystem.stop(); } } + + @Test + @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "cdn") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") + void cdnConfigurationSourceStartsCdnWithoutRemoteConfig() { + RemoteConfigService cdnService = mock(RemoteConfigService.class); + RemoteConfigService remoteConfigService = mock(RemoteConfigService.class); + FeatureFlaggingSystem.setServiceFactoriesForTest( + (ignoredSco, ignoredConfig) -> cdnService, + (ignoredSco, ignoredConfig) -> remoteConfigService); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects()); + + verify(cdnService).init(); + verifyNoInteractions(remoteConfigService); + } finally { + FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.resetServiceFactoriesForTest(); + } + } + + @Test + @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") + void explicitRemoteConfigSuppressesCdnService() { + RemoteConfigService cdnService = mock(RemoteConfigService.class); + RemoteConfigService remoteConfigService = mock(RemoteConfigService.class); + FeatureFlaggingSystem.setServiceFactoriesForTest( + (ignoredSco, ignoredConfig) -> cdnService, + (ignoredSco, ignoredConfig) -> remoteConfigService); + + try { + FeatureFlaggingSystem.start(sharedCommunicationObjects()); + + verify(remoteConfigService).init(); + verifyNoInteractions(cdnService); + } finally { + FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.resetServiceFactoriesForTest(); + } + } + + @Test + @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "invalid") + void invalidConfigurationSourceFailsBeforeStartingCdn() { + RemoteConfigService cdnService = mock(RemoteConfigService.class); + RemoteConfigService remoteConfigService = mock(RemoteConfigService.class); + FeatureFlaggingSystem.setServiceFactoriesForTest( + (ignoredSco, ignoredConfig) -> cdnService, + (ignoredSco, ignoredConfig) -> remoteConfigService); + + try { + assertThrows( + IllegalArgumentException.class, + () -> FeatureFlaggingSystem.start(sharedCommunicationObjects())); + verifyNoInteractions(cdnService); + verifyNoInteractions(remoteConfigService); + } finally { + FeatureFlaggingSystem.stop(); + FeatureFlaggingSystem.resetServiceFactoriesForTest(); + } + } + + private static SharedCommunicationObjects sharedCommunicationObjects() { + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.supportsEvpProxy()).thenReturn(true); + when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); + sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); + sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + return sharedCommunicationObjects; + } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java new file mode 100644 index 00000000000..3857cfd8cd4 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java @@ -0,0 +1,284 @@ +package com.datadog.featureflag; + +import static datadog.communication.http.OkHttpUtils.prepareRequest; + +import datadog.communication.http.OkHttpUtils; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URLEncoder; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class CdnConfigService implements RemoteConfigService { + private static final Logger LOGGER = LoggerFactory.getLogger(CdnConfigService.class); + + private static final String SERVER_DISTRIBUTION_PATH = + "/api/v2/feature-flagging/config/server-distribution"; + private static final int MAX_ATTEMPTS = 3; + + private final HttpUrl endpoint; + private final Config config; + private final Map extraHeaders; + private final long pollIntervalMillis; + private final CdnClient client; + private final ScheduledExecutorService executor; + private final AtomicBoolean polling = new AtomicBoolean(); + private volatile boolean closed; + private volatile ScheduledFuture scheduledPoll; + private volatile String etag; + + CdnConfigService(final Config config) { + this(config, endpoint(config)); + } + + private CdnConfigService(final Config config, final HttpUrl endpoint) { + this( + endpoint, + config, + millis(config.getFlaggingConfigurationSourcePollIntervalSeconds()), + new OkHttpCdnClient( + OkHttpUtils.buildHttpClient( + endpoint, millis(config.getFlaggingConfigurationSourceRequestTimeoutSeconds()))), + Executors.newSingleThreadScheduledExecutor(new CdnThreadFactory())); + } + + CdnConfigService( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final CdnClient client, + final ScheduledExecutorService executor) { + this.endpoint = endpoint; + this.config = config; + final Map configuredExtraHeaders = + config.getFlaggingConfigurationSourceExtraHeaders(); + this.extraHeaders = + configuredExtraHeaders == null ? Collections.emptyMap() : configuredExtraHeaders; + this.pollIntervalMillis = pollIntervalMillis; + this.client = client; + this.executor = executor; + } + + @Override + public void init() { + if (closed) { + return; + } + scheduledPoll = + executor.scheduleWithFixedDelay( + this::pollOnceSafely, 0, pollIntervalMillis, TimeUnit.MILLISECONDS); + } + + boolean pollOnce() { + if (closed || !polling.compareAndSet(false, true)) { + return false; + } + try { + return fetchAndApply(); + } finally { + polling.set(false); + } + } + + @Override + public void close() { + closed = true; + if (scheduledPoll != null) { + scheduledPoll.cancel(true); + scheduledPoll = null; + } + executor.shutdownNow(); + } + + private void pollOnceSafely() { + try { + pollOnce(); + } catch (final RuntimeException e) { + LOGGER.debug("Unexpected error while polling Feature Flagging CDN", e); + } + } + + private boolean fetchAndApply() { + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + final CdnResponse response = client.fetch(endpoint, config, extraHeaders, etag); + if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { + continue; + } + return apply(response); + } catch (final IOException e) { + if (attempt == MAX_ATTEMPTS) { + LOGGER.debug("Feature Flagging CDN request failed", e); + return false; + } + } + } + return false; + } + + private boolean apply(final CdnResponse response) { + if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) { + updateEtag(response.etag); + return true; + } + if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED + || response.status == HttpURLConnection.HTTP_FORBIDDEN + || response.status != HttpURLConnection.HTTP_OK + || response.body == null) { + return false; + } + final ServerConfiguration configuration; + try { + configuration = + RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( + response.body); + } catch (final IOException | RuntimeException e) { + LOGGER.debug("Feature Flagging CDN returned malformed UFC payload", e); + return false; + } + if (configuration == null) { + return false; + } + updateEtag(response.etag); + FeatureFlaggingGateway.dispatch(configuration); + return true; + } + + private static boolean isRetryableStatus(final int status) { + return status == HttpURLConnection.HTTP_CLIENT_TIMEOUT + || status == 429 + || (status >= 500 && status <= 599); + } + + private void updateEtag(final String nextEtag) { + if (nextEtag != null && !nextEtag.trim().isEmpty()) { + etag = nextEtag; + } + } + + static HttpUrl endpoint(final Config config) { + final String configuredBaseUrl = config.getFlaggingConfigurationSourceBaseUrl(); + final String endpoint = + configuredBaseUrl == null + ? defaultEndpoint(config) + : endpointFromConfiguredUrl(configuredBaseUrl); + final HttpUrl parsed = HttpUrl.parse(endpoint); + if (parsed == null) { + throw new IllegalArgumentException("Invalid Feature Flagging CDN URL: " + endpoint); + } + return parsed; + } + + private static String endpointFromConfiguredUrl(final String configuredUrl) { + final HttpUrl parsed = HttpUrl.parse(configuredUrl.trim()); + if (parsed == null) { + throw new IllegalArgumentException("Invalid Feature Flagging CDN URL: " + configuredUrl); + } + if (isRootPath(parsed)) { + return parsed.newBuilder().addPathSegments("mock/ufc/config").build().toString(); + } + return parsed.toString(); + } + + private static boolean isRootPath(final HttpUrl url) { + return "/".equals(url.encodedPath()) || url.encodedPath().isEmpty(); + } + + private static String defaultEndpoint(final Config config) { + final StringBuilder endpoint = + new StringBuilder("https://api.") + .append(config.getSite().toLowerCase(Locale.ROOT)) + .append(SERVER_DISTRIBUTION_PATH); + final String env = config.getEnv(); + if (env != null && !env.isEmpty()) { + endpoint.append("?dd_env=").append(urlEncode(env)); + } + return endpoint.toString(); + } + + private static String urlEncode(final String value) { + try { + return URLEncoder.encode(value, "UTF-8"); + } catch (final IOException e) { + throw new IllegalArgumentException("Unable to encode Feature Flagging environment", e); + } + } + + static long millis(final double seconds) { + return Math.max(1L, Math.round(seconds * 1000)); + } + + interface CdnClient { + CdnResponse fetch( + HttpUrl endpoint, Config config, Map extraHeaders, String etag) + throws IOException; + } + + static final class CdnResponse { + final int status; + final String etag; + final byte[] body; + + CdnResponse(final int status, final String etag, final byte[] body) { + this.status = status; + this.etag = etag; + this.body = body; + } + } + + private static final class OkHttpCdnClient implements CdnClient { + private final OkHttpClient httpClient; + + private OkHttpCdnClient(final OkHttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public CdnResponse fetch( + final HttpUrl endpoint, + final Config config, + final Map extraHeaders, + final String etag) + throws IOException { + final Map headers = new HashMap<>(extraHeaders); + if (etag != null) { + headers.put("If-None-Match", etag); + } + final Request request = prepareRequest(endpoint, headers, config, true).get().build(); + try (Response response = httpClient.newCall(request).execute()) { + final ResponseBody responseBody = response.body(); + return new CdnResponse( + response.code(), + response.header("ETag"), + responseBody == null ? null : responseBody.bytes()); + } + } + } + + private static final class CdnThreadFactory implements ThreadFactory { + @Override + public Thread newThread(final Runnable runnable) { + final Thread thread = new Thread(runnable, "dd-feature-flagging-cdn-poller"); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java new file mode 100644 index 00000000000..0bcdf1567c9 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java @@ -0,0 +1,303 @@ +package com.datadog.featureflag; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class CdnConfigServiceTest { + + @Mock private FeatureFlaggingGateway.ConfigListener listener; + + @AfterEach + void cleanup() { + FeatureFlaggingGateway.removeConfigListener(listener); + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + + @Test + void appendsMockConfigPathWhenConfiguredUrlIsRoot() { + final Config config = config("http://mock-cdn:8092", "datadoghq.com", ""); + + assertEquals( + "http://mock-cdn:8092/mock/ufc/config", CdnConfigService.endpoint(config).toString()); + } + + @Test + void preservesConfiguredFullEndpointUrl() { + final Config config = + config("http://mock-cdn:8092/mock/ufc/config?fixture=valid", "datadoghq.com", ""); + + assertEquals( + "http://mock-cdn:8092/mock/ufc/config?fixture=valid", + CdnConfigService.endpoint(config).toString()); + } + + @Test + void derivesDefaultEndpointFromSiteAndEnv() { + final Config config = config(null, "datad0g.com", "staging env"); + + assertEquals( + "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", + CdnConfigService.endpoint(config).toString()); + } + + @Test + void appliesAcceptedUfcThroughGatewayAndSendsHeaders() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final CdnConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("test-api-key", client.requests.get(0).apiKey); + assertEquals("app-key", client.requests.get(0).extraHeaders.get("DD-APPLICATION-KEY")); + assertEquals("1", client.requests.get(0).extraHeaders.get("fastly-client")); + assertNull(client.requests.get(0).etag); + } + + @Test + void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { + final FakeClient client = + new FakeClient(response(200, "etag-a", emptyConfig()), response(304, null, null)); + final CdnConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("etag-a", client.requests.get(1).etag); + } + + @Test + void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { + final FakeClient client = + new FakeClient( + response(401, null, null), + response(200, null, "{not-json}"), + response(200, null, "{\"flags\":[]}")); + final CdnConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verifyNoInteractions(listener); + } + + @Test + void retriesTimeoutBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow CDN"), + new SocketTimeoutException("slow CDN"), + response(200, "etag-a", emptyConfig())); + final CdnConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { + final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); + final CdnConfigService service = service(client, extraHeaders()); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(2, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void rejectsOverlappingPolls() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + client.block(requestStarted, releaseRequest); + final CdnConfigService service = service(client, extraHeaders()); + final ExecutorService runner = Executors.newFixedThreadPool(2); + + try { + final Future first = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + final Future second = runner.submit(service::pollOnce); + + assertFalse(second.get(1, TimeUnit.SECONDS)); + releaseRequest.countDown(); + assertTrue(first.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void closePreventsFurtherPolls() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final CdnConfigService service = service(client, extraHeaders()); + + service.close(); + + assertFalse(service.pollOnce()); + assertEquals(0, client.calls.get()); + } + + private static CdnConfigService service( + final FakeClient client, final Map extraHeaders) { + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + return new CdnConfigService( + HttpUrl.get("http://localhost/mock/ufc/config"), + config("http://localhost", "datadoghq.com", "", extraHeaders), + 60_000, + client, + executor); + } + + private static Config config(final String baseUrl, final String site, final String env) { + return config(baseUrl, site, env, new HashMap<>()); + } + + private static Config config( + final String baseUrl, + final String site, + final String env, + final Map extraHeaders) { + final Config config = mock(Config.class); + lenient().when(config.getFlaggingConfigurationSourceBaseUrl()).thenReturn(baseUrl); + lenient().when(config.getFlaggingConfigurationSourceExtraHeaders()).thenReturn(extraHeaders); + lenient().when(config.getFlaggingConfigurationSourcePollIntervalSeconds()).thenReturn(30.0D); + lenient().when(config.getFlaggingConfigurationSourceRequestTimeoutSeconds()).thenReturn(2.0D); + lenient().when(config.getApiKey()).thenReturn("test-api-key"); + lenient().when(config.getSite()).thenReturn(site); + lenient().when(config.getEnv()).thenReturn(env); + return config; + } + + private static Map extraHeaders() { + final Map headers = new HashMap<>(); + headers.put("DD-APPLICATION-KEY", "app-key"); + headers.put("fastly-client", "1"); + return headers; + } + + private static CdnConfigService.CdnResponse response( + final int status, final String etag, final String body) { + return new CdnConfigService.CdnResponse( + status, etag, body == null ? null : body.getBytes(UTF_8)); + } + + private static String emptyConfig() { + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } + + private static final class FakeClient implements CdnConfigService.CdnClient { + private final AtomicInteger calls = new AtomicInteger(); + private final List requests = new ArrayList<>(); + private final BlockingQueue responses = new LinkedBlockingQueue<>(); + private CountDownLatch requestStarted; + private CountDownLatch releaseRequest; + + private FakeClient(final Object... responses) { + for (final Object response : responses) { + this.responses.add(response); + } + } + + private void block(final CountDownLatch requestStarted, final CountDownLatch releaseRequest) { + this.requestStarted = requestStarted; + this.releaseRequest = releaseRequest; + } + + @Override + public CdnConfigService.CdnResponse fetch( + final HttpUrl endpoint, + final Config config, + final Map extraHeaders, + final String etag) + throws IOException { + calls.incrementAndGet(); + requests.add(new Request(config.getApiKey(), etag, extraHeaders)); + if (requestStarted != null) { + requestStarted.countDown(); + } + if (releaseRequest != null) { + await(releaseRequest); + } + final Object response = responses.remove(); + if (response instanceof IOException) { + throw (IOException) response; + } + return (CdnConfigService.CdnResponse) response; + } + + private static void await(final CountDownLatch latch) throws IOException { + try { + if (!latch.await(1, TimeUnit.SECONDS)) { + throw new SocketTimeoutException("test request did not release"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + } + + private static final class Request { + private final String apiKey; + private final String etag; + private final Map extraHeaders; + + private Request( + final String apiKey, final String etag, final Map extraHeaders) { + this.apiKey = apiKey; + this.etag = etag; + this.extraHeaders = new HashMap<>(extraHeaders); + } + } +} From 1c8eef89066f8d1fe75f9a0967f37eab1352b459 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 8 Jul 2026 14:26:12 -0400 Subject: [PATCH 28/37] Clean up feature flagging configuration source naming --- .../featureflag/FeatureFlaggingSystem.java | 52 +++++-------- .../FeatureFlaggingSystemTest.java | 78 +++++++------------ ...e.java => ConfigurationSourceService.java} | 2 +- .../featureflag/RemoteConfigServiceImpl.java | 2 +- ...Service.java => UfcHttpConfigService.java} | 54 ++++++------- ...est.java => UfcHttpConfigServiceTest.java} | 40 +++++----- 6 files changed, 97 insertions(+), 131 deletions(-) rename products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/{RemoteConfigService.java => ConfigurationSourceService.java} (60%) rename products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/{CdnConfigService.java => UfcHttpConfigService.java} (82%) rename products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/{CdnConfigServiceTest.java => UfcHttpConfigServiceTest.java} (88%) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 8d2ab24fb8f..6128d43cd34 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -12,45 +12,45 @@ public class FeatureFlaggingSystem { private static final String SOURCE_REMOTE_CONFIG = "remote_config"; private static final String SOURCE_OFFLINE = "offline"; - private static volatile RemoteConfigService CONFIG_SERVICE; + private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; - private static final ConfigServiceFactory DEFAULT_CDN_CONFIG_SERVICE_FACTORY = - (sco, config) -> new CdnConfigService(config); - private static final ConfigServiceFactory DEFAULT_REMOTE_CONFIG_SERVICE_FACTORY = - RemoteConfigServiceImpl::new; - private static volatile ConfigServiceFactory CDN_CONFIG_SERVICE_FACTORY = - DEFAULT_CDN_CONFIG_SERVICE_FACTORY; - private static volatile ConfigServiceFactory REMOTE_CONFIG_SERVICE_FACTORY = - DEFAULT_REMOTE_CONFIG_SERVICE_FACTORY; private FeatureFlaggingSystem() {} public static void start(final SharedCommunicationObjects sco) { LOGGER.debug("Feature Flagging system starting"); final Config config = Config.get(); + CONFIG_SERVICE = createConfigurationSourceService(sco, config); + + if (CONFIG_SERVICE != null) { + CONFIG_SERVICE.init(); + } + + EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); + EXPOSURE_WRITER.init(); + + LOGGER.debug("Feature Flagging system started"); + } + + static ConfigurationSourceService createConfigurationSourceService( + final SharedCommunicationObjects sco, final Config config) { final String configurationSource = config.getFlaggingConfigurationSource(); if (SOURCE_REMOTE_CONFIG.equals(configurationSource)) { if (!config.isRemoteConfigEnabled()) { throw new IllegalStateException("Feature Flagging system started without RC"); } - CONFIG_SERVICE = REMOTE_CONFIG_SERVICE_FACTORY.create(sco, config); - CONFIG_SERVICE.init(); + return new RemoteConfigServiceImpl(sco, config); } else if (SOURCE_CDN.equals(configurationSource)) { - CONFIG_SERVICE = CDN_CONFIG_SERVICE_FACTORY.create(sco, config); - CONFIG_SERVICE.init(); + return new UfcHttpConfigService(config); } else if (SOURCE_OFFLINE.equals(configurationSource)) { LOGGER.debug( "Feature Flagging offline configuration source selected; no config service started"); + return null; } else { throw new IllegalArgumentException( "Unsupported Feature Flagging configuration source: " + configurationSource); } - - EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); - EXPOSURE_WRITER.init(); - - LOGGER.debug("Feature Flagging system started"); } public static void stop() { @@ -64,20 +64,4 @@ public static void stop() { } LOGGER.debug("Feature Flagging system stopped"); } - - public static void setServiceFactoriesForTest( - final ConfigServiceFactory cdnConfigServiceFactory, - final ConfigServiceFactory remoteConfigServiceFactory) { - CDN_CONFIG_SERVICE_FACTORY = cdnConfigServiceFactory; - REMOTE_CONFIG_SERVICE_FACTORY = remoteConfigServiceFactory; - } - - public static void resetServiceFactoriesForTest() { - CDN_CONFIG_SERVICE_FACTORY = DEFAULT_CDN_CONFIG_SERVICE_FACTORY; - REMOTE_CONFIG_SERVICE_FACTORY = DEFAULT_REMOTE_CONFIG_SERVICE_FACTORY; - } - - public interface ConfigServiceFactory { - RemoteConfigService create(SharedCommunicationObjects sco, Config config); - } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index c98617fd8f1..c70ec92bafd 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -2,12 +2,13 @@ import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; @@ -69,64 +70,43 @@ void testThatRemoteConfigIsRequired() { @Test @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "cdn") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") - void cdnConfigurationSourceStartsCdnWithoutRemoteConfig() { - RemoteConfigService cdnService = mock(RemoteConfigService.class); - RemoteConfigService remoteConfigService = mock(RemoteConfigService.class); - FeatureFlaggingSystem.setServiceFactoriesForTest( - (ignoredSco, ignoredConfig) -> cdnService, - (ignoredSco, ignoredConfig) -> remoteConfigService); - - try { - FeatureFlaggingSystem.start(sharedCommunicationObjects()); - - verify(cdnService).init(); - verifyNoInteractions(remoteConfigService); - } finally { - FeatureFlaggingSystem.stop(); - FeatureFlaggingSystem.resetServiceFactoriesForTest(); - } + void cdnConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { + assertInstanceOf( + UfcHttpConfigService.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); } @Test @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") - void explicitRemoteConfigSuppressesCdnService() { - RemoteConfigService cdnService = mock(RemoteConfigService.class); - RemoteConfigService remoteConfigService = mock(RemoteConfigService.class); - FeatureFlaggingSystem.setServiceFactoriesForTest( - (ignoredSco, ignoredConfig) -> cdnService, - (ignoredSco, ignoredConfig) -> remoteConfigService); - - try { - FeatureFlaggingSystem.start(sharedCommunicationObjects()); - - verify(remoteConfigService).init(); - verifyNoInteractions(cdnService); - } finally { - FeatureFlaggingSystem.stop(); - FeatureFlaggingSystem.resetServiceFactoriesForTest(); - } + void explicitRemoteConfigUsesRemoteConfigService() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + when(sharedCommunicationObjects.configurationPoller(any(Config.class))) + .thenReturn(mock(ConfigurationPoller.class)); + + assertInstanceOf( + RemoteConfigServiceImpl.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects, Config.get())); } @Test @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "invalid") - void invalidConfigurationSourceFailsBeforeStartingCdn() { - RemoteConfigService cdnService = mock(RemoteConfigService.class); - RemoteConfigService remoteConfigService = mock(RemoteConfigService.class); - FeatureFlaggingSystem.setServiceFactoriesForTest( - (ignoredSco, ignoredConfig) -> cdnService, - (ignoredSco, ignoredConfig) -> remoteConfigService); + void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { + assertThrows( + IllegalArgumentException.class, + () -> + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } - try { - assertThrows( - IllegalArgumentException.class, - () -> FeatureFlaggingSystem.start(sharedCommunicationObjects())); - verifyNoInteractions(cdnService); - verifyNoInteractions(remoteConfigService); - } finally { - FeatureFlaggingSystem.stop(); - FeatureFlaggingSystem.resetServiceFactoriesForTest(); - } + @Test + @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "offline") + void offlineConfigurationSourceDoesNotStartNetworkSource() { + assertNull( + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); } private static SharedCommunicationObjects sharedCommunicationObjects() { diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java similarity index 60% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java index 5f84a78f7d4..83495545717 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java @@ -2,7 +2,7 @@ import java.io.Closeable; -public interface RemoteConfigService extends Closeable { +public interface ConfigurationSourceService extends Closeable { void init(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index ec28f684a96..a5a9b2d7f36 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -32,7 +32,7 @@ import okio.Okio; public class RemoteConfigServiceImpl - implements RemoteConfigService, ConfigurationChangesTypedListener { + implements ConfigurationSourceService, ConfigurationChangesTypedListener { private final ConfigurationPoller configurationPoller; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java similarity index 82% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java index 3857cfd8cd4..f7b42909d36 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/CdnConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java @@ -27,8 +27,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -final class CdnConfigService implements RemoteConfigService { - private static final Logger LOGGER = LoggerFactory.getLogger(CdnConfigService.class); +final class UfcHttpConfigService implements ConfigurationSourceService { + private static final Logger LOGGER = LoggerFactory.getLogger(UfcHttpConfigService.class); private static final String SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; @@ -38,33 +38,33 @@ final class CdnConfigService implements RemoteConfigService { private final Config config; private final Map extraHeaders; private final long pollIntervalMillis; - private final CdnClient client; + private final UfcHttpClient client; private final ScheduledExecutorService executor; private final AtomicBoolean polling = new AtomicBoolean(); private volatile boolean closed; private volatile ScheduledFuture scheduledPoll; private volatile String etag; - CdnConfigService(final Config config) { + UfcHttpConfigService(final Config config) { this(config, endpoint(config)); } - private CdnConfigService(final Config config, final HttpUrl endpoint) { + private UfcHttpConfigService(final Config config, final HttpUrl endpoint) { this( endpoint, config, millis(config.getFlaggingConfigurationSourcePollIntervalSeconds()), - new OkHttpCdnClient( + new OkHttpUfcHttpClient( OkHttpUtils.buildHttpClient( endpoint, millis(config.getFlaggingConfigurationSourceRequestTimeoutSeconds()))), - Executors.newSingleThreadScheduledExecutor(new CdnThreadFactory())); + Executors.newSingleThreadScheduledExecutor(new UfcHttpThreadFactory())); } - CdnConfigService( + UfcHttpConfigService( final HttpUrl endpoint, final Config config, final long pollIntervalMillis, - final CdnClient client, + final UfcHttpClient client, final ScheduledExecutorService executor) { this.endpoint = endpoint; this.config = config; @@ -112,21 +112,21 @@ private void pollOnceSafely() { try { pollOnce(); } catch (final RuntimeException e) { - LOGGER.debug("Unexpected error while polling Feature Flagging CDN", e); + LOGGER.debug("Unexpected error while polling Feature Flagging HTTP configuration source", e); } } private boolean fetchAndApply() { for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { - final CdnResponse response = client.fetch(endpoint, config, extraHeaders, etag); + final UfcHttpResponse response = client.fetch(endpoint, config, extraHeaders, etag); if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { continue; } return apply(response); } catch (final IOException e) { if (attempt == MAX_ATTEMPTS) { - LOGGER.debug("Feature Flagging CDN request failed", e); + LOGGER.debug("Feature Flagging HTTP configuration source request failed", e); return false; } } @@ -134,7 +134,7 @@ private boolean fetchAndApply() { return false; } - private boolean apply(final CdnResponse response) { + private boolean apply(final UfcHttpResponse response) { if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) { updateEtag(response.etag); return true; @@ -151,7 +151,7 @@ private boolean apply(final CdnResponse response) { RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( response.body); } catch (final IOException | RuntimeException e) { - LOGGER.debug("Feature Flagging CDN returned malformed UFC payload", e); + LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); return false; } if (configuration == null) { @@ -182,7 +182,8 @@ static HttpUrl endpoint(final Config config) { : endpointFromConfiguredUrl(configuredBaseUrl); final HttpUrl parsed = HttpUrl.parse(endpoint); if (parsed == null) { - throw new IllegalArgumentException("Invalid Feature Flagging CDN URL: " + endpoint); + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + endpoint); } return parsed; } @@ -190,7 +191,8 @@ static HttpUrl endpoint(final Config config) { private static String endpointFromConfiguredUrl(final String configuredUrl) { final HttpUrl parsed = HttpUrl.parse(configuredUrl.trim()); if (parsed == null) { - throw new IllegalArgumentException("Invalid Feature Flagging CDN URL: " + configuredUrl); + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + configuredUrl); } if (isRootPath(parsed)) { return parsed.newBuilder().addPathSegments("mock/ufc/config").build().toString(); @@ -226,33 +228,33 @@ static long millis(final double seconds) { return Math.max(1L, Math.round(seconds * 1000)); } - interface CdnClient { - CdnResponse fetch( + interface UfcHttpClient { + UfcHttpResponse fetch( HttpUrl endpoint, Config config, Map extraHeaders, String etag) throws IOException; } - static final class CdnResponse { + static final class UfcHttpResponse { final int status; final String etag; final byte[] body; - CdnResponse(final int status, final String etag, final byte[] body) { + UfcHttpResponse(final int status, final String etag, final byte[] body) { this.status = status; this.etag = etag; this.body = body; } } - private static final class OkHttpCdnClient implements CdnClient { + private static final class OkHttpUfcHttpClient implements UfcHttpClient { private final OkHttpClient httpClient; - private OkHttpCdnClient(final OkHttpClient httpClient) { + private OkHttpUfcHttpClient(final OkHttpClient httpClient) { this.httpClient = httpClient; } @Override - public CdnResponse fetch( + public UfcHttpResponse fetch( final HttpUrl endpoint, final Config config, final Map extraHeaders, @@ -265,7 +267,7 @@ public CdnResponse fetch( final Request request = prepareRequest(endpoint, headers, config, true).get().build(); try (Response response = httpClient.newCall(request).execute()) { final ResponseBody responseBody = response.body(); - return new CdnResponse( + return new UfcHttpResponse( response.code(), response.header("ETag"), responseBody == null ? null : responseBody.bytes()); @@ -273,10 +275,10 @@ public CdnResponse fetch( } } - private static final class CdnThreadFactory implements ThreadFactory { + private static final class UfcHttpThreadFactory implements ThreadFactory { @Override public Thread newThread(final Runnable runnable) { - final Thread thread = new Thread(runnable, "dd-feature-flagging-cdn-poller"); + final Thread thread = new Thread(runnable, "dd-feature-flagging-http-poller"); thread.setDaemon(true); return thread; } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java similarity index 88% rename from products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java rename to products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java index 0bcdf1567c9..6bdc7b408a6 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/CdnConfigServiceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java @@ -37,7 +37,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -class CdnConfigServiceTest { +class UfcHttpConfigServiceTest { @Mock private FeatureFlaggingGateway.ConfigListener listener; @@ -52,7 +52,7 @@ void appendsMockConfigPathWhenConfiguredUrlIsRoot() { final Config config = config("http://mock-cdn:8092", "datadoghq.com", ""); assertEquals( - "http://mock-cdn:8092/mock/ufc/config", CdnConfigService.endpoint(config).toString()); + "http://mock-cdn:8092/mock/ufc/config", UfcHttpConfigService.endpoint(config).toString()); } @Test @@ -62,7 +62,7 @@ void preservesConfiguredFullEndpointUrl() { assertEquals( "http://mock-cdn:8092/mock/ufc/config?fixture=valid", - CdnConfigService.endpoint(config).toString()); + UfcHttpConfigService.endpoint(config).toString()); } @Test @@ -71,13 +71,13 @@ void derivesDefaultEndpointFromSiteAndEnv() { assertEquals( "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", - CdnConfigService.endpoint(config).toString()); + UfcHttpConfigService.endpoint(config).toString()); } @Test void appliesAcceptedUfcThroughGatewayAndSendsHeaders() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -93,7 +93,7 @@ void appliesAcceptedUfcThroughGatewayAndSendsHeaders() throws Exception { void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig()), response(304, null, null)); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -110,7 +110,7 @@ void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { response(401, null, null), response(200, null, "{not-json}"), response(200, null, "{\"flags\":[]}")); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); FeatureFlaggingGateway.addConfigListener(listener); assertFalse(service.pollOnce()); @@ -124,10 +124,10 @@ void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { void retriesTimeoutBeforeApplyingConfig() throws Exception { final FakeClient client = new FakeClient( - new SocketTimeoutException("slow CDN"), - new SocketTimeoutException("slow CDN"), + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), response(200, "etag-a", emptyConfig())); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -139,7 +139,7 @@ void retriesTimeoutBeforeApplyingConfig() throws Exception { @Test void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -154,7 +154,7 @@ void rejectsOverlappingPolls() throws Exception { final CountDownLatch releaseRequest = new CountDownLatch(1); final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); client.block(requestStarted, releaseRequest); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); final ExecutorService runner = Executors.newFixedThreadPool(2); try { @@ -175,7 +175,7 @@ void rejectsOverlappingPolls() throws Exception { @Test void closePreventsFurtherPolls() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); - final CdnConfigService service = service(client, extraHeaders()); + final UfcHttpConfigService service = service(client, extraHeaders()); service.close(); @@ -183,10 +183,10 @@ void closePreventsFurtherPolls() throws Exception { assertEquals(0, client.calls.get()); } - private static CdnConfigService service( + private static UfcHttpConfigService service( final FakeClient client, final Map extraHeaders) { final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - return new CdnConfigService( + return new UfcHttpConfigService( HttpUrl.get("http://localhost/mock/ufc/config"), config("http://localhost", "datadoghq.com", "", extraHeaders), 60_000, @@ -221,9 +221,9 @@ private static Map extraHeaders() { return headers; } - private static CdnConfigService.CdnResponse response( + private static UfcHttpConfigService.UfcHttpResponse response( final int status, final String etag, final String body) { - return new CdnConfigService.CdnResponse( + return new UfcHttpConfigService.UfcHttpResponse( status, etag, body == null ? null : body.getBytes(UTF_8)); } @@ -236,7 +236,7 @@ private static String emptyConfig() { + "}"; } - private static final class FakeClient implements CdnConfigService.CdnClient { + private static final class FakeClient implements UfcHttpConfigService.UfcHttpClient { private final AtomicInteger calls = new AtomicInteger(); private final List requests = new ArrayList<>(); private final BlockingQueue responses = new LinkedBlockingQueue<>(); @@ -255,7 +255,7 @@ private void block(final CountDownLatch requestStarted, final CountDownLatch rel } @Override - public CdnConfigService.CdnResponse fetch( + public UfcHttpConfigService.UfcHttpResponse fetch( final HttpUrl endpoint, final Config config, final Map extraHeaders, @@ -273,7 +273,7 @@ public CdnConfigService.CdnResponse fetch( if (response instanceof IOException) { throw (IOException) response; } - return (CdnConfigService.CdnResponse) response; + return (UfcHttpConfigService.UfcHttpResponse) response; } private static void await(final CountDownLatch latch) throws IOException { From ce756567e57a6a49c3ba8d0d838c5ca22f081ce3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 8 Jul 2026 14:48:08 -0400 Subject: [PATCH 29/37] Auto-enable feature flagging for explicit CDN source --- .../java/datadog/trace/bootstrap/Agent.java | 26 ++++++++++++++++--- .../api/config/FeatureFlaggingConfig.java | 4 --- metadata/supported-configurations.json | 8 ------ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index ea9793ce86d..45f9ecb3acf 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -135,8 +135,7 @@ private enum AgentFeature { APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_ENABLED, false), - DEPRECATED_FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); + FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); private final String configKey; private final String systemProp; @@ -285,7 +284,7 @@ public static void start( llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGGING) - || isFeatureEnabled(AgentFeature.DEPRECATED_FEATURE_FLAGGING); + || isFeatureFlaggingCdnConfigurationSourceConfigured(); // setup writers when llmobs is enabled to accomodate apm and llmobs if (llmObsEnabled) { @@ -1634,6 +1633,27 @@ private static boolean isFeatureEnabled(AgentFeature feature) { } } + private static boolean isFeatureFlaggingCdnConfigurationSourceConfigured() { + final String configurationSource = + getConfiguredValue(FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE); + return configurationSource != null && "cdn".equalsIgnoreCase(configurationSource.trim()); + } + + private static String getConfiguredValue(final String configKey) { + final String systemProp = propertyNameToSystemPropertyName(configKey); + String settingValue = getNullIfEmpty(SystemProperties.get(systemProp)); + if (settingValue == null) { + settingValue = getNullIfEmpty(getStableConfig(FLEET, configKey)); + } + if (settingValue == null) { + settingValue = getNullIfEmpty(ddGetEnv(systemProp)); + } + if (settingValue == null) { + settingValue = getNullIfEmpty(getStableConfig(LOCAL, configKey)); + } + return settingValue; + } + /** * @see datadog.trace.api.ProductActivation#fromString(String) */ diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 25faf9da12f..99a96797041 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -2,10 +2,6 @@ public class FeatureFlaggingConfig { - public static final String FLAGGING_ENABLED = "flagging.enabled"; - - /** Replaced by {@link #FLAGGING_ENABLED}. */ - @Deprecated public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; public static final String FLAGGING_CONFIGURATION_SOURCE = "flagging.configuration.source"; diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 4c99c583841..425948da222 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1553,14 +1553,6 @@ "aliases": [] } ], - "DD_FLAGGING_ENABLED": [ - { - "version": "A", - "type": "boolean", - "default": "false", - "aliases": ["DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"] - } - ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", From 23c0b229998f3c8e59fd0fe8e97b541ddb717872 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 8 Jul 2026 21:01:51 -0400 Subject: [PATCH 30/37] Add Java agentless feature flag source --- .../java/datadog/trace/bootstrap/Agent.java | 39 +++++++++++--- .../datadog/trace/api/ConfigDefaults.java | 2 +- .../api/config/FeatureFlaggingConfig.java | 20 +++---- .../main/java/datadog/trace/api/Config.java | 21 ++++---- metadata/supported-configurations.json | 52 +++++++++++-------- .../featureflag/FeatureFlaggingSystem.java | 4 +- .../FeatureFlaggingSystemTest.java | 16 +++--- .../feature-flagging-api/README.md | 4 +- .../featureflag/UfcHttpConfigServiceTest.java | 9 ++-- 9 files changed, 102 insertions(+), 65 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 45f9ecb3acf..98ec5e57718 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -72,6 +72,7 @@ import java.net.URL; import java.security.CodeSource; import java.util.EnumSet; +import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.PatternSyntaxException; @@ -135,7 +136,7 @@ private enum AgentFeature { APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); + FEATURE_FLAGGING(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED, false); private final String configKey; private final String systemProp; @@ -282,9 +283,7 @@ public static void start( agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION); llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); - featureFlaggingEnabled = - isFeatureEnabled(AgentFeature.FEATURE_FLAGGING) - || isFeatureFlaggingCdnConfigurationSourceConfigured(); + featureFlaggingEnabled = isFeatureFlaggingEnabled(); // setup writers when llmobs is enabled to accomodate apm and llmobs if (llmObsEnabled) { @@ -1633,10 +1632,36 @@ private static boolean isFeatureEnabled(AgentFeature feature) { } } - private static boolean isFeatureFlaggingCdnConfigurationSourceConfigured() { + private static boolean isFeatureFlaggingEnabled() { + final String enabled = getConfiguredValue(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED); + if ("false".equalsIgnoreCase(enabled) || "0".equals(enabled)) { + return false; + } + if ("true".equalsIgnoreCase(enabled) || "1".equals(enabled)) { + return true; + } + + final String oldEnabled = getConfiguredValue(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED); + if ("false".equalsIgnoreCase(oldEnabled) || "0".equals(oldEnabled)) { + return false; + } + if ("true".equalsIgnoreCase(oldEnabled) || "1".equals(oldEnabled)) { + return true; + } + + return isFeatureFlaggingConfigurationSourceAutoEnabled(); + } + + private static boolean isFeatureFlaggingConfigurationSourceAutoEnabled() { final String configurationSource = - getConfiguredValue(FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE); - return configurationSource != null && "cdn".equalsIgnoreCase(configurationSource.trim()); + getConfiguredValue(FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE); + if (configurationSource == null) { + return true; + } + final String normalized = configurationSource.trim().toLowerCase(Locale.ROOT); + return "agentless".equals(normalized) + || "remote_config".equals(normalized) + || "offline".equals(normalized); } private static String getConfiguredValue(final String configKey) { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 42293cdea28..84f1428fe94 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -49,7 +49,7 @@ public final class ConfigDefaults { static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; - public static final String DEFAULT_FLAGGING_CONFIGURATION_SOURCE = "cdn"; + public static final String DEFAULT_FLAGGING_CONFIGURATION_SOURCE = "agentless"; public static final double DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30.0D; public static final double DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2.0D; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 99a96797041..97d7c105caa 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -2,17 +2,19 @@ public class FeatureFlaggingConfig { + public static final String FEATURE_FLAGS_ENABLED = "feature.flags.enabled"; public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; - public static final String FLAGGING_CONFIGURATION_SOURCE = "flagging.configuration.source"; - public static final String FLAGGING_CONFIGURATION_SOURCE_BASE_URL = - "flagging.configuration.source.base.url"; - public static final String FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = - "flagging.configuration.source.poll.interval.seconds"; - public static final String FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = - "flagging.configuration.source.request.timeout.seconds"; - public static final String FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS = - "flagging.configuration.source.extra.headers"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = + "feature.flags.configuration.source"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = + "feature.flags.configuration.source.agentless.base.url"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS = + "feature.flags.configuration.source.agentless.poll.interval.seconds"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = + "feature.flags.configuration.source.agentless.request.timeout.seconds"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS = + "feature.flags.configuration.source.agentless.extra.headers"; private FeatureFlaggingConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 5a64fcf8aa6..f691a415f8e 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -364,11 +364,11 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_EXCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; -import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE; -import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_BASE_URL; -import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS; -import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; -import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_QUEUE_SIZE; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL; @@ -2852,19 +2852,20 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) flaggingConfigurationSource = normalizeFlaggingConfigurationSource( configProvider.getString( - FLAGGING_CONFIGURATION_SOURCE, DEFAULT_FLAGGING_CONFIGURATION_SOURCE)); + FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FLAGGING_CONFIGURATION_SOURCE)); flaggingConfigurationSourceBaseUrl = - configProvider.getStringNotEmpty(FLAGGING_CONFIGURATION_SOURCE_BASE_URL, null); + configProvider.getStringNotEmpty( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); flaggingConfigurationSourcePollIntervalSeconds = configProvider.getDouble( - FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS, + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); flaggingConfigurationSourceRequestTimeoutSeconds = configProvider.getDouble( - FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS, + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); flaggingConfigurationSourceExtraHeaders = - configProvider.getMergedMap(FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS, '='); + configProvider.getMergedMap(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS, '='); dynamicInstrumentationEnabled = configProvider.getBoolean( diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 425948da222..19c2b2fdce4 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1497,31 +1497,15 @@ "aliases": [] } ], - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": [ - { - "version": "A", - "type": "boolean", - "default": "false", - "aliases": [] - } - ], - "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": [ - { - "version": "B", - "type": "boolean", - "default": "true", - "aliases": [] - } - ], - "DD_FLAGGING_CONFIGURATION_SOURCE": [ + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ { "version": "A", "type": "string", - "default": "cdn", + "default": "agentless", "aliases": [] } ], - "DD_FLAGGING_CONFIGURATION_SOURCE_BASE_URL": [ + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ { "version": "A", "type": "string", @@ -1529,7 +1513,7 @@ "aliases": [] } ], - "DD_FLAGGING_CONFIGURATION_SOURCE_EXTRA_HEADERS": [ + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS": [ { "version": "A", "type": "map", @@ -1537,7 +1521,7 @@ "aliases": [] } ], - "DD_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS": [ + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ { "version": "A", "type": "decimal", @@ -1545,7 +1529,7 @@ "aliases": [] } ], - "DD_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS": [ + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ { "version": "A", "type": "decimal", @@ -1553,6 +1537,30 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"] + } + ], + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], + "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": [ + { + "version": "B", + "type": "boolean", + "default": "true", + "aliases": [] + } + ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 6128d43cd34..10fc0bdbd5f 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -8,7 +8,7 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); - private static final String SOURCE_CDN = "cdn"; + private static final String SOURCE_AGENTLESS = "agentless"; private static final String SOURCE_REMOTE_CONFIG = "remote_config"; private static final String SOURCE_OFFLINE = "offline"; @@ -41,7 +41,7 @@ static ConfigurationSourceService createConfigurationSourceService( throw new IllegalStateException("Feature Flagging system started without RC"); } return new RemoteConfigServiceImpl(sco, config); - } else if (SOURCE_CDN.equals(configurationSource)) { + } else if (SOURCE_AGENTLESS.equals(configurationSource)) { return new UfcHttpConfigService(config); } else if (SOURCE_OFFLINE.equals(configurationSource)) { LOGGER.debug( diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index c70ec92bafd..df8511aa3a7 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,6 +1,6 @@ package com.datadog.featureflag; -import static datadog.trace.api.config.FeatureFlaggingConfig.FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; @@ -26,7 +26,7 @@ class FeatureFlaggingSystemTest { @Test - @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); @@ -53,7 +53,7 @@ void testFeatureFlagSystemInitialization() { } @Test - @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); @@ -68,9 +68,9 @@ void testThatRemoteConfigIsRequired() { } @Test - @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "cdn") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") - void cdnConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { + void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { assertInstanceOf( UfcHttpConfigService.class, FeatureFlaggingSystem.createConfigurationSourceService( @@ -78,7 +78,7 @@ void cdnConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { } @Test - @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void explicitRemoteConfigUsesRemoteConfigService() { SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); @@ -92,7 +92,7 @@ void explicitRemoteConfigUsesRemoteConfigService() { } @Test - @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "invalid") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { assertThrows( IllegalArgumentException.class, @@ -102,7 +102,7 @@ void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { } @Test - @WithConfig(key = FLAGGING_CONFIGURATION_SOURCE, value = "offline") + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") void offlineConfigurationSourceDoesNotStartNetworkSource() { assertNull( FeatureFlaggingSystem.createConfigurationSourceService( diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index 47733020559..b55f9ad988c 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -85,5 +85,5 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc ## Requirements - Java 11+ -- Datadog Agent with Remote Configuration enabled -- `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=true` +- Feature Flags provider enabled with `DD_FEATURE_FLAGS_ENABLED=true`, or left unset + for the default `agentless` configuration source. diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java index 6bdc7b408a6..d90f85b92f4 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java @@ -49,19 +49,20 @@ void cleanup() { @Test void appendsMockConfigPathWhenConfiguredUrlIsRoot() { - final Config config = config("http://mock-cdn:8092", "datadoghq.com", ""); + final Config config = config("http://mock-backend:8092", "datadoghq.com", ""); assertEquals( - "http://mock-cdn:8092/mock/ufc/config", UfcHttpConfigService.endpoint(config).toString()); + "http://mock-backend:8092/mock/ufc/config", + UfcHttpConfigService.endpoint(config).toString()); } @Test void preservesConfiguredFullEndpointUrl() { final Config config = - config("http://mock-cdn:8092/mock/ufc/config?fixture=valid", "datadoghq.com", ""); + config("http://mock-backend:8092/mock/ufc/config?fixture=valid", "datadoghq.com", ""); assertEquals( - "http://mock-cdn:8092/mock/ufc/config?fixture=valid", + "http://mock-backend:8092/mock/ufc/config?fixture=valid", UfcHttpConfigService.endpoint(config).toString()); } From 82e864f68be1c8059cbbd3b27ccec66fb11078f8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 8 Jul 2026 21:24:55 -0400 Subject: [PATCH 31/37] Address agentless configuration source feedback --- .../datadog/trace/api/ConfigDefaults.java | 8 +- .../main/java/datadog/trace/api/Config.java | 83 ++++++++++--------- .../featureflag/FeatureFlaggingSystem.java | 58 +++++++++---- .../featureflag/UfcHttpConfigService.java | 27 ++++-- .../featureflag/UfcHttpConfigServiceTest.java | 29 ++++--- 5 files changed, 124 insertions(+), 81 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 84f1428fe94..c3c46201c69 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -49,9 +49,11 @@ public final class ConfigDefaults { static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; - public static final String DEFAULT_FLAGGING_CONFIGURATION_SOURCE = "agentless"; - public static final double DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30.0D; - public static final double DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2.0D; + public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; + public static final double DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = + 30.0D; + public static final double DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = + 2.0D; static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index f691a415f8e..82c64929f4f 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -86,9 +86,9 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_BODY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_PARAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_EXPERIMENTATAL_JEE_SPLIT_BY_DEPLOYMENT; -import static datadog.trace.api.ConfigDefaults.DEFAULT_FLAGGING_CONFIGURATION_SOURCE; -import static datadog.trace.api.ConfigDefaults.DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; -import static datadog.trace.api.ConfigDefaults.DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_CLIENT_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_SERVER_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_HEALTH_METRICS_ENABLED; @@ -1221,11 +1221,11 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; - private final String flaggingConfigurationSource; - private final String flaggingConfigurationSourceBaseUrl; - private final double flaggingConfigurationSourcePollIntervalSeconds; - private final double flaggingConfigurationSourceRequestTimeoutSeconds; - private final Map flaggingConfigurationSourceExtraHeaders; + private final String featureFlaggingConfigurationSource; + private final String featureFlaggingConfigurationSourceBaseUrl; + private final double featureFlaggingConfigurationSourcePollIntervalSeconds; + private final double featureFlaggingConfigurationSourceRequestTimeoutSeconds; + private final Map featureFlaggingConfigurationSourceExtraHeaders; private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; @@ -2849,23 +2849,24 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getInteger( REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES); - flaggingConfigurationSource = - normalizeFlaggingConfigurationSource( + featureFlaggingConfigurationSource = + normalizeFeatureFlaggingConfigurationSource( configProvider.getString( - FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FLAGGING_CONFIGURATION_SOURCE)); - flaggingConfigurationSourceBaseUrl = + FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); + featureFlaggingConfigurationSourceBaseUrl = configProvider.getStringNotEmpty( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); - flaggingConfigurationSourcePollIntervalSeconds = + featureFlaggingConfigurationSourcePollIntervalSeconds = configProvider.getDouble( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, - DEFAULT_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); - flaggingConfigurationSourceRequestTimeoutSeconds = + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); + featureFlaggingConfigurationSourceRequestTimeoutSeconds = configProvider.getDouble( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, - DEFAULT_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); - flaggingConfigurationSourceExtraHeaders = - configProvider.getMergedMap(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS, '='); + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); + featureFlaggingConfigurationSourceExtraHeaders = + configProvider.getMergedMap( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS, '='); dynamicInstrumentationEnabled = configProvider.getBoolean( @@ -3779,12 +3780,12 @@ public boolean isInferredProxyPropagationEnabled() { return traceInferredProxyEnabled; } - private static String normalizeFlaggingConfigurationSource(final String source) { + private static String normalizeFeatureFlaggingConfigurationSource(final String source) { if (source == null) { - return DEFAULT_FLAGGING_CONFIGURATION_SOURCE; + return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; } final String normalized = source.trim().toLowerCase(Locale.ROOT); - return normalized.isEmpty() ? DEFAULT_FLAGGING_CONFIGURATION_SOURCE : normalized; + return normalized.isEmpty() ? DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE : normalized; } public boolean isBaggageExtract() { @@ -4694,24 +4695,24 @@ public int getRemoteConfigMaxExtraServices() { return remoteConfigMaxExtraServices; } - public String getFlaggingConfigurationSource() { - return flaggingConfigurationSource; + public String getFeatureFlaggingConfigurationSource() { + return featureFlaggingConfigurationSource; } - public String getFlaggingConfigurationSourceBaseUrl() { - return flaggingConfigurationSourceBaseUrl; + public String getFeatureFlaggingConfigurationSourceBaseUrl() { + return featureFlaggingConfigurationSourceBaseUrl; } - public double getFlaggingConfigurationSourcePollIntervalSeconds() { - return flaggingConfigurationSourcePollIntervalSeconds; + public double getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { + return featureFlaggingConfigurationSourcePollIntervalSeconds; } - public double getFlaggingConfigurationSourceRequestTimeoutSeconds() { - return flaggingConfigurationSourceRequestTimeoutSeconds; + public double getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { + return featureFlaggingConfigurationSourceRequestTimeoutSeconds; } - public Map getFlaggingConfigurationSourceExtraHeaders() { - return flaggingConfigurationSourceExtraHeaders; + public Map getFeatureFlaggingConfigurationSourceExtraHeaders() { + return featureFlaggingConfigurationSourceExtraHeaders; } public boolean isDynamicInstrumentationEnabled() { @@ -6546,16 +6547,16 @@ public String toString() { + remoteConfigMaxPayloadSize + ", remoteConfigIntegrityCheckEnabled=" + remoteConfigIntegrityCheckEnabled - + ", flaggingConfigurationSource=" - + flaggingConfigurationSource - + ", flaggingConfigurationSourceBaseUrl=" - + flaggingConfigurationSourceBaseUrl - + ", flaggingConfigurationSourcePollIntervalSeconds=" - + flaggingConfigurationSourcePollIntervalSeconds - + ", flaggingConfigurationSourceRequestTimeoutSeconds=" - + flaggingConfigurationSourceRequestTimeoutSeconds - + ", flaggingConfigurationSourceExtraHeaderNames=" - + flaggingConfigurationSourceExtraHeaders.keySet() + + ", featureFlaggingConfigurationSource=" + + featureFlaggingConfigurationSource + + ", featureFlaggingConfigurationSourceBaseUrl=" + + featureFlaggingConfigurationSourceBaseUrl + + ", featureFlaggingConfigurationSourcePollIntervalSeconds=" + + featureFlaggingConfigurationSourcePollIntervalSeconds + + ", featureFlaggingConfigurationSourceRequestTimeoutSeconds=" + + featureFlaggingConfigurationSourceRequestTimeoutSeconds + + ", featureFlaggingConfigurationSourceExtraHeaderNames=" + + featureFlaggingConfigurationSourceExtraHeaders.keySet() + ", debuggerEnabled=" + dynamicInstrumentationEnabled + ", debuggerUploadTimeout=" diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 10fc0bdbd5f..f2db8754a27 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -8,9 +8,6 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); - private static final String SOURCE_AGENTLESS = "agentless"; - private static final String SOURCE_REMOTE_CONFIG = "remote_config"; - private static final String SOURCE_OFFLINE = "offline"; private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; @@ -34,22 +31,25 @@ public static void start(final SharedCommunicationObjects sco) { static ConfigurationSourceService createConfigurationSourceService( final SharedCommunicationObjects sco, final Config config) { - final String configurationSource = config.getFlaggingConfigurationSource(); + final ConfigurationSource configurationSource = + ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource()); - if (SOURCE_REMOTE_CONFIG.equals(configurationSource)) { - if (!config.isRemoteConfigEnabled()) { - throw new IllegalStateException("Feature Flagging system started without RC"); - } - return new RemoteConfigServiceImpl(sco, config); - } else if (SOURCE_AGENTLESS.equals(configurationSource)) { - return new UfcHttpConfigService(config); - } else if (SOURCE_OFFLINE.equals(configurationSource)) { - LOGGER.debug( - "Feature Flagging offline configuration source selected; no config service started"); - return null; - } else { - throw new IllegalArgumentException( - "Unsupported Feature Flagging configuration source: " + configurationSource); + switch (configurationSource) { + case REMOTE_CONFIG: + if (!config.isRemoteConfigEnabled()) { + throw new IllegalStateException("Feature Flagging system started without RC"); + } + return new RemoteConfigServiceImpl(sco, config); + case AGENTLESS: + return new UfcHttpConfigService(config); + case OFFLINE: + LOGGER.debug( + "Feature Flagging offline configuration source selected; no config service started"); + return null; + default: + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + + config.getFeatureFlaggingConfigurationSource()); } } @@ -64,4 +64,26 @@ public static void stop() { } LOGGER.debug("Feature Flagging system stopped"); } + + private enum ConfigurationSource { + AGENTLESS("agentless"), + REMOTE_CONFIG("remote_config"), + OFFLINE("offline"); + + private final String value; + + ConfigurationSource(final String value) { + this.value = value; + } + + private static ConfigurationSource from(final String value) { + for (final ConfigurationSource source : values()) { + if (source.value.equals(value)) { + return source; + } + } + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + value); + } + } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java index f7b42909d36..f0ecfa049ab 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java @@ -30,7 +30,7 @@ final class UfcHttpConfigService implements ConfigurationSourceService { private static final Logger LOGGER = LoggerFactory.getLogger(UfcHttpConfigService.class); - private static final String SERVER_DISTRIBUTION_PATH = + private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; private static final int MAX_ATTEMPTS = 3; @@ -53,10 +53,11 @@ private UfcHttpConfigService(final Config config, final HttpUrl endpoint) { this( endpoint, config, - millis(config.getFlaggingConfigurationSourcePollIntervalSeconds()), + millis(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()), new OkHttpUfcHttpClient( OkHttpUtils.buildHttpClient( - endpoint, millis(config.getFlaggingConfigurationSourceRequestTimeoutSeconds()))), + endpoint, + millis(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()))), Executors.newSingleThreadScheduledExecutor(new UfcHttpThreadFactory())); } @@ -69,7 +70,7 @@ endpoint, millis(config.getFlaggingConfigurationSourceRequestTimeoutSeconds()))) this.endpoint = endpoint; this.config = config; final Map configuredExtraHeaders = - config.getFlaggingConfigurationSourceExtraHeaders(); + config.getFeatureFlaggingConfigurationSourceExtraHeaders(); this.extraHeaders = configuredExtraHeaders == null ? Collections.emptyMap() : configuredExtraHeaders; this.pollIntervalMillis = pollIntervalMillis; @@ -175,10 +176,10 @@ private void updateEtag(final String nextEtag) { } static HttpUrl endpoint(final Config config) { - final String configuredBaseUrl = config.getFlaggingConfigurationSourceBaseUrl(); + final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceBaseUrl(); final String endpoint = configuredBaseUrl == null - ? defaultEndpoint(config) + ? datadogApiServerDistributionEndpoint(config) : endpointFromConfiguredUrl(configuredBaseUrl); final HttpUrl parsed = HttpUrl.parse(endpoint); if (parsed == null) { @@ -195,7 +196,11 @@ private static String endpointFromConfiguredUrl(final String configuredUrl) { "Invalid Feature Flagging HTTP configuration source URL: " + configuredUrl); } if (isRootPath(parsed)) { - return parsed.newBuilder().addPathSegments("mock/ufc/config").build().toString(); + return parsed + .newBuilder() + .addPathSegments(datadogApiServerDistributionPath()) + .build() + .toString(); } return parsed.toString(); } @@ -204,11 +209,11 @@ private static boolean isRootPath(final HttpUrl url) { return "/".equals(url.encodedPath()) || url.encodedPath().isEmpty(); } - private static String defaultEndpoint(final Config config) { + private static String datadogApiServerDistributionEndpoint(final Config config) { final StringBuilder endpoint = new StringBuilder("https://api.") .append(config.getSite().toLowerCase(Locale.ROOT)) - .append(SERVER_DISTRIBUTION_PATH); + .append(DATADOG_API_SERVER_DISTRIBUTION_PATH); final String env = config.getEnv(); if (env != null && !env.isEmpty()) { endpoint.append("?dd_env=").append(urlEncode(env)); @@ -216,6 +221,10 @@ private static String defaultEndpoint(final Config config) { return endpoint.toString(); } + private static String datadogApiServerDistributionPath() { + return DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1); + } + private static String urlEncode(final String value) { try { return URLEncoder.encode(value, "UTF-8"); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java index d90f85b92f4..107b06bf53f 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java @@ -48,26 +48,29 @@ void cleanup() { } @Test - void appendsMockConfigPathWhenConfiguredUrlIsRoot() { + void appendsDatadogApiServerDistributionPathWhenConfiguredUrlIsRoot() { final Config config = config("http://mock-backend:8092", "datadoghq.com", ""); assertEquals( - "http://mock-backend:8092/mock/ufc/config", + "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution", UfcHttpConfigService.endpoint(config).toString()); } @Test void preservesConfiguredFullEndpointUrl() { final Config config = - config("http://mock-backend:8092/mock/ufc/config?fixture=valid", "datadoghq.com", ""); + config( + "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution?fixture=valid", + "datadoghq.com", + ""); assertEquals( - "http://mock-backend:8092/mock/ufc/config?fixture=valid", + "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution?fixture=valid", UfcHttpConfigService.endpoint(config).toString()); } @Test - void derivesDefaultEndpointFromSiteAndEnv() { + void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { final Config config = config(null, "datad0g.com", "staging env"); assertEquals( @@ -188,7 +191,7 @@ private static UfcHttpConfigService service( final FakeClient client, final Map extraHeaders) { final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); return new UfcHttpConfigService( - HttpUrl.get("http://localhost/mock/ufc/config"), + HttpUrl.get("http://localhost/api/v2/feature-flagging/config/server-distribution"), config("http://localhost", "datadoghq.com", "", extraHeaders), 60_000, client, @@ -205,10 +208,16 @@ private static Config config( final String env, final Map extraHeaders) { final Config config = mock(Config.class); - lenient().when(config.getFlaggingConfigurationSourceBaseUrl()).thenReturn(baseUrl); - lenient().when(config.getFlaggingConfigurationSourceExtraHeaders()).thenReturn(extraHeaders); - lenient().when(config.getFlaggingConfigurationSourcePollIntervalSeconds()).thenReturn(30.0D); - lenient().when(config.getFlaggingConfigurationSourceRequestTimeoutSeconds()).thenReturn(2.0D); + lenient().when(config.getFeatureFlaggingConfigurationSourceBaseUrl()).thenReturn(baseUrl); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceExtraHeaders()) + .thenReturn(extraHeaders); + lenient() + .when(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()) + .thenReturn(30.0D); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) + .thenReturn(2.0D); lenient().when(config.getApiKey()).thenReturn("test-api-key"); lenient().when(config.getSite()).thenReturn(site); lenient().when(config.getEnv()).thenReturn(env); From e4393c21d57c19f3ff2a5858b66746606a74e8e1 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 9 Jul 2026 22:57:25 -0400 Subject: [PATCH 32/37] Refine agentless configuration source contract --- .../java/datadog/trace/bootstrap/Agent.java | 52 +--------- .../api/config/FeatureFlaggingConfig.java | 5 - .../main/java/datadog/trace/api/Config.java | 22 ----- metadata/supported-configurations.json | 24 ----- .../featureflag/FeatureFlaggingSystem.java | 2 +- .../FeatureFlaggingSystemTest.java | 2 +- .../feature-flagging-api/README.md | 6 +- ...java => AgentlessConfigurationSource.java} | 61 +++--------- ... => AgentlessConfigurationSourceTest.java} | 99 +++++-------------- 9 files changed, 42 insertions(+), 231 deletions(-) rename products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/{UfcHttpConfigService.java => AgentlessConfigurationSource.java} (77%) rename products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/{UfcHttpConfigServiceTest.java => AgentlessConfigurationSourceTest.java} (69%) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 98ec5e57718..876a07f1fd1 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -72,7 +72,6 @@ import java.net.URL; import java.security.CodeSource; import java.util.EnumSet; -import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.PatternSyntaxException; @@ -136,7 +135,7 @@ private enum AgentFeature { APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false), LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false), LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false), - FEATURE_FLAGGING(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED, false); + FEATURE_FLAGGING(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED, false); private final String configKey; private final String systemProp; @@ -283,7 +282,7 @@ public static void start( agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION); appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION); llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS); - featureFlaggingEnabled = isFeatureFlaggingEnabled(); + featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGGING); // setup writers when llmobs is enabled to accomodate apm and llmobs if (llmObsEnabled) { @@ -1632,53 +1631,6 @@ private static boolean isFeatureEnabled(AgentFeature feature) { } } - private static boolean isFeatureFlaggingEnabled() { - final String enabled = getConfiguredValue(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED); - if ("false".equalsIgnoreCase(enabled) || "0".equals(enabled)) { - return false; - } - if ("true".equalsIgnoreCase(enabled) || "1".equals(enabled)) { - return true; - } - - final String oldEnabled = getConfiguredValue(FeatureFlaggingConfig.FLAGGING_PROVIDER_ENABLED); - if ("false".equalsIgnoreCase(oldEnabled) || "0".equals(oldEnabled)) { - return false; - } - if ("true".equalsIgnoreCase(oldEnabled) || "1".equals(oldEnabled)) { - return true; - } - - return isFeatureFlaggingConfigurationSourceAutoEnabled(); - } - - private static boolean isFeatureFlaggingConfigurationSourceAutoEnabled() { - final String configurationSource = - getConfiguredValue(FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE); - if (configurationSource == null) { - return true; - } - final String normalized = configurationSource.trim().toLowerCase(Locale.ROOT); - return "agentless".equals(normalized) - || "remote_config".equals(normalized) - || "offline".equals(normalized); - } - - private static String getConfiguredValue(final String configKey) { - final String systemProp = propertyNameToSystemPropertyName(configKey); - String settingValue = getNullIfEmpty(SystemProperties.get(systemProp)); - if (settingValue == null) { - settingValue = getNullIfEmpty(getStableConfig(FLEET, configKey)); - } - if (settingValue == null) { - settingValue = getNullIfEmpty(ddGetEnv(systemProp)); - } - if (settingValue == null) { - settingValue = getNullIfEmpty(getStableConfig(LOCAL, configKey)); - } - return settingValue; - } - /** * @see datadog.trace.api.ProductActivation#fromString(String) */ diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 97d7c105caa..9fbc047b434 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -2,19 +2,14 @@ public class FeatureFlaggingConfig { - public static final String FEATURE_FLAGS_ENABLED = "feature.flags.enabled"; public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = "feature.flags.configuration.source"; - public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = - "feature.flags.configuration.source.agentless.base.url"; public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS = "feature.flags.configuration.source.agentless.poll.interval.seconds"; public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = "feature.flags.configuration.source.agentless.request.timeout.seconds"; - public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS = - "feature.flags.configuration.source.agentless.extra.headers"; private FeatureFlaggingConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 82c64929f4f..acf383fbffe 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -365,8 +365,6 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; -import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; -import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS; import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; @@ -1222,10 +1220,8 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; private final String featureFlaggingConfigurationSource; - private final String featureFlaggingConfigurationSourceBaseUrl; private final double featureFlaggingConfigurationSourcePollIntervalSeconds; private final double featureFlaggingConfigurationSourceRequestTimeoutSeconds; - private final Map featureFlaggingConfigurationSourceExtraHeaders; private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; @@ -2853,9 +2849,6 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) normalizeFeatureFlaggingConfigurationSource( configProvider.getString( FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); - featureFlaggingConfigurationSourceBaseUrl = - configProvider.getStringNotEmpty( - FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); featureFlaggingConfigurationSourcePollIntervalSeconds = configProvider.getDouble( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, @@ -2864,9 +2857,6 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getDouble( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); - featureFlaggingConfigurationSourceExtraHeaders = - configProvider.getMergedMap( - FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS, '='); dynamicInstrumentationEnabled = configProvider.getBoolean( @@ -4699,10 +4689,6 @@ public String getFeatureFlaggingConfigurationSource() { return featureFlaggingConfigurationSource; } - public String getFeatureFlaggingConfigurationSourceBaseUrl() { - return featureFlaggingConfigurationSourceBaseUrl; - } - public double getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { return featureFlaggingConfigurationSourcePollIntervalSeconds; } @@ -4711,10 +4697,6 @@ public double getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { return featureFlaggingConfigurationSourceRequestTimeoutSeconds; } - public Map getFeatureFlaggingConfigurationSourceExtraHeaders() { - return featureFlaggingConfigurationSourceExtraHeaders; - } - public boolean isDynamicInstrumentationEnabled() { return dynamicInstrumentationEnabled; } @@ -6549,14 +6531,10 @@ public String toString() { + remoteConfigIntegrityCheckEnabled + ", featureFlaggingConfigurationSource=" + featureFlaggingConfigurationSource - + ", featureFlaggingConfigurationSourceBaseUrl=" - + featureFlaggingConfigurationSourceBaseUrl + ", featureFlaggingConfigurationSourcePollIntervalSeconds=" + featureFlaggingConfigurationSourcePollIntervalSeconds + ", featureFlaggingConfigurationSourceRequestTimeoutSeconds=" + featureFlaggingConfigurationSourceRequestTimeoutSeconds - + ", featureFlaggingConfigurationSourceExtraHeaderNames=" - + featureFlaggingConfigurationSourceExtraHeaders.keySet() + ", debuggerEnabled=" + dynamicInstrumentationEnabled + ", debuggerUploadTimeout=" diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 19c2b2fdce4..f2c62bc6eff 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1505,22 +1505,6 @@ "aliases": [] } ], - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ - { - "version": "A", - "type": "string", - "default": null, - "aliases": [] - } - ], - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_EXTRA_HEADERS": [ - { - "version": "A", - "type": "map", - "default": null, - "aliases": [] - } - ], "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ { "version": "A", @@ -1537,14 +1521,6 @@ "aliases": [] } ], - "DD_FEATURE_FLAGS_ENABLED": [ - { - "version": "A", - "type": "boolean", - "default": "true", - "aliases": ["DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"] - } - ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index f2db8754a27..4c38a30c879 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -41,7 +41,7 @@ static ConfigurationSourceService createConfigurationSourceService( } return new RemoteConfigServiceImpl(sco, config); case AGENTLESS: - return new UfcHttpConfigService(config); + return new AgentlessConfigurationSource(config); case OFFLINE: LOGGER.debug( "Feature Flagging offline configuration source selected; no config service started"); diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index df8511aa3a7..e1608757a23 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -72,7 +72,7 @@ void testThatRemoteConfigIsRequired() { @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { assertInstanceOf( - UfcHttpConfigService.class, + AgentlessConfigurationSource.class, FeatureFlaggingSystem.createConfigurationSourceService( sharedCommunicationObjects(), Config.get())); } diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index b55f9ad988c..1eedec6eae0 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -85,5 +85,7 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc ## Requirements - Java 11+ -- Feature Flags provider enabled with `DD_FEATURE_FLAGS_ENABLED=true`, or left unset - for the default `agentless` configuration source. +- `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless` uses the Datadog agentless + backend. `remote_config` uses the existing Agent Remote Configuration path. + `offline` is reserved for startup-provided UFC bytes; until those bytes are + implemented, no network source starts and evaluations use defaults. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java similarity index 77% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index f0ecfa049ab..20df2941ec2 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UfcHttpConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.net.URLEncoder; -import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -27,16 +26,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -final class UfcHttpConfigService implements ConfigurationSourceService { - private static final Logger LOGGER = LoggerFactory.getLogger(UfcHttpConfigService.class); +final class AgentlessConfigurationSource implements ConfigurationSourceService { + private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessConfigurationSource.class); + // TODO before merge: confirm the final backend route with the server-distribution API owners. private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; private static final int MAX_ATTEMPTS = 3; private final HttpUrl endpoint; private final Config config; - private final Map extraHeaders; private final long pollIntervalMillis; private final UfcHttpClient client; private final ScheduledExecutorService executor; @@ -45,11 +44,11 @@ final class UfcHttpConfigService implements ConfigurationSourceService { private volatile ScheduledFuture scheduledPoll; private volatile String etag; - UfcHttpConfigService(final Config config) { + AgentlessConfigurationSource(final Config config) { this(config, endpoint(config)); } - private UfcHttpConfigService(final Config config, final HttpUrl endpoint) { + private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint) { this( endpoint, config, @@ -61,7 +60,7 @@ private UfcHttpConfigService(final Config config, final HttpUrl endpoint) { Executors.newSingleThreadScheduledExecutor(new UfcHttpThreadFactory())); } - UfcHttpConfigService( + AgentlessConfigurationSource( final HttpUrl endpoint, final Config config, final long pollIntervalMillis, @@ -69,10 +68,6 @@ private UfcHttpConfigService(final Config config, final HttpUrl endpoint) { final ScheduledExecutorService executor) { this.endpoint = endpoint; this.config = config; - final Map configuredExtraHeaders = - config.getFeatureFlaggingConfigurationSourceExtraHeaders(); - this.extraHeaders = - configuredExtraHeaders == null ? Collections.emptyMap() : configuredExtraHeaders; this.pollIntervalMillis = pollIntervalMillis; this.client = client; this.executor = executor; @@ -120,7 +115,7 @@ private void pollOnceSafely() { private boolean fetchAndApply() { for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { - final UfcHttpResponse response = client.fetch(endpoint, config, extraHeaders, etag); + final UfcHttpResponse response = client.fetch(endpoint, config, etag); if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { continue; } @@ -176,11 +171,7 @@ private void updateEtag(final String nextEtag) { } static HttpUrl endpoint(final Config config) { - final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceBaseUrl(); - final String endpoint = - configuredBaseUrl == null - ? datadogApiServerDistributionEndpoint(config) - : endpointFromConfiguredUrl(configuredBaseUrl); + final String endpoint = datadogApiServerDistributionEndpoint(config); final HttpUrl parsed = HttpUrl.parse(endpoint); if (parsed == null) { throw new IllegalArgumentException( @@ -189,26 +180,6 @@ static HttpUrl endpoint(final Config config) { return parsed; } - private static String endpointFromConfiguredUrl(final String configuredUrl) { - final HttpUrl parsed = HttpUrl.parse(configuredUrl.trim()); - if (parsed == null) { - throw new IllegalArgumentException( - "Invalid Feature Flagging HTTP configuration source URL: " + configuredUrl); - } - if (isRootPath(parsed)) { - return parsed - .newBuilder() - .addPathSegments(datadogApiServerDistributionPath()) - .build() - .toString(); - } - return parsed.toString(); - } - - private static boolean isRootPath(final HttpUrl url) { - return "/".equals(url.encodedPath()) || url.encodedPath().isEmpty(); - } - private static String datadogApiServerDistributionEndpoint(final Config config) { final StringBuilder endpoint = new StringBuilder("https://api.") @@ -221,10 +192,6 @@ private static String datadogApiServerDistributionEndpoint(final Config config) return endpoint.toString(); } - private static String datadogApiServerDistributionPath() { - return DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1); - } - private static String urlEncode(final String value) { try { return URLEncoder.encode(value, "UTF-8"); @@ -238,9 +205,7 @@ static long millis(final double seconds) { } interface UfcHttpClient { - UfcHttpResponse fetch( - HttpUrl endpoint, Config config, Map extraHeaders, String etag) - throws IOException; + UfcHttpResponse fetch(HttpUrl endpoint, Config config, String etag) throws IOException; } static final class UfcHttpResponse { @@ -263,13 +228,9 @@ private OkHttpUfcHttpClient(final OkHttpClient httpClient) { } @Override - public UfcHttpResponse fetch( - final HttpUrl endpoint, - final Config config, - final Map extraHeaders, - final String etag) + public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final String etag) throws IOException { - final Map headers = new HashMap<>(extraHeaders); + final Map headers = new HashMap<>(); if (etag != null) { headers.put("If-None-Match", etag); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java similarity index 69% rename from products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java rename to products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 107b06bf53f..d5410634e37 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/UfcHttpConfigServiceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -17,9 +17,7 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -37,7 +35,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -class UfcHttpConfigServiceTest { +class AgentlessConfigurationSourceTest { @Mock private FeatureFlaggingGateway.ConfigListener listener; @@ -47,49 +45,25 @@ void cleanup() { FeatureFlaggingGateway.dispatch((ServerConfiguration) null); } - @Test - void appendsDatadogApiServerDistributionPathWhenConfiguredUrlIsRoot() { - final Config config = config("http://mock-backend:8092", "datadoghq.com", ""); - - assertEquals( - "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution", - UfcHttpConfigService.endpoint(config).toString()); - } - - @Test - void preservesConfiguredFullEndpointUrl() { - final Config config = - config( - "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution?fixture=valid", - "datadoghq.com", - ""); - - assertEquals( - "http://mock-backend:8092/api/v2/feature-flagging/config/server-distribution?fixture=valid", - UfcHttpConfigService.endpoint(config).toString()); - } - @Test void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { - final Config config = config(null, "datad0g.com", "staging env"); + final Config config = config("datad0g.com", "staging env"); assertEquals( "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", - UfcHttpConfigService.endpoint(config).toString()); + AgentlessConfigurationSource.endpoint(config).toString()); } @Test - void appliesAcceptedUfcThroughGatewayAndSendsHeaders() throws Exception { + void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); verify(listener).accept(any(ServerConfiguration.class)); assertEquals("test-api-key", client.requests.get(0).apiKey); - assertEquals("app-key", client.requests.get(0).extraHeaders.get("DD-APPLICATION-KEY")); - assertEquals("1", client.requests.get(0).extraHeaders.get("fastly-client")); assertNull(client.requests.get(0).etag); } @@ -97,7 +71,7 @@ void appliesAcceptedUfcThroughGatewayAndSendsHeaders() throws Exception { void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig()), response(304, null, null)); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -114,7 +88,7 @@ void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { response(401, null, null), response(200, null, "{not-json}"), response(200, null, "{\"flags\":[]}")); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertFalse(service.pollOnce()); @@ -131,7 +105,7 @@ void retriesTimeoutBeforeApplyingConfig() throws Exception { new SocketTimeoutException("slow HTTP configuration source"), new SocketTimeoutException("slow HTTP configuration source"), response(200, "etag-a", emptyConfig())); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -143,7 +117,7 @@ void retriesTimeoutBeforeApplyingConfig() throws Exception { @Test void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); @@ -158,7 +132,7 @@ void rejectsOverlappingPolls() throws Exception { final CountDownLatch releaseRequest = new CountDownLatch(1); final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); client.block(requestStarted, releaseRequest); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); final ExecutorService runner = Executors.newFixedThreadPool(2); try { @@ -179,7 +153,7 @@ void rejectsOverlappingPolls() throws Exception { @Test void closePreventsFurtherPolls() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); - final UfcHttpConfigService service = service(client, extraHeaders()); + final AgentlessConfigurationSource service = service(client); service.close(); @@ -187,31 +161,18 @@ void closePreventsFurtherPolls() throws Exception { assertEquals(0, client.calls.get()); } - private static UfcHttpConfigService service( - final FakeClient client, final Map extraHeaders) { + private static AgentlessConfigurationSource service(final FakeClient client) { final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - return new UfcHttpConfigService( + return new AgentlessConfigurationSource( HttpUrl.get("http://localhost/api/v2/feature-flagging/config/server-distribution"), - config("http://localhost", "datadoghq.com", "", extraHeaders), + config("datadoghq.com", ""), 60_000, client, executor); } - private static Config config(final String baseUrl, final String site, final String env) { - return config(baseUrl, site, env, new HashMap<>()); - } - - private static Config config( - final String baseUrl, - final String site, - final String env, - final Map extraHeaders) { + private static Config config(final String site, final String env) { final Config config = mock(Config.class); - lenient().when(config.getFeatureFlaggingConfigurationSourceBaseUrl()).thenReturn(baseUrl); - lenient() - .when(config.getFeatureFlaggingConfigurationSourceExtraHeaders()) - .thenReturn(extraHeaders); lenient() .when(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()) .thenReturn(30.0D); @@ -224,16 +185,9 @@ private static Config config( return config; } - private static Map extraHeaders() { - final Map headers = new HashMap<>(); - headers.put("DD-APPLICATION-KEY", "app-key"); - headers.put("fastly-client", "1"); - return headers; - } - - private static UfcHttpConfigService.UfcHttpResponse response( + private static AgentlessConfigurationSource.UfcHttpResponse response( final int status, final String etag, final String body) { - return new UfcHttpConfigService.UfcHttpResponse( + return new AgentlessConfigurationSource.UfcHttpResponse( status, etag, body == null ? null : body.getBytes(UTF_8)); } @@ -246,7 +200,7 @@ private static String emptyConfig() { + "}"; } - private static final class FakeClient implements UfcHttpConfigService.UfcHttpClient { + private static final class FakeClient implements AgentlessConfigurationSource.UfcHttpClient { private final AtomicInteger calls = new AtomicInteger(); private final List requests = new ArrayList<>(); private final BlockingQueue responses = new LinkedBlockingQueue<>(); @@ -265,14 +219,10 @@ private void block(final CountDownLatch requestStarted, final CountDownLatch rel } @Override - public UfcHttpConfigService.UfcHttpResponse fetch( - final HttpUrl endpoint, - final Config config, - final Map extraHeaders, - final String etag) - throws IOException { + public AgentlessConfigurationSource.UfcHttpResponse fetch( + final HttpUrl endpoint, final Config config, final String etag) throws IOException { calls.incrementAndGet(); - requests.add(new Request(config.getApiKey(), etag, extraHeaders)); + requests.add(new Request(config.getApiKey(), etag)); if (requestStarted != null) { requestStarted.countDown(); } @@ -283,7 +233,7 @@ public UfcHttpConfigService.UfcHttpResponse fetch( if (response instanceof IOException) { throw (IOException) response; } - return (UfcHttpConfigService.UfcHttpResponse) response; + return (AgentlessConfigurationSource.UfcHttpResponse) response; } private static void await(final CountDownLatch latch) throws IOException { @@ -301,13 +251,10 @@ private static void await(final CountDownLatch latch) throws IOException { private static final class Request { private final String apiKey; private final String etag; - private final Map extraHeaders; - private Request( - final String apiKey, final String etag, final Map extraHeaders) { + private Request(final String apiKey, final String etag) { this.apiKey = apiKey; this.etag = etag; - this.extraHeaders = new HashMap<>(extraHeaders); } } } From a207eb9a2782eac6d09147c077640c0f866e5edf Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 16:28:32 -0400 Subject: [PATCH 33/37] Fix agentless feature flagging CI coverage --- .../OpenFeatureProviderSmokeTest.groovy | 1 + .../featureflag/FeatureFlaggingSystem.java | 27 +- .../FeatureFlaggingSystemTest.java | 11 + .../AgentlessConfigurationSource.java | 14 +- .../AgentlessConfigurationSourceTest.java | 242 +++++++++++++++++- 5 files changed, 265 insertions(+), 30 deletions(-) diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy index cb4c641d667..ca80ca47f8b 100644 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy @@ -42,6 +42,7 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED', 'true') + builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') return builder } diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 4c38a30c879..6b4833ca920 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -34,23 +34,18 @@ static ConfigurationSourceService createConfigurationSourceService( final ConfigurationSource configurationSource = ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource()); - switch (configurationSource) { - case REMOTE_CONFIG: - if (!config.isRemoteConfigEnabled()) { - throw new IllegalStateException("Feature Flagging system started without RC"); - } - return new RemoteConfigServiceImpl(sco, config); - case AGENTLESS: - return new AgentlessConfigurationSource(config); - case OFFLINE: - LOGGER.debug( - "Feature Flagging offline configuration source selected; no config service started"); - return null; - default: - throw new IllegalArgumentException( - "Unsupported Feature Flagging configuration source: " - + config.getFeatureFlaggingConfigurationSource()); + if (configurationSource == ConfigurationSource.REMOTE_CONFIG) { + if (!config.isRemoteConfigEnabled()) { + throw new IllegalStateException("Feature Flagging system started without RC"); + } + return new RemoteConfigServiceImpl(sco, config); + } + if (configurationSource == ConfigurationSource.AGENTLESS) { + return new AgentlessConfigurationSource(config); } + LOGGER.debug( + "Feature Flagging offline configuration source selected; no config service started"); + return null; } public static void stop() { diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index e1608757a23..ea367b87741 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -2,6 +2,7 @@ import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -109,6 +110,16 @@ void offlineConfigurationSourceDoesNotStartNetworkSource() { sharedCommunicationObjects(), Config.get())); } + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void startWithOfflineConfigurationSourceSkipsConfigService() { + try { + assertDoesNotThrow(() -> FeatureFlaggingSystem.start(sharedCommunicationObjects())); + } finally { + FeatureFlaggingSystem.stop(); + } + } + private static SharedCommunicationObjects sharedCommunicationObjects() { DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); when(discovery.supportsEvpProxy()).thenReturn(true); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 20df2941ec2..2d5a42b08da 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -113,7 +113,7 @@ private void pollOnceSafely() { } private boolean fetchAndApply() { - for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + for (int attempt = 1; ; attempt++) { try { final UfcHttpResponse response = client.fetch(endpoint, config, etag); if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { @@ -127,7 +127,6 @@ private boolean fetchAndApply() { } } } - return false; } private boolean apply(final UfcHttpResponse response) { @@ -220,10 +219,10 @@ static final class UfcHttpResponse { } } - private static final class OkHttpUfcHttpClient implements UfcHttpClient { + static final class OkHttpUfcHttpClient implements UfcHttpClient { private final OkHttpClient httpClient; - private OkHttpUfcHttpClient(final OkHttpClient httpClient) { + OkHttpUfcHttpClient(final OkHttpClient httpClient) { this.httpClient = httpClient; } @@ -237,15 +236,12 @@ public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final final Request request = prepareRequest(endpoint, headers, config, true).get().build(); try (Response response = httpClient.newCall(request).execute()) { final ResponseBody responseBody = response.body(); - return new UfcHttpResponse( - response.code(), - response.header("ETag"), - responseBody == null ? null : responseBody.bytes()); + return new UfcHttpResponse(response.code(), response.header("ETag"), responseBody.bytes()); } } } - private static final class UfcHttpThreadFactory implements ThreadFactory { + static final class UfcHttpThreadFactory implements ThreadFactory { @Override public Thread newThread(final Runnable runnable) { final Thread thread = new Thread(runnable, "dd-feature-flagging-http-poller"); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index d5410634e37..6e5f13ddc7d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -4,17 +4,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; @@ -28,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,6 +41,7 @@ @ExtendWith(MockitoExtension.class) class AgentlessConfigurationSourceTest { + private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/server-distribution"; @Mock private FeatureFlaggingGateway.ConfigListener listener; @@ -54,6 +60,97 @@ void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { AgentlessConfigurationSource.endpoint(config).toString()); } + @Test + void derivesDatadogApiServerDistributionEndpointWithoutEnv() { + assertEquals( + "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config("datadoghq.com", "")).toString()); + assertEquals( + "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); + } + + @Test + void rejectsInvalidDatadogApiServerDistributionEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> AgentlessConfigurationSource.endpoint(config("datadoghq.com:bad", ""))); + } + + @Test + void defaultConstructorBuildsHttpClientFromConfig() { + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource(config("datad0g.com", "staging")); + + service.close(); + } + + @Test + void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> + api.getResponse() + .addHeader("ETag", "etag-b") + .send(emptyConfig()))))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), "etag-a"); + + assertEquals(HttpURLConnection.HTTP_OK, response.status); + assertEquals("etag-b", response.etag); + assertEquals(emptyConfig(), new String(response.body, UTF_8)); + assertEquals("test-api-key", server.getLastRequest().getHeader("DD-API-KEY")); + assertEquals("etag-a", server.getLastRequest().getHeader("If-None-Match")); + assertEquals("java", server.getLastRequest().getHeader("Datadog-Meta-Lang")); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> + api.getResponse() + .status(HttpURLConnection.HTTP_NO_CONTENT) + .send())))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null); + + assertEquals(HttpURLConnection.HTTP_NO_CONTENT, response.status); + assertNull(response.etag); + assertEquals(0, response.body.length); + assertNull(server.getLastRequest().getHeader("If-None-Match")); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + @Test void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); @@ -67,6 +164,20 @@ void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { assertNull(client.requests.get(0).etag); } + @Test + void ignoresBlankEtag() throws Exception { + final FakeClient client = + new FakeClient(response(200, " ", emptyConfig()), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + verify(listener).accept(any(ServerConfiguration.class)); + } + @Test void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { final FakeClient client = @@ -98,6 +209,27 @@ void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { verifyNoInteractions(listener); } + @Test + void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { + final FakeClient client = + new FakeClient( + response(403, null, null), + response(404, null, null), + response(600, null, null), + response(200, null, null), + response(200, null, "null")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verifyNoInteractions(listener); + } + @Test void retriesTimeoutBeforeApplyingConfig() throws Exception { final FakeClient client = @@ -114,6 +246,24 @@ void retriesTimeoutBeforeApplyingConfig() throws Exception { verify(listener).accept(any(ServerConfiguration.class)); } + @Test + void retriesClientTimeoutAndRateLimitStatusBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + response(408, null, null), + response(200, "etag-a", emptyConfig()), + response(429, null, null), + response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertEquals(4, client.calls.get()); + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + } + @Test void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); @@ -126,6 +276,36 @@ void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { verifyNoInteractions(listener); } + @Test + void givesUpAfterRetryableFailuresAreExhausted() throws Exception { + final FakeClient client = + new FakeClient( + response(503, null, null), response(503, null, null), response(503, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void givesUpAfterIoFailuresAreExhausted() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verifyNoInteractions(listener); + } + @Test void rejectsOverlappingPolls() throws Exception { final CountDownLatch requestStarted = new CountDownLatch(1); @@ -150,6 +330,25 @@ void rejectsOverlappingPolls() throws Exception { } } + @Test + void initSchedulesPollAndCloseCancelsFuture() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 1, + client, + Executors.newSingleThreadScheduledExecutor()); + FeatureFlaggingGateway.addConfigListener(listener); + + service.init(); + awaitCalls(client, 1); + service.close(); + + verify(listener).accept(any(ServerConfiguration.class)); + } + @Test void closePreventsFurtherPolls() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); @@ -161,14 +360,34 @@ void closePreventsFurtherPolls() throws Exception { assertEquals(0, client.calls.get()); } + @Test + void initAfterCloseDoesNotSchedulePoll() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.close(); + service.init(); + + assertEquals(0, client.calls.get()); + } + + @Test + void threadFactoryCreatesDaemonPollerThread() { + final Thread thread = + new AgentlessConfigurationSource.UfcHttpThreadFactory().newThread(() -> {}); + + assertEquals("dd-feature-flagging-http-poller", thread.getName()); + assertTrue(thread.isDaemon()); + } + private static AgentlessConfigurationSource service(final FakeClient client) { final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); return new AgentlessConfigurationSource( - HttpUrl.get("http://localhost/api/v2/feature-flagging/config/server-distribution"), - config("datadoghq.com", ""), - 60_000, - client, - executor); + HttpUrl.get("http://localhost" + CONFIG_PATH), config(), 60_000, client, executor); + } + + private static Config config() { + return config("datadoghq.com", ""); } private static Config config(final String site, final String env) { @@ -200,6 +419,16 @@ private static String emptyConfig() { + "}"; } + private static void awaitCalls(final FakeClient client, final int count) throws Exception { + for (int i = 0; i < 100; i++) { + if (client.calls.get() >= count) { + return; + } + TimeUnit.MILLISECONDS.sleep(10); + } + assertEquals(count, client.calls.get()); + } + private static final class FakeClient implements AgentlessConfigurationSource.UfcHttpClient { private final AtomicInteger calls = new AtomicInteger(); private final List requests = new ArrayList<>(); @@ -233,6 +462,9 @@ public AgentlessConfigurationSource.UfcHttpResponse fetch( if (response instanceof IOException) { throw (IOException) response; } + if (response instanceof RuntimeException) { + throw (RuntimeException) response; + } return (AgentlessConfigurationSource.UfcHttpResponse) response; } From 814dfbfc8d9e5cd75b117656c472c4f3fac1253d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 17:57:34 -0400 Subject: [PATCH 34/37] Add custom endpoint for agentless feature flags --- .../api/config/FeatureFlaggingConfig.java | 2 ++ .../main/java/datadog/trace/api/Config.java | 11 ++++++ metadata/supported-configurations.json | 8 +++++ .../feature-flagging-api/README.md | 10 ++++-- .../AgentlessConfigurationSource.java | 22 +++++++++++- .../AgentlessConfigurationSourceTest.java | 36 +++++++++++++++++++ 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 9fbc047b434..05d53a0c2ad 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -6,6 +6,8 @@ public class FeatureFlaggingConfig { public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = "feature.flags.configuration.source"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = + "feature.flags.configuration.source.agentless.base.url"; public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS = "feature.flags.configuration.source.agentless.poll.interval.seconds"; public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index acf383fbffe..1a58083dc90 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -365,6 +365,7 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; @@ -1220,6 +1221,7 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; private final String featureFlaggingConfigurationSource; + private final String featureFlaggingConfigurationSourceAgentlessBaseUrl; private final double featureFlaggingConfigurationSourcePollIntervalSeconds; private final double featureFlaggingConfigurationSourceRequestTimeoutSeconds; @@ -2849,6 +2851,9 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) normalizeFeatureFlaggingConfigurationSource( configProvider.getString( FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); + featureFlaggingConfigurationSourceAgentlessBaseUrl = + configProvider.getStringNotEmpty( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); featureFlaggingConfigurationSourcePollIntervalSeconds = configProvider.getDouble( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, @@ -4689,6 +4694,10 @@ public String getFeatureFlaggingConfigurationSource() { return featureFlaggingConfigurationSource; } + public String getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() { + return featureFlaggingConfigurationSourceAgentlessBaseUrl; + } + public double getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { return featureFlaggingConfigurationSourcePollIntervalSeconds; } @@ -6531,6 +6540,8 @@ public String toString() { + remoteConfigIntegrityCheckEnabled + ", featureFlaggingConfigurationSource=" + featureFlaggingConfigurationSource + + ", featureFlaggingConfigurationSourceAgentlessBaseUrl=" + + featureFlaggingConfigurationSourceAgentlessBaseUrl + ", featureFlaggingConfigurationSourcePollIntervalSeconds=" + featureFlaggingConfigurationSourcePollIntervalSeconds + ", featureFlaggingConfigurationSourceRequestTimeoutSeconds=" diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index f2c62bc6eff..534bddee26e 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1505,6 +1505,14 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index 1eedec6eae0..ca9dc9e7d0a 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -86,6 +86,10 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc - Java 11+ - `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless` uses the Datadog agentless - backend. `remote_config` uses the existing Agent Remote Configuration path. - `offline` is reserved for startup-provided UFC bytes; until those bytes are - implemented, no network source starts and evaluations use defaults. + backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a + different HTTP backend while keeping agentless delivery semantics. A bare + host uses the standard server-distribution path; a URL with a path is used as + the exact UFC endpoint. `remote_config` uses the existing Agent Remote + Configuration path. `offline` is reserved for startup-provided UFC bytes; + until those bytes are implemented, no network source starts and evaluations + use defaults. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 2d5a42b08da..9d77e4fdd0b 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -170,7 +170,11 @@ private void updateEtag(final String nextEtag) { } static HttpUrl endpoint(final Config config) { - final String endpoint = datadogApiServerDistributionEndpoint(config); + final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl(); + final String endpoint = + configuredBaseUrl == null + ? datadogApiServerDistributionEndpoint(config) + : endpointFromConfiguredBaseUrl(configuredBaseUrl); final HttpUrl parsed = HttpUrl.parse(endpoint); if (parsed == null) { throw new IllegalArgumentException( @@ -179,6 +183,22 @@ static HttpUrl endpoint(final Config config) { return parsed; } + private static String endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { + final HttpUrl parsed = HttpUrl.parse(configuredBaseUrl.trim()); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + configuredBaseUrl); + } + if ("/".equals(parsed.encodedPath()) || parsed.encodedPath().isEmpty()) { + return parsed + .newBuilder() + .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) + .build() + .toString(); + } + return parsed.toString(); + } + private static String datadogApiServerDistributionEndpoint(final Config config) { final StringBuilder endpoint = new StringBuilder("https://api.") diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 6e5f13ddc7d..1291b616058 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -70,6 +70,41 @@ void derivesDatadogApiServerDistributionEndpointWithoutEnv() { AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); } + @Test + void appendsServerDistributionPathToConfiguredAgentlessBaseUrl() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://mock-backend:8080"); + + assertEquals( + "http://mock-backend:8080/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void usesConfiguredAgentlessEndpointWithPathUnchanged() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://mock-backend:8080/custom/ufc?tenant=test"); + + assertEquals( + "http://mock-backend:8080/custom/ufc?tenant=test", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void rejectsInvalidConfiguredAgentlessBaseUrl() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("not a URL"); + + assertThrows( + IllegalArgumentException.class, () -> AgentlessConfigurationSource.endpoint(config)); + } + @Test void rejectsInvalidDatadogApiServerDistributionEndpoint() { assertThrows( @@ -398,6 +433,7 @@ private static Config config(final String site, final String env) { lenient() .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) .thenReturn(2.0D); + lenient().when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()).thenReturn(null); lenient().when(config.getApiKey()).thenReturn("test-api-key"); lenient().when(config.getSite()).thenReturn(site); lenient().when(config.getEnv()).thenReturn(env); From 91a36947ba1e2b638eca2d4a0dc215c93019fa1b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 18:32:49 -0400 Subject: [PATCH 35/37] Use integer agentless polling durations --- .../main/java/datadog/trace/api/ConfigDefaults.java | 7 +++---- .../src/main/java/datadog/trace/api/Config.java | 12 ++++++------ metadata/supported-configurations.json | 4 ++-- .../featureflag/AgentlessConfigurationSource.java | 4 ++-- .../AgentlessConfigurationSourceTest.java | 4 ++-- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index c3c46201c69..62f254ee2ec 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -50,10 +50,9 @@ public final class ConfigDefaults { static final String DEFAULT_SITE = "datadoghq.com"; public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; - public static final double DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = - 30.0D; - public static final double DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = - 2.0D; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = + 2; static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 1a58083dc90..fb147ac363a 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -1222,8 +1222,8 @@ public static String getHostName() { private final String featureFlaggingConfigurationSource; private final String featureFlaggingConfigurationSourceAgentlessBaseUrl; - private final double featureFlaggingConfigurationSourcePollIntervalSeconds; - private final double featureFlaggingConfigurationSourceRequestTimeoutSeconds; + private final int featureFlaggingConfigurationSourcePollIntervalSeconds; + private final int featureFlaggingConfigurationSourceRequestTimeoutSeconds; private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; @@ -2855,11 +2855,11 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getStringNotEmpty( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); featureFlaggingConfigurationSourcePollIntervalSeconds = - configProvider.getDouble( + configProvider.getInteger( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); featureFlaggingConfigurationSourceRequestTimeoutSeconds = - configProvider.getDouble( + configProvider.getInteger( FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); @@ -4698,11 +4698,11 @@ public String getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() { return featureFlaggingConfigurationSourceAgentlessBaseUrl; } - public double getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { + public int getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { return featureFlaggingConfigurationSourcePollIntervalSeconds; } - public double getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { + public int getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { return featureFlaggingConfigurationSourceRequestTimeoutSeconds; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 534bddee26e..74b6ba24df8 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1516,7 +1516,7 @@ "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ { "version": "A", - "type": "decimal", + "type": "int", "default": "30", "aliases": [] } @@ -1524,7 +1524,7 @@ "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ { "version": "A", - "type": "decimal", + "type": "int", "default": "2", "aliases": [] } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 9d77e4fdd0b..f6d3c1d65aa 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -219,8 +219,8 @@ private static String urlEncode(final String value) { } } - static long millis(final double seconds) { - return Math.max(1L, Math.round(seconds * 1000)); + static long millis(final int seconds) { + return Math.max(1L, seconds * 1000L); } interface UfcHttpClient { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 1291b616058..234407b8ccc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -429,10 +429,10 @@ private static Config config(final String site, final String env) { final Config config = mock(Config.class); lenient() .when(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()) - .thenReturn(30.0D); + .thenReturn(30); lenient() .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) - .thenReturn(2.0D); + .thenReturn(2); lenient().when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()).thenReturn(null); lenient().when(config.getApiKey()).thenReturn("test-api-key"); lenient().when(config.getSite()).thenReturn(site); From 1858f3256ea0023692e1ab6ea34d835db3347d67 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 18:40:09 -0400 Subject: [PATCH 36/37] Fix agentless duration formatting --- .../src/main/java/datadog/trace/api/ConfigDefaults.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 62f254ee2ec..532e9f47d71 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -51,8 +51,7 @@ public final class ConfigDefaults { public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; - public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = - 2; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; From ff586da6124e0df6bedaf0be7b1f25cfa7ffee85 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:35:32 -0400 Subject: [PATCH 37/37] fix(openfeature): serialize instant context values --- .../main/java/datadog/trace/api/openfeature/DDEvaluator.java | 2 +- .../java/datadog/trace/api/openfeature/DDEvaluatorTest.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 0a137cc5669..a871d68de5e 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -607,7 +607,7 @@ private static Object convertValue(final Value value) { } else if (value.isString()) { return value.asString(); } else if (value.isInstant()) { - return value.asInstant(); + return value.asInstant().toString(); } throw new IllegalArgumentException("Unsupported OpenFeature value type: " + value); } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index a4813ad8091..da02bf72706 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -36,6 +36,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Date; @@ -213,6 +214,8 @@ private static Arguments[] flatteningTestCases() { arguments.add( Arguments.of( mapOf("plan", "gold", "cohort", "gold"), mapOf("plan", "gold", "cohort", "gold"))); + final Instant instant = Instant.parse("2026-07-10T12:34:56Z"); + arguments.add(Arguments.of(mapOf("instant", instant), mapOf("instant", instant.toString()))); return arguments.toArray(new Arguments[0]); }