diff --git a/CHANGELOG.next-release.md b/CHANGELOG.next-release.md
index 6850e010886..662320a0d94 100644
--- a/CHANGELOG.next-release.md
+++ b/CHANGELOG.next-release.md
@@ -13,6 +13,7 @@ This file contains all changes which are not released yet.
# Features and enhancements
+- Add experimental auto-instrumentation for Spring AI 2.x chat models, embedding models, vector stores, and tool callbacks, with opt-in AI content capture
# Deprecations
diff --git a/apm-agent-builds/pom.xml b/apm-agent-builds/pom.xml
index 50948a497ba..dd395214543 100644
--- a/apm-agent-builds/pom.xml
+++ b/apm-agent-builds/pom.xml
@@ -346,6 +346,11 @@
apm-sparkjava-plugin
${project.version}
+
+ ${project.groupId}
+ apm-spring-ai-plugin
+ ${project.version}
+
${project.groupId}
apm-spring-resttemplate-plugin
diff --git a/apm-agent-common/src/test/java/co/elastic/apm/agent/test/AgentTestContainer.java b/apm-agent-common/src/test/java/co/elastic/apm/agent/test/AgentTestContainer.java
index e73a391afe1..251333b2468 100644
--- a/apm-agent-common/src/test/java/co/elastic/apm/agent/test/AgentTestContainer.java
+++ b/apm-agent-common/src/test/java/co/elastic/apm/agent/test/AgentTestContainer.java
@@ -212,7 +212,7 @@ public AppServer withHttpPort(int port) {
}
public String getBaseUrl() {
- return String.format("http://%s:%d", getContainerIpAddress(), getMappedPort(webPort));
+ return String.format("http://%s:%d", getHost(), getMappedPort(webPort));
}
public int getHttpPort() {
diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfigurationImpl.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfigurationImpl.java
index d73e4185dd9..00e07a73d1e 100644
--- a/apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfigurationImpl.java
+++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfigurationImpl.java
@@ -261,6 +261,7 @@ public class CoreConfigurationImpl extends ConfigurationOptionProvider implement
"- `transaction.context.request.body`, `error.context.request.body`\n" +
"- `transaction.context.message.body`, `error.context.message.body`\n" +
"- `span.context.db.statement`\n" +
+ "- captured AI content values\n" +
"\nNote that tracing data is limited at the upstream APM server to \n" +
"{apm-guide-ref}/configuration-process.html#max_event_size[`max_event_size`], \n" +
"which defaults to 300kB. If you configure `long_field_max_length` too large, it \n" +
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/pom.xml b/apm-agent-plugins/apm-spring-ai-plugin/pom.xml
new file mode 100644
index 00000000000..7a987240a0e
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/pom.xml
@@ -0,0 +1,80 @@
+
+
+ 4.0.0
+
+
+ co.elastic.apm
+ apm-agent-plugins
+ 1.57.2-SNAPSHOT
+
+
+ apm-spring-ai-plugin
+ ${project.groupId}:${project.artifactId}
+
+
+ ${project.basedir}/../..
+ true
+ 8
+ 8
+ 2.0.2
+ 3.8.8
+ 2.24.3
+
+
+
+
+ ${project.groupId}
+ apm-agent-core
+ ${project.version}
+
+
+ ${project.groupId}
+ apm-opentelemetry-embedded-metrics-sdk
+ ${project.version}
+
+
+ org.springframework.ai
+ spring-ai-model
+ ${version.spring-ai}
+ provided
+
+
+ org.springframework.ai
+ spring-ai-vector-store
+ ${version.spring-ai}
+ provided
+
+
+ io.projectreactor
+ reactor-core
+ ${version.reactor}
+ provided
+
+
+ ${project.groupId}
+ apm-opentelemetry-metricsdk-plugin
+ ${project.version}
+ test
+
+
+ org.apache.ivy
+ ivy
+ test
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+ ${version.log4j2-test}
+ test
+
+
+ org.apache.logging.log4j
+ log4j-core
+ ${version.log4j2-test}
+ test
+
+
+
+
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/AbstractSpringAiInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/AbstractSpringAiInstrumentation.java
new file mode 100644
index 00000000000..2d06a2adb78
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/AbstractSpringAiInstrumentation.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.sdk.ElasticApmInstrumentation;
+import net.bytebuddy.matcher.ElementMatcher;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static co.elastic.apm.agent.sdk.bytebuddy.CustomElementMatchers.classLoaderCanLoadClass;
+
+abstract class AbstractSpringAiInstrumentation extends ElasticApmInstrumentation {
+
+ @Override
+ public ElementMatcher.Junction getClassLoaderMatcher() {
+ return classLoaderCanLoadClass("org.springframework.ai.model.Model")
+ .and(classLoaderCanLoadClass("org.springframework.ai.observation.conventions.AiOperationType"))
+ // DefaultEmbeddingOptions.Builder was introduced in Spring AI 2.0.
+ .and(classLoaderCanLoadClass("org.springframework.ai.embedding.DefaultEmbeddingOptions$Builder"));
+ }
+
+ @Override
+ public Collection getInstrumentationGroupNames() {
+ return Arrays.asList("spring-ai", "experimental");
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/ChatModelInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/ChatModelInstrumentation.java
new file mode 100644
index 00000000000..1bef082a15e
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/ChatModelInstrumentation.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.tracer.Span;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.NamedElement;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.prompt.Prompt;
+
+import javax.annotation.Nullable;
+
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isInterface;
+import static net.bytebuddy.matcher.ElementMatchers.nameContains;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class ChatModelInstrumentation extends AbstractSpringAiInstrumentation {
+
+ @Override
+ public ElementMatcher super NamedElement> getTypeMatcherPreFilter() {
+ return nameContains("ChatModel");
+ }
+
+ @Override
+ public ElementMatcher super TypeDescription> getTypeMatcher() {
+ return not(isInterface()).and(hasSuperType(named("org.springframework.ai.chat.model.ChatModel")));
+ }
+
+ @Override
+ public ElementMatcher super MethodDescription> getMethodMatcher() {
+ return named("call").and(takesArguments(1))
+ .and(takesArgument(0, named("org.springframework.ai.chat.prompt.Prompt")));
+ }
+
+ @Override
+ public String getAdviceClassName() {
+ return getClass().getName() + "$ChatModelAdvice";
+ }
+
+ public static class ChatModelAdvice {
+
+ @Nullable
+ @Advice.OnMethodEnter(suppress = Throwable.class, inline = false)
+ public static Object onEnter(@Advice.Argument(0) Prompt prompt) {
+ Span> span = SpringAiSpanHelper.startModelSpan("chat", prompt, false);
+ return span == null ? null : span.activate();
+ }
+
+ @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false)
+ public static void onExit(@Advice.Enter @Nullable Object spanObject,
+ @Advice.Return @Nullable ChatResponse response,
+ @Advice.Thrown @Nullable Throwable thrown) {
+ SpringAiSpanHelper.endChatSpan((Span>) spanObject, response, thrown);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/EmbeddingModelInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/EmbeddingModelInstrumentation.java
new file mode 100644
index 00000000000..08c540a3aca
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/EmbeddingModelInstrumentation.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.tracer.Span;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.NamedElement;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.springframework.ai.embedding.EmbeddingRequest;
+import org.springframework.ai.embedding.EmbeddingResponse;
+
+import javax.annotation.Nullable;
+
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isInterface;
+import static net.bytebuddy.matcher.ElementMatchers.nameContains;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class EmbeddingModelInstrumentation extends AbstractSpringAiInstrumentation {
+
+ @Override
+ public ElementMatcher super NamedElement> getTypeMatcherPreFilter() {
+ return nameContains("Embedding");
+ }
+
+ @Override
+ public ElementMatcher super TypeDescription> getTypeMatcher() {
+ return not(isInterface()).and(hasSuperType(named("org.springframework.ai.embedding.EmbeddingModel")));
+ }
+
+ @Override
+ public ElementMatcher super MethodDescription> getMethodMatcher() {
+ return named("call").and(takesArguments(1))
+ .and(takesArgument(0, named("org.springframework.ai.embedding.EmbeddingRequest")));
+ }
+
+ @Override
+ public String getAdviceClassName() {
+ return getClass().getName() + "$EmbeddingModelAdvice";
+ }
+
+ public static class EmbeddingModelAdvice {
+
+ @Nullable
+ @Advice.OnMethodEnter(suppress = Throwable.class, inline = false)
+ public static Object onEnter(@Advice.Argument(0) EmbeddingRequest request) {
+ Span> span = SpringAiSpanHelper.startEmbeddingSpan(request);
+ return span == null ? null : span.activate();
+ }
+
+ @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false)
+ public static void onExit(@Advice.Enter @Nullable Object spanObject,
+ @Advice.Return @Nullable EmbeddingResponse response,
+ @Advice.Thrown @Nullable Throwable thrown) {
+ SpringAiSpanHelper.endEmbeddingSpan((Span>) spanObject, response, thrown);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiConfiguration.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiConfiguration.java
new file mode 100644
index 00000000000..64c43e353af
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiConfiguration.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import org.stagemonitor.configuration.ConfigurationOption;
+import org.stagemonitor.configuration.ConfigurationOptionProvider;
+
+public class SpringAiConfiguration extends ConfigurationOptionProvider {
+
+ private final ConfigurationOption captureSpringAiContent = ConfigurationOption.booleanOption()
+ .key("capture_spring_ai_content")
+ .configurationCategory("AI")
+ .tags("security", "performance", "experimental", "added[1.58.0]")
+ .description("Enables capture of Spring AI prompt and response messages, embedding inputs, vector store documents and queries, and tool arguments and results.\n\n" +
+ "Captured content can contain sensitive information and can significantly increase storage usage. Content capture is disabled by default. " +
+ "Individual captured content values are truncated according to `long_field_max_length`.")
+ .dynamic(true)
+ .buildWithDefault(false);
+
+ public boolean isCaptureSpringAiContent() {
+ return captureSpringAiContent.get();
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiMetrics.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiMetrics.java
new file mode 100644
index 00000000000..e10221cb733
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiMetrics.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.embeddedotel.EmbeddedSdkManager;
+import co.elastic.apm.agent.embeddedotel.proxy.ProxyAttributeKey;
+import co.elastic.apm.agent.embeddedotel.proxy.ProxyAttributes;
+import co.elastic.apm.agent.embeddedotel.proxy.ProxyAttributesBuilder;
+import co.elastic.apm.agent.embeddedotel.proxy.ProxyDoubleHistogram;
+import co.elastic.apm.agent.impl.ElasticApmTracer;
+import co.elastic.apm.agent.tracer.GlobalTracer;
+import co.elastic.apm.agent.tracer.Tracer;
+
+import javax.annotation.Nullable;
+
+final class SpringAiMetrics {
+
+ static final String TIME_TO_FIRST_CHUNK = "gen_ai.client.inference.time_to_first_chunk";
+
+ private static final Tracer tracer = GlobalTracer.get();
+ private static final ProxyAttributeKey OPERATION_NAME =
+ ProxyAttributeKey.stringKey("gen_ai.operation.name");
+ private static final ProxyAttributeKey REQUEST_MODEL =
+ ProxyAttributeKey.stringKey("gen_ai.request.model");
+ @Nullable
+ private static final ProxyDoubleHistogram timeToFirstChunk = createTimeToFirstChunkHistogram();
+
+ private SpringAiMetrics() {
+ }
+
+ static void recordTimeToFirstChunk(@Nullable String requestModel, double seconds) {
+ if (timeToFirstChunk == null) {
+ return;
+ }
+ timeToFirstChunk.record(seconds, createAttributes(requestModel));
+ }
+
+ private static ProxyAttributes createAttributes(@Nullable String requestModel) {
+ ProxyAttributesBuilder builder = ProxyAttributes.builder();
+ builder.put(OPERATION_NAME, "chat");
+ if (requestModel != null) {
+ builder.put(REQUEST_MODEL, requestModel);
+ }
+ return builder.build();
+ }
+
+ @Nullable
+ private static ProxyDoubleHistogram createTimeToFirstChunkHistogram() {
+ EmbeddedSdkManager sdkManager = tracer.require(ElasticApmTracer.class)
+ .getLifecycleListener(EmbeddedSdkManager.class);
+ if (sdkManager == null) {
+ return null;
+ }
+ return sdkManager.getMeterProvider()
+ .get("co.elastic.apm.spring-ai")
+ .histogramBuilder(TIME_TO_FIRST_CHUNK)
+ .setUnit("s")
+ .setDescription("Time to receive the first chunk of a streaming generative AI response")
+ .build();
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiSpanHelper.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiSpanHelper.java
new file mode 100644
index 00000000000..0c74332ba57
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/SpringAiSpanHelper.java
@@ -0,0 +1,475 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.tracer.GlobalTracer;
+import co.elastic.apm.agent.tracer.Outcome;
+import co.elastic.apm.agent.tracer.Span;
+import co.elastic.apm.agent.tracer.TraceState;
+import co.elastic.apm.agent.tracer.Tracer;
+import co.elastic.apm.agent.tracer.configuration.CoreConfiguration;
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.messages.Message;
+import org.springframework.ai.chat.messages.ToolResponseMessage;
+import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
+import org.springframework.ai.chat.metadata.ChatResponseMetadata;
+import org.springframework.ai.chat.metadata.Usage;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.model.MessageAggregator;
+import org.springframework.ai.chat.prompt.ChatOptions;
+import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.embedding.EmbeddingOptions;
+import org.springframework.ai.embedding.EmbeddingRequest;
+import org.springframework.ai.embedding.EmbeddingResponse;
+import org.springframework.ai.embedding.EmbeddingResponseMetadata;
+import org.springframework.ai.tool.ToolCallback;
+import org.springframework.ai.tool.definition.ToolDefinition;
+import org.springframework.ai.vectorstore.SearchRequest;
+import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
+import reactor.core.publisher.Flux;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+final class SpringAiSpanHelper {
+
+ private static final Tracer tracer = GlobalTracer.get();
+ private static final SpringAiConfiguration config = tracer.getConfig(SpringAiConfiguration.class);
+ private static final CoreConfiguration coreConfig = tracer.getConfig(CoreConfiguration.class);
+
+ private SpringAiSpanHelper() {
+ }
+
+ @Nullable
+ static Span> startModelSpan(String operation, Prompt prompt, boolean streaming) {
+ Span> span = tracer.currentContext().createSpan();
+ if (span == null) {
+ return null;
+ }
+
+ ChatOptions options = prompt.getOptions();
+ String requestModel = options == null ? null : options.getModel();
+ span.withName(spanName(operation, requestModel))
+ .withType("app")
+ .withSubtype("internal")
+ .withOtelAttribute("gen_ai.operation.name", operation)
+ .withOtelAttribute("gen_ai.request.model", requestModel)
+ .withOtelAttribute("gen_ai.request.stream", streaming);
+
+ if (options != null) {
+ span.withOtelAttribute("gen_ai.request.frequency_penalty", options.getFrequencyPenalty())
+ .withOtelAttribute("gen_ai.request.max_tokens", options.getMaxTokens())
+ .withOtelAttribute("gen_ai.request.presence_penalty", options.getPresencePenalty())
+ .withOtelAttribute("gen_ai.request.stop_sequences", stringify(options.getStopSequences()))
+ .withOtelAttribute("gen_ai.request.temperature", options.getTemperature())
+ .withOtelAttribute("gen_ai.request.top_k", options.getTopK())
+ .withOtelAttribute("gen_ai.request.top_p", options.getTopP());
+ }
+ if (config.isCaptureSpringAiContent()) {
+ captureStructuredContent(span, "gen_ai.input.messages", serializeMessages(prompt.getInstructions()));
+ }
+ return span;
+ }
+
+ @Nullable
+ static Span> startEmbeddingSpan(EmbeddingRequest request) {
+ EmbeddingOptions options = request.getOptions();
+ Span> span = tracer.currentContext().createSpan();
+ if (span == null) {
+ return null;
+ }
+ String requestModel = options == null ? null : options.getModel();
+ span.withName(spanName("embedding", requestModel))
+ .withType("app")
+ .withSubtype("internal")
+ .withOtelAttribute("gen_ai.operation.name", "embedding")
+ .withOtelAttribute("gen_ai.request.model", requestModel)
+ .withOtelAttribute("gen_ai.request.embedding.dimensions", options == null ? null : options.getDimensions());
+ if (config.isCaptureSpringAiContent()) {
+ captureStructuredContent(span, "spring.ai.embedding.request.inputs", serializeStrings(request.getInstructions()));
+ }
+ return span;
+ }
+
+ @Nullable
+ static Span> startVectorStoreSpan(AbstractObservationVectorStore vectorStore, String operation,
+ @Nullable Object argument) {
+ Span> span = tracer.currentContext().createExitSpan();
+ if (span == null) {
+ return null;
+ }
+ VectorStoreObservationContext.Builder builder = vectorStore.createObservationContextBuilder(operation);
+ SearchRequest searchRequest = argument instanceof SearchRequest ? (SearchRequest) argument : null;
+ if (searchRequest != null) {
+ builder.queryRequest(searchRequest);
+ }
+ VectorStoreObservationContext context = builder.build();
+ String dbSystem = context.getDatabaseSystem();
+ boolean hasDbSystem = dbSystem != null && !dbSystem.isEmpty();
+ span.withName((hasDbSystem ? dbSystem : "vector_store") + " " + operation)
+ .withType("db")
+ .withSubtype("unknown")
+ .withOtelAttribute("spring.ai.kind", "vector_store")
+ .withOtelAttribute("db.operation.name", operation)
+ .withOtelAttribute("db.collection.name", context.getCollectionName())
+ .withOtelAttribute("db.namespace", context.getNamespace())
+ .withOtelAttribute("db.search.similarity_metric", context.getSimilarityMetric())
+ .withOtelAttribute("db.vector.dimension_count", context.getDimensions())
+ .withOtelAttribute("db.vector.field_name", context.getFieldName());
+ if (searchRequest != null) {
+ span.withOtelAttribute("db.vector.query.top_k", searchRequest.getTopK())
+ .withOtelAttribute("db.vector.query.similarity_threshold", searchRequest.getSimilarityThreshold());
+ captureContent(span, "db.vector.query.content", searchRequest.getQuery());
+ if (searchRequest.getFilterExpression() != null) {
+ captureContent(span, "db.vector.query.filter", searchRequest.getFilterExpression().toString());
+ }
+ } else if ("add".equals(operation) && argument instanceof List) {
+ if (config.isCaptureSpringAiContent()) {
+ captureStructuredContent(span, "db.vector.documents", serializeDocuments((List>) argument));
+ }
+ }
+ if (hasDbSystem) {
+ span.withOtelAttribute("db.system", dbSystem);
+ span.getContext().getServiceTarget().withType(dbSystem).withName(context.getCollectionName());
+ }
+ return span;
+ }
+
+ @Nullable
+ static Span> startToolSpan(ToolCallback callback, String toolInput) {
+ Span> span = tracer.currentContext().createSpan();
+ if (span == null) {
+ return null;
+ }
+ ToolDefinition definition = callback.getToolDefinition();
+ String name = definition.name();
+ span.withName("execute_tool " + name)
+ .withType("app")
+ .withSubtype("internal")
+ .withOtelAttribute("gen_ai.operation.name", "execute_tool")
+ .withOtelAttribute("spring.ai.kind", "tool_call")
+ .withOtelAttribute("spring.ai.tool.type", "function")
+ .withOtelAttribute("spring.ai.tool.definition.name", name)
+ .withOtelAttribute("spring.ai.tool.definition.description", definition.description())
+ .withOtelAttribute("spring.ai.tool.definition.schema", definition.inputSchema());
+ captureContent(span, "spring.ai.tool.call.arguments", toolInput);
+ return span;
+ }
+
+ @SuppressWarnings("unchecked")
+ static Flux> wrapChatFlux(final Prompt prompt, final Flux> source) {
+ final Flux typedSource = (Flux) source;
+ return Flux.defer(() -> {
+ final long startNanos = System.nanoTime();
+ final Span> span = startModelSpan("chat", prompt, true);
+ final AtomicBoolean firstChunkRecorded = new AtomicBoolean();
+ final Flux timedSource = typedSource.doOnNext(value -> {
+ if (firstChunkRecorded.compareAndSet(false, true)) {
+ double elapsedSeconds = Math.max(0, System.nanoTime() - startNanos) / 1_000_000_000d;
+ if (span != null) {
+ span.withOtelAttribute("gen_ai.response.time_to_first_chunk", elapsedSeconds);
+ }
+ ChatOptions options = prompt.getOptions();
+ SpringAiMetrics.recordTimeToFirstChunk(
+ options == null ? null : options.getModel(), elapsedSeconds);
+ }
+ });
+ if (span == null) {
+ return timedSource;
+ }
+ final AtomicReference lastResponse = new AtomicReference<>();
+ final AtomicReference aggregatedResponse = new AtomicReference<>();
+ final AtomicReference failure = new AtomicReference<>();
+ final AtomicBoolean ended = new AtomicBoolean();
+ span.withSync(false);
+ Flux observedSource = config.isCaptureSpringAiContent()
+ ? new MessageAggregator().aggregate(timedSource, aggregatedResponse::set)
+ : timedSource;
+ return observedSource
+ .doOnNext(lastResponse::set)
+ .doOnError(failure::set)
+ .doFinally(signal -> {
+ if (ended.compareAndSet(false, true)) {
+ ChatResponse response = aggregatedResponse.get();
+ addChatResponseAttributes(span, response == null ? lastResponse.get() : response);
+ endAsyncSpan(span, failure.get());
+ }
+ })
+ .contextWrite(context -> context.put(TraceState.class, span));
+ });
+ }
+
+ static void endChatSpan(@Nullable Span> span, @Nullable ChatResponse response, @Nullable Throwable thrown) {
+ if (span == null) {
+ return;
+ }
+ addChatResponseAttributes(span, response);
+ endSpan(span, thrown);
+ }
+
+ private static void addChatResponseAttributes(Span> span, @Nullable ChatResponse response) {
+ if (response != null) {
+ ChatResponseMetadata metadata = response.getMetadata();
+ if (metadata != null) {
+ span.withOtelAttribute("gen_ai.response.id", metadata.getId())
+ .withOtelAttribute("gen_ai.response.model", metadata.getModel());
+ addUsage(span, metadata.getUsage());
+ }
+ List finishReasons = new ArrayList<>();
+ for (Generation generation : response.getResults()) {
+ ChatGenerationMetadata generationMetadata = generation.getMetadata();
+ if (generationMetadata != null && generationMetadata.getFinishReason() != null) {
+ finishReasons.add(generationMetadata.getFinishReason());
+ }
+ }
+ span.withOtelAttribute("gen_ai.response.finish_reasons", stringify(finishReasons));
+ if (config.isCaptureSpringAiContent()) {
+ captureStructuredContent(span, "gen_ai.output.messages", serializeGenerations(response.getResults()));
+ }
+ }
+ }
+
+ static void endVectorStoreSpan(@Nullable Span> span, @Nullable Object response, @Nullable Throwable thrown) {
+ if (span != null && response instanceof List) {
+ if (config.isCaptureSpringAiContent()) {
+ captureStructuredContent(span, "db.vector.query.response.documents", serializeDocuments((List>) response));
+ }
+ }
+ endSpan(span, thrown);
+ }
+
+ static void endToolSpan(@Nullable Span> span, @Nullable String result, @Nullable Throwable thrown) {
+ if (span != null) {
+ captureContent(span, "spring.ai.tool.call.result", result);
+ }
+ endSpan(span, thrown);
+ }
+
+ static void endEmbeddingSpan(@Nullable Span> span, @Nullable EmbeddingResponse response,
+ @Nullable Throwable thrown) {
+ if (span != null && response != null) {
+ EmbeddingResponseMetadata metadata = response.getMetadata();
+ if (metadata != null) {
+ span.withOtelAttribute("gen_ai.response.model", metadata.getModel());
+ addUsage(span, metadata.getUsage());
+ }
+ }
+ endSpan(span, thrown);
+ }
+
+ static void endSpan(@Nullable Span> span, @Nullable Throwable thrown) {
+ if (span != null) {
+ span.captureException(thrown)
+ .withOutcome(thrown == null ? Outcome.SUCCESS : Outcome.FAILURE)
+ .deactivate()
+ .end();
+ }
+ }
+
+ private static void endAsyncSpan(Span> span, @Nullable Throwable thrown) {
+ span.captureException(thrown)
+ .withOutcome(thrown == null ? Outcome.SUCCESS : Outcome.FAILURE)
+ .end();
+ }
+
+ private static void addUsage(Span> span, @Nullable Usage usage) {
+ if (usage != null) {
+ span.withOtelAttribute("gen_ai.usage.input_tokens", usage.getPromptTokens())
+ .withOtelAttribute("gen_ai.usage.output_tokens", usage.getCompletionTokens())
+ .withOtelAttribute("gen_ai.usage.total_tokens", usage.getTotalTokens())
+ .withOtelAttribute("gen_ai.usage.cache_read.input_tokens", usage.getCacheReadInputTokens())
+ .withOtelAttribute("gen_ai.usage.cache_creation.input_tokens", usage.getCacheWriteInputTokens());
+ }
+ }
+
+ private static String spanName(String operation, @Nullable String model) {
+ return model == null || model.isEmpty() ? operation : operation + " " + model;
+ }
+
+ @Nullable
+ private static String stringify(@Nullable List> values) {
+ return values == null || values.isEmpty() ? null : values.toString();
+ }
+
+ private static void captureContent(Span> span, String key, @Nullable String content) {
+ if (config.isCaptureSpringAiContent() && content != null) {
+ span.withOtelAttribute(key, truncate(content, coreConfig.getLongFieldMaxLength()));
+ }
+ }
+
+ private static void captureStructuredContent(Span> span, String key, String content) {
+ span.withOtelAttribute(key, content);
+ }
+
+ private static String truncate(String value, int maxCodePoints) {
+ if (maxCodePoints < 0 || value.codePointCount(0, value.length()) <= maxCodePoints) {
+ return value;
+ }
+ return value.substring(0, value.offsetByCodePoints(0, maxCodePoints));
+ }
+
+ private static String serializeMessages(List extends Message> messages) {
+ StringBuilder result = new StringBuilder("[");
+ for (int i = 0; i < messages.size(); i++) {
+ if (i > 0) {
+ result.append(',');
+ }
+ Message message = messages.get(i);
+ result.append("{\"role\":");
+ appendJsonString(result, message.getMessageType().getValue());
+ result.append(",\"parts\":[");
+ boolean partAdded = false;
+ String text = message.getText();
+ if (text != null && !text.isEmpty()) {
+ appendTextPart(result, text);
+ partAdded = true;
+ }
+ if (message instanceof AssistantMessage) {
+ for (AssistantMessage.ToolCall toolCall : ((AssistantMessage) message).getToolCalls()) {
+ appendPartSeparator(result, partAdded);
+ appendToolCallPart(result, toolCall);
+ partAdded = true;
+ }
+ }
+ if (message instanceof ToolResponseMessage) {
+ for (ToolResponseMessage.ToolResponse response : ((ToolResponseMessage) message).getResponses()) {
+ appendPartSeparator(result, partAdded);
+ appendToolResponsePart(result, response);
+ partAdded = true;
+ }
+ }
+ if (!partAdded) {
+ appendTextPart(result, text);
+ }
+ result.append("]}");
+ }
+ return result.append(']').toString();
+ }
+
+ private static void appendPartSeparator(StringBuilder target, boolean partAdded) {
+ if (partAdded) {
+ target.append(',');
+ }
+ }
+
+ private static void appendTextPart(StringBuilder target, @Nullable String content) {
+ target.append("{\"type\":\"text\",\"content\":");
+ appendJsonString(target, content);
+ target.append('}');
+ }
+
+ private static void appendToolCallPart(StringBuilder target, AssistantMessage.ToolCall toolCall) {
+ target.append("{\"type\":\"tool_call\",\"id\":");
+ appendJsonString(target, toolCall.id());
+ target.append(",\"name\":");
+ appendJsonString(target, toolCall.name());
+ target.append(",\"arguments\":");
+ appendJsonString(target, toolCall.arguments());
+ target.append('}');
+ }
+
+ private static void appendToolResponsePart(StringBuilder target,
+ ToolResponseMessage.ToolResponse response) {
+ target.append("{\"type\":\"tool_call_response\",\"id\":");
+ appendJsonString(target, response.id());
+ target.append(",\"name\":");
+ appendJsonString(target, response.name());
+ target.append(",\"response\":");
+ appendJsonString(target, response.responseData());
+ target.append('}');
+ }
+
+ private static String serializeGenerations(List generations) {
+ List messages = new ArrayList<>();
+ for (Generation generation : generations) {
+ if (generation.getOutput() != null) {
+ messages.add(generation.getOutput());
+ }
+ }
+ return serializeMessages(messages);
+ }
+
+ private static String serializeStrings(List values) {
+ StringBuilder result = new StringBuilder("[");
+ for (int i = 0; i < values.size(); i++) {
+ if (i > 0) {
+ result.append(',');
+ }
+ appendJsonString(result, values.get(i));
+ }
+ return result.append(']').toString();
+ }
+
+ private static String serializeDocuments(List> values) {
+ StringBuilder result = new StringBuilder("[");
+ boolean first = true;
+ for (Object value : values) {
+ if (!(value instanceof Document)) {
+ continue;
+ }
+ if (!first) {
+ result.append(',');
+ }
+ first = false;
+ Document document = (Document) value;
+ result.append("{\"id\":");
+ appendJsonString(result, document.getId());
+ result.append(",\"content\":");
+ appendJsonString(result, document.getText());
+ result.append('}');
+ }
+ return result.append(']').toString();
+ }
+
+ private static void appendJsonString(StringBuilder target, @Nullable String value) {
+ if (value == null) {
+ target.append("null");
+ return;
+ }
+ value = truncate(value, coreConfig.getLongFieldMaxLength());
+ target.append('"');
+ for (int i = 0; i < value.length(); i++) {
+ char c = value.charAt(i);
+ switch (c) {
+ case '"': target.append("\\\""); break;
+ case '\\': target.append("\\\\"); break;
+ case '\b': target.append("\\b"); break;
+ case '\f': target.append("\\f"); break;
+ case '\n': target.append("\\n"); break;
+ case '\r': target.append("\\r"); break;
+ case '\t': target.append("\\t"); break;
+ default:
+ if (c < 0x20) {
+ target.append(String.format(Locale.ROOT, "\\u%04x", (int) c));
+ } else {
+ target.append(c);
+ }
+ }
+ }
+ target.append('"');
+ }
+
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/StreamingChatModelInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/StreamingChatModelInstrumentation.java
new file mode 100644
index 00000000000..c0d9d84f472
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/StreamingChatModelInstrumentation.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.NamedElement;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.springframework.ai.chat.prompt.Prompt;
+import reactor.core.publisher.Flux;
+
+import javax.annotation.Nullable;
+
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isInterface;
+import static net.bytebuddy.matcher.ElementMatchers.nameContains;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class StreamingChatModelInstrumentation extends AbstractSpringAiInstrumentation {
+
+ @Override
+ public ElementMatcher super NamedElement> getTypeMatcherPreFilter() {
+ return nameContains("ChatModel");
+ }
+
+ @Override
+ public ElementMatcher super TypeDescription> getTypeMatcher() {
+ return not(isInterface()).and(hasSuperType(named("org.springframework.ai.chat.model.StreamingChatModel")));
+ }
+
+ @Override
+ public ElementMatcher super MethodDescription> getMethodMatcher() {
+ return named("stream").and(takesArguments(1))
+ .and(takesArgument(0, named("org.springframework.ai.chat.prompt.Prompt")));
+ }
+
+ @Override
+ public String getAdviceClassName() {
+ return getClass().getName() + "$StreamingChatModelAdvice";
+ }
+
+ public static class StreamingChatModelAdvice {
+ @Nullable
+ @Advice.AssignReturned.ToReturned
+ @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false)
+ public static Flux> onExit(@Advice.Argument(0) Prompt prompt,
+ @Advice.Return @Nullable Flux> flux,
+ @Advice.Thrown @Nullable Throwable thrown) {
+ if (thrown == null && flux != null) {
+ return SpringAiSpanHelper.wrapChatFlux(prompt, flux);
+ }
+ return flux;
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/ToolCallbackInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/ToolCallbackInstrumentation.java
new file mode 100644
index 00000000000..0a0e0605cda
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/ToolCallbackInstrumentation.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.tracer.Span;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.NamedElement;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.springframework.ai.tool.ToolCallback;
+
+import javax.annotation.Nullable;
+
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isInterface;
+import static net.bytebuddy.matcher.ElementMatchers.nameContains;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class ToolCallbackInstrumentation extends AbstractSpringAiInstrumentation {
+
+ @Override
+ public ElementMatcher super NamedElement> getTypeMatcherPreFilter() {
+ return nameContains("Tool").or(nameContains("Callback"));
+ }
+
+ @Override
+ public ElementMatcher super TypeDescription> getTypeMatcher() {
+ return not(isInterface()).and(hasSuperType(named("org.springframework.ai.tool.ToolCallback")));
+ }
+
+ @Override
+ public ElementMatcher super MethodDescription> getMethodMatcher() {
+ return named("call").and(takesArguments(String.class));
+ }
+
+ @Override
+ public String getAdviceClassName() {
+ return getClass().getName() + "$ToolCallbackAdvice";
+ }
+
+ public static class ToolCallbackAdvice {
+
+ @Nullable
+ @Advice.OnMethodEnter(suppress = Throwable.class, inline = false)
+ public static Object onEnter(@Advice.This ToolCallback callback,
+ @Advice.Argument(0) String toolInput) {
+ Span> span = SpringAiSpanHelper.startToolSpan(callback, toolInput);
+ return span == null ? null : span.activate();
+ }
+
+ @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false)
+ public static void onExit(@Advice.Enter @Nullable Object spanObject,
+ @Advice.Return @Nullable String result,
+ @Advice.Thrown @Nullable Throwable thrown) {
+ SpringAiSpanHelper.endToolSpan((Span>) spanObject, result, thrown);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/VectorStoreInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/VectorStoreInstrumentation.java
new file mode 100644
index 00000000000..b7b1c36a139
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/VectorStoreInstrumentation.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.tracer.Span;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.NamedElement;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
+
+import javax.annotation.Nullable;
+
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isInterface;
+import static net.bytebuddy.matcher.ElementMatchers.nameContains;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.namedOneOf;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class VectorStoreInstrumentation extends AbstractSpringAiInstrumentation {
+
+ @Override
+ public ElementMatcher super NamedElement> getTypeMatcherPreFilter() {
+ return nameContains("VectorStore");
+ }
+
+ @Override
+ public ElementMatcher super TypeDescription> getTypeMatcher() {
+ return not(isInterface()).and(hasSuperType(named(
+ "org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore")));
+ }
+
+ @Override
+ public ElementMatcher super MethodDescription> getMethodMatcher() {
+ return namedOneOf("add", "delete").and(takesArguments(1));
+ }
+
+ @Override
+ public String getAdviceClassName() {
+ return getClass().getName() + "$VectorStoreAdvice";
+ }
+
+ public static class VectorStoreAdvice {
+
+ @Nullable
+ @Advice.OnMethodEnter(suppress = Throwable.class, inline = false)
+ public static Object onEnter(@Advice.This AbstractObservationVectorStore vectorStore,
+ @Advice.Origin("#m") String method,
+ @Advice.Argument(0) @Nullable Object argument) {
+ Span> span = SpringAiSpanHelper.startVectorStoreSpan(vectorStore, method, argument);
+ return span == null ? null : span.activate();
+ }
+
+ @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false)
+ public static void onExit(@Advice.Enter @Nullable Object spanObject,
+ @Advice.Thrown @Nullable Throwable thrown) {
+ SpringAiSpanHelper.endSpan((Span>) spanObject, thrown);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/VectorStoreQueryInstrumentation.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/VectorStoreQueryInstrumentation.java
new file mode 100644
index 00000000000..f70678256b4
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/VectorStoreQueryInstrumentation.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.tracer.Span;
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.description.NamedElement;
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.springframework.ai.vectorstore.SearchRequest;
+import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
+
+import javax.annotation.Nullable;
+
+import static net.bytebuddy.matcher.ElementMatchers.hasSuperType;
+import static net.bytebuddy.matcher.ElementMatchers.isInterface;
+import static net.bytebuddy.matcher.ElementMatchers.nameContains;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+
+public class VectorStoreQueryInstrumentation extends AbstractSpringAiInstrumentation {
+
+ @Override
+ public ElementMatcher super NamedElement> getTypeMatcherPreFilter() {
+ return nameContains("VectorStore");
+ }
+
+ @Override
+ public ElementMatcher super TypeDescription> getTypeMatcher() {
+ return not(isInterface()).and(hasSuperType(named(
+ "org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore")));
+ }
+
+ @Override
+ public ElementMatcher super MethodDescription> getMethodMatcher() {
+ return named("similaritySearch").and(takesArguments(1))
+ .and(takesArgument(0, named("org.springframework.ai.vectorstore.SearchRequest")));
+ }
+
+ @Override
+ public String getAdviceClassName() {
+ return getClass().getName() + "$VectorStoreQueryAdvice";
+ }
+
+ public static class VectorStoreQueryAdvice {
+
+ @Nullable
+ @Advice.OnMethodEnter(suppress = Throwable.class, inline = false)
+ public static Object onEnter(@Advice.This AbstractObservationVectorStore vectorStore,
+ @Advice.Argument(0) SearchRequest request) {
+ Span> span = SpringAiSpanHelper.startVectorStoreSpan(vectorStore, "query", request);
+ return span == null ? null : span.activate();
+ }
+
+ @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false)
+ public static void onExit(@Advice.Enter @Nullable Object spanObject,
+ @Advice.Return @Nullable Object response,
+ @Advice.Thrown @Nullable Throwable thrown) {
+ SpringAiSpanHelper.endVectorStoreSpan((Span>) spanObject, response, thrown);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/package-info.java b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/package-info.java
new file mode 100644
index 00000000000..ad73cf12f8c
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/java/co/elastic/apm/agent/springai/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+@NonnullApi
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.sdk.NonnullApi;
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation b/apm-agent-plugins/apm-spring-ai-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation
new file mode 100644
index 00000000000..e765d63a9d2
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation
@@ -0,0 +1,6 @@
+co.elastic.apm.agent.springai.ChatModelInstrumentation
+co.elastic.apm.agent.springai.StreamingChatModelInstrumentation
+co.elastic.apm.agent.springai.EmbeddingModelInstrumentation
+co.elastic.apm.agent.springai.VectorStoreInstrumentation
+co.elastic.apm.agent.springai.VectorStoreQueryInstrumentation
+co.elastic.apm.agent.springai.ToolCallbackInstrumentation
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/main/resources/META-INF/services/org.stagemonitor.configuration.ConfigurationOptionProvider b/apm-agent-plugins/apm-spring-ai-plugin/src/main/resources/META-INF/services/org.stagemonitor.configuration.ConfigurationOptionProvider
new file mode 100644
index 00000000000..877ea3fe66a
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/main/resources/META-INF/services/org.stagemonitor.configuration.ConfigurationOptionProvider
@@ -0,0 +1 @@
+co.elastic.apm.agent.springai.SpringAiConfiguration
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiClassLoaderMatcherAssertions.java b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiClassLoaderMatcherAssertions.java
new file mode 100644
index 00000000000..80eb5766da5
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiClassLoaderMatcherAssertions.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class SpringAiClassLoaderMatcherAssertions {
+
+ private static final String MODEL_CLASS = "org/springframework/ai/model/Model.class";
+ private static final String AI_OPERATION_TYPE_CLASS =
+ "org/springframework/ai/observation/conventions/AiOperationType.class";
+ private static final String VECTOR_STORE_CLASS =
+ "org/springframework/ai/vectorstore/observation/AbstractObservationVectorStore.class";
+ private static final String SPRING_AI_2_MARKER_CLASS =
+ "org/springframework/ai/embedding/DefaultEmbeddingOptions$Builder.class";
+
+ public static void assertClassLoaderMatcher(ClassLoader classLoader, boolean supported) {
+ assertCommonSpringAiClassesPresent(classLoader);
+ if (supported) {
+ assertThat(classLoader.getResource(SPRING_AI_2_MARKER_CLASS)).isNotNull();
+ } else {
+ assertThat(classLoader.getResource(SPRING_AI_2_MARKER_CLASS)).isNull();
+ }
+ assertThat(instrumentations()).allSatisfy(instrumentation ->
+ assertThat(instrumentation.getClassLoaderMatcher().matches(classLoader)).isEqualTo(supported));
+ }
+
+ private static void assertCommonSpringAiClassesPresent(ClassLoader classLoader) {
+ assertThat(classLoader.getResource(MODEL_CLASS)).isNotNull();
+ assertThat(classLoader.getResource(AI_OPERATION_TYPE_CLASS)).isNotNull();
+ assertThat(classLoader.getResource(VECTOR_STORE_CLASS)).isNotNull();
+ }
+
+ private static List instrumentations() {
+ return Arrays.asList(
+ new ChatModelInstrumentation(),
+ new StreamingChatModelInstrumentation(),
+ new EmbeddingModelInstrumentation(),
+ new VectorStoreInstrumentation(),
+ new VectorStoreQueryInstrumentation(),
+ new ToolCallbackInstrumentation()
+ );
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiInstrumentationTest.java b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiInstrumentationTest.java
new file mode 100644
index 00000000000..3f7669c9af6
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiInstrumentationTest.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.AbstractInstrumentationTest;
+import co.elastic.apm.agent.embeddedotel.EmbeddedSdkManager;
+import co.elastic.apm.agent.impl.transaction.SpanImpl;
+import co.elastic.apm.agent.impl.transaction.TransactionImpl;
+import co.elastic.apm.agent.tracer.Outcome;
+import co.elastic.apm.agent.tracer.configuration.CoreConfiguration;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.messages.SystemMessage;
+import org.springframework.ai.chat.messages.ToolResponseMessage;
+import org.springframework.ai.chat.messages.UserMessage;
+import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
+import org.springframework.ai.chat.metadata.ChatResponseMetadata;
+import org.springframework.ai.chat.metadata.DefaultUsage;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.prompt.ChatOptions;
+import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.embedding.Embedding;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.embedding.EmbeddingOptions;
+import org.springframework.ai.embedding.EmbeddingRequest;
+import org.springframework.ai.embedding.EmbeddingResponse;
+import org.springframework.ai.embedding.EmbeddingResponseMetadata;
+import org.springframework.ai.tool.ToolCallback;
+import org.springframework.ai.tool.definition.ToolDefinition;
+import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
+import org.springframework.ai.vectorstore.SearchRequest;
+import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
+import reactor.core.publisher.Flux;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+
+import static co.elastic.apm.agent.testutils.assertions.Assertions.assertThat;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.mockito.Mockito.doReturn;
+
+class SpringAiInstrumentationTest extends AbstractInstrumentationTest {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ @Test
+ void capturesChatCallAndSemanticAttributes() {
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ ChatOptions options = ChatOptions.builder().model("gpt-test").temperature(0.2).maxTokens(42).build();
+ new OpenAiTestChatModel().call(new Prompt("secret prompt", options));
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ SpanImpl span = reporter.getFirstSpan();
+ assertThat(span)
+ .hasName("chat gpt-test")
+ .hasType("app")
+ .hasSubType("internal")
+ .hasOutcome(Outcome.SUCCESS)
+ .hasOtelAttribute("gen_ai.operation.name", "chat")
+ .hasOtelAttribute("gen_ai.request.model", "gpt-test")
+ .hasOtelAttribute("gen_ai.response.model", "gpt-test-result")
+ .hasOtelAttribute("gen_ai.usage.input_tokens", 10)
+ .hasOtelAttribute("gen_ai.usage.output_tokens", 5)
+ .hasOtelAttribute("gen_ai.usage.total_tokens", 15);
+ assertThat(span.getOtelAttributes())
+ .doesNotContainKeys("gen_ai.system", "gen_ai.provider.name",
+ "gen_ai.input.messages", "gen_ai.output.messages");
+ }
+
+ @Test
+ void capturesStreamingChatUntilFluxTerminates() throws Exception {
+ enableContentCapture();
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ ChatOptions options = ChatOptions.builder().model("gpt-stream").build();
+ Prompt prompt = new Prompt("secret stream prompt", options);
+ new OpenAiTestChatModel().stream(prompt).blockLast();
+ new DelayedOpenAiTestChatModel(20).stream(prompt).blockLast();
+ new DelayedOpenAiTestChatModel(200).stream(prompt).blockLast();
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ SpanImpl span = reporter.getFirstSpan();
+ assertThat(span)
+ .hasName("chat gpt-stream")
+ .hasOtelAttribute("gen_ai.request.stream", true)
+ .hasOtelAttribute("gen_ai.usage.total_tokens", 15)
+ .hasOtelAttribute("gen_ai.output.messages",
+ "[{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"secret response\"}]}]");
+ assertThat(
+ ((Number) span.getOtelAttributes().get("gen_ai.response.time_to_first_chunk")).doubleValue())
+ .isGreaterThanOrEqualTo(0);
+
+ EmbeddedSdkManager sdkManager = tracer.getLifecycleListener(EmbeddedSdkManager.class);
+ ((SdkMeterProvider) sdkManager.getMeterProvider().getDelegate()).forceFlush().join(10, SECONDS);
+ assertThat(reporter.getBytes()).hasSize(1);
+ JsonNode metricSet = OBJECT_MAPPER.readTree(reporter.getBytes().get(0)).get("metricset");
+ JsonNode sample = metricSet.get("samples").get(SpringAiMetrics.TIME_TO_FIRST_CHUNK);
+ assertThat(sample.get("type").asText()).isEqualTo("histogram");
+ long count = 0;
+ for (JsonNode bucketCount : sample.get("counts")) {
+ count += bucketCount.asLong();
+ }
+ assertThat(count).isEqualTo(3);
+ assertThat(sample.get("values").size()).isGreaterThanOrEqualTo(2);
+ assertThat(metricSet.get("tags").toString())
+ .contains("gen_ai_operation_name", "chat", "gen_ai_request_model", "gpt-stream")
+ .doesNotContain("gen_ai_provider_name");
+
+ }
+
+ @Test
+ void doesNotRecordTimeToFirstChunkForEmptyStream() {
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ new EmptyOpenAiTestChatModel().stream(new Prompt("empty stream")).blockLast();
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ assertThat(reporter.getFirstSpan().getOtelAttributes())
+ .doesNotContainKey("gen_ai.response.time_to_first_chunk");
+ }
+
+ @Test
+ void capturesEmbeddingCall() {
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ EmbeddingOptions options = EmbeddingOptions.builder().model("embedding-test").dimensions(3).build();
+ new OpenAiTestEmbeddingModel().call(new EmbeddingRequest(Collections.singletonList("secret"), options));
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ SpanImpl span = reporter.getFirstSpan();
+ assertThat(span)
+ .hasName("embedding embedding-test")
+ .hasOtelAttribute("gen_ai.request.embedding.dimensions", 3)
+ .hasOtelAttribute("gen_ai.usage.input_tokens", 4);
+ assertThat(span.getOtelAttributes())
+ .doesNotContainKey("spring.ai.embedding.request.inputs");
+ }
+
+ @Test
+ void capturesToolDefinitionButNotArgumentsOrResult() {
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ new TestToolCallback().call("{\"password\":\"secret\"}");
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ SpanImpl span = reporter.getFirstSpan();
+ assertThat(span)
+ .hasName("execute_tool weather")
+ .hasType("app")
+ .hasSubType("internal")
+ .hasOtelAttribute("spring.ai.tool.definition.name", "weather");
+ assertThat(span.getOtelAttributes())
+ .doesNotContainKeys("spring.ai.tool.call.arguments", "spring.ai.tool.call.result");
+ }
+
+ @Test
+ void capturesVectorStoreQuery() {
+ TestVectorStore vectorStore = new TestVectorStore(new OpenAiTestEmbeddingModel());
+
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ vectorStore.similaritySearch(SearchRequest.builder()
+ .query("secret search query")
+ .topK(3)
+ .similarityThreshold(0.5)
+ .build());
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ SpanImpl span = reporter.getFirstSpan();
+ assertThat(span)
+ .hasName("simple query")
+ .hasType("db")
+ .hasSubType("unknown")
+ .hasOtelAttribute("db.system", "simple")
+ .hasOtelAttribute("db.operation.name", "query")
+ .hasOtelAttribute("db.vector.query.top_k", 3)
+ .hasOtelAttribute("db.vector.query.similarity_threshold", 0.5);
+ }
+
+ @Test
+ void capturesChatContentWhenEnabled() {
+ enableContentCapture();
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ new OpenAiTestChatModel().call(new Prompt("secret prompt"));
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ assertThat(reporter.getFirstSpan().getOtelAttributes())
+ .extractingByKeys("gen_ai.input.messages", "gen_ai.output.messages")
+ .allSatisfy(value -> assertThat(value.toString()).contains("secret"));
+ }
+
+ @Test
+ void capturesToolCallsAndResponsesInChatMessages() throws Exception {
+ enableContentCapture();
+ AssistantMessage toolCall = AssistantMessage.builder()
+ .content("")
+ .toolCalls(Collections.singletonList(new AssistantMessage.ToolCall(
+ "call-1", "function", "echo", "{\"message\":\"Hello from Spring AI\"}")))
+ .build();
+ ToolResponseMessage toolResponse = ToolResponseMessage.builder()
+ .responses(Collections.singletonList(new ToolResponseMessage.ToolResponse(
+ "call-1", "echo", "[{\"text\":\"Hello from Spring AI\"}]")))
+ .build();
+ Prompt prompt = new Prompt(List.of(
+ new SystemMessage("Call the echo tool"),
+ new UserMessage("Echo this message"),
+ toolCall,
+ toolResponse
+ ));
+
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ new OpenAiTestChatModel().call(prompt);
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ String messages = reporter.getFirstSpan().getOtelAttributes().get("gen_ai.input.messages").toString();
+ JsonNode parsed = OBJECT_MAPPER.readTree(messages);
+ assertThat(parsed.get(2).get("role").asText()).isEqualTo("assistant");
+ assertThat(parsed.get(2).get("parts").get(0).get("type").asText())
+ .isEqualTo("tool_call");
+ assertThat(parsed.get(2).get("parts").get(0).get("name").asText())
+ .isEqualTo("echo");
+ assertThat(parsed.get(2).get("parts").get(0).get("arguments").asText())
+ .contains("Hello from Spring AI");
+ assertThat(parsed.get(3).get("role").asText()).isEqualTo("tool");
+ assertThat(parsed.get(3).get("parts").get(0).get("type").asText())
+ .isEqualTo("tool_call_response");
+ assertThat(parsed.get(3).get("parts").get(0).get("response").asText())
+ .contains("Hello from Spring AI");
+ }
+
+ @Test
+ void capturesEmbeddingAndToolContentWhenEnabled() {
+ enableContentCapture();
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ new OpenAiTestEmbeddingModel().call(new EmbeddingRequest(
+ Collections.singletonList("secret embedding document"), EmbeddingOptions.builder().build()));
+ } finally {
+ transaction.deactivate().end();
+ }
+ assertThat(reporter.getFirstSpan())
+ .hasOtelAttribute("spring.ai.embedding.request.inputs", "[\"secret embedding document\"]");
+
+ reporter.reset();
+ transaction = startTestRootTransaction();
+ try {
+ new TestToolCallback().call("{\"password\":\"secret\"}");
+ } finally {
+ transaction.deactivate().end();
+ }
+ assertThat(reporter.getFirstSpan())
+ .hasOtelAttribute("spring.ai.tool.call.arguments", "{\"password\":\"secret\"}")
+ .hasOtelAttribute("spring.ai.tool.call.result", "secret result");
+ }
+
+ @Test
+ void capturesVectorDocumentsAndQueryContentWhenEnabled() {
+ enableContentCapture();
+ TestVectorStore vectorStore = new TestVectorStore(new OpenAiTestEmbeddingModel());
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ vectorStore.add(Collections.singletonList(new Document("secret stored document")));
+ } finally {
+ transaction.deactivate().end();
+ }
+ assertThat(
+ reporter.getFirstSpan().getOtelAttributes().get("db.vector.documents").toString())
+ .contains("secret stored document");
+
+ reporter.reset();
+ transaction = startTestRootTransaction();
+ try {
+ vectorStore.similaritySearch(SearchRequest.builder()
+ .query("secret search query")
+ .filterExpression("category == 'secret'")
+ .build());
+ } finally {
+ transaction.deactivate().end();
+ }
+ assertThat(reporter.getFirstSpan())
+ .hasOtelAttribute("db.vector.query.content", "secret search query");
+ assertThat(
+ reporter.getFirstSpan().getOtelAttributes())
+ .hasEntrySatisfying("db.vector.query.filter",
+ value -> assertThat(value.toString()).contains("secret"))
+ .hasEntrySatisfying("db.vector.query.response.documents",
+ value -> assertThat(value.toString()).contains("secret result document"));
+ }
+
+ @Test
+ void truncatesCapturedContentAtLongFieldLimit() {
+ enableContentCapture();
+ doReturn(6).when(tracer.getConfig(CoreConfiguration.class)).getLongFieldMaxLength();
+ TransactionImpl transaction = startTestRootTransaction();
+ try {
+ new TestToolCallback().call("123456789");
+ } finally {
+ transaction.deactivate().end();
+ }
+
+ assertThat(reporter.getFirstSpan())
+ .hasOtelAttribute("spring.ai.tool.call.arguments", "123456")
+ .hasOtelAttribute("spring.ai.tool.call.result", "secret");
+ }
+
+ private static void enableContentCapture() {
+ doReturn(true).when(config.getConfig(SpringAiConfiguration.class)).isCaptureSpringAiContent();
+ }
+
+ static class OpenAiTestChatModel implements ChatModel {
+ @Override
+ public ChatResponse call(Prompt prompt) {
+ return response();
+ }
+
+ @Override
+ public Flux stream(Prompt prompt) {
+ return Flux.just(
+ new ChatResponse(Collections.singletonList(
+ new Generation(new AssistantMessage("secret ")))),
+ new ChatResponse(Collections.singletonList(
+ new Generation(new AssistantMessage("response")))),
+ new ChatResponse(Collections.emptyList(), response().getMetadata())
+ );
+ }
+
+ static ChatResponse response() {
+ Generation generation = new Generation(new AssistantMessage("secret response"),
+ ChatGenerationMetadata.builder().finishReason("stop").build());
+ ChatResponseMetadata metadata = ChatResponseMetadata.builder()
+ .id("response-id")
+ .model("gpt-test-result")
+ .usage(new DefaultUsage(10, 5, 15))
+ .build();
+ return new ChatResponse(Collections.singletonList(generation), metadata);
+ }
+ }
+
+ static class DelayedOpenAiTestChatModel extends OpenAiTestChatModel {
+ private final long delayMillis;
+
+ DelayedOpenAiTestChatModel(long delayMillis) {
+ this.delayMillis = delayMillis;
+ }
+
+ @Override
+ public Flux stream(Prompt prompt) {
+ return Flux.just(response()).delaySubscription(Duration.ofMillis(delayMillis));
+ }
+ }
+
+ static class EmptyOpenAiTestChatModel extends OpenAiTestChatModel {
+ @Override
+ public Flux stream(Prompt prompt) {
+ return Flux.empty();
+ }
+ }
+
+ static class OpenAiTestEmbeddingModel implements EmbeddingModel {
+ @Override
+ public EmbeddingResponse call(EmbeddingRequest request) {
+ return new EmbeddingResponse(
+ Collections.singletonList(new Embedding(new float[]{1, 2, 3}, 0)),
+ new EmbeddingResponseMetadata("embedding-result", new DefaultUsage(4, 0, 4)));
+ }
+
+ @Override
+ public float[] embed(Document document) {
+ return new float[]{1, 2, 3};
+ }
+ }
+
+ static class TestVectorStore extends AbstractObservationVectorStore {
+
+ TestVectorStore(EmbeddingModel embeddingModel) {
+ super(new TestVectorStoreBuilder(embeddingModel));
+ }
+
+ @Override
+ public void doAdd(List documents) {
+ }
+
+ @Override
+ public void doDelete(List ids) {
+ }
+
+ @Override
+ public List doSimilaritySearch(SearchRequest request) {
+ return Collections.singletonList(new Document("secret result document"));
+ }
+
+ @Override
+ public VectorStoreObservationContext.Builder createObservationContextBuilder(String operation) {
+ return VectorStoreObservationContext.builder("simple", operation)
+ .collectionName("test-collection")
+ .dimensions(3)
+ .similarityMetric("cosine");
+ }
+ }
+
+ static class TestVectorStoreBuilder extends AbstractVectorStoreBuilder {
+
+ TestVectorStoreBuilder(EmbeddingModel embeddingModel) {
+ super(embeddingModel);
+ }
+
+ @Override
+ public TestVectorStore build() {
+ return new TestVectorStore(embeddingModel);
+ }
+ }
+
+ static class TestToolCallback implements ToolCallback {
+ @Override
+ public ToolDefinition getToolDefinition() {
+ return ToolDefinition.builder()
+ .name("weather")
+ .description("Gets the weather")
+ .inputSchema("{\"type\":\"object\"}")
+ .build();
+ }
+
+ @Override
+ public String call(String toolInput) {
+ return "secret result";
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiSupportedVersionTest.java b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiSupportedVersionTest.java
new file mode 100644
index 00000000000..2ccce3db17d
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiSupportedVersionTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.testutils.TestClassWithDependencyRunner;
+import org.junit.jupiter.api.Test;
+
+import java.net.URLClassLoader;
+
+@TestClassWithDependencyRunner.DisableOutsideOfRunner
+class SpringAiSupportedVersionTest {
+
+ @Test
+ void acceptsSpringAiVersion() throws Exception {
+ URLClassLoader versionClassLoader = (URLClassLoader) getClass().getClassLoader();
+ try (URLClassLoader isolatedClassLoader = new URLClassLoader(versionClassLoader.getURLs(), null)) {
+ SpringAiClassLoaderMatcherAssertions.assertClassLoaderMatcher(isolatedClassLoader, true);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiUnsupportedVersionTest.java b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiUnsupportedVersionTest.java
new file mode 100644
index 00000000000..9582d6e6bea
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiUnsupportedVersionTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.testutils.TestClassWithDependencyRunner;
+import org.junit.jupiter.api.Test;
+
+import java.net.URLClassLoader;
+
+@TestClassWithDependencyRunner.DisableOutsideOfRunner
+class SpringAiUnsupportedVersionTest {
+
+ @Test
+ void rejectsSpringAiVersion() throws Exception {
+ URLClassLoader versionClassLoader = (URLClassLoader) getClass().getClassLoader();
+ try (URLClassLoader isolatedClassLoader = new URLClassLoader(versionClassLoader.getURLs(), null)) {
+ SpringAiClassLoaderMatcherAssertions.assertClassLoaderMatcher(isolatedClassLoader, false);
+ }
+ }
+}
diff --git a/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiVersionIT.java b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiVersionIT.java
new file mode 100644
index 00000000000..e8a8b3f8597
--- /dev/null
+++ b/apm-agent-plugins/apm-spring-ai-plugin/src/test/java/co/elastic/apm/agent/springai/SpringAiVersionIT.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.springai;
+
+import co.elastic.apm.agent.testutils.TestClassWithDependencyRunner;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.List;
+
+class SpringAiVersionIT {
+
+ @ParameterizedTest
+ @ValueSource(strings = {"1.0.8", "1.1.7"})
+ void rejectsUnsupportedSpringAiVersions(String version) throws Exception {
+ runVersionTest(version, SpringAiUnsupportedVersionTest.class);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"2.0.0", "2.0.2"})
+ void acceptsSupportedSpringAiVersions(String version) throws Exception {
+ runVersionTest(version, SpringAiSupportedVersionTest.class);
+ }
+
+ private static void runVersionTest(String version, Class> testClass) throws Exception {
+ List dependencies = List.of(
+ "org.springframework.ai:spring-ai-model:" + version,
+ "org.springframework.ai:spring-ai-commons:" + version,
+ "org.springframework.ai:spring-ai-vector-store:" + version);
+ TestClassWithDependencyRunner runner = new TestClassWithDependencyRunner(
+ dependencies, testClass.getName());
+ runner.run();
+ }
+}
diff --git a/apm-agent-plugins/pom.xml b/apm-agent-plugins/pom.xml
index ddca5256d7e..c67cf2097b3 100644
--- a/apm-agent-plugins/pom.xml
+++ b/apm-agent-plugins/pom.xml
@@ -28,6 +28,7 @@
apm-api-plugin
apm-apache-httpclient
apm-spring-resttemplate
+ apm-spring-ai-plugin
apm-httpclient-core
apm-httpserver-core
apm-logging-plugin
diff --git a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/CoreConfiguration.java b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/CoreConfiguration.java
index 50cf86f6cc2..8807257077c 100644
--- a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/CoreConfiguration.java
+++ b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/CoreConfiguration.java
@@ -34,6 +34,8 @@ public interface CoreConfiguration {
EventType getCaptureBody();
+ int getLongFieldMaxLength();
+
boolean isEnablePublicApiAnnotationInheritance();
List getSanitizeFieldNames();
diff --git a/docs/reference/config-ai.md b/docs/reference/config-ai.md
new file mode 100644
index 00000000000..cd5e16e6454
--- /dev/null
+++ b/docs/reference/config-ai.md
@@ -0,0 +1,33 @@
+---
+navigation_title: "AI"
+applies_to:
+ stack:
+ serverless:
+ observability:
+ product:
+ apm_agent_java: ga
+---
+
+# AI configuration options [config-ai]
+
+## `capture_spring_ai_content` (experimental, security, performance) [config-capture-spring-ai-content]
+
+```{applies_to}
+apm_agent_java: ga 1.58.0
+```
+
+Enables capture of Spring AI prompt and response messages, embedding inputs, vector store documents and queries, and tool arguments and results.
+
+Captured content can contain sensitive information and can significantly increase heap usage, network traffic, and storage requirements. Review your data handling requirements before enabling this option. Individual captured content values are truncated according to [`long_field_max_length`](/reference/config-core.md#config-long-field-max-length).
+
+[ ](/reference/configuration.md#configuration-dynamic)
+
+Support for configuring this option through Kibana central configuration will be added in a future Kibana version.
+
+| Default | Type | Dynamic |
+| --- | --- | --- |
+| `false` | Boolean | true |
+
+| Java System Properties | Property file | Environment |
+| --- | --- | --- |
+| `elastic.apm.capture_spring_ai_content` | `capture_spring_ai_content` | `ELASTIC_APM_CAPTURE_SPRING_AI_CONTENT` |
diff --git a/docs/reference/config-core.md b/docs/reference/config-core.md
index 3d005060247..5bae1f35df6 100644
--- a/docs/reference/config-core.md
+++ b/docs/reference/config-core.md
@@ -281,6 +281,7 @@ The following transaction, span, and error fields will be truncated at this numb
* `transaction.context.request.body`, `error.context.request.body`
* `transaction.context.message.body`, `error.context.message.body`
* `span.context.db.statement`
+* Captured AI content values
Note that tracing data is limited at the upstream APM server to [`max_event_size`](docs-content://solutions/observability/apm/general-configuration-options.md#apm-max_event_size), which defaults to 300kB. If you configure `long_field_max_length` too large, it could result in transactions, spans, or errors that are rejected by APM server.
@@ -328,7 +329,7 @@ Review the data captured by Elastic APM carefully to make sure it does not captu
apm_agent_java: ga 1.28.0
```
-A list of instrumentations which should be selectively enabled. Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`. When set to non-empty value, only listed instrumentations will be enabled if they are not disabled through [`disable_instrumentations` ([1.0.0])](#config-disable-instrumentations) or [`enable_experimental_instrumentations` ([1.25.0])](#config-enable-experimental-instrumentations). When not set or empty (default), all instrumentations enabled by default will be enabled unless they are disabled through [`disable_instrumentations` ([1.0.0])](#config-disable-instrumentations) or [`enable_experimental_instrumentations` ([1.25.0])](#config-enable-experimental-instrumentations).
+A list of instrumentations which should be selectively enabled. Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-ai`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`. When set to non-empty value, only listed instrumentations will be enabled if they are not disabled through [`disable_instrumentations` ([1.0.0])](#config-disable-instrumentations) or [`enable_experimental_instrumentations` ([1.25.0])](#config-enable-experimental-instrumentations). When not set or empty (default), all instrumentations enabled by default will be enabled unless they are disabled through [`disable_instrumentations` ([1.0.0])](#config-disable-instrumentations) or [`enable_experimental_instrumentations` ([1.25.0])](#config-enable-experimental-instrumentations).
::::{note}
Changing this value at runtime can slow down the application temporarily.
@@ -352,7 +353,7 @@ Changing this value at runtime can slow down the application temporarily.
apm_agent_java: ga 1.0.0
```
-A list of instrumentations which should be disabled. Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`. For version `1.25.0` and later, use [`enable_experimental_instrumentations` ([1.25.0])](#config-enable-experimental-instrumentations) to enable experimental instrumentations.
+A list of instrumentations which should be disabled. Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-ai`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`. For version `1.25.0` and later, use [`enable_experimental_instrumentations` ([1.25.0])](#config-enable-experimental-instrumentations) to enable experimental instrumentations.
::::{note}
Changing this value at runtime can slow down the application temporarily.
diff --git a/docs/reference/config-reference-properties-file.md b/docs/reference/config-reference-properties-file.md
index 66c462ad83b..5ed6a4c35cd 100644
--- a/docs/reference/config-reference-properties-file.md
+++ b/docs/reference/config-reference-properties-file.md
@@ -12,6 +12,20 @@ applies_to:
# Property file reference [config-reference-properties-file]
```properties
+############################################
+# AI #
+############################################
+
+# Enables capture of Spring AI prompt and response messages, embedding inputs, vector store documents and queries, and tool arguments and results.
+#
+# Captured content can contain sensitive information and can significantly increase storage usage. Content capture is disabled by default. Individual captured content values are truncated according to `long_field_max_length`.
+#
+# This setting can be changed at runtime
+# Type: Boolean
+# Default value: false
+#
+# capture_spring_ai_content=false
+
############################################
# Circuit-Breaker #
############################################
@@ -294,6 +308,7 @@ applies_to:
# - `transaction.context.request.body`, `error.context.request.body`
# - `transaction.context.message.body`, `error.context.message.body`
# - `span.context.db.statement`
+# - captured AI content values
#
# Note that tracing data is limited at the upstream APM server to
# {apm-guide-ref}/configuration-process.html#max_event_size[`max_event_size`],
@@ -332,7 +347,7 @@ applies_to:
# sanitize_field_names=password,passwd,pwd,secret,*key,*token*,*session*,*credit*,*card*,*auth*,*principal*,set-cookie
# A list of instrumentations which should be selectively enabled.
-# Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`.
+# Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-ai`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`.
# When set to non-empty value, only listed instrumentations will be enabled if they are not disabled through <> or <>.
# When not set or empty (default), all instrumentations enabled by default will be enabled unless they are disabled through <> or <>.
#
@@ -345,7 +360,7 @@ applies_to:
# enable_instrumentations=
# A list of instrumentations which should be disabled.
-# Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`.
+# Valid options are `annotations`, `annotations-capture-span`, `annotations-capture-transaction`, `annotations-traced`, `apache-commons-exec`, `apache-httpclient`, `asynchttpclient`, `aws-lambda`, `aws-sdk`, `cassandra`, `concurrent`, `dubbo`, `elasticsearch-restclient`, `exception-handler`, `executor`, `executor-collection`, `experimental`, `finagle-httpclient`, `fork-join`, `grails`, `grpc`, `hibernate-search`, `http-client`, `jakarta-websocket`, `java-ldap`, `javalin`, `javax-websocket`, `jax-rs`, `jax-ws`, `jdbc`, `jdk-httpclient`, `jdk-httpserver`, `jedis`, `jms`, `jsf`, `kafka`, `lettuce`, `log-correlation`, `log-error`, `log-reformatting`, `logging`, `micrometer`, `mongodb`, `mongodb-client`, `okhttp`, `opentelemetry`, `opentelemetry-annotations`, `opentelemetry-metrics`, `opentracing`, `process`, `public-api`, `quartz`, `rabbitmq`, `reactor`, `redis`, `redisson`, `render`, `scala-future`, `scheduled`, `servlet-api`, `servlet-api-async`, `servlet-api-dispatch`, `servlet-input-stream`, `servlet-service-name`, `servlet-version`, `sparkjava`, `spring-ai`, `spring-amqp`, `spring-mvc`, `spring-resttemplate`, `spring-service-name`, `spring-view-render`, `spring-webclient`, `spring-webflux`, `ssl-context`, `struts`, `timer-task`, `urlconnection`, `vertx`, `vertx-web`, `vertx-webclient`, `websocket`.
# For version `1.25.0` and later, use <> to enable experimental instrumentations.
#
# NOTE: Changing this value at runtime can slow down the application temporarily.
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index 34a7283f2ef..fdb34bf65a9 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -78,6 +78,10 @@ ELASTIC_APM_SERVER_URL=http://127.0.0.1:8200
This is a list of all configuration options grouped by their category. Click on a key to get more information.
+* [AI](/reference/config-ai.md)
+
+ * [`capture_spring_ai_content` (experimental, security, performance [1.58.0])](/reference/config-ai.md#config-capture-spring-ai-content)
+
* [Circuit-Breaker](/reference/config-circuit-breaker.md)
* [`circuit_breaker_enabled` ([1.14.0] performance)](/reference/config-circuit-breaker.md#config-circuit-breaker-enabled)
diff --git a/docs/reference/supported-technologies.md b/docs/reference/supported-technologies.md
index ce1f62e017c..6e5397307ec 100644
--- a/docs/reference/supported-technologies.md
+++ b/docs/reference/supported-technologies.md
@@ -19,6 +19,7 @@ Elastic supports OpenTelemetry, which allows logs, metrics, and trace signal col
* [Web Frameworks](#supported-web-frameworks)
* [Application Servers/Servlet Containers](#supported-app-servers)
* [Data Stores](#supported-databases)
+* [Generative AI](#supported-generative-ai)
* [Networking frameworks](#supported-networking-frameworks)
* [Asynchronous frameworks](#supported-async-frameworks)
* [Messaging frameworks](#supported-messaging-frameworks)
@@ -128,6 +129,13 @@ The Elastic APM Java agent has generic support for the Servlet API 3+. However,
| AWS S3 | ≥ 1.x, ≤ 2.39.x | The agent creates spans for interactions with the AWS S3 service through the AWS Java SDK. | 1.31.0, 2.21+ since 1.44.0 |
+## Generative AI [supported-generative-ai]
+
+| Framework | Supported versions | Description | Since |
+|-----------|--------------------|-------------|-------|
+| Spring AI | 2.0.0 - 2.0.2 | Creates spans for synchronous and streaming chat model calls, embedding model calls, vector store operations, and tool callbacks. Prompt, response, document, query, tool argument, and tool result content is not captured by default; use [`capture_spring_ai_content`](/reference/config-ai.md#config-capture-spring-ai-content) to opt in. This experimental instrumentation is disabled by default; enable it with [`enable_experimental_instrumentations`](/reference/config-core.md#config-enable-experimental-instrumentations). | 1.58.0 (experimental) |
+
+
## Networking frameworks [supported-networking-frameworks]
Distributed tracing will only work if you are using one of the supported networking frameworks.
diff --git a/docs/reference/toc.yml b/docs/reference/toc.yml
index 2f5cbc5e788..b75ea0e59b6 100644
--- a/docs/reference/toc.yml
+++ b/docs/reference/toc.yml
@@ -11,6 +11,7 @@ toc:
- file: supported-technologies.md
- file: configuration.md
children:
+ - file: config-ai.md
- file: config-circuit-breaker.md
- file: config-core.md
- file: config-datastore.md
@@ -45,4 +46,4 @@ toc:
- file: community-plugins.md
- file: upgrading.md
- title: Troubleshooting
- crosslink: docs-content://troubleshoot/observability/apm-agent-java/apm-java-agent.md
\ No newline at end of file
+ crosslink: docs-content://troubleshoot/observability/apm-agent-java/apm-java-agent.md
diff --git a/pom.xml b/pom.xml
index 434b341cdae..2fbe7f91284 100644
--- a/pom.xml
+++ b/pom.xml
@@ -361,7 +361,7 @@
central
true
- apm-agent-common,apm-agent-cached-lookup-key,apm-agent-bootstrap,apm-agent-core,elastic-apm-agent-premain,apm-agent-plugins,apm-httpclient-core,apm-apache-httpclient,apm-apache-httpclient3-plugin,apm-apache-httpclient-common,apm-apache-httpclient4-plugin,apm-apache-httpclient5-plugin,apm-api-plugin,apm-asynchttpclient-plugin,apm-java-concurrent-plugin,apm-cassandra,apm-cassandra-core-plugin,apm-cassandra3-plugin,apm-cassandra4-plugin,apm-okhttp-plugin,apm-dubbo-plugin,apm-es-restclient-plugin,apm-es-restclient-plugin-common,apm-es-restclient-plugin-5_6,apm-es-restclient-plugin-6_4,apm-es-restclient-plugin-7_x,apm-es-restclient-plugin-8_x,apm-grails-plugin,apm-grpc,apm-grpc-plugin,apm-hibernate-search-plugin,apm-hibernate-search-plugin-common,apm-hibernate-search-plugin-5_x,apm-hibernate-search-plugin-6_x,apm-httpserver-core,apm-servlet-plugin,apm-javalin-plugin,apm-jaxrs-plugin,apm-jaxws-plugin,apm-jdbc-plugin,apm-jdk-httpclient-plugin,apm-jdk-httpserver-plugin,apm-redis-plugin,apm-redis-common,apm-jedis-plugin,apm-jedis-4-plugin,apm-jms-plugin,apm-jms-plugin-base,apm-jms-javax,apm-jms-jakarta,apm-jmx-plugin,apm-jsf-plugin,apm-kafka-plugin,apm-kafka-base-plugin,apm-kafka-headers-plugin,apm-kafka-spring-plugin,apm-lettuce-plugin,apm-logging-plugin,apm-logging-plugin-common,apm-log4j1-plugin,apm-log4j2-plugin,apm-opentelemetry,apm-opentelemetry-embedded-metrics-sdk,apm-opentelemetry-metrics-bridge-parent,apm-opentelemetry-metrics-bridge-common,apm-opentelemetry-metrics-bridge-v1_14,apm-opentelemetry-metrics-bridge-latest,apm-opentelemetry-metricsdk-plugin,apm-opentelemetry-plugin,apm-slf4j-plugin,apm-logback-plugin,apm-logback-plugin-impl,apm-jboss-logging-plugin,apm-jul-plugin,apm-tomcat-logging-plugin,apm-micrometer-plugin,apm-mongodb,apm-mongodb-common,apm-mongodb3-plugin,apm-mongodb4-plugin,apm-opentracing-plugin,apm-process-plugin,apm-profiling-plugin,apm-quartz,apm-quartz-common,apm-quartz-plugin-1,apm-quartz-plugin-2,apm-rabbitmq,apm-rabbitmq-plugin,apm-rabbitmq-spring5,apm-reactor-plugin,apm-redisson-plugin,apm-scala-concurrent-plugin,apm-scheduled-annotation-plugin,apm-sparkjava-plugin,apm-urlconnection-plugin,apm-spring-resttemplate,apm-spring-resttemplate-plugin,apm-spring-webflux,apm-spring-webflux-common-spring5,apm-spring-webflux-common,apm-spring-webclient-plugin,apm-spring-webflux-testapp,apm-spring-webflux-testapp-spring7,apm-spring-webflux-spring5,apm-spring-webflux-plugin,apm-spring-webflux-plugin-spring7,apm-spring-webmvc,apm-spring-webmvc-spring5,apm-spring-webmvc-plugin,apm-struts-plugin,apm-vertx,apm-vertx-common,apm-vertx3-plugin,apm-vertx4-plugin,apm-awslambda-plugin,apm-jakarta-websocket-plugin,apm-ecs-logging-plugin,apm-aws-sdk,apm-aws-sdk-common,apm-aws-sdk-1-plugin,apm-aws-sdk-2-plugin,apm-finagle-httpclient-plugin,apm-java-ldap-plugin,apm-agent-builds,apm-agent,apm-agent-lambda-layer,apm-agent-java8,apm-agent-benchmarks,apm-spring-resttemplate-test,apm-spring-restclient-test,apm-logback-plugin-legacy-tests,apm-log4j2-plugin-tests,apm-jms-spring-plugin,apm-jedis-2-tests,apm-jedis-3-tests,apm-jedis-5-tests,apm-lettuce-3-tests,apm-grpc-test-1.6.1,apm-grpc-test-latest,apm-rabbitmq-test-3,apm-rabbitmq-test-4,apm-rabbitmq-test-spring6,apm-okhttp-test,apm-opentelemetry-test,apm-vertx3-test-latest,apm-servlet-jakarta-test,apm-jaxws-plugin-jakartaee-test,apm-jaxrs-plugin-jakartaee-test,apm-scheduled-annotation-plugin-jakartaee-test,integration-tests,simple-webapp,jakartaee-simple-webapp,jsf-app,jsf-app-dependent,jsf-app-standalone,jakartaee-jsf-app,jakartaee-jsf-app-dependent,jakartaee-jsf-app-standalone,cdi-app,cdi-app-dependent,cdi-app-standalone,soap-test,external-plugin-test,plugin-instrumentation-target,external-plugin-app,external-plugin-jakarta-app,external-plugin,application-server-integration-tests,spring-boot-1-5,spring-boot-2,spring-boot-2-base,spring-boot-2-jetty,spring-boot-2-tomcat,spring-boot-2-undertow,cdi-jakartaee-app,cdi-jakartaee-app-dependent,cdi-jakartaee-app-standalone,quarkus,quarkus-jaxrs-base,quarkus-jaxrs-undertow,quarkus-jaxrs-vertx,external-plugin-otel-test,external-plugin-otel-test-app,external-plugin-otel-test-plugin1,external-plugin-otel-test-plugin2,runtime-attach,runtime-attach-app,runtime-attach-test,main-app-test,spring-boot-3,spring-boot-3-jetty,spring-boot-3-tomcat,spring-boot-3-undertow,aws-lambda-test,spring-boot-4,spring-boot-4-base,spring-boot-4-jetty,spring-boot-4-tomcat,apm-spring-webmvc-spring7-test
+ apm-agent-common,apm-agent-cached-lookup-key,apm-agent-bootstrap,apm-agent-core,elastic-apm-agent-premain,apm-agent-plugins,apm-httpclient-core,apm-apache-httpclient,apm-apache-httpclient3-plugin,apm-apache-httpclient-common,apm-apache-httpclient4-plugin,apm-apache-httpclient5-plugin,apm-api-plugin,apm-asynchttpclient-plugin,apm-java-concurrent-plugin,apm-cassandra,apm-cassandra-core-plugin,apm-cassandra3-plugin,apm-cassandra4-plugin,apm-okhttp-plugin,apm-dubbo-plugin,apm-es-restclient-plugin,apm-es-restclient-plugin-common,apm-es-restclient-plugin-5_6,apm-es-restclient-plugin-6_4,apm-es-restclient-plugin-7_x,apm-es-restclient-plugin-8_x,apm-grails-plugin,apm-grpc,apm-grpc-plugin,apm-hibernate-search-plugin,apm-hibernate-search-plugin-common,apm-hibernate-search-plugin-5_x,apm-hibernate-search-plugin-6_x,apm-httpserver-core,apm-servlet-plugin,apm-javalin-plugin,apm-jaxrs-plugin,apm-jaxws-plugin,apm-jdbc-plugin,apm-jdk-httpclient-plugin,apm-jdk-httpserver-plugin,apm-redis-plugin,apm-redis-common,apm-jedis-plugin,apm-jedis-4-plugin,apm-jms-plugin,apm-jms-plugin-base,apm-jms-javax,apm-jms-jakarta,apm-jmx-plugin,apm-jsf-plugin,apm-kafka-plugin,apm-kafka-base-plugin,apm-kafka-headers-plugin,apm-kafka-spring-plugin,apm-lettuce-plugin,apm-logging-plugin,apm-logging-plugin-common,apm-log4j1-plugin,apm-log4j2-plugin,apm-opentelemetry,apm-opentelemetry-embedded-metrics-sdk,apm-opentelemetry-metrics-bridge-parent,apm-opentelemetry-metrics-bridge-common,apm-opentelemetry-metrics-bridge-v1_14,apm-opentelemetry-metrics-bridge-latest,apm-opentelemetry-metricsdk-plugin,apm-opentelemetry-plugin,apm-slf4j-plugin,apm-logback-plugin,apm-logback-plugin-impl,apm-jboss-logging-plugin,apm-jul-plugin,apm-tomcat-logging-plugin,apm-micrometer-plugin,apm-mongodb,apm-mongodb-common,apm-mongodb3-plugin,apm-mongodb4-plugin,apm-opentracing-plugin,apm-process-plugin,apm-profiling-plugin,apm-quartz,apm-quartz-common,apm-quartz-plugin-1,apm-quartz-plugin-2,apm-rabbitmq,apm-rabbitmq-plugin,apm-rabbitmq-spring5,apm-reactor-plugin,apm-redisson-plugin,apm-scala-concurrent-plugin,apm-scheduled-annotation-plugin,apm-sparkjava-plugin,apm-urlconnection-plugin,apm-spring-ai-plugin,apm-spring-resttemplate,apm-spring-resttemplate-plugin,apm-spring-webflux,apm-spring-webflux-common-spring5,apm-spring-webflux-common,apm-spring-webclient-plugin,apm-spring-webflux-testapp,apm-spring-webflux-testapp-spring7,apm-spring-webflux-spring5,apm-spring-webflux-plugin,apm-spring-webflux-plugin-spring7,apm-spring-webmvc,apm-spring-webmvc-spring5,apm-spring-webmvc-plugin,apm-struts-plugin,apm-vertx,apm-vertx-common,apm-vertx3-plugin,apm-vertx4-plugin,apm-awslambda-plugin,apm-jakarta-websocket-plugin,apm-ecs-logging-plugin,apm-aws-sdk,apm-aws-sdk-common,apm-aws-sdk-1-plugin,apm-aws-sdk-2-plugin,apm-finagle-httpclient-plugin,apm-java-ldap-plugin,apm-agent-builds,apm-agent,apm-agent-lambda-layer,apm-agent-java8,apm-agent-benchmarks,apm-spring-resttemplate-test,apm-spring-restclient-test,apm-logback-plugin-legacy-tests,apm-log4j2-plugin-tests,apm-jms-spring-plugin,apm-jedis-2-tests,apm-jedis-3-tests,apm-jedis-5-tests,apm-lettuce-3-tests,apm-grpc-test-1.6.1,apm-grpc-test-latest,apm-rabbitmq-test-3,apm-rabbitmq-test-4,apm-rabbitmq-test-spring6,apm-okhttp-test,apm-opentelemetry-test,apm-vertx3-test-latest,apm-servlet-jakarta-test,apm-jaxws-plugin-jakartaee-test,apm-jaxrs-plugin-jakartaee-test,apm-scheduled-annotation-plugin-jakartaee-test,integration-tests,simple-webapp,jakartaee-simple-webapp,jsf-app,jsf-app-dependent,jsf-app-standalone,jakartaee-jsf-app,jakartaee-jsf-app-dependent,jakartaee-jsf-app-standalone,cdi-app,cdi-app-dependent,cdi-app-standalone,soap-test,external-plugin-test,plugin-instrumentation-target,external-plugin-app,external-plugin-jakarta-app,external-plugin,application-server-integration-tests,spring-boot-1-5,spring-boot-2,spring-boot-2-base,spring-boot-2-jetty,spring-boot-2-tomcat,spring-boot-2-undertow,cdi-jakartaee-app,cdi-jakartaee-app-dependent,cdi-jakartaee-app-standalone,quarkus,quarkus-jaxrs-base,quarkus-jaxrs-undertow,external-plugin-otel-test,external-plugin-otel-test-app,external-plugin-otel-test-plugin1,external-plugin-otel-test-plugin2,runtime-attach,runtime-attach-app,runtime-attach-test,main-app-test,spring-boot-3,spring-boot-3-jetty,spring-boot-3-tomcat,spring-boot-3-undertow,aws-lambda-test,spring-boot-4,spring-boot-4-base,spring-boot-4-jetty,spring-boot-4-tomcat,apm-spring-webmvc-spring7-test