From e29c3a90818cc85372724eb9d5bce8a67b550775 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Mon, 16 Mar 2026 21:39:21 +0800 Subject: [PATCH 01/11] Support Virtual-GenAI monitoring --- .github/workflows/skywalking.yaml | 2 + docs/en/setup/service-agent/virtual-genai.md | 16 + oap-server/analyzer/agent-analyzer/pom.xml | 5 + .../VirtualServiceAnalysisListener.java | 21 +- .../vservice/VirtualGenAIProcessor.java | 101 ++++++ oap-server/analyzer/genAI-analyzer/pom.xml | 37 ++ .../analyzer/GenAIAnalyzerModuleProvider.java | 94 +++++ .../meter/analyzer/config/GenAIConfig.java | 50 +++ .../analyzer/config/GenAIConfigLoader.java | 110 ++++++ .../meter/analyzer/config/GenAIOALDefine.java | 33 ++ .../meter/analyzer/config/GenAITagKey.java | 28 ++ .../matcher/GenAIProviderPrefixMatcher.java | 115 ++++++ .../analyzer/module/GenAIAnalyzerModule.java | 38 ++ .../analyzer/service/GenAIMeterAnalyzer.java | 130 +++++++ .../service/GenAIModelAccessDispatcher.java | 37 ++ .../service/IGenAIMeterAnalyzerService.java | 30 ++ ...ing.oap.server.library.module.ModuleDefine | 19 + ...g.oap.server.library.module.ModuleProvider | 18 + oap-server/analyzer/pom.xml | 1 + .../skywalking/oal/rt/grammar/OALLexer.g4 | 2 + .../skywalking/oal/rt/grammar/OALParser.g4 | 3 +- .../generator/RuntimeOALGenerationTest.java | 6 + .../ui/template/UITemplateInitializer.java | 1 + .../core/source/DefaultScopeDefine.java | 2 + .../oap/server/core/source/GenAIMetrics.java | 45 +++ .../server/core/source/GenAIModelAccess.java | 71 ++++ .../core/source/GenAIProviderAccess.java | 62 ++++ oap-server/server-starter/pom.xml | 1 + .../src/main/resources/application.yml | 4 + .../src/main/resources/gen-ai-config.yml | 95 +++++ .../src/main/resources/oal/virtual-gen-ai.oal | 45 +++ .../ui-initialized-templates/menu.yaml | 10 + .../rocketmq/rocketmq-root.json | 2 +- .../virtual_genai/virtual-genai-model.json | 326 ++++++++++++++++++ .../virtual_genai/virtual-genai-provider.json | 280 +++++++++++++++ .../virtual_genai/virtual-genai-root.json | 57 +++ .../cases/storage/expected/config-dump.yml | 1 + .../cases/virtual-genai/Dockerfile.provider | 41 +++ .../cases/virtual-genai/docker-compose.yml | 69 ++++ test/e2e-v2/cases/virtual-genai/e2e.yaml | 44 +++ .../cases/virtual-genai/expected/instance.yml | 22 ++ .../expected/metrics-has-value-label.yml | 38 ++ .../expected/metrics-has-value.yml | 34 ++ .../cases/virtual-genai/expected/service.yml | 24 ++ .../cases/virtual-genai/virtual-genai.yaml | 65 ++++ .../e2e-service-provider/pom.xml | 6 + .../e2e/controller/LLMMockController.java | 107 ++++++ test/e2e-v2/script/env | 2 +- 48 files changed, 2341 insertions(+), 9 deletions(-) create mode 100644 docs/en/setup/service-agent/virtual-genai.md create mode 100644 oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java create mode 100644 oap-server/analyzer/genAI-analyzer/pom.xml create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine create mode 100644 oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java create mode 100644 oap-server/server-starter/src/main/resources/gen-ai-config.yml create mode 100644 oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal create mode 100644 oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json create mode 100644 oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json create mode 100644 oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json create mode 100644 test/e2e-v2/cases/virtual-genai/Dockerfile.provider create mode 100644 test/e2e-v2/cases/virtual-genai/docker-compose.yml create mode 100644 test/e2e-v2/cases/virtual-genai/e2e.yaml create mode 100644 test/e2e-v2/cases/virtual-genai/expected/instance.yml create mode 100644 test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml create mode 100644 test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml create mode 100644 test/e2e-v2/cases/virtual-genai/expected/service.yml create mode 100644 test/e2e-v2/cases/virtual-genai/virtual-genai.yaml create mode 100644 test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 3e4158e9699a..a663789297d3 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -627,6 +627,8 @@ jobs: config: test/e2e-v2/cases/zipkin/kafka/e2e.yaml - name: Zipkin BanyanDB config: test/e2e-v2/cases/zipkin/banyandb/e2e.yaml + - name: Virtual-genai + config: test/e2e-v2/cases/virtual-genai/e2e.yaml - name: Nginx config: test/e2e-v2/cases/nginx/e2e.yaml diff --git a/docs/en/setup/service-agent/virtual-genai.md b/docs/en/setup/service-agent/virtual-genai.md new file mode 100644 index 000000000000..1c9feb68dbdc --- /dev/null +++ b/docs/en/setup/service-agent/virtual-genai.md @@ -0,0 +1,16 @@ +# Virtual GenAI + +Virtual cache represent the Generative AI service nodes detected by [server agents' plugins](server-agents.md). The performance +metrics of the GenAI operations are also from the GenAI client-side perspective. + +For example, an Spring-ai plugin in the Java agent could detect the latency of a chat completion request. +As a result, SkyWalking would show traffic, latency, success rate, and token usage (input/output) powered by backend analysis capabilities in this dashboard. + +The GenAI operation span should have +- It is an **Exit** span +- **Span's layer == GENAI** +- Tag key = `gen_ai.provider.name`, value = The Generative AI provider, e.g. openai, anthropic, ollama +- Tag key = `gen_ai.response.model`, value = The name of the GenAI model a response is being made to, e.g. gpt-4o, claude-3-5-sonnet +- Tag key = `gen_ai.usage.input_tokens`, value = The number of tokens used in the GenAI input (prompt) +- Tag key = `gen_ai.usage.output_tokens`, value = The number of tokens used in the GenAI response (completion) +- If the GenAI service is a remote API (e.g. OpenAI), the span's peer would be the network address (IP or domain) of the GenAI server. diff --git a/oap-server/analyzer/agent-analyzer/pom.xml b/oap-server/analyzer/agent-analyzer/pom.xml index 5a281800589b..391d0765035e 100644 --- a/oap-server/analyzer/agent-analyzer/pom.xml +++ b/oap-server/analyzer/agent-analyzer/pom.xml @@ -43,6 +43,11 @@ meter-analyzer ${project.version} + + org.apache.skywalking + genAI-analyzer + ${project.version} + org.apache.skywalking server-testing diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java index 95c0ac47fccb..90e15c41ec76 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java @@ -20,12 +20,16 @@ import java.util.Arrays; import java.util.List; + import lombok.RequiredArgsConstructor; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.oap.meter.analyzer.module.GenAIAnalyzerModule; +import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualCacheProcessor; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualDatabaseProcessor; +import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualGenAIProcessor; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualMQProcessor; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualServiceProcessor; import org.apache.skywalking.oap.server.core.CoreModule; @@ -71,23 +75,28 @@ public void parseEntry(final SpanObject span, final SegmentObject segmentObject) public static class Factory implements AnalysisListenerFactory { private final SourceReceiver sourceReceiver; private final NamingControl namingControl; + private final IGenAIMeterAnalyzerService genAIMeterAnalyzerService; public Factory(ModuleManager moduleManager) { this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); this.namingControl = moduleManager.find(CoreModule.NAME) .provider() .getService(NamingControl.class); + this.genAIMeterAnalyzerService = moduleManager.find(GenAIAnalyzerModule.NAME) + .provider() + .getService(IGenAIMeterAnalyzerService.class); } @Override public AnalysisListener create(ModuleManager moduleManager, AnalyzerModuleConfig config) { return new VirtualServiceAnalysisListener( - sourceReceiver, - Arrays.asList( - new VirtualCacheProcessor(namingControl, config), - new VirtualDatabaseProcessor(namingControl, config), - new VirtualMQProcessor(namingControl) - ) + sourceReceiver, + Arrays.asList( + new VirtualCacheProcessor(namingControl, config), + new VirtualDatabaseProcessor(namingControl, config), + new VirtualMQProcessor(namingControl), + new VirtualGenAIProcessor(genAIMeterAnalyzerService) + ) ); } } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java new file mode 100644 index 000000000000..1b1c3e7eb423 --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.source.GenAIMetrics; +import org.apache.skywalking.oap.server.core.source.GenAIModelAccess; +import org.apache.skywalking.oap.server.core.source.GenAIProviderAccess; +import org.apache.skywalking.oap.server.core.source.ServiceMeta; +import org.apache.skywalking.oap.server.core.source.Source; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +@RequiredArgsConstructor +public class VirtualGenAIProcessor implements VirtualServiceProcessor { + + private final IGenAIMeterAnalyzerService meterAnalyzerService; + + private List recordList = new ArrayList<>(); + + @Override + public void prepareVSIfNecessary(SpanObject span, SegmentObject segmentObject) { + if (span.getSpanLayer() != SpanLayer.GenAI) { + return; + } + + GenAIMetrics metrics = meterAnalyzerService.extractMetricsFromSWSpan(span, segmentObject); + if (metrics == null) { + return; + } + + recordList.add(toServiceMeta(metrics)); + recordList.add(toProviderAccess(metrics)); + recordList.add(toModelAccess(metrics)); + } + + private ServiceMeta toServiceMeta(GenAIMetrics metrics) { + ServiceMeta service = new ServiceMeta(); + service.setName(metrics.getProviderName()); + service.setLayer(Layer.VIRTUAL_GENAI); + service.setTimeBucket(metrics.getTimeBucket()); + return service; + } + + private GenAIProviderAccess toProviderAccess(GenAIMetrics metrics) { + GenAIProviderAccess source = new GenAIProviderAccess(); + source.setName(metrics.getProviderName()); + source.setInputTokens(metrics.getInputTokens()); + source.setOutputTokens(metrics.getOutputTokens()); + source.setTotalCost(metrics.getTotalCost()); + source.setLatency(metrics.getLatency()); + source.setStatus(metrics.isStatus()); + source.setTimeBucket(metrics.getTimeBucket()); + return source; + } + + private GenAIModelAccess toModelAccess(GenAIMetrics metrics) { + GenAIModelAccess source = new GenAIModelAccess(); + source.setServiceName(metrics.getProviderName()); + source.setModelName(metrics.getModelName()); + source.setInputTokens(metrics.getInputTokens()); + source.setOutputTokens(metrics.getOutputTokens()); + source.setTotalCost(metrics.getTotalCost()); + source.setTimeToFirstToken(metrics.getTimeToFirstToken()); + source.setLatency(metrics.getLatency()); + source.setStatus(metrics.isStatus()); + source.setTimeBucket(metrics.getTimeBucket()); + return source; + } + + @Override + public void emitTo(Consumer consumer) { + for (Source source : recordList) { + if (source != null) { + consumer.accept(source); + } + } + } +} diff --git a/oap-server/analyzer/genAI-analyzer/pom.xml b/oap-server/analyzer/genAI-analyzer/pom.xml new file mode 100644 index 000000000000..52ee1bddf325 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/pom.xml @@ -0,0 +1,37 @@ + + + + + + analyzer + org.apache.skywalking + ${revision} + + 4.0.0 + + genAI-analyzer + + + + org.apache.skywalking + server-core + ${project.version} + + + diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java new file mode 100644 index 000000000000..fa6b2a2798fa --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer; + +import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfig; +import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfigLoader; +import org.apache.skywalking.oap.meter.analyzer.config.GenAIOALDefine; +import org.apache.skywalking.oap.meter.analyzer.matcher.GenAIProviderPrefixMatcher; +import org.apache.skywalking.oap.meter.analyzer.module.GenAIAnalyzerModule; +import org.apache.skywalking.oap.meter.analyzer.service.GenAIMeterAnalyzer; +import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.yaml.snakeyaml.Yaml; + +public class GenAIAnalyzerModuleProvider extends ModuleProvider { + + private GenAIConfig config; + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return GenAIAnalyzerModule.class; + } + + @Override + public ConfigCreator newConfigCreator() { + return new ConfigCreator() { + @Override + public Class type() { + return GenAIConfig.class; + } + + @Override + public void onInitialized(final GenAIConfig initialized) { + config = initialized; + } + }; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + GenAIConfigLoader loader = new GenAIConfigLoader(config, new Yaml()); + config = loader.loadConfig(); + GenAIProviderPrefixMatcher matcher = GenAIProviderPrefixMatcher.build(config); + this.registerServiceImplementation( + IGenAIMeterAnalyzerService.class, + new GenAIMeterAnalyzer(matcher) + ); + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + getManager().find(CoreModule.NAME) + .provider() + .getService(OALEngineLoaderService.class) + .load(GenAIOALDefine.INSTANCE); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public String[] requiredModules() { + return new String[0]; + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java new file mode 100644 index 000000000000..41682ca97637 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.config; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +import java.util.ArrayList; +import java.util.List; + +public class GenAIConfig extends ModuleConfig { + + @Getter + @Setter + private List providers = new ArrayList<>(); + + @Getter + @Setter + public static class Provider { + private String provider; + private String baseUrl; + private List prefixMatch = new ArrayList<>(); + private List models = new ArrayList<>(); + } + + @Getter + @Setter + public static class Model { + private String name; + private double inputCostPerM; + private double outputCostPerM; + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java new file mode 100644 index 000000000000..4fcede0b3ed7 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.config; + +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.yaml.snakeyaml.Yaml; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.Reader; +import java.util.List; +import java.util.Map; + +public class GenAIConfigLoader { + + private final GenAIConfig config; + + public GenAIConfigLoader(GenAIConfig config, Yaml yaml) { + this.config = config; + } + + public GenAIConfig loadConfig() throws ModuleStartException { + Map>> configMap; + try (Reader applicationReader = ResourceUtils.read("gen-ai-config.yml")) { + Yaml yaml = new Yaml(); + configMap = yaml.loadAs(applicationReader, Map.class); + } catch (FileNotFoundException e) { + throw new ModuleStartException( + "Cannot find the GenAI configuration file [gen-ai-config.yml].", e); + } catch (IOException e) { + throw new ModuleStartException( + "Failed to read the GenAI configuration file [gen-ai-config.yml].", e); + } + + if (configMap == null || !configMap.containsKey("providers")) { + return config; + } + + List> providersConfig = configMap.get("providers"); + for (Map providerMap : providersConfig) { + GenAIConfig.Provider provider = new GenAIConfig.Provider(); + + Object name = providerMap.get("provider"); + if (name == null) { + throw new ModuleStartException("Provider name is missing in [gen-ai-config.yml]."); + } + provider.setProvider(name.toString()); + + Object baseUrl = providerMap.get("base-url"); + if (baseUrl != null && StringUtil.isNotBlank(baseUrl.toString())) { + provider.setBaseUrl(baseUrl.toString()); + } + + Object prefixMatch = providerMap.get("prefix-match"); + if (prefixMatch instanceof List) { + provider.getPrefixMatch().addAll((List) prefixMatch); + } else if (prefixMatch != null) { + throw new ModuleStartException("prefix-match must be a list in [gen-ai-config.yml] for provider: " + name); + } + + // Parse specific model overrides + Object modelsConfig = providerMap.get("models"); + if (modelsConfig instanceof List) { + for (Object modelObj : (List) modelsConfig) { + if (modelObj instanceof Map) { + Map modelMap = (Map) modelObj; + GenAIConfig.Model model = new GenAIConfig.Model(); + model.setName(String.valueOf(modelMap.get("name"))); + model.setInputCostPerM(parseCost(modelMap.get("input-cost-per-m"))); + model.setOutputCostPerM(parseCost(modelMap.get("output-cost-per-m"))); + provider.getModels().add(model); + } + } + } + + config.getProviders().add(provider); + } + + return config; + } + + private double parseCost(Object value) { + if (value == null) { + return 0.0; + } + try { + return Double.parseDouble(value.toString()); + } catch (NumberFormatException e) { + return 0.0; + } + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java new file mode 100644 index 000000000000..97c399def037 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.config; + +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; + +public class GenAIOALDefine extends OALDefine { + + public static final GenAIOALDefine INSTANCE = new GenAIOALDefine(); + + private GenAIOALDefine() { + super( + "oal/virtual-gen-ai.oal", + "org.apache.skywalking.oap.server.core.source" + ); + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java new file mode 100644 index 000000000000..456b36e7275f --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.meter.analyzer.config; + +public class GenAITagKey { + + public static final String PROVIDER_NAME = "gen_ai.provider.name"; + + public static final String RESPONSE_MODEL = "gen_ai.response.model"; + public static final String INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + public static final String STREAM_TTFT = "gen_ai.stream.ttfr"; +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java new file mode 100644 index 000000000000..a0802f41d396 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.matcher; + +import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfig; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GenAIProviderPrefixMatcher { + private static final String UNKNOWN = "unknown"; + private final TrieNode root; + private final Map modelMap; + + private static final MatchResult UNKNOWN_RESULT = new MatchResult(UNKNOWN, null); + + private GenAIProviderPrefixMatcher(TrieNode root, Map modelMap) { + this.root = root; + this.modelMap = modelMap; + } + + private static class TrieNode { + final Map children = new HashMap<>(); + String providerName; + } + + public static class MatchResult { + private final String provider; + private final GenAIConfig.Model modelConfig; + + public MatchResult(String provider, GenAIConfig.Model modelConfig) { + this.provider = provider; + this.modelConfig = modelConfig; + } + + public String getProvider() { + return provider; + } + + public GenAIConfig.Model getModelConfig() { + return modelConfig; + } + } + + public static GenAIProviderPrefixMatcher build(GenAIConfig config) { + TrieNode root = new TrieNode(); + Map modelMap = new HashMap<>(); + + for (GenAIConfig.Provider p : config.getProviders()) { + List prefixes = p.getPrefixMatch(); + if (prefixes != null) { + for (String prefix : prefixes) { + if (prefix == null || prefix.isEmpty()) continue; + + TrieNode current = root; + for (int i = 0; i < prefix.length(); i++) { + char c = prefix.charAt(i); + current = current.children.computeIfAbsent(c, k -> new TrieNode()); + } + current.providerName = p.getProvider(); + } + } + + List models = p.getModels(); + if (models != null) { + for (GenAIConfig.Model model : models) { + if (model.getName() != null) { + modelMap.put(model.getName(), model); + } + } + } + } + + return new GenAIProviderPrefixMatcher(root, modelMap); + } + + public MatchResult match(String modelName) { + if (modelName == null || modelName.isEmpty()) { + return UNKNOWN_RESULT; + } + + TrieNode current = root; + String matchedProvider = null; + + for (int i = 0; i < modelName.length(); i++) { + current = current.children.get(modelName.charAt(i)); + if (current == null) break; + if (current.providerName != null) { + matchedProvider = current.providerName; + } + } + + String provider = matchedProvider != null ? matchedProvider : UNKNOWN; + GenAIConfig.Model modelConfig = modelMap.get(modelName); + + return new MatchResult(provider, modelConfig); + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java new file mode 100644 index 000000000000..0945b7f89d95 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.module; + +import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class GenAIAnalyzerModule extends ModuleDefine { + + public static final String NAME = "genAI-analyzer"; + + public GenAIAnalyzerModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] { + IGenAIMeterAnalyzerService.class, + }; + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java new file mode 100644 index 000000000000..1e67f1c11053 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.meter.analyzer.service; + +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfig; +import org.apache.skywalking.oap.meter.analyzer.config.GenAITagKey; +import org.apache.skywalking.oap.meter.analyzer.matcher.GenAIProviderPrefixMatcher; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.source.GenAIMetrics; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static java.util.stream.Collectors.toMap; + +public class GenAIMeterAnalyzer implements IGenAIMeterAnalyzerService { + + private static final Logger LOG = LoggerFactory.getLogger(GenAIMeterAnalyzer.class); + + private final GenAIProviderPrefixMatcher matcher; + + public GenAIMeterAnalyzer(GenAIProviderPrefixMatcher matcher) { + this.matcher = matcher; + } + + @Override + public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segment) { + Map tags = span.getTagsList().stream() + .collect(toMap( + KeyStringValuePair::getKey, + KeyStringValuePair::getValue, + (v1, v2) -> v1 + )); + + String modelName = tags.get(GenAITagKey.RESPONSE_MODEL); + + if (StringUtil.isBlank(modelName)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Model name is missing in span [{}], skipping GenAI analysis", span.getOperationName()); + } + return null; + } + String provider = tags.get(GenAITagKey.PROVIDER_NAME); + GenAIProviderPrefixMatcher.MatchResult matchResult = matcher.match(modelName); + + if (StringUtil.isBlank(provider)) { + provider = matchResult.getProvider(); + } + + GenAIConfig.Model modelConfig = matchResult.getModelConfig(); + + long inputTokens = parseSafeLong(tags.get(GenAITagKey.INPUT_TOKENS)); + long outputTokens = parseSafeLong(tags.get(GenAITagKey.OUTPUT_TOKENS)); + + // calculate the total cost by the cost configs + double totalCost = 0.0; + if (modelConfig != null) { + if (modelConfig.getInputCostPerM() > 0) { + totalCost += inputTokens * modelConfig.getInputCostPerM(); + } + if (modelConfig.getOutputCostPerM() > 0) { + totalCost += outputTokens * modelConfig.getOutputCostPerM(); + } + } + + GenAIMetrics metrics = new GenAIMetrics(); + + metrics.setServiceId(IDManager.ServiceID.buildId(provider, Layer.VIRTUAL_GENAI.isNormal())); + metrics.setProviderName(provider); + metrics.setModelName(modelName); + metrics.setInputTokens(inputTokens); + metrics.setOutputTokens(outputTokens); + + metrics.setTimeToFirstToken(parseSafeInt(tags.get(GenAITagKey.STREAM_TTFT))); + metrics.setTotalCost(totalCost); + + long latency = span.getEndTime() - span.getStartTime(); + metrics.setLatency(latency); + metrics.setStatus(!span.getIsError()); + metrics.setTimeBucket(TimeBucket.getMinuteTimeBucket(span.getStartTime())); + + return metrics; + } + + private long parseSafeLong(String value) { + if (StringUtil.isEmpty(value)) { + return 0; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + LOG.warn("Failed to parse token count: {}", value); + return 0; + } + } + + private int parseSafeInt(String value) { + if (StringUtil.isEmpty(value)) { + return 0; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + LOG.warn("Failed to parse token count: {}", value); + return 0; + } + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java new file mode 100644 index 000000000000..9c0fa069b8a2 --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.service; + +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; +import org.apache.skywalking.oap.server.core.source.GenAIModelAccess; + +public class GenAIModelAccessDispatcher implements SourceDispatcher { + + @Override + public void dispatch(GenAIModelAccess source) { + InstanceTraffic traffic = new InstanceTraffic(); + traffic.setTimeBucket(source.getTimeBucket()); + traffic.setName(source.getModelName()); + traffic.setServiceId(source.getServiceId()); + traffic.setLastPingTimestamp(source.getTimeBucket()); + MetricsStreamProcessor.getInstance().in(traffic); + } +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java new file mode 100644 index 000000000000..6b00eeb024da --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.meter.analyzer.service; + +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.oap.server.core.source.GenAIMetrics; +import org.apache.skywalking.oap.server.library.module.Service; + +public interface IGenAIMeterAnalyzerService extends Service { + + GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segment); + +} diff --git a/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000000..c648cced34ee --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.meter.analyzer.module.GenAIAnalyzerModule \ No newline at end of file diff --git a/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..1eef579246fa --- /dev/null +++ b/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.skywalking.oap.meter.analyzer.GenAIAnalyzerModuleProvider \ No newline at end of file diff --git a/oap-server/analyzer/pom.xml b/oap-server/analyzer/pom.xml index 8cad4dff5a9d..fa78c963f434 100644 --- a/oap-server/analyzer/pom.xml +++ b/oap-server/analyzer/pom.xml @@ -34,6 +34,7 @@ meter-analyzer log-analyzer hierarchy + genAI-analyzer diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index 75a37da627db..497822d54c24 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -63,6 +63,8 @@ SRC_CILIUM_ENDPOINT: 'CiliumEndpoint'; SRC_CILIUM_SERVICE_RELATION: 'CiliumServiceRelation'; SRC_CILIUM_SERVICE_INSTANCE_RELATION: 'CiliumServiceInstanceRelation'; SRC_CILIUM_ENDPOINT_RELATION: 'CiliumEndpointRelation'; +SRC_GEN_AI_PROVIDER_ACCESS: 'GenAIProviderAccess'; +SRC_GEN_AI_MODEL_ACCESS: 'GenAIModelAccess'; DECORATOR: 'decorator'; diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index 38713ff208ca..fcefe29d53c8 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -63,7 +63,8 @@ source SRC_BROWSER_APP_TRAFFIC | SRC_BROWSER_APP_PAGE_TRAFFIC | SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC | SRC_EVENT | SRC_MQ_ACCESS | SRC_MQ_ENDPOINT_ACCESS | SRC_K8S_SERVICE | SRC_K8S_SERVICE_INSTANCE | SRC_K8S_ENDPOINT | SRC_K8S_SERVICE_RELATION | SRC_K8S_SERVICE_INSTANCE_RELATION | - SRC_CILIUM_SERVICE | SRC_CILIUM_SERVICE_INSTANCE | SRC_CILIUM_ENDPOINT | SRC_CILIUM_SERVICE_RELATION | SRC_CILIUM_SERVICE_INSTANCE_RELATION | SRC_CILIUM_ENDPOINT_RELATION + SRC_CILIUM_SERVICE | SRC_CILIUM_SERVICE_INSTANCE | SRC_CILIUM_ENDPOINT | SRC_CILIUM_SERVICE_RELATION | SRC_CILIUM_SERVICE_INSTANCE_RELATION | SRC_CILIUM_ENDPOINT_RELATION | + SRC_GEN_AI_PROVIDER_ACCESS | SRC_GEN_AI_MODEL_ACCESS ; disableSource diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java index 27e63c884a05..87691eae094c 100644 --- a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java @@ -98,6 +98,8 @@ public static void setup() { // DisableOALDefine - no catalog registerOALDefine("disable", createOALDefine("oal/disable.oal", SOURCE_PACKAGE, "")); + registerOALDefine("disable", createOALDefine("oal/virtual-gen-ai.oal", SOURCE_PACKAGE, "")); + // Set generated file path for IDE inspection OALClassGeneratorV2.setGeneratedFilePath("target/test-classes"); } @@ -185,6 +187,10 @@ private static void initializeScopes() { notifyClass(listener, SOURCE_PACKAGE, "Process"); notifyClass(listener, SOURCE_PACKAGE, "ProcessRelation"); + // gen_ai + notifyClass(listener, SOURCE_PACKAGE, "GenAIProviderAccess"); + notifyClass(listener, SOURCE_PACKAGE, "GenAIModelAccess"); + // Register decorators registerDecorator(SOURCE_PACKAGE, "ServiceDecorator"); registerDecorator(SOURCE_PACKAGE, "EndpointDecorator"); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java index dd01e97c19d8..525ccf11e2fc 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/ui/template/UITemplateInitializer.java @@ -81,6 +81,7 @@ public class UITemplateInitializer { Layer.SO11Y_GO_AGENT.name(), Layer.FLINK.name(), Layer.BANYANDB.name(), + Layer.VIRTUAL_GENAI.name(), "custom" }; private final UITemplateManagementService uiTemplateManagementService; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 2650babbb125..0916fe4fdd59 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -156,6 +156,8 @@ public class DefaultScopeDefine { public static final int PPROF_PROFILING_DATA = 93; public static final int PPROF_TASK_LOG = 94; public static final int ALARM_RECOVERY = 95; + public static final int GEN_AI_PROVIDER_ACCESS = 96; + public static final int GEN_AI_MODEL_ACCESS = 97; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java new file mode 100644 index 000000000000..a6e91947f927 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; + +@Data +public class GenAIMetrics { + + private String serviceId; + + private String providerName; + + private String modelName; + + private long inputTokens; + + private long outputTokens; + + private double totalCost; + + private int timeToFirstToken; + + private long latency; + + private boolean status; + + private long timeBucket; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java new file mode 100644 index 000000000000..3636a6dfe058 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.GEN_AI_MODEL_ACCESS; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; + +@Data +@ScopeDeclaration(id = GEN_AI_MODEL_ACCESS, name = "GenAIModelAccess", catalog = SERVICE_INSTANCE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class GenAIModelAccess extends Source { + + @Override + public int scope() { + return GEN_AI_MODEL_ACCESS; + } + + @Override + public String getEntityId() { + return entityId; + } + + private String entityId; + + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) + private String serviceName; + + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + private String serviceId; + + @ScopeDefaultColumn.DefinedByField(columnName = "name") + private String modelName; + + private long inputTokens; + + private long outputTokens; + + private double totalCost; + + private int timeToFirstToken; + + private long latency; + + private boolean status; + + @Override + public void prepare() { + serviceId = IDManager.ServiceID.buildId(serviceName, Layer.VIRTUAL_GENAI.isNormal()); + entityId = IDManager.ServiceInstanceID.buildId(serviceId, modelName); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java new file mode 100644 index 000000000000..cd08fb541419 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.GEN_AI_PROVIDER_ACCESS; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_CATALOG_NAME; + +@Data +@ScopeDeclaration(id = GEN_AI_PROVIDER_ACCESS, name = "GenAIProviderAccess", catalog = SERVICE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class GenAIProviderAccess extends Source { + + @Override + public int scope() { + return GEN_AI_PROVIDER_ACCESS; + } + + @Override + public String getEntityId() { + return entityId; + } + + private String entityId; + + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) + private String name; + + private long inputTokens; + + private long outputTokens; + + private double totalCost; + + private long latency; + + private boolean status; + + @Override + public void prepare() { + entityId = IDManager.ServiceID.buildId(name, Layer.VIRTUAL_GENAI.isNormal()); + } +} diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index ce1601659e81..09b5138c2f62 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -346,6 +346,7 @@ log-mal-rules/ telegraf-rules/ cilium-rules/ + gen-ai-settings.yml diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 3d9a06e6c6f6..2c866302fa9b 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -244,6 +244,10 @@ event-analyzer: selector: ${SW_EVENT_ANALYZER:default} default: +genAI-analyzer: + selector: ${SW_GENAI_ANALYZER:default} + default: + receiver-sharing-server: selector: ${SW_RECEIVER_SHARING_SERVER:default} default: diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml new file mode 100644 index 000000000000..c95667dc1e64 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file configures GenAI provider matching rules and estimated pricing. +# Rules define how to map model names to providers and calculate token costs. + +#Here is a example +#- provider: ollama +# # If a model name starts with these strings, it will be mapped to this provider +# prefix-match: +# - ollama +# models: +# - name: ollama +# # --- PRICING CONFIGURATION --- +# # Costs are defined as the price per 1,000,000 (one million) tokens. +# # The currency depends on your system's default setting (e.g., USD or CNY). +# +# # Estimated cost for every 1,000,000 input (prompt) tokens +# input-cost-per-m: 1 +# +# # Estimated cost for every 1,000,000 output (completion) tokens +# output-cost-per-m: 1 + +providers: + - provider: openai + prefix-match: + - gpt + + - provider: anthropic + prefix-match: + - claude + models: + - name: claude-3-7-sonnet-20250219-thinking + input-cost-per-m: 100 + output-cost-per-m: 100 + + - provider: gemini + prefix-match: + - gemini + + - provider: mistral + prefix-match: + - mistral- + + - provider: groq + prefix-match: + - llama-3 + + - provider: deepseek + prefix-match: + - deepseek + + - provider: bytedance + prefix-match: + - doubao + + - provider: zhipu_ai + prefix-match: + - glm + + - provider: alibaba + prefix-match: + - qwen + + - provider: tencent + prefix-match: + - hunyuan + + - provider: moonshot + prefix-match: + - kimi + + - provider: minimax + prefix-match: + - minimax + + - provider: ollama + prefix-match: + - ollama + + - provider: azure_openai + prefix-match: + - azure \ No newline at end of file diff --git a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal new file mode 100644 index 000000000000..dd5331c85200 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +gen_ai_provider_resp_time = from(GenAIProviderAccess.latency).longAvg(); +gen_ai_provider_sla = from(GenAIProviderAccess.*).percent(status == true); +gen_ai_provider_cpm = from(GenAIProviderAccess.*).cpm(); + +gen_ai_provider_latency_avg = from(GenAIProviderAccess.latency).longAvg(); +gen_ai_provider_latency_percentile = from(GenAIProviderAccess.latency).percentile(10); + +gen_ai_provider_input_tokens_sum = from(GenAIProviderAccess.inputTokens).sum(); +gen_ai_provider_output_tokens_sum = from(GenAIProviderAccess.outputTokens).sum(); + +gen_ai_provider_total_cost = from(GenAIProviderAccess.totalCost).sum(); + +gen_ai_model_call_cpm = from(GenAIModelAccess.*).cpm(); + +gen_ai_model_sla = from(GenAIModelAccess.*).percent(status == true); + +gen_ai_model_latency_avg = from(GenAIModelAccess.latency).longAvg(); + +gen_ai_model_latency_percentile = from(GenAIModelAccess.latency).percentile(10); + +gen_ai_model_ttft_avg = from(GenAIModelAccess.timeToFirstToken).filter(timeToFirstToken > 0).longAvg(); +gen_ai_model_ttft_percentile = from(GenAIModelAccess.timeToFirstToken).filter(timeToFirstToken > 0).percentile(10); + +gen_ai_model_input_tokens_sum = from(GenAIModelAccess.inputTokens).sum(); +gen_ai_model_output_tokens_sum = from(GenAIModelAccess.outputTokens).sum(); + +gen_ai_model_total_cost = from(GenAIModelAccess.totalCost).sum(); \ No newline at end of file diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml b/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml index ffe5059942d4..73b11a265eed 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml @@ -272,3 +272,13 @@ menus: description: The Go Agent for Apache SkyWalking, which provides the native tracing/metrics/logging abilities for Golang projects. documentLink: https://skywalking.apache.org/docs/main/next/en/setup/backend/dashboards-so11y-go-agent/ i18nKey: self_observability_go_agent + - title: GenAI + icon: gen_ai + description: Generative AI (GenAI) observability provides comprehensive monitoring and performance insights for various AI services and large language models (LLM). + i18nKey: virtual_genai + menus: + - title: Virtual GenAI + layer: VIRTUAL_GENAI + description: Observe the virtual GenAI providers and models which are conjectured by language agents through various plugins. + documentLink: https://skywalking.apache.org/docs/main/next/en/setup/service-agent/virtual-genai/ + i18nKey: virtual_gen_ai \ No newline at end of file diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/rocketmq/rocketmq-root.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/rocketmq/rocketmq-root.json index e89b31d355e4..5602bccbd6fd 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/rocketmq/rocketmq-root.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/rocketmq/rocketmq-root.json @@ -32,7 +32,7 @@ "content": "Provide RocketMQ monitoring through OpenTelemetry's Prometheus Receiver", "fontSize": 14, "textAlign": "left", - "url": "https://skywalking.apache.org/docs/main/next/en/setup/backend/backend-Rocketmq-monitoring/" + "url": "https://skywalking.apache.org/docs/main/next/en/setup/backend/backend-rocketmq-monitoring/" } } ], diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json new file mode 100644 index 000000000000..dbeafb0e38ac --- /dev/null +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json @@ -0,0 +1,326 @@ +[ + { + "id": "Virtual-GenAI-Model", + "configuration": { + "children": [ + { + "x": 0, + "y": 0, + "w": 24, + "h": 42, + "i": "0", + "type": "Tab", + "children": [ + { + "name": "Overview", + "children": [ + { + "x": 0, + "y": 0, + "w": 4, + "h": 11, + "i": "2", + "type": "Widget", + "expressions": [ + "latest(gen_ai_model_total_cost)/1000000" + ], + "graph": { + "type": "Card", + "fontSize": 24, + "textAlign": "center", + "showUnit": true + }, + "widget": { + "name": "TotalCost", + "title": "Total Cost", + "tips": "The total cost of GenAI model calls." + }, + "id": "0-0-2", + "moved": false, + "typesOfMQE": [ + "SINGLE_VALUE" + ] + }, + { + "x": 4, + "y": 0, + "w": 4, + "h": 11, + "i": "18", + "type": "Widget", + "expressions": [ + "latest(gen_ai_model_output_tokens_sum)" + ], + "graph": { + "type": "Card", + "fontSize": 24, + "textAlign": "center", + "showUnit": true + }, + "metricConfig": [ + { + "label": "Output Tokens" + } + ], + "widget": { + "name": "OutputTokens", + "title": "Output Tokens", + "tips": "The sum of output tokens." + }, + "id": "0-0-18", + "moved": false, + "typesOfMQE": [ + "SINGLE_VALUE" + ] + }, + { + "x": 8, + "y": 0, + "w": 4, + "h": 11, + "i": "19", + "type": "Widget", + "expressions": [ + "latest(gen_ai_model_input_tokens_sum)" + ], + "graph": { + "type": "Card", + "fontSize": 24, + "textAlign": "center", + "showUnit": true + }, + "metricConfig": [ + { + "label": "Input Tokens" + } + ], + "widget": { + "name": "InputTokens", + "title": "Input Tokens", + "tips": "The sum of input tokens." + }, + "id": "0-0-19", + "moved": false, + "typesOfMQE": [ + "SINGLE_VALUE" + ] + }, + { + "x": 12, + "y": 0, + "w": 12, + "h": 11, + "i": "1", + "type": "Widget", + "expressions": [ + "gen_ai_model_sla/100" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "name": "ServiceSLA", + "title": "Service SLA", + "tips": "Successful rate of GenAI model calls." + }, + "metricConfig": [ + { + "label": "SLA", + "unit": "%" + } + ], + "id": "0-0-1", + "moved": false + }, + { + "x": 0, + "y": 11, + "w": 12, + "h": 11, + "i": "0", + "type": "Widget", + "expressions": [ + "gen_ai_model_latency_avg" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "metricConfig": [ + { + "label": "Response Time", + "unit": "ms" + } + ], + "widget": { + "name": "ResponseTime", + "title": "Response Time", + "tips": "The average response time of GenAI model." + }, + "id": "0-0-0", + "moved": false + }, + { + "x": 12, + "y": 11, + "w": 12, + "h": 11, + "i": "5", + "type": "Widget", + "expressions": [ + "gen_ai_model_call_cpm" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "metricConfig": [ + { + "label": "CPM" + } + ], + "widget": { + "name": "CPM", + "title": "Calls Per Minute", + "tips": "The number of calls per minute." + }, + "id": "0-0-5", + "moved": false + }, + { + "x": 0, + "y": 22, + "w": 12, + "h": 12, + "i": "14", + "type": "Widget", + "expressions": [ + "gen_ai_model_latency_avg" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "name": "AvgLatency", + "title": "Average Latency", + "tips": "The average latency of model access." + }, + "metricConfig": [ + { + "label": "Avg Latency", + "unit": "ms" + } + ], + "id": "0-0-14", + "moved": false + }, + { + "x": 12, + "y": 22, + "w": 12, + "h": 12, + "i": "13", + "type": "Widget", + "expressions": [ + "gen_ai_model_latency_percentile" + ], + "graph": { + "type": "Bar", + "showBackground": true + }, + "widget": { + "name": "LatencyPercentile", + "title": "Latency Percentile", + "tips": "The percentile of model access latency." + }, + "id": "0-0-13", + "moved": false + }, + { + "x": 0, + "y": 34, + "w": 12, + "h": 10, + "i": "20", + "type": "Widget", + "widget": { + "name": "AvgTTFT", + "title": "Average TTFT", + "tips": "The average time from the start of the request until the model returns the first token (Time To First Token)." + }, + "metricConfig": [ + { + "label": "Avg TTFT", + "unit": "ms" + } + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "id": "0-0-20", + "moved": false, + "expressions": [ + "gen_ai_model_ttft_avg" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] + }, + { + "x": 12, + "y": 34, + "w": 12, + "h": 10, + "i": "21", + "type": "Widget", + "widget": { + "name": "TTFTPercentile", + "title": "TTFT Percentile", + "tips": "The percentile distribution of Time to First Token." + }, + "metricConfig": [ + { + "label": "TTFT Percentile", + "unit": "ms" + } + ], + "graph": { + "type": "Bar", + "showBackground": true + }, + "id": "0-0-21", + "moved": false, + "expressions": [ + "gen_ai_model_ttft_percentile" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] + } + ] + } + ], + "id": "0", + "activedTabIndex": 0, + "moved": false + } + ], + "layer": "VIRTUAL_GENAI", + "entity": "ServiceInstance", + "name": "Virtual-GenAI-Model", + "id": "Virtual-GenAI-Model", + "isRoot": false + } + } +] \ No newline at end of file diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json new file mode 100644 index 000000000000..dca377314f3c --- /dev/null +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json @@ -0,0 +1,280 @@ +[ + { + "id": "Virtual-GenAI-Provider", + "configuration": { + "children": [ + { + "x": 0, + "y": 0, + "w": 24, + "h": 42, + "i": "0", + "type": "Tab", + "children": [ + { + "name": "Overview", + "children": [ + { + "x": 0, + "y": 0, + "w": 4, + "h": 11, + "i": "2", + "type": "Widget", + "expressions": [ + "latest(gen_ai_provider_total_cost)/1000000" + ], + "graph": { + "type": "Card", + "fontSize": 24, + "textAlign": "center", + "showUnit": true + }, + "widget": { + "name": "TotalCost", + "title": "Total Cost", + "tips": "The total cost of GenAI model calls." + }, + "id": "0-0-2", + "moved": false, + "typesOfMQE": [ + "SINGLE_VALUE" + ] + }, + { + "x": 4, + "y": 0, + "w": 4, + "h": 11, + "i": "18", + "type": "Widget", + "expressions": [ + "latest(gen_ai_provider_output_tokens_sum)" + ], + "graph": { + "type": "Card", + "fontSize": 24, + "textAlign": "center", + "showUnit": true + }, + "metricConfig": [ + { + "label": "Output Tokens" + } + ], + "widget": { + "name": "OutputTokens", + "title": "Output Tokens", + "tips": "The sum of output tokens." + }, + "id": "0-0-18", + "moved": false, + "typesOfMQE": [ + "SINGLE_VALUE" + ] + }, + { + "x": 8, + "y": 0, + "w": 4, + "h": 11, + "i": "19", + "type": "Widget", + "expressions": [ + "latest(gen_ai_provider_input_tokens_sum)" + ], + "graph": { + "type": "Card", + "fontSize": 24, + "textAlign": "center", + "showUnit": true + }, + "metricConfig": [ + { + "label": "Input Tokens" + } + ], + "widget": { + "name": "InputTokens", + "title": "Input Tokens", + "tips": "The sum of input tokens." + }, + "id": "0-0-19", + "moved": false, + "typesOfMQE": [ + "SINGLE_VALUE" + ] + }, + { + "x": 12, + "y": 0, + "w": 12, + "h": 11, + "i": "1", + "type": "Widget", + "expressions": [ + "gen_ai_provider_sla/100" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "name": "ServiceSLA", + "title": "Service SLA", + "tips": "Successful rate of GenAI provider calls." + }, + "metricConfig": [ + { + "label": "SLA", + "unit": "%" + } + ], + "id": "0-0-1", + "moved": false + }, + { + "x": 0, + "y": 11, + "w": 12, + "h": 11, + "i": "0", + "type": "Widget", + "expressions": [ + "gen_ai_provider_resp_time" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "metricConfig": [ + { + "label": "Response Time", + "unit": "ms" + } + ], + "widget": { + "name": "ResponseTime", + "title": "Response Time", + "tips": "The average response time of GenAI provider." + }, + "id": "0-0-0", + "moved": false + }, + { + "x": 12, + "y": 11, + "w": 12, + "h": 11, + "i": "5", + "type": "Widget", + "expressions": [ + "gen_ai_provider_cpm" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "metricConfig": [ + { + "label": "CPM" + } + ], + "widget": { + "name": "CPM", + "title": "Calls Per Minute", + "tips": "The number of calls per minute." + }, + "id": "0-0-5", + "moved": false + }, + { + "x": 0, + "y": 22, + "w": 12, + "h": 12, + "i": "14", + "type": "Widget", + "expressions": [ + "gen_ai_provider_latency_avg" + ], + "graph": { + "type": "Line", + "showXAxis": true, + "showYAxis": true + }, + "widget": { + "name": "AvgLatency", + "title": "Average Latency", + "tips": "The average latency of model access." + }, + "metricConfig": [ + { + "label": "Avg Latency", + "unit": "ms" + } + ], + "id": "0-0-14", + "moved": false + }, + { + "x": 12, + "y": 22, + "w": 12, + "h": 12, + "i": "13", + "type": "Widget", + "expressions": [ + "gen_ai_provider_latency_percentile" + ], + "graph": { + "type": "Bar", + "showBackground": true + }, + "widget": { + "name": "LatencyPercentile", + "title": "Latency Percentile", + "tips": "The percentile of model access latency." + }, + "id": "0-0-13", + "moved": false + } + ] + }, + { + "name": "model", + "children": [ + { + "x": 0, + "y": 0, + "w": 24, + "h": 30, + "i": "0", + "type": "Widget", + "graph": { + "type": "InstanceList", + "dashboardName": "Virtual-GenAI-Model", + "fontSize": 12 + }, + "id": "0-1-0", + "moved": false + } + ] + } + ], + "id": "0", + "activedTabIndex": 0, + "moved": false + } + ], + "layer": "VIRTUAL_GENAI", + "entity": "Service", + "name": "Virtual-GenAI-Provider", + "id": "Virtual-GenAI-Provider", + "isRoot": false + } + } +] diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json new file mode 100644 index 000000000000..21f037425094 --- /dev/null +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json @@ -0,0 +1,57 @@ +[ + { + "id": "Virtual-GenAI-Root", + "configuration": { + "children": [ + { + "x": 0, + "y": 2, + "w": 24, + "h": 52, + "i": "0", + "type": "Widget", + "widget": { + "title": "Virtual GenAI" + }, + "graph": { + "type": "ServiceList", + "dashboardName": "Virtual-GenAI-Provider", + "fontSize": 12, + "showXAxis": false, + "showYAxis": false, + "showGroup": false + }, + "expressions": [ + "avg(gen_ai_provider_resp_time)", + "avg(gen_ai_provider_sla)/100", + "avg(gen_ai_provider_cpm)" + ], + "subExpressions": [ + "gen_ai_provider_resp_time", + "gen_ai_provider_sla/100", + "gen_ai_provider_cpm" + ], + "metricConfig": [ + { + "unit": "ms", + "label": "Access Latency" + }, + { + "label": "Successful Access Rate", + "unit": "%" + }, + { + "label": "Access Traffic", + "unit": "calls / min" + } + ] + } + ], + "id": "Virtual-GenAI-Root", + "layer": "VIRTUAL_GENAI", + "entity": "All", + "name": "Virtual-GenAI-Root", + "isRoot": true + } + } +] diff --git a/test/e2e-v2/cases/storage/expected/config-dump.yml b/test/e2e-v2/cases/storage/expected/config-dump.yml index e5f259f5ef9d..8c0590d6527c 100644 --- a/test/e2e-v2/cases/storage/expected/config-dump.yml +++ b/test/e2e-v2/cases/storage/expected/config-dump.yml @@ -64,6 +64,7 @@ telemetry.provider=prometheus core.default.trainingPeriodHttpUriRecognitionPattern=60 promql.default.restContextPath=/ core.default.maxHeapMemoryUsagePercent=96 +genAI-analyzer.provider=default aws-firehose.default.contextPath=/ agent-analyzer.default.slowCacheWriteThreshold=default:20,redis:10 envoy-metric.default.maxMessageSize=0 diff --git a/test/e2e-v2/cases/virtual-genai/Dockerfile.provider b/test/e2e-v2/cases/virtual-genai/Dockerfile.provider new file mode 100644 index 000000000000..256187e6c495 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/Dockerfile.provider @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG SW_AGENT_JDK_VERSION +ARG SW_AGENT_JAVA_COMMIT + +FROM eclipse-temurin:17-jdk AS builder + +RUN apt-get update && apt-get install -y git maven && rm -rf /var/lib/apt/lists/* + +WORKDIR /source +RUN git clone https://github.com/spring-projects/spring-ai-examples.git + +WORKDIR /source/spring-ai-examples/misc/openai-streaming-response +RUN mvn clean package -DskipTests + +RUN find target/ -maxdepth 1 -name "*.jar" ! -name "*-plain.jar" -exec cp {} /app.jar \; + +FROM ghcr.io/apache/skywalking-java/skywalking-java:${SW_AGENT_JAVA_COMMIT}-java${SW_AGENT_JDK_VERSION} + +WORKDIR /skywalking + +VOLUME /services + +COPY --from=builder /app.jar /services/app.jar + +EXPOSE 8080 + +CMD ["sh", "-c", "java -jar /services/app.jar"] diff --git a/test/e2e-v2/cases/virtual-genai/docker-compose.yml b/test/e2e-v2/cases/virtual-genai/docker-compose.yml new file mode 100644 index 000000000000..e1839a1a17c4 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/docker-compose.yml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" + +services: + oap: + extends: + file: ../../script/docker-compose/base-compose.yml + service: oap + ports: + - "12800:12800" + - "11800:11800" + networks: + - e2e + + banyandb: + extends: + file: ../../script/docker-compose/base-compose.yml + service: banyandb + ports: + - 17912 + + provider: + extends: + file: ../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + depends_on: + oap: + condition: service_healthy + + spring-ai-examples: + build: + context: . + dockerfile: Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=17 + - SW_AGENT_JAVA_COMMIT=${SW_AGENT_JAVA_COMMIT} + ports: + - "9260:8080" + networks: + - e2e + environment: + OPENAI_API_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + SPRING_AI_OPENAI_BASE_URL: http://provider:9090/llm + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_LOGGING_OUTPUT: CONSOLE + SW_AGENT_NAME: e2e-spring-ai + SW_AGENT_INSTANCE_NAME: spring-ai-examples + depends_on: + oap: + condition: service_healthy + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e-v2/cases/virtual-genai/e2e.yaml b/test/e2e-v2/cases/virtual-genai/e2e.yaml new file mode 100644 index 000000000000..cfd52903aa70 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/e2e.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +trigger: + action: http + interval: 3s + times: 5 + url: http://localhost:9260/ai/generateStream + method: GET + +verify: + retry: + count: 60 + interval: 3s + cases: + - includes: + - ./virtual-genai.yaml diff --git a/test/e2e-v2/cases/virtual-genai/expected/instance.yml b/test/e2e-v2/cases/virtual-genai/expected/instance.yml new file mode 100644 index 000000000000..d3ffc6ca30a3 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/expected/instance.yml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ notEmpty .id }} + name: gpt-4.1-mini + instanceuuid: {{ notEmpty .instanceuuid }} + attributes: [] + language: UNKNOWN +{{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml new file mode 100644 index 000000000000..1df2937d4846 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +debuggingtrace: null +type: TIME_SERIES_VALUES +results: + {{- contains .results }} + - metric: + labels: + {{- contains .metric.labels }} + - key: "_" + value: {{ notEmpty .value }} + {{- end}} + values: + {{- contains .values }} + - id: {{ notEmpty .id }} + value: {{ .value }} + traceid: null + owner: null + - id: {{ notEmpty .id }} + value: null + traceid: null + owner: null + {{- end}} + {{- end}} +error: null diff --git a/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml new file mode 100644 index 000000000000..f68a07155e03 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +debuggingtrace: null +type: TIME_SERIES_VALUES +results: + {{- contains .results }} + - metric: + labels: [] + values: + {{- contains .values }} + - id: {{ notEmpty .id }} + value: {{ notEmpty .value }} + traceid: null + owner: null + - id: {{ notEmpty .id }} + value: null + traceid: null + owner: null + {{- end}} + {{- end}} +error: null \ No newline at end of file diff --git a/test/e2e-v2/cases/virtual-genai/expected/service.yml b/test/e2e-v2/cases/virtual-genai/expected/service.yml new file mode 100644 index 000000000000..498ee501899a --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/expected/service.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "openai" }}.0 + name: openai + group: "" + shortname: openai + layers: + - VIRTUAL_GENAI + normal: false +{{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml b/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml new file mode 100644 index 000000000000..ca5f4dc66706 --- /dev/null +++ b/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +cases: + # service cases + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_resp_time --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_sla --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_cpm --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_latency_avg --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_latency_percentile --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value-label.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_input_tokens_sum --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_output_tokens_sum --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_total_cost --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml + + + # instance cases + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance ls --service-id=b3BlbmFp.0 + expected: expected/instance.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_call_cpm --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_sla --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_latency_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_latency_percentile --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value-label.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_ttft_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_ttft_percentile --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value-label.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_input_tokens_sum --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_output_tokens_sum --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_total_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + expected: expected/metrics-has-value.yml + + + + diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml b/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml index 5f535cc0fd19..048bd92375b0 100644 --- a/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml +++ b/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml @@ -111,6 +111,12 @@ guava 23.0 + + + com.alibaba + fastjson + 1.2.83 + diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java new file mode 100644 index 000000000000..24baa68d0088 --- /dev/null +++ b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.controller; + +import com.alibaba.fastjson.JSONObject; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.time.Instant; + +@RestController +@RequestMapping("/llm") +public class LLMMockController { + @PostMapping("/v1/chat/completions") + public Object completions(@RequestBody JSONObject request, HttpServletResponse response) throws Exception { + + response.setContentType("text/event-stream"); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Cache-Control", "no-cache"); + response.setHeader("Connection", "keep-alive"); + + String id = "chatcmpl-simple-mock-001"; + long created = Instant.now().getEpochSecond(); + String model = "gpt-4.1-mini-2025-04-14"; + + try (PrintWriter writer = response.getWriter()) { + Thread.sleep(1000); + writeStreamChunk(writer, id, created, model, "{\"role\":\"assistant\"}", "null"); + + String fullContent = "Why did the scarecrow win an award? Because he was outstanding in his field!"; + String[] words = fullContent.split(" "); + + for (int i = 0; i < words.length; i++) { + String chunk = words[i] + (i == words.length - 1 ? "" : " "); + Thread.sleep(50); + writeStreamChunk(writer, id, created, model, "{\"content\":\"" + chunk + "\"}", "null"); + } + + writeStreamChunk(writer, id, created, model, "{}", "\"stop\""); + + writer.write("data: [DONE]\n\n"); + writer.flush(); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return null; + } + + private void writeStreamChunk(PrintWriter writer, String id, long created, String model, String delta, String finishReason) { + String json = "{" + + "\"id\": \"%s\"," + + "\"object\": \"chat.completion.chunk\"," + + "\"created\": %d," + + "\"model\": \"%s\"," + + "\"system_fingerprint\": null," + + "\"choices\": [" + + "{" + + "\"index\": 0," + + "\"delta\": %s," + + "\"finish_reason\": %s" + + "}" + + "]," + + "\"usage\": {" + + "\"completion_tokens\": 17," + + "\"completion_tokens_details\": {" + + "\"accepted_prediction_tokens\": 0," + + "\"audio_tokens\": 0," + + "\"reasoning_tokens\": 0," + + "\"rejected_prediction_tokens\": 0" + + "}," + + "\"prompt_tokens\": 52," + + "\"prompt_tokens_details\": {" + + "\"audio_tokens\": 0," + + "\"cached_tokens\": 0" + + "}," + + "\"total_tokens\": 69" + + "}" + + "}"; + + String formattedJson = String.format(json, id, created, model, delta, finishReason); + + String cleanJson = formattedJson.replace("\n", "").replace("\r", ""); + writer.write("data: " + cleanJson + "\n\n"); + writer.flush(); + } +} diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 96f105986f51..9d7ff36f3dc0 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SW_AGENT_JAVA_COMMIT=2a61027e5eb74ed1258c764ae2ffeabd499416a6 +SW_AGENT_JAVA_COMMIT=2f1d9e94d6d1ac22d92f4e9c6905901fe646ffdf SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e From 3642ce354579e5161bd97dab829e226443ae189c Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Mon, 16 Mar 2026 21:55:52 +0800 Subject: [PATCH 02/11] fix changes --- docs/en/changes/changes.md | 1 + .../server-starter/src/main/resources/gen-ai-config.yml | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index a8465dc26a29..77dae2645187 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -166,6 +166,7 @@ * Update hierarchy rule documentation: `auto-matching-rules` in `hierarchy-definition.yml` no longer use Groovy scripts. Rules now use a dedicated expression grammar supporting property access, String methods, if/else, comparisons, and logical operators. All shipped rules are fully compatible. * Activate `otlp-traces` handler in `receiver-otel` by default. * Update Istio E2E test versions: remove EOL 1.20.0, add 1.25.0–1.29.0 for ALS/Metrics/Ambient tests. Update Rover with Istio Process test from 1.15.0 to 1.28.0 with Kubernetes 1.28. +* Support Virtual-GenAI monitoring. #### UI * Fix the missing icon in new native trace view. diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index c95667dc1e64..5d6ad7291847 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -41,10 +41,6 @@ providers: - provider: anthropic prefix-match: - claude - models: - - name: claude-3-7-sonnet-20250219-thinking - input-cost-per-m: 100 - output-cost-per-m: 100 - provider: gemini prefix-match: From d2c21655d7cc5c0485b30f3ad735ae572e640858 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Fri, 20 Mar 2026 12:33:34 +0800 Subject: [PATCH 03/11] fix some issues --- .github/workflows/skywalking.yaml | 2 +- apm-dist/src/main/assembly/binary.xml | 1 + docs/en/setup/service-agent/virtual-genai.md | 3 +- docs/menu.yml | 4 + oap-server/analyzer/agent-analyzer/pom.xml | 2 +- .../VirtualServiceAnalysisListener.java | 6 +- .../vservice/VirtualGenAIProcessor.java | 13 +- .../pom.xml | 2 +- .../genai}/GenAIAnalyzerModuleProvider.java | 23 +-- .../analyzer/genai}/config/GenAIConfig.java | 2 +- .../genai}/config/GenAIConfigLoader.java | 4 +- .../genai}/config/GenAIOALDefine.java | 2 +- .../analyzer/genai}/config/GenAITagKey.java | 4 +- .../matcher/GenAIProviderPrefixMatcher.java | 4 +- .../genai}/module/GenAIAnalyzerModule.java | 6 +- .../genai}/service/GenAIMeterAnalyzer.java | 10 +- .../service/GenAIModelAccessDispatcher.java | 2 +- .../service/IGenAIMeterAnalyzerService.java | 2 +- ...ing.oap.server.library.module.ModuleDefine | 2 +- ...g.oap.server.library.module.ModuleProvider | 2 +- oap-server/analyzer/pom.xml | 2 +- oap-server/server-starter/pom.xml | 2 +- .../src/main/resources/application.yml | 2 +- .../src/main/resources/gen-ai-config.yml | 2 +- .../src/main/resources/oal/virtual-gen-ai.oal | 16 +- .../ui-initialized-templates/menu.yaml | 2 +- .../virtual_genai/virtual-genai-model.json | 155 +++++++++++++----- .../virtual_genai/virtual-genai-provider.json | 83 ++++++++-- .../virtual_genai/virtual-genai-root.json | 1 + skywalking-ui | 2 +- .../cases/storage/expected/config-dump.yml | 2 +- .../cases/virtual-genai/Dockerfile.provider | 5 +- .../cases/virtual-genai/docker-compose.yml | 2 +- test/e2e-v2/cases/virtual-genai/e2e.yaml | 1 + .../cases/virtual-genai/expected/instance.yml | 4 +- .../expected/metrics-has-value-label.yml | 3 +- .../expected/metrics-has-value.yml | 2 +- .../cases/virtual-genai/expected/service.yml | 2 +- .../cases/virtual-genai/virtual-genai.yaml | 33 ++-- .../e2e-service-provider/pom.xml | 5 - .../e2e/controller/LLMMockController.java | 4 +- test/e2e-v2/script/env | 2 +- 42 files changed, 295 insertions(+), 133 deletions(-) rename oap-server/analyzer/{genAI-analyzer => gen-ai-analyzer}/pom.xml (97%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/GenAIAnalyzerModuleProvider.java (82%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/config/GenAIConfig.java (96%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/config/GenAIConfigLoader.java (97%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/config/GenAIOALDefine.java (95%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/config/GenAITagKey.java (88%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/matcher/GenAIProviderPrefixMatcher.java (96%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/module/GenAIAnalyzerModule.java (86%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/service/GenAIMeterAnalyzer.java (94%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/service/GenAIModelAccessDispatcher.java (96%) rename oap-server/analyzer/{genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer => gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai}/service/IGenAIMeterAnalyzerService.java (95%) rename oap-server/analyzer/{genAI-analyzer => gen-ai-analyzer}/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine (92%) rename oap-server/analyzer/{genAI-analyzer => gen-ai-analyzer}/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider (91%) diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index a663789297d3..b5a88ec4a698 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -627,7 +627,7 @@ jobs: config: test/e2e-v2/cases/zipkin/kafka/e2e.yaml - name: Zipkin BanyanDB config: test/e2e-v2/cases/zipkin/banyandb/e2e.yaml - - name: Virtual-genai + - name: Virtual GenAI config: test/e2e-v2/cases/virtual-genai/e2e.yaml - name: Nginx diff --git a/apm-dist/src/main/assembly/binary.xml b/apm-dist/src/main/assembly/binary.xml index f6dbf452d329..81f7b2a281c4 100644 --- a/apm-dist/src/main/assembly/binary.xml +++ b/apm-dist/src/main/assembly/binary.xml @@ -75,6 +75,7 @@ log-mal-rules/** telegraf-rules/* cilium-rules/* + gen-ai-config.yml config diff --git a/docs/en/setup/service-agent/virtual-genai.md b/docs/en/setup/service-agent/virtual-genai.md index 1c9feb68dbdc..ce91ecc2c045 100644 --- a/docs/en/setup/service-agent/virtual-genai.md +++ b/docs/en/setup/service-agent/virtual-genai.md @@ -1,6 +1,6 @@ # Virtual GenAI -Virtual cache represent the Generative AI service nodes detected by [server agents' plugins](server-agents.md). The performance +Virtual GenAI represent the Generative AI service nodes detected by [server agents' plugins](server-agents.md). The performance metrics of the GenAI operations are also from the GenAI client-side perspective. For example, an Spring-ai plugin in the Java agent could detect the latency of a chat completion request. @@ -13,4 +13,5 @@ The GenAI operation span should have - Tag key = `gen_ai.response.model`, value = The name of the GenAI model a response is being made to, e.g. gpt-4o, claude-3-5-sonnet - Tag key = `gen_ai.usage.input_tokens`, value = The number of tokens used in the GenAI input (prompt) - Tag key = `gen_ai.usage.output_tokens`, value = The number of tokens used in the GenAI response (completion) +- Tag key = `gen_ai.server.time_to_first_token`, value = The duration in milliseconds from the start of the request until the first token is received. Note: This metric is only available for streaming requests. - If the GenAI service is a remote API (e.g. OpenAI), the span's peer would be the network address (IP or domain) of the GenAI server. diff --git a/docs/menu.yml b/docs/menu.yml index 4558a252eac7..e347015d95bc 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -156,6 +156,10 @@ catalog: path: "/en/setup/backend/dashboards-so11y-java-agent" - name: "SkyWalking Go Agent self telemetry" path: "/en/setup/backend/dashboards-so11y-go-agent" + - name: "GenAI" + catalog: + - name: "Virtual Genai" + path: "/en/setup/service-agent/virtual-genai" - name: "Configuration Vocabulary" path: "/en/setup/backend/configuration-vocabulary" - name: "Advanced Setup" diff --git a/oap-server/analyzer/agent-analyzer/pom.xml b/oap-server/analyzer/agent-analyzer/pom.xml index 391d0765035e..1fe05db26720 100644 --- a/oap-server/analyzer/agent-analyzer/pom.xml +++ b/oap-server/analyzer/agent-analyzer/pom.xml @@ -45,7 +45,7 @@ org.apache.skywalking - genAI-analyzer + gen-ai-analyzer ${project.version} diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java index 90e15c41ec76..861f699ce830 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/VirtualServiceAnalysisListener.java @@ -24,8 +24,8 @@ import lombok.RequiredArgsConstructor; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; -import org.apache.skywalking.oap.meter.analyzer.module.GenAIAnalyzerModule; -import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.analyzer.genai.module.GenAIAnalyzerModule; +import org.apache.skywalking.oap.analyzer.genai.service.IGenAIMeterAnalyzerService; import org.apache.skywalking.oap.server.analyzer.provider.AnalyzerModuleConfig; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualCacheProcessor; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.vservice.VirtualDatabaseProcessor; @@ -95,7 +95,7 @@ public AnalysisListener create(ModuleManager moduleManager, AnalyzerModuleConfig new VirtualCacheProcessor(namingControl, config), new VirtualDatabaseProcessor(namingControl, config), new VirtualMQProcessor(namingControl), - new VirtualGenAIProcessor(genAIMeterAnalyzerService) + new VirtualGenAIProcessor(namingControl, genAIMeterAnalyzerService) ) ); } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java index 1b1c3e7eb423..e5e8c4af33a5 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java @@ -21,8 +21,9 @@ import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; -import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.analyzer.genai.service.IGenAIMeterAnalyzerService; import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.core.source.GenAIMetrics; import org.apache.skywalking.oap.server.core.source.GenAIModelAccess; import org.apache.skywalking.oap.server.core.source.GenAIProviderAccess; @@ -36,9 +37,11 @@ @RequiredArgsConstructor public class VirtualGenAIProcessor implements VirtualServiceProcessor { + private final NamingControl namingControl; + private final IGenAIMeterAnalyzerService meterAnalyzerService; - private List recordList = new ArrayList<>(); + private final List recordList = new ArrayList<>(); @Override public void prepareVSIfNecessary(SpanObject span, SegmentObject segmentObject) { @@ -66,7 +69,7 @@ private ServiceMeta toServiceMeta(GenAIMetrics metrics) { private GenAIProviderAccess toProviderAccess(GenAIMetrics metrics) { GenAIProviderAccess source = new GenAIProviderAccess(); - source.setName(metrics.getProviderName()); + source.setName(namingControl.formatServiceName(metrics.getProviderName())); source.setInputTokens(metrics.getInputTokens()); source.setOutputTokens(metrics.getOutputTokens()); source.setTotalCost(metrics.getTotalCost()); @@ -78,8 +81,8 @@ private GenAIProviderAccess toProviderAccess(GenAIMetrics metrics) { private GenAIModelAccess toModelAccess(GenAIMetrics metrics) { GenAIModelAccess source = new GenAIModelAccess(); - source.setServiceName(metrics.getProviderName()); - source.setModelName(metrics.getModelName()); + source.setServiceName(namingControl.formatServiceName(metrics.getProviderName())); + source.setModelName(namingControl.formatInstanceName(metrics.getModelName())); source.setInputTokens(metrics.getInputTokens()); source.setOutputTokens(metrics.getOutputTokens()); source.setTotalCost(metrics.getTotalCost()); diff --git a/oap-server/analyzer/genAI-analyzer/pom.xml b/oap-server/analyzer/gen-ai-analyzer/pom.xml similarity index 97% rename from oap-server/analyzer/genAI-analyzer/pom.xml rename to oap-server/analyzer/gen-ai-analyzer/pom.xml index 52ee1bddf325..be35fa818e9e 100644 --- a/oap-server/analyzer/genAI-analyzer/pom.xml +++ b/oap-server/analyzer/gen-ai-analyzer/pom.xml @@ -25,7 +25,7 @@ 4.0.0 - genAI-analyzer + gen-ai-analyzer diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/GenAIAnalyzerModuleProvider.java similarity index 82% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/GenAIAnalyzerModuleProvider.java index fa6b2a2798fa..6248a9b1b329 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/GenAIAnalyzerModuleProvider.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/GenAIAnalyzerModuleProvider.java @@ -16,15 +16,15 @@ * */ -package org.apache.skywalking.oap.meter.analyzer; +package org.apache.skywalking.oap.analyzer.genai; -import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfig; -import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfigLoader; -import org.apache.skywalking.oap.meter.analyzer.config.GenAIOALDefine; -import org.apache.skywalking.oap.meter.analyzer.matcher.GenAIProviderPrefixMatcher; -import org.apache.skywalking.oap.meter.analyzer.module.GenAIAnalyzerModule; -import org.apache.skywalking.oap.meter.analyzer.service.GenAIMeterAnalyzer; -import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfigLoader; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIOALDefine; +import org.apache.skywalking.oap.analyzer.genai.matcher.GenAIProviderPrefixMatcher; +import org.apache.skywalking.oap.analyzer.genai.module.GenAIAnalyzerModule; +import org.apache.skywalking.oap.analyzer.genai.service.GenAIMeterAnalyzer; +import org.apache.skywalking.oap.analyzer.genai.service.IGenAIMeterAnalyzerService; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; import org.apache.skywalking.oap.server.library.module.ModuleConfig; @@ -32,7 +32,6 @@ import org.apache.skywalking.oap.server.library.module.ModuleProvider; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; -import org.yaml.snakeyaml.Yaml; public class GenAIAnalyzerModuleProvider extends ModuleProvider { @@ -65,7 +64,7 @@ public void onInitialized(final GenAIConfig initialized) { @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException { - GenAIConfigLoader loader = new GenAIConfigLoader(config, new Yaml()); + GenAIConfigLoader loader = new GenAIConfigLoader(config); config = loader.loadConfig(); GenAIProviderPrefixMatcher matcher = GenAIProviderPrefixMatcher.build(config); this.registerServiceImplementation( @@ -89,6 +88,8 @@ public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleSta @Override public String[] requiredModules() { - return new String[0]; + return new String[] { + CoreModule.NAME + }; } } diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java similarity index 96% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java index 41682ca97637..46d83174c586 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfig.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.config; +package org.apache.skywalking.oap.analyzer.genai.config; import lombok.Getter; import lombok.Setter; diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java similarity index 97% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java index 4fcede0b3ed7..ea16c7cc7112 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIConfigLoader.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.config; +package org.apache.skywalking.oap.analyzer.genai.config; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.util.ResourceUtils; @@ -33,7 +33,7 @@ public class GenAIConfigLoader { private final GenAIConfig config; - public GenAIConfigLoader(GenAIConfig config, Yaml yaml) { + public GenAIConfigLoader(GenAIConfig config) { this.config = config; } diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIOALDefine.java similarity index 95% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIOALDefine.java index 97c399def037..2509b3772581 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAIOALDefine.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIOALDefine.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.config; +package org.apache.skywalking.oap.analyzer.genai.config; import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKey.java similarity index 88% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKey.java index 456b36e7275f..df33c32262f4 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/config/GenAITagKey.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKey.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.skywalking.oap.meter.analyzer.config; +package org.apache.skywalking.oap.analyzer.genai.config; public class GenAITagKey { @@ -24,5 +24,5 @@ public class GenAITagKey { public static final String RESPONSE_MODEL = "gen_ai.response.model"; public static final String INPUT_TOKENS = "gen_ai.usage.input_tokens"; public static final String OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; - public static final String STREAM_TTFT = "gen_ai.stream.ttfr"; + public static final String SERVER_TIME_TO_FIRST_TOKEN = "gen_ai.server.time_to_first_token"; } diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java similarity index 96% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java index a0802f41d396..6bbd71d390bb 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/matcher/GenAIProviderPrefixMatcher.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/matcher/GenAIProviderPrefixMatcher.java @@ -16,9 +16,9 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.matcher; +package org.apache.skywalking.oap.analyzer.genai.matcher; -import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfig; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; import java.util.HashMap; import java.util.List; diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/module/GenAIAnalyzerModule.java similarity index 86% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/module/GenAIAnalyzerModule.java index 0945b7f89d95..833cb30e8e5b 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/module/GenAIAnalyzerModule.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/module/GenAIAnalyzerModule.java @@ -16,14 +16,14 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.module; +package org.apache.skywalking.oap.analyzer.genai.module; -import org.apache.skywalking.oap.meter.analyzer.service.IGenAIMeterAnalyzerService; +import org.apache.skywalking.oap.analyzer.genai.service.IGenAIMeterAnalyzerService; import org.apache.skywalking.oap.server.library.module.ModuleDefine; public class GenAIAnalyzerModule extends ModuleDefine { - public static final String NAME = "genAI-analyzer"; + public static final String NAME = "gen-ai-analyzer"; public GenAIAnalyzerModule() { super(NAME); diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java similarity index 94% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java index 1e67f1c11053..735acac69dd1 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIMeterAnalyzer.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.skywalking.oap.meter.analyzer.service; +package org.apache.skywalking.oap.analyzer.genai.service; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; -import org.apache.skywalking.oap.meter.analyzer.config.GenAIConfig; -import org.apache.skywalking.oap.meter.analyzer.config.GenAITagKey; -import org.apache.skywalking.oap.meter.analyzer.matcher.GenAIProviderPrefixMatcher; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; +import org.apache.skywalking.oap.analyzer.genai.config.GenAITagKey; +import org.apache.skywalking.oap.analyzer.genai.matcher.GenAIProviderPrefixMatcher; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; @@ -93,7 +93,7 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm metrics.setInputTokens(inputTokens); metrics.setOutputTokens(outputTokens); - metrics.setTimeToFirstToken(parseSafeInt(tags.get(GenAITagKey.STREAM_TTFT))); + metrics.setTimeToFirstToken(parseSafeInt(tags.get(GenAITagKey.SERVER_TIME_TO_FIRST_TOKEN))); metrics.setTotalCost(totalCost); long latency = span.getEndTime() - span.getStartTime(); diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java similarity index 96% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java index 9c0fa069b8a2..d5b0337a1d93 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/GenAIModelAccessDispatcher.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.service; +package org.apache.skywalking.oap.analyzer.genai.service; import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; diff --git a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/IGenAIMeterAnalyzerService.java similarity index 95% rename from oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/IGenAIMeterAnalyzerService.java index 6b00eeb024da..efbf2192b030 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/service/IGenAIMeterAnalyzerService.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/IGenAIMeterAnalyzerService.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oap.meter.analyzer.service; +package org.apache.skywalking.oap.analyzer.genai.service; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; diff --git a/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/analyzer/gen-ai-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine similarity index 92% rename from oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine rename to oap-server/analyzer/gen-ai-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine index c648cced34ee..ea7d0042b839 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -16,4 +16,4 @@ # # -org.apache.skywalking.oap.meter.analyzer.module.GenAIAnalyzerModule \ No newline at end of file +org.apache.skywalking.oap.analyzer.genai.module.GenAIAnalyzerModule diff --git a/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/analyzer/gen-ai-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider similarity index 91% rename from oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider rename to oap-server/analyzer/gen-ai-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider index 1eef579246fa..b0256f986574 100644 --- a/oap-server/analyzer/genAI-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -15,4 +15,4 @@ # limitations under the License. # -org.apache.skywalking.oap.meter.analyzer.GenAIAnalyzerModuleProvider \ No newline at end of file +org.apache.skywalking.oap.analyzer.genai.GenAIAnalyzerModuleProvider diff --git a/oap-server/analyzer/pom.xml b/oap-server/analyzer/pom.xml index fa78c963f434..091eea2f35f6 100644 --- a/oap-server/analyzer/pom.xml +++ b/oap-server/analyzer/pom.xml @@ -34,7 +34,7 @@ meter-analyzer log-analyzer hierarchy - genAI-analyzer + gen-ai-analyzer diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index 09b5138c2f62..cc510317a504 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -346,7 +346,7 @@ log-mal-rules/ telegraf-rules/ cilium-rules/ - gen-ai-settings.yml + gen-ai-config.yml diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 2c866302fa9b..7f2223f8fc4a 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -244,7 +244,7 @@ event-analyzer: selector: ${SW_EVENT_ANALYZER:default} default: -genAI-analyzer: +gen-ai-analyzer: selector: ${SW_GENAI_ANALYZER:default} default: diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index 5d6ad7291847..49cba6f37352 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -88,4 +88,4 @@ providers: - provider: azure_openai prefix-match: - - azure \ No newline at end of file + - azure diff --git a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal index dd5331c85200..ee07e548bec5 100644 --- a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal +++ b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal @@ -20,13 +20,16 @@ gen_ai_provider_resp_time = from(GenAIProviderAccess.latency).longAvg(); gen_ai_provider_sla = from(GenAIProviderAccess.*).percent(status == true); gen_ai_provider_cpm = from(GenAIProviderAccess.*).cpm(); -gen_ai_provider_latency_avg = from(GenAIProviderAccess.latency).longAvg(); -gen_ai_provider_latency_percentile = from(GenAIProviderAccess.latency).percentile(10); +gen_ai_provider_latency_percentile = from(GenAIProviderAccess.latency).percentile2(10); gen_ai_provider_input_tokens_sum = from(GenAIProviderAccess.inputTokens).sum(); +gen_ai_provider_input_tokens_avg = from(GenAIProviderAccess.inputTokens).longAvg(); + gen_ai_provider_output_tokens_sum = from(GenAIProviderAccess.outputTokens).sum(); +gen_ai_provider_output_tokens_avg = from(GenAIProviderAccess.outputTokens).longAvg(); gen_ai_provider_total_cost = from(GenAIProviderAccess.totalCost).sum(); +gen_ai_provider_avg_cost = from(GenAIProviderAccess.totalCost).doubleAvg(); gen_ai_model_call_cpm = from(GenAIModelAccess.*).cpm(); @@ -34,12 +37,15 @@ gen_ai_model_sla = from(GenAIModelAccess.*).percent(status == true); gen_ai_model_latency_avg = from(GenAIModelAccess.latency).longAvg(); -gen_ai_model_latency_percentile = from(GenAIModelAccess.latency).percentile(10); +gen_ai_model_latency_percentile = from(GenAIModelAccess.latency).percentile2(10); gen_ai_model_ttft_avg = from(GenAIModelAccess.timeToFirstToken).filter(timeToFirstToken > 0).longAvg(); -gen_ai_model_ttft_percentile = from(GenAIModelAccess.timeToFirstToken).filter(timeToFirstToken > 0).percentile(10); +gen_ai_model_ttft_percentile = from(GenAIModelAccess.timeToFirstToken).filter(timeToFirstToken > 0).percentile2(10); gen_ai_model_input_tokens_sum = from(GenAIModelAccess.inputTokens).sum(); +gen_ai_model_input_tokens_avg = from(GenAIModelAccess.inputTokens).longAvg(); gen_ai_model_output_tokens_sum = from(GenAIModelAccess.outputTokens).sum(); +gen_ai_model_output_tokens_avg = from(GenAIModelAccess.outputTokens).longAvg(); -gen_ai_model_total_cost = from(GenAIModelAccess.totalCost).sum(); \ No newline at end of file +gen_ai_model_total_cost = from(GenAIModelAccess.totalCost).sum(); +gen_ai_model_avg_cost = from(GenAIModelAccess.totalCost).doubleAvg(); \ No newline at end of file diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml b/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml index 73b11a265eed..2a1c862e8764 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/menu.yaml @@ -281,4 +281,4 @@ menus: layer: VIRTUAL_GENAI description: Observe the virtual GenAI providers and models which are conjectured by language agents through various plugins. documentLink: https://skywalking.apache.org/docs/main/next/en/setup/service-agent/virtual-genai/ - i18nKey: virtual_gen_ai \ No newline at end of file + i18nKey: virtual_gen_ai diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json index dbeafb0e38ac..610cf90adea1 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json @@ -107,9 +107,9 @@ }, { "x": 12, - "y": 0, + "y": 24, "w": 12, - "h": 11, + "h": 12, "i": "1", "type": "Widget", "expressions": [ @@ -136,9 +136,9 @@ }, { "x": 0, - "y": 11, + "y": 24, "w": 12, - "h": 11, + "h": 12, "i": "0", "type": "Widget", "expressions": [ @@ -165,7 +165,7 @@ }, { "x": 12, - "y": 11, + "y": 36, "w": 12, "h": 11, "i": "5", @@ -191,38 +191,9 @@ "id": "0-0-5", "moved": false }, - { - "x": 0, - "y": 22, - "w": 12, - "h": 12, - "i": "14", - "type": "Widget", - "expressions": [ - "gen_ai_model_latency_avg" - ], - "graph": { - "type": "Line", - "showXAxis": true, - "showYAxis": true - }, - "widget": { - "name": "AvgLatency", - "title": "Average Latency", - "tips": "The average latency of model access." - }, - "metricConfig": [ - { - "label": "Avg Latency", - "unit": "ms" - } - ], - "id": "0-0-14", - "moved": false - }, { "x": 12, - "y": 22, + "y": 47, "w": 12, "h": 12, "i": "13", @@ -244,9 +215,9 @@ }, { "x": 0, - "y": 34, + "y": 36, "w": 12, - "h": 10, + "h": 11, "i": "20", "type": "Widget", "widget": { @@ -278,10 +249,10 @@ ] }, { - "x": 12, - "y": 34, + "x": 0, + "y": 47, "w": 12, - "h": 10, + "h": 12, "i": "21", "type": "Widget", "widget": { @@ -307,6 +278,108 @@ "typesOfMQE": [ "TIME_SERIES_VALUES" ] + }, + { + "x": 0, + "y": 11, + "w": 12, + "h": 13, + "i": "22", + "type": "Widget", + "widget": { + "name": "AvgInputTokens", + "title": "Average Input Tokens", + "tips": "The average number of input tokens used per model call." + }, + "metricConfig": [ + { + "label": "Avg Input Tokens" + } + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "id": "0-0-22", + "moved": false, + "expressions": [ + "gen_ai_model_input_tokens_avg" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] + }, + { + "x": 12, + "y": 11, + "w": 12, + "h": 13, + "i": "23", + "type": "Widget", + "widget": { + "name": "AvgOutputTokens", + "title": "Average Output Tokens", + "tips": "The average number of output tokens used per model call." + }, + "metricConfig": [ + { + "label": "Avg Output Tokens" + } + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "id": "0-0-23", + "moved": false, + "expressions": [ + "gen_ai_model_output_tokens_avg" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] + }, + { + "x": 12, + "y": 0, + "w": 12, + "h": 11, + "i": "24", + "type": "Widget", + "widget": { + "name": "AvgCost", + "title": "Average Cost", + "tips": "The average cost of model calls." + }, + "metricConfig": [ + { + "label": "Avg Cost" + } + ], + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "id": "0-0-24", + "moved": false, + "expressions": [ + "gen_ai_model_avg_cost" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] } ] } @@ -323,4 +396,4 @@ "isRoot": false } } -] \ No newline at end of file +] diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json index dca377314f3c..3bbbe6923f9e 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json @@ -136,9 +136,9 @@ }, { "x": 0, - "y": 11, + "y": 34, "w": 12, - "h": 11, + "h": 12, "i": "0", "type": "Widget", "expressions": [ @@ -165,9 +165,9 @@ }, { "x": 12, - "y": 11, + "y": 22, "w": 12, - "h": 11, + "h": 12, "i": "5", "type": "Widget", "expressions": [ @@ -199,7 +199,7 @@ "i": "14", "type": "Widget", "expressions": [ - "gen_ai_provider_latency_avg" + "gen_ai_provider_avg_cost" ], "graph": { "type": "Line", @@ -207,9 +207,9 @@ "showYAxis": true }, "widget": { - "name": "AvgLatency", - "title": "Average Latency", - "tips": "The average latency of model access." + "name": "AvgCost", + "title": "Average Cost", + "tips": "The average cost of provider access." }, "metricConfig": [ { @@ -218,11 +218,14 @@ } ], "id": "0-0-14", - "moved": false + "moved": false, + "typesOfMQE": [ + "UNKNOWN" + ] }, { "x": 12, - "y": 22, + "y": 34, "w": 12, "h": 12, "i": "13", @@ -241,6 +244,64 @@ }, "id": "0-0-13", "moved": false + }, + { + "x": 12, + "y": 11, + "w": 12, + "h": 11, + "i": "20", + "type": "Widget", + "widget": { + "name": "AvgOutputTokens", + "title": "Average Output Tokens", + "tips": "The average number of output tokens used per provider access." + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "id": "0-0-20", + "moved": false, + "expressions": [ + "gen_ai_provider_output_tokens_avg" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] + }, + { + "x": 0, + "y": 11, + "w": 12, + "h": 11, + "i": "21", + "type": "Widget", + "widget": { + "name": "AvgInputTokens", + "title": "Average Input Tokens", + "tips": "The average number of input tokens used per provider access." + }, + "graph": { + "type": "Line", + "step": false, + "smooth": false, + "showSymbol": true, + "showXAxis": true, + "showYAxis": true + }, + "id": "0-0-21", + "moved": false, + "expressions": [ + "gen_ai_provider_input_tokens_avg" + ], + "typesOfMQE": [ + "TIME_SERIES_VALUES" + ] } ] }, @@ -278,3 +339,5 @@ } } ] + + diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json index 21f037425094..b2e76a1168d2 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-root.json @@ -55,3 +55,4 @@ } } ] + diff --git a/skywalking-ui b/skywalking-ui index 6be09fb26b24..6538cc401d19 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 6be09fb26b248814f45224e8fded0b1a5fc7a9cf +Subproject commit 6538cc401d19f768d8b1e075785d991ce7e4739f diff --git a/test/e2e-v2/cases/storage/expected/config-dump.yml b/test/e2e-v2/cases/storage/expected/config-dump.yml index 8c0590d6527c..118fa76f7012 100644 --- a/test/e2e-v2/cases/storage/expected/config-dump.yml +++ b/test/e2e-v2/cases/storage/expected/config-dump.yml @@ -38,6 +38,7 @@ core.default.enableDataKeeperExecutor=true agent-analyzer.default.slowCacheReadThreshold=default:20,redis:10 receiver-ebpf.default.continuousPolicyCacheTimeout=60 receiver-ebpf.default.gRPCSslKeyPath= +gen-ai-analyzer.provider=default receiver-browser.provider=default agent-analyzer.default.segmentStatusAnalysisStrategy=FROM_SPAN_STATUS envoy-metric.default.maxConcurrentCallsPerConnection=0 @@ -64,7 +65,6 @@ telemetry.provider=prometheus core.default.trainingPeriodHttpUriRecognitionPattern=60 promql.default.restContextPath=/ core.default.maxHeapMemoryUsagePercent=96 -genAI-analyzer.provider=default aws-firehose.default.contextPath=/ agent-analyzer.default.slowCacheWriteThreshold=default:20,redis:10 envoy-metric.default.maxMessageSize=0 diff --git a/test/e2e-v2/cases/virtual-genai/Dockerfile.provider b/test/e2e-v2/cases/virtual-genai/Dockerfile.provider index 256187e6c495..93f3935f944a 100644 --- a/test/e2e-v2/cases/virtual-genai/Dockerfile.provider +++ b/test/e2e-v2/cases/virtual-genai/Dockerfile.provider @@ -21,7 +21,9 @@ FROM eclipse-temurin:17-jdk AS builder RUN apt-get update && apt-get install -y git maven && rm -rf /var/lib/apt/lists/* WORKDIR /source -RUN git clone https://github.com/spring-projects/spring-ai-examples.git +RUN git clone https://github.com/spring-projects/spring-ai-examples.git \ + && cd spring-ai-examples \ + && git checkout 2a6088db3d18d5fa6fc208b12adf1172d22f77fd WORKDIR /source/spring-ai-examples/misc/openai-streaming-response RUN mvn clean package -DskipTests @@ -39,3 +41,4 @@ COPY --from=builder /app.jar /services/app.jar EXPOSE 8080 CMD ["sh", "-c", "java -jar /services/app.jar"] + diff --git a/test/e2e-v2/cases/virtual-genai/docker-compose.yml b/test/e2e-v2/cases/virtual-genai/docker-compose.yml index e1839a1a17c4..da9253f2df33 100644 --- a/test/e2e-v2/cases/virtual-genai/docker-compose.yml +++ b/test/e2e-v2/cases/virtual-genai/docker-compose.yml @@ -66,4 +66,4 @@ services: condition: service_healthy networks: - e2e: \ No newline at end of file + e2e: diff --git a/test/e2e-v2/cases/virtual-genai/e2e.yaml b/test/e2e-v2/cases/virtual-genai/e2e.yaml index cfd52903aa70..3416deb91cec 100644 --- a/test/e2e-v2/cases/virtual-genai/e2e.yaml +++ b/test/e2e-v2/cases/virtual-genai/e2e.yaml @@ -42,3 +42,4 @@ verify: cases: - includes: - ./virtual-genai.yaml + diff --git a/test/e2e-v2/cases/virtual-genai/expected/instance.yml b/test/e2e-v2/cases/virtual-genai/expected/instance.yml index d3ffc6ca30a3..f74704b2b875 100644 --- a/test/e2e-v2/cases/virtual-genai/expected/instance.yml +++ b/test/e2e-v2/cases/virtual-genai/expected/instance.yml @@ -15,8 +15,8 @@ {{- contains . }} - id: {{ notEmpty .id }} - name: gpt-4.1-mini + name: gpt-4.1-mini-2025-04-14 instanceuuid: {{ notEmpty .instanceuuid }} attributes: [] language: UNKNOWN -{{- end }} \ No newline at end of file +{{- end }} diff --git a/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml index 1df2937d4846..c983c0e19ba3 100644 --- a/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml +++ b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value-label.yml @@ -20,7 +20,7 @@ results: - metric: labels: {{- contains .metric.labels }} - - key: "_" + - key: "p" value: {{ notEmpty .value }} {{- end}} values: @@ -36,3 +36,4 @@ results: {{- end}} {{- end}} error: null + diff --git a/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml index f68a07155e03..979b9b25775c 100644 --- a/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml +++ b/test/e2e-v2/cases/virtual-genai/expected/metrics-has-value.yml @@ -31,4 +31,4 @@ results: owner: null {{- end}} {{- end}} -error: null \ No newline at end of file +error: null diff --git a/test/e2e-v2/cases/virtual-genai/expected/service.yml b/test/e2e-v2/cases/virtual-genai/expected/service.yml index 498ee501899a..5471c7d596d8 100644 --- a/test/e2e-v2/cases/virtual-genai/expected/service.yml +++ b/test/e2e-v2/cases/virtual-genai/expected/service.yml @@ -21,4 +21,4 @@ layers: - VIRTUAL_GENAI normal: false -{{- end }} \ No newline at end of file +{{- end }} diff --git a/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml b/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml index ca5f4dc66706..aacfa76bdd48 100644 --- a/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml +++ b/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml @@ -26,39 +26,50 @@ cases: expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_cpm --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_latency_avg --service-id=b3BlbmFp.0 - expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_latency_percentile --service-id=b3BlbmFp.0 expected: expected/metrics-has-value-label.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_input_tokens_sum --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_input_tokens_avg --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_output_tokens_sum --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_output_tokens_avg --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_total_cost --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_avg_cost --service-id=b3BlbmFp.0 + expected: expected/metrics-has-value.yml # instance cases - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance ls --service-id=b3BlbmFp.0 expected: expected/instance.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_call_cpm --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_call_cpm --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_sla --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_sla --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_latency_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_latency_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_latency_percentile --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_latency_percentile --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value-label.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_ttft_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_ttft_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_ttft_percentile --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_ttft_percentile --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value-label.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_input_tokens_sum --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_input_tokens_sum --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_output_tokens_sum --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_input_tokens_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_total_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_output_tokens_sum --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_output_tokens_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_total_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_avg_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 + expected: expected/metrics-has-value.yml + diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml b/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml index 048bd92375b0..2de07cc331c9 100644 --- a/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml +++ b/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml @@ -112,11 +112,6 @@ 23.0 - - com.alibaba - fastjson - 1.2.83 - diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java index 24baa68d0088..1d29883ebdd7 100644 --- a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java +++ b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LLMMockController.java @@ -18,9 +18,7 @@ package org.apache.skywalking.e2e.controller; -import com.alibaba.fastjson.JSONObject; import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -32,7 +30,7 @@ @RequestMapping("/llm") public class LLMMockController { @PostMapping("/v1/chat/completions") - public Object completions(@RequestBody JSONObject request, HttpServletResponse response) throws Exception { + public Object completions(HttpServletResponse response) throws Exception { response.setContentType("text/event-stream"); response.setCharacterEncoding("UTF-8"); diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 9d7ff36f3dc0..f9a89d8849f8 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SW_AGENT_JAVA_COMMIT=2f1d9e94d6d1ac22d92f4e9c6905901fe646ffdf +SW_AGENT_JAVA_COMMIT=ac0df43d7140e726eba9e5e5b1b75cf364c71dff SW_AGENT_SATELLITE_COMMIT=ea27a3f4e126a24775fe12e2aa2695bcb23d99c3 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e From ca9704e17be8868d29f38f766068307b88863008 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Fri, 20 Mar 2026 20:57:28 +0800 Subject: [PATCH 04/11] fix --- .../server-starter/src/main/resources/oal/virtual-gen-ai.oal | 2 +- skywalking-ui | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal index ee07e548bec5..d828517b9118 100644 --- a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal +++ b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal @@ -48,4 +48,4 @@ gen_ai_model_output_tokens_sum = from(GenAIModelAccess.outputTokens).sum(); gen_ai_model_output_tokens_avg = from(GenAIModelAccess.outputTokens).longAvg(); gen_ai_model_total_cost = from(GenAIModelAccess.totalCost).sum(); -gen_ai_model_avg_cost = from(GenAIModelAccess.totalCost).doubleAvg(); \ No newline at end of file +gen_ai_model_avg_cost = from(GenAIModelAccess.totalCost).doubleAvg(); diff --git a/skywalking-ui b/skywalking-ui index 6538cc401d19..8b004ef3167c 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 6538cc401d19f768d8b1e075785d991ce7e4739f +Subproject commit 8b004ef3167c44d1e4176db0bdeaf41efbad016b From 8e915bdf05e56b0efc7d524a20c036fcfe537b81 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Sun, 22 Mar 2026 21:18:41 +0800 Subject: [PATCH 05/11] fix some suggestions --- docs/en/setup/service-agent/virtual-genai.md | 61 ++++++++++++++++--- .../vservice/VirtualGenAIProcessor.java | 11 ++++ .../service/GenAIModelAccessDispatcher.java | 37 ----------- .../cases/virtual-genai/docker-compose.yml | 5 ++ 4 files changed, 69 insertions(+), 45 deletions(-) delete mode 100644 oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java diff --git a/docs/en/setup/service-agent/virtual-genai.md b/docs/en/setup/service-agent/virtual-genai.md index ce91ecc2c045..905350cacdd3 100644 --- a/docs/en/setup/service-agent/virtual-genai.md +++ b/docs/en/setup/service-agent/virtual-genai.md @@ -1,17 +1,62 @@ # Virtual GenAI -Virtual GenAI represent the Generative AI service nodes detected by [server agents' plugins](server-agents.md). The performance -metrics of the GenAI operations are also from the GenAI client-side perspective. +Virtual GenAI represents the Generative AI service nodes detected by [server agents' plugins](server-agents.md). The performance +metrics of the GenAI operations are from the GenAI client-side perspective. -For example, an Spring-ai plugin in the Java agent could detect the latency of a chat completion request. -As a result, SkyWalking would show traffic, latency, success rate, and token usage (input/output) powered by backend analysis capabilities in this dashboard. +For example, a Spring AI plugin in the Java agent could detect the latency of a chat completion request. +As a result, SkyWalking would show traffic, latency, success rate, token usage (input/output), and estimated cost in the GenAI dashboard. -The GenAI operation span should have +## Span Contract + +The GenAI operation span should have the following properties: - It is an **Exit** span - **Span's layer == GENAI** - Tag key = `gen_ai.provider.name`, value = The Generative AI provider, e.g. openai, anthropic, ollama -- Tag key = `gen_ai.response.model`, value = The name of the GenAI model a response is being made to, e.g. gpt-4o, claude-3-5-sonnet +- Tag key = `gen_ai.response.model`, value = The name of the GenAI model, e.g. gpt-4o, claude-3-5-sonnet - Tag key = `gen_ai.usage.input_tokens`, value = The number of tokens used in the GenAI input (prompt) - Tag key = `gen_ai.usage.output_tokens`, value = The number of tokens used in the GenAI response (completion) -- Tag key = `gen_ai.server.time_to_first_token`, value = The duration in milliseconds from the start of the request until the first token is received. Note: This metric is only available for streaming requests. -- If the GenAI service is a remote API (e.g. OpenAI), the span's peer would be the network address (IP or domain) of the GenAI server. +- Tag key = `gen_ai.server.time_to_first_token`, value = The duration in milliseconds until the first token is received (streaming requests only) +- If the GenAI service is a remote API (e.g. OpenAI), the span's peer should be the network address (IP or domain) of the GenAI server. + +## Provider Configuration + +SkyWalking uses `gen-ai-config.yml` to map model names to providers and configure cost estimation. + +When the `gen_ai.provider.name` tag is present in the span, it is used directly. Otherwise, SkyWalking matches the model name +against `prefix-match` rules to identify the provider. For example, a model name starting with `gpt` is mapped to `openai`. + +To configure cost estimation, add `models` with pricing under the provider: + +​```yaml +providers: +- provider: openai + prefix-match: + - gpt + models: + - name: gpt-4o + input-cost-per-m: 2.5 # cost per 1,000,000 input tokens + output-cost-per-m: 10 # cost per 1,000,000 output tokens + ​``` + +## Metrics + +The following metrics are available at the **provider** (service) level: +- `gen_ai_provider_cpm` - Calls per minute +- `gen_ai_provider_sla` - Success rate +- `gen_ai_provider_resp_time` - Average response time +- `gen_ai_provider_latency_percentile` - Latency percentiles +- `gen_ai_provider_input_tokens_sum / avg` - Input token usage +- `gen_ai_provider_output_tokens_sum / avg` - Output token usage +- `gen_ai_provider_total_cost / avg_cost` - Estimated cost + +The following metrics are available at the **model** (service instance) level: +- `gen_ai_model_call_cpm` - Calls per minute +- `gen_ai_model_sla` - Success rate +- `gen_ai_model_latency_avg / percentile` - Latency +- `gen_ai_model_ttft_avg / percentile` - Time to first token (streaming only) +- `gen_ai_model_input_tokens_sum / avg` - Input token usage +- `gen_ai_model_output_tokens_sum / avg` - Output token usage +- `gen_ai_model_total_cost / avg_cost` - Estimated cost + +## Requirement +`skwaylking java agent` version >= 9.7 \ No newline at end of file diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java index e5e8c4af33a5..2417f627fcb1 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java @@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.source.GenAIMetrics; import org.apache.skywalking.oap.server.core.source.GenAIModelAccess; import org.apache.skywalking.oap.server.core.source.GenAIProviderAccess; +import org.apache.skywalking.oap.server.core.source.ServiceInstance; import org.apache.skywalking.oap.server.core.source.ServiceMeta; import org.apache.skywalking.oap.server.core.source.Source; @@ -55,6 +56,7 @@ public void prepareVSIfNecessary(SpanObject span, SegmentObject segmentObject) { } recordList.add(toServiceMeta(metrics)); + recordList.add(toInstance(metrics)); recordList.add(toProviderAccess(metrics)); recordList.add(toModelAccess(metrics)); } @@ -67,6 +69,15 @@ private ServiceMeta toServiceMeta(GenAIMetrics metrics) { return service; } + private Source toInstance(GenAIMetrics metrics) { + ServiceInstance instance = new ServiceInstance(); + instance.setTimeBucket(metrics.getTimeBucket()); + instance.setName(metrics.getModelName()); + instance.setServiceLayer(Layer.VIRTUAL_GENAI); + instance.setServiceName(metrics.getProviderName()); + return instance; + } + private GenAIProviderAccess toProviderAccess(GenAIMetrics metrics) { GenAIProviderAccess source = new GenAIProviderAccess(); source.setName(namingControl.formatServiceName(metrics.getProviderName())); diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java deleted file mode 100644 index d5b0337a1d93..000000000000 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIModelAccessDispatcher.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.analyzer.genai.service; - -import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; -import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; -import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; -import org.apache.skywalking.oap.server.core.source.GenAIModelAccess; - -public class GenAIModelAccessDispatcher implements SourceDispatcher { - - @Override - public void dispatch(GenAIModelAccess source) { - InstanceTraffic traffic = new InstanceTraffic(); - traffic.setTimeBucket(source.getTimeBucket()); - traffic.setName(source.getModelName()); - traffic.setServiceId(source.getServiceId()); - traffic.setLastPingTimestamp(source.getTimeBucket()); - MetricsStreamProcessor.getInstance().in(traffic); - } -} diff --git a/test/e2e-v2/cases/virtual-genai/docker-compose.yml b/test/e2e-v2/cases/virtual-genai/docker-compose.yml index da9253f2df33..25e44de2d73b 100644 --- a/test/e2e-v2/cases/virtual-genai/docker-compose.yml +++ b/test/e2e-v2/cases/virtual-genai/docker-compose.yml @@ -61,6 +61,11 @@ services: SW_LOGGING_OUTPUT: CONSOLE SW_AGENT_NAME: e2e-spring-ai SW_AGENT_INSTANCE_NAME: spring-ai-examples + healthcheck: + test: [ "CMD", "sh", "-c", "nc -nz 127.0.0.1 8080" ] + interval: 5s + timeout: 60s + retries: 120 depends_on: oap: condition: service_healthy From 30da5b4509a240ff173576160069512f5a692880 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Mon, 23 Mar 2026 08:55:20 +0800 Subject: [PATCH 06/11] fix some suggestions --- docs/en/setup/service-agent/virtual-genai.md | 17 +++++++------- .../generator/RuntimeOALGenerationTest.java | 2 +- .../src/main/resources/oal/virtual-gen-ai.oal | 15 +++++-------- .../virtual_genai/virtual-genai-model.json | 21 +++++++----------- .../virtual_genai/virtual-genai-provider.json | 22 +++++++------------ .../cases/virtual-genai/virtual-genai.yaml | 19 +++++----------- 6 files changed, 37 insertions(+), 59 deletions(-) diff --git a/docs/en/setup/service-agent/virtual-genai.md b/docs/en/setup/service-agent/virtual-genai.md index 905350cacdd3..e5e518ad8883 100644 --- a/docs/en/setup/service-agent/virtual-genai.md +++ b/docs/en/setup/service-agent/virtual-genai.md @@ -27,16 +27,17 @@ against `prefix-match` rules to identify the provider. For example, a model name To configure cost estimation, add `models` with pricing under the provider: -​```yaml +yaml +``` providers: - provider: openai prefix-match: - gpt - models: + models: - name: gpt-4o - input-cost-per-m: 2.5 # cost per 1,000,000 input tokens - output-cost-per-m: 10 # cost per 1,000,000 output tokens - ​``` + input-estimated-cost-per-m: 2.5 # estimated cost per 1,000,000 input tokens + output-estimated-cost-per-m: 10 # estimated cost per 1,000,000 output tokens +``` ## Metrics @@ -47,7 +48,7 @@ The following metrics are available at the **provider** (service) level: - `gen_ai_provider_latency_percentile` - Latency percentiles - `gen_ai_provider_input_tokens_sum / avg` - Input token usage - `gen_ai_provider_output_tokens_sum / avg` - Output token usage -- `gen_ai_provider_total_cost / avg_cost` - Estimated cost +- `gen_ai_provider_total_estimated_cost / avg_estimated_cost` - Estimated cost The following metrics are available at the **model** (service instance) level: - `gen_ai_model_call_cpm` - Calls per minute @@ -56,7 +57,7 @@ The following metrics are available at the **model** (service instance) level: - `gen_ai_model_ttft_avg / percentile` - Time to first token (streaming only) - `gen_ai_model_input_tokens_sum / avg` - Input token usage - `gen_ai_model_output_tokens_sum / avg` - Output token usage -- `gen_ai_model_total_cost / avg_cost` - Estimated cost +- `gen_ai_model_total_estimated_cost / avg_estimated_cost` - Estimated cost ## Requirement -`skwaylking java agent` version >= 9.7 \ No newline at end of file +`Skwaylking java agent` version >= 9.7 \ No newline at end of file diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java index 87691eae094c..1665c5550e99 100644 --- a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/v2/generator/RuntimeOALGenerationTest.java @@ -98,7 +98,7 @@ public static void setup() { // DisableOALDefine - no catalog registerOALDefine("disable", createOALDefine("oal/disable.oal", SOURCE_PACKAGE, "")); - registerOALDefine("disable", createOALDefine("oal/virtual-gen-ai.oal", SOURCE_PACKAGE, "")); + registerOALDefine("virtual-gen-ai", createOALDefine("oal/virtual-gen-ai.oal", SOURCE_PACKAGE, "")); // Set generated file path for IDE inspection OALClassGeneratorV2.setGeneratedFilePath("target/test-classes"); diff --git a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal index d828517b9118..f0bc47e91c87 100644 --- a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal +++ b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,24 +19,19 @@ gen_ai_provider_resp_time = from(GenAIProviderAccess.latency).longAvg(); gen_ai_provider_sla = from(GenAIProviderAccess.*).percent(status == true); gen_ai_provider_cpm = from(GenAIProviderAccess.*).cpm(); - gen_ai_provider_latency_percentile = from(GenAIProviderAccess.latency).percentile2(10); gen_ai_provider_input_tokens_sum = from(GenAIProviderAccess.inputTokens).sum(); gen_ai_provider_input_tokens_avg = from(GenAIProviderAccess.inputTokens).longAvg(); - gen_ai_provider_output_tokens_sum = from(GenAIProviderAccess.outputTokens).sum(); gen_ai_provider_output_tokens_avg = from(GenAIProviderAccess.outputTokens).longAvg(); -gen_ai_provider_total_cost = from(GenAIProviderAccess.totalCost).sum(); -gen_ai_provider_avg_cost = from(GenAIProviderAccess.totalCost).doubleAvg(); +gen_ai_provider_total_estimated_cost = from(GenAIProviderAccess.totalCost).sum(); +gen_ai_provider_avg_estimated_cost = from(GenAIProviderAccess.totalCost).doubleAvg(); gen_ai_model_call_cpm = from(GenAIModelAccess.*).cpm(); - gen_ai_model_sla = from(GenAIModelAccess.*).percent(status == true); - gen_ai_model_latency_avg = from(GenAIModelAccess.latency).longAvg(); - gen_ai_model_latency_percentile = from(GenAIModelAccess.latency).percentile2(10); gen_ai_model_ttft_avg = from(GenAIModelAccess.timeToFirstToken).filter(timeToFirstToken > 0).longAvg(); @@ -47,5 +42,5 @@ gen_ai_model_input_tokens_avg = from(GenAIModelAccess.inputTokens).longAvg(); gen_ai_model_output_tokens_sum = from(GenAIModelAccess.outputTokens).sum(); gen_ai_model_output_tokens_avg = from(GenAIModelAccess.outputTokens).longAvg(); -gen_ai_model_total_cost = from(GenAIModelAccess.totalCost).sum(); -gen_ai_model_avg_cost = from(GenAIModelAccess.totalCost).doubleAvg(); +gen_ai_model_total_estimated_cost = from(GenAIModelAccess.totalCost).sum(); +gen_ai_model_avg_estimated_cost = from(GenAIModelAccess.totalCost).doubleAvg(); \ No newline at end of file diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json index 610cf90adea1..5035050dcaf6 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-model.json @@ -22,7 +22,7 @@ "i": "2", "type": "Widget", "expressions": [ - "latest(gen_ai_model_total_cost)/1000000" + "latest(gen_ai_model_total_estimated_cost)/1000000" ], "graph": { "type": "Card", @@ -31,9 +31,9 @@ "showUnit": true }, "widget": { - "name": "TotalCost", - "title": "Total Cost", - "tips": "The total cost of GenAI model calls." + "name": "TotalEstimatedCost", + "title": "Total Estimated Cost", + "tips": "The total estimated cost of GenAI model calls." }, "id": "0-0-2", "moved": false, @@ -355,15 +355,10 @@ "i": "24", "type": "Widget", "widget": { - "name": "AvgCost", - "title": "Average Cost", - "tips": "The average cost of model calls." + "name": "AvgEstimatedCost", + "title": "Average Estimated Cost", + "tips": "The average estimated cost of model calls." }, - "metricConfig": [ - { - "label": "Avg Cost" - } - ], "graph": { "type": "Line", "step": false, @@ -375,7 +370,7 @@ "id": "0-0-24", "moved": false, "expressions": [ - "gen_ai_model_avg_cost" + "gen_ai_model_avg_estimated_cost" ], "typesOfMQE": [ "TIME_SERIES_VALUES" diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json index 3bbbe6923f9e..7a2d7406f751 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/virtual_genai/virtual-genai-provider.json @@ -22,7 +22,7 @@ "i": "2", "type": "Widget", "expressions": [ - "latest(gen_ai_provider_total_cost)/1000000" + "latest(gen_ai_provider_total_estimated_cost)/1000000" ], "graph": { "type": "Card", @@ -31,9 +31,9 @@ "showUnit": true }, "widget": { - "name": "TotalCost", - "title": "Total Cost", - "tips": "The total cost of GenAI model calls." + "name": "TotalEstimatedCost", + "title": "Total Estimated Cost", + "tips": "The total estimated cost of GenAI model calls." }, "id": "0-0-2", "moved": false, @@ -199,7 +199,7 @@ "i": "14", "type": "Widget", "expressions": [ - "gen_ai_provider_avg_cost" + "gen_ai_provider_avg_estimated_cost" ], "graph": { "type": "Line", @@ -207,16 +207,10 @@ "showYAxis": true }, "widget": { - "name": "AvgCost", - "title": "Average Cost", - "tips": "The average cost of provider access." + "name": "AvgEstimatedCost", + "title": "Average Estimated Cost", + "tips": "The average estimated cost of provider access." }, - "metricConfig": [ - { - "label": "Avg Latency", - "unit": "ms" - } - ], "id": "0-0-14", "moved": false, "typesOfMQE": [ diff --git a/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml b/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml index aacfa76bdd48..7f7274b1fa47 100644 --- a/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml +++ b/test/e2e-v2/cases/virtual-genai/virtual-genai.yaml @@ -19,7 +19,6 @@ cases: # service cases - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls expected: expected/service.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_resp_time --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_sla --service-id=b3BlbmFp.0 @@ -36,13 +35,12 @@ cases: expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_output_tokens_avg --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_total_cost --service-id=b3BlbmFp.0 + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_total_estimated_cost --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_avg_cost --service-id=b3BlbmFp.0 + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_provider_avg_estimated_cost --service-id=b3BlbmFp.0 expected: expected/metrics-has-value.yml - - # instance cases + # instance cases - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance ls --service-id=b3BlbmFp.0 expected: expected/instance.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_call_cpm --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 @@ -65,12 +63,7 @@ cases: expected: expected/metrics-has-value.yml - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_output_tokens_avg --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_total_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_total_estimated_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_avg_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 - expected: expected/metrics-has-value.yml - - - - - + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=gen_ai_model_avg_estimated_cost --service-id=b3BlbmFp.0 --instance-name=gpt-4.1-mini-2025-04-14 + expected: expected/metrics-has-value.yml \ No newline at end of file From 25394b0e2ac728ef1e2d8e3ac6815f3f5023b10a Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Mon, 23 Mar 2026 22:17:20 +0800 Subject: [PATCH 07/11] fix some suggestions and add some default model pricing --- docs/en/setup/service-agent/virtual-genai.md | 6 +- docs/menu.yml | 2 +- .../analyzer/genai/config/GenAIConfig.java | 4 +- .../genai/config/GenAIConfigLoader.java | 4 +- .../genai/service/GenAIMeterAnalyzer.java | 12 +- .../src/main/resources/gen-ai-config.yml | 300 +++++++++++++++++- 6 files changed, 305 insertions(+), 23 deletions(-) diff --git a/docs/en/setup/service-agent/virtual-genai.md b/docs/en/setup/service-agent/virtual-genai.md index e5e518ad8883..86b9d5376429 100644 --- a/docs/en/setup/service-agent/virtual-genai.md +++ b/docs/en/setup/service-agent/virtual-genai.md @@ -27,8 +27,8 @@ against `prefix-match` rules to identify the provider. For example, a model name To configure cost estimation, add `models` with pricing under the provider: -yaml -``` + +```yaml providers: - provider: openai prefix-match: @@ -60,4 +60,4 @@ The following metrics are available at the **model** (service instance) level: - `gen_ai_model_total_estimated_cost / avg_estimated_cost` - Estimated cost ## Requirement -`Skwaylking java agent` version >= 9.7 \ No newline at end of file +`SkyWalking Java Agent` version >= 9.7 \ No newline at end of file diff --git a/docs/menu.yml b/docs/menu.yml index e347015d95bc..0eb6a07f13c8 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -158,7 +158,7 @@ catalog: path: "/en/setup/backend/dashboards-so11y-go-agent" - name: "GenAI" catalog: - - name: "Virtual Genai" + - name: "Virtual GenAI" path: "/en/setup/service-agent/virtual-genai" - name: "Configuration Vocabulary" path: "/en/setup/backend/configuration-vocabulary" diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java index 46d83174c586..a4a667a80ff2 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfig.java @@ -44,7 +44,7 @@ public static class Provider { @Setter public static class Model { private String name; - private double inputCostPerM; - private double outputCostPerM; + private double inputEstimatedCostPerM; + private double outputEstimatedCostPerM; } } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java index ea16c7cc7112..c39d126b8c86 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAIConfigLoader.java @@ -84,8 +84,8 @@ public GenAIConfig loadConfig() throws ModuleStartException { Map modelMap = (Map) modelObj; GenAIConfig.Model model = new GenAIConfig.Model(); model.setName(String.valueOf(modelMap.get("name"))); - model.setInputCostPerM(parseCost(modelMap.get("input-cost-per-m"))); - model.setOutputCostPerM(parseCost(modelMap.get("output-cost-per-m"))); + model.setInputEstimatedCostPerM(parseCost(modelMap.get("input-estimated-cost-per-m"))); + model.setOutputEstimatedCostPerM(parseCost(modelMap.get("output-estimated-cost-per-m"))); provider.getModels().add(model); } } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java index 735acac69dd1..82d0c6b62906 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java @@ -77,11 +77,11 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm // calculate the total cost by the cost configs double totalCost = 0.0; if (modelConfig != null) { - if (modelConfig.getInputCostPerM() > 0) { - totalCost += inputTokens * modelConfig.getInputCostPerM(); + if (modelConfig.getInputEstimatedCostPerM() > 0) { + totalCost += inputTokens * modelConfig.getInputEstimatedCostPerM(); } - if (modelConfig.getOutputCostPerM() > 0) { - totalCost += outputTokens * modelConfig.getOutputCostPerM(); + if (modelConfig.getOutputEstimatedCostPerM() > 0) { + totalCost += outputTokens * modelConfig.getOutputEstimatedCostPerM(); } } @@ -111,7 +111,7 @@ private long parseSafeLong(String value) { try { return Long.parseLong(value); } catch (NumberFormatException e) { - LOG.warn("Failed to parse token count: {}", value); + LOG.warn("Failed to parse value to long: {}", value); return 0; } } @@ -123,7 +123,7 @@ private int parseSafeInt(String value) { try { return Integer.parseInt(value); } catch (NumberFormatException e) { - LOG.warn("Failed to parse token count: {}", value); + LOG.warn("Failed to parse value to int: {}", value); return 0; } } diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index 49cba6f37352..49fcff0030ce 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -28,35 +28,243 @@ # # The currency depends on your system's default setting (e.g., USD or CNY). # # # Estimated cost for every 1,000,000 input (prompt) tokens -# input-cost-per-m: 1 +# input-estimated-cost-per-m: 1 # # # Estimated cost for every 1,000,000 output (completion) tokens -# output-cost-per-m: 1 +# output-estimated-cost-per-m: 1 + +last-updated: 2026-03-23 + +# --- IMPORTANT NOTES +# 1. ESTIMATED COSTS ONLY: The pricing values below are for estimation purposes +# within the system and MUST NOT be treated as actual billing or invoice amounts. +# +# 2. DATA SOURCE: These figures are based on official provider pricing pages +# available as of the 'last-updated' date above. +# +# 3. MARKET VOLATILITY: GenAI model pricing is subject to frequent and rapid +# changes without prior notice. +# +# 4. USER RESPONSIBILITY & CURRENCY: Users are encouraged to verify current rates regularly. +# Default unit is USD; prices from providers using other currencies have been +# converted based on exchange rates at the time of update. + providers: - provider: openai prefix-match: - gpt + models: + - name: gpt-5.4-pro + # Pricing for Short context + input-estimated-cost-per-m: 30.0 + output-estimated-cost-per-m: 180.0 + - name: gpt-5.4 + # Pricing for Short context + input-estimated-cost-per-m: 2.5 + output-estimated-cost-per-m: 15.0 + - name: gpt-5.4-mini + # Pricing for Short context + input-estimated-cost-per-m: 0.75 + output-estimated-cost-per-m: 4.5 + - name: gpt-5.4-nano + # Pricing for Short context + input-estimated-cost-per-m: 0.2 + output-estimated-cost-per-m: 1.25 + - name: gpt-5.2-pro + input-estimated-cost-per-m: 21.0 + output-estimated-cost-per-m: 168.0 + - name: gpt-5.2 + input-estimated-cost-per-m: 1.75 + output-estimated-cost-per-m: 14.0 + - name: gpt-5.1 + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: gpt-5-pro + input-estimated-cost-per-m: 15.0 + output-estimated-cost-per-m: 120.0 + - name: gpt-5 + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: gpt-5-mini + input-estimated-cost-per-m: 0.25 + output-estimated-cost-per-m: 2.0 + - name: gpt-5-nano + input-estimated-cost-per-m: 0.05 + output-estimated-cost-per-m: 0.4 + - name: gpt-4.1 + input-estimated-cost-per-m: 2.0 + output-estimated-cost-per-m: 8.0 + - name: gpt-4.1-mini-2025-04-14 + input-estimated-cost-per-m: 0.4 + output-estimated-cost-per-m: 1.6 + - name: gpt-4.1-nano + input-estimated-cost-per-m: 0.1 + output-estimated-cost-per-m: 0.4 + - name: gpt-4o + input-estimated-cost-per-m: 2.5 + output-estimated-cost-per-m: 10.0 + - name: gpt-4o-mini + input-estimated-cost-per-m: 0.15 + output-estimated-cost-per-m: 0.6 - provider: anthropic prefix-match: - claude + models: + - name: claude-4.6-opus + # Base Input pricing + input-estimated-cost-per-m: 5.0 + output-estimated-cost-per-m: 25.0 + - name: claude-4.5-opus + # Base Input pricing + input-estimated-cost-per-m: 5.0 + output-estimated-cost-per-m: 25.0 + - name: claude-4.1-opus + # Base Input pricing + input-estimated-cost-per-m: 15.0 + output-estimated-cost-per-m: 75.0 + - name: claude-4-opus + # Base Input pricing + input-estimated-cost-per-m: 15.0 + output-estimated-cost-per-m: 75.0 + - name: claude-4.6-sonnet + # Base Input pricing + input-estimated-cost-per-m: 3.0 + output-estimated-cost-per-m: 15.0 + - name: claude-4.5-sonnet + # Base Input pricing + input-estimated-cost-per-m: 3.0 + output-estimated-cost-per-m: 15.0 + - name: claude-4-sonnet + # Base Input pricing + input-estimated-cost-per-m: 3.0 + output-estimated-cost-per-m: 15.0 + - name: claude-3.7-sonnet + # Deprecated, Base Input pricing + input-estimated-cost-per-m: 3.0 + output-estimated-cost-per-m: 15.0 + - name: claude-4.5-haiku + # Base Input pricing + input-estimated-cost-per-m: 1.0 + output-estimated-cost-per-m: 5.0 + - name: claude-3.5-haiku + # Base Input pricing + input-estimated-cost-per-m: 0.8 + output-estimated-cost-per-m: 4.0 + - name: claude-3-opus + # Deprecated, Base Input pricing + input-estimated-cost-per-m: 15.0 + output-estimated-cost-per-m: 75.0 + - name: claude-3-haiku + # Base Input pricing + input-estimated-cost-per-m: 0.25 + output-estimated-cost-per-m: 1.25 - provider: gemini prefix-match: - - gemini + - gemini + models: + # --- Gemini 3.1 Series --- + - name: gemini-3.1-pro-preview + + # Pricing for prompt <= 200k (Tier 1) + input-estimated-cost-per-m: 2.0 + output-estimated-cost-per-m: 12.0 + - name: gemini-3.1-pro-preview-customtools + input-estimated-cost-per-m: 2.0 + output-estimated-cost-per-m: 12.0 + + # Pricing for text/picture/video + - name: gemini-3.1-flash-lite-preview + input-estimated-cost-per-m: 0.25 + output-estimated-cost-per-m: 1.5 + - name: gemini-3.1-flash-image-preview + input-estimated-cost-per-m: 0.5 + output-estimated-cost-per-m: 3.0 + + # --- Gemini 3 Series --- + - name: gemini-3-flash-preview + input-estimated-cost-per-m: 0.5 + output-estimated-cost-per-m: 3.0 + - name: gemini-3-pro-image-preview + input-estimated-cost-per-m: 2.0 + output-estimated-cost-per-m: 12.0 + + # --- Gemini 2.5 Series --- + - name: gemini-2.5-pro + # Pricing for prompt <= 200k (Tier 1) + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: gemini-2.5-flash + input-estimated-cost-per-m: 0.3 + output-estimated-cost-per-m: 2.5 + - name: gemini-2.5-flash-lite + input-estimated-cost-per-m: 0.1 + output-estimated-cost-per-m: 0.4 + - name: gemini-2.5-flash-lite-preview-09-2025 + input-estimated-cost-per-m: 0.1 + output-estimated-cost-per-m: 0.4 + - name: gemini-2.5-flash-native-audio-preview-12-2025 + input-estimated-cost-per-m: 0.5 + output-estimated-cost-per-m: 2.0 + - name: gemini-2.5-flash-image + input-estimated-cost-per-m: 0.3 + output-estimated-cost-per-m: 2.5 + - name: gemini-2.5-flash-preview-tts + input-estimated-cost-per-m: 0.5 + output-estimated-cost-per-m: 10.0 + - name: gemini-2.5-pro-preview-tts + input-estimated-cost-per-m: 1.0 + output-estimated-cost-per-m: 20.0 + + # --- Gemini 2.0 Series (Deprecated) --- + - name: gemini-2.0-flash + input-estimated-cost-per-m: 0.1 + output-estimated-cost-per-m: 0.4 - provider: mistral prefix-match: - - mistral- + - mistral + models: + - name: mistral-large-3 + input-estimated-cost-per-m: 0.5 + output-estimated-cost-per-m: 1.5 + - name: mistral-medium-3 + input-estimated-cost-per-m: 0.4 + output-estimated-cost-per-m: 2.0 + - name: mistral-small-4 + input-estimated-cost-per-m: 0.15 + output-estimated-cost-per-m: 0.6 - provider: groq prefix-match: - llama-3 + models: + - name: llama-4-scout-17bx16e-128k + input-estimated-cost-per-m: 0.11 + output-estimated-cost-per-m: 0.34 + - name: llama-3.3-70b-versatile-128k + input-estimated-cost-per-m: 0.59 + output-estimated-cost-per-m: 0.79 + - name: llama-3.1-8b-instant-128k + input-estimated-cost-per-m: 0.05 + output-estimated-cost-per-m: 0.08 - provider: deepseek prefix-match: - deepseek + models: + - name: deepseek-chat + # DeepSeek-V3.2 (Non-thinking Mode), 128K context + # Input pricing for cache miss + input-estimated-cost-per-m: 0.28 + output-estimated-cost-per-m: 0.42 + - name: deepseek-reasoner + # DeepSeek-V3.2 (Thinking Mode), 128K context + # Input pricing for cache miss + input-estimated-cost-per-m: 0.28 + output-estimated-cost-per-m: 0.42 - provider: bytedance prefix-match: @@ -65,27 +273,101 @@ providers: - provider: zhipu_ai prefix-match: - glm + models: + - name: glm-5-turbo + # Input [0, 32) + input-estimated-cost-per-m: 0.73 + output-estimated-cost-per-m: 3.20 + - name: glm-5 + # Input [0, 32) + input-estimated-cost-per-m: 0.58 + output-estimated-cost-per-m: 2.61 + - name: glm-4.7 + # Input [0, 32) + input-estimated-cost-per-m: 0.29 + output-estimated-cost-per-m: 1.16 + - name: glm-4.5-air + # Input [0, 32) + input-estimated-cost-per-m: 0.12 + output-estimated-cost-per-m: 0.29 + - name: glm-4.7-flashx + # For 200K context + input-estimated-cost-per-m: 0.07 + output-estimated-cost-per-m: 0.44 - provider: alibaba prefix-match: - qwen + models: + - name: qwen3-max + input-estimated-cost-per-m: 0.36 + output-estimated-cost-per-m: 1.45 + - name: qwen3.5-plus + input-estimated-cost-per-m: 0.12 + output-estimated-cost-per-m: 0.70 + - name: qwen3.5-flash + input-estimated-cost-per-m: 0.03 + output-estimated-cost-per-m: 0.29 - provider: tencent prefix-match: - hunyuan + - Tencent + models: + - name: hunyuan-2.0-think + # For Input (0, 32k] + input-estimated-cost-per-m: 0.58 + output-estimated-cost-per-m: 2.31 + - name: hunyuan-2.0-instruct + # For Input (0, 32k] + input-estimated-cost-per-m: 0.46 + output-estimated-cost-per-m: 1.15 + - name: hunyuan-turbos + input-estimated-cost-per-m: 0.12 + output-estimated-cost-per-m: 0.29 + - name: hunyuan-t1 + input-estimated-cost-per-m: 0.15 + output-estimated-cost-per-m: 0.58 + - name: hunyuan-turbos + input-estimated-cost-per-m: 0.12 + output-estimated-cost-per-m: 0.29 + - name: hunyuan-a13b + input-estimated-cost-per-m: 0.07 + output-estimated-cost-per-m: 0.29 - provider: moonshot prefix-match: - kimi + models: + - name: kimi-k2.5 + input-estimated-cost-per-m: 0.58 + output-estimated-cost-per-m: 3.05 + - name: kimi-k2-0905 + input-estimated-cost-per-m: 0.58 + output-estimated-cost-per-m: 2.32 + - name: kimi-k2-thinking + input-estimated-cost-per-m: 0.58 + output-estimated-cost-per-m: 2.32 - provider: minimax prefix-match: - minimax + models: + - name: minimax-m2.7 + input-estimated-cost-per-m: 0.3 + output-estimated-cost-per-m: 1.2 + - name: minimax-m2.7-highspeed + input-estimated-cost-per-m: 0.6 + output-estimated-cost-per-m: 2.4 + - name: minimax-m2.5 + input-estimated-cost-per-m: 0.3 + output-estimated-cost-per-m: 1.2 + - name: minimax-m2.5-highspeed + input-estimated-cost-per-m: 0.6 + output-estimated-cost-per-m: 2.4 + - name: m2-her + input-estimated-cost-per-m: 0.3 + output-estimated-cost-per-m: 1.2 - provider: ollama prefix-match: - - ollama - - - provider: azure_openai - prefix-match: - - azure From ea7e330222a5b7c35071a032c78cc06429f34244 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Tue, 24 Mar 2026 19:32:21 +0800 Subject: [PATCH 08/11] fix --- .../vservice/VirtualGenAIProcessor.java | 4 +- .../{GenAITagKey.java => GenAITagKeys.java} | 2 +- .../genai/service/GenAIMeterAnalyzer.java | 34 ++- .../genai/GenAIMeterAnalyzerTest.java | 234 ++++++++++++++++++ .../oap/server/core/source/GenAIMetrics.java | 7 +- .../server/core/source/GenAIModelAccess.java | 2 +- .../core/source/GenAIProviderAccess.java | 2 +- .../src/main/resources/gen-ai-config.yml | 66 ++++- 8 files changed, 323 insertions(+), 28 deletions(-) rename oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/{GenAITagKey.java => GenAITagKeys.java} (97%) create mode 100644 oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java index 2417f627fcb1..0aa1cdbc3d1b 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java @@ -83,7 +83,7 @@ private GenAIProviderAccess toProviderAccess(GenAIMetrics metrics) { source.setName(namingControl.formatServiceName(metrics.getProviderName())); source.setInputTokens(metrics.getInputTokens()); source.setOutputTokens(metrics.getOutputTokens()); - source.setTotalCost(metrics.getTotalCost()); + source.setTotalEstimatedCost(metrics.getTotalEstimatedCost()); source.setLatency(metrics.getLatency()); source.setStatus(metrics.isStatus()); source.setTimeBucket(metrics.getTimeBucket()); @@ -96,7 +96,7 @@ private GenAIModelAccess toModelAccess(GenAIMetrics metrics) { source.setModelName(namingControl.formatInstanceName(metrics.getModelName())); source.setInputTokens(metrics.getInputTokens()); source.setOutputTokens(metrics.getOutputTokens()); - source.setTotalCost(metrics.getTotalCost()); + source.setTotalEstimatedCost(metrics.getTotalEstimatedCost()); source.setTimeToFirstToken(metrics.getTimeToFirstToken()); source.setLatency(metrics.getLatency()); source.setStatus(metrics.isStatus()); diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKey.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKeys.java similarity index 97% rename from oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKey.java rename to oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKeys.java index df33c32262f4..e98b49d55a15 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKey.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/config/GenAITagKeys.java @@ -17,7 +17,7 @@ package org.apache.skywalking.oap.analyzer.genai.config; -public class GenAITagKey { +public class GenAITagKeys { public static final String PROVIDER_NAME = "gen_ai.provider.name"; diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java index 82d0c6b62906..15f5bc9f5845 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java @@ -17,28 +17,26 @@ package org.apache.skywalking.oap.analyzer.genai.service; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; -import org.apache.skywalking.oap.analyzer.genai.config.GenAITagKey; +import org.apache.skywalking.oap.analyzer.genai.config.GenAITagKeys; import org.apache.skywalking.oap.analyzer.genai.matcher.GenAIProviderPrefixMatcher; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.source.GenAIMetrics; import org.apache.skywalking.oap.server.library.util.StringUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; import static java.util.stream.Collectors.toMap; +@Slf4j public class GenAIMeterAnalyzer implements IGenAIMeterAnalyzerService { - private static final Logger LOG = LoggerFactory.getLogger(GenAIMeterAnalyzer.class); - private final GenAIProviderPrefixMatcher matcher; public GenAIMeterAnalyzer(GenAIProviderPrefixMatcher matcher) { @@ -54,15 +52,15 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm (v1, v2) -> v1 )); - String modelName = tags.get(GenAITagKey.RESPONSE_MODEL); + String modelName = tags.get(GenAITagKeys.RESPONSE_MODEL); if (StringUtil.isBlank(modelName)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Model name is missing in span [{}], skipping GenAI analysis", span.getOperationName()); + if (log.isDebugEnabled()) { + log.debug("Model name is missing in span [{}], skipping GenAI analysis", span.getOperationName()); } return null; } - String provider = tags.get(GenAITagKey.PROVIDER_NAME); + String provider = tags.get(GenAITagKeys.PROVIDER_NAME); GenAIProviderPrefixMatcher.MatchResult matchResult = matcher.match(modelName); if (StringUtil.isBlank(provider)) { @@ -71,17 +69,17 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm GenAIConfig.Model modelConfig = matchResult.getModelConfig(); - long inputTokens = parseSafeLong(tags.get(GenAITagKey.INPUT_TOKENS)); - long outputTokens = parseSafeLong(tags.get(GenAITagKey.OUTPUT_TOKENS)); + long inputTokens = parseSafeLong(tags.get(GenAITagKeys.INPUT_TOKENS)); + long outputTokens = parseSafeLong(tags.get(GenAITagKeys.OUTPUT_TOKENS)); // calculate the total cost by the cost configs - double totalCost = 0.0; + long totalCost = 0L; if (modelConfig != null) { if (modelConfig.getInputEstimatedCostPerM() > 0) { - totalCost += inputTokens * modelConfig.getInputEstimatedCostPerM(); + totalCost += (long) (inputTokens * modelConfig.getInputEstimatedCostPerM()); } if (modelConfig.getOutputEstimatedCostPerM() > 0) { - totalCost += outputTokens * modelConfig.getOutputEstimatedCostPerM(); + totalCost += (long) (outputTokens * modelConfig.getOutputEstimatedCostPerM()); } } @@ -93,8 +91,8 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm metrics.setInputTokens(inputTokens); metrics.setOutputTokens(outputTokens); - metrics.setTimeToFirstToken(parseSafeInt(tags.get(GenAITagKey.SERVER_TIME_TO_FIRST_TOKEN))); - metrics.setTotalCost(totalCost); + metrics.setTimeToFirstToken(parseSafeInt(tags.get(GenAITagKeys.SERVER_TIME_TO_FIRST_TOKEN))); + metrics.setTotalEstimatedCost(totalCost); long latency = span.getEndTime() - span.getStartTime(); metrics.setLatency(latency); @@ -111,7 +109,7 @@ private long parseSafeLong(String value) { try { return Long.parseLong(value); } catch (NumberFormatException e) { - LOG.warn("Failed to parse value to long: {}", value); + log.warn("Failed to parse value to long: {}", value); return 0; } } @@ -123,7 +121,7 @@ private int parseSafeInt(String value) { try { return Integer.parseInt(value); } catch (NumberFormatException e) { - LOG.warn("Failed to parse value to int: {}", value); + log.warn("Failed to parse value to int: {}", value); return 0; } } diff --git a/oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java b/oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java new file mode 100644 index 000000000000..f8e069784697 --- /dev/null +++ b/oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.analyzer.genai; + +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; +import org.apache.skywalking.oap.analyzer.genai.config.GenAITagKeys; +import org.apache.skywalking.oap.analyzer.genai.matcher.GenAIProviderPrefixMatcher; +import org.apache.skywalking.oap.analyzer.genai.service.GenAIMeterAnalyzer; +import org.apache.skywalking.oap.server.core.source.GenAIMetrics; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GenAIMeterAnalyzerTest { + private GenAIProviderPrefixMatcher matcher; + private GenAIMeterAnalyzer analyzer; + + @BeforeEach + void setUp() { + matcher = mock(GenAIProviderPrefixMatcher.class); + analyzer = new GenAIMeterAnalyzer(matcher); + } + + @Test + void testExtractMetricsWithValidSpan() { + SpanObject span = SpanObject.newBuilder() + .setOperationName("genai_call") + .setStartTime(1000000L) + .setEndTime(1005000L) + .setIsError(false) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.RESPONSE_MODEL) + .setValue("gpt-4o") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.PROVIDER_NAME) + .setValue("openai") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.INPUT_TOKENS) + .setValue("1000") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.OUTPUT_TOKENS) + .setValue("500") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.SERVER_TIME_TO_FIRST_TOKEN) + .setValue("100") + .build()) + .build(); + + GenAIConfig.Model modelConfig = new GenAIConfig.Model(); + modelConfig.setName("gpt-4o"); + modelConfig.setInputEstimatedCostPerM(2.5); + modelConfig.setOutputEstimatedCostPerM(10.0); + + GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("openai", modelConfig); + when(matcher.match("gpt-4o")).thenReturn(matchResult); + SegmentObject segment = SegmentObject.newBuilder().build(); + GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); + + assertNotNull(metrics); + assertEquals("openai", metrics.getProviderName()); + assertEquals("gpt-4o", metrics.getModelName()); + assertEquals(1000L, metrics.getInputTokens()); + assertEquals(500L, metrics.getOutputTokens()); + assertEquals(100, metrics.getTimeToFirstToken()); + assertEquals(7500L, metrics.getTotalEstimatedCost()); + assertEquals(5000L, metrics.getLatency()); + assertTrue(metrics.isStatus()); + } + + @Test + void testExtractMetricsWithMissingModelName() { + SpanObject span = SpanObject.newBuilder() + .setOperationName("genai_call") + .setStartTime(1000000L) + .setEndTime(1005000L) + .build(); + + SegmentObject segment = SegmentObject.newBuilder().build(); + GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); + + assertNull(metrics); + } + + @Test + void testExtractMetricsWithoutProviderName() { + SpanObject span = SpanObject.newBuilder() + .setOperationName("genai_call") + .setStartTime(1000000L) + .setEndTime(1005000L) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.RESPONSE_MODEL) + .setValue("deepseek-chat") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.INPUT_TOKENS) + .setValue("2000") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.OUTPUT_TOKENS) + .setValue("1000") + .build()) + .build(); + + GenAIConfig.Model modelConfig = new GenAIConfig.Model(); + modelConfig.setName("deepseek-chat"); + modelConfig.setInputEstimatedCostPerM(0.28); + modelConfig.setOutputEstimatedCostPerM(0.42); + + GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("deepseek", modelConfig); + when(matcher.match("deepseek-chat")).thenReturn(matchResult); + + SegmentObject segment = SegmentObject.newBuilder().build(); + GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); + + assertNotNull(metrics); + assertEquals("deepseek", metrics.getProviderName()); + assertEquals("deepseek-chat", metrics.getModelName()); + assertEquals(980L, metrics.getTotalEstimatedCost()); // (2000 * 0.28) + (1000 * 0.42) + } + + @Test + void testExtractMetricsWithNoModelConfig() { + SpanObject span = SpanObject.newBuilder() + .setOperationName("genai_call") + .setStartTime(1000000L) + .setEndTime(1005000L) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.RESPONSE_MODEL) + .setValue("unknown-model") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.INPUT_TOKENS) + .setValue("1000") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.OUTPUT_TOKENS) + .setValue("500") + .build()) + .build(); + + GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("unknown", null); + when(matcher.match("unknown-model")).thenReturn(matchResult); + + SegmentObject segment = SegmentObject.newBuilder().build(); + GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); + + assertNotNull(metrics); + assertEquals(0L, metrics.getTotalEstimatedCost()); + } + + @Test + void testExtractMetricsWithInvalidTokenValues() { + SpanObject span = SpanObject.newBuilder() + .setOperationName("genai_call") + .setStartTime(1000000L) + .setEndTime(1005000L) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.RESPONSE_MODEL) + .setValue("gpt-4o") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.INPUT_TOKENS) + .setValue("invalid") + .build()) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.OUTPUT_TOKENS) + .setValue("not-a-number") + .build()) + .build(); + + GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("openai", null); + when(matcher.match("gpt-4o")).thenReturn(matchResult); + + SegmentObject segment = SegmentObject.newBuilder().build(); + GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); + + assertNotNull(metrics); + assertEquals(0L, metrics.getInputTokens()); + assertEquals(0L, metrics.getOutputTokens()); + } + + @Test + void testExtractMetricsWithErrorSpan() { + SpanObject span = SpanObject.newBuilder() + .setOperationName("genai_call") + .setStartTime(1000000L) + .setEndTime(1005000L) + .setIsError(true) + .addTags(KeyStringValuePair.newBuilder() + .setKey(GenAITagKeys.RESPONSE_MODEL) + .setValue("claude-4-sonnet") + .build()) + .build(); + + GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("anthropic", null); + when(matcher.match("claude-4-sonnet")).thenReturn(matchResult); + + SegmentObject segment = SegmentObject.newBuilder().build(); + GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); + + assertNotNull(metrics); + assertFalse(metrics.isStatus()); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java index a6e91947f927..2ef4562143bb 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIMetrics.java @@ -33,7 +33,12 @@ public class GenAIMetrics { private long outputTokens; - private double totalCost; + /** + * The total estimated cost of GenAI model calls. + * This value is amplified by 10^6 (multiplied by 1,000,000) to be stored as a long + * and to avoid precision issues with double in SumMetrics. + */ + private long totalEstimatedCost; private int timeToFirstToken; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java index 3636a6dfe058..0fbb752ccb07 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIModelAccess.java @@ -55,7 +55,7 @@ public String getEntityId() { private long outputTokens; - private double totalCost; + private long totalEstimatedCost; private int timeToFirstToken; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java index cd08fb541419..f16634ffe2e2 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/GenAIProviderAccess.java @@ -49,7 +49,7 @@ public String getEntityId() { private long outputTokens; - private double totalCost; + private long totalEstimatedCost; private long latency; diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index 49fcff0030ce..5dd3704af72f 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -239,7 +239,7 @@ providers: - provider: groq prefix-match: - - llama-3 + - llama models: - name: llama-4-scout-17bx16e-128k input-estimated-cost-per-m: 0.11 @@ -300,12 +300,15 @@ providers: - qwen models: - name: qwen3-max + # Lowest output price input-estimated-cost-per-m: 0.36 output-estimated-cost-per-m: 1.45 - name: qwen3.5-plus + # Lowest output price input-estimated-cost-per-m: 0.12 output-estimated-cost-per-m: 0.70 - name: qwen3.5-flash + # Lowest output price input-estimated-cost-per-m: 0.03 output-estimated-cost-per-m: 0.29 @@ -322,9 +325,6 @@ providers: # For Input (0, 32k] input-estimated-cost-per-m: 0.46 output-estimated-cost-per-m: 1.15 - - name: hunyuan-turbos - input-estimated-cost-per-m: 0.12 - output-estimated-cost-per-m: 0.29 - name: hunyuan-t1 input-estimated-cost-per-m: 0.15 output-estimated-cost-per-m: 0.58 @@ -371,3 +371,61 @@ providers: - provider: ollama prefix-match: + + - provider: azure_openai + prefix-match: + - GPT + models: + - name: GPT-5.2 + # GPT-5.2 Global + input-estimated-cost-per-m: 1.75 + output-estimated-cost-per-m: 14.0 + - name: GPT-5.2-chat + input-estimated-cost-per-m: 1.75 + output-estimated-cost-per-m: 14.0 + + # --- GPT-5.1 Series --- + - name: GPT-5.1 + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: GPT-5.1-chat + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: GPT-5.1-codex-Global + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: GPT-5.1-codex-max-Global + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: GPT-5.1-codex-mini-Global + input-estimated-cost-per-m: 0.25 + output-estimated-cost-per-m: 2.0 + + # --- GPT-5 Series --- + - name: GPT-5-2025-08-07-Global + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: GPT-5-Data-Zone + input-estimated-cost-per-m: 1.38 + output-estimated-cost-per-m: 11.0 + - name: GPT-5-Pro-Global + input-estimated-cost-per-m: 15.0 + output-estimated-cost-per-m: 120.0 + - name: GPT-5-Codex-Global + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 + - name: GPT-5-mini-Global + input-estimated-cost-per-m: 0.25 + output-estimated-cost-per-m: 2.0 + - name: GPT-5-mini-Data-Zone + input-estimated-cost-per-m: 0.28 + output-estimated-cost-per-m: 2.20 + - name: GPT-5-nano-Global + input-estimated-cost-per-m: 0.05 + output-estimated-cost-per-m: 0.40 + - name: GPT-5-nano-Data-Zone + input-estimated-cost-per-m: 0.06 + output-estimated-cost-per-m: 0.44 + - name: GPT-5-chat-Global + input-estimated-cost-per-m: 1.25 + output-estimated-cost-per-m: 10.0 From 1f5d5ac79f564efd5c4c3f7a38592df2731c0a13 Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Tue, 24 Mar 2026 19:56:18 +0800 Subject: [PATCH 09/11] fix --- .../config}/GenAIMeterAnalyzerTest.java | 79 +++++++++++-------- 1 file changed, 45 insertions(+), 34 deletions(-) rename oap-server/{analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai => server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config}/GenAIMeterAnalyzerTest.java (80%) diff --git a/oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java similarity index 80% rename from oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java rename to oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java index f8e069784697..dd838cf33519 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/test/java/org/apache/skywalking/oap/analyzer/genai/GenAIMeterAnalyzerTest.java +++ b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java @@ -16,16 +16,18 @@ * */ -package org.apache.skywalking.oap.analyzer.genai; +package org.apache.skywalking.oap.server.starter.config; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfig; +import org.apache.skywalking.oap.analyzer.genai.config.GenAIConfigLoader; import org.apache.skywalking.oap.analyzer.genai.config.GenAITagKeys; import org.apache.skywalking.oap.analyzer.genai.matcher.GenAIProviderPrefixMatcher; import org.apache.skywalking.oap.analyzer.genai.service.GenAIMeterAnalyzer; import org.apache.skywalking.oap.server.core.source.GenAIMetrics; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -34,19 +36,36 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class GenAIMeterAnalyzerTest { + private GenAIConfig loadedConfig; private GenAIProviderPrefixMatcher matcher; private GenAIMeterAnalyzer analyzer; @BeforeEach - void setUp() { - matcher = mock(GenAIProviderPrefixMatcher.class); + void setUp() throws ModuleStartException { + GenAIConfig config = new GenAIConfig(); + GenAIConfigLoader loader = new GenAIConfigLoader(config); + loadedConfig = loader.loadConfig(); + + matcher = GenAIProviderPrefixMatcher.build(loadedConfig); analyzer = new GenAIMeterAnalyzer(matcher); } + @Test + void testLoadConfig() { + assertNotNull(loadedConfig); + assertFalse(loadedConfig.getProviders().isEmpty(), "Providers list should not be empty after loading config"); + } + + @Test + void testProviderMatching() { + assertEquals("openai", matcher.match("gpt-5.4-pro").getProvider()); + assertEquals("deepseek", matcher.match("deepseek-chat").getProvider()); + assertEquals("anthropic", matcher.match("claude-4.6-opus").getProvider()); + assertEquals("gemini", matcher.match("gemini-3.1-pro-preview").getProvider()); + } + @Test void testExtractMetricsWithValidSpan() { SpanObject span = SpanObject.newBuilder() @@ -56,7 +75,7 @@ void testExtractMetricsWithValidSpan() { .setIsError(false) .addTags(KeyStringValuePair.newBuilder() .setKey(GenAITagKeys.RESPONSE_MODEL) - .setValue("gpt-4o") + .setValue("gpt-5.4") .build()) .addTags(KeyStringValuePair.newBuilder() .setKey(GenAITagKeys.PROVIDER_NAME) @@ -76,23 +95,18 @@ void testExtractMetricsWithValidSpan() { .build()) .build(); - GenAIConfig.Model modelConfig = new GenAIConfig.Model(); - modelConfig.setName("gpt-4o"); - modelConfig.setInputEstimatedCostPerM(2.5); - modelConfig.setOutputEstimatedCostPerM(10.0); - - GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("openai", modelConfig); - when(matcher.match("gpt-4o")).thenReturn(matchResult); SegmentObject segment = SegmentObject.newBuilder().build(); GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); assertNotNull(metrics); assertEquals("openai", metrics.getProviderName()); - assertEquals("gpt-4o", metrics.getModelName()); + assertEquals("gpt-5.4", metrics.getModelName()); assertEquals(1000L, metrics.getInputTokens()); assertEquals(500L, metrics.getOutputTokens()); assertEquals(100, metrics.getTimeToFirstToken()); - assertEquals(7500L, metrics.getTotalEstimatedCost()); + + + assertEquals(10000L, metrics.getTotalEstimatedCost()); assertEquals(5000L, metrics.getLatency()); assertTrue(metrics.isStatus()); } @@ -131,21 +145,13 @@ void testExtractMetricsWithoutProviderName() { .build()) .build(); - GenAIConfig.Model modelConfig = new GenAIConfig.Model(); - modelConfig.setName("deepseek-chat"); - modelConfig.setInputEstimatedCostPerM(0.28); - modelConfig.setOutputEstimatedCostPerM(0.42); - - GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("deepseek", modelConfig); - when(matcher.match("deepseek-chat")).thenReturn(matchResult); - SegmentObject segment = SegmentObject.newBuilder().build(); GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); assertNotNull(metrics); assertEquals("deepseek", metrics.getProviderName()); assertEquals("deepseek-chat", metrics.getModelName()); - assertEquals(980L, metrics.getTotalEstimatedCost()); // (2000 * 0.28) + (1000 * 0.42) + assertEquals(980L, metrics.getTotalEstimatedCost()); // 基于 0.28 和 0.42 预估价 } @Test @@ -168,9 +174,6 @@ void testExtractMetricsWithNoModelConfig() { .build()) .build(); - GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("unknown", null); - when(matcher.match("unknown-model")).thenReturn(matchResult); - SegmentObject segment = SegmentObject.newBuilder().build(); GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); @@ -186,7 +189,7 @@ void testExtractMetricsWithInvalidTokenValues() { .setEndTime(1005000L) .addTags(KeyStringValuePair.newBuilder() .setKey(GenAITagKeys.RESPONSE_MODEL) - .setValue("gpt-4o") + .setValue("gpt-5.4") .build()) .addTags(KeyStringValuePair.newBuilder() .setKey(GenAITagKeys.INPUT_TOKENS) @@ -198,9 +201,6 @@ void testExtractMetricsWithInvalidTokenValues() { .build()) .build(); - GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("openai", null); - when(matcher.match("gpt-4o")).thenReturn(matchResult); - SegmentObject segment = SegmentObject.newBuilder().build(); GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); @@ -222,13 +222,24 @@ void testExtractMetricsWithErrorSpan() { .build()) .build(); - GenAIProviderPrefixMatcher.MatchResult matchResult = new GenAIProviderPrefixMatcher.MatchResult("anthropic", null); - when(matcher.match("claude-4-sonnet")).thenReturn(matchResult); - SegmentObject segment = SegmentObject.newBuilder().build(); GenAIMetrics metrics = analyzer.extractMetricsFromSWSpan(span, segment); assertNotNull(metrics); assertFalse(metrics.isStatus()); } + + @Test + void testEstimatedCost() throws ModuleStartException { + GenAIConfig config = new GenAIConfig(); + GenAIConfigLoader loader = new GenAIConfigLoader(config); + GenAIConfig loadedConfig = loader.loadConfig(); + + GenAIProviderPrefixMatcher matcher = GenAIProviderPrefixMatcher.build(loadedConfig); + + GenAIProviderPrefixMatcher.MatchResult result = matcher.match("gpt-5.4-pro"); + assertNotNull(result.getModelConfig()); + assertEquals(30.0, result.getModelConfig().getInputEstimatedCostPerM(), 0.001); + assertEquals(180.0, result.getModelConfig().getOutputEstimatedCostPerM(), 0.001); + } } From 227fef455d5cab56e6d7370d8d4f91e986cb870f Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Tue, 24 Mar 2026 20:01:46 +0800 Subject: [PATCH 10/11] fix --- .../src/main/resources/gen-ai-config.yml | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index 5dd3704af72f..2eaf16b6acb4 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -371,61 +371,3 @@ providers: - provider: ollama prefix-match: - - - provider: azure_openai - prefix-match: - - GPT - models: - - name: GPT-5.2 - # GPT-5.2 Global - input-estimated-cost-per-m: 1.75 - output-estimated-cost-per-m: 14.0 - - name: GPT-5.2-chat - input-estimated-cost-per-m: 1.75 - output-estimated-cost-per-m: 14.0 - - # --- GPT-5.1 Series --- - - name: GPT-5.1 - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 - - name: GPT-5.1-chat - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 - - name: GPT-5.1-codex-Global - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 - - name: GPT-5.1-codex-max-Global - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 - - name: GPT-5.1-codex-mini-Global - input-estimated-cost-per-m: 0.25 - output-estimated-cost-per-m: 2.0 - - # --- GPT-5 Series --- - - name: GPT-5-2025-08-07-Global - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 - - name: GPT-5-Data-Zone - input-estimated-cost-per-m: 1.38 - output-estimated-cost-per-m: 11.0 - - name: GPT-5-Pro-Global - input-estimated-cost-per-m: 15.0 - output-estimated-cost-per-m: 120.0 - - name: GPT-5-Codex-Global - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 - - name: GPT-5-mini-Global - input-estimated-cost-per-m: 0.25 - output-estimated-cost-per-m: 2.0 - - name: GPT-5-mini-Data-Zone - input-estimated-cost-per-m: 0.28 - output-estimated-cost-per-m: 2.20 - - name: GPT-5-nano-Global - input-estimated-cost-per-m: 0.05 - output-estimated-cost-per-m: 0.40 - - name: GPT-5-nano-Data-Zone - input-estimated-cost-per-m: 0.06 - output-estimated-cost-per-m: 0.44 - - name: GPT-5-chat-Global - input-estimated-cost-per-m: 1.25 - output-estimated-cost-per-m: 10.0 From a6208893ea4709e993cb3d81c0330e0f11a1c2de Mon Sep 17 00:00:00 2001 From: peachisai <2581009893@qq.com> Date: Tue, 24 Mar 2026 21:07:32 +0800 Subject: [PATCH 11/11] fix --- .../parser/listener/vservice/VirtualGenAIProcessor.java | 4 ++-- .../oap/analyzer/genai/service/GenAIMeterAnalyzer.java | 8 ++++---- .../server-starter/src/main/resources/gen-ai-config.yml | 2 ++ .../src/main/resources/oal/virtual-gen-ai.oal | 8 ++++---- .../oap/server/starter/config/GenAIMeterAnalyzerTest.java | 4 +--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java index 0aa1cdbc3d1b..246e1e41c0ec 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualGenAIProcessor.java @@ -63,7 +63,7 @@ public void prepareVSIfNecessary(SpanObject span, SegmentObject segmentObject) { private ServiceMeta toServiceMeta(GenAIMetrics metrics) { ServiceMeta service = new ServiceMeta(); - service.setName(metrics.getProviderName()); + service.setName(namingControl.formatServiceName(metrics.getProviderName())); service.setLayer(Layer.VIRTUAL_GENAI); service.setTimeBucket(metrics.getTimeBucket()); return service; @@ -72,7 +72,7 @@ private ServiceMeta toServiceMeta(GenAIMetrics metrics) { private Source toInstance(GenAIMetrics metrics) { ServiceInstance instance = new ServiceInstance(); instance.setTimeBucket(metrics.getTimeBucket()); - instance.setName(metrics.getModelName()); + instance.setName(namingControl.formatInstanceName(metrics.getModelName())); instance.setServiceLayer(Layer.VIRTUAL_GENAI); instance.setServiceName(metrics.getProviderName()); return instance; diff --git a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java index 15f5bc9f5845..41b13dd0652f 100644 --- a/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java +++ b/oap-server/analyzer/gen-ai-analyzer/src/main/java/org/apache/skywalking/oap/analyzer/genai/service/GenAIMeterAnalyzer.java @@ -73,13 +73,13 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm long outputTokens = parseSafeLong(tags.get(GenAITagKeys.OUTPUT_TOKENS)); // calculate the total cost by the cost configs - long totalCost = 0L; + double totalCost = 0.0D; if (modelConfig != null) { if (modelConfig.getInputEstimatedCostPerM() > 0) { - totalCost += (long) (inputTokens * modelConfig.getInputEstimatedCostPerM()); + totalCost += inputTokens * modelConfig.getInputEstimatedCostPerM(); } if (modelConfig.getOutputEstimatedCostPerM() > 0) { - totalCost += (long) (outputTokens * modelConfig.getOutputEstimatedCostPerM()); + totalCost += outputTokens * modelConfig.getOutputEstimatedCostPerM(); } } @@ -92,7 +92,7 @@ public GenAIMetrics extractMetricsFromSWSpan(SpanObject span, SegmentObject segm metrics.setOutputTokens(outputTokens); metrics.setTimeToFirstToken(parseSafeInt(tags.get(GenAITagKeys.SERVER_TIME_TO_FIRST_TOKEN))); - metrics.setTotalEstimatedCost(totalCost); + metrics.setTotalEstimatedCost(Math.round(totalCost)); long latency = span.getEndTime() - span.getStartTime(); metrics.setLatency(latency); diff --git a/oap-server/server-starter/src/main/resources/gen-ai-config.yml b/oap-server/server-starter/src/main/resources/gen-ai-config.yml index 2eaf16b6acb4..b3b82c71f4c2 100644 --- a/oap-server/server-starter/src/main/resources/gen-ai-config.yml +++ b/oap-server/server-starter/src/main/resources/gen-ai-config.yml @@ -370,4 +370,6 @@ providers: output-estimated-cost-per-m: 1.2 - provider: ollama + # OLLAMA (Local Models): This section serves as a template for self-hosted models. + # Users can define their own prefix-match and specific models here. prefix-match: diff --git a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal index f0bc47e91c87..6a7b9a1d9a05 100644 --- a/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal +++ b/oap-server/server-starter/src/main/resources/oal/virtual-gen-ai.oal @@ -26,8 +26,8 @@ gen_ai_provider_input_tokens_avg = from(GenAIProviderAccess.inputTokens).longAvg gen_ai_provider_output_tokens_sum = from(GenAIProviderAccess.outputTokens).sum(); gen_ai_provider_output_tokens_avg = from(GenAIProviderAccess.outputTokens).longAvg(); -gen_ai_provider_total_estimated_cost = from(GenAIProviderAccess.totalCost).sum(); -gen_ai_provider_avg_estimated_cost = from(GenAIProviderAccess.totalCost).doubleAvg(); +gen_ai_provider_total_estimated_cost = from(GenAIProviderAccess.totalEstimatedCost).sum(); +gen_ai_provider_avg_estimated_cost = from(GenAIProviderAccess.totalEstimatedCost).doubleAvg(); gen_ai_model_call_cpm = from(GenAIModelAccess.*).cpm(); gen_ai_model_sla = from(GenAIModelAccess.*).percent(status == true); @@ -42,5 +42,5 @@ gen_ai_model_input_tokens_avg = from(GenAIModelAccess.inputTokens).longAvg(); gen_ai_model_output_tokens_sum = from(GenAIModelAccess.outputTokens).sum(); gen_ai_model_output_tokens_avg = from(GenAIModelAccess.outputTokens).longAvg(); -gen_ai_model_total_estimated_cost = from(GenAIModelAccess.totalCost).sum(); -gen_ai_model_avg_estimated_cost = from(GenAIModelAccess.totalCost).doubleAvg(); \ No newline at end of file +gen_ai_model_total_estimated_cost = from(GenAIModelAccess.totalEstimatedCost).sum(); +gen_ai_model_avg_estimated_cost = from(GenAIModelAccess.totalEstimatedCost).doubleAvg(); \ No newline at end of file diff --git a/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java index dd838cf33519..b0ebe05939aa 100644 --- a/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java +++ b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/GenAIMeterAnalyzerTest.java @@ -104,8 +104,6 @@ void testExtractMetricsWithValidSpan() { assertEquals(1000L, metrics.getInputTokens()); assertEquals(500L, metrics.getOutputTokens()); assertEquals(100, metrics.getTimeToFirstToken()); - - assertEquals(10000L, metrics.getTotalEstimatedCost()); assertEquals(5000L, metrics.getLatency()); assertTrue(metrics.isStatus()); @@ -151,7 +149,7 @@ void testExtractMetricsWithoutProviderName() { assertNotNull(metrics); assertEquals("deepseek", metrics.getProviderName()); assertEquals("deepseek-chat", metrics.getModelName()); - assertEquals(980L, metrics.getTotalEstimatedCost()); // 基于 0.28 和 0.42 预估价 + assertEquals(980L, metrics.getTotalEstimatedCost()); } @Test