Support for OpenTelemetry tracing instrumentation

agent/conf/log4j-cloud.xml.in

@@ -30,7 +30,7 @@ under the License.
          <Policies>
             <TimeBasedTriggeringPolicy/>
          </Policies>
-         <PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
+         <PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) (traceid:%X{traceid}) %m%ex%n"/>
       </RollingFile>
 
       <!-- ============================== -->
@@ -39,7 +39,7 @@ under the License.
 
       <Console name="CONSOLE" target="SYSTEM_OUT">
           <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
-          <PatternLayout pattern="%-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
+          <PatternLayout pattern="%-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) (traceid:%X{traceid}) %m%ex%n"/>
       </Console>
    </Appenders>
 

api/pom.xml

@@ -71,6 +71,15 @@
             <artifactId>cloud-framework-direct-download</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry.instrumentation</groupId>
+            <artifactId>opentelemetry-instrumentation-annotations</artifactId>
+            <version>${cs.opentelemetry-instrumentation.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.api.filter;
+
+import org.apache.cloudstack.context.LogContext;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.util.UUID;
+import javax.servlet.FilterConfig;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+
+public class ApiTraceFilter implements Filter {
+
+    // Cap the accepted trace id length to avoid log/DB bloat from a crafted header.
+    private static final int MAX_TRACE_ID_LENGTH = 128;
+
+    @Override
+    public void init(FilterConfig filterConfig) throws ServletException {
+    }
+
+    @Override
+    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+            throws IOException, ServletException {
+        try {
+            HttpServletRequest httpReq = (HttpServletRequest) request;
+            String traceId = sanitizeTraceId(httpReq.getHeader(LogContext.TRACEID_KEY));
+            if (StringUtils.isBlank(traceId)) {
+                traceId = UUID.randomUUID().toString();
+            }
+
+            LogContext.current().putContextParameter(LogContext.TRACEID_KEY, traceId);
+            chain.doFilter(request, response);
+        } finally {
+            LogContext.current().removeContextParameter(LogContext.TRACEID_KEY);
+        }
+    }
+
+    /**
+     * Returns the caller-supplied trace id only if it is safe to log and store: no control
+     * characters (prevents log forging) and within a bounded length. Otherwise returns null so a
+     * fresh id is generated.
+     */
+    private String sanitizeTraceId(String traceId) {
+        if (traceId == null) {
+            return null;
+        }
+        String trimmed = traceId.trim();
+        if (trimmed.isEmpty() || trimmed.length() > MAX_TRACE_ID_LENGTH) {
+            return null;
+        }
+        for (int i = 0; i < trimmed.length(); i++) {
+            if (Character.isISOControl(trimmed.charAt(i))) {
+                return null;
+            }
+        }
+        return trimmed;
+    }
+
+    @Override
+    public void destroy() {
+    }
+}

api/src/main/java/org/apache/cloudstack/context/LogContext.java

@@ -16,10 +16,16 @@
 // under the License.
 package org.apache.cloudstack.context;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 import java.util.UUID;
 
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
@@ -53,6 +59,44 @@ public class LogContext {
     private long userId;
     private final Map<String, String> context = new HashMap<String, String>();
 
+    public final static String TRACEID_KEY = "traceid";
+
+    /**
+     * MDC keys under which the active OpenTelemetry ids are published. The names are
+     * deployment specific, so they are read from server.properties and fall back to a
+     * neutral default when the property is absent or blank.
+     */
+    public final static String TRACE_ID_KEY_PROPERTY = "otel.trace.id.mdc.key";
+    public final static String SPAN_ID_KEY_PROPERTY = "otel.span.id.mdc.key";
+
+    public final static String DEFAULT_TRACE_ID_KEY = "otel_trace_id";
+    public final static String DEFAULT_SPAN_ID_KEY = "otel_span_id";
+
+    private final static Properties SERVER_PROPERTIES = loadServerProperties();
+
+    public final static String TRACE_ID_KEY = traceKeyFromProperties();
+    public final static String SPAN_ID_KEY = spanKeyFromProperties();
+
+    private static Properties loadServerProperties() {
+        try {
+            File file = PropertiesUtil.findConfigFile("server.properties");
+            return file == null ? new Properties() : PropertiesUtil.loadFromFile(file);
+        } catch (IOException e) {
+            LOGGER.warn("Could not read server.properties, using the default MDC key names", e);
+            return new Properties();
+        }
+    }
+
+    private static String traceKeyFromProperties() {
+        String value = SERVER_PROPERTIES.getProperty(TRACE_ID_KEY_PROPERTY);
+        return StringUtils.isBlank(value) ? DEFAULT_TRACE_ID_KEY : value.trim();
+    }
+
+    private static String spanKeyFromProperties() {
+        String value = SERVER_PROPERTIES.getProperty(SPAN_ID_KEY_PROPERTY);
+        return StringUtils.isBlank(value) ? DEFAULT_SPAN_ID_KEY : value.trim();
+    }
+
     static EntityManager s_entityMgr;
 
     public static void init(EntityManager entityMgr) {
@@ -78,6 +122,25 @@ protected LogContext(User user, Account account, String logContextId) {
 
     public void putContextParameter(String key, String value) {
         context.put(key, value);
+        ThreadContext.put(key, value);
+        if (value == null) {
+            ThreadContext.remove(key);
+        } else {
+            ThreadContext.put(key, value);
+        }
+    }
+
+    public void removeContextParameter(String key) {
+        context.remove(key);
+        ThreadContext.remove(key);
+    }
+
+    public void removeContextParameters() {
+        // Iterate over a copy of the keys: removeContextParameter mutates the context
+        // map, so iterating the live keySet/entrySet would throw ConcurrentModificationException.
+        for (String key : new ArrayList<>(context.keySet())) {
+            removeContextParameter(key);
+        }
     }
 
     public String getContextParameter(String key) {

api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.context;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextStorage;
+import io.opentelemetry.context.Scope;
+import org.apache.logging.log4j.ThreadContext;
+
+/**
+ * Mirrors the active OpenTelemetry span onto the Log4j MDC so management-server log
+ * lines carry trace and span ids on every thread that has an active
+ * span (API requests, agent-command dispatch, async jobs), not just the servlet path.
+ * The MDC key names come from {@link LogContext#TRACE_ID_KEY} and
+ * {@link LogContext#SPAN_ID_KEY}, which are environment driven.
+ *
+ * The OpenTelemetry agent populates the log MDC automatically for Log4j2 and Logback,
+ * but not for Log4j 1.2 (reload4j), which the management server uses. This wrapper
+ * fills that gap by hooking the OpenTelemetry context lifecycle: whenever a span
+ * becomes current on a thread it copies the ids into the MDC, and restores the
+ * previous values when that scope closes. Install once at startup via {@link #register()}.
+ */
+public class TraceContextMdcWrapper implements ContextStorage {
+
+    private final ContextStorage delegate;
+
+    TraceContextMdcWrapper(ContextStorage delegate) {
+        this.delegate = delegate;
+    }
+
+    /**
+     * Install the wrapper. Must be called before the first OpenTelemetry context is
+     * used, i.e. at management-server startup, before the server accepts requests.
+     */
+    public static void register() {
+        ContextStorage.addWrapper(TraceContextMdcWrapper::new);
+    }
+
+    @Override
+    public Scope attach(Context toAttach) {
+        String previousTraceId = ThreadContext.get(LogContext.TRACE_ID_KEY);
+        String previousSpanId = ThreadContext.get(LogContext.SPAN_ID_KEY);
+        SpanContext spanContext = Span.fromContext(toAttach).getSpanContext();
+        if (spanContext.isValid()) {
+            ThreadContext.put(LogContext.TRACE_ID_KEY, spanContext.getTraceId());
+            ThreadContext.put(LogContext.SPAN_ID_KEY, spanContext.getSpanId());
+        } else {
+            ThreadContext.remove(LogContext.TRACE_ID_KEY);
+            ThreadContext.remove(LogContext.SPAN_ID_KEY);
+        }
+        Scope delegateScope = delegate.attach(toAttach);
+        return () -> {
+            delegateScope.close();
+            restore(LogContext.TRACE_ID_KEY, previousTraceId);
+            restore(LogContext.SPAN_ID_KEY, previousSpanId);
+        };
+    }
+
+    private static void restore(String key, String previous) {
+        if (previous != null) {
+            ThreadContext.put(key, previous);
+        } else {
+            ThreadContext.remove(key);
+        }
+    }
+
+    @Override
+    public Context current() {
+        return delegate.current();
+    }
+}

api/src/main/resources/META-INF/cloudstack/api-config/spring-api-config-context.xml

@@ -28,5 +28,6 @@
                       >
 
     <bean id="apiServiceConfiguration" class="org.apache.cloudstack.config.ApiServiceConfiguration" />
+    <bean id="apiTraceFilter" class="org.apache.cloudstack.api.filter.ApiTraceFilter"/>
 
 </beans>

api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.context;
+
+import org.apache.logging.log4j.ThreadContext;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.TraceFlags;
+import io.opentelemetry.api.trace.TraceState;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextStorage;
+import io.opentelemetry.context.Scope;
+
+public class TraceContextMdcWrapperTest {
+
+    private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736";
+    private static final String SPAN_ID = "00f067aa0ba902b7";
+    private static final String OTHER_TRACE_ID = "d75597dcda4f6e7b9c1a2b3c4d5e6f70";
+    private static final String OTHER_SPAN_ID = "aabbccddeeff0011";
+
+    // Minimal delegate so we test the wrapper in isolation, no real context storage.
+    private final ContextStorage noopDelegate = new ContextStorage() {
+        @Override
+        public Scope attach(Context toAttach) {
+            return () -> { };
+        }
+
+        @Override
+        public Context current() {
+            return Context.root();
+        }
+    };
+
+    private final TraceContextMdcWrapper wrapper = new TraceContextMdcWrapper(noopDelegate);
+
+    @After
+    public void tearDown() {
+        ThreadContext.remove(LogContext.TRACE_ID_KEY);
+        ThreadContext.remove(LogContext.SPAN_ID_KEY);
+    }
+
+    private static Context contextWithSpan(String traceId, String spanId) {
+        return Context.root().with(Span.wrap(
+                SpanContext.create(traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault())));
+    }
+
+    @Test
+    public void putsTraceContextOnMdcWhileScopeOpenAndRestoresOnClose() {
+        Scope scope = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID));
+        Assert.assertEquals(TRACE_ID, ThreadContext.get(LogContext.TRACE_ID_KEY));
+        Assert.assertEquals(SPAN_ID, ThreadContext.get(LogContext.SPAN_ID_KEY));
+
+        scope.close();
+        Assert.assertNull(ThreadContext.get(LogContext.TRACE_ID_KEY));
+        Assert.assertNull(ThreadContext.get(LogContext.SPAN_ID_KEY));
+    }
+
+    @Test
+    public void leavesMdcUnsetWhenNoActiveSpan() {
+        Scope scope = wrapper.attach(Context.root());
+        Assert.assertNull(ThreadContext.get(LogContext.TRACE_ID_KEY));
+        Assert.assertNull(ThreadContext.get(LogContext.SPAN_ID_KEY));
+        scope.close();
+    }
+
+    @Test
+    public void restoresOuterSpanWhenNestedScopeCloses() {
+        Scope outer = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID));
+        Scope inner = wrapper.attach(contextWithSpan(OTHER_TRACE_ID, OTHER_SPAN_ID));
+        Assert.assertEquals(OTHER_TRACE_ID, ThreadContext.get(LogContext.TRACE_ID_KEY));
+
+        inner.close();
+        Assert.assertEquals(TRACE_ID, ThreadContext.get(LogContext.TRACE_ID_KEY));
+
+        outer.close();
+        Assert.assertNull(ThreadContext.get(LogContext.TRACE_ID_KEY));
+    }
+}