diff --git a/agent/agent-tooling/build.gradle.kts b/agent/agent-tooling/build.gradle.kts index 644f01e2308..d39042d3048 100644 --- a/agent/agent-tooling/build.gradle.kts +++ b/agent/agent-tooling/build.gradle.kts @@ -11,6 +11,7 @@ dependencies { annotationProcessor("com.google.auto.value:auto-value") implementation("io.opentelemetry.contrib:opentelemetry-jfr-connection") + compileOnly("org.gradle.jfr.polyfill:jfr-polyfill:1.0.2") implementation("com.azure:azure-storage-blob") implementation(project(":agent:agent-profiler:agent-alerting-api")) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index 26005a97585..531c35c4acd 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -1531,6 +1531,7 @@ public static class ProfilerConfiguration { public boolean enableRequestTriggering = false; public List requestTriggerEndpoints = new ArrayList<>(); @Nullable public String cgroupPath = null; + @Nullable public String localProfilerConfigDir = null; } public static class GcEventConfiguration { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java index 5e877d9f1fb..fdc950e5837 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/Profiler.java @@ -23,6 +23,8 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; @@ -30,6 +32,7 @@ import java.util.function.Consumer; import javax.annotation.Nullable; import javax.management.MBeanServerConnection; +import jdk.jfr.FlightRecorder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +68,13 @@ public class Profiler { private final RecordingConfiguration spanRecordingConfiguration; private final RecordingConfiguration manualRecordingConfiguration; + // Events to force-disable regardless of .jfc configuration + private static final List DISABLED_EVENTS = Arrays.asList( + "jdk.InitialSystemProperty", + "jdk.InitialEnvironmentVariable", + "jdk.InitialSecurityProperty", + "jdk.OldObjectSample"); + private final File temporaryDirectory; public Profiler(Configuration.ProfilerConfiguration config, File tempDir) { @@ -194,6 +204,7 @@ private void executeProfile( try { newRecording.start(); + disableEvents(newRecording.getId()); // schedule closing the recording scheduledExecutorService.schedule( @@ -210,6 +221,24 @@ private void executeProfile( } } + /** + * Disable configured events on the recording using the direct JFR API. This acts as a safety net + * to override .jfc settings that may have been misconfigured. + */ + @SuppressWarnings("Java8ApiChecker") + private static void disableEvents(long recordingId) { + FlightRecorder.getFlightRecorder().getRecordings().stream() + .filter(r -> r.getId() == recordingId) + .findFirst() + .ifPresent( + recording -> { + for (String event : DISABLED_EVENTS) { + recording.disable(event); + logger.debug("Disabled JFR event: {}", event); + } + }); + } + /** When a profile has been created, upload it to service profiler. */ @SuppressWarnings( "CatchingUnchecked") // catching unchecked exception is necessary for proper error handling diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java index b6fac1be160..a0b2dc4b55a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilingInitializer.java @@ -15,6 +15,7 @@ import com.microsoft.applicationinsights.agent.internal.diagnostics.SdkVersionFinder; import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; import com.microsoft.applicationinsights.agent.internal.profiler.config.ConfigService; +import com.microsoft.applicationinsights.agent.internal.profiler.config.LocalProfilerConfigService; import com.microsoft.applicationinsights.agent.internal.profiler.config.ProfilerConfiguration; import com.microsoft.applicationinsights.agent.internal.profiler.service.ServiceProfilerClient; import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertConfigParser; @@ -29,6 +30,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +61,7 @@ public class ProfilingInitializer { private HttpPipeline httpPipeline; private ScheduledExecutorService serviceProfilerExecutorService; private ServiceProfilerClient serviceProfilerClient; + @Nullable private LocalProfilerConfigService localConfigService; ////////////////////////////////////////////////////////// private PerformanceMonitoringService performanceMonitoringService; @@ -142,6 +145,14 @@ private synchronized void performInit() { httpPipeline, userAgent); + if (configuration.localProfilerConfigDir != null) { + File localConfigDir = new File(configuration.localProfilerConfigDir); + localConfigService = new LocalProfilerConfigService(localConfigDir); + logger.info( + "Local profiler configuration directory configured: {}", + localConfigDir.getAbsolutePath()); + } + // Monitor service remains alive permanently due to scheduling an periodic config pull startPollingForConfigUpdates(); } @@ -157,12 +168,27 @@ private void startPollingForConfigUpdates() { private void pullProfilerSettings(ConfigService configService) { try { + // Local config takes precedence over remote when the local file is present + if (localConfigService != null && localConfigService.isLocalConfigPresent()) { + localConfigService + .pullSettings() + .subscribe(this::applyConfiguration, ProfilingInitializer::logLocalConfigError); + return; + } + configService.pullSettings().subscribe(this::applyConfiguration, this::logProfilerPullError); } catch (Throwable t) { logProfilerPullError(t); } } + private static void logLocalConfigError(Throwable e) { + logger.error( + "Error reading local profiler configuration. " + + "Fix or remove the file to restore normal operation.", + e); + } + private void logProfilerPullError(Throwable e) { if (currentlyEnabled.get()) { logger.error("Error pulling service profiler settings", e); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/config/LocalProfilerConfigService.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/config/LocalProfilerConfigService.java new file mode 100644 index 00000000000..7aeef909d31 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/config/LocalProfilerConfigService.java @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler.config; + +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Date; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; + +/** + * Checks a local directory for a profiler configuration file. If a valid file exists, it takes + * precedence over the remote Azure service profiler configuration. + * + *

The configuration file must be named {@code profiler-config.json} and use the same JSON format + * as {@link ProfilerConfiguration}. + */ +public class LocalProfilerConfigService { + + private static final Logger logger = LoggerFactory.getLogger(LocalProfilerConfigService.class); + + static final String CONFIG_FILE_NAME = "profiler-config.json"; + + private final File configFile; + private volatile long lastModifiedTime; + + public LocalProfilerConfigService(File configDir) { + this.configFile = new File(configDir, CONFIG_FILE_NAME); + this.lastModifiedTime = 0; + } + + /** + * Checks for a local profiler configuration file. Returns a configuration if the file exists and + * has been modified since the last check. Returns empty if no file exists or if it has not + * changed. + * + * @return Mono containing the configuration if changed, or Mono.error if the file is malformed + */ + public Mono pullSettings() { + if (!configFile.exists()) { + return Mono.empty(); + } + + long fileLastModified = configFile.lastModified(); + if (fileLastModified == lastModifiedTime) { + // File has not changed since last successful read + return Mono.empty(); + } + + try { + ProfilerConfiguration config = readConfigFile(); + logger.info( + "Successfully read local profiler configuration from: {}", configFile.getAbsolutePath()); + + // Delete the file after successful read - this is a one-shot configuration mechanism + if (!configFile.delete()) { + logger.warn( + "Failed to delete local profiler configuration file after reading: {}", + configFile.getAbsolutePath()); + } else { + logger.info( + "Deleted local profiler configuration file after successful read: {}", + configFile.getAbsolutePath()); + } + + lastModifiedTime = fileLastModified; + return Mono.just(config); + } catch (Exception e) { + logger.error( + "Failed to parse local profiler configuration file: {}. " + + "Fix or remove the file to restore normal operation.", + configFile.getAbsolutePath(), + e); + return Mono.error(e); + } + } + + /** + * Returns true if a local config file is present (regardless of whether it has changed), meaning + * local override mode is active. + */ + public boolean isLocalConfigPresent() { + return configFile.exists(); + } + + private ProfilerConfiguration readConfigFile() throws IOException { + try (Reader reader = + new InputStreamReader(new FileInputStream(configFile), StandardCharsets.UTF_8); + JsonReader jsonReader = JsonProviders.createReader(reader)) { + + ProfilerConfiguration config = ProfilerConfiguration.fromJson(jsonReader); + + // Default lastModified from file timestamp if not provided + if (config.getLastModified() == null + || config.getLastModified().compareTo(ProfilerConfiguration.DEFAULT_DATE) == 0) { + config.setLastModified(new Date(configFile.lastModified())); + } + + // Default enabledLastModified if not set + if (config.getEnabledLastModified() == null) { + config.setEnabledLastModified(config.getLastModified()); + } + + // Default requestTriggerConfiguration to empty list if null + if (config.getRequestTriggerConfiguration() == null) { + config.setRequestTriggerConfiguration(new ArrayList<>()); + } + + return config; + } + } + + // visible for testing + @Nullable + File getConfigFile() { + return configFile; + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/config/LocalProfilerConfigServiceTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/config/LocalProfilerConfigServiceTest.java new file mode 100644 index 00000000000..ea66295481d --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/profiler/config/LocalProfilerConfigServiceTest.java @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.profiler.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import reactor.core.publisher.Mono; + +class LocalProfilerConfigServiceTest { + + @TempDir File tempDir; + + @Test + void returnsEmptyWhenNoConfigFileExists() { + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + Mono result = service.pullSettings(); + + assertThat(result.blockOptional()).isEmpty(); + assertThat(service.isLocalConfigPresent()).isFalse(); + } + + @Test + void returnsConfigWhenValidFileExists() throws IOException { + writeConfigFile(validConfigJson()); + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + ProfilerConfiguration config = service.pullSettings().block(); + + assertThat(config).isNotNull(); + assertThat(config.isEnabled()).isTrue(); + assertThat(config.getCpuTriggerConfiguration()).contains("--cpu-threshold 80"); + // File should be deleted after successful read + assertThat(service.isLocalConfigPresent()).isFalse(); + } + + @Test + void returnsEmptyOnSecondCallBecauseFileDeletedAfterRead() throws IOException { + writeConfigFile(validConfigJson()); + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + // First call returns config and deletes file + assertThat(service.pullSettings().blockOptional()).isPresent(); + assertThat(service.isLocalConfigPresent()).isFalse(); + + // Second call returns empty (file was deleted) + assertThat(service.pullSettings().blockOptional()).isEmpty(); + } + + @Test + void returnsConfigWhenNewFileDroppedAfterPreviousConsumed() throws IOException { + writeConfigFile(validConfigJson()); + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + // First call consumes and deletes the file + ProfilerConfiguration config1 = service.pullSettings().block(); + assertThat(config1).isNotNull(); + assertThat(config1.getCpuTriggerConfiguration()).contains("--cpu-threshold 80"); + assertThat(service.isLocalConfigPresent()).isFalse(); + + // Drop a new file with different config + writeConfigFile(validConfigJson().replace("80", "90")); + + // New file is read and deleted + ProfilerConfiguration config2 = service.pullSettings().block(); + assertThat(config2).isNotNull(); + assertThat(config2.getCpuTriggerConfiguration()).contains("--cpu-threshold 90"); + assertThat(service.isLocalConfigPresent()).isFalse(); + } + + @Test + void returnsErrorWhenFileMalformed() throws IOException { + writeConfigFile("this is not json"); + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + Mono result = service.pullSettings(); + + // Malformed file produces an error signal + assertThatThrownBy(result::block).isNotNull(); + } + + @Test + void defaultsLastModifiedFromFileTimestamp() throws IOException { + // Config without lastModified field + String json = + "{" + + "\"enabled\": true," + + "\"cpuTriggerConfiguration\": \"--cpu-threshold 80 --cpu-trigger-enabled true\"," + + "\"memoryTriggerConfiguration\": \"--memory-threshold 80 --memory-trigger-enabled true\"," + + "\"collectionPlan\": \"\"," + + "\"requestTriggerConfiguration\": []" + + "}"; + writeConfigFile(json); + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + ProfilerConfiguration config = service.pullSettings().block(); + + assertThat(config).isNotNull(); + assertThat(config.getLastModified()).isNotNull(); + assertThat(config.getLastModified().getTime()).isGreaterThan(0); + // File should be deleted after successful read + assertThat(service.isLocalConfigPresent()).isFalse(); + } + + @Test + void doesNotDeleteFileWhenParsingFails() throws IOException { + writeConfigFile("this is not valid json"); + LocalProfilerConfigService service = new LocalProfilerConfigService(tempDir); + + // Parsing fails + Mono result = service.pullSettings(); + assertThatThrownBy(result::block).isNotNull(); + + // File should still exist so user can fix it + assertThat(service.isLocalConfigPresent()).isTrue(); + } + + private void writeConfigFile(String content) throws IOException { + File configFile = new File(tempDir, LocalProfilerConfigService.CONFIG_FILE_NAME); + try (Writer writer = + new OutputStreamWriter(new FileOutputStream(configFile), StandardCharsets.UTF_8)) { + writer.write(content); + } + } + + private static String validConfigJson() { + return "{" + + "\"id\": \"local-config\"," + + "\"lastModified\": \"2024-01-01T00:00:00+00:00\"," + + "\"enabledLastModified\": \"2024-01-01T00:00:00+00:00\"," + + "\"enabled\": true," + + "\"collectionPlan\": \"--single --mode immediate --immediate-profiling-duration 120 --expiration 5249691022697135638 --settings-moniker local-test\"," + + "\"cpuTriggerConfiguration\": \"--cpu-threshold 80 --cpu-trigger-profilingDuration 120 --cpu-trigger-cooldown 14400 --cpu-trigger-enabled true\"," + + "\"memoryTriggerConfiguration\": \"--memory-threshold 80 --memory-trigger-profilingDuration 120 --memory-trigger-cooldown 14400 --memory-trigger-enabled true\"," + + "\"defaultConfiguration\": null," + + "\"requestTriggerConfiguration\": []" + + "}"; + } +}