From 8069c09cf000c48524980832f032668d724259c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 17 Sep 2026 09:39:50 +0200 Subject: [PATCH 01/45] feat: schedule polling and retries on managed executors instead of Timers PollingEventSource and TimerEventSource each created a java.util.Timer, and PerResourcePollingEventSource a ScheduledThreadPoolExecutor of its own, so every polling event source and every controller cost a thread that the operator neither sized nor shut down. They now schedule on executors managed by ExecutorServiceManager, which builds them from ConfigurationService: - getScheduledExecutorService() backs the polling event sources - getRetryAndRescheduleExecutorService() backs the retry and reschedule timer of every controller, kept separate so that a slow poll cannot delay a retry Both are sized by concurrentScheduledTaskThreads() (4 by default) and concurrentRetryAndRescheduleThreads() (2 by default), overridable programmatically or through the josdk.scheduled-tasks.concurrent-threads and josdk.retry-and-reschedule.concurrent-threads keys. Both default to daemon threads, discard the tasks scheduled for later on shutdown so they don't hold up the termination of the operator, and drop cancelled tasks eagerly. The previous Executors.newScheduledThreadPool(0) was effectively single threaded and created non daemon threads. An event source resolves its executor on every start rather than at creation time, since the manager replaces its pools when the operator is restarted, and only shuts one down if it created it itself. That also fixes PerResourcePollingEventSource shutting down a user supplied executor and not registering its tasks again after a restart. PollingEventSource gains a constructor taking an EventSourceContext, the one without it is deprecated, and both polling configurations accept an executor to run a single event source on a pool of its own. --- .../content/en/docs/documentation/eventing.md | 22 +++ .../documentation/operations/configuration.md | 7 + .../api/config/ConfigurationService.java | 80 +++++++++++ .../config/ConfigurationServiceOverrider.java | 104 ++++++++++++++ .../api/config/ExecutorServiceManager.java | 41 ++++-- .../operator/api/config/Utils.java | 48 +++++++ .../external/PollingDependentResource.java | 1 + .../processing/event/EventSourceManager.java | 4 +- .../processing/event/EventSources.java | 13 +- .../PerResourcePollingConfiguration.java | 12 +- .../PerResourcePollingEventSource.java | 57 +++++++- .../source/polling/PollingConfiguration.java | 18 ++- .../polling/PollingConfigurationBuilder.java | 20 ++- .../source/polling/PollingEventSource.java | 129 ++++++++++++++---- .../event/source/timer/TimerEventSource.java | 91 +++++++++--- .../ConfigurationServiceOverriderTest.java | 59 ++++++++ .../config/ExecutorServiceManagerTest.java | 6 + .../polling/PollingEventSourceTest.java | 77 ++++++++++- .../source/timer/TimerEventSourceTest.java | 57 ++++++++ .../operator/config/loader/ConfigLoader.java | 8 ++ .../config/loader/ConfigLoaderTest.java | 24 ++++ ...edExternalDependentResourceReconciler.java | 1 + 22 files changed, 804 insertions(+), 75 deletions(-) diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index 4b890407e4..e954f8a31f 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -222,6 +222,28 @@ is similar to `PerResourcePollingEventSource` except that, contrary to that even doesn't poll a specific API separately per resource, but periodically and independently of actually observed primary resources. +#### Threading of the polling event sources + +Both polling event sources schedule their polls on an executor the operator shares between all of +its polling event sources. A poll therefore only starts once one of that executor's threads is +free, so if your operator registers many polling event sources, or if fetching your external +resources is slow, size the pool accordingly with +`ConfigurationServiceOverrider.withConcurrentScheduledTaskThreads` (4 threads by default): + +```java +Operator operator = new Operator(overrider -> overrider.withConcurrentScheduledTaskThreads(20)); +``` + +Retried and rescheduled reconciliations are triggered on a separate executor, so a slow poll can +never delay them. It is sized with `withConcurrentRetryAndRescheduleThreads` (2 threads by +default); few threads are needed there since triggering a reconciliation only enqueues an event +for one of the reconciliation threads to pick up. + +Use `ConfigurationServiceOverrider.withScheduledExecutorService` to replace the polling executor +altogether, or the `withExecutorService` method of the event source's own configuration builder to +poll a single event source on an executor of its own. An executor provided that way is not managed +by the operator: it is your responsibility to shut it down. + #### Inbound event sources [SimpleInboundEventSource](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/inbound/SimpleInboundEventSource.java) diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index 86d5ff90df..959e1728cd 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -301,6 +301,13 @@ All operator-level keys are prefixed with `josdk.`. |---|---|---| | `josdk.workflow.executor-threads` | `Integer` | Thread pool size for workflow execution | +#### Scheduled Tasks + +| Key | Type | Description | +|---|---|---| +| `josdk.scheduled-tasks.concurrent-threads` | `Integer` | Thread pool size for the operator's scheduled tasks, i.e. the polling event sources | +| `josdk.retry-and-reschedule.concurrent-threads` | `Integer` | Thread pool size for triggering retried and rescheduled reconciliations | + #### Informer | Key | Type | Description | diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 336af9e2a8..bf972bfcc7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Consumer; import org.slf4j.Logger; @@ -64,6 +65,20 @@ public interface ConfigurationService { /** The default number of threads used to process dependent workflows */ int DEFAULT_WORKFLOW_EXECUTOR_THREAD_NUMBER = DEFAULT_RECONCILIATION_THREADS_NUMBER; + /** + * The default number of threads used to run the operator's scheduled tasks, i.e. the periodic + * polls of {@link io.javaoperatorsdk.operator.processing.event.source.polling.PollingEventSource} + * and {@link + * io.javaoperatorsdk.operator.processing.event.source.polling.PerResourcePollingEventSource} + */ + int DEFAULT_SCHEDULED_TASK_THREADS_NUMBER = 4; + + /** + * The default number of threads used to trigger the operator's retried and rescheduled + * reconciliations + */ + int DEFAULT_RETRY_AND_RESCHEDULE_THREADS_NUMBER = 2; + /** * Creates a new {@link ConfigurationService} instance used to configure an {@link * io.javaoperatorsdk.operator.Operator} instance, starting from the specified base configuration @@ -219,6 +234,33 @@ default int concurrentWorkflowExecutorThreads() { return DEFAULT_WORKFLOW_EXECUTOR_THREAD_NUMBER; } + /** + * Number of threads the operator can spin out to run its scheduled (i.e. periodic or delayed) + * tasks with the default executor. These threads are shared by all the polling event sources of + * the operator, so this number should be raised when many, or slow, polling event sources are + * registered: a poll only starts once a thread is available, and a slow poll therefore delays the + * polls of the other event sources. + * + * @return the maximum number of concurrent scheduled task threads + * @since 5.6.0 + */ + default int concurrentScheduledTaskThreads() { + return DEFAULT_SCHEDULED_TASK_THREADS_NUMBER; + } + + /** + * Number of threads the operator can spin out to trigger its retried and rescheduled + * reconciliations with the default executor. These threads are shared by all the controllers of + * the operator, but the tasks they run only enqueue an event for the reconciliation to happen on + * a reconciliation thread, so few of them are needed. + * + * @return the maximum number of concurrent retry and reschedule threads + * @since 5.6.0 + */ + default int concurrentRetryAndRescheduleThreads() { + return DEFAULT_RETRY_AND_RESCHEDULE_THREADS_NUMBER; + } + /** * Override to provide a custom {@link Metrics} implementation * @@ -249,6 +291,44 @@ default ExecutorService getWorkflowExecutorService() { return Executors.newFixedThreadPool(concurrentWorkflowExecutorThreads()); } + /** + * Override to provide a custom {@link ScheduledExecutorService} implementation to change how the + * operator's scheduled (i.e. periodic or delayed) tasks are run. This executor is shared by all + * the polling event sources of the operator. The retried and rescheduled reconciliations run on + * an executor of their own, see {@link #getRetryAndRescheduleExecutorService()}, so that a slow + * poll can't delay them. + * + *

Note that the default implementation lets the executor discard the tasks that were scheduled + * for later when it is shut down, so that they don't delay the termination of the operator, and + * that it creates daemon threads so that a never stopped operator doesn't keep the JVM alive. + * Custom implementations are advised to do the same. + * + * @return the {@link ScheduledExecutorService} implementation to use to run scheduled tasks + * @since 5.6.0 + */ + default ScheduledExecutorService getScheduledExecutorService() { + return Utils.daemonScheduledThreadPool( + concurrentScheduledTaskThreads(), "josdk-scheduled-task"); + } + + /** + * Override to provide a custom {@link ScheduledExecutorService} implementation to change how the + * operator's retried and rescheduled reconciliations are triggered. This executor is kept + * separate from the one the polling event sources use, see {@link + * #getScheduledExecutorService()}, so that a slow poll can't delay a retry. + * + *

The same notes as for {@link #getScheduledExecutorService()} apply to custom + * implementations. + * + * @return the {@link ScheduledExecutorService} implementation to use to trigger retried and + * rescheduled reconciliations + * @since 5.6.0 + */ + default ScheduledExecutorService getRetryAndRescheduleExecutorService() { + return Utils.daemonScheduledThreadPool( + concurrentRetryAndRescheduleThreads(), "josdk-retry-reschedule"); + } + /** * Determines whether the associated Kubernetes client should be closed when the associated {@link * io.javaoperatorsdk.operator.Operator} is stopped. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index 2cf6540af0..9ac8668b20 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -21,6 +21,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Function; import org.slf4j.Logger; @@ -45,11 +46,15 @@ public class ConfigurationServiceOverrider { private Boolean checkCR; private Integer concurrentReconciliationThreads; private Integer concurrentWorkflowExecutorThreads; + private Integer concurrentScheduledTaskThreads; + private Integer concurrentRetryAndRescheduleThreads; private Cloner cloner; private Boolean closeClientOnStop; private KubernetesClient client; private ExecutorService executorService; private ExecutorService workflowExecutorService; + private ScheduledExecutorService scheduledExecutorService; + private ScheduledExecutorService retryAndRescheduleExecutorService; private LeaderElectionConfiguration leaderElectionConfiguration; private String clusterScopedEventNamespace; private EventRecorder eventRecorder; @@ -86,6 +91,34 @@ public ConfigurationServiceOverrider withConcurrentWorkflowExecutorThreads(int t return this; } + /** + * Sets the number of threads used to run the operator's scheduled (i.e. periodic or delayed) + * tasks, which are shared by all its polling event sources. + * + * @param threadNumber the maximum number of concurrent scheduled task threads + * @return this {@link ConfigurationServiceOverrider} for chained customization + * @see ConfigurationService#concurrentScheduledTaskThreads() + * @since 5.6.0 + */ + public ConfigurationServiceOverrider withConcurrentScheduledTaskThreads(int threadNumber) { + this.concurrentScheduledTaskThreads = threadNumber; + return this; + } + + /** + * Sets the number of threads used to trigger the operator's retried and rescheduled + * reconciliations, which are shared by all its controllers. + * + * @param threadNumber the maximum number of concurrent retry and reschedule threads + * @return this {@link ConfigurationServiceOverrider} for chained customization + * @see ConfigurationService#concurrentRetryAndRescheduleThreads() + * @since 5.6.0 + */ + public ConfigurationServiceOverrider withConcurrentRetryAndRescheduleThreads(int threadNumber) { + this.concurrentRetryAndRescheduleThreads = threadNumber; + return this; + } + @SuppressWarnings("rawtypes") public ConfigurationServiceOverrider withDependentResourceFactory( DependentResourceFactory dependentResourceFactory) { @@ -119,6 +152,37 @@ public ConfigurationServiceOverrider withWorkflowExecutorService( return this; } + /** + * Replaces the executor used to run the operator's scheduled (i.e. periodic or delayed) tasks, + * which are shared by all its polling event sources. + * + * @param scheduledExecutorService the executor to run scheduled tasks on + * @return this {@link ConfigurationServiceOverrider} for chained customization + * @see ConfigurationService#getScheduledExecutorService() + * @since 5.6.0 + */ + public ConfigurationServiceOverrider withScheduledExecutorService( + ScheduledExecutorService scheduledExecutorService) { + this.scheduledExecutorService = scheduledExecutorService; + return this; + } + + /** + * Replaces the executor used to trigger the operator's retried and rescheduled reconciliations, + * which is shared by all its controllers. + * + * @param retryAndRescheduleExecutorService the executor to trigger retried and rescheduled + * reconciliations on + * @return this {@link ConfigurationServiceOverrider} for chained customization + * @see ConfigurationService#getRetryAndRescheduleExecutorService() + * @since 5.6.0 + */ + public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( + ScheduledExecutorService retryAndRescheduleExecutorService) { + this.retryAndRescheduleExecutorService = retryAndRescheduleExecutorService; + return this; + } + /** * Replaces the default {@link KubernetesClient} instance by the specified one. This is the * preferred mechanism to configure which client will be used to access the cluster. @@ -312,6 +376,28 @@ public int concurrentWorkflowExecutorThreads() { original.concurrentWorkflowExecutorThreads()); } + @Override + public int concurrentScheduledTaskThreads() { + return Utils.ensureValid( + overriddenValueOrDefault( + concurrentScheduledTaskThreads, + ConfigurationService::concurrentScheduledTaskThreads), + "maximum scheduled task threads", + 1, + original.concurrentScheduledTaskThreads()); + } + + @Override + public int concurrentRetryAndRescheduleThreads() { + return Utils.ensureValid( + overriddenValueOrDefault( + concurrentRetryAndRescheduleThreads, + ConfigurationService::concurrentRetryAndRescheduleThreads), + "maximum retry and reschedule threads", + 1, + original.concurrentRetryAndRescheduleThreads()); + } + @Override public Metrics getMetrics() { return overriddenValueOrDefault(metrics, ConfigurationService::getMetrics); @@ -340,6 +426,24 @@ public ExecutorService getWorkflowExecutorService() { } } + @Override + public ScheduledExecutorService getScheduledExecutorService() { + if (scheduledExecutorService != null) { + return scheduledExecutorService; + } else { + return super.getScheduledExecutorService(); + } + } + + @Override + public ScheduledExecutorService getRetryAndRescheduleExecutorService() { + if (retryAndRescheduleExecutorService != null) { + return retryAndRescheduleExecutorService; + } else { + return super.getRetryAndRescheduleExecutorService(); + } + } + @Override public Optional getLeaderElectionConfiguration() { return leaderElectionConfiguration != null diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index cdcafcaa46..437bbfeae9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -42,6 +42,7 @@ public class ExecutorServiceManager { private ExecutorService workflowExecutor; private ExecutorService cachingExecutorService; private ScheduledExecutorService scheduledExecutorService; + private ScheduledExecutorService retryAndRescheduleExecutorService; private boolean started; private ConfigurationService configurationService; @@ -128,30 +129,54 @@ public ExecutorService cachingExecutorService() { return cachingExecutorService; } + /** + * The executor the operator runs its scheduled (i.e. periodic or delayed) tasks on, shared by its + * polling event sources. Note that it is only valid while the manager is started: it is shut down + * by {@link #stop(Duration)} and replaced by a fresh one on the next {@link + * #start(ConfigurationService)}, so callers should retrieve it when they start rather than hold + * on to it. + * + * @return the executor to run scheduled tasks on + */ public ScheduledExecutorService scheduledExecutorService() { return scheduledExecutorService; } + /** + * The executor the operator triggers its retried and rescheduled reconciliations on, kept + * separate from {@link #scheduledExecutorService()} so that a slow poll can't delay a retry. The + * same lifecycle caveat as for {@link #scheduledExecutorService()} applies. + * + * @return the executor to trigger retried and rescheduled reconciliations on + */ + public ScheduledExecutorService retryAndRescheduleExecutorService() { + return retryAndRescheduleExecutorService; + } + public synchronized void start(ConfigurationService configurationService) { if (!started) { this.configurationService = configurationService; // used to lazy init workflow executor this.cachingExecutorService = Executors.newCachedThreadPool(); - this.scheduledExecutorService = Executors.newScheduledThreadPool(0); + this.scheduledExecutorService = configurationService.getScheduledExecutorService(); + this.retryAndRescheduleExecutorService = + configurationService.getRetryAndRescheduleExecutorService(); this.executor = new InstrumentedExecutorService(configurationService.getExecutorService()); started = true; } } public synchronized void stop(Duration gracefulShutdownTimeout) { - var parallelExec = Executors.newFixedThreadPool(4); + var shutdowns = + List.of( + shutdown(executor, gracefulShutdownTimeout), + shutdown(workflowExecutor, gracefulShutdownTimeout), + shutdown(cachingExecutorService, gracefulShutdownTimeout), + shutdown(scheduledExecutorService, gracefulShutdownTimeout), + shutdown(retryAndRescheduleExecutorService, gracefulShutdownTimeout)); + var parallelExec = Executors.newFixedThreadPool(shutdowns.size()); try { log.debug("Closing executor"); - parallelExec.invokeAll( - List.of( - shutdown(executor, gracefulShutdownTimeout), - shutdown(workflowExecutor, gracefulShutdownTimeout), - shutdown(cachingExecutorService, gracefulShutdownTimeout), - shutdown(scheduledExecutorService, gracefulShutdownTimeout))); + parallelExec.invokeAll(shutdowns); } catch (InterruptedException e) { log.debug("Exception closing executor: {}", e.getLocalizedMessage()); Thread.currentThread().interrupt(); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java index 4613d9645d..95e4fba6e1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java @@ -26,6 +26,11 @@ import java.util.Date; import java.util.Optional; import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,6 +113,49 @@ public static int ensureValid(int value, String description, int minValue, int d return value; } + /** + * Creates a {@link ThreadFactory} producing daemon threads named after the specified prefix. + * Daemon threads don't keep the JVM alive if the {@link io.javaoperatorsdk.operator.Operator} is + * never stopped, and naming them makes the pool they belong to identifiable in thread dumps. + * + * @param namePrefix the prefix the created threads are named after + * @return a {@link ThreadFactory} creating named daemon threads + * @since 5.6.0 + */ + public static ThreadFactory daemonThreadFactory(String namePrefix) { + final var defaultThreadFactory = Executors.defaultThreadFactory(); + final var counter = new AtomicLong(); + return runnable -> { + final var thread = defaultThreadFactory.newThread(runnable); + thread.setName(namePrefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } + + /** + * Creates the kind of {@link ScheduledExecutorService} the operator runs its scheduled (i.e. + * periodic or delayed) tasks on: one whose threads are daemon threads named after the specified + * prefix, and which doesn't let the tasks that were scheduled for later delay its shutdown. + * + * @param corePoolSize the number of threads to keep in the pool + * @param threadNamePrefix the prefix the threads of the pool are named after + * @return a {@link ScheduledExecutorService} to run scheduled tasks on + * @since 5.6.0 + */ + public static ScheduledExecutorService daemonScheduledThreadPool( + int corePoolSize, String threadNamePrefix) { + final var executor = + new ScheduledThreadPoolExecutor(corePoolSize, daemonThreadFactory(threadNamePrefix)); + // tasks that are scheduled far out (a reconciliation rescheduled in an hour, say) would + // otherwise keep the pool from terminating until the graceful shutdown timeout expires + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + // cancelled tasks are frequent (every retry that is superseded by a new event cancels one) and + // would otherwise be retained until their delay elapses + executor.setRemoveOnCancelPolicy(true); + return executor; + } + @SuppressWarnings("unused") // this is used in the Quarkus extension public static boolean isValidateCustomResourcesEnvVarSet() { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PollingDependentResource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PollingDependentResource.java index 894e359d57..a524fc2ec1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PollingDependentResource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/external/PollingDependentResource.java @@ -55,6 +55,7 @@ protected ExternalResourceCachingEventSource createEventSource( EventSourceContext

context) { return new PollingEventSource<>( resourceType(), + context, new PollingConfiguration<>(name(), this, getPollingPeriod(), resourceIDMapper)); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java index d553d14cf9..bd5222fbbb 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSourceManager.java @@ -55,7 +55,9 @@ public class EventSourceManager

public EventSourceManager(Controller

controller) { this( controller, - new EventSources<>(controller.getConfiguration().triggerReconcilerOnAllEvents())); + new EventSources<>( + controller.getConfiguration().triggerReconcilerOnAllEvents(), + () -> controller.getExecutorServiceManager().retryAndRescheduleExecutorService())); } EventSourceManager(Controller

controller, EventSources

eventSources) { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSources.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSources.java index b482d26dca..2e8a523de0 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSources.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventSources.java @@ -23,6 +23,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Supplier; import java.util.stream.Stream; import io.fabric8.kubernetes.api.model.HasMetadata; @@ -41,8 +43,17 @@ class EventSources

{ private ControllerEventSource

controllerEventSource; public EventSources(boolean triggerReconcilerOnAllEvents) { + this(triggerReconcilerOnAllEvents, null); + } + + public EventSources( + boolean triggerReconcilerOnAllEvents, + Supplier scheduledExecutorServiceSupplier) { retryAndRescheduleTimerEventSource = - new TimerEventSource<>("RetryAndRescheduleTimerEventSource", triggerReconcilerOnAllEvents); + new TimerEventSource<>( + "RetryAndRescheduleTimerEventSource", + triggerReconcilerOnAllEvents, + scheduledExecutorServiceSupplier); } EventSources() { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java index 599647ff29..5c84a800a7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java @@ -18,12 +18,15 @@ import java.time.Duration; import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.function.Predicate; import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.processing.ResourceIDMapper; +/** + * @param executorService the executor to run the polls on, {@code null} (the default) to run them + * on the executor the operator shares between all its scheduled tasks + */ public record PerResourcePollingConfiguration( String name, ScheduledExecutorService executorService, @@ -32,8 +35,6 @@ public record PerResourcePollingConfiguration( Predicate

registerPredicate, Duration defaultPollingPeriod) { - public static final int DEFAULT_EXECUTOR_THREAD_NUMBER = 1; - public PerResourcePollingConfiguration( String name, ScheduledExecutorService executorService, @@ -42,10 +43,7 @@ public PerResourcePollingConfiguration( Predicate

registerPredicate, Duration defaultPollingPeriod) { this.name = name; - this.executorService = - executorService == null - ? new ScheduledThreadPoolExecutor(DEFAULT_EXECUTOR_THREAD_NUMBER) - : executorService; + this.executorService = executorService; this.resourceIDMapper = resourceIDMapper == null ? ResourceIDMapper.resourceIdProviderMapper() : resourceIDMapper; this.resourceFetcher = Objects.requireNonNull(resourceFetcher); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 1ab750d8f0..6303912c5a 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -21,6 +21,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -31,6 +32,8 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.OperatorException; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.javaoperatorsdk.operator.processing.event.source.Cache; @@ -56,11 +59,15 @@ public class PerResourcePollingEventSource private final Cache

primaryResourceCache; private final Set fetchedForPrimaries = ConcurrentHashMap.newKeySet(); - private final ScheduledExecutorService executorService; + private final ScheduledExecutorService configuredExecutorService; + private final ConfigurationService configurationService; + private final boolean ownsExecutorService; private final ResourceFetcher resourceFetcher; private final Predicate

registerPredicate; private final Duration period; + private volatile ScheduledExecutorService executorService; + public PerResourcePollingEventSource( Class resourceClass, EventSourceContext

context, @@ -69,10 +76,48 @@ public PerResourcePollingEventSource( this.primaryResourceCache = context.getPrimaryCache(); this.resourceFetcher = config.resourceFetcher(); this.registerPredicate = config.registerPredicate(); - this.executorService = config.executorService(); + this.configuredExecutorService = config.executorService(); + this.configurationService = configurationServiceOf(context); + // when neither the configuration nor the operator provides one, the event source has to create + // an executor of its own, and is then the one responsible for shutting it down + this.ownsExecutorService = configuredExecutorService == null && configurationService == null; this.period = config.defaultPollingPeriod(); } + /** + * The configuration of the operator the event source belongs to, or {@code null} if it doesn't + * belong to one, which only happens when the event source is used standalone, outside an + * operator. + */ + private static ConfigurationService configurationServiceOf(EventSourceContext context) { + final var controllerConfiguration = context.getControllerConfiguration(); + return controllerConfiguration == null + ? null + : controllerConfiguration.getConfigurationService(); + } + + @Override + public void start() throws OperatorException { + executorService = resolveExecutorService(); + super.start(); + } + + /** + * Resolves the executor to poll on. Note that this happens on every start, and not once at + * creation time, since the operator shuts its executors down when it is stopped and creates new + * ones if it is started again. + */ + private ScheduledExecutorService resolveExecutorService() { + if (configuredExecutorService != null) { + return configuredExecutorService; + } + if (configurationService != null) { + return configurationService.getExecutorServiceManager().scheduledExecutorService(); + } + return Executors.newSingleThreadScheduledExecutor( + Utils.daemonThreadFactory("josdk-polling-" + name())); + } + private Set getAndCacheResource(P primary, boolean fromGetter) { var values = resourceFetcher.fetchResources(primary); var primaryID = ResourceID.fromResource(primary); @@ -202,6 +247,12 @@ default Optional fetchDelay(Set lastFetchedResource, P primary) { @Override public void stop() throws OperatorException { super.stop(); - executorService.shutdownNow(); + // the tasks have to be cancelled explicitly now that the executor can be shared with the rest + // of the operator, and the map cleared so that they are registered again on a restart + scheduledFutures.values().forEach(future -> future.cancel(true)); + scheduledFutures.clear(); + if (ownsExecutorService && executorService != null) { + executorService.shutdownNow(); + } } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java index 9ac1b8cc96..0836058cf4 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java @@ -17,24 +17,40 @@ import java.time.Duration; import java.util.Objects; +import java.util.concurrent.ScheduledExecutorService; import io.javaoperatorsdk.operator.processing.ResourceIDMapper; +/** + * @param executorService the executor to run the polls on, {@code null} (the default) to run them + * on the executor the operator shares between all its scheduled tasks + */ public record PollingConfiguration( String name, PollingEventSource.GenericResourceFetcher genericResourceFetcher, Duration period, - ResourceIDMapper resourceIDMapper) { + ResourceIDMapper resourceIDMapper, + ScheduledExecutorService executorService) { public PollingConfiguration( String name, PollingEventSource.GenericResourceFetcher genericResourceFetcher, Duration period, ResourceIDMapper resourceIDMapper) { + this(name, genericResourceFetcher, period, resourceIDMapper, null); + } + + public PollingConfiguration( + String name, + PollingEventSource.GenericResourceFetcher genericResourceFetcher, + Duration period, + ResourceIDMapper resourceIDMapper, + ScheduledExecutorService executorService) { this.name = name; this.genericResourceFetcher = Objects.requireNonNull(genericResourceFetcher); this.period = period; this.resourceIDMapper = resourceIDMapper == null ? ResourceIDMapper.resourceIdProviderMapper() : resourceIDMapper; + this.executorService = executorService; } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java index 0e68876b60..5a12fc4d50 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java @@ -16,6 +16,7 @@ package io.javaoperatorsdk.operator.processing.event.source.polling; import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; import io.javaoperatorsdk.operator.processing.ResourceIDMapper; @@ -24,6 +25,7 @@ public final class PollingConfigurationBuilder { private final PollingEventSource.GenericResourceFetcher genericResourceFetcher; private ResourceIDMapper resourceIDMapper; private String name; + private ScheduledExecutorService executorService; public PollingConfigurationBuilder( PollingEventSource.GenericResourceFetcher fetcher, Duration period) { @@ -42,7 +44,23 @@ public PollingConfigurationBuilder withName(String name) { return this; } + /** + * Runs the polls on the specified executor instead of the one the operator shares between all its + * scheduled tasks. Note that an explicitly provided executor is not managed by the operator: it + * is the caller's responsibility to shut it down. + * + * @param executorService the executor to run the polls on + * @return this builder for chained customization + * @since 5.6.0 + */ + public PollingConfigurationBuilder withExecutorService( + ScheduledExecutorService executorService) { + this.executorService = executorService; + return this; + } + public PollingConfiguration build() { - return new PollingConfiguration<>(name, genericResourceFetcher, period, resourceIDMapper); + return new PollingConfiguration<>( + name, genericResourceFetcher, period, resourceIDMapper, executorService); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index 5c85c24f90..8e615a6059 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -18,8 +18,10 @@ import java.time.Duration; import java.util.Map; import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.slf4j.Logger; @@ -27,6 +29,9 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.OperatorException; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.Utils; +import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.javaoperatorsdk.operator.processing.event.source.ExternalResourceCachingEventSource; @@ -62,45 +67,109 @@ public class PollingEventSource private static final Logger log = LoggerFactory.getLogger(PollingEventSource.class); - private Timer timer; private final GenericResourceFetcher genericResourceFetcher; private final Duration period; private final AtomicBoolean healthy = new AtomicBoolean(true); + private final ScheduledExecutorService configuredExecutorService; + private final ConfigurationService configurationService; + private final boolean ownsExecutorService; + private volatile ScheduledExecutorService executorService; + private volatile ScheduledFuture pollingTask; + + /** + * Creates an event source polling on the executor the operator shares between all its scheduled + * tasks, unless the configuration provides one of its own. + * + * @param resourceClass the type of the polled resource + * @param context the context this event source is created for + * @param config the configuration of the polling + * @since 5.6.0 + */ + public PollingEventSource( + Class resourceClass, EventSourceContext

context, PollingConfiguration config) { + this(resourceClass, config, configurationServiceOf(context)); + } + + /** + * @deprecated use {@link #PollingEventSource(Class, EventSourceContext, PollingConfiguration)} + * instead: without a context, and unless the configuration provides an executor, this event + * source has to create a thread of its own to poll on instead of using the one the operator + * shares between all its scheduled tasks. + */ + @Deprecated(since = "5.6.0") public PollingEventSource(Class resourceClass, PollingConfiguration config) { + this(resourceClass, config, null); + } + + private PollingEventSource( + Class resourceClass, + PollingConfiguration config, + ConfigurationService configurationService) { super(config.name(), resourceClass, config.resourceIDMapper()); this.genericResourceFetcher = config.genericResourceFetcher(); this.period = config.period(); + this.configuredExecutorService = config.executorService(); + this.configurationService = configurationService; + // when neither the configuration nor the operator provides one, the event source has to create + // an executor of its own, and is then the one responsible for shutting it down + this.ownsExecutorService = configuredExecutorService == null && configurationService == null; + } + + /** + * The configuration of the operator the event source belongs to, or {@code null} if it doesn't + * belong to one, which only happens when the event source is used standalone, outside an + * operator. + */ + private static ConfigurationService configurationServiceOf(EventSourceContext context) { + final var controllerConfiguration = context.getControllerConfiguration(); + return controllerConfiguration == null + ? null + : controllerConfiguration.getConfigurationService(); } @Override public void start() throws OperatorException { - if (timer != null) { + if (pollingTask != null) { return; } super.start(); + executorService = resolveExecutorService(); getStateAndFillCache(); - timer = new Timer(true); - timer.schedule( - new TimerTask() { - @Override - public void run() { - try { - if (!isRunning()) { - log.debug("Event source not yet started. Will not run."); - return; - } - getStateAndFillCache(); - healthy.set(true); - } catch (Exception e) { - // Exception is required because of Kotlin - healthy.set(false); - log.error("Error during polling.", e); - } - } - }, - period.toMillis(), - period.toMillis()); + pollingTask = + executorService.scheduleWithFixedDelay( + this::poll, period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS); + } + + private void poll() { + try { + if (!isRunning()) { + log.debug("Event source not yet started. Will not run."); + return; + } + getStateAndFillCache(); + healthy.set(true); + } catch (Exception e) { + // Exception is required because of Kotlin + healthy.set(false); + log.error("Error during polling.", e); + } + } + + /** + * Resolves the executor to poll on. Note that this happens on every start, and not once at + * creation time, since the operator shuts its executors down when it is stopped and creates new + * ones if it is started again. + */ + private ScheduledExecutorService resolveExecutorService() { + if (configuredExecutorService != null) { + return configuredExecutorService; + } + if (configurationService != null) { + return configurationService.getExecutorServiceManager().scheduledExecutorService(); + } + return Executors.newSingleThreadScheduledExecutor( + Utils.daemonThreadFactory("josdk-polling-" + name())); } protected synchronized void getStateAndFillCache() { @@ -115,9 +184,13 @@ public interface GenericResourceFetcher { @Override public void stop() throws OperatorException { super.stop(); - if (timer != null) { - timer.cancel(); - timer = null; + if (pollingTask != null) { + // as with the java.util.Timer this replaces, an ongoing poll is left to finish + pollingTask.cancel(false); + pollingTask = null; + } + if (ownsExecutorService && executorService != null) { + executorService.shutdownNow(); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index eae9663fe6..b4d7aa36a3 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -17,14 +17,18 @@ import java.util.Map; import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.BaseControl; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.Event; @@ -36,17 +40,47 @@ public class TimerEventSource extends AbstractEventSource implements ResourceEventAware { private static final Logger log = LoggerFactory.getLogger(TimerEventSource.class); - private Timer timer; - private final Map onceTasks = new ConcurrentHashMap<>(); + private final Map> onceTasks = new ConcurrentHashMap<>(); + private final Supplier executorServiceSupplier; + private final boolean ownsExecutorService; private boolean triggerReconcilerOnAllEvents; + private volatile ScheduledExecutorService executorService; public TimerEventSource() { - super(Void.class); + this((Supplier) null); } public TimerEventSource(String name, boolean triggerReconcilerOnAllEvents) { + this(name, triggerReconcilerOnAllEvents, null); + } + + /** + * Creates an event source scheduling on the executor provided by the specified supplier. The + * supplier is called on every start, and not once at creation time, since the operator shuts its + * executors down when it is stopped and creates new ones if it is started again. + * + * @param executorServiceSupplier supplies the executor to schedule on, {@code null} to have the + * event source create, and shut down, an executor of its own + * @since 5.6.0 + */ + public TimerEventSource(Supplier executorServiceSupplier) { + super(Void.class); + this.executorServiceSupplier = executorServiceSupplier; + this.ownsExecutorService = executorServiceSupplier == null; + } + + /** + * @see #TimerEventSource(Supplier) + * @since 5.6.0 + */ + public TimerEventSource( + String name, + boolean triggerReconcilerOnAllEvents, + Supplier executorServiceSupplier) { super(Void.class, name); this.triggerReconcilerOnAllEvents = triggerReconcilerOnAllEvents; + this.executorServiceSupplier = executorServiceSupplier; + this.ownsExecutorService = executorServiceSupplier == null; } @SuppressWarnings("unused") @@ -55,20 +89,25 @@ public void scheduleOnce(R resource, long delay) { } public void scheduleOnce(ResourceID resourceID, long delay) { - if (!isRunning()) { + final var executor = executorService; + if (!isRunning() || executor == null) { throw new IllegalStateException("The TimerEventSource is not running"); } - if (onceTasks.containsKey(resourceID)) { - cancelOnceSchedule(resourceID); - } - EventProducerTimeTask task = new EventProducerTimeTask(resourceID); if (delay == BaseControl.INSTANT_RESCHEDULE) { - task.run(); - } else { - onceTasks.put(resourceID, task); - timer.schedule(task, delay); + cancelOnceSchedule(resourceID); + new EventProducerTimeTask(resourceID).run(); + return; } + + onceTasks.compute( + resourceID, + (id, alreadyScheduled) -> { + if (alreadyScheduled != null) { + alreadyScheduled.cancel(false); + } + return executor.schedule(new EventProducerTimeTask(id), delay, TimeUnit.MILLISECONDS); + }); } @Override @@ -81,9 +120,11 @@ public void onResourceDeleted(R resource) { } public void cancelOnceSchedule(ResourceID customResourceUid) { - TimerTask timerTask = onceTasks.remove(customResourceUid); - if (timerTask != null) { - timerTask.cancel(); + var scheduled = onceTasks.remove(customResourceUid); + if (scheduled != null) { + // as with the java.util.TimerTask this replaces, a task that is already running is left to + // finish + scheduled.cancel(false); } } @@ -91,16 +132,26 @@ public void cancelOnceSchedule(ResourceID customResourceUid) { public void start() { if (!isRunning()) { super.start(); - timer = new Timer(true); + executorService = resolveExecutorService(); } } + private ScheduledExecutorService resolveExecutorService() { + if (executorServiceSupplier != null) { + return executorServiceSupplier.get(); + } + return Executors.newSingleThreadScheduledExecutor( + Utils.daemonThreadFactory("josdk-timer-" + name())); + } + @Override public void stop() { if (isRunning()) { onceTasks.keySet().forEach(this::cancelOnceSchedule); - timer.cancel(); super.stop(); + if (ownsExecutorService && executorService != null) { + executorService.shutdownNow(); + } } } @@ -114,7 +165,7 @@ public Set getSecondaryResources(HasMetadata primary) { return Set.of(); } - public class EventProducerTimeTask extends TimerTask { + public class EventProducerTimeTask implements Runnable { protected final ResourceID customResourceUid; diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java index aec8381135..aebd7bc9b0 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java @@ -20,6 +20,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor; import org.junit.jupiter.api.Test; @@ -117,6 +118,64 @@ public R clone(R object) { config.reconciliationTerminationTimeout(), overridden.reconciliationTerminationTimeout()); } + @Test + void scheduledExecutorCanBeOverridden() { + final var scheduledExecutorService = Executors.newScheduledThreadPool(1); + final var retryExecutorService = Executors.newScheduledThreadPool(1); + try { + final var overridden = + new ConfigurationServiceOverrider(config) + .withConcurrentScheduledTaskThreads(7) + .withScheduledExecutorService(scheduledExecutorService) + .withConcurrentRetryAndRescheduleThreads(5) + .withRetryAndRescheduleExecutorService(retryExecutorService) + .build(); + + assertThat(overridden.concurrentScheduledTaskThreads()).isEqualTo(7); + assertThat(overridden.getScheduledExecutorService()).isSameAs(scheduledExecutorService); + assertThat(overridden.concurrentRetryAndRescheduleThreads()).isEqualTo(5); + assertThat(overridden.getRetryAndRescheduleExecutorService()).isSameAs(retryExecutorService); + } finally { + scheduledExecutorService.shutdownNow(); + retryExecutorService.shutdownNow(); + } + } + + @Test + void scheduledExecutorDefaultsToADaemonPoolOfTheConfiguredSize() { + final var overridden = + (ScheduledThreadPoolExecutor) + new ConfigurationServiceOverrider(config) + .withConcurrentScheduledTaskThreads(3) + .build() + .getScheduledExecutorService(); + + try { + assertThat(overridden.getCorePoolSize()).isEqualTo(3); + // scheduled tasks must not delay the termination of the operator, nor keep the JVM alive + assertThat(overridden.getExecuteExistingDelayedTasksAfterShutdownPolicy()).isFalse(); + assertThat(overridden.getThreadFactory().newThread(() -> {}).isDaemon()).isTrue(); + } finally { + overridden.shutdownNow(); + } + } + + @Test + void retryAndRescheduleExecutorIsSeparateFromTheScheduledTaskOne() { + final var scheduled = config.getScheduledExecutorService(); + final var retryAndReschedule = config.getRetryAndRescheduleExecutorService(); + + try { + // a slow poll on the scheduled task executor must not be able to delay a retry + assertThat(retryAndReschedule).isNotSameAs(scheduled); + assertThat(((ScheduledThreadPoolExecutor) retryAndReschedule).getCorePoolSize()) + .isEqualTo(ConfigurationService.DEFAULT_RETRY_AND_RESCHEDULE_THREADS_NUMBER); + } finally { + scheduled.shutdownNow(); + retryAndReschedule.shutdownNow(); + } + } + @Test void eventRecorderIsNotConfiguredByDefaultAndCanBeOverridden() { final var eventRecorder = diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java index 40ffded241..995a6e374f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java @@ -30,14 +30,19 @@ void stopShutsDownTheScheduledExecutorService() { ConfigurationService configurationService = new BaseConfigurationService(); var manager = configurationService.getExecutorServiceManager(); var scheduled = manager.scheduledExecutorService(); + var retryAndReschedule = manager.retryAndRescheduleExecutorService(); try { assertThat(scheduled.isShutdown()).isFalse(); + assertThat(retryAndReschedule.isShutdown()).isFalse(); + // retries are triggered on an executor of their own so that a slow poll can't delay them + assertThat(retryAndReschedule).isNotSameAs(scheduled); } finally { manager.stop(SHUTDOWN_TIMEOUT); } assertThat(scheduled.isShutdown()).isTrue(); + assertThat(retryAndReschedule.isShutdown()).isTrue(); } @Test @@ -53,6 +58,7 @@ void canBeRestartedAfterStop() { assertThat(manager.reconcileExecutorService().isShutdown()).isFalse(); assertThat(manager.cachingExecutorService().isShutdown()).isFalse(); assertThat(manager.scheduledExecutorService().isShutdown()).isFalse(); + assertThat(manager.retryAndRescheduleExecutorService().isShutdown()).isFalse(); manager.stop(SHUTDOWN_TIMEOUT); } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java index 92400df4de..43b20d368f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java @@ -19,11 +19,16 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.EventHandler; import io.javaoperatorsdk.operator.processing.event.ResourceID; @@ -46,6 +51,7 @@ class PollingEventSourceTest private final PollingEventSource.GenericResourceFetcher resourceFetcher = mock(PollingEventSource.GenericResourceFetcher.class); + @SuppressWarnings("deprecation") private final PollingEventSource pollingEventSource = new PollingEventSource<>( SampleExternalResource.class, @@ -67,7 +73,7 @@ void canBeRestartedAfterStop() throws InterruptedException { Thread.sleep(DEFAULT_WAIT_PERIOD); pollingEventSource.stop(); - // a cancelled java.util.Timer cannot be reused, a new one has to be created on start + // the polling task is cancelled on stop, a new one has to be scheduled on start pollingEventSource.start(); Thread.sleep(DEFAULT_WAIT_PERIOD); @@ -75,7 +81,20 @@ void canBeRestartedAfterStop() throws InterruptedException { } @Test - void timerThreadIsADaemonSoItDoesNotKeepTheJvmAlive() throws InterruptedException { + void stopCancelsThePollingTask() throws InterruptedException { + when(resourceFetcher.fetchResources()).thenReturn(testResponseWithTwoValues()); + pollingEventSource.start(); + Thread.sleep(DEFAULT_WAIT_PERIOD); + pollingEventSource.stop(); + clearInvocations(resourceFetcher); + + Thread.sleep(DEFAULT_WAIT_PERIOD); + + verify(resourceFetcher, never()).fetchResources(); + } + + @Test + void pollingThreadIsADaemonSoItDoesNotKeepTheJvmAlive() throws InterruptedException { when(resourceFetcher.fetchResources()).thenReturn(testResponseWithTwoValues()); var threadsBeforeStart = Thread.getAllStackTraces().keySet(); @@ -83,13 +102,13 @@ void timerThreadIsADaemonSoItDoesNotKeepTheJvmAlive() throws InterruptedExceptio pollingEventSource.start(); Thread.sleep(DEFAULT_WAIT_PERIOD); - var newTimerThreads = + var newPollingThreads = Thread.getAllStackTraces().keySet().stream() - .filter(t -> t.getName().startsWith("Timer-")) + .filter(t -> t.getName().startsWith("josdk-polling-")) .filter(t -> !threadsBeforeStart.contains(t)) .toList(); - assertThat(newTimerThreads).isNotEmpty().allMatch(Thread::isDaemon); + assertThat(newPollingThreads).isNotEmpty().allMatch(Thread::isDaemon); } @Test @@ -154,6 +173,54 @@ void updatesHealthIndicatorBasedOnExceptionsInFetcher() { .untilAsserted(() -> assertThat(pollingEventSource.getStatus()).isEqualTo(Status.HEALTHY)); } + @Test + void pollsOnTheOperatorsSharedSchedulerWhenCreatedWithAContext() { + var pollingThreadNames = new CopyOnWriteArrayList(); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + pollingThreadNames.add(Thread.currentThread().getName()); + return testResponseWithOneValue(); + }); + + var configurationService = new BaseConfigurationService(); + var executorServiceManager = configurationService.getExecutorServiceManager(); + var eventSource = + new PollingEventSource( + SampleExternalResource.class, + contextFor(configurationService), + new PollingConfiguration<>(null, resourceFetcher, POLL_PERIOD, null)); + eventSource.setEventHandler(mock(EventHandler.class)); + + try { + eventSource.start(); + + // the initial fetch happens on the calling thread, the scheduled ones on the shared pool + await() + .untilAsserted( + () -> + assertThat(pollingThreadNames) + .anyMatch(name -> name.startsWith("josdk-scheduled-task-"))); + + eventSource.stop(); + + // the shared executor belongs to the operator, stopping an event source must not shut it down + assertThat(executorServiceManager.scheduledExecutorService().isShutdown()).isFalse(); + } finally { + executorServiceManager.stop(Duration.ofMillis(100)); + } + } + + @SuppressWarnings("unchecked") + private static EventSourceContext contextFor( + ConfigurationService configurationService) { + var controllerConfiguration = mock(ControllerConfiguration.class); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + var context = mock(EventSourceContext.class); + when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); + return context; + } + private Map> testResponseWithTwoValueForSameId() { Map> res = new HashMap<>(); res.put(primaryID1(), Set.of(testResource1(), testResource2())); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index 3a4e1cb80d..e8d36c0034 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; @@ -26,6 +27,7 @@ import org.junit.jupiter.api.Test; import io.javaoperatorsdk.operator.TestUtils; +import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.BaseControl; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.Event; @@ -125,6 +127,59 @@ public void handlesInstanceReschedule() { assertThat(eventHandler.events).hasSize(1); } + @Test + public void schedulesOnTheProvidedExecutorAndLeavesItRunningOnStop() { + var providedExecutor = + Executors.newSingleThreadScheduledExecutor( + Utils.daemonThreadFactory("provided-timer-executor")); + var handler = new CapturingEventHandler(); + var eventSource = new TimerEventSource(() -> providedExecutor); + eventSource.setEventHandler(handler); + + try { + eventSource.start(); + eventSource.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); + + untilAsserted( + () -> + assertThat(handler.eventProducingThreadNames) + .containsExactly("provided-timer-executor-1")); + + eventSource.stop(); + + // the executor is not the event source's to shut down + assertThat(providedExecutor.isShutdown()).isFalse(); + } finally { + providedExecutor.shutdownNow(); + } + } + + @Test + public void shutsDownTheExecutorItCreatedItselfAndCreatesANewOneOnRestart() { + var eventSource = new TimerEventSource(); + var handler = new CapturingEventHandler(); + eventSource.setEventHandler(handler); + + eventSource.start(); + eventSource.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); + untilAsserted(() -> assertThat(handler.events).hasSize(1)); + var firstThreadName = handler.eventProducingThreadNames.get(0); + assertThat(firstThreadName).startsWith("josdk-timer-"); + + eventSource.stop(); + Awaitility.await() + .untilAsserted( + () -> + assertThat(Thread.getAllStackTraces().keySet()) + .noneMatch(t -> t.getName().equals(firstThreadName))); + + eventSource.start(); + eventSource.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); + untilAsserted(() -> assertThat(handler.events).hasSize(2)); + + eventSource.stop(); + } + private void untilAsserted(ThrowingRunnable assertion) { untilAsserted(INITIAL_DELAY, PERIOD, assertion); } @@ -150,10 +205,12 @@ private void untilAsserted(long initialDelay, long interval, ThrowingRunnable as public static class CapturingEventHandler implements EventHandler { private final List events = new CopyOnWriteArrayList<>(); + private final List eventProducingThreadNames = new CopyOnWriteArrayList<>(); @Override public void handleEvent(Event event) { events.add(event); + eventProducingThreadNames.add(Thread.currentThread().getName()); } } } diff --git a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java index 1a221fb76c..8837e9e797 100644 --- a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java +++ b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java @@ -82,6 +82,14 @@ public static ConfigLoader getDefault() { "workflow.executor-threads", Integer.class, ConfigurationServiceOverrider::withConcurrentWorkflowExecutorThreads), + new ConfigBinding<>( + "scheduled-tasks.concurrent-threads", + Integer.class, + ConfigurationServiceOverrider::withConcurrentScheduledTaskThreads), + new ConfigBinding<>( + "retry-and-reschedule.concurrent-threads", + Integer.class, + ConfigurationServiceOverrider::withConcurrentRetryAndRescheduleThreads), new ConfigBinding<>( "close-client-on-stop", Boolean.class, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java index 0b014bb6f7..e22794e2b5 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java @@ -86,6 +86,30 @@ void applyConfigsAppliesConcurrentWorkflowExecutorThreads() { assertThat(result.concurrentWorkflowExecutorThreads()).isEqualTo(3); } + @Test + void applyConfigsAppliesConcurrentScheduledTaskThreads() { + var loader = + new ConfigLoader(mapProvider(Map.of("josdk.scheduled-tasks.concurrent-threads", 8))); + + var base = new BaseConfigurationService(null); + var result = + ConfigurationService.newOverriddenConfigurationService(base, loader.applyConfigs()); + + assertThat(result.concurrentScheduledTaskThreads()).isEqualTo(8); + } + + @Test + void applyConfigsAppliesConcurrentRetryAndRescheduleThreads() { + var loader = + new ConfigLoader(mapProvider(Map.of("josdk.retry-and-reschedule.concurrent-threads", 6))); + + var base = new BaseConfigurationService(null); + var result = + ConfigurationService.newOverriddenConfigurationService(base, loader.applyConfigs()); + + assertThat(result.concurrentRetryAndRescheduleThreads()).isEqualTo(6); + } + @Test void applyConfigsAppliesBooleanFlags() { var values = new HashMap(); diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanagedexternaldependenttype/MultipleManagedExternalDependentResourceReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanagedexternaldependenttype/MultipleManagedExternalDependentResourceReconciler.java index dec9422486..33b7f965f3 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanagedexternaldependenttype/MultipleManagedExternalDependentResourceReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/dependent/multiplemanagedexternaldependenttype/MultipleManagedExternalDependentResourceReconciler.java @@ -96,6 +96,7 @@ public int getNumberOfExecutions() { pollingEventSource = new PollingEventSource<>( ExternalResource.class, + context, new PollingConfigurationBuilder( fetcher, Duration.ofMillis(1000L)) .withName(EVENT_SOURCE_NAME) From ff3f56c131a3c5627cf1c83073be360c40739f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 25 Sep 2026 14:28:03 +0200 Subject: [PATCH 02/45] fix: keep DEFAULT_EXECUTOR_THREAD_NUMBER and withdraw polls scheduled after stop --- .../source/polling/PerResourcePollingConfiguration.java | 7 +++++++ .../source/polling/PerResourcePollingEventSource.java | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java index 5c84a800a7..49bc8dd7ba 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java @@ -35,6 +35,13 @@ public record PerResourcePollingConfiguration( Predicate

registerPredicate, Duration defaultPollingPeriod) { + /** + * @deprecated not used anymore, polls now run on the executor the operator shares between all its + * scheduled tasks unless an executor is explicitly configured + */ + @Deprecated(forRemoval = true) + public static final int DEFAULT_EXECUTOR_THREAD_NUMBER = 1; + public PerResourcePollingConfiguration( String name, ScheduledExecutorService executorService, diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 6303912c5a..0e02482d9e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -137,6 +137,12 @@ private void scheduleNextExecution(P primary, Set actualResources) { executorService.schedule( new FetchingExecutor(primaryID), fetchDuration.toMillis(), TimeUnit.MILLISECONDS); scheduledFutures.put(primaryID, scheduledFuture); + // a fetch that was already running when the event source got stopped can get here after stop + // cleared the map; since the shared executor stays alive, the task has to be withdrawn, + // otherwise the stale entry would prevent the resource from being registered on a restart + if (!isRunning() && scheduledFutures.remove(primaryID, scheduledFuture)) { + scheduledFuture.cancel(true); + } } @Override From 7e9f474e1dc6883906dd1b90970b25d4c2a1fffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Sun, 25 Oct 2026 18:59:01 +0100 Subject: [PATCH 03/45] test: cover restarting polling event sources, including after the operator replaces its executors --- .../PerResourcePollingEventSourceTest.java | 54 +++++++++++++++++++ .../polling/PollingEventSourceTest.java | 35 +++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 907ed19fe5..2f406f0489 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -24,6 +24,8 @@ import org.junit.jupiter.api.Test; import io.javaoperatorsdk.operator.TestUtils; +import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.ResourceIDMapper; import io.javaoperatorsdk.operator.processing.event.EventHandler; @@ -215,6 +217,58 @@ void supportsDynamicPollingDelay() { .untilAsserted(() -> verify(supplier, times(2)).fetchResources(any())); } + @Test + void pollsAgainAfterRestart() { + source.onResourceCreated(testCustomResource); + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + source.stop(); + + source.start(); + clearInvocations(supplier); + source.onResourceCreated(testCustomResource); + + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + } + + @Test + void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { + var configurationService = new BaseConfigurationService(); + var executorServiceManager = configurationService.getExecutorServiceManager(); + var controllerConfiguration = mock(ControllerConfiguration.class); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); + setUpSource( + new PerResourcePollingEventSource<>( + SampleExternalResource.class, + context, + new PerResourcePollingConfigurationBuilder< + SampleExternalResource, TestCustomResource, String>( + supplier, Duration.ofMillis(PERIOD)) + .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) + .build())); + + try { + source.onResourceCreated(testCustomResource); + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + var firstExecutor = executorServiceManager.scheduledExecutorService(); + source.stop(); + + executorServiceManager.stop(Duration.ofMillis(100)); + executorServiceManager.start(configurationService); + assertThat(firstExecutor.isShutdown()).isTrue(); + assertThat(executorServiceManager.scheduledExecutorService()).isNotSameAs(firstExecutor); + + source.start(); + clearInvocations(supplier); + source.onResourceCreated(testCustomResource); + + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + } finally { + source.stop(); + executorServiceManager.stop(Duration.ofMillis(100)); + } + } + @Test void deleteEventCancelsTheScheduling() { when(supplier.fetchResources(any())).thenReturn(Set.of(SampleExternalResource.testResource1())); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java index 43b20d368f..c9be8caf58 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java @@ -75,11 +75,44 @@ void canBeRestartedAfterStop() throws InterruptedException { // the polling task is cancelled on stop, a new one has to be scheduled on start pollingEventSource.start(); - Thread.sleep(DEFAULT_WAIT_PERIOD); + clearInvocations(resourceFetcher); + await().untilAsserted(() -> verify(resourceFetcher, atLeast(2)).fetchResources()); assertThat(pollingEventSource.getStatus()).isEqualTo(Status.HEALTHY); } + @Test + void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { + when(resourceFetcher.fetchResources()).thenReturn(testResponseWithOneValue()); + var configurationService = new BaseConfigurationService(); + var executorServiceManager = configurationService.getExecutorServiceManager(); + var eventSource = + new PollingEventSource( + SampleExternalResource.class, + contextFor(configurationService), + new PollingConfiguration<>(null, resourceFetcher, POLL_PERIOD, null)); + eventSource.setEventHandler(mock(EventHandler.class)); + + try { + eventSource.start(); + var firstExecutor = executorServiceManager.scheduledExecutorService(); + eventSource.stop(); + + executorServiceManager.stop(Duration.ofMillis(100)); + executorServiceManager.start(configurationService); + assertThat(firstExecutor.isShutdown()).isTrue(); + assertThat(executorServiceManager.scheduledExecutorService()).isNotSameAs(firstExecutor); + + eventSource.start(); + clearInvocations(resourceFetcher); + + await().untilAsserted(() -> verify(resourceFetcher, atLeast(2)).fetchResources()); + } finally { + eventSource.stop(); + executorServiceManager.stop(Duration.ofMillis(100)); + } + } + @Test void stopCancelsThePollingTask() throws InterruptedException { when(resourceFetcher.fetchResources()).thenReturn(testResponseWithTwoValues()); From 5b4ccec0f67241d2ae5618a8766c40bfeffe1427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Sun, 25 Oct 2026 19:31:36 +0100 Subject: [PATCH 04/45] fix: close timer scheduling race on stop, guard repeated start of per resource polling, clarify executor ownership in docs --- .../content/en/docs/documentation/eventing.md | 9 +++-- .../PerResourcePollingEventSource.java | 3 ++ .../event/source/timer/TimerEventSource.java | 36 ++++++++++++++----- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index e954f8a31f..aec261bbf0 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -240,9 +240,12 @@ default); few threads are needed there since triggering a reconciliation only en for one of the reconciliation threads to pick up. Use `ConfigurationServiceOverrider.withScheduledExecutorService` to replace the polling executor -altogether, or the `withExecutorService` method of the event source's own configuration builder to -poll a single event source on an executor of its own. An executor provided that way is not managed -by the operator: it is your responsibility to shut it down. +altogether. The operator still manages the lifecycle of that executor: it shuts it down when the +operator is stopped, so don't shut it down yourself. + +Use the `withExecutorService` method of the event source's own configuration builder to poll a +single event source on an executor of its own. An executor provided that way is not managed by the +operator: it is your responsibility to shut it down. #### Inbound event sources diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 0e02482d9e..7b9f1b5fba 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -98,6 +98,9 @@ private static ConfigurationService configurationServiceOf(EventSourceContext @Override public void start() throws OperatorException { + if (isRunning()) { + return; + } executorService = resolveExecutorService(); super.start(); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index b4d7aa36a3..f9207c6e1b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -19,9 +19,11 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import org.slf4j.Logger; @@ -100,14 +102,30 @@ public void scheduleOnce(ResourceID resourceID, long delay) { return; } - onceTasks.compute( - resourceID, - (id, alreadyScheduled) -> { - if (alreadyScheduled != null) { - alreadyScheduled.cancel(false); - } - return executor.schedule(new EventProducerTimeTask(id), delay, TimeUnit.MILLISECONDS); - }); + var scheduled = new AtomicReference>(); + try { + onceTasks.compute( + resourceID, + (id, alreadyScheduled) -> { + if (alreadyScheduled != null) { + alreadyScheduled.cancel(false); + } + var future = + executor.schedule(new EventProducerTimeTask(id), delay, TimeUnit.MILLISECONDS); + scheduled.set(future); + return future; + }); + } catch (RejectedExecutionException e) { + if (isRunning()) { + throw e; + } + throw new IllegalStateException("The TimerEventSource is not running", e); + } + // stop() clears the running flag before cancelling the tasks, so a task that got registered + // concurrently with it is either cancelled by stop, or withdrawn here + if (!isRunning() && onceTasks.remove(resourceID, scheduled.get())) { + scheduled.get().cancel(false); + } } @Override @@ -147,8 +165,8 @@ private ScheduledExecutorService resolveExecutorService() { @Override public void stop() { if (isRunning()) { - onceTasks.keySet().forEach(this::cancelOnceSchedule); super.stop(); + onceTasks.keySet().forEach(this::cancelOnceSchedule); if (ownsExecutorService && executorService != null) { executorService.shutdownNow(); } From df3de17c53f08bc56d695a090fa453c8f101b280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Sun, 25 Oct 2026 19:43:55 +0100 Subject: [PATCH 05/45] fix: keep EventProducerTimeTask a TimerTask and cancel a poll scheduled after stop during start --- .../event/source/polling/PollingEventSource.java | 9 ++++++++- .../processing/event/source/timer/TimerEventSource.java | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index 8e615a6059..d25caee74c 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -136,9 +136,16 @@ public void start() throws OperatorException { super.start(); executorService = resolveExecutorService(); getStateAndFillCache(); - pollingTask = + var task = executorService.scheduleWithFixedDelay( this::poll, period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS); + pollingTask = task; + // stop() clears the running flag before it looks for the task, so a stop that happened during + // the initial fetch is either seen by stop (which cancels the task) or caught here + if (!isRunning()) { + task.cancel(false); + pollingTask = null; + } } private void poll() { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index f9207c6e1b..e8a0e351d0 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -17,6 +17,7 @@ import java.util.Map; import java.util.Set; +import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; @@ -183,7 +184,7 @@ public Set getSecondaryResources(HasMetadata primary) { return Set.of(); } - public class EventProducerTimeTask implements Runnable { + public class EventProducerTimeTask extends TimerTask { protected final ResourceID customResourceUid; From a706ff45358f3064863f82b1342839bac0d62fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 7 Sep 2026 15:57:55 +0200 Subject: [PATCH 06/45] feat: virtual thread support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../documentation/operations/configuration.md | 29 +++ .../api/config/ConfigurationService.java | 34 +++- .../config/ConfigurationServiceOverrider.java | 19 ++ .../api/config/ExecutorServiceManager.java | 34 ++++ .../operator/api/config/VirtualThreads.java | 178 ++++++++++++++++++ .../ConfigurationServiceOverriderTest.java | 7 + .../api/config/VirtualThreadsTest.java | 162 ++++++++++++++++ .../operator/config/loader/ConfigLoader.java | 4 + .../VirtualThreadsCustomResource.java | 30 +++ .../virtualthreads/VirtualThreadsIT.java | 92 +++++++++ .../VirtualThreadsTestReconciler.java | 84 +++++++++ .../config/loader/ConfigLoaderTest.java | 2 + 12 files changed, 673 insertions(+), 2 deletions(-) create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsCustomResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsTestReconciler.java diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index 959e1728cd..f3e3c4f4b6 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -23,6 +23,34 @@ Operator operator = new Operator( override -> override .withLeaderElectionConfiguration(new LeaderElectionConfiguration("bar", "barNS"))); ``` +### Virtual Threads + +Reconciliation is mostly about blocking: talking to the Kubernetes API server or to external +systems. Virtual threads make such blocking calls much cheaper than platform threads, and the +framework can be switched over to them with a single flag: + +```java +Operator operator = new Operator(override -> override.withUseVirtualThreads(true)); +``` + +When enabled, reconciliations, dependent resource workflows and the framework's internal +housekeeping (starting the informers, for example) all run on virtual threads. + +Enabling virtual threads does **not** remove the concurrency limits, parallelism is configured +exactly as before: `withConcurrentReconciliationThreads(int)` still caps how many reconciliations +run at the same time and `withConcurrentWorkflowExecutorThreads(int)` how many dependent resources +of a workflow are processed concurrently. Only the threads backing those limits change. Since +virtual threads are cheap, these limits can usually be raised significantly compared to what is +reasonable with platform threads. + +Two things to keep in mind: + +- Virtual threads require Java 21 or later at runtime. When the flag is set on an older JVM, a + warning is logged and platform threads are used instead, so the same configuration works on any + supported Java version. +- A custom `ExecutorService` provided through `withExecutorService(...)` or + `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. + ## Reconciler-Level Configuration While reconcilers are typically configured using the `@ControllerConfiguration` annotation, you can also override configuration at runtime when registering the reconciler with the operator. You can either: @@ -285,6 +313,7 @@ All operator-level keys are prefixed with `josdk.`. |---|---|---| | `josdk.check-crd` | `Boolean` | Validate CRDs against local model on startup | | `josdk.close-client-on-stop` | `Boolean` | Close the Kubernetes client when the operator stops | +| `josdk.use-virtual-threads` | `Boolean` | Run the framework's concurrent work on virtual threads (requires Java 21+ at runtime) | | `josdk.use-ssa-to-patch-primary-resource` | `Boolean` | Use Server-Side Apply to patch the primary resource | | `josdk.clone-secondary-resources-when-getting-from-cache` | `Boolean` | Clone secondary resources on cache reads | diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index bf972bfcc7..7e77975f0e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -270,6 +270,34 @@ default Metrics getMetrics() { return Metrics.NOOP; } + /** + * Whether the framework should run the tasks it executes concurrently — reconciliations, + * dependent workflows and internal housekeeping such as starting the informers — on virtual + * threads instead of platform threads. + * + *

Virtual threads make blocking operations, which is essentially all a reconciler does while + * talking to the Kubernetes API server or to external systems, much cheaper. Enabling them does + * not lift the configured concurrency limits: {@link #concurrentReconciliationThreads()} + * and {@link #concurrentWorkflowExecutorThreads()} still cap how many reconciliations, + * respectively dependent resources, are processed at the same time, they just aren't backed by a + * pool of platform threads anymore. Since virtual threads are cheap, those limits can be set + * considerably higher than what would be reasonable for platform threads. + * + *

Requires Java 21 or later at runtime. When enabled on an older JVM, a warning is logged and + * platform threads are used, so that the same configuration works regardless of the Java version + * the operator runs on. + * + *

Note that this only affects the executors created by the framework: a custom {@link + * ExecutorService} provided through {@link #getExecutorService()} or {@link + * #getWorkflowExecutorService()} is used as is. + * + * @return {@code true} to use virtual threads, {@code false} (default) to use platform threads + * @since 5.7.0 + */ + default boolean useVirtualThreads() { + return false; + } + /** * Override to provide a custom {@link ExecutorService} implementation to change how threads * handle concurrent reconciliations @@ -278,7 +306,8 @@ default Metrics getMetrics() { * processing */ default ExecutorService getExecutorService() { - return Executors.newFixedThreadPool(concurrentReconciliationThreads()); + return ExecutorServiceManager.newBoundedExecutorService( + concurrentReconciliationThreads(), useVirtualThreads()); } /** @@ -288,7 +317,8 @@ default ExecutorService getExecutorService() { * @return the {@link ExecutorService} implementation to use for dependent workflow processing */ default ExecutorService getWorkflowExecutorService() { - return Executors.newFixedThreadPool(concurrentWorkflowExecutorThreads()); + return ExecutorServiceManager.newBoundedExecutorService( + concurrentWorkflowExecutorThreads(), useVirtualThreads()); } /** diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index 9ac8668b20..e747570b96 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -53,6 +53,7 @@ public class ConfigurationServiceOverrider { private KubernetesClient client; private ExecutorService executorService; private ExecutorService workflowExecutorService; + private Boolean useVirtualThreads; private ScheduledExecutorService scheduledExecutorService; private ScheduledExecutorService retryAndRescheduleExecutorService; private LeaderElectionConfiguration leaderElectionConfiguration; @@ -183,6 +184,19 @@ public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( return this; } + /** + * Makes the framework run the tasks it executes concurrently on virtual threads instead of + * platform threads. Requires Java 21 or later at runtime, see {@link + * ConfigurationService#useVirtualThreads()} for the details. + * + * @param useVirtualThreads {@code true} to use virtual threads + * @return this {@link ConfigurationServiceOverrider} for chained customization + */ + public ConfigurationServiceOverrider withUseVirtualThreads(boolean useVirtualThreads) { + this.useVirtualThreads = useVirtualThreads; + return this; + } + /** * Replaces the default {@link KubernetesClient} instance by the specified one. This is the * preferred mechanism to configure which client will be used to access the cluster. @@ -408,6 +422,11 @@ public boolean closeClientOnStop() { return overriddenValueOrDefault(closeClientOnStop, ConfigurationService::closeClientOnStop); } + @Override + public boolean useVirtualThreads() { + return overriddenValueOrDefault(useVirtualThreads, ConfigurationService::useVirtualThreads); + } + @Override public ExecutorService getExecutorService() { if (executorService != null) { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index 437bbfeae9..c063dc21ec 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -50,6 +50,37 @@ public class ExecutorServiceManager { start(configurationService); } + /** + * Creates the executor service used to run a bounded number of tasks concurrently, either backed + * by virtual threads or by a fixed size pool of platform threads. The concurrency limit is + * enforced in both cases. + * + * @param maxConcurrency the maximal number of tasks executed at the same time + * @param useVirtualThreads whether virtual threads should be used, see {@link + * ConfigurationService#useVirtualThreads()} + * @return the created {@link ExecutorService} + */ + public static ExecutorService newBoundedExecutorService( + int maxConcurrency, boolean useVirtualThreads) { + return VirtualThreads.shouldUse(useVirtualThreads) + ? VirtualThreads.newBoundedVirtualThreadExecutor(maxConcurrency) + : Executors.newFixedThreadPool(maxConcurrency); + } + + /** + * Creates the executor service used to run an unbounded number of tasks concurrently, either + * backed by virtual threads or by a cached pool of platform threads. + * + * @param useVirtualThreads whether virtual threads should be used, see {@link + * ConfigurationService#useVirtualThreads()} + * @return the created {@link ExecutorService} + */ + public static ExecutorService newUnboundedExecutorService(boolean useVirtualThreads) { + return VirtualThreads.shouldUse(useVirtualThreads) + ? VirtualThreads.newVirtualThreadPerTaskExecutor() + : Executors.newCachedThreadPool(); + } + /** * Uses cachingExecutorService from this manager. Use this only for tasks, that don't have dynamic * nature, in sense that won't grow with the number of inputs (thus kubernetes resources) @@ -156,6 +187,9 @@ public ScheduledExecutorService retryAndRescheduleExecutorService() { public synchronized void start(ConfigurationService configurationService) { if (!started) { this.configurationService = configurationService; // used to lazy init workflow executor + this.cachingExecutorService = + newUnboundedExecutorService(configurationService.useVirtualThreads()); + this.scheduledExecutorService = Executors.newScheduledThreadPool(0); this.cachingExecutorService = Executors.newCachedThreadPool(); this.scheduledExecutorService = configurationService.getScheduledExecutorService(); this.retryAndRescheduleExecutorService = diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java new file mode 100644 index 0000000000..dd7a8a80a0 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -0,0 +1,178 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.config; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javaoperatorsdk.operator.OperatorException; + +/** + * Creates the virtual thread based executors used when {@link + * ConfigurationService#useVirtualThreads()} is enabled. + * + *

The SDK is compiled for Java 17, in which virtual threads don't exist yet, so {@code + * Executors.newVirtualThreadPerTaskExecutor()} is looked up reflectively and is only available when + * the operator actually runs on Java 21 or later. + */ +final class VirtualThreads { + + private static final Logger log = LoggerFactory.getLogger(VirtualThreads.class); + + private static final MethodHandle NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR = lookupFactoryMethod(); + private static final AtomicBoolean UNSUPPORTED_WARNING_LOGGED = new AtomicBoolean(); + + private VirtualThreads() {} + + private static MethodHandle lookupFactoryMethod() { + try { + return MethodHandles.publicLookup() + .findStatic( + Executors.class, + "newVirtualThreadPerTaskExecutor", + MethodType.methodType(ExecutorService.class)); + } catch (NoSuchMethodException | IllegalAccessException e) { + log.debug("Virtual threads are not available on this JVM", e); + return null; + } + } + + /** Whether the JVM the operator runs on supports virtual threads, i.e. is Java 21 or later. */ + static boolean isSupported() { + return NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR != null; + } + + /** + * Whether virtual threads should effectively be used, i.e. they were requested through {@link + * ConfigurationService#useVirtualThreads()} and the JVM supports them. Requesting them + * on a JVM that doesn't support them is only warned about, so that the same configuration can be + * used regardless of the Java version the operator ends up running on, the only consequence being + * that platform threads are used instead. Concurrency limits are enforced either way. + */ + static boolean shouldUse(boolean requested) { + if (!requested || isSupported()) { + return requested; + } + if (UNSUPPORTED_WARNING_LOGGED.compareAndSet(false, true)) { + log.warn( + "Virtual threads were requested but are not supported by the JVM in use (Java {}, Java 21" + + " or later is required). Falling back to platform threads.", + Runtime.version().feature()); + } + return false; + } + + /** An unbounded executor starting a new virtual thread for each submitted task. */ + static ExecutorService newVirtualThreadPerTaskExecutor() { + if (!isSupported()) { + throw new OperatorException( + "Virtual threads are not supported by the JVM in use, Java 21 or later is required"); + } + try { + return (ExecutorService) NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR.invokeExact(); + } catch (Throwable e) { + throw new OperatorException("Couldn't create a virtual thread per task executor", e); + } + } + + /** + * A virtual thread based executor executing at most {@code maxConcurrency} tasks at the same + * time, the equivalent of a fixed size platform thread pool. + */ + static ExecutorService newBoundedVirtualThreadExecutor(int maxConcurrency) { + return new BoundedExecutorService(newVirtualThreadPerTaskExecutor(), maxConcurrency); + } + + /** + * Limits how many of the tasks submitted to the wrapped executor run at the same time. + * + *

A thread is started for each task as soon as it is submitted, the task then waits for a + * permit before it actually runs. This only makes sense with virtual threads, which are cheap + * enough to be parked in large numbers, and has the property that submitting a task never blocks + * the submitting thread, just like queuing it on a fixed size platform thread pool wouldn't. + */ + private static final class BoundedExecutorService extends AbstractExecutorService { + + private final ExecutorService delegate; + private final Semaphore permits; + + private BoundedExecutorService(ExecutorService delegate, int maxConcurrency) { + this.delegate = delegate; + // fair, so that tasks run roughly in submission order as they would on a thread pool + this.permits = new Semaphore(maxConcurrency, true); + } + + @Override + public void execute(Runnable command) { + delegate.execute( + () -> { + try { + permits.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // shutdownNow interrupted us before the task even started: cancel it so that whoever + // waits on the associated future isn't left hanging + if (command instanceof Future) { + ((Future) command).cancel(false); + } + return; + } + try { + command.run(); + } finally { + permits.release(); + } + }); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java index aebd7bc9b0..7acf3fff84 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java @@ -211,6 +211,13 @@ void threadCountConfiguredProperly() { .isEqualTo(14); } + @Test + void virtualThreadsAreDisabledByDefaultAndCanBeOverridden() { + assertThat(config.useVirtualThreads()).isFalse(); + assertThat(new ConfigurationServiceOverrider(config).withUseVirtualThreads(true).build()) + .returns(true, ConfigurationService::useVirtualThreads); + } + @SuppressWarnings("rawtypes") @Test void dependentResourceFactoryDefaultsToTheSharedOneAndCanBeOverridden() { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java new file mode 100644 index 0000000000..60d7e26f54 --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java @@ -0,0 +1,162 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.config; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +import static org.assertj.core.api.Assertions.assertThat; + +class VirtualThreadsTest { + + private static final int MAX_CONCURRENCY = 3; + private static final int TASK_NUMBER = 20; + private static final int TIMEOUT_SECONDS = 30; + + @Test + void usesPlatformThreadPoolWhenVirtualThreadsAreNotRequested() throws Exception { + var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, false); + try { + assertThat(executor).isInstanceOf(ThreadPoolExecutor.class); + assertThat(executor.submit(VirtualThreadsTest::onVirtualThread).get()).isFalse(); + } finally { + executor.shutdownNow(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void usesVirtualThreadsWhenRequested() throws Exception { + var virtualExecutor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); + try { + assertThat(virtualExecutor.submit(VirtualThreadsTest::onVirtualThread).get()).isTrue(); + } finally { + virtualExecutor.shutdownNow(); + } + + var unbounded = ExecutorServiceManager.newUnboundedExecutorService(true); + try { + assertThat(unbounded.submit(VirtualThreadsTest::onVirtualThread).get()).isTrue(); + } finally { + unbounded.shutdownNow(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void boundedVirtualThreadExecutorRespectsTheConfiguredConcurrency() throws Exception { + var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); + try { + final var running = new AtomicInteger(); + final var maxObservedConcurrency = new AtomicInteger(); + final var done = new CountDownLatch(TASK_NUMBER); + // each task gets its own (virtual) thread, only the number of concurrently running ones is + // capped + final Set usedThreads = ConcurrentHashMap.newKeySet(); + + IntStream.range(0, TASK_NUMBER) + .forEach( + i -> + executor.execute( + () -> { + usedThreads.add(Thread.currentThread()); + maxObservedConcurrency.accumulateAndGet( + running.incrementAndGet(), Math::max); + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + running.decrementAndGet(); + done.countDown(); + } + })); + + assertThat(done.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)).isTrue(); + assertThat(maxObservedConcurrency).hasValue(MAX_CONCURRENCY); + assertThat(usedThreads).hasSize(TASK_NUMBER); + } finally { + executor.shutdownNow(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void invokeAllIsBoundedTooSinceItIsUsedToStartTheEventSources() { + var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); + try { + final var running = new AtomicInteger(); + final var maxObservedConcurrency = new AtomicInteger(); + + ExecutorServiceManager.executeAndWaitForAllToComplete( + IntStream.range(0, TASK_NUMBER).boxed(), + i -> { + maxObservedConcurrency.accumulateAndGet(running.incrementAndGet(), Math::max); + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + running.decrementAndGet(); + } + return null; + }, + i -> "task-" + i, + executor); + + assertThat(maxObservedConcurrency).hasValue(MAX_CONCURRENCY); + } finally { + executor.shutdownNow(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void shutdownTerminatesOnceTheAlreadySubmittedTasksAreDone() throws Exception { + ExecutorService executor = + ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); + final var done = new CountDownLatch(TASK_NUMBER); + + IntStream.range(0, TASK_NUMBER).forEach(i -> executor.execute(done::countDown)); + executor.shutdown(); + + assertThat(executor.awaitTermination(TIMEOUT_SECONDS, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.isShutdown()).isTrue(); + assertThat(done.getCount()).isZero(); + } + + /** + * {@code Thread.isVirtual} only exists as of Java 21 while the tests are compiled for Java 17, + * hence the reflective call. + */ + static boolean onVirtualThread() { + try { + return (Boolean) Thread.class.getMethod("isVirtual").invoke(Thread.currentThread()); + } catch (ReflectiveOperationException e) { + return false; + } + } +} diff --git a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java index 8837e9e797..35e0bda882 100644 --- a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java +++ b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java @@ -94,6 +94,10 @@ public static ConfigLoader getDefault() { "close-client-on-stop", Boolean.class, ConfigurationServiceOverrider::withCloseClientOnStop), + new ConfigBinding<>( + "use-virtual-threads", + Boolean.class, + ConfigurationServiceOverrider::withUseVirtualThreads), new ConfigBinding<>( "informer.stop-on-error-during-startup", Boolean.class, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsCustomResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsCustomResource.java new file mode 100644 index 0000000000..be086a5f1b --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsCustomResource.java @@ -0,0 +1,30 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.virtualthreads; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Kind; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@Kind("VirtualThreadsCustomResource") +@ShortNames("vtc") +public class VirtualThreadsCustomResource extends CustomResource + implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java new file mode 100644 index 0000000000..fbb38042aa --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java @@ -0,0 +1,92 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.virtualthreads; + +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.javaoperatorsdk.annotation.Sample; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; + +import static io.javaoperatorsdk.operator.baseapi.virtualthreads.VirtualThreadsTestReconciler.onVirtualThread; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +@Sample( + tldr = "Running reconciliations on virtual threads", + description = + """ + Demonstrates how to make the framework execute its concurrent work on virtual threads by \ + simply setting a flag on the ConfigurationService. Virtual threads make the blocking calls \ + a reconciler typically performs much cheaper, while the configured concurrency limits are \ + still enforced: the test verifies that the reconciler runs on virtual threads and that no \ + more than the configured number of reconciliations happen at the same time. + """) +class VirtualThreadsIT { + + static final int CONCURRENT_RECONCILIATION_THREADS = 2; + static final int NUMBER_OF_RESOURCES = 10; + + /** + * Virtual threads require Java 21, on an older JVM the framework transparently falls back to + * platform threads, which this test also covers since only the concurrency assertions apply then. + */ + private static final boolean VIRTUAL_THREADS_SUPPORTED = Runtime.version().feature() >= 21; + + @RegisterExtension + LocallyRunOperatorExtension operator = + LocallyRunOperatorExtension.builder() + .withConfigurationService( + o -> + o.withUseVirtualThreads(true) + .withConcurrentReconciliationThreads(CONCURRENT_RECONCILIATION_THREADS)) + .withReconciler(new VirtualThreadsTestReconciler()) + .build(); + + @Test + void reconciliationsRunOnVirtualThreadsWithinTheConfiguredConcurrency() { + // the test itself runs on a platform thread, so the reconciler assertion below can only pass + // if the framework actually switched to virtual threads + assertThat(onVirtualThread()).isFalse(); + + IntStream.range(0, NUMBER_OF_RESOURCES).forEach(i -> operator.create(testResource(i))); + + var reconciler = operator.getReconcilerOfType(VirtualThreadsTestReconciler.class); + await() + .atMost(2, TimeUnit.MINUTES) + .untilAsserted( + () -> + assertThat(reconciler.getNumberOfExecutions()) + .isGreaterThanOrEqualTo(NUMBER_OF_RESOURCES)); + + // parallelism is retained: reconciliations do happen concurrently, but never more than the + // configured number of them + assertThat(reconciler.getMaxConcurrentReconciliations()) + .isEqualTo(CONCURRENT_RECONCILIATION_THREADS); + assertThat(reconciler.allExecutionsOnVirtualThreads()).isEqualTo(VIRTUAL_THREADS_SUPPORTED); + } + + private VirtualThreadsCustomResource testResource(int index) { + var resource = new VirtualThreadsCustomResource(); + resource.setMetadata(new ObjectMeta()); + resource.getMetadata().setName("virtual-threads-test-" + index); + return resource; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsTestReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsTestReconciler.java new file mode 100644 index 0000000000..ee6f00f499 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsTestReconciler.java @@ -0,0 +1,84 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.virtualthreads; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.support.TestExecutionInfoProvider; + +/** + * Blocks for a while during reconciliation, recording on what kind of thread it ran and how many + * reconciliations were in flight at the same time. + */ +@ControllerConfiguration +public class VirtualThreadsTestReconciler + implements Reconciler, TestExecutionInfoProvider { + + public static final Duration RECONCILIATION_DURATION = Duration.ofMillis(300); + + private final AtomicInteger numberOfExecutions = new AtomicInteger(); + private final AtomicInteger runningReconciliations = new AtomicInteger(); + private final AtomicInteger maxConcurrentReconciliations = new AtomicInteger(); + private final AtomicBoolean allExecutionsOnVirtualThreads = new AtomicBoolean(true); + + @Override + public UpdateControl reconcile( + VirtualThreadsCustomResource resource, Context context) + throws InterruptedException { + if (!onVirtualThread()) { + allExecutionsOnVirtualThreads.set(false); + } + maxConcurrentReconciliations.accumulateAndGet( + runningReconciliations.incrementAndGet(), Math::max); + try { + Thread.sleep(RECONCILIATION_DURATION.toMillis()); + } finally { + runningReconciliations.decrementAndGet(); + numberOfExecutions.incrementAndGet(); + } + return UpdateControl.noUpdate(); + } + + public int getNumberOfExecutions() { + return numberOfExecutions.get(); + } + + public int getMaxConcurrentReconciliations() { + return maxConcurrentReconciliations.get(); + } + + public boolean allExecutionsOnVirtualThreads() { + return allExecutionsOnVirtualThreads.get(); + } + + /** + * {@code Thread.isVirtual} only exists as of Java 21 while the tests are compiled for Java 17, + * hence the reflective call. + */ + static boolean onVirtualThread() { + try { + return (Boolean) Thread.class.getMethod("isVirtual").invoke(Thread.currentThread()); + } catch (ReflectiveOperationException e) { + return false; + } + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java index e22794e2b5..3a5887d572 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java @@ -119,6 +119,7 @@ void applyConfigsAppliesBooleanFlags() { values.put("josdk.dependent-resources.ssa-based-create-update-match", false); values.put("josdk.use-ssa-to-patch-primary-resource", false); values.put("josdk.clone-secondary-resources-when-getting-from-cache", true); + values.put("josdk.use-virtual-threads", true); var loader = new ConfigLoader(mapProvider(values)); var base = new BaseConfigurationService(null); @@ -131,6 +132,7 @@ void applyConfigsAppliesBooleanFlags() { assertThat(result.ssaBasedCreateUpdateMatchForDependentResources()).isFalse(); assertThat(result.useSSAToPatchPrimaryResource()).isFalse(); assertThat(result.cloneSecondaryResourcesWhenGettingFromCache()).isTrue(); + assertThat(result.useVirtualThreads()).isTrue(); } @Test From d32c23de6060ddac41df7bd5b72f3a1a050d45ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 17 Sep 2026 08:50:05 +0200 Subject: [PATCH 07/45] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/api/config/ExecutorServiceManager.java | 2 ++ .../operator/api/config/VirtualThreads.java | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index c063dc21ec..a1937c9ab9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -189,6 +189,8 @@ public synchronized void start(ConfigurationService configurationService) { this.configurationService = configurationService; // used to lazy init workflow executor this.cachingExecutorService = newUnboundedExecutorService(configurationService.useVirtualThreads()); + // stays on platform threads even when virtual threads are requested: there is no virtual + // thread backed ScheduledExecutorService in the JDK, see VirtualThreads this.scheduledExecutorService = Executors.newScheduledThreadPool(0); this.cachingExecutorService = Executors.newCachedThreadPool(); this.scheduledExecutorService = configurationService.getScheduledExecutorService(); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index dd7a8a80a0..d80d3928b8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -39,6 +39,14 @@ *

The SDK is compiled for Java 17, in which virtual threads don't exist yet, so {@code * Executors.newVirtualThreadPerTaskExecutor()} is looked up reflectively and is only available when * the operator actually runs on Java 21 or later. + * + *

There is intentionally no scheduled variant here: the JDK provides no virtual thread backed + * {@link java.util.concurrent.ScheduledExecutorService}, scheduling was deliberately left out of + * virtual threads (the Loom runtime itself uses a platform thread scheduler to unpark timed out + * virtual threads). Running scheduled tasks on virtual threads would mean keeping a platform thread + * scheduler purely for the timing and handing each fired task off to a virtual thread executor, + * which the SDK doesn't do, so {@link ExecutorServiceManager#scheduledExecutorService()} is always + * backed by platform threads. */ final class VirtualThreads { From 6f2e1cebe4197c4724f84a1bd3ac6a0ba76bf302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 17 Sep 2026 09:20:50 +0200 Subject: [PATCH 08/45] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/api/config/ConfigurationService.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 7e77975f0e..1faed2be7b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -218,6 +218,11 @@ default boolean checkCRDAndValidateLocalModel() { * The number of threads the operator can spin out to dispatch reconciliation requests to * reconcilers with the default executors * + *

This is a concurrency limit and applies regardless of whether the default executor is backed + * by platform or by virtual threads, see {@link #useVirtualThreads()}: with virtual threads it + * caps how many reconciliations run at the same time rather than the size of a thread pool. Since + * virtual threads are cheap, the limit can be set considerably higher when they are enabled. + * * @return the number of concurrent reconciliation threads */ default int concurrentReconciliationThreads() { @@ -228,6 +233,12 @@ default int concurrentReconciliationThreads() { * Number of threads the operator can spin out to be used in the workflows with the default * executor. * + *

This is a concurrency limit and applies regardless of whether the default executor is backed + * by platform or by virtual threads, see {@link #useVirtualThreads()}: with virtual threads it + * caps how many dependent resources are processed at the same time rather than the size of a + * thread pool. Since virtual threads are cheap, the limit can be set considerably higher when + * they are enabled. + * * @return the maximum number of concurrent workflow threads */ default int concurrentWorkflowExecutorThreads() { From 09f88aec72571b5f744bdc115580f44a19829153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 18 Sep 2026 14:45:45 +0200 Subject: [PATCH 09/45] remove unnecessary comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../io/javaoperatorsdk/operator/api/config/VirtualThreads.java | 1 - 1 file changed, 1 deletion(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index d80d3928b8..1d84c82e25 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -131,7 +131,6 @@ private static final class BoundedExecutorService extends AbstractExecutorServic private BoundedExecutorService(ExecutorService delegate, int maxConcurrency) { this.delegate = delegate; - // fair, so that tasks run roughly in submission order as they would on a thread pool this.permits = new Semaphore(maxConcurrency, true); } From 7deda64d6967aa93bb326901518b0e1ab73ae624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 21 Sep 2026 13:53:05 +0200 Subject: [PATCH 10/45] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../en/docs/documentation/operations/configuration.md | 8 ++++++-- .../operator/api/config/ConfigurationService.java | 8 +++++--- .../api/config/ConfigurationServiceOverrider.java | 2 +- .../operator/api/config/VirtualThreads.java | 6 ++++-- .../javaoperatorsdk/operator/sample/WebPageOperator.java | 4 +++- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index f3e3c4f4b6..f4745a495c 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -45,7 +45,11 @@ reasonable with platform threads. Two things to keep in mind: -- Virtual threads require Java 21 or later at runtime. When the flag is set on an older JVM, a +- Virtual threads are officially supported on **Java 25 or later**. They exist from Java 21 on and + the flag does enable them there, but before Java 25 a virtual thread pins its carrier thread + while it is inside a `synchronized` block, which can starve the carrier pool. [JEP + 491](https://openjdk.org/jeps/491), delivered in Java 25, removed that pinning, so this is the + baseline the framework supports. On a JVM without virtual threads at all (below Java 21) a warning is logged and platform threads are used instead, so the same configuration works on any supported Java version. - A custom `ExecutorService` provided through `withExecutorService(...)` or @@ -313,7 +317,7 @@ All operator-level keys are prefixed with `josdk.`. |---|---|---| | `josdk.check-crd` | `Boolean` | Validate CRDs against local model on startup | | `josdk.close-client-on-stop` | `Boolean` | Close the Kubernetes client when the operator stops | -| `josdk.use-virtual-threads` | `Boolean` | Run the framework's concurrent work on virtual threads (requires Java 21+ at runtime) | +| `josdk.use-virtual-threads` | `Boolean` | Run the framework's concurrent work on virtual threads (officially supported on Java 25+ at runtime) | | `josdk.use-ssa-to-patch-primary-resource` | `Boolean` | Use Server-Side Apply to patch the primary resource | | `josdk.clone-secondary-resources-when-getting-from-cache` | `Boolean` | Clone secondary resources on cache reads | diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 1faed2be7b..f10fc1f4cc 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -294,9 +294,11 @@ default Metrics getMetrics() { * pool of platform threads anymore. Since virtual threads are cheap, those limits can be set * considerably higher than what would be reasonable for platform threads. * - *

Requires Java 21 or later at runtime. When enabled on an older JVM, a warning is logged and - * platform threads are used, so that the same configuration works regardless of the Java version - * the operator runs on. + *

Officially supported on Java 25 or later. Virtual threads exist as of Java 21 and are used + * there as well, but before Java 25 a virtual thread pins its carrier thread while inside a + * {@code synchronized} block, which JEP 491 removed in + * Java 25. On a JVM without virtual threads at all, a warning is logged and platform threads are + * used, so that the same configuration works regardless of the Java version the operator runs on. * *

Note that this only affects the executors created by the framework: a custom {@link * ExecutorService} provided through {@link #getExecutorService()} or {@link diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index e747570b96..e222fed62e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -186,7 +186,7 @@ public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( /** * Makes the framework run the tasks it executes concurrently on virtual threads instead of - * platform threads. Requires Java 21 or later at runtime, see {@link + * platform threads. Officially supported on Java 25 or later, see {@link * ConfigurationService#useVirtualThreads()} for the details. * * @param useVirtualThreads {@code true} to use virtual threads diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index 1d84c82e25..ae76823fbc 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -89,7 +89,8 @@ static boolean shouldUse(boolean requested) { if (UNSUPPORTED_WARNING_LOGGED.compareAndSet(false, true)) { log.warn( "Virtual threads were requested but are not supported by the JVM in use (Java {}, Java 21" - + " or later is required). Falling back to platform threads.", + + " or later is required, Java 25 or later is officially supported). Falling back to" + + " platform threads.", Runtime.version().feature()); } return false; @@ -99,7 +100,8 @@ static boolean shouldUse(boolean requested) { static ExecutorService newVirtualThreadPerTaskExecutor() { if (!isSupported()) { throw new OperatorException( - "Virtual threads are not supported by the JVM in use, Java 21 or later is required"); + "Virtual threads are not supported by the JVM in use, Java 21 or later is required" + + " (Java 25 or later is officially supported)"); } try { return (ExecutorService) NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR.invokeExact(); diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index 5366dc2e9a..392172886f 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -40,7 +40,9 @@ public class WebPageOperator { public static void main(String[] args) throws IOException { log.info("WebServer Operator starting!"); - Operator operator = new Operator(o -> o.withStopOnInformerErrorDuringStartup(false)); + Operator operator = + new Operator( + o -> o.withStopOnInformerErrorDuringStartup(false).withUseVirtualThreads(true)); String reconcilerEnvVar = System.getenv(WEBPAGE_RECONCILER_ENV); if (WEBPAGE_CLASSIC_RECONCILER_ENV_VALUE.equals(reconcilerEnvVar)) { operator.register(new WebPageReconciler()); From f4b7e5e76cef8287baa16c14a40915bd3f8753d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 23 Sep 2026 13:33:51 +0200 Subject: [PATCH 11/45] feat: run default Kubernetes client tasks on virtual threads When useVirtualThreads() is enabled, the client created by the default ConfigurationService#getKubernetesClient() now uses a virtual thread task executor (informer event dispatching, watch event delivery) instead of fabric8's cached platform thread pool, shut down when the client is closed. User provided clients are used as is. --- .../documentation/operations/configuration.md | 9 +++++ .../api/config/ConfigurationService.java | 32 ++++++++++++----- .../operator/api/config/VirtualThreads.java | 22 ++++++++++++ .../api/config/VirtualThreadsTest.java | 35 +++++++++++++++++++ 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index f4745a495c..af3d39e815 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -54,6 +54,15 @@ Two things to keep in mind: supported Java version. - A custom `ExecutorService` provided through `withExecutorService(...)` or `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. +- The Kubernetes client the framework creates when none is provided also switches its internal + task executor (used to dispatch informer events to their handlers and to deliver watch events) + to virtual threads. A client you provide through `withKubernetesClient(...)` is used as is: to + get the same behavior, configure it yourself, e.g. with + `new KubernetesClientBuilder().withTaskExecutor(Executors.newVirtualThreadPerTaskExecutor())` + (the client doesn't shut down an executor passed that way, use `withTaskExecutorSupplier(...)` + if it should be shut down when the client is closed). + Either way, the blocking calls your reconciler makes through the client work well on virtual + threads without any client-side change. ## Reconciler-Level Configuration diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index f10fc1f4cc..c4fdb4ce0d 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -170,6 +170,12 @@ public R clone(R object) { * io.javaoperatorsdk.operator.Operator#Operator(Consumer)}, passing your custom instance with * {@link ConfigurationServiceOverrider#withKubernetesClient(KubernetesClient)}. * + *

When {@link #useVirtualThreads()} is enabled (and supported by the JVM), the default client + * runs its internal asynchronous tasks, such as dispatching informer events to their handlers and + * delivering watch events, on virtual threads instead of its default cached platform thread pool. + * A client provided by overriding this method or through {@link + * ConfigurationServiceOverrider#withKubernetesClient(KubernetesClient)} is used as is. + * *

NOTE: It is strongly suggested that implementors override this method since the * default implementation creates a new {@link KubernetesClient} instance each time this method is * called. @@ -178,13 +184,17 @@ public R clone(R object) { * @since 4.4.0 */ default KubernetesClient getKubernetesClient() { - return new KubernetesClientBuilder() - .withConfig( - new ConfigBuilder(Config.autoConfigure(null)) - .withMaxConcurrentRequests(DEFAULT_MAX_CONCURRENT_REQUEST) - .build()) - .withKubernetesSerialization(new KubernetesSerialization()) - .build(); + final var builder = + new KubernetesClientBuilder() + .withConfig( + new ConfigBuilder(Config.autoConfigure(null)) + .withMaxConcurrentRequests(DEFAULT_MAX_CONCURRENT_REQUEST) + .build()) + .withKubernetesSerialization(new KubernetesSerialization()); + if (VirtualThreads.shouldUse(useVirtualThreads())) { + builder.withTaskExecutorSupplier(VirtualThreads.newKubernetesClientTaskExecutorSupplier()); + } + return builder.build(); } /** @@ -302,7 +312,13 @@ default Metrics getMetrics() { * *

Note that this only affects the executors created by the framework: a custom {@link * ExecutorService} provided through {@link #getExecutorService()} or {@link - * #getWorkflowExecutorService()} is used as is. + * #getWorkflowExecutorService()} is used as is. The same goes for the {@link KubernetesClient}: + * the default client created by {@link #getKubernetesClient()} then also runs its internal + * asynchronous tasks (informer event dispatching, watch event delivery) on virtual threads, while + * a custom client, e.g. provided through {@link + * ConfigurationServiceOverrider#withKubernetesClient(KubernetesClient)}, keeps whatever task + * executor it was built with. The blocking calls a reconciler performs through the client don't + * need any of this: they already park the calling virtual thread rather than its carrier. * * @return {@code true} to use virtual threads, {@code false} (default) to use platform threads * @since 5.7.0 diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index ae76823fbc..7c7b8edc2b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -20,6 +20,7 @@ import java.lang.invoke.MethodType; import java.util.List; import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -30,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.kubernetes.client.KubernetesClientBuilder.ExecutorSupplier; import io.javaoperatorsdk.operator.OperatorException; /** @@ -110,6 +112,26 @@ static ExecutorService newVirtualThreadPerTaskExecutor() { } } + /** + * Supplies the task executor of the default {@link io.fabric8.kubernetes.client.KubernetesClient} + * created by {@link ConfigurationService#getKubernetesClient()}: an unbounded virtual thread + * executor, replacing the client's default cached platform thread pool, that is shut down when + * the client is closed. + */ + static ExecutorSupplier newKubernetesClientTaskExecutorSupplier() { + return new ExecutorSupplier() { + @Override + public Executor get() { + return newVirtualThreadPerTaskExecutor(); + } + + @Override + public void onClose(Executor executor) { + ((ExecutorService) executor).shutdownNow(); + } + }; + } + /** * A virtual thread based executor executing at most {@code maxConcurrency} tasks at the same * time, the equivalent of a fixed size platform thread pool. diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java index 60d7e26f54..c28be4b4fc 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java @@ -16,6 +16,7 @@ package io.javaoperatorsdk.operator.api.config; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -28,6 +29,9 @@ import org.junit.jupiter.api.condition.EnabledForJreRange; import org.junit.jupiter.api.condition.JRE; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.impl.BaseClient; + import static org.assertj.core.api.Assertions.assertThat; class VirtualThreadsTest { @@ -148,6 +152,37 @@ void shutdownTerminatesOnceTheAlreadySubmittedTasksAreDone() throws Exception { assertThat(done.getCount()).isZero(); } + @Test + void defaultKubernetesClientKeepsPlatformThreadsWhenVirtualThreadsAreNotRequested() + throws Exception { + try (var client = defaultKubernetesClient(false)) { + assertThat(runsTasksOnVirtualThread(client)).isFalse(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void defaultKubernetesClientRunsItsTasksOnVirtualThreadsWhenRequested() throws Exception { + ExecutorService executor; + try (var client = defaultKubernetesClient(true)) { + assertThat(runsTasksOnVirtualThread(client)).isTrue(); + executor = (ExecutorService) client.adapt(BaseClient.class).getExecutor(); + } + assertThat(executor.isShutdown()).isTrue(); + } + + private static KubernetesClient defaultKubernetesClient(boolean useVirtualThreads) { + return ConfigurationService.newOverriddenConfigurationService( + o -> o.withUseVirtualThreads(useVirtualThreads)) + .getKubernetesClient(); + } + + private static boolean runsTasksOnVirtualThread(KubernetesClient client) throws Exception { + return CompletableFuture.supplyAsync( + VirtualThreadsTest::onVirtualThread, client.adapt(BaseClient.class).getExecutor()) + .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + /** * {@code Thread.isVirtual} only exists as of Java 21 while the tests are compiled for Java 17, * hence the reflective call. From d1cad7a689143b0a83b4c32f3c4c139efccdba6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 26 Oct 2026 15:23:20 +0100 Subject: [PATCH 12/45] fix: correctness issues in bounded virtual thread executor, event recording and desired state aspects - bounded virtual thread executor rejects non-positive concurrency and returns tasks that never started from shutdownNow - event sink aggregation also updates type, reason and action - long supplied event names are shortened with a hash so they stay distinct - events for cluster scoped resources in an overridden namespace carry an eventTime - desired state aspects can look up the actual resource of their dependent, receive the user's bulk dependent, and tolerate a missing controller configuration --- .../api/config/ConfigurationService.java | 1 - .../operator/api/config/VirtualThreads.java | 81 +++++-- .../api/event/DefaultEventRecorder.java | 39 ++- .../operator/api/event/DefaultEventSink.java | 15 +- .../api/reconciler/DefaultContext.java | 35 ++- .../BulkDependentResourceReconciler.java | 10 + ...BoundedVirtualThreadExecutorAuditTest.java | 88 +++++++ .../api/event/DefaultEventRecorderTest.java | 30 ++- .../api/event/EventRecorderAuditTest.java | 213 +++++++++++++++++ .../DesiredStateAspectAuditTest.java | 222 ++++++++++++++++++ 10 files changed, 696 insertions(+), 38 deletions(-) create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index c4fdb4ce0d..2b5b3d6689 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -20,7 +20,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Consumer; diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index 7c7b8edc2b..ad7ff3a230 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -18,12 +18,15 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -137,6 +140,9 @@ public void onClose(Executor executor) { * time, the equivalent of a fixed size platform thread pool. */ static ExecutorService newBoundedVirtualThreadExecutor(int maxConcurrency) { + if (maxConcurrency <= 0) { + throw new IllegalArgumentException("maxConcurrency must be positive: " + maxConcurrency); + } return new BoundedExecutorService(newVirtualThreadPerTaskExecutor(), maxConcurrency); } @@ -152,6 +158,7 @@ private static final class BoundedExecutorService extends AbstractExecutorServic private final ExecutorService delegate; private final Semaphore permits; + private final Set pending = ConcurrentHashMap.newKeySet(); private BoundedExecutorService(ExecutorService delegate, int maxConcurrency) { this.delegate = delegate; @@ -160,25 +167,49 @@ private BoundedExecutorService(ExecutorService delegate, int maxConcurrency) { @Override public void execute(Runnable command) { - delegate.execute( - () -> { - try { - permits.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - // shutdownNow interrupted us before the task even started: cancel it so that whoever - // waits on the associated future isn't left hanging - if (command instanceof Future) { - ((Future) command).cancel(false); - } - return; - } - try { - command.run(); - } finally { - permits.release(); - } - }); + Objects.requireNonNull(command); + var task = new PendingTask(command); + pending.add(task); + try { + delegate.execute(task); + } catch (RuntimeException e) { + pending.remove(task); + throw e; + } + } + + private final class PendingTask implements Runnable { + + private final Runnable command; + private final AtomicBoolean claimed = new AtomicBoolean(); + + private PendingTask(Runnable command) { + this.command = command; + } + + private boolean claim() { + return claimed.compareAndSet(false, true); + } + + @Override + public void run() { + try { + permits.acquire(); + } catch (InterruptedException e) { + // shutdownNow interrupted us and already handed the task back to its caller + Thread.currentThread().interrupt(); + return; + } + try { + if (!claim()) { + return; + } + pending.remove(this); + command.run(); + } finally { + permits.release(); + } + } } @Override @@ -188,7 +219,15 @@ public void shutdown() { @Override public List shutdownNow() { - return delegate.shutdownNow(); + var notStarted = new ArrayList(); + for (var task : pending) { + if (task.claim()) { + pending.remove(task); + notStarted.add(task.command); + } + } + delegate.shutdownNow(); + return notStarted; } @Override diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java index 67764b7146..1d070d7e4e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java @@ -21,6 +21,8 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.HexFormat; import java.util.Objects; @@ -32,6 +34,7 @@ import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.api.model.EventBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.MicroTime; import io.fabric8.kubernetes.api.model.ObjectReference; import io.fabric8.kubernetes.api.model.ObjectReferenceBuilder; import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; @@ -75,6 +78,12 @@ public class DefaultEventRecorder implements EventRecorder { private static final int IDENTITY_HASH_LENGTH = 32; + /** + * The fixed width format of {@code MicroTime}, which the API server refuses to read otherwise. + */ + private static final DateTimeFormatter EVENT_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC); + /** What the API server accepts as an object name, see RFC 1123 on DNS subdomains. */ private static final Pattern RFC_1123_SUBDOMAIN = Pattern.compile("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*"); @@ -211,11 +220,12 @@ protected Event toEvent(Context context, EventRecord record) { var regarding = context.getPrimaryResource(); var now = Instant.now().truncatedTo(ChronoUnit.SECONDS).toString(); var involvedObject = objectReferenceFor(regarding); + var namespace = eventNamespace(regarding, context); var builder = new EventBuilder() .withNewMetadata() .withName(eventName(regarding, record, controllerName)) - .withNamespace(eventNamespace(regarding, context)) + .withNamespace(namespace) .withLabels(record.labels()) .withAnnotations(record.annotations()) .endMetadata() @@ -256,9 +266,26 @@ protected Event toEvent(Context context, EventRecord record) { .endMetadata(); } record.action().ifPresent(builder::withAction); + if (requiresEventTime(regarding, namespace)) { + builder.withEventTime(new MicroTime(EVENT_TIME_FORMAT.format(Instant.now()))); + if (record.action().isEmpty()) { + builder.withAction(record.reason()); + } + } return builder.build(); } + /** + * The API server only accepts an event without {@code eventTime} for a cluster scoped object if + * the event lives in the {@code default} namespace. Events that carry an {@code eventTime} are + * not subject to that restriction, but then have to set an {@code action} too. + */ + private static boolean requiresEventTime(HasMetadata regarding, String eventNamespace) { + return regarding.getMetadata().getNamespace() == null + && !CLUSTER_SCOPED_EVENT_NAMESPACE.equals(eventNamespace) + && !eventNamespace.isEmpty(); + } + private String eventNamespace(HasMetadata regarding, Context context) { var namespace = regarding.getMetadata().getNamespace(); if (namespace != null) { @@ -295,15 +322,17 @@ private static String truncateToMaxNameLength(String name) { if (name.length() <= MAX_NAME_LENGTH) { return name; } - var truncated = name.substring(0, MAX_NAME_LENGTH); + var suffix = "." + identityDigest(name); + var truncated = name.substring(0, MAX_NAME_LENGTH - suffix.length()); while (truncated.endsWith("-") || truncated.endsWith(".")) { truncated = truncated.substring(0, truncated.length() - 1); } + var shortened = truncated + suffix; log.warn( - "Truncated the name of event {} to {} to stay within the Kubernetes name limit", + "Shortened the name of event {} to {} to stay within the Kubernetes name limit", name, - truncated); - return truncated; + shortened); + return shortened; } /** diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java index c2f5461ae8..4ac42589c1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java @@ -30,8 +30,10 @@ *

An event that does not exist yet is created. An event that does is a repeat of one already * recorded, since the name of an event is derived from everything that identifies it (see {@link * DefaultEventRecorder}), and is aggregated onto the recorded one by patching its {@code count} and - * {@code lastTimestamp}. That is what makes {@code kubectl describe} report a repeating event once, - * as {@code (x12 over 3m)}, rather than filling the event list with copies of it. + * {@code lastTimestamp}, along with its {@code message}, {@code type}, {@code reason} and {@code + * action}, which events sharing a name explicitly may differ in. That is what makes {@code kubectl + * describe} report a repeating event once, as {@code (x12 over 3m)}, rather than filling the event + * list with copies of it. * *

Recording therefore needs {@code get}, {@code create} and {@code patch} on {@code events} in * the core API group. The patch is made on the resource version the event was read at, so two @@ -55,12 +57,17 @@ public void emit(Event event, Context context) { if (existing == null) { events.resource(event).create(); } else { - var aggregated = + var builder = new EventBuilder(existing) .withCount(requireNonNullElse(existing.getCount(), 1) + 1) .withLastTimestamp(event.getLastTimestamp()) .withMessage(event.getMessage()) - .build(); + .withType(event.getType()) + .withReason(event.getReason()); + if (event.getAction() != null) { + builder.withAction(event.getAction()); + } + var aggregated = builder.build(); events.resource(existing).patch(aggregated); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java index 50e61526c4..c5e3e9c4e8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java @@ -50,6 +50,8 @@ public class DefaultContext

implements Context

{ private final boolean primaryResourceDeleted; private final boolean primaryResourceFinalStateUnknown; private final Map, Object> desiredStates = new ConcurrentHashMap<>(); + private final Map, DesiredStateInProgress> desiredStatesInProgress = + new ConcurrentHashMap<>(); private final ResourceOperations

resourceOperations; public DefaultContext( @@ -254,15 +256,40 @@ public DefaultContext

setRetryInfo(RetryInfo retryInfo) { return this; } - @SuppressWarnings("unchecked") public R getOrComputeDesiredStateFor( DependentResource dependentResource, Function desiredStateComputer) { + return getOrComputeDesiredStateFor(dependentResource, dependentResource, desiredStateComputer); + } + + /** + * Same as {@link #getOrComputeDesiredStateFor(DependentResource, Function)} but hands {@code + * aspectTarget} rather than {@code dependentResource} to the {@link DesiredStateAspect}s, for + * dependents that are only a stand-in for the one the user configured. + */ + @SuppressWarnings("unchecked") + public R getOrComputeDesiredStateFor( + DependentResource dependentResource, + DependentResource aspectTarget, + Function desiredStateComputer) { + // an aspect can look up the actual resource of its dependent, which asks for the desired state + // being computed: the thread computing it is handed the state as it is so far, instead of + // re-entering the computation + final var computing = desiredStatesInProgress.get(dependentResource); + if (computing != null && computing.thread() == Thread.currentThread()) { + return (R) computing.desired(); + } return (R) desiredStates.computeIfAbsent( dependentResource, ignored -> { final var desired = desiredStateComputer.apply(getPrimaryResource()); - applyDesiredStateAspects(desired, dependentResource); + desiredStatesInProgress.put( + dependentResource, new DesiredStateInProgress(Thread.currentThread(), desired)); + try { + applyDesiredStateAspects(desired, aspectTarget); + } finally { + desiredStatesInProgress.remove(dependentResource); + } return desired; }); } @@ -273,11 +300,13 @@ public R getOrComputeDesiredStateFor( * resources are therefore left untouched. */ private void applyDesiredStateAspects(Object desired, DependentResource dependentResource) { - if (desired instanceof HasMetadata hasMetadata) { + if (desired instanceof HasMetadata hasMetadata && controllerConfiguration != null) { controllerConfiguration .getConfigurationService() .desiredStateAspects() .forEach(aspect -> aspect.apply(hasMetadata, dependentResource, this)); } } + + private record DesiredStateInProgress(Thread thread, Object desired) {} } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java index 2428eeba96..bfe46332c9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java @@ -24,8 +24,10 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.DefaultContext; import io.javaoperatorsdk.operator.api.reconciler.Ignore; import io.javaoperatorsdk.operator.api.reconciler.dependent.Deleter; +import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource; import io.javaoperatorsdk.operator.api.reconciler.dependent.ReconcileResult; class BulkDependentResourceReconciler @@ -115,6 +117,14 @@ public R update(R actual, R desired, P primary, Context

context) { return ((Updater) bulkDependentResource).update(actual, desired, primary, context); } + @Override + @SuppressWarnings("unchecked") + protected R getOrComputeDesired(Context

context) { + return ((DefaultContext

) context) + .getOrComputeDesiredStateFor( + this, (DependentResource) bulkDependentResource, p -> desired); + } + @Override public Result match(R resource, P primary, Context

context) { // retrieve the desired state via the context so that it is processed the same way as for diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java new file mode 100644 index 0000000000..18d0239b5c --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java @@ -0,0 +1,88 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.config; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +/** Throwaway audit tests for VirtualThreads.BoundedExecutorService. */ +class BoundedVirtualThreadExecutorAuditTest { + + @Test + void controlPlatformPoolRejectsNonPositiveConcurrency() { + assertThatIllegalArgumentException() + .isThrownBy(() -> ExecutorServiceManager.newBoundedExecutorService(0, false)); + } + + /** + * The platform variant fails fast for a non positive limit, the virtual variant creates a + * Semaphore(0) (or a negative one) whose tasks wait for a permit forever. + */ + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void virtualThreadVariantRejectsNonPositiveConcurrencyToo() { + assertThatIllegalArgumentException() + .isThrownBy(() -> ExecutorServiceManager.newBoundedExecutorService(0, true).shutdownNow()); + } + + @Test + void controlPlatformPoolShutdownNowReturnsTasksThatNeverStarted() throws Exception { + assertThat(queuedTasksReturnedByShutdownNow(false)).isEqualTo(5); + } + + /** + * ExecutorService#shutdownNow must return the tasks that never commenced execution; the bounded + * virtual executor returns an empty list, and plain Runnables handed to execute() that were still + * waiting for a permit are silently dropped (neither run nor returned). + */ + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void virtualThreadVariantShutdownNowReturnsTasksThatNeverStarted() throws Exception { + assertThat(queuedTasksReturnedByShutdownNow(true)).isEqualTo(5); + } + + private static int queuedTasksReturnedByShutdownNow(boolean useVirtualThreads) throws Exception { + final var executor = ExecutorServiceManager.newBoundedExecutorService(1, useVirtualThreads); + final var running = new CountDownLatch(1); + final var ran = new AtomicInteger(); + executor.execute( + () -> { + running.countDown(); + try { + Thread.sleep(60_000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + assertThat(running.await(10, TimeUnit.SECONDS)).isTrue(); + for (int i = 0; i < 5; i++) { + executor.execute(ran::incrementAndGet); + } + Thread.sleep(200); // let the virtual threads reach the permit acquisition + final var notStarted = executor.shutdownNow(); + assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + assertThat(ran).hasValue(0); + return notStarted.size(); + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java index 388217b9be..3cedc6cc9f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java @@ -436,19 +436,41 @@ void stripsTrailingSeparatorsFromTruncatedNames() { EventRecord.builder() .reason("Created") .message("created") - .name("a".repeat(252) + "." + "b".repeat(47)) + .name("a".repeat(219) + "." + "b".repeat(47)) .build(), context(configMap())); recorder.record( EventRecord.builder() .reason("Created") .message("created") - .name("b".repeat(252) + "-" + "c".repeat(47)) + .name("b".repeat(219) + "-" + "c".repeat(47)) .build(), context(configMap())); - assertThat(emitted.get(0).getMetadata().getName()).isEqualTo("a".repeat(252)); - assertThat(emitted.get(1).getMetadata().getName()).isEqualTo("b".repeat(252)); + assertThat(emitted.get(0).getMetadata().getName()).matches("a{219}\\.[0-9a-f]{32}"); + assertThat(emitted.get(1).getMetadata().getName()).matches("b{219}\\.[0-9a-f]{32}"); + } + + @Test + void keepsLongSuppliedNamesSharingTheirPrefixDistinct() { + recorder.record( + EventRecord.builder() + .reason("Created") + .message("created") + .name("a".repeat(300) + "-one") + .build(), + context(configMap())); + recorder.record( + EventRecord.builder() + .reason("Created") + .message("created") + .name("a".repeat(300) + "-two") + .build(), + context(configMap())); + + assertThat(emitted.get(0).getMetadata().getName()).hasSize(253); + assertThat(emitted.get(0).getMetadata().getName()) + .isNotEqualTo(emitted.get(1).getMetadata().getName()); } @Test diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java new file mode 100644 index 0000000000..bf2599de15 --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java @@ -0,0 +1,213 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.event; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.Event; +import io.fabric8.kubernetes.api.model.EventList; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.dsl.V1APIGroupDSL; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Context; + +import static io.javaoperatorsdk.operator.api.config.LeaderElectionConfigurationBuilder.aLeaderElectionConfiguration; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Audit repros for the configurable event recorder. Each test fails on the current code. */ +class EventRecorderAuditTest { + + private final List emitted = new ArrayList<>(); + + /** + * The core/v1 API server (legacyValidateEvent in pkg/apis/core/validation/events.go) rejects an + * event without eventTime whose involvedObject.namespace is empty unless the event itself lives + * in "" or "default". Overriding the cluster scoped event namespace, as + * ConfigurationService#clusterScopedEventNamespace recommends, therefore requires the event to + * carry an eventTime, and with it an action. + */ + @Test + void overriddenClusterScopedEventNamespaceProducesEventsTheCoreApiAccepts() { + var recorder = new DefaultEventRecorder((event, context) -> emitted.add(event)); + var namespace = + new NamespaceBuilder() + .withNewMetadata() + .withName("ns1") + .withUid("uid-2") + .endMetadata() + .build(); + + recorder.record(EventRecord.normal("Created", "created"), context(namespace, "operator-ns")); + + var event = emitted.get(0); + assertThat(event.getInvolvedObject().getNamespace()).isNull(); + assertThat(event.getMetadata().getNamespace()).isEqualTo("operator-ns"); + assertThat(event.getEventTime().getTime()) + .matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}Z"); + assertThat(event.getAction()).isEqualTo("Created"); + } + + @Test + void clusterScopedEventsInTheDefaultNamespaceDoNotCarryAnEventTime() { + var recorder = new DefaultEventRecorder((event, context) -> emitted.add(event)); + var namespace = + new NamespaceBuilder() + .withNewMetadata() + .withName("ns1") + .withUid("uid-2") + .endMetadata() + .build(); + + recorder.record(EventRecord.normal("Created", "created"), context(namespace, "default")); + + assertThat(emitted.get(0).getEventTime()).isNull(); + assertThat(emitted.get(0).getMetadata().getNamespace()).isEqualTo("default"); + } + + /** + * A naming strategy (here the one EventNamingStrategy's javadoc suggests, a fixed suffix per + * object) makes the name the aggregation identity on its own, so records with different type and + * reason resolve to the same event. DefaultEventSink then only patches count, lastTimestamp and + * message: the Warning is published under the type and reason of the earlier Normal event. + */ + @Test + void aggregatingOntoANamedEventKeepsTheTypeAndReasonOfTheFirstOccurrence() { + var stored = new AtomicReference(); + var patched = new AtomicReference(); + var sink = new DefaultEventSink(mockClient(stored, patched)); + var recorder = + DefaultEventRecorder.builder(sink) + .namingStrategy( + (regarding, record) -> + Optional.of(regarding.getMetadata().getName() + "-status-report")) + .build(); + var context = context(configMap("test1"), "default"); + + recorder.record(EventRecord.normal("Ready", "all good"), context); + recorder.record(EventRecord.warning("Degraded", "replica down"), context); + + assertThat(patched.get()).isNotNull(); + assertThat(patched.get().getCount()).isEqualTo(2); + assertThat(patched.get().getMessage()).isEqualTo("replica down"); + assertThat(patched.get().getType()).isEqualTo("Warning"); + assertThat(patched.get().getReason()).isEqualTo("Degraded"); + } + + /** + * Supplied names are truncated to 253 characters, which silently maps distinct names onto the + * same event, so two different events of a long-named object are aggregated as one. + */ + @Test + void truncatingSuppliedNamesMergesDistinctEvents() { + var recorder = + DefaultEventRecorder.builder((event, context) -> emitted.add(event)) + .namingStrategy( + (regarding, record) -> + Optional.of( + regarding.getMetadata().getName() + "-" + record.reason().toLowerCase())) + .build(); + var context = context(configMap("a".repeat(245)), "default"); + + recorder.record(EventRecord.warning("BackoffLimitExceeded", "job failed"), context); + recorder.record(EventRecord.warning("BackoffStarted", "job retrying"), context); + + assertThat(emitted.get(0).getMetadata().getName()) + .isNotEqualTo(emitted.get(1).getMetadata().getName()); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private KubernetesClient mockClient( + AtomicReference stored, AtomicReference patched) { + var client = mock(KubernetesClient.class); + var v1 = mock(V1APIGroupDSL.class); + MixedOperation> events = mock(MixedOperation.class); + NonNamespaceOperation> inNs = + mock(NonNamespaceOperation.class); + when(client.v1()).thenReturn(v1); + when(v1.events()).thenReturn(events); + when(events.inNamespace(anyString())).thenReturn(inNs); + Resource byName = mock(Resource.class); + when(inNs.withName(anyString())).thenReturn(byName); + when(byName.get()).thenAnswer(i -> stored.get()); + when(inNs.resource(any(Event.class))) + .thenAnswer( + i -> { + Event target = i.getArgument(0); + Resource r = mock(Resource.class); + when(r.create()) + .thenAnswer( + c -> { + stored.set(target); + return target; + }); + when(r.patch(any(Event.class))) + .thenAnswer( + p -> { + Event e = p.getArgument(0); + patched.set(e); + stored.set(e); + return e; + }); + return r; + }); + return client; + } + + @SuppressWarnings("rawtypes") + private Context context(HasMetadata primaryResource, String clusterScopedEventNamespace) { + var configurationService = mock(ConfigurationService.class); + when(configurationService.getLeaderElectionConfiguration()) + .thenReturn( + Optional.of(aLeaderElectionConfiguration("lease").withIdentity("pod-1").build())); + when(configurationService.clusterScopedEventNamespace()) + .thenReturn(clusterScopedEventNamespace); + ControllerConfiguration controllerConfiguration = mock(ControllerConfiguration.class); + when(controllerConfiguration.getName()).thenReturn("testcontroller"); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + Context context = mock(Context.class); + when(context.getPrimaryResource()).thenReturn(primaryResource); + when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); + return context; + } + + private ConfigMap configMap(String name) { + return new ConfigMapBuilder() + .withNewMetadata() + .withName(name) + .withNamespace("ns1") + .withUid("uid-1") + .withResourceVersion("42") + .endMetadata() + .build(); + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java new file mode 100644 index 0000000000..595327ec50 --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java @@ -0,0 +1,222 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.processing.dependent; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.DefaultContext; +import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource; +import io.javaoperatorsdk.operator.api.reconciler.dependent.DesiredStateAspect; +import io.javaoperatorsdk.operator.processing.Controller; +import io.javaoperatorsdk.operator.processing.dependent.Matcher.Result; +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource; +import io.javaoperatorsdk.operator.processing.event.EventSourceManager; +import io.javaoperatorsdk.operator.processing.event.source.EventSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Audit repro tests for the DesiredStateAspect feature (commit 110199c47). */ +@SuppressWarnings({"unchecked", "rawtypes"}) +class DesiredStateAspectAuditTest { + + private static final ConfigMap PRIMARY = + new ConfigMapBuilder() + .withNewMetadata() + .withName("primary") + .withNamespace("ns") + .endMetadata() + .build(); + + private static Controller controllerWith(List aspects) { + final ConfigurationService configurationService = mock(); + when(configurationService.desiredStateAspects()).thenReturn(aspects); + final ControllerConfiguration controllerConfiguration = mock(); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + final Controller controller = mock(); + when(controller.getConfiguration()).thenReturn(controllerConfiguration); + return controller; + } + + /** + * The aspect contract says it receives "the DependentResource the desired state was computed + * for". For bulk dependents, it instead receives the private per-item wrapper + * BulkDependentResourceReconciler.BulkDependentResourceInstance, so aspects keyed on the + * dependent (type checks, name(), getClass().getSimpleName() as in DesiredStateAspectIT) + * misbehave. + */ + @Test + void bulkDependentAspectReceivesTheUserBulkDependentResource() { + final List> seen = new ArrayList<>(); + final DesiredStateAspect aspect = + (desired, dependentResource, ctx) -> seen.add(dependentResource); + final var context = + new DefaultContext<>(null, controllerWith(List.of(aspect)), PRIMARY, false, false); + + final var bulk = new CreatingBulk(); + bulk.reconcile(PRIMARY, context); + + assertThat(seen).hasSize(1); + assertThat(seen.get(0).name()).isEqualTo(bulk.name()); + assertThat(seen.get(0)).isSameAs(bulk); + } + + /** + * Aspects run inside ConcurrentHashMap.computeIfAbsent. An aspect that looks up the actual + * resource of the dependent it is given (e.g. to preserve an existing label) re-enters + * getOrComputeDesired for the same key through KubernetesDependentResource + * .selectTargetSecondaryResource -> targetSecondaryResourceID, and CHM throws "Recursive update". + */ + @Test + void aspectCanLookUpActualResourceOfTheDependent() { + final DesiredStateAspect aspect = + (desired, dependentResource, ctx) -> + ((DependentResource) dependentResource) + .getSecondaryResource(ctx.getPrimaryResource(), ctx); + final var controller = controllerWith(List.of(aspect)); + final EventSourceManager esm = mock(); + final EventSource es = mock(); + when(es.getSecondaryResources(any())).thenReturn(Set.of(configMap("child", "actual"))); + when(esm.getEventSourcesFor(ConfigMap.class)).thenReturn(List.of(es)); + when(controller.getEventSourceManager()).thenReturn(esm); + final var context = new DefaultContext<>(null, controller, PRIMARY, false, false); + + final var dependent = new ChildConfigMapDependent(); + + assertThatCode(() -> dependent.getSecondaryResource(PRIMARY, context)) + .doesNotThrowAnyException(); + } + + /** + * Before 110199c47, a DefaultContext built on a Controller whose getConfiguration() is not + * stubbed (the pattern the existing tests used: new DefaultContext<>(mock(), mock(), ...)) could + * compute desired states. Now applyDesiredStateAspects dereferences the null + * ControllerConfiguration for every HasMetadata desired state. + */ + @Test + void desiredComputationDoesNotRequireAspectConfiguration() { + final var context = new DefaultContext<>(null, mock(Controller.class), PRIMARY, false, false); + final var dependent = new ChildConfigMapDependent(); + + assertThatCode(() -> dependent.getOrComputeDesired(context)).doesNotThrowAnyException(); + } + + private static ConfigMap configMap(String name, String value) { + return new ConfigMapBuilder() + .withNewMetadata() + .withName(name) + .withNamespace("ns") + .endMetadata() + .withData(Map.of("key", value)) + .build(); + } + + private static class ChildConfigMapDependent + extends KubernetesDependentResource { + ChildConfigMapDependent() { + super(ConfigMap.class); + } + + @Override + protected ConfigMap desired(ConfigMap primary, Context context) { + return configMap("child", "desired"); + } + + @Override + public ConfigMap getOrComputeDesired(Context context) { + return super.getOrComputeDesired(context); + } + } + + private static class CreatingBulk extends AbstractDependentResource + implements BulkDependentResource, + Creator { + + CreatingBulk() { + super("my-bulk"); + } + + @Override + public Map desiredResources(ConfigMap primary, Context context) { + var desired = new LinkedHashMap(); + desired.put("a", configMap("a", "desired")); + return desired; + } + + @Override + public Map getSecondaryResources( + ConfigMap primary, Context context) { + return Map.of(); + } + + @Override + public ConfigMap create(ConfigMap desired, ConfigMap primary, Context context) { + return desired; + } + + @Override + public void deleteTargetResource( + ConfigMap primary, ConfigMap resource, String key, Context context) {} + + @Override + public Result match( + ConfigMap actualResource, + ConfigMap desired, + ConfigMap primary, + Context context) { + return Result.computed(false, desired); + } + + @Override + public Result match( + ConfigMap resource, ConfigMap primary, Context context) { + return Result.computed(false, resource); + } + + @Override + protected Optional selectTargetSecondaryResource( + Set secondaryResources, ConfigMap primary, Context context) { + return Optional.empty(); + } + + @Override + protected void onCreated(ConfigMap primary, ConfigMap created, Context context) {} + + @Override + protected void onUpdated( + ConfigMap primary, ConfigMap updated, ConfigMap actual, Context context) {} + + @Override + public Class resourceType() { + return ConfigMap.class; + } + } +} From c557818bc280480d1faa868b8bd68cd7d72437c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 27 Oct 2026 15:28:08 +0100 Subject: [PATCH 13/45] fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/config/loader/ConfigLoaderTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java index 3a5887d572..13f44b036a 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java @@ -337,7 +337,11 @@ public Optional getValue(String key, Class type) { Map.entry( "withCloneSecondaryResourcesWhenGettingFromCache", "clone-secondary-resources-when-getting-from-cache"), - Map.entry("withClusterScopedEventNamespace", "events.cluster-scoped-namespace")); + Map.entry("withClusterScopedEventNamespace", "events.cluster-scoped-namespace"), + Map.entry( + "withConcurrentRetryAndRescheduleThreads", "retry-and-reschedule.concurrent-threads"), + Map.entry("withConcurrentScheduledTaskThreads", "scheduled-tasks.concurrent-threads"), + Map.entry("withUseVirtualThreads", "use-virtual-threads")); /** * Maps every scalar setter of {@link ControllerConfigurationOverrider} to the property key suffix From efad7a61c184c9e6550b4979c984c761ccc90418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 27 Oct 2026 17:22:18 +0100 Subject: [PATCH 14/45] fix: address review comments on virtual thread support - remove leftover platform thread executor assignments that overrode the virtual thread based caching executor in ExecutorServiceManager - buffer per resource polling registrations that arrive before the event source started and replay them once the executor is resolved - close the race between execute and shutdownNow of the bounded virtual thread executor with a shared lifecycle lock - reject aggregating events that carry an eventTime when type, reason or action differ - keep the JVM alive with a non-daemon thread while virtual threads are in use - run the startup deadlock test on Java 25 or later only --- .../api/config/ExecutorServiceManager.java | 32 +++++- .../operator/api/config/VirtualThreads.java | 29 +++-- .../operator/api/event/DefaultEventSink.java | 23 +++- .../PerResourcePollingEventSource.java | 28 +++++ .../VirtualThreadsStartupDeadlockTest.java | 103 ++++++++++++++++++ ...BoundedVirtualThreadExecutorAuditTest.java | 36 ++++++ .../config/ExecutorServiceManagerTest.java | 51 +++++++++ .../config/VirtualThreadsJvmLifetimeTest.java | 88 +++++++++++++++ .../api/event/EventRecorderAuditTest.java | 31 ++++++ .../PerResourcePollingEventSourceTest.java | 46 ++++++++ 10 files changed, 449 insertions(+), 18 deletions(-) create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index a1937c9ab9..149c990142 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -26,6 +26,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.LockSupport; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -45,6 +46,7 @@ public class ExecutorServiceManager { private ScheduledExecutorService retryAndRescheduleExecutorService; private boolean started; private ConfigurationService configurationService; + private Thread keepAliveThread; ExecutorServiceManager(ConfigurationService configurationService) { start(configurationService); @@ -189,19 +191,41 @@ public synchronized void start(ConfigurationService configurationService) { this.configurationService = configurationService; // used to lazy init workflow executor this.cachingExecutorService = newUnboundedExecutorService(configurationService.useVirtualThreads()); - // stays on platform threads even when virtual threads are requested: there is no virtual - // thread backed ScheduledExecutorService in the JDK, see VirtualThreads - this.scheduledExecutorService = Executors.newScheduledThreadPool(0); - this.cachingExecutorService = Executors.newCachedThreadPool(); this.scheduledExecutorService = configurationService.getScheduledExecutorService(); this.retryAndRescheduleExecutorService = configurationService.getRetryAndRescheduleExecutorService(); this.executor = new InstrumentedExecutorService(configurationService.getExecutorService()); + if (VirtualThreads.shouldUse(configurationService.useVirtualThreads())) { + startKeepAliveThread(); + } started = true; } } + /** + * Virtual threads are always daemon threads, so with them nothing is left to keep the JVM running + * once the main method of an operator returns after starting it, as opposed to the non-daemon + * threads of the platform thread pools. This thread restores that, until the executors are + * stopped. + */ + private void startKeepAliveThread() { + keepAliveThread = + new Thread( + () -> { + while (!Thread.currentThread().isInterrupted()) { + LockSupport.park(); + } + }, + "josdk-keep-alive"); + keepAliveThread.setDaemon(false); + keepAliveThread.start(); + } + public synchronized void stop(Duration gracefulShutdownTimeout) { + if (keepAliveThread != null) { + keepAliveThread.interrupt(); + keepAliveThread = null; + } var shutdowns = List.of( shutdown(executor, gracefulShutdownTimeout), diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index ad7ff3a230..f650123547 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -159,6 +159,7 @@ private static final class BoundedExecutorService extends AbstractExecutorServic private final ExecutorService delegate; private final Semaphore permits; private final Set pending = ConcurrentHashMap.newKeySet(); + private final Object lifecycleLock = new Object(); private BoundedExecutorService(ExecutorService delegate, int maxConcurrency) { this.delegate = delegate; @@ -169,12 +170,16 @@ private BoundedExecutorService(ExecutorService delegate, int maxConcurrency) { public void execute(Runnable command) { Objects.requireNonNull(command); var task = new PendingTask(command); - pending.add(task); - try { - delegate.execute(task); - } catch (RuntimeException e) { - pending.remove(task); - throw e; + // admission is closed under the same lock that shutdownNow drains pending tasks with, so a + // task is either rejected or seen by the drain, and never interrupted while unaccounted for + synchronized (lifecycleLock) { + pending.add(task); + try { + delegate.execute(task); + } catch (RuntimeException e) { + pending.remove(task); + throw e; + } } } @@ -220,13 +225,15 @@ public void shutdown() { @Override public List shutdownNow() { var notStarted = new ArrayList(); - for (var task : pending) { - if (task.claim()) { - pending.remove(task); - notStarted.add(task.command); + synchronized (lifecycleLock) { + for (var task : pending) { + if (task.claim()) { + pending.remove(task); + notStarted.add(task.command); + } } + delegate.shutdownNow(); } - delegate.shutdownNow(); return notStarted; } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java index 4ac42589c1..8269759e95 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java @@ -15,6 +15,8 @@ */ package io.javaoperatorsdk.operator.api.event; +import java.util.Objects; + import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.api.model.EventBuilder; import io.fabric8.kubernetes.client.KubernetesClient; @@ -31,9 +33,11 @@ * recorded, since the name of an event is derived from everything that identifies it (see {@link * DefaultEventRecorder}), and is aggregated onto the recorded one by patching its {@code count} and * {@code lastTimestamp}, along with its {@code message}, {@code type}, {@code reason} and {@code - * action}, which events sharing a name explicitly may differ in. That is what makes {@code kubectl - * describe} report a repeating event once, as {@code (x12 over 3m)}, rather than filling the event - * list with copies of it. + * action}, which events sharing a name explicitly may differ in, unless the event carries an {@code + * eventTime} (see {@link DefaultEventRecorder}), in which case the API server treats them as + * immutable and recording an event that differs in them is rejected. That is what makes {@code + * kubectl describe} report a repeating event once, as {@code (x12 over 3m)}, rather than filling + * the event list with copies of it. * *

Recording therefore needs {@code get}, {@code create} and {@code patch} on {@code events} in * the core API group. The patch is made on the resource version the event was read at, so two @@ -57,6 +61,13 @@ public void emit(Event event, Context context) { if (existing == null) { events.resource(event).create(); } else { + if (existing.getEventTime() != null && differsInImmutableFields(existing, event)) { + throw new IllegalStateException( + "Event " + + name + + " carries an eventTime, so its type, reason and action cannot change, but the" + + " event to record differs in them: use distinct event names for them"); + } var builder = new EventBuilder(existing) .withCount(requireNonNullElse(existing.getCount(), 1) + 1) @@ -71,4 +82,10 @@ public void emit(Event event, Context context) { events.resource(existing).patch(aggregated); } } + + private static boolean differsInImmutableFields(Event existing, Event event) { + return !Objects.equals(existing.getType(), event.getType()) + || !Objects.equals(existing.getReason(), event.getReason()) + || (event.getAction() != null && !Objects.equals(existing.getAction(), event.getAction())); + } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 7b9f1b5fba..2cad388795 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -66,6 +66,8 @@ public class PerResourcePollingEventSource private final Predicate

registerPredicate; private final Duration period; + private final Map registrationsBeforeStart = new ConcurrentHashMap<>(); + private volatile ScheduledExecutorService executorService; public PerResourcePollingEventSource( @@ -103,6 +105,16 @@ public void start() throws OperatorException { } executorService = resolveExecutorService(); super.start(); + replayRegistrationsBeforeStart(); + } + + private void replayRegistrationsBeforeStart() { + for (var primaryID : registrationsBeforeStart.keySet()) { + var resource = registrationsBeforeStart.remove(primaryID); + if (resource != null) { + checkAndRegisterTask(resource); + } + } } /** @@ -161,6 +173,7 @@ public void onResourceUpdated(P newResource, P oldResource) { @Override public void onResourceDeleted(P resource) { var resourceID = ResourceID.fromResource(resource); + registrationsBeforeStart.remove(resourceID); var scheduledFuture = scheduledFutures.remove(resourceID); if (scheduledFuture != null) { log.debug("Canceling scheduledFuture for resource: {}", resource); @@ -175,6 +188,20 @@ public void onResourceDeleted(P resource) { // important because otherwise there will be a race condition related to the timerTasks. private void checkAndRegisterTask(P resource) { var primaryID = ResourceID.fromResource(resource); + // the primary informer is started before this event source, and the executor to poll on is + // only resolved when this one starts, so resources seen in the meantime are registered then + if (!isRunning()) { + registrationsBeforeStart.put(primaryID, resource); + if (!isRunning()) { + return; + } + replayRegistrationsBeforeStart(); + return; + } + registerTask(primaryID, resource); + } + + private synchronized void registerTask(ResourceID primaryID, P resource) { if (scheduledFutures.get(primaryID) == null && (registerPredicate == null || registerPredicate.test(resource))) { var cachedResources = cachedResourcesFor(primaryID); @@ -260,6 +287,7 @@ public void stop() throws OperatorException { // of the operator, and the map cleared so that they are registered again on a restart scheduledFutures.values().forEach(future -> future.cancel(true)); scheduledFutures.clear(); + registrationsBeforeStart.clear(); if (ownsExecutorService && executorService != null) { executorService.shutdownNow(); } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java new file mode 100644 index 0000000000..914559eeb8 --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java @@ -0,0 +1,103 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.MockControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.processing.Controller; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * With virtual threads enabled on Java 21 to 24 (no JEP 491, hence only run on Java 25 or later, + * the official baseline for virtual thread support), starting as many controllers as there are + * virtual thread carriers deadlocks, because each controller is started on a virtual thread of the + * caching executor that blocks, while holding the monitors of the synchronized Controller.start / + * EventSourceManager.start / ControllerEventSource.start, on informer-start tasks submitted to the + * very same virtual thread executor. + */ +class VirtualThreadsStartupDeadlockTest { + + @Test + @EnabledForJreRange(min = JRE.JAVA_25) + @SuppressWarnings({"rawtypes", "unchecked"}) + void startingControllersOnVirtualThreadsDoesNotDeadlock() throws Exception { + assertStartsWithoutDeadlock(true); + } + + @Test + void controlStartingControllersOnPlatformThreads() throws Exception { + assertStartsWithoutDeadlock(false); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static void assertStartsWithoutDeadlock(boolean useVirtualThreads) throws Exception { + final var mockClient = MockKubernetesClient.client(ConfigMap.class); + final ConfigurationService configService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), + o -> o.withKubernetesClient(mockClient).withUseVirtualThreads(useVirtualThreads)); + final var executorServiceManager = configService.getExecutorServiceManager(); + final var controllerManager = new ControllerManager(executorServiceManager); + + // default virtual thread scheduler parallelism == number of available processors, e.g. 1 or 2 + // in a typical resource-limited operator pod + final int controllers = + System.getenv("AUDIT_N") != null + ? Integer.parseInt(System.getenv("AUDIT_N")) + : Runtime.getRuntime().availableProcessors(); + for (int i = 0; i < controllers; i++) { + final var configuration = MockControllerConfiguration.forResource(ConfigMap.class); + when(configuration.getConfigurationService()).thenReturn(configService); + when(configuration.getName()).thenReturn("controller-" + i); + controllerManager.add(new Controller(mock(Reconciler.class), configuration, mockClient)); + } + + final var start = CompletableFuture.runAsync(() -> controllerManager.start(false)); + try { + start.get(20, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // break the deadlock (interrupts the pinned virtual threads) so that the shared virtual + // thread scheduler isn't left starved for the rest of the test JVM + executorServiceManager.stop(Duration.ZERO); + try { + start.get(20, TimeUnit.SECONDS); + } catch (Exception ignored) { + // expected once the stuck start was interrupted + } + assertThat(false) + .as( + "ControllerManager.start deadlocked starting %d controllers on virtual threads", + controllers) + .isTrue(); + } + executorServiceManager.stop(Duration.ZERO); + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java index 18d0239b5c..c319790267 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java @@ -16,6 +16,7 @@ package io.javaoperatorsdk.operator.api.config; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -62,6 +63,41 @@ void virtualThreadVariantShutdownNowReturnsTasksThatNeverStarted() throws Except assertThat(queuedTasksReturnedByShutdownNow(true)).isEqualTo(5); } + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void everyTaskSubmittedWhileShuttingDownIsRunReturnedOrRejected() throws Exception { + for (int round = 0; round < 50; round++) { + final var executor = ExecutorServiceManager.newBoundedExecutorService(1, true); + final var accepted = new AtomicInteger(); + final var ran = new AtomicInteger(); + final var submitters = new CountDownLatch(4); + final var go = new CountDownLatch(1); + for (int t = 0; t < 4; t++) { + new Thread( + () -> { + try { + go.await(); + for (int i = 0; i < 200; i++) { + executor.execute(ran::incrementAndGet); + accepted.incrementAndGet(); + } + } catch (RejectedExecutionException | InterruptedException e) { + // the executor was shut down + } finally { + submitters.countDown(); + } + }) + .start(); + } + go.countDown(); + final var notStarted = executor.shutdownNow(); + assertThat(submitters.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + // tasks admitted after shutdownNow returned can't exist, so none may go missing + assertThat(ran.get() + notStarted.size()).isEqualTo(accepted.get()); + } + } + private static int queuedTasksReturnedByShutdownNow(boolean useVirtualThreads) throws Exception { final var executor = ExecutorServiceManager.newBoundedExecutorService(1, useVirtualThreads); final var running = new CountDownLatch(1); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java index 995a6e374f..272bcf914f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java @@ -16,10 +16,14 @@ package io.javaoperatorsdk.operator.api.config; import java.time.Duration; +import java.util.Optional; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; class ExecutorServiceManagerTest { @@ -62,4 +66,51 @@ void canBeRestartedAfterStop() { manager.stop(SHUTDOWN_TIMEOUT); } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void cachingExecutorRunsOnVirtualThreadsWhenRequested() throws Exception { + var configurationService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); + var manager = configurationService.getExecutorServiceManager(); + try { + var virtual = + manager.cachingExecutorService().submit(() -> VirtualThreadsTest.onVirtualThread()).get(); + assertThat(virtual).isTrue(); + } finally { + manager.stop(SHUTDOWN_TIMEOUT); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void keepAliveThreadRunsOnlyWhileTheManagerIsStartedWithVirtualThreads() throws Exception { + var configurationService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); + var manager = configurationService.getExecutorServiceManager(); + + assertThat(keepAliveThread()).hasValueSatisfying(t -> assertThat(t.isDaemon()).isFalse()); + + manager.stop(SHUTDOWN_TIMEOUT); + + await().untilAsserted(() -> assertThat(keepAliveThread()).isEmpty()); + } + + @Test + void noKeepAliveThreadWithPlatformThreads() { + var manager = new BaseConfigurationService().getExecutorServiceManager(); + try { + assertThat(keepAliveThread()).isEmpty(); + } finally { + manager.stop(SHUTDOWN_TIMEOUT); + } + } + + private static Optional keepAliveThread() { + return Thread.getAllStackTraces().keySet().stream() + .filter(t -> t.getName().equals("josdk-keep-alive") && t.isAlive()) + .findFirst(); + } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java new file mode 100644 index 0000000000..d5d68c150b --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java @@ -0,0 +1,88 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.config; + +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Throwaway audit test: the usual operator main method ends right after {@code operator.start()} + * (see docs/.../operations/configuration.md and several samples) and relies on the framework's + * non-daemon executor threads to keep the JVM alive. Virtual threads are always daemon threads, so + * with virtual threads enabled nothing keeps the JVM running once main returns. + */ +class VirtualThreadsJvmLifetimeTest { + + /** Mimics an operator main: start the executors, run a reconciliation, return from main. */ + public static class OperatorLikeMain { + public static void main(String[] args) throws Exception { + final boolean useVirtualThreads = Boolean.parseBoolean(args[0]); + final var configService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), o -> o.withUseVirtualThreads(useVirtualThreads)); + final var esm = configService.getExecutorServiceManager(); + // what Operator.start() does: start informers on the caching executor ... + esm.cachingExecutorService().submit(() -> {}).get(); + // ... and a first reconciliation dispatched by the EventProcessor + esm.reconcileExecutorService().submit(() -> {}).get(); + System.out.println("main returning"); + } + } + + private static boolean jvmStillAliveAfterMainReturns(boolean useVirtualThreads) throws Exception { + final var java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + final var process = + new ProcessBuilder( + java, + "-cp", + System.getProperty("java.class.path"), + OperatorLikeMain.class.getName(), + Boolean.toString(useVirtualThreads)) + .redirectErrorStream(true) + .start(); + try { + final boolean exited = process.waitFor(10, TimeUnit.SECONDS); + if (exited) { + final var output = new String(process.getInputStream().readAllBytes()); + // make sure the child exited normally after main returned, not because it crashed + assertThat(output).contains("main returning"); + assertThat(process.exitValue()).isZero(); + } + return !exited; + } finally { + process.destroyForcibly(); + } + } + + @Test + void controlPlatformThreadsKeepTheOperatorJvmAlive() throws Exception { + assertThat(jvmStillAliveAfterMainReturns(false)).isTrue(); + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void virtualThreadsKeepTheOperatorJvmAliveToo() throws Exception { + assertThat(jvmStillAliveAfterMainReturns(true)) + .as("JVM exited right after main returned with useVirtualThreads=true") + .isTrue(); + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java index bf2599de15..ccaf2394d2 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java @@ -123,6 +123,28 @@ void aggregatingOntoANamedEventKeepsTheTypeAndReasonOfTheFirstOccurrence() { assertThat(patched.get().getReason()).isEqualTo("Degraded"); } + @Test + void aggregatingOntoAnEventWithAnEventTimeRejectsChangedImmutableFields() { + var stored = new AtomicReference(); + var patched = new AtomicReference(); + var sink = new DefaultEventSink(mockClient(stored, patched)); + var recorder = + DefaultEventRecorder.builder(sink) + .namingStrategy((regarding, record) -> Optional.of("fixed-name")) + .build(); + var context = context(clusterScoped("ns1"), "operator-ns"); + + recorder.record(EventRecord.normal("Ready", "all good"), context); + recorder.record(EventRecord.warning("Degraded", "replica down"), context); + + assertThat(patched.get()).isNull(); + assertThat(stored.get().getType()).isEqualTo("Normal"); + + recorder.record(EventRecord.normal("Ready", "still good"), context); + + assertThat(patched.get().getCount()).isEqualTo(2); + } + /** * Supplied names are truncated to 253 characters, which silently maps distinct names onto the * same event, so two different events of a long-named object are aggregated as one. @@ -200,6 +222,15 @@ private Context context(HasMetadata primaryResource, String clusterScopedEven return context; } + private HasMetadata clusterScoped(String name) { + return new NamespaceBuilder() + .withNewMetadata() + .withName(name) + .withUid("uid-3") + .endMetadata() + .build(); + } + private ConfigMap configMap(String name) { return new ConfigMapBuilder() .withNewMetadata() diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 2f406f0489..c6736dc741 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -90,6 +90,52 @@ void pollsTheResourceAfterAwareOfIt() { }); } + @Test + void resourcesSeenBeforeStartArePolledOnceStarted() { + source.stop(); + setUpSource( + new PerResourcePollingEventSource<>( + SampleExternalResource.class, + context, + new PerResourcePollingConfigurationBuilder< + SampleExternalResource, TestCustomResource, String>( + supplier, Duration.ofMillis(PERIOD)) + .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) + .build()), + false); + + source.onResourceCreated(testCustomResource); + verify(supplier, never()).fetchResources(any()); + + source.start(); + + await() + .untilAsserted(() -> verify(supplier, atLeast(1)).fetchResources(eq(testCustomResource))); + } + + @Test + void resourcesDeletedBeforeStartAreNotPolled() { + source.stop(); + setUpSource( + new PerResourcePollingEventSource<>( + SampleExternalResource.class, + context, + new PerResourcePollingConfigurationBuilder< + SampleExternalResource, TestCustomResource, String>( + supplier, Duration.ofMillis(PERIOD)) + .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) + .build()), + false); + + source.onResourceCreated(testCustomResource); + source.onResourceDeleted(testCustomResource); + source.start(); + + await() + .pollDelay(Duration.ofMillis(2 * PERIOD)) + .untilAsserted(() -> verify(supplier, never()).fetchResources(any())); + } + @Test void registeringTaskOnAPredicate() { setUpSource( From 7cf09e1fcdbaad9a60c55cb040a07364e751cc76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 27 Oct 2026 17:59:11 +0100 Subject: [PATCH 15/45] revert: move event recorder fixes to their own PR The event recorder aggregation, long name and cluster scoped event namespace fixes now live in fix/event-recorder-correctness. --- .../api/event/DefaultEventRecorder.java | 39 +-- .../operator/api/event/DefaultEventSink.java | 32 +-- .../api/event/DefaultEventRecorderTest.java | 30 +-- .../api/event/EventRecorderAuditTest.java | 244 ------------------ 4 files changed, 13 insertions(+), 332 deletions(-) delete mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java index 1d070d7e4e..67764b7146 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorder.java @@ -21,8 +21,6 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.HexFormat; import java.util.Objects; @@ -34,7 +32,6 @@ import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.api.model.EventBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.MicroTime; import io.fabric8.kubernetes.api.model.ObjectReference; import io.fabric8.kubernetes.api.model.ObjectReferenceBuilder; import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; @@ -78,12 +75,6 @@ public class DefaultEventRecorder implements EventRecorder { private static final int IDENTITY_HASH_LENGTH = 32; - /** - * The fixed width format of {@code MicroTime}, which the API server refuses to read otherwise. - */ - private static final DateTimeFormatter EVENT_TIME_FORMAT = - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'").withZone(ZoneOffset.UTC); - /** What the API server accepts as an object name, see RFC 1123 on DNS subdomains. */ private static final Pattern RFC_1123_SUBDOMAIN = Pattern.compile("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*"); @@ -220,12 +211,11 @@ protected Event toEvent(Context context, EventRecord record) { var regarding = context.getPrimaryResource(); var now = Instant.now().truncatedTo(ChronoUnit.SECONDS).toString(); var involvedObject = objectReferenceFor(regarding); - var namespace = eventNamespace(regarding, context); var builder = new EventBuilder() .withNewMetadata() .withName(eventName(regarding, record, controllerName)) - .withNamespace(namespace) + .withNamespace(eventNamespace(regarding, context)) .withLabels(record.labels()) .withAnnotations(record.annotations()) .endMetadata() @@ -266,26 +256,9 @@ protected Event toEvent(Context context, EventRecord record) { .endMetadata(); } record.action().ifPresent(builder::withAction); - if (requiresEventTime(regarding, namespace)) { - builder.withEventTime(new MicroTime(EVENT_TIME_FORMAT.format(Instant.now()))); - if (record.action().isEmpty()) { - builder.withAction(record.reason()); - } - } return builder.build(); } - /** - * The API server only accepts an event without {@code eventTime} for a cluster scoped object if - * the event lives in the {@code default} namespace. Events that carry an {@code eventTime} are - * not subject to that restriction, but then have to set an {@code action} too. - */ - private static boolean requiresEventTime(HasMetadata regarding, String eventNamespace) { - return regarding.getMetadata().getNamespace() == null - && !CLUSTER_SCOPED_EVENT_NAMESPACE.equals(eventNamespace) - && !eventNamespace.isEmpty(); - } - private String eventNamespace(HasMetadata regarding, Context context) { var namespace = regarding.getMetadata().getNamespace(); if (namespace != null) { @@ -322,17 +295,15 @@ private static String truncateToMaxNameLength(String name) { if (name.length() <= MAX_NAME_LENGTH) { return name; } - var suffix = "." + identityDigest(name); - var truncated = name.substring(0, MAX_NAME_LENGTH - suffix.length()); + var truncated = name.substring(0, MAX_NAME_LENGTH); while (truncated.endsWith("-") || truncated.endsWith(".")) { truncated = truncated.substring(0, truncated.length() - 1); } - var shortened = truncated + suffix; log.warn( - "Shortened the name of event {} to {} to stay within the Kubernetes name limit", + "Truncated the name of event {} to {} to stay within the Kubernetes name limit", name, - shortened); - return shortened; + truncated); + return truncated; } /** diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java index 8269759e95..c2f5461ae8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/event/DefaultEventSink.java @@ -15,8 +15,6 @@ */ package io.javaoperatorsdk.operator.api.event; -import java.util.Objects; - import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.api.model.EventBuilder; import io.fabric8.kubernetes.client.KubernetesClient; @@ -32,12 +30,8 @@ *

An event that does not exist yet is created. An event that does is a repeat of one already * recorded, since the name of an event is derived from everything that identifies it (see {@link * DefaultEventRecorder}), and is aggregated onto the recorded one by patching its {@code count} and - * {@code lastTimestamp}, along with its {@code message}, {@code type}, {@code reason} and {@code - * action}, which events sharing a name explicitly may differ in, unless the event carries an {@code - * eventTime} (see {@link DefaultEventRecorder}), in which case the API server treats them as - * immutable and recording an event that differs in them is rejected. That is what makes {@code - * kubectl describe} report a repeating event once, as {@code (x12 over 3m)}, rather than filling - * the event list with copies of it. + * {@code lastTimestamp}. That is what makes {@code kubectl describe} report a repeating event once, + * as {@code (x12 over 3m)}, rather than filling the event list with copies of it. * *

Recording therefore needs {@code get}, {@code create} and {@code patch} on {@code events} in * the core API group. The patch is made on the resource version the event was read at, so two @@ -61,31 +55,13 @@ public void emit(Event event, Context context) { if (existing == null) { events.resource(event).create(); } else { - if (existing.getEventTime() != null && differsInImmutableFields(existing, event)) { - throw new IllegalStateException( - "Event " - + name - + " carries an eventTime, so its type, reason and action cannot change, but the" - + " event to record differs in them: use distinct event names for them"); - } - var builder = + var aggregated = new EventBuilder(existing) .withCount(requireNonNullElse(existing.getCount(), 1) + 1) .withLastTimestamp(event.getLastTimestamp()) .withMessage(event.getMessage()) - .withType(event.getType()) - .withReason(event.getReason()); - if (event.getAction() != null) { - builder.withAction(event.getAction()); - } - var aggregated = builder.build(); + .build(); events.resource(existing).patch(aggregated); } } - - private static boolean differsInImmutableFields(Event existing, Event event) { - return !Objects.equals(existing.getType(), event.getType()) - || !Objects.equals(existing.getReason(), event.getReason()) - || (event.getAction() != null && !Objects.equals(existing.getAction(), event.getAction())); - } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java index 3cedc6cc9f..388217b9be 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/DefaultEventRecorderTest.java @@ -436,41 +436,19 @@ void stripsTrailingSeparatorsFromTruncatedNames() { EventRecord.builder() .reason("Created") .message("created") - .name("a".repeat(219) + "." + "b".repeat(47)) + .name("a".repeat(252) + "." + "b".repeat(47)) .build(), context(configMap())); recorder.record( EventRecord.builder() .reason("Created") .message("created") - .name("b".repeat(219) + "-" + "c".repeat(47)) + .name("b".repeat(252) + "-" + "c".repeat(47)) .build(), context(configMap())); - assertThat(emitted.get(0).getMetadata().getName()).matches("a{219}\\.[0-9a-f]{32}"); - assertThat(emitted.get(1).getMetadata().getName()).matches("b{219}\\.[0-9a-f]{32}"); - } - - @Test - void keepsLongSuppliedNamesSharingTheirPrefixDistinct() { - recorder.record( - EventRecord.builder() - .reason("Created") - .message("created") - .name("a".repeat(300) + "-one") - .build(), - context(configMap())); - recorder.record( - EventRecord.builder() - .reason("Created") - .message("created") - .name("a".repeat(300) + "-two") - .build(), - context(configMap())); - - assertThat(emitted.get(0).getMetadata().getName()).hasSize(253); - assertThat(emitted.get(0).getMetadata().getName()) - .isNotEqualTo(emitted.get(1).getMetadata().getName()); + assertThat(emitted.get(0).getMetadata().getName()).isEqualTo("a".repeat(252)); + assertThat(emitted.get(1).getMetadata().getName()).isEqualTo("b".repeat(252)); } @Test diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java deleted file mode 100644 index ccaf2394d2..0000000000 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/event/EventRecorderAuditTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright Java Operator SDK Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.javaoperatorsdk.operator.api.event; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; - -import org.junit.jupiter.api.Test; - -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.ConfigMapBuilder; -import io.fabric8.kubernetes.api.model.Event; -import io.fabric8.kubernetes.api.model.EventList; -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.NamespaceBuilder; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.dsl.MixedOperation; -import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; -import io.fabric8.kubernetes.client.dsl.Resource; -import io.fabric8.kubernetes.client.dsl.V1APIGroupDSL; -import io.javaoperatorsdk.operator.api.config.ConfigurationService; -import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; -import io.javaoperatorsdk.operator.api.reconciler.Context; - -import static io.javaoperatorsdk.operator.api.config.LeaderElectionConfigurationBuilder.aLeaderElectionConfiguration; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** Audit repros for the configurable event recorder. Each test fails on the current code. */ -class EventRecorderAuditTest { - - private final List emitted = new ArrayList<>(); - - /** - * The core/v1 API server (legacyValidateEvent in pkg/apis/core/validation/events.go) rejects an - * event without eventTime whose involvedObject.namespace is empty unless the event itself lives - * in "" or "default". Overriding the cluster scoped event namespace, as - * ConfigurationService#clusterScopedEventNamespace recommends, therefore requires the event to - * carry an eventTime, and with it an action. - */ - @Test - void overriddenClusterScopedEventNamespaceProducesEventsTheCoreApiAccepts() { - var recorder = new DefaultEventRecorder((event, context) -> emitted.add(event)); - var namespace = - new NamespaceBuilder() - .withNewMetadata() - .withName("ns1") - .withUid("uid-2") - .endMetadata() - .build(); - - recorder.record(EventRecord.normal("Created", "created"), context(namespace, "operator-ns")); - - var event = emitted.get(0); - assertThat(event.getInvolvedObject().getNamespace()).isNull(); - assertThat(event.getMetadata().getNamespace()).isEqualTo("operator-ns"); - assertThat(event.getEventTime().getTime()) - .matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}Z"); - assertThat(event.getAction()).isEqualTo("Created"); - } - - @Test - void clusterScopedEventsInTheDefaultNamespaceDoNotCarryAnEventTime() { - var recorder = new DefaultEventRecorder((event, context) -> emitted.add(event)); - var namespace = - new NamespaceBuilder() - .withNewMetadata() - .withName("ns1") - .withUid("uid-2") - .endMetadata() - .build(); - - recorder.record(EventRecord.normal("Created", "created"), context(namespace, "default")); - - assertThat(emitted.get(0).getEventTime()).isNull(); - assertThat(emitted.get(0).getMetadata().getNamespace()).isEqualTo("default"); - } - - /** - * A naming strategy (here the one EventNamingStrategy's javadoc suggests, a fixed suffix per - * object) makes the name the aggregation identity on its own, so records with different type and - * reason resolve to the same event. DefaultEventSink then only patches count, lastTimestamp and - * message: the Warning is published under the type and reason of the earlier Normal event. - */ - @Test - void aggregatingOntoANamedEventKeepsTheTypeAndReasonOfTheFirstOccurrence() { - var stored = new AtomicReference(); - var patched = new AtomicReference(); - var sink = new DefaultEventSink(mockClient(stored, patched)); - var recorder = - DefaultEventRecorder.builder(sink) - .namingStrategy( - (regarding, record) -> - Optional.of(regarding.getMetadata().getName() + "-status-report")) - .build(); - var context = context(configMap("test1"), "default"); - - recorder.record(EventRecord.normal("Ready", "all good"), context); - recorder.record(EventRecord.warning("Degraded", "replica down"), context); - - assertThat(patched.get()).isNotNull(); - assertThat(patched.get().getCount()).isEqualTo(2); - assertThat(patched.get().getMessage()).isEqualTo("replica down"); - assertThat(patched.get().getType()).isEqualTo("Warning"); - assertThat(patched.get().getReason()).isEqualTo("Degraded"); - } - - @Test - void aggregatingOntoAnEventWithAnEventTimeRejectsChangedImmutableFields() { - var stored = new AtomicReference(); - var patched = new AtomicReference(); - var sink = new DefaultEventSink(mockClient(stored, patched)); - var recorder = - DefaultEventRecorder.builder(sink) - .namingStrategy((regarding, record) -> Optional.of("fixed-name")) - .build(); - var context = context(clusterScoped("ns1"), "operator-ns"); - - recorder.record(EventRecord.normal("Ready", "all good"), context); - recorder.record(EventRecord.warning("Degraded", "replica down"), context); - - assertThat(patched.get()).isNull(); - assertThat(stored.get().getType()).isEqualTo("Normal"); - - recorder.record(EventRecord.normal("Ready", "still good"), context); - - assertThat(patched.get().getCount()).isEqualTo(2); - } - - /** - * Supplied names are truncated to 253 characters, which silently maps distinct names onto the - * same event, so two different events of a long-named object are aggregated as one. - */ - @Test - void truncatingSuppliedNamesMergesDistinctEvents() { - var recorder = - DefaultEventRecorder.builder((event, context) -> emitted.add(event)) - .namingStrategy( - (regarding, record) -> - Optional.of( - regarding.getMetadata().getName() + "-" + record.reason().toLowerCase())) - .build(); - var context = context(configMap("a".repeat(245)), "default"); - - recorder.record(EventRecord.warning("BackoffLimitExceeded", "job failed"), context); - recorder.record(EventRecord.warning("BackoffStarted", "job retrying"), context); - - assertThat(emitted.get(0).getMetadata().getName()) - .isNotEqualTo(emitted.get(1).getMetadata().getName()); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private KubernetesClient mockClient( - AtomicReference stored, AtomicReference patched) { - var client = mock(KubernetesClient.class); - var v1 = mock(V1APIGroupDSL.class); - MixedOperation> events = mock(MixedOperation.class); - NonNamespaceOperation> inNs = - mock(NonNamespaceOperation.class); - when(client.v1()).thenReturn(v1); - when(v1.events()).thenReturn(events); - when(events.inNamespace(anyString())).thenReturn(inNs); - Resource byName = mock(Resource.class); - when(inNs.withName(anyString())).thenReturn(byName); - when(byName.get()).thenAnswer(i -> stored.get()); - when(inNs.resource(any(Event.class))) - .thenAnswer( - i -> { - Event target = i.getArgument(0); - Resource r = mock(Resource.class); - when(r.create()) - .thenAnswer( - c -> { - stored.set(target); - return target; - }); - when(r.patch(any(Event.class))) - .thenAnswer( - p -> { - Event e = p.getArgument(0); - patched.set(e); - stored.set(e); - return e; - }); - return r; - }); - return client; - } - - @SuppressWarnings("rawtypes") - private Context context(HasMetadata primaryResource, String clusterScopedEventNamespace) { - var configurationService = mock(ConfigurationService.class); - when(configurationService.getLeaderElectionConfiguration()) - .thenReturn( - Optional.of(aLeaderElectionConfiguration("lease").withIdentity("pod-1").build())); - when(configurationService.clusterScopedEventNamespace()) - .thenReturn(clusterScopedEventNamespace); - ControllerConfiguration controllerConfiguration = mock(ControllerConfiguration.class); - when(controllerConfiguration.getName()).thenReturn("testcontroller"); - when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); - Context context = mock(Context.class); - when(context.getPrimaryResource()).thenReturn(primaryResource); - when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); - return context; - } - - private HasMetadata clusterScoped(String name) { - return new NamespaceBuilder() - .withNewMetadata() - .withName(name) - .withUid("uid-3") - .endMetadata() - .build(); - } - - private ConfigMap configMap(String name) { - return new ConfigMapBuilder() - .withNewMetadata() - .withName(name) - .withNamespace("ns1") - .withUid("uid-1") - .withResourceVersion("42") - .endMetadata() - .build(); - } -} From 9c6112150603ca48f99710681db3f5933614f3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 28 Oct 2026 09:37:45 +0100 Subject: [PATCH 16/45] refactor: move executor helpers from Utils to internal ExecutorUtilsInternal Add a virtual thread backed scheduled executor and related configuration, docs and tests. Co-Authored-By: Claude Sonnet 5.5 --- .../content/en/docs/documentation/eventing.md | 4 + .../documentation/operations/configuration.md | 28 +- .../operator/ExecutorUtilsInternal.java | 69 ++++ .../api/config/ConfigurationService.java | 23 +- .../config/ConfigurationServiceOverrider.java | 2 +- .../api/config/ExecutorServiceManager.java | 21 ++ .../operator/api/config/Utils.java | 48 --- .../VirtualThreadScheduledExecutor.java | 305 ++++++++++++++++++ .../operator/api/config/VirtualThreads.java | 59 +++- .../PerResourcePollingEventSource.java | 4 +- .../source/polling/PollingEventSource.java | 4 +- .../event/source/timer/TimerEventSource.java | 4 +- ...BoundedVirtualThreadExecutorAuditTest.java | 6 +- .../config/ExecutorServiceManagerTest.java | 4 +- .../VirtualThreadScheduledExecutorTest.java | 276 ++++++++++++++++ .../config/VirtualThreadsJvmLifetimeTest.java | 2 +- .../api/config/VirtualThreadsTest.java | 34 +- .../source/timer/TimerEventSourceTest.java | 4 +- .../virtualthreads/VirtualThreadsIT.java | 4 +- 19 files changed, 795 insertions(+), 106 deletions(-) create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index aec261bbf0..431c1b0eb8 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -234,6 +234,10 @@ resources is slow, size the pool accordingly with Operator operator = new Operator(overrider -> overrider.withConcurrentScheduledTaskThreads(20)); ``` +With virtual threads enabled, see [virtual threads](../operations/configuration/#virtual-threads), the +polls run on virtual threads, the executor only keeping one platform thread to do the timing; the +number of threads then caps how many polls run at the same time. + Retried and rescheduled reconciliations are triggered on a separate executor, so a slow poll can never delay them. It is sized with `withConcurrentRetryAndRescheduleThreads` (2 threads by default); few threads are needed there since triggering a reconciliation only enqueues an event diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index af3d39e815..61f301a120 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -43,15 +43,23 @@ of a workflow are processed concurrently. Only the threads backing those limits virtual threads are cheap, these limits can usually be raised significantly compared to what is reasonable with platform threads. -Two things to keep in mind: - -- Virtual threads are officially supported on **Java 25 or later**. They exist from Java 21 on and - the flag does enable them there, but before Java 25 a virtual thread pins its carrier thread - while it is inside a `synchronized` block, which can starve the carrier pool. [JEP - 491](https://openjdk.org/jeps/491), delivered in Java 25, removed that pinning, so this is the - baseline the framework supports. On a JVM without virtual threads at all (below Java 21) a - warning is logged and platform threads are used instead, so the same configuration works on any - supported Java version. +Things to keep in mind: + +- Virtual threads are only supported on **Java 25 or later**. Before Java 25 a virtual thread pins + its carrier thread while it is inside a `synchronized` block, which can starve the carrier pool + and even deadlock the operator on startup. [JEP 491](https://openjdk.org/jeps/491), delivered in + Java 25, removed that pinning. This is checked at runtime: on an older JVM, including the ones + where virtual threads exist (Java 21 to 24), a warning is logged and platform threads are used + instead, so the same configuration works on any Java version the operator runs on. +- The scheduled tasks (the polling event sources, and the retried and rescheduled reconciliations) + run on virtual threads too. The JDK has no scheduled executor backed by virtual threads, so the + default ones keep a single platform thread for the timing only and run each task they fire on a + virtual thread. `withConcurrentScheduledTaskThreads(int)` and + `withConcurrentRetryAndRescheduleThreads(int)` still cap how many of those tasks run at the same + time. +- With virtual threads, which are always daemon threads, the framework keeps one idle platform + thread (`josdk-keep-alive`) running until the operator is stopped, so that an operator whose + `main` method returns after starting it doesn't let the JVM exit. - A custom `ExecutorService` provided through `withExecutorService(...)` or `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. - The Kubernetes client the framework creates when none is provided also switches its internal @@ -326,7 +334,7 @@ All operator-level keys are prefixed with `josdk.`. |---|---|---| | `josdk.check-crd` | `Boolean` | Validate CRDs against local model on startup | | `josdk.close-client-on-stop` | `Boolean` | Close the Kubernetes client when the operator stops | -| `josdk.use-virtual-threads` | `Boolean` | Run the framework's concurrent work on virtual threads (officially supported on Java 25+ at runtime) | +| `josdk.use-virtual-threads` | `Boolean` | Run the framework's concurrent work on virtual threads (only supported on Java 25+, platform threads are used on older versions) | | `josdk.use-ssa-to-patch-primary-resource` | `Boolean` | Use Server-Side Apply to patch the primary resource | | `josdk.clone-secondary-resources-when-getting-from-cache` | `Boolean` | Clone secondary resources on cache reads | diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java new file mode 100644 index 0000000000..c989a9e0db --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java @@ -0,0 +1,69 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; + +public class ExecutorUtilsInternal { + + // prevent instantiation of util class + private ExecutorUtilsInternal() {} + + /** + * Creates a {@link ThreadFactory} producing daemon threads named after the specified prefix. + * Daemon threads don't keep the JVM alive if the {@link Operator} is never stopped, and naming + * them makes the pool they belong to identifiable in thread dumps. + * + * @param namePrefix the prefix the created threads are named after + * @return a {@link ThreadFactory} creating named daemon threads + */ + public static ThreadFactory daemonThreadFactory(String namePrefix) { + final var defaultThreadFactory = Executors.defaultThreadFactory(); + final var counter = new AtomicLong(); + return runnable -> { + final var thread = defaultThreadFactory.newThread(runnable); + thread.setName(namePrefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } + + /** + * Creates the kind of {@link ScheduledExecutorService} the operator runs its scheduled (i.e. + * periodic or delayed) tasks on: one whose threads are daemon threads named after the specified + * prefix, and which doesn't let the tasks that were scheduled for later delay its shutdown. + * + * @param corePoolSize the number of threads to keep in the pool + * @param threadNamePrefix the prefix the threads of the pool are named after + * @return a {@link ScheduledExecutorService} to run scheduled tasks on + */ + public static ScheduledExecutorService daemonScheduledThreadPool( + int corePoolSize, String threadNamePrefix) { + final var executor = + new ScheduledThreadPoolExecutor(corePoolSize, daemonThreadFactory(threadNamePrefix)); + // tasks that are scheduled far out (a reconciliation rescheduled in an hour, say) would + // otherwise keep the pool from terminating until the graceful shutdown timeout expires + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + // cancelled tasks are frequent (every retry that is superseded by a new event cancels one) and + // would otherwise be retained until their delay elapses + executor.setRemoveOnCancelPolicy(true); + return executor; + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 2b5b3d6689..e729744e8d 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -303,15 +303,18 @@ default Metrics getMetrics() { * pool of platform threads anymore. Since virtual threads are cheap, those limits can be set * considerably higher than what would be reasonable for platform threads. * - *

Officially supported on Java 25 or later. Virtual threads exist as of Java 21 and are used - * there as well, but before Java 25 a virtual thread pins its carrier thread while inside a - * {@code synchronized} block, which JEP 491 removed in - * Java 25. On a JVM without virtual threads at all, a warning is logged and platform threads are - * used, so that the same configuration works regardless of the Java version the operator runs on. + *

Virtual threads are only supported on Java 25 or later, where JEP 491 removed the pinning of the carrier thread by + * {@code synchronized} blocks. This is checked at runtime: on an older JVM, a warning is logged + * and platform threads are used, so that the same configuration works regardless of the Java + * version the operator runs on. * *

Note that this only affects the executors created by the framework: a custom {@link * ExecutorService} provided through {@link #getExecutorService()} or {@link - * #getWorkflowExecutorService()} is used as is. The same goes for the {@link KubernetesClient}: + * #getWorkflowExecutorService()}, or a custom {@link ScheduledExecutorService} provided through + * {@link #getScheduledExecutorService()} or {@link #getRetryAndRescheduleExecutorService()}, is + * used as is. The default scheduled executors keep a single platform thread for the timing and + * run each task they fire on a virtual thread. The same goes for the {@link KubernetesClient}: * the default client created by {@link #getKubernetesClient()} then also runs its internal * asynchronous tasks (informer event dispatching, watch event delivery) on virtual threads, while * a custom client, e.g. provided through {@link @@ -365,8 +368,8 @@ default ExecutorService getWorkflowExecutorService() { * @since 5.6.0 */ default ScheduledExecutorService getScheduledExecutorService() { - return Utils.daemonScheduledThreadPool( - concurrentScheduledTaskThreads(), "josdk-scheduled-task"); + return ExecutorServiceManager.newScheduledExecutorService( + concurrentScheduledTaskThreads(), useVirtualThreads(), "josdk-scheduled-task"); } /** @@ -383,8 +386,8 @@ default ScheduledExecutorService getScheduledExecutorService() { * @since 5.6.0 */ default ScheduledExecutorService getRetryAndRescheduleExecutorService() { - return Utils.daemonScheduledThreadPool( - concurrentRetryAndRescheduleThreads(), "josdk-retry-reschedule"); + return ExecutorServiceManager.newScheduledExecutorService( + concurrentRetryAndRescheduleThreads(), useVirtualThreads(), "josdk-retry-reschedule"); } /** diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index e222fed62e..7215cc29ba 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -186,7 +186,7 @@ public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( /** * Makes the framework run the tasks it executes concurrently on virtual threads instead of - * platform threads. Officially supported on Java 25 or later, see {@link + * platform threads. Only supported on Java 25 or later, see {@link * ConfigurationService#useVirtualThreads()} for the details. * * @param useVirtualThreads {@code true} to use virtual threads diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index 149c990142..2ca1591536 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -34,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.OperatorException; public class ExecutorServiceManager { @@ -69,6 +70,26 @@ public static ExecutorService newBoundedExecutorService( : Executors.newFixedThreadPool(maxConcurrency); } + /** + * Creates the executor service used to run scheduled (i.e. periodic or delayed) tasks, either + * firing them on virtual threads or on a pool of platform threads. The platform thread variant is + * the one of {@link ExecutorUtilsInternal#daemonScheduledThreadPool(int, String)}. In both cases + * the threads are daemon threads and the concurrency limit is enforced. + * + * @param maxConcurrency the maximal number of tasks executed at the same time + * @param useVirtualThreads whether virtual threads should be used, see {@link + * ConfigurationService#useVirtualThreads()} + * @param threadNamePrefix the prefix the threads of the executor are named after + * @return the {@link ScheduledExecutorService} to run scheduled tasks on + * @since 5.7.0 + */ + public static ScheduledExecutorService newScheduledExecutorService( + int maxConcurrency, boolean useVirtualThreads, String threadNamePrefix) { + return VirtualThreads.shouldUse(useVirtualThreads) + ? VirtualThreads.newScheduledExecutor(maxConcurrency, threadNamePrefix) + : ExecutorUtilsInternal.daemonScheduledThreadPool(maxConcurrency, threadNamePrefix); + } + /** * Creates the executor service used to run an unbounded number of tasks concurrently, either * backed by virtual threads or by a cached pool of platform threads. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java index 95e4fba6e1..4613d9645d 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/Utils.java @@ -26,11 +26,6 @@ import java.util.Date; import java.util.Optional; import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,49 +108,6 @@ public static int ensureValid(int value, String description, int minValue, int d return value; } - /** - * Creates a {@link ThreadFactory} producing daemon threads named after the specified prefix. - * Daemon threads don't keep the JVM alive if the {@link io.javaoperatorsdk.operator.Operator} is - * never stopped, and naming them makes the pool they belong to identifiable in thread dumps. - * - * @param namePrefix the prefix the created threads are named after - * @return a {@link ThreadFactory} creating named daemon threads - * @since 5.6.0 - */ - public static ThreadFactory daemonThreadFactory(String namePrefix) { - final var defaultThreadFactory = Executors.defaultThreadFactory(); - final var counter = new AtomicLong(); - return runnable -> { - final var thread = defaultThreadFactory.newThread(runnable); - thread.setName(namePrefix + "-" + counter.incrementAndGet()); - thread.setDaemon(true); - return thread; - }; - } - - /** - * Creates the kind of {@link ScheduledExecutorService} the operator runs its scheduled (i.e. - * periodic or delayed) tasks on: one whose threads are daemon threads named after the specified - * prefix, and which doesn't let the tasks that were scheduled for later delay its shutdown. - * - * @param corePoolSize the number of threads to keep in the pool - * @param threadNamePrefix the prefix the threads of the pool are named after - * @return a {@link ScheduledExecutorService} to run scheduled tasks on - * @since 5.6.0 - */ - public static ScheduledExecutorService daemonScheduledThreadPool( - int corePoolSize, String threadNamePrefix) { - final var executor = - new ScheduledThreadPoolExecutor(corePoolSize, daemonThreadFactory(threadNamePrefix)); - // tasks that are scheduled far out (a reconciliation rescheduled in an hour, say) would - // otherwise keep the pool from terminating until the graceful shutdown timeout expires - executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - // cancelled tasks are frequent (every retry that is superseded by a new event cancels one) and - // would otherwise be retained until their delay elapses - executor.setRemoveOnCancelPolicy(true); - return executor; - } - @SuppressWarnings("unused") // this is used in the Quarkus extension public static boolean isValidateCustomResourcesEnvVarSet() { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java new file mode 100644 index 0000000000..d38c5abe96 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java @@ -0,0 +1,305 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.config; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Delayed; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import io.javaoperatorsdk.operator.ExecutorUtilsInternal; + +/** + * A {@link ScheduledExecutorService} running the tasks it fires on the threads of another executor, + * which is how scheduled tasks end up on virtual threads: the JDK has no virtual thread backed + * {@link ScheduledExecutorService}, so a single daemon platform thread does the timing and only + * hands each fired task off. + * + *

The semantics follow the ones of the default scheduled executor of the SDK, see {@link + * ExecutorUtilsInternal#daemonScheduledThreadPool(int, String)}: tasks that were scheduled for + * later, and periodic tasks, are discarded when the executor is shut down, while tasks that already + * started are let finish. + */ +final class VirtualThreadScheduledExecutor extends AbstractExecutorService + implements ScheduledExecutorService { + + private final ExecutorService worker; + private final ScheduledExecutorService timer; + private final Set> waiting = ConcurrentHashMap.newKeySet(); + private final Object lifecycleLock = new Object(); + + VirtualThreadScheduledExecutor(ExecutorService worker, String threadNamePrefix) { + this.worker = worker; + this.timer = ExecutorUtilsInternal.daemonScheduledThreadPool(1, threadNamePrefix + "-timer"); + } + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + Objects.requireNonNull(command); + return start(new Handle(Executors.callable(command, null), 0), delay, unit); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + Objects.requireNonNull(callable); + return start(new Handle<>(callable, 0), delay, unit); + } + + @Override + public ScheduledFuture scheduleAtFixedRate( + Runnable command, long initialDelay, long period, TimeUnit unit) { + Objects.requireNonNull(command); + if (period <= 0) { + throw new IllegalArgumentException("period must be positive"); + } + return start( + new Handle(Executors.callable(command, null), unit.toNanos(period)), + initialDelay, + unit); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay( + Runnable command, long initialDelay, long delay, TimeUnit unit) { + Objects.requireNonNull(command); + if (delay <= 0) { + throw new IllegalArgumentException("delay must be positive"); + } + return start( + new Handle(Executors.callable(command, null), -unit.toNanos(delay)), + initialDelay, + unit); + } + + private Handle start(Handle handle, long initialDelay, TimeUnit unit) { + handle.arm(Math.max(0, unit.toNanos(initialDelay))); + return handle; + } + + @Override + public void execute(Runnable command) { + worker.execute(Objects.requireNonNull(command)); + } + + @Override + public void shutdown() { + synchronized (lifecycleLock) { + cancelWaiting(false); + timer.shutdown(); + worker.shutdown(); + } + } + + @Override + public List shutdownNow() { + final var notStarted = new ArrayList(); + synchronized (lifecycleLock) { + cancelWaiting(true); + timer.shutdownNow(); + for (var task : worker.shutdownNow()) { + if (task instanceof Dispatched dispatched) { + dispatched.handle.cancel(false); + } else { + notStarted.add(task); + } + } + } + return Collections.unmodifiableList(notStarted); + } + + private void cancelWaiting(boolean mayInterruptIfRunning) { + for (var handle : waiting) { + handle.cancel(mayInterruptIfRunning); + } + } + + @Override + public boolean isShutdown() { + return timer.isShutdown() && worker.isShutdown(); + } + + @Override + public boolean isTerminated() { + return timer.isTerminated() && worker.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + final var deadline = System.nanoTime() + unit.toNanos(timeout); + return timer.awaitTermination(timeout, unit) + && worker.awaitTermination(deadline - System.nanoTime(), TimeUnit.NANOSECONDS); + } + + /** + * What is handed to the worker once a task fires, so that it can be traced back to its handle. + */ + private record Dispatched(Handle handle) implements Runnable { + @Override + public void run() { + handle.runOnWorker(); + } + } + + /** + * The future of a scheduled task. A positive {@code period} makes it a task at a fixed rate, a + * negative one a task with a fixed delay, whose absolute value is the delay. + */ + private final class Handle implements ScheduledFuture { + + private final Callable callable; + private final long period; + private final CompletableFuture result = new CompletableFuture<>(); + + private volatile ScheduledFuture timerFuture; + private volatile Thread runner; + private volatile long triggerTime; + + private Handle(Callable callable, long period) { + this.callable = callable; + this.period = period; + } + + private void arm(long delayNanos) { + triggerTime = System.nanoTime() + delayNanos; + synchronized (lifecycleLock) { + waiting.add(this); + try { + timerFuture = timer.schedule(this::dispatch, delayNanos, TimeUnit.NANOSECONDS); + } catch (RejectedExecutionException e) { + waiting.remove(this); + throw e; + } + } + if (result.isDone()) { + timerFuture.cancel(false); + } + } + + private void dispatch() { + waiting.remove(this); + if (result.isDone()) { + return; + } + try { + worker.execute(new Dispatched(this)); + } catch (RejectedExecutionException e) { + result.cancel(false); + } + } + + private void runOnWorker() { + if (result.isDone()) { + return; + } + runner = Thread.currentThread(); + try { + if (period == 0) { + result.complete(callable.call()); + return; + } + callable.call(); + } catch (Throwable t) { + result.completeExceptionally(t); + return; + } finally { + runner = null; + } + rearm(); + } + + private void rearm() { + if (result.isDone()) { + return; + } + final long delay; + if (period > 0) { + delay = Math.max(0, triggerTime + period - System.nanoTime()); + } else { + delay = -period; + } + try { + arm(delay); + } catch (RejectedExecutionException e) { + result.cancel(false); + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + final var cancelled = result.cancel(mayInterruptIfRunning); + if (cancelled) { + waiting.remove(this); + final var future = timerFuture; + if (future != null) { + future.cancel(false); + } + final var thread = runner; + if (mayInterruptIfRunning && thread != null) { + thread.interrupt(); + } + } + return cancelled; + } + + @Override + public boolean isCancelled() { + return result.isCancelled(); + } + + @Override + public boolean isDone() { + return result.isDone(); + } + + @Override + public V get() throws InterruptedException, ExecutionException { + return result.get(); + } + + @Override + public V get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return result.get(timeout, unit); + } + + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(triggerTime - System.nanoTime(), TimeUnit.NANOSECONDS); + } + + @Override + public int compareTo(Delayed other) { + if (other == this) { + return 0; + } + return Long.compare(getDelay(TimeUnit.NANOSECONDS), other.getDelay(TimeUnit.NANOSECONDS)); + } + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index f650123547..0cf512587f 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -27,6 +27,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -41,22 +42,28 @@ * Creates the virtual thread based executors used when {@link * ConfigurationService#useVirtualThreads()} is enabled. * + *

Virtual threads are only supported on Java 25 or later, where JEP 491 removed the pinning of the carrier thread by + * {@code synchronized} blocks. On older JVMs, including the ones where virtual threads already + * exist, the request is ignored with a warning and platform threads are used, see {@link + * #shouldUse(boolean)}. + * *

The SDK is compiled for Java 17, in which virtual threads don't exist yet, so {@code - * Executors.newVirtualThreadPerTaskExecutor()} is looked up reflectively and is only available when - * the operator actually runs on Java 21 or later. + * Executors.newVirtualThreadPerTaskExecutor()} is looked up reflectively. * - *

There is intentionally no scheduled variant here: the JDK provides no virtual thread backed - * {@link java.util.concurrent.ScheduledExecutorService}, scheduling was deliberately left out of - * virtual threads (the Loom runtime itself uses a platform thread scheduler to unpark timed out - * virtual threads). Running scheduled tasks on virtual threads would mean keeping a platform thread - * scheduler purely for the timing and handing each fired task off to a virtual thread executor, - * which the SDK doesn't do, so {@link ExecutorServiceManager#scheduledExecutorService()} is always - * backed by platform threads. + *

The JDK provides no virtual thread backed {@link + * java.util.concurrent.ScheduledExecutorService} (the Loom runtime itself uses a platform thread + * scheduler to unpark timed out virtual threads), so the scheduled variant, see {@link + * #newScheduledExecutor(int, String)}, keeps a single platform thread for the timing only and hands + * each fired task off to a virtual thread. */ final class VirtualThreads { private static final Logger log = LoggerFactory.getLogger(VirtualThreads.class); + /** The first Java version virtual threads are supported on. */ + static final int MINIMUM_SUPPORTED_JAVA_VERSION = 25; + private static final MethodHandle NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR = lookupFactoryMethod(); private static final AtomicBoolean UNSUPPORTED_WARNING_LOGGED = new AtomicBoolean(); @@ -75,9 +82,17 @@ private static MethodHandle lookupFactoryMethod() { } } - /** Whether the JVM the operator runs on supports virtual threads, i.e. is Java 21 or later. */ + /** + * Whether the JVM the operator runs on supports virtual threads, i.e. is Java {@value + * #MINIMUM_SUPPORTED_JAVA_VERSION} or later. + */ static boolean isSupported() { - return NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR != null; + return isSupported(Runtime.version().feature()); + } + + static boolean isSupported(int javaFeatureVersion) { + return NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR != null + && javaFeatureVersion >= MINIMUM_SUPPORTED_JAVA_VERSION; } /** @@ -93,9 +108,9 @@ static boolean shouldUse(boolean requested) { } if (UNSUPPORTED_WARNING_LOGGED.compareAndSet(false, true)) { log.warn( - "Virtual threads were requested but are not supported by the JVM in use (Java {}, Java 21" - + " or later is required, Java 25 or later is officially supported). Falling back to" - + " platform threads.", + "Virtual threads were requested but are only supported on Java {} or later, and the JVM" + + " in use is Java {}. Falling back to platform threads.", + MINIMUM_SUPPORTED_JAVA_VERSION, Runtime.version().feature()); } return false; @@ -105,8 +120,9 @@ static boolean shouldUse(boolean requested) { static ExecutorService newVirtualThreadPerTaskExecutor() { if (!isSupported()) { throw new OperatorException( - "Virtual threads are not supported by the JVM in use, Java 21 or later is required" - + " (Java 25 or later is officially supported)"); + "Virtual threads are only supported on Java " + + MINIMUM_SUPPORTED_JAVA_VERSION + + " or later"); } try { return (ExecutorService) NEW_VIRTUAL_THREAD_PER_TASK_EXECUTOR.invokeExact(); @@ -135,6 +151,17 @@ public void onClose(Executor executor) { }; } + /** + * A {@link ScheduledExecutorService} running the tasks it fires on virtual threads, at most + * {@code maxConcurrency} of them at the same time. The timing is done by a single daemon platform + * thread named after {@code threadNamePrefix}, which only hands the tasks off. + */ + static ScheduledExecutorService newScheduledExecutor( + int maxConcurrency, String threadNamePrefix) { + return new VirtualThreadScheduledExecutor( + newBoundedVirtualThreadExecutor(maxConcurrency), threadNamePrefix); + } + /** * A virtual thread based executor executing at most {@code maxConcurrency} tasks at the same * time, the equivalent of a fixed size platform thread pool. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 2cad388795..0799788689 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -31,9 +31,9 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.api.config.ConfigurationService; -import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.javaoperatorsdk.operator.processing.event.source.Cache; @@ -130,7 +130,7 @@ private ScheduledExecutorService resolveExecutorService() { return configurationService.getExecutorServiceManager().scheduledExecutorService(); } return Executors.newSingleThreadScheduledExecutor( - Utils.daemonThreadFactory("josdk-polling-" + name())); + ExecutorUtilsInternal.daemonThreadFactory("josdk-polling-" + name())); } private Set getAndCacheResource(P primary, boolean fromGetter) { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index d25caee74c..b5cb976a5e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -28,9 +28,9 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.api.config.ConfigurationService; -import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.ResourceID; @@ -176,7 +176,7 @@ private ScheduledExecutorService resolveExecutorService() { return configurationService.getExecutorServiceManager().scheduledExecutorService(); } return Executors.newSingleThreadScheduledExecutor( - Utils.daemonThreadFactory("josdk-polling-" + name())); + ExecutorUtilsInternal.daemonThreadFactory("josdk-polling-" + name())); } protected synchronized void getStateAndFillCache() { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index e8a0e351d0..368d8bf2e7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.javaoperatorsdk.operator.api.config.Utils; +import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.api.reconciler.BaseControl; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.Event; @@ -160,7 +160,7 @@ private ScheduledExecutorService resolveExecutorService() { return executorServiceSupplier.get(); } return Executors.newSingleThreadScheduledExecutor( - Utils.daemonThreadFactory("josdk-timer-" + name())); + ExecutorUtilsInternal.daemonThreadFactory("josdk-timer-" + name())); } @Override diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java index c319790267..5e17c52ed9 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java @@ -41,7 +41,7 @@ void controlPlatformPoolRejectsNonPositiveConcurrency() { * Semaphore(0) (or a negative one) whose tasks wait for a permit forever. */ @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void virtualThreadVariantRejectsNonPositiveConcurrencyToo() { assertThatIllegalArgumentException() .isThrownBy(() -> ExecutorServiceManager.newBoundedExecutorService(0, true).shutdownNow()); @@ -58,13 +58,13 @@ void controlPlatformPoolShutdownNowReturnsTasksThatNeverStarted() throws Excepti * waiting for a permit are silently dropped (neither run nor returned). */ @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void virtualThreadVariantShutdownNowReturnsTasksThatNeverStarted() throws Exception { assertThat(queuedTasksReturnedByShutdownNow(true)).isEqualTo(5); } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void everyTaskSubmittedWhileShuttingDownIsRunReturnedOrRejected() throws Exception { for (int round = 0; round < 50; round++) { final var executor = ExecutorServiceManager.newBoundedExecutorService(1, true); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java index 272bcf914f..33900d834a 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java @@ -68,7 +68,7 @@ void canBeRestartedAfterStop() { } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void cachingExecutorRunsOnVirtualThreadsWhenRequested() throws Exception { var configurationService = ConfigurationService.newOverriddenConfigurationService( @@ -84,7 +84,7 @@ void cachingExecutorRunsOnVirtualThreadsWhenRequested() throws Exception { } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void keepAliveThreadRunsOnlyWhileTheManagerIsStartedWithVirtualThreads() throws Exception { var configurationService = ConfigurationService.newOverriddenConfigurationService( diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java new file mode 100644 index 0000000000..ae9f617863 --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java @@ -0,0 +1,276 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.api.config; + +import java.time.Duration; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.awaitility.Awaitility.await; + +@EnabledForJreRange(min = JRE.JAVA_25) +class VirtualThreadScheduledExecutorTest { + + private static final int MAX_CONCURRENCY = 2; + + private ScheduledExecutorService executor; + + @BeforeEach + void setUp() { + executor = ExecutorServiceManager.newScheduledExecutorService(MAX_CONCURRENCY, true, "test"); + } + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void runsDelayedTasksOnVirtualThreads() throws Exception { + var future = executor.schedule(VirtualThreadsTest::onVirtualThread, 50, TimeUnit.MILLISECONDS); + + assertThat(future.get(10, TimeUnit.SECONDS)).isTrue(); + } + + @Test + void keepsTheTimingOnADaemonPlatformThread() { + executor.schedule(() -> {}, 10, TimeUnit.MINUTES); + + assertThat(Thread.getAllStackTraces().keySet()) + .filteredOn(t -> t.getName().startsWith("test-timer")) + .isNotEmpty() + .allSatisfy(t -> assertThat(t.isDaemon()).isTrue()); + } + + @Test + void delayedTasksDoNotRunEarly() throws Exception { + var start = System.nanoTime(); + + executor.schedule(() -> {}, 200, TimeUnit.MILLISECONDS).get(10, TimeUnit.SECONDS); + + assertThat(Duration.ofNanos(System.nanoTime() - start)) + .isGreaterThanOrEqualTo(Duration.ofMillis(190)); + } + + @Test + void cancellingBeforeTheTaskFiresPreventsIt() throws Exception { + var ran = new AtomicBoolean(); + var future = executor.schedule(() -> ran.set(true), 200, TimeUnit.MILLISECONDS); + + assertThat(future.cancel(false)).isTrue(); + + assertThat(future.isCancelled()).isTrue(); + assertThatExceptionOfType(CancellationException.class).isThrownBy(future::get); + Thread.sleep(400); + assertThat(ran).isFalse(); + } + + @Test + void propagatesTheFailureOfATask() { + var future = + executor.schedule( + () -> { + throw new IllegalStateException("failed"); + }, + 0, + TimeUnit.MILLISECONDS); + + assertThatExceptionOfType(ExecutionException.class) + .isThrownBy(() -> future.get(10, TimeUnit.SECONDS)) + .withCauseInstanceOf(IllegalStateException.class); + } + + @Test + void repeatsTasksWithAFixedDelayUntilCancelled() { + var runs = new AtomicInteger(); + var onVirtualThread = new AtomicBoolean(true); + + var future = + executor.scheduleWithFixedDelay( + () -> { + onVirtualThread.compareAndSet(true, VirtualThreadsTest.onVirtualThread()); + runs.incrementAndGet(); + }, + 0, + 20, + TimeUnit.MILLISECONDS); + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(runs).hasValueGreaterThan(3)); + assertThat(future.cancel(false)).isTrue(); + var runsAtCancel = runs.get(); + await() + .pollDelay(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(runs.get()).isLessThanOrEqualTo(runsAtCancel + 1)); + assertThat(onVirtualThread).isTrue(); + } + + @Test + void repeatsTasksAtAFixedRate() { + var runs = new AtomicInteger(); + + executor.scheduleAtFixedRate(runs::incrementAndGet, 0, 20, TimeUnit.MILLISECONDS); + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(runs).hasValueGreaterThan(3)); + } + + @Test + void aFailingPeriodicTaskIsNotRepeated() { + var runs = new AtomicInteger(); + + var future = + executor.scheduleWithFixedDelay( + () -> { + runs.incrementAndGet(); + throw new IllegalStateException("failed"); + }, + 0, + 20, + TimeUnit.MILLISECONDS); + + assertThatExceptionOfType(ExecutionException.class) + .isThrownBy(() -> future.get(10, TimeUnit.SECONDS)); + await().pollDelay(Duration.ofMillis(200)).untilAsserted(() -> assertThat(runs).hasValue(1)); + } + + @Test + void runsAtMostTheConfiguredNumberOfTasksAtTheSameTime() throws Exception { + var running = new AtomicInteger(); + var maxRunning = new AtomicInteger(); + var done = new CountDownLatch(10); + for (int i = 0; i < 10; i++) { + executor.schedule( + () -> { + maxRunning.accumulateAndGet(running.incrementAndGet(), Math::max); + try { + Thread.sleep(30); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + running.decrementAndGet(); + done.countDown(); + }, + 0, + TimeUnit.MILLISECONDS); + } + + assertThat(done.await(30, TimeUnit.SECONDS)).isTrue(); + assertThat(maxRunning).hasValueLessThanOrEqualTo(MAX_CONCURRENCY); + } + + @Test + void shutdownDiscardsTasksScheduledForLaterAndRejectsNewOnes() throws Exception { + var ran = new AtomicBoolean(); + var later = executor.schedule(() -> ran.set(true), 10, TimeUnit.MINUTES); + var periodic = executor.scheduleWithFixedDelay(() -> {}, 10, 10, TimeUnit.MINUTES); + + executor.shutdown(); + + assertThat(later.isCancelled()).isTrue(); + assertThat(periodic.isCancelled()).isTrue(); + assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + assertThat(ran).isFalse(); + assertThatExceptionOfType(RejectedExecutionException.class) + .isThrownBy(() -> executor.schedule(() -> {}, 0, TimeUnit.MILLISECONDS)); + } + + @Test + void shutdownLetsStartedTasksFinish() throws Exception { + var started = new CountDownLatch(1); + var finished = new AtomicBoolean(); + var future = + executor.schedule( + () -> { + started.countDown(); + Thread.sleep(200); + finished.set(true); + return true; + }, + 0, + TimeUnit.MILLISECONDS); + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + + executor.shutdown(); + + assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + assertThat(future.get()).isTrue(); + assertThat(finished).isTrue(); + } + + @Test + void shutdownNowInterruptsRunningTasks() throws Exception { + var started = new CountDownLatch(1); + var interrupted = new CountDownLatch(1); + executor.schedule( + () -> { + started.countDown(); + try { + Thread.sleep(60_000); + } catch (InterruptedException e) { + interrupted.countDown(); + } + }, + 0, + TimeUnit.MILLISECONDS); + assertThat(started.await(10, TimeUnit.SECONDS)).isTrue(); + + executor.shutdownNow(); + + assertThat(interrupted.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + } + + @Test + void theOperatorsDefaultScheduledExecutorsRunOnVirtualThreadsWhenRequested() throws Exception { + var configurationService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); + var manager = configurationService.getExecutorServiceManager(); + try { + assertThat( + manager + .scheduledExecutorService() + .schedule(VirtualThreadsTest::onVirtualThread, 0, TimeUnit.MILLISECONDS) + .get(10, TimeUnit.SECONDS)) + .isTrue(); + assertThat( + manager + .retryAndRescheduleExecutorService() + .schedule(VirtualThreadsTest::onVirtualThread, 0, TimeUnit.MILLISECONDS) + .get(10, TimeUnit.SECONDS)) + .isTrue(); + } finally { + manager.stop(Duration.ofMillis(100)); + } + } +} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java index d5d68c150b..6ff0cc32fd 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java @@ -79,7 +79,7 @@ void controlPlatformThreadsKeepTheOperatorJvmAlive() throws Exception { } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void virtualThreadsKeepTheOperatorJvmAliveToo() throws Exception { assertThat(jvmStillAliveAfterMainReturns(true)) .as("JVM exited right after main returned with useVirtualThreads=true") diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java index c28be4b4fc..6c3b0854e3 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java @@ -52,7 +52,31 @@ void usesPlatformThreadPoolWhenVirtualThreadsAreNotRequested() throws Exception } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + void virtualThreadsAreOnlySupportedFromJava25() { + assertThat(VirtualThreads.isSupported(17)).isFalse(); + assertThat(VirtualThreads.isSupported(21)).isFalse(); + assertThat(VirtualThreads.isSupported(24)).isFalse(); + assertThat(VirtualThreads.isSupported(Runtime.version().feature())) + .isEqualTo(Runtime.version().feature() >= 25); + } + + @Test + @EnabledForJreRange(max = JRE.JAVA_24) + void fallsBackToPlatformThreadsBeforeJava25() throws Exception { + var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); + var scheduled = ExecutorServiceManager.newScheduledExecutorService(MAX_CONCURRENCY, true, "t"); + try { + assertThat(executor).isInstanceOf(ThreadPoolExecutor.class); + assertThat(executor.submit(VirtualThreadsTest::onVirtualThread).get()).isFalse(); + assertThat(scheduled.submit(VirtualThreadsTest::onVirtualThread).get()).isFalse(); + } finally { + executor.shutdownNow(); + scheduled.shutdownNow(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_25) void usesVirtualThreadsWhenRequested() throws Exception { var virtualExecutor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); try { @@ -70,7 +94,7 @@ void usesVirtualThreadsWhenRequested() throws Exception { } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void boundedVirtualThreadExecutorRespectsTheConfiguredConcurrency() throws Exception { var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); try { @@ -108,7 +132,7 @@ void boundedVirtualThreadExecutorRespectsTheConfiguredConcurrency() throws Excep } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void invokeAllIsBoundedTooSinceItIsUsedToStartTheEventSources() { var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); try { @@ -138,7 +162,7 @@ void invokeAllIsBoundedTooSinceItIsUsedToStartTheEventSources() { } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void shutdownTerminatesOnceTheAlreadySubmittedTasksAreDone() throws Exception { ExecutorService executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); @@ -161,7 +185,7 @@ void defaultKubernetesClientKeepsPlatformThreadsWhenVirtualThreadsAreNotRequeste } @Test - @EnabledForJreRange(min = JRE.JAVA_21) + @EnabledForJreRange(min = JRE.JAVA_25) void defaultKubernetesClientRunsItsTasksOnVirtualThreadsWhenRequested() throws Exception { ExecutorService executor; try (var client = defaultKubernetesClient(true)) { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index e8d36c0034..3e6fbeaef3 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -26,8 +26,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.TestUtils; -import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.BaseControl; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.Event; @@ -131,7 +131,7 @@ public void handlesInstanceReschedule() { public void schedulesOnTheProvidedExecutorAndLeavesItRunningOnStop() { var providedExecutor = Executors.newSingleThreadScheduledExecutor( - Utils.daemonThreadFactory("provided-timer-executor")); + ExecutorUtilsInternal.daemonThreadFactory("provided-timer-executor")); var handler = new CapturingEventHandler(); var eventSource = new TimerEventSource(() -> providedExecutor); eventSource.setEventHandler(handler); diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java index fbb38042aa..67ed2c780d 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/virtualthreads/VirtualThreadsIT.java @@ -45,10 +45,10 @@ class VirtualThreadsIT { static final int NUMBER_OF_RESOURCES = 10; /** - * Virtual threads require Java 21, on an older JVM the framework transparently falls back to + * Virtual threads require Java 25, on an older JVM the framework transparently falls back to * platform threads, which this test also covers since only the concurrency assertions apply then. */ - private static final boolean VIRTUAL_THREADS_SUPPORTED = Runtime.version().feature() >= 21; + private static final boolean VIRTUAL_THREADS_SUPPORTED = Runtime.version().feature() >= 25; @RegisterExtension LocallyRunOperatorExtension operator = From 8414999132281e24ef27e5ca7bf71dcc254d0e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 28 Oct 2026 10:16:20 +0100 Subject: [PATCH 17/45] fix: scheduled executor defaults, timer restart ordering and polling resilience - use the daemon scheduled thread pool for standalone timer and polling executors - resolve the timer executor before marking the event source as running - keep polling a resource after its fetcher failed - exclude the intentional compareTo/equals mismatch of scheduled handles from SpotBugs Co-Authored-By: Claude Sonnet 5.5 --- .../polling/PerResourcePollingEventSource.java | 13 ++++++++----- .../event/source/polling/PollingEventSource.java | 4 +--- .../event/source/timer/TimerEventSource.java | 6 ++---- .../polling/PerResourcePollingEventSourceTest.java | 12 ++++++++++++ spotbugs-exclude.xml | 10 ++++++++++ 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 0799788689..7faf8832fd 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -21,7 +21,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -129,8 +128,7 @@ private ScheduledExecutorService resolveExecutorService() { if (configurationService != null) { return configurationService.getExecutorServiceManager().scheduledExecutorService(); } - return Executors.newSingleThreadScheduledExecutor( - ExecutorUtilsInternal.daemonThreadFactory("josdk-polling-" + name())); + return ExecutorUtilsInternal.daemonScheduledThreadPool(1, "josdk-polling-" + name()); } private Set getAndCacheResource(P primary, boolean fromGetter) { @@ -231,8 +229,13 @@ public void run() { log.warn("No resource in cache for resource ID: {}", primaryID); // no new execution is scheduled in this case, an on delete event should be received shortly } else { - var actualResources = primary.map(p -> getAndCacheResource(p, false)); - scheduleNextExecution(primary.get(), actualResources.orElse(null)); + Set actualResources = null; + try { + actualResources = getAndCacheResource(primary.get(), false); + } catch (RuntimeException e) { + log.error("Error during polling resources for: {}", primaryID, e); + } + scheduleNextExecution(primary.get(), actualResources); } } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index b5cb976a5e..de2d7a9bc1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -18,7 +18,6 @@ import java.time.Duration; import java.util.Map; import java.util.Set; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -175,8 +174,7 @@ private ScheduledExecutorService resolveExecutorService() { if (configurationService != null) { return configurationService.getExecutorServiceManager().scheduledExecutorService(); } - return Executors.newSingleThreadScheduledExecutor( - ExecutorUtilsInternal.daemonThreadFactory("josdk-polling-" + name())); + return ExecutorUtilsInternal.daemonScheduledThreadPool(1, "josdk-polling-" + name()); } protected synchronized void getStateAndFillCache() { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index 368d8bf2e7..07de4f0d7e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -19,7 +19,6 @@ import java.util.Set; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -150,8 +149,8 @@ public void cancelOnceSchedule(ResourceID customResourceUid) { @Override public void start() { if (!isRunning()) { - super.start(); executorService = resolveExecutorService(); + super.start(); } } @@ -159,8 +158,7 @@ private ScheduledExecutorService resolveExecutorService() { if (executorServiceSupplier != null) { return executorServiceSupplier.get(); } - return Executors.newSingleThreadScheduledExecutor( - ExecutorUtilsInternal.daemonThreadFactory("josdk-timer-" + name())); + return ExecutorUtilsInternal.daemonScheduledThreadPool(1, "josdk-timer-" + name()); } @Override diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index c6736dc741..cb9b82e93c 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -90,6 +90,18 @@ void pollsTheResourceAfterAwareOfIt() { }); } + @Test + void keepsPollingAfterTheFetcherFailed() { + when(supplier.fetchResources(any())) + .thenThrow(new IllegalStateException("temporary failure")) + .thenReturn(Set.of(SampleExternalResource.testResource1())); + + source.onResourceCreated(testCustomResource); + + await() + .untilAsserted(() -> verify(supplier, atLeast(3)).fetchResources(eq(testCustomResource))); + } + @Test void resourcesSeenBeforeStartArePolledOnceStarted() { source.stop(); diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml index f6107ac4ea..9c516e059e 100644 --- a/spotbugs-exclude.xml +++ b/spotbugs-exclude.xml @@ -67,4 +67,14 @@ + + + + + + From ef9f7d99ac936af8b0e35c85861683873d86c6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 28 Oct 2026 10:47:43 +0100 Subject: [PATCH 18/45] fix: keep the scheduled executors of the original configuration when overriding Co-Authored-By: Claude Sonnet 5.5 --- .../config/ConfigurationServiceOverrider.java | 4 ++ .../ConfigurationServiceOverriderTest.java | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index 7215cc29ba..b5ccf75a5c 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -449,6 +449,8 @@ public ExecutorService getWorkflowExecutorService() { public ScheduledExecutorService getScheduledExecutorService() { if (scheduledExecutorService != null) { return scheduledExecutorService; + } else if (concurrentScheduledTaskThreads == null && useVirtualThreads == null) { + return original.getScheduledExecutorService(); } else { return super.getScheduledExecutorService(); } @@ -458,6 +460,8 @@ public ScheduledExecutorService getScheduledExecutorService() { public ScheduledExecutorService getRetryAndRescheduleExecutorService() { if (retryAndRescheduleExecutorService != null) { return retryAndRescheduleExecutorService; + } else if (concurrentRetryAndRescheduleThreads == null && useVirtualThreads == null) { + return original.getRetryAndRescheduleExecutorService(); } else { return super.getRetryAndRescheduleExecutorService(); } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java index 7acf3fff84..e91688152e 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java @@ -20,6 +20,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor; @@ -141,6 +142,47 @@ void scheduledExecutorCanBeOverridden() { } } + @Test + void executorsOfTheOriginalConfigurationAreKeptUnlessTheirSettingsAreOverridden() { + final var scheduledExecutorService = Executors.newScheduledThreadPool(1); + final var retryExecutorService = Executors.newScheduledThreadPool(1); + try { + final var original = + new BaseConfigurationService() { + @Override + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + @Override + public ScheduledExecutorService getRetryAndRescheduleExecutorService() { + return retryExecutorService; + } + }; + + final var overridden = + new ConfigurationServiceOverrider(original).withCloseClientOnStop(false); + + assertThat(overridden.build().getScheduledExecutorService()) + .isSameAs(scheduledExecutorService); + assertThat(overridden.build().getRetryAndRescheduleExecutorService()) + .isSameAs(retryExecutorService); + + final var resized = overridden.withConcurrentScheduledTaskThreads(3).build(); + final var defaultScheduled = resized.getScheduledExecutorService(); + try { + assertThat(defaultScheduled).isNotSameAs(scheduledExecutorService); + assertThat(((ScheduledThreadPoolExecutor) defaultScheduled).getCorePoolSize()).isEqualTo(3); + assertThat(resized.getRetryAndRescheduleExecutorService()).isSameAs(retryExecutorService); + } finally { + defaultScheduled.shutdownNow(); + } + } finally { + scheduledExecutorService.shutdownNow(); + retryExecutorService.shutdownNow(); + } + } + @Test void scheduledExecutorDefaultsToADaemonPoolOfTheConfiguredSize() { final var overridden = From 32725ec09c8b604e3d0ff2d6d8a648496a13a179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 28 Oct 2026 11:29:10 +0100 Subject: [PATCH 19/45] feat: do not limit the scheduled tasks running on virtual threads Co-Authored-By: Claude Sonnet 5.5 --- .../content/en/docs/documentation/eventing.md | 5 +++-- .../documentation/operations/configuration.md | 6 +++--- .../api/config/ConfigurationService.java | 7 +++++-- .../api/config/ExecutorServiceManager.java | 8 ++++--- .../operator/api/config/VirtualThreads.java | 14 ++++++------- .../VirtualThreadScheduledExecutorTest.java | 21 ++++++++++--------- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index 431c1b0eb8..4f776145c0 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -235,8 +235,9 @@ Operator operator = new Operator(overrider -> overrider.withConcurrentScheduledT ``` With virtual threads enabled, see [virtual threads](../operations/configuration/#virtual-threads), the -polls run on virtual threads, the executor only keeping one platform thread to do the timing; the -number of threads then caps how many polls run at the same time. +polls run on virtual threads, the executor only keeping one platform thread to do the timing. The +number of polls running at the same time is then not limited, so `withConcurrentScheduledTaskThreads` +doesn't apply, and a slow poll doesn't delay the others. Retried and rescheduled reconciliations are triggered on a separate executor, so a slow poll can never delay them. It is sized with `withConcurrentRetryAndRescheduleThreads` (2 threads by diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index 61f301a120..ac48696603 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -54,9 +54,9 @@ Things to keep in mind: - The scheduled tasks (the polling event sources, and the retried and rescheduled reconciliations) run on virtual threads too. The JDK has no scheduled executor backed by virtual threads, so the default ones keep a single platform thread for the timing only and run each task they fire on a - virtual thread. `withConcurrentScheduledTaskThreads(int)` and - `withConcurrentRetryAndRescheduleThreads(int)` still cap how many of those tasks run at the same - time. + virtual thread. The number of those tasks running at the same time is not limited, so + `withConcurrentScheduledTaskThreads(int)` and `withConcurrentRetryAndRescheduleThreads(int)` only + apply to platform threads. - With virtual threads, which are always daemon threads, the framework keeps one idle platform thread (`josdk-keep-alive`) running until the operator is stopped, so that an operator whose `main` method returns after starting it doesn't let the JVM exit. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index e729744e8d..25572333a3 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -259,7 +259,9 @@ default int concurrentWorkflowExecutorThreads() { * tasks with the default executor. These threads are shared by all the polling event sources of * the operator, so this number should be raised when many, or slow, polling event sources are * registered: a poll only starts once a thread is available, and a slow poll therefore delays the - * polls of the other event sources. + * polls of the other event sources. Not used when virtual threads are enabled, see {@link + * #useVirtualThreads()}, since the number of scheduled tasks running at the same time is not + * limited then. * * @return the maximum number of concurrent scheduled task threads * @since 5.6.0 @@ -272,7 +274,8 @@ default int concurrentScheduledTaskThreads() { * Number of threads the operator can spin out to trigger its retried and rescheduled * reconciliations with the default executor. These threads are shared by all the controllers of * the operator, but the tasks they run only enqueue an event for the reconciliation to happen on - * a reconciliation thread, so few of them are needed. + * a reconciliation thread, so few of them are needed. Not used when virtual threads are enabled, + * see {@link #useVirtualThreads()}. * * @return the maximum number of concurrent retry and reschedule threads * @since 5.6.0 diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index 2ca1591536..b937a4c022 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -74,9 +74,11 @@ public static ExecutorService newBoundedExecutorService( * Creates the executor service used to run scheduled (i.e. periodic or delayed) tasks, either * firing them on virtual threads or on a pool of platform threads. The platform thread variant is * the one of {@link ExecutorUtilsInternal#daemonScheduledThreadPool(int, String)}. In both cases - * the threads are daemon threads and the concurrency limit is enforced. + * the threads are daemon threads. Virtual threads are cheap and the tasks scheduled are typically + * blocking calls, so their number is not limited. * - * @param maxConcurrency the maximal number of tasks executed at the same time + * @param maxConcurrency the number of threads of the platform thread pool, not used with virtual + * threads * @param useVirtualThreads whether virtual threads should be used, see {@link * ConfigurationService#useVirtualThreads()} * @param threadNamePrefix the prefix the threads of the executor are named after @@ -86,7 +88,7 @@ public static ExecutorService newBoundedExecutorService( public static ScheduledExecutorService newScheduledExecutorService( int maxConcurrency, boolean useVirtualThreads, String threadNamePrefix) { return VirtualThreads.shouldUse(useVirtualThreads) - ? VirtualThreads.newScheduledExecutor(maxConcurrency, threadNamePrefix) + ? VirtualThreads.newScheduledExecutor(threadNamePrefix) : ExecutorUtilsInternal.daemonScheduledThreadPool(maxConcurrency, threadNamePrefix); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java index 0cf512587f..a47f09b320 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreads.java @@ -54,8 +54,8 @@ *

The JDK provides no virtual thread backed {@link * java.util.concurrent.ScheduledExecutorService} (the Loom runtime itself uses a platform thread * scheduler to unpark timed out virtual threads), so the scheduled variant, see {@link - * #newScheduledExecutor(int, String)}, keeps a single platform thread for the timing only and hands - * each fired task off to a virtual thread. + * #newScheduledExecutor(String)}, keeps a single platform thread for the timing only and hands each + * fired task off to a virtual thread. */ final class VirtualThreads { @@ -152,14 +152,12 @@ public void onClose(Executor executor) { } /** - * A {@link ScheduledExecutorService} running the tasks it fires on virtual threads, at most - * {@code maxConcurrency} of them at the same time. The timing is done by a single daemon platform + * A {@link ScheduledExecutorService} running the tasks it fires on virtual threads, without + * limiting how many of them run at the same time. The timing is done by a single daemon platform * thread named after {@code threadNamePrefix}, which only hands the tasks off. */ - static ScheduledExecutorService newScheduledExecutor( - int maxConcurrency, String threadNamePrefix) { - return new VirtualThreadScheduledExecutor( - newBoundedVirtualThreadExecutor(maxConcurrency), threadNamePrefix); + static ScheduledExecutorService newScheduledExecutor(String threadNamePrefix) { + return new VirtualThreadScheduledExecutor(newVirtualThreadPerTaskExecutor(), threadNamePrefix); } /** diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java index ae9f617863..a7adca62f2 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java @@ -39,6 +39,7 @@ class VirtualThreadScheduledExecutorTest { private static final int MAX_CONCURRENCY = 2; + private static final int TASKS = 10; private ScheduledExecutorService executor; @@ -164,28 +165,28 @@ void aFailingPeriodicTaskIsNotRepeated() { } @Test - void runsAtMostTheConfiguredNumberOfTasksAtTheSameTime() throws Exception { - var running = new AtomicInteger(); - var maxRunning = new AtomicInteger(); - var done = new CountDownLatch(10); - for (int i = 0; i < 10; i++) { + void doesNotLimitTheNumberOfTasksRunningAtTheSameTime() throws Exception { + var allStarted = new CountDownLatch(TASKS); + var done = new CountDownLatch(TASKS); + for (int i = 0; i < TASKS; i++) { executor.schedule( () -> { - maxRunning.accumulateAndGet(running.incrementAndGet(), Math::max); + allStarted.countDown(); try { - Thread.sleep(30); + // only returns if all the tasks are running at the same time, which with more tasks + // than the (platform thread) concurrency would never be the case if it was limited + if (allStarted.await(30, TimeUnit.SECONDS)) { + done.countDown(); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - running.decrementAndGet(); - done.countDown(); }, 0, TimeUnit.MILLISECONDS); } assertThat(done.await(30, TimeUnit.SECONDS)).isTrue(); - assertThat(maxRunning).hasValueLessThanOrEqualTo(MAX_CONCURRENCY); } @Test From b3527581bd6435553ad8b3f065da7e18ba63da1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 10:18:31 +0100 Subject: [PATCH 20/45] fix: address review comments on the scheduled executor and polling Publish the runner of a scheduled task before checking whether it was cancelled, make fixed-rate tasks catch up after an overrun, drop the stale scheduled future of a primary missing from the cache, and fix a race in TimerEventSourceTest. --- .../VirtualThreadScheduledExecutor.java | 21 ++++++------ .../PerResourcePollingEventSource.java | 5 ++- .../VirtualThreadScheduledExecutorTest.java | 32 +++++++++++++++++++ .../PerResourcePollingEventSourceTest.java | 15 +++++++++ .../source/timer/TimerEventSourceTest.java | 2 +- 5 files changed, 63 insertions(+), 12 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java index d38c5abe96..3bd273d85f 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java @@ -99,7 +99,7 @@ public ScheduledFuture scheduleWithFixedDelay( } private Handle start(Handle handle, long initialDelay, TimeUnit unit) { - handle.arm(Math.max(0, unit.toNanos(initialDelay))); + handle.arm(System.nanoTime() + Math.max(0, unit.toNanos(initialDelay))); return handle; } @@ -186,8 +186,9 @@ private Handle(Callable callable, long period) { this.period = period; } - private void arm(long delayNanos) { - triggerTime = System.nanoTime() + delayNanos; + private void arm(long triggerNanos) { + triggerTime = triggerNanos; + final var delayNanos = Math.max(0, triggerNanos - System.nanoTime()); synchronized (lifecycleLock) { waiting.add(this); try { @@ -215,11 +216,11 @@ private void dispatch() { } private void runOnWorker() { - if (result.isDone()) { - return; - } runner = Thread.currentThread(); try { + if (result.isDone()) { + return; + } if (period == 0) { result.complete(callable.call()); return; @@ -238,14 +239,14 @@ private void rearm() { if (result.isDone()) { return; } - final long delay; + final long nextTrigger; if (period > 0) { - delay = Math.max(0, triggerTime + period - System.nanoTime()); + nextTrigger = triggerTime + period; } else { - delay = -period; + nextTrigger = System.nanoTime() - period; } try { - arm(delay); + arm(nextTrigger); } catch (RejectedExecutionException e) { result.cancel(false); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 7faf8832fd..f6259dd57a 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -227,7 +227,10 @@ public void run() { var primary = primaryResourceCache.get(primaryID); if (primary.isEmpty()) { log.warn("No resource in cache for resource ID: {}", primaryID); - // no new execution is scheduled in this case, an on delete event should be received shortly + // no new execution is scheduled in this case, an on delete event should be received + // shortly; + // the entry is dropped so that it can't block a registration of a recreated resource + scheduledFutures.remove(primaryID); } else { Set actualResources = null; try { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java index a7adca62f2..c2d2fb5089 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java @@ -17,6 +17,7 @@ import java.time.Duration; import java.util.concurrent.CancellationException; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; @@ -24,6 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -145,6 +147,36 @@ void repeatsTasksAtAFixedRate() { .untilAsserted(() -> assertThat(runs).hasValueGreaterThan(3)); } + @Test + void aFixedRateTaskCatchesUpAfterAnOverrun() { + var starts = new CopyOnWriteArrayList(); + var firstRunEnd = new AtomicLong(); + + executor.scheduleAtFixedRate( + () -> { + starts.add(System.nanoTime()); + if (starts.size() == 1) { + try { + Thread.sleep(350); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + firstRunEnd.set(System.nanoTime()); + } + }, + 0, + 100, + TimeUnit.MILLISECONDS); + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(starts).hasSizeGreaterThanOrEqualTo(4)); + // the three executions missed during the first run are made up for right after it, instead of + // being skipped, which would space the fourth one a full period behind the third + assertThat(Duration.ofNanos(starts.get(3) - firstRunEnd.get())) + .isLessThan(Duration.ofMillis(150)); + } + @Test void aFailingPeriodicTaskIsNotRepeated() { var runs = new AtomicInteger(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index cb9b82e93c..50d8fd6b7d 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -148,6 +148,21 @@ void resourcesDeletedBeforeStartAreNotPolled() { .untilAsserted(() -> verify(supplier, never()).fetchResources(any())); } + @Test + void resourceAbsentFromTheCacheCanBeRegisteredAgain() { + when(resourceCache.get(any())).thenReturn(Optional.empty()); + source.onResourceCreated(testCustomResource); + await() + .pollDelay(Duration.ofMillis(2 * PERIOD)) + .untilAsserted(() -> verify(supplier, never()).fetchResources(any())); + + when(resourceCache.get(any())).thenReturn(Optional.of(testCustomResource)); + source.onResourceUpdated(testCustomResource, testCustomResource); + + await() + .untilAsserted(() -> verify(supplier, atLeast(1)).fetchResources(eq(testCustomResource))); + } + @Test void registeringTaskOnAPredicate() { setUpSource( diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index 3e6fbeaef3..5620d9cb1b 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -162,7 +162,7 @@ public void shutsDownTheExecutorItCreatedItselfAndCreatesANewOneOnRestart() { eventSource.start(); eventSource.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); - untilAsserted(() -> assertThat(handler.events).hasSize(1)); + untilAsserted(() -> assertThat(handler.eventProducingThreadNames).hasSize(1)); var firstThreadName = handler.eventProducingThreadNames.get(0); assertThat(firstThreadName).startsWith("josdk-timer-"); From 2575c9f4900cc933cf106216eaa6afb13c9d8321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 10:48:45 +0100 Subject: [PATCH 21/45] use virtual threads in web app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java | 2 +- .../javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java | 1 + .../io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java | 1 + .../sample/WebPageOperatorManagedDependentResourcesE2E.java | 1 + .../sample/WebPageOperatorStandaloneDependentResourcesE2E.java | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java index 3e8a9df13f..039c9954b5 100644 --- a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java +++ b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java @@ -43,7 +43,7 @@ public static void main(String[] args) throws IOException { overrider -> overrider.withMetrics( new MicrometerMetricsV2.MicrometerMetricsV2Builder(new LoggingMeterRegistry()) - .build())); + .build()).withUseVirtualThreads(true)); MySQLSchemaReconciler schemaReconciler = new MySQLSchemaReconciler(); diff --git a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java index 622c693fd5..b5672f9340 100644 --- a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java +++ b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java @@ -74,6 +74,7 @@ boolean isLocal() { .withReconciler(new MySQLSchemaReconciler()) // configuration for schema comes from // SchemaDependentResource annotation .withInfrastructure(infrastructure) + .withConfigurationService(o->o.withUseVirtualThreads(true)) .withDeleteCRDs(false) .withPortForward(MY_SQL_NS, "app", "mysql", 3306, SchemaDependentResource.LOCAL_PORT) .build() diff --git a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java index 8b0ed5fe69..ac6e595da6 100644 --- a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java +++ b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java @@ -41,6 +41,7 @@ public WebPageOperatorE2E() throws FileNotFoundException {} AbstractOperatorExtension operator = isLocal() ? LocallyRunOperatorExtension.builder() + .withConfigurationService(o -> o.withUseVirtualThreads(true)) .waitForNamespaceDeletion(false) .withDeleteCRDs(false) .withReconciler(new WebPageReconciler()) diff --git a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java index 20c27529d4..7665e82608 100644 --- a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java +++ b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java @@ -41,6 +41,7 @@ public WebPageOperatorManagedDependentResourcesE2E() throws FileNotFoundExceptio ? LocallyRunOperatorExtension.builder() .waitForNamespaceDeletion(false) .withDeleteCRDs(false) + .withConfigurationService(o->o.withUseVirtualThreads(true)) .withReconciler(new WebPageManagedDependentsReconciler()) .build() : ClusterDeployedOperatorExtension.builder() diff --git a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorStandaloneDependentResourcesE2E.java b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorStandaloneDependentResourcesE2E.java index 2073bf3daa..8e667e60e4 100644 --- a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorStandaloneDependentResourcesE2E.java +++ b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorStandaloneDependentResourcesE2E.java @@ -35,6 +35,7 @@ public WebPageOperatorStandaloneDependentResourcesE2E() throws FileNotFoundExcep .waitForNamespaceDeletion(false) .withDeleteCRDs(false) .withReconciler(new WebPageStandaloneDependentsReconciler()) + .withConfigurationService(o -> o.withUseVirtualThreads(true)) .build() : ClusterDeployedOperatorExtension.builder() .waitForNamespaceDeletion(false) From 9e15697f0cc306edd44c0f2d52f063549e7dfc8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 10:59:09 +0100 Subject: [PATCH 22/45] fix: let idle threads of the scheduled thread pools time out ScheduledThreadPoolExecutor starts a core thread on every schedule call until the core size is reached, regardless of the load, and never gives them up. Allow the core threads to time out so that the pools shrink back when idle. --- .../operator/ExecutorUtilsInternal.java | 10 +++- .../operator/ExecutorUtilsInternalTest.java | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/ExecutorUtilsInternalTest.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java index c989a9e0db..e9ebdb6673 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/ExecutorUtilsInternal.java @@ -19,10 +19,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; public class ExecutorUtilsInternal { + private static final long SCHEDULED_THREAD_KEEP_ALIVE_SECONDS = 60; + // prevent instantiation of util class private ExecutorUtilsInternal() {} @@ -50,7 +53,8 @@ public static ThreadFactory daemonThreadFactory(String namePrefix) { * periodic or delayed) tasks on: one whose threads are daemon threads named after the specified * prefix, and which doesn't let the tasks that were scheduled for later delay its shutdown. * - * @param corePoolSize the number of threads to keep in the pool + * @param corePoolSize the maximal number of threads of the pool, which are created on demand and + * let go if idle for a while * @param threadNamePrefix the prefix the threads of the pool are named after * @return a {@link ScheduledExecutorService} to run scheduled tasks on */ @@ -64,6 +68,10 @@ public static ScheduledExecutorService daemonScheduledThreadPool( // cancelled tasks are frequent (every retry that is superseded by a new event cancels one) and // would otherwise be retained until their delay elapses executor.setRemoveOnCancelPolicy(true); + // the threads are created as tasks get scheduled, not as the load requires, so without this + // they would all stay around idle for good; the pool always keeps a thread for delayed tasks + executor.setKeepAliveTime(SCHEDULED_THREAD_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS); + executor.allowCoreThreadTimeOut(true); return executor; } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/ExecutorUtilsInternalTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/ExecutorUtilsInternalTest.java new file mode 100644 index 0000000000..ac2775714e --- /dev/null +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/ExecutorUtilsInternalTest.java @@ -0,0 +1,53 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class ExecutorUtilsInternalTest { + + @Test + void idleThreadsOfTheScheduledPoolAreLetGoButDelayedTasksStillRun() throws Exception { + var executor = + (ScheduledThreadPoolExecutor) + ExecutorUtilsInternal.daemonScheduledThreadPool(4, "idle-test"); + try { + executor.setKeepAliveTime(100, TimeUnit.MILLISECONDS); + var ran = new CountDownLatch(1); + for (int i = 0; i < 4; i++) { + executor.schedule(() -> {}, 0, TimeUnit.MILLISECONDS); + } + executor.schedule(ran::countDown, 1, TimeUnit.SECONDS); + assertThat(executor.getPoolSize()).isGreaterThan(1); + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(executor.getPoolSize()).isLessThanOrEqualTo(1)); + + assertThat(ran.await(10, TimeUnit.SECONDS)).isTrue(); + } finally { + executor.shutdownNow(); + } + } +} From 5aceb5e2a1af618753dff3878557120e67367d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 11:29:11 +0100 Subject: [PATCH 23/45] revert: move the desired state aspect fixes to their own branch --- .../api/reconciler/DefaultContext.java | 35 +-- .../BulkDependentResourceReconciler.java | 10 - .../DesiredStateAspectAuditTest.java | 222 ------------------ 3 files changed, 3 insertions(+), 264 deletions(-) delete mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java index c5e3e9c4e8..50e61526c4 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java @@ -50,8 +50,6 @@ public class DefaultContext

implements Context

{ private final boolean primaryResourceDeleted; private final boolean primaryResourceFinalStateUnknown; private final Map, Object> desiredStates = new ConcurrentHashMap<>(); - private final Map, DesiredStateInProgress> desiredStatesInProgress = - new ConcurrentHashMap<>(); private final ResourceOperations

resourceOperations; public DefaultContext( @@ -256,40 +254,15 @@ public DefaultContext

setRetryInfo(RetryInfo retryInfo) { return this; } - public R getOrComputeDesiredStateFor( - DependentResource dependentResource, Function desiredStateComputer) { - return getOrComputeDesiredStateFor(dependentResource, dependentResource, desiredStateComputer); - } - - /** - * Same as {@link #getOrComputeDesiredStateFor(DependentResource, Function)} but hands {@code - * aspectTarget} rather than {@code dependentResource} to the {@link DesiredStateAspect}s, for - * dependents that are only a stand-in for the one the user configured. - */ @SuppressWarnings("unchecked") public R getOrComputeDesiredStateFor( - DependentResource dependentResource, - DependentResource aspectTarget, - Function desiredStateComputer) { - // an aspect can look up the actual resource of its dependent, which asks for the desired state - // being computed: the thread computing it is handed the state as it is so far, instead of - // re-entering the computation - final var computing = desiredStatesInProgress.get(dependentResource); - if (computing != null && computing.thread() == Thread.currentThread()) { - return (R) computing.desired(); - } + DependentResource dependentResource, Function desiredStateComputer) { return (R) desiredStates.computeIfAbsent( dependentResource, ignored -> { final var desired = desiredStateComputer.apply(getPrimaryResource()); - desiredStatesInProgress.put( - dependentResource, new DesiredStateInProgress(Thread.currentThread(), desired)); - try { - applyDesiredStateAspects(desired, aspectTarget); - } finally { - desiredStatesInProgress.remove(dependentResource); - } + applyDesiredStateAspects(desired, dependentResource); return desired; }); } @@ -300,13 +273,11 @@ public R getOrComputeDesiredStateFor( * resources are therefore left untouched. */ private void applyDesiredStateAspects(Object desired, DependentResource dependentResource) { - if (desired instanceof HasMetadata hasMetadata && controllerConfiguration != null) { + if (desired instanceof HasMetadata hasMetadata) { controllerConfiguration .getConfigurationService() .desiredStateAspects() .forEach(aspect -> aspect.apply(hasMetadata, dependentResource, this)); } } - - private record DesiredStateInProgress(Thread thread, Object desired) {} } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java index bfe46332c9..2428eeba96 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/dependent/BulkDependentResourceReconciler.java @@ -24,10 +24,8 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.api.reconciler.Context; -import io.javaoperatorsdk.operator.api.reconciler.DefaultContext; import io.javaoperatorsdk.operator.api.reconciler.Ignore; import io.javaoperatorsdk.operator.api.reconciler.dependent.Deleter; -import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource; import io.javaoperatorsdk.operator.api.reconciler.dependent.ReconcileResult; class BulkDependentResourceReconciler @@ -117,14 +115,6 @@ public R update(R actual, R desired, P primary, Context

context) { return ((Updater) bulkDependentResource).update(actual, desired, primary, context); } - @Override - @SuppressWarnings("unchecked") - protected R getOrComputeDesired(Context

context) { - return ((DefaultContext

) context) - .getOrComputeDesiredStateFor( - this, (DependentResource) bulkDependentResource, p -> desired); - } - @Override public Result match(R resource, P primary, Context

context) { // retrieve the desired state via the context so that it is processed the same way as for diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java deleted file mode 100644 index 595327ec50..0000000000 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/dependent/DesiredStateAspectAuditTest.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright Java Operator SDK Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.javaoperatorsdk.operator.processing.dependent; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -import org.junit.jupiter.api.Test; - -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.ConfigMapBuilder; -import io.javaoperatorsdk.operator.api.config.ConfigurationService; -import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; -import io.javaoperatorsdk.operator.api.reconciler.Context; -import io.javaoperatorsdk.operator.api.reconciler.DefaultContext; -import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource; -import io.javaoperatorsdk.operator.api.reconciler.dependent.DesiredStateAspect; -import io.javaoperatorsdk.operator.processing.Controller; -import io.javaoperatorsdk.operator.processing.dependent.Matcher.Result; -import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource; -import io.javaoperatorsdk.operator.processing.event.EventSourceManager; -import io.javaoperatorsdk.operator.processing.event.source.EventSource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** Audit repro tests for the DesiredStateAspect feature (commit 110199c47). */ -@SuppressWarnings({"unchecked", "rawtypes"}) -class DesiredStateAspectAuditTest { - - private static final ConfigMap PRIMARY = - new ConfigMapBuilder() - .withNewMetadata() - .withName("primary") - .withNamespace("ns") - .endMetadata() - .build(); - - private static Controller controllerWith(List aspects) { - final ConfigurationService configurationService = mock(); - when(configurationService.desiredStateAspects()).thenReturn(aspects); - final ControllerConfiguration controllerConfiguration = mock(); - when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); - final Controller controller = mock(); - when(controller.getConfiguration()).thenReturn(controllerConfiguration); - return controller; - } - - /** - * The aspect contract says it receives "the DependentResource the desired state was computed - * for". For bulk dependents, it instead receives the private per-item wrapper - * BulkDependentResourceReconciler.BulkDependentResourceInstance, so aspects keyed on the - * dependent (type checks, name(), getClass().getSimpleName() as in DesiredStateAspectIT) - * misbehave. - */ - @Test - void bulkDependentAspectReceivesTheUserBulkDependentResource() { - final List> seen = new ArrayList<>(); - final DesiredStateAspect aspect = - (desired, dependentResource, ctx) -> seen.add(dependentResource); - final var context = - new DefaultContext<>(null, controllerWith(List.of(aspect)), PRIMARY, false, false); - - final var bulk = new CreatingBulk(); - bulk.reconcile(PRIMARY, context); - - assertThat(seen).hasSize(1); - assertThat(seen.get(0).name()).isEqualTo(bulk.name()); - assertThat(seen.get(0)).isSameAs(bulk); - } - - /** - * Aspects run inside ConcurrentHashMap.computeIfAbsent. An aspect that looks up the actual - * resource of the dependent it is given (e.g. to preserve an existing label) re-enters - * getOrComputeDesired for the same key through KubernetesDependentResource - * .selectTargetSecondaryResource -> targetSecondaryResourceID, and CHM throws "Recursive update". - */ - @Test - void aspectCanLookUpActualResourceOfTheDependent() { - final DesiredStateAspect aspect = - (desired, dependentResource, ctx) -> - ((DependentResource) dependentResource) - .getSecondaryResource(ctx.getPrimaryResource(), ctx); - final var controller = controllerWith(List.of(aspect)); - final EventSourceManager esm = mock(); - final EventSource es = mock(); - when(es.getSecondaryResources(any())).thenReturn(Set.of(configMap("child", "actual"))); - when(esm.getEventSourcesFor(ConfigMap.class)).thenReturn(List.of(es)); - when(controller.getEventSourceManager()).thenReturn(esm); - final var context = new DefaultContext<>(null, controller, PRIMARY, false, false); - - final var dependent = new ChildConfigMapDependent(); - - assertThatCode(() -> dependent.getSecondaryResource(PRIMARY, context)) - .doesNotThrowAnyException(); - } - - /** - * Before 110199c47, a DefaultContext built on a Controller whose getConfiguration() is not - * stubbed (the pattern the existing tests used: new DefaultContext<>(mock(), mock(), ...)) could - * compute desired states. Now applyDesiredStateAspects dereferences the null - * ControllerConfiguration for every HasMetadata desired state. - */ - @Test - void desiredComputationDoesNotRequireAspectConfiguration() { - final var context = new DefaultContext<>(null, mock(Controller.class), PRIMARY, false, false); - final var dependent = new ChildConfigMapDependent(); - - assertThatCode(() -> dependent.getOrComputeDesired(context)).doesNotThrowAnyException(); - } - - private static ConfigMap configMap(String name, String value) { - return new ConfigMapBuilder() - .withNewMetadata() - .withName(name) - .withNamespace("ns") - .endMetadata() - .withData(Map.of("key", value)) - .build(); - } - - private static class ChildConfigMapDependent - extends KubernetesDependentResource { - ChildConfigMapDependent() { - super(ConfigMap.class); - } - - @Override - protected ConfigMap desired(ConfigMap primary, Context context) { - return configMap("child", "desired"); - } - - @Override - public ConfigMap getOrComputeDesired(Context context) { - return super.getOrComputeDesired(context); - } - } - - private static class CreatingBulk extends AbstractDependentResource - implements BulkDependentResource, - Creator { - - CreatingBulk() { - super("my-bulk"); - } - - @Override - public Map desiredResources(ConfigMap primary, Context context) { - var desired = new LinkedHashMap(); - desired.put("a", configMap("a", "desired")); - return desired; - } - - @Override - public Map getSecondaryResources( - ConfigMap primary, Context context) { - return Map.of(); - } - - @Override - public ConfigMap create(ConfigMap desired, ConfigMap primary, Context context) { - return desired; - } - - @Override - public void deleteTargetResource( - ConfigMap primary, ConfigMap resource, String key, Context context) {} - - @Override - public Result match( - ConfigMap actualResource, - ConfigMap desired, - ConfigMap primary, - Context context) { - return Result.computed(false, desired); - } - - @Override - public Result match( - ConfigMap resource, ConfigMap primary, Context context) { - return Result.computed(false, resource); - } - - @Override - protected Optional selectTargetSecondaryResource( - Set secondaryResources, ConfigMap primary, Context context) { - return Optional.empty(); - } - - @Override - protected void onCreated(ConfigMap primary, ConfigMap created, Context context) {} - - @Override - protected void onUpdated( - ConfigMap primary, ConfigMap updated, ConfigMap actual, Context context) {} - - @Override - public Class resourceType() { - return ConfigMap.class; - } - } -} From 3324293d7d33da0d17682b1b3a282e9972d5db6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 11:49:10 +0100 Subject: [PATCH 24/45] revert: move the per resource polling resilience fixes to their own branch Co-Authored-By: Claude Sonnet 5.5 --- .../PerResourcePollingEventSource.java | 14 +++------- .../PerResourcePollingEventSourceTest.java | 27 ------------------- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index f6259dd57a..658e2cd73e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -227,18 +227,10 @@ public void run() { var primary = primaryResourceCache.get(primaryID); if (primary.isEmpty()) { log.warn("No resource in cache for resource ID: {}", primaryID); - // no new execution is scheduled in this case, an on delete event should be received - // shortly; - // the entry is dropped so that it can't block a registration of a recreated resource - scheduledFutures.remove(primaryID); + // no new execution is scheduled in this case, an on delete event should be received shortly } else { - Set actualResources = null; - try { - actualResources = getAndCacheResource(primary.get(), false); - } catch (RuntimeException e) { - log.error("Error during polling resources for: {}", primaryID, e); - } - scheduleNextExecution(primary.get(), actualResources); + var actualResources = primary.map(p -> getAndCacheResource(p, false)); + scheduleNextExecution(primary.get(), actualResources.orElse(null)); } } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 50d8fd6b7d..c6736dc741 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -90,18 +90,6 @@ void pollsTheResourceAfterAwareOfIt() { }); } - @Test - void keepsPollingAfterTheFetcherFailed() { - when(supplier.fetchResources(any())) - .thenThrow(new IllegalStateException("temporary failure")) - .thenReturn(Set.of(SampleExternalResource.testResource1())); - - source.onResourceCreated(testCustomResource); - - await() - .untilAsserted(() -> verify(supplier, atLeast(3)).fetchResources(eq(testCustomResource))); - } - @Test void resourcesSeenBeforeStartArePolledOnceStarted() { source.stop(); @@ -148,21 +136,6 @@ void resourcesDeletedBeforeStartAreNotPolled() { .untilAsserted(() -> verify(supplier, never()).fetchResources(any())); } - @Test - void resourceAbsentFromTheCacheCanBeRegisteredAgain() { - when(resourceCache.get(any())).thenReturn(Optional.empty()); - source.onResourceCreated(testCustomResource); - await() - .pollDelay(Duration.ofMillis(2 * PERIOD)) - .untilAsserted(() -> verify(supplier, never()).fetchResources(any())); - - when(resourceCache.get(any())).thenReturn(Optional.of(testCustomResource)); - source.onResourceUpdated(testCustomResource, testCustomResource); - - await() - .untilAsserted(() -> verify(supplier, atLeast(1)).fetchResources(eq(testCustomResource))); - } - @Test void registeringTaskOnAPredicate() { setUpSource( From 410b32ba13d4672985971ce728483cf199727cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 12:12:20 +0100 Subject: [PATCH 25/45] fix format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/sample/MySQLSchemaOperator.java | 9 ++++++--- .../operator/sample/MySQLSchemaOperatorE2E.java | 2 +- sample-operators/webpage/src/main/resources/log4j2.xml | 2 +- .../WebPageOperatorManagedDependentResourcesE2E.java | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java index 039c9954b5..e8a2d6838c 100644 --- a/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java +++ b/sample-operators/mysql-schema/src/main/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperator.java @@ -41,9 +41,12 @@ public static void main(String[] args) throws IOException { Operator operator = new Operator( overrider -> - overrider.withMetrics( - new MicrometerMetricsV2.MicrometerMetricsV2Builder(new LoggingMeterRegistry()) - .build()).withUseVirtualThreads(true)); + overrider + .withMetrics( + new MicrometerMetricsV2.MicrometerMetricsV2Builder( + new LoggingMeterRegistry()) + .build()) + .withUseVirtualThreads(true)); MySQLSchemaReconciler schemaReconciler = new MySQLSchemaReconciler(); diff --git a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java index b5672f9340..70e98d5c92 100644 --- a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java +++ b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java @@ -74,7 +74,7 @@ boolean isLocal() { .withReconciler(new MySQLSchemaReconciler()) // configuration for schema comes from // SchemaDependentResource annotation .withInfrastructure(infrastructure) - .withConfigurationService(o->o.withUseVirtualThreads(true)) + .withConfigurationService(o -> o.withUseVirtualThreads(true)) .withDeleteCRDs(false) .withPortForward(MY_SQL_NS, "app", "mysql", 3306, SchemaDependentResource.LOCAL_PORT) .build() diff --git a/sample-operators/webpage/src/main/resources/log4j2.xml b/sample-operators/webpage/src/main/resources/log4j2.xml index c8c007835f..2979258355 100644 --- a/sample-operators/webpage/src/main/resources/log4j2.xml +++ b/sample-operators/webpage/src/main/resources/log4j2.xml @@ -23,7 +23,7 @@ - + diff --git a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java index 7665e82608..1c8df953ae 100644 --- a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java +++ b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorManagedDependentResourcesE2E.java @@ -41,7 +41,7 @@ public WebPageOperatorManagedDependentResourcesE2E() throws FileNotFoundExceptio ? LocallyRunOperatorExtension.builder() .waitForNamespaceDeletion(false) .withDeleteCRDs(false) - .withConfigurationService(o->o.withUseVirtualThreads(true)) + .withConfigurationService(o -> o.withUseVirtualThreads(true)) .withReconciler(new WebPageManagedDependentsReconciler()) .build() : ClusterDeployedOperatorExtension.builder() From e518f6555215ea39e26d09a5e970bd79541af823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 13:45:41 +0100 Subject: [PATCH 26/45] revert: move the keep alive thread to its own branch Co-Authored-By: Claude Sonnet 5.5 --- .../documentation/operations/configuration.md | 6 +- .../api/config/ExecutorServiceManager.java | 28 ------ .../config/ExecutorServiceManagerTest.java | 33 ------- .../config/VirtualThreadsJvmLifetimeTest.java | 88 ------------------- 4 files changed, 3 insertions(+), 152 deletions(-) delete mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index ac48696603..6b877af846 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -57,9 +57,9 @@ Things to keep in mind: virtual thread. The number of those tasks running at the same time is not limited, so `withConcurrentScheduledTaskThreads(int)` and `withConcurrentRetryAndRescheduleThreads(int)` only apply to platform threads. -- With virtual threads, which are always daemon threads, the framework keeps one idle platform - thread (`josdk-keep-alive`) running until the operator is stopped, so that an operator whose - `main` method returns after starting it doesn't let the JVM exit. +- Virtual threads are always daemon threads, so unlike the threads of the platform thread pools they + don't keep the JVM running: if the `main` method of your application returns after starting the + operator, make sure something else keeps the JVM alive. - A custom `ExecutorService` provided through `withExecutorService(...)` or `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. - The Kubernetes client the framework creates when none is provided also switches its internal diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index b937a4c022..0b06f8b9b7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.locks.LockSupport; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -47,7 +46,6 @@ public class ExecutorServiceManager { private ScheduledExecutorService retryAndRescheduleExecutorService; private boolean started; private ConfigurationService configurationService; - private Thread keepAliveThread; ExecutorServiceManager(ConfigurationService configurationService) { start(configurationService); @@ -218,37 +216,11 @@ public synchronized void start(ConfigurationService configurationService) { this.retryAndRescheduleExecutorService = configurationService.getRetryAndRescheduleExecutorService(); this.executor = new InstrumentedExecutorService(configurationService.getExecutorService()); - if (VirtualThreads.shouldUse(configurationService.useVirtualThreads())) { - startKeepAliveThread(); - } started = true; } } - /** - * Virtual threads are always daemon threads, so with them nothing is left to keep the JVM running - * once the main method of an operator returns after starting it, as opposed to the non-daemon - * threads of the platform thread pools. This thread restores that, until the executors are - * stopped. - */ - private void startKeepAliveThread() { - keepAliveThread = - new Thread( - () -> { - while (!Thread.currentThread().isInterrupted()) { - LockSupport.park(); - } - }, - "josdk-keep-alive"); - keepAliveThread.setDaemon(false); - keepAliveThread.start(); - } - public synchronized void stop(Duration gracefulShutdownTimeout) { - if (keepAliveThread != null) { - keepAliveThread.interrupt(); - keepAliveThread = null; - } var shutdowns = List.of( shutdown(executor, gracefulShutdownTimeout), diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java index 33900d834a..ea6725896f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java @@ -16,14 +16,12 @@ package io.javaoperatorsdk.operator.api.config; import java.time.Duration; -import java.util.Optional; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledForJreRange; import org.junit.jupiter.api.condition.JRE; import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; class ExecutorServiceManagerTest { @@ -82,35 +80,4 @@ void cachingExecutorRunsOnVirtualThreadsWhenRequested() throws Exception { manager.stop(SHUTDOWN_TIMEOUT); } } - - @Test - @EnabledForJreRange(min = JRE.JAVA_25) - void keepAliveThreadRunsOnlyWhileTheManagerIsStartedWithVirtualThreads() throws Exception { - var configurationService = - ConfigurationService.newOverriddenConfigurationService( - new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); - var manager = configurationService.getExecutorServiceManager(); - - assertThat(keepAliveThread()).hasValueSatisfying(t -> assertThat(t.isDaemon()).isFalse()); - - manager.stop(SHUTDOWN_TIMEOUT); - - await().untilAsserted(() -> assertThat(keepAliveThread()).isEmpty()); - } - - @Test - void noKeepAliveThreadWithPlatformThreads() { - var manager = new BaseConfigurationService().getExecutorServiceManager(); - try { - assertThat(keepAliveThread()).isEmpty(); - } finally { - manager.stop(SHUTDOWN_TIMEOUT); - } - } - - private static Optional keepAliveThread() { - return Thread.getAllStackTraces().keySet().stream() - .filter(t -> t.getName().equals("josdk-keep-alive") && t.isAlive()) - .findFirst(); - } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java deleted file mode 100644 index 6ff0cc32fd..0000000000 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsJvmLifetimeTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Java Operator SDK Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.javaoperatorsdk.operator.api.config; - -import java.nio.file.Path; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledForJreRange; -import org.junit.jupiter.api.condition.JRE; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Throwaway audit test: the usual operator main method ends right after {@code operator.start()} - * (see docs/.../operations/configuration.md and several samples) and relies on the framework's - * non-daemon executor threads to keep the JVM alive. Virtual threads are always daemon threads, so - * with virtual threads enabled nothing keeps the JVM running once main returns. - */ -class VirtualThreadsJvmLifetimeTest { - - /** Mimics an operator main: start the executors, run a reconciliation, return from main. */ - public static class OperatorLikeMain { - public static void main(String[] args) throws Exception { - final boolean useVirtualThreads = Boolean.parseBoolean(args[0]); - final var configService = - ConfigurationService.newOverriddenConfigurationService( - new BaseConfigurationService(), o -> o.withUseVirtualThreads(useVirtualThreads)); - final var esm = configService.getExecutorServiceManager(); - // what Operator.start() does: start informers on the caching executor ... - esm.cachingExecutorService().submit(() -> {}).get(); - // ... and a first reconciliation dispatched by the EventProcessor - esm.reconcileExecutorService().submit(() -> {}).get(); - System.out.println("main returning"); - } - } - - private static boolean jvmStillAliveAfterMainReturns(boolean useVirtualThreads) throws Exception { - final var java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); - final var process = - new ProcessBuilder( - java, - "-cp", - System.getProperty("java.class.path"), - OperatorLikeMain.class.getName(), - Boolean.toString(useVirtualThreads)) - .redirectErrorStream(true) - .start(); - try { - final boolean exited = process.waitFor(10, TimeUnit.SECONDS); - if (exited) { - final var output = new String(process.getInputStream().readAllBytes()); - // make sure the child exited normally after main returned, not because it crashed - assertThat(output).contains("main returning"); - assertThat(process.exitValue()).isZero(); - } - return !exited; - } finally { - process.destroyForcibly(); - } - } - - @Test - void controlPlatformThreadsKeepTheOperatorJvmAlive() throws Exception { - assertThat(jvmStillAliveAfterMainReturns(false)).isTrue(); - } - - @Test - @EnabledForJreRange(min = JRE.JAVA_25) - void virtualThreadsKeepTheOperatorJvmAliveToo() throws Exception { - assertThat(jvmStillAliveAfterMainReturns(true)) - .as("JVM exited right after main returned with useVirtualThreads=true") - .isTrue(); - } -} From 1e89aff8f536d5a5582661f8184d6330d5558763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 29 Oct 2026 14:28:20 +0100 Subject: [PATCH 27/45] fix: reject polls of a previous run after a restart and drop cancelled handles from the waiting set - a per resource poll that outlives a stop and start of the event source no longer reschedules itself next to the polling chain of the new run - a periodic task cancelled while being re-armed is removed from the waiting set of the scheduled executor Co-Authored-By: Claude Sonnet 5.5 --- .../VirtualThreadScheduledExecutor.java | 1 + .../PerResourcePollingEventSource.java | 35 ++++++++++------- .../PerResourcePollingEventSourceTest.java | 38 +++++++++++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java index 3bd273d85f..61d7c74d66 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java @@ -199,6 +199,7 @@ private void arm(long triggerNanos) { } } if (result.isDone()) { + waiting.remove(this); timerFuture.cancel(false); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 658e2cd73e..1315f627b9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -69,6 +69,10 @@ public class PerResourcePollingEventSource private volatile ScheduledExecutorService executorService; + // incremented on every stop, to tell the polls of previous runs apart, which can outlive the + // stop when the fetcher ignores the interruption; guarded by this + private long lifecycleGeneration; + public PerResourcePollingEventSource( Class resourceClass, EventSourceContext

context, @@ -140,7 +144,11 @@ private Set getAndCacheResource(P primary, boolean fromGetter) { } @SuppressWarnings("unchecked") - private void scheduleNextExecution(P primary, Set actualResources) { + private synchronized void scheduleNextExecution( + P primary, Set actualResources, long generation) { + if (generation != lifecycleGeneration) { + return; + } var primaryID = ResourceID.fromResource(primary); var fetchDelay = resourceFetcher.fetchDelay(actualResources, primary); var fetchDuration = fetchDelay.orElse(period); @@ -148,14 +156,10 @@ private void scheduleNextExecution(P primary, Set actualResources) { ScheduledFuture scheduledFuture = (ScheduledFuture) executorService.schedule( - new FetchingExecutor(primaryID), fetchDuration.toMillis(), TimeUnit.MILLISECONDS); + new FetchingExecutor(primaryID, generation), + fetchDuration.toMillis(), + TimeUnit.MILLISECONDS); scheduledFutures.put(primaryID, scheduledFuture); - // a fetch that was already running when the event source got stopped can get here after stop - // cleared the map; since the shared executor stays alive, the task has to be withdrawn, - // otherwise the stale entry would prevent the resource from being registered on a restart - if (!isRunning() && scheduledFutures.remove(primaryID, scheduledFuture)) { - scheduledFuture.cancel(true); - } } @Override @@ -206,15 +210,17 @@ private synchronized void registerTask(ResourceID primaryID, P resource) { var actualResources = cachedResources.isEmpty() ? null : cachedResources; // note that there is a delay, to not do two fetches when the resources first appeared // and getSecondaryResource is called on reconciliation. - scheduleNextExecution(resource, actualResources); + scheduleNextExecution(resource, actualResources, lifecycleGeneration); } } private class FetchingExecutor implements Runnable { private final ResourceID primaryID; + private final long generation; - public FetchingExecutor(ResourceID primaryID) { + public FetchingExecutor(ResourceID primaryID, long generation) { this.primaryID = primaryID; + this.generation = generation; } @Override @@ -230,7 +236,7 @@ public void run() { // no new execution is scheduled in this case, an on delete event should be received shortly } else { var actualResources = primary.map(p -> getAndCacheResource(p, false)); - scheduleNextExecution(primary.get(), actualResources.orElse(null)); + scheduleNextExecution(primary.get(), actualResources.orElse(null), generation); } } } @@ -283,8 +289,11 @@ public void stop() throws OperatorException { super.stop(); // the tasks have to be cancelled explicitly now that the executor can be shared with the rest // of the operator, and the map cleared so that they are registered again on a restart - scheduledFutures.values().forEach(future -> future.cancel(true)); - scheduledFutures.clear(); + synchronized (this) { + lifecycleGeneration++; + scheduledFutures.values().forEach(future -> future.cancel(true)); + scheduledFutures.clear(); + } registrationsBeforeStart.clear(); if (ownsExecutorService && executorService != null) { executorService.shutdownNow(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index c6736dc741..93d5f593ce 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -19,6 +19,8 @@ import java.util.Collections; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -276,6 +278,42 @@ void pollsAgainAfterRestart() { await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); } + @Test + void aFetchOutlivingARestartDoesNotStartASecondPollingChain() throws Exception { + var fetching = new CountDownLatch(1); + var release = new CountDownLatch(1); + when(supplier.fetchResources(any())) + .thenAnswer( + invocation -> { + fetching.countDown(); + var interrupted = false; + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return Set.of(SampleExternalResource.testResource1()); + }) + .thenReturn(Set.of(SampleExternalResource.testResource1())); + + source.onResourceCreated(testCustomResource); + assertThat(fetching.await(5, TimeUnit.SECONDS)).isTrue(); + source.stop(); + source.start(); + source.onResourceCreated(testCustomResource); + clearInvocations(supplier); + release.countDown(); + + await() + .pollDelay(Duration.ofMillis(10 * PERIOD)) + .untilAsserted(() -> verify(supplier, atMost(15)).fetchResources(any())); + } + @Test void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { var configurationService = new BaseConfigurationService(); From 0ad0137d9a63dd0026f6acd3b98a3d12bf403f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 30 Oct 2026 09:48:08 +0100 Subject: [PATCH 28/45] fix: keep the scheduled executors of the original configuration unless an override changes their settings Co-Authored-By: Claude Sonnet 5.5 --- .../config/ConfigurationServiceOverrider.java | 14 ++++++-- .../ConfigurationServiceOverriderTest.java | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index b5ccf75a5c..be7fb8e6c5 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -449,7 +449,8 @@ public ExecutorService getWorkflowExecutorService() { public ScheduledExecutorService getScheduledExecutorService() { if (scheduledExecutorService != null) { return scheduledExecutorService; - } else if (concurrentScheduledTaskThreads == null && useVirtualThreads == null) { + } else if (!changes( + concurrentScheduledTaskThreads, original.concurrentScheduledTaskThreads())) { return original.getScheduledExecutorService(); } else { return super.getScheduledExecutorService(); @@ -460,13 +461,22 @@ public ScheduledExecutorService getScheduledExecutorService() { public ScheduledExecutorService getRetryAndRescheduleExecutorService() { if (retryAndRescheduleExecutorService != null) { return retryAndRescheduleExecutorService; - } else if (concurrentRetryAndRescheduleThreads == null && useVirtualThreads == null) { + } else if (!changes( + concurrentRetryAndRescheduleThreads, original.concurrentRetryAndRescheduleThreads())) { return original.getRetryAndRescheduleExecutorService(); } else { return super.getRetryAndRescheduleExecutorService(); } } + // the default scheduled executors are only built anew when an override actually changes the + // way they are built, otherwise the ones of the original configuration are kept, which also + // preserves a custom executor the original provides + private boolean changes(Integer overriddenThreads, int originalThreads) { + return (overriddenThreads != null && overriddenThreads != originalThreads) + || (useVirtualThreads != null && useVirtualThreads != original.useVirtualThreads()); + } + @Override public Optional getLeaderElectionConfiguration() { return leaderElectionConfiguration != null diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java index e91688152e..c5ae255d60 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java @@ -183,6 +183,40 @@ public ScheduledExecutorService getRetryAndRescheduleExecutorService() { } } + @Test + void executorsOfTheOriginalConfigurationAreKeptWhenOverridesDoNotChangeTheirSettings() { + final var scheduledExecutorService = Executors.newScheduledThreadPool(1); + final var retryExecutorService = Executors.newScheduledThreadPool(1); + try { + final var original = + new BaseConfigurationService() { + @Override + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + @Override + public ScheduledExecutorService getRetryAndRescheduleExecutorService() { + return retryExecutorService; + } + }; + + final var overridden = + new ConfigurationServiceOverrider(original) + .withUseVirtualThreads(original.useVirtualThreads()) + .withConcurrentScheduledTaskThreads(original.concurrentScheduledTaskThreads()) + .withConcurrentRetryAndRescheduleThreads( + original.concurrentRetryAndRescheduleThreads()) + .build(); + + assertThat(overridden.getScheduledExecutorService()).isSameAs(scheduledExecutorService); + assertThat(overridden.getRetryAndRescheduleExecutorService()).isSameAs(retryExecutorService); + } finally { + scheduledExecutorService.shutdownNow(); + retryExecutorService.shutdownNow(); + } + } + @Test void scheduledExecutorDefaultsToADaemonPoolOfTheConfiguredSize() { final var overridden = From b5b89460dfbbb70ac71378f0c26a96f187f7aaad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 30 Oct 2026 10:14:19 +0100 Subject: [PATCH 29/45] fix: hold back registrations racing stop and follow the operator's executors when started again while running - a per resource registration that races stop() is buffered until the next start instead of leaving a stale entry that blocks polling after a restart - the timer and per resource polling event sources resolve their executor again when started while running, which happens when the operator is started again after a failed start and replaces its executors Co-Authored-By: Claude Sonnet 5.5 --- .../PerResourcePollingEventSource.java | 29 +++++++ .../event/source/timer/TimerEventSource.java | 10 ++- .../PerResourcePollingEventSourceTest.java | 75 +++++++++++++++++++ .../source/timer/TimerEventSourceTest.java | 22 ++++++ 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 1315f627b9..3874836233 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -104,6 +104,7 @@ private static ConfigurationService configurationServiceOf(EventSourceContext @Override public void start() throws OperatorException { if (isRunning()) { + switchToCurrentExecutor(); return; } executorService = resolveExecutorService(); @@ -111,6 +112,28 @@ public void start() throws OperatorException { replayRegistrationsBeforeStart(); } + /** + * The operator replaces its executors when it is started again, which can happen while this event + * source keeps running, when a previous start failed after it started. The polls scheduled on the + * replaced executor are gone then, so they are scheduled again on the new one. + */ + private void switchToCurrentExecutor() { + if (ownsExecutorService) { + return; + } + var resolved = resolveExecutorService(); + synchronized (this) { + if (resolved == executorService) { + return; + } + executorService = resolved; + lifecycleGeneration++; + scheduledFutures.values().forEach(future -> future.cancel(true)); + scheduledFutures.clear(); + } + primaryResourceCache.list().forEach(this::checkAndRegisterTask); + } + private void replayRegistrationsBeforeStart() { for (var primaryID : registrationsBeforeStart.keySet()) { var resource = registrationsBeforeStart.remove(primaryID); @@ -204,6 +227,12 @@ private void checkAndRegisterTask(P resource) { } private synchronized void registerTask(ResourceID primaryID, P resource) { + // stop() clears the running flag before it takes the monitor, so a registration that raced it + // has to be held back, otherwise a task scheduled after the stop would block a restart + if (!isRunning()) { + registrationsBeforeStart.put(primaryID, resource); + return; + } if (scheduledFutures.get(primaryID) == null && (registerPredicate == null || registerPredicate.test(resource))) { var cachedResources = cachedResourcesFor(primaryID); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index 07de4f0d7e..dd7d05e16b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -63,7 +63,7 @@ public TimerEventSource(String name, boolean triggerReconcilerOnAllEvents) { * * @param executorServiceSupplier supplies the executor to schedule on, {@code null} to have the * event source create, and shut down, an executor of its own - * @since 5.6.0 + * @since 5.7.0 */ public TimerEventSource(Supplier executorServiceSupplier) { super(Void.class); @@ -73,7 +73,7 @@ public TimerEventSource(Supplier executorServiceSuppli /** * @see #TimerEventSource(Supplier) - * @since 5.6.0 + * @since 5.7.0 */ public TimerEventSource( String name, @@ -148,8 +148,12 @@ public void cancelOnceSchedule(ResourceID customResourceUid) { @Override public void start() { - if (!isRunning()) { + // the operator replaces its executors when it is started again, which can happen while this + // event source keeps running, when a previous start failed after it started + if (!isRunning() || !ownsExecutorService) { executorService = resolveExecutorService(); + } + if (!isRunning()) { super.start(); } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 93d5f593ce..11a97308c0 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -314,6 +315,80 @@ void aFetchOutlivingARestartDoesNotStartASecondPollingChain() throws Exception { .untilAsserted(() -> verify(supplier, atMost(15)).fetchResources(any())); } + @Test + void aRegistrationRacingStopDoesNotBlockPollingAfterARestart() throws Exception { + var configurationService = new BaseConfigurationService(); + var executorServiceManager = configurationService.getExecutorServiceManager(); + var controllerConfiguration = mock(ControllerConfiguration.class); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); + setUpSource( + new PerResourcePollingEventSource<>( + SampleExternalResource.class, + context, + new PerResourcePollingConfigurationBuilder< + SampleExternalResource, TestCustomResource, String>( + supplier, Duration.ofMillis(PERIOD)) + .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) + .build())); + + try { + Thread informer; + synchronized (source) { + informer = new Thread(() -> source.onResourceCreated(testCustomResource)); + informer.start(); + await().until(() -> informer.getState() == Thread.State.BLOCKED); + source.stop(); + } + informer.join(); + executorServiceManager.stop(Duration.ofMillis(100)); + executorServiceManager.start(configurationService); + + source.start(); + clearInvocations(supplier); + source.onResourceCreated(testCustomResource); + + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + } finally { + source.stop(); + executorServiceManager.stop(Duration.ofMillis(100)); + } + } + + @Test + void pollsOnTheNewSharedSchedulerWhenStartedAgainWhileRunning() { + var configurationService = new BaseConfigurationService(); + var executorServiceManager = configurationService.getExecutorServiceManager(); + var controllerConfiguration = mock(ControllerConfiguration.class); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); + when(resourceCache.list()).thenAnswer(i -> Stream.of(testCustomResource)); + setUpSource( + new PerResourcePollingEventSource<>( + SampleExternalResource.class, + context, + new PerResourcePollingConfigurationBuilder< + SampleExternalResource, TestCustomResource, String>( + supplier, Duration.ofMillis(PERIOD)) + .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) + .build())); + + try { + source.onResourceCreated(testCustomResource); + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + + executorServiceManager.stop(Duration.ofMillis(100)); + executorServiceManager.start(configurationService); + clearInvocations(supplier); + source.start(); + + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + } finally { + source.stop(); + executorServiceManager.stop(Duration.ofMillis(100)); + } + } + @Test void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { var configurationService = new BaseConfigurationService(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index 5620d9cb1b..769968b738 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -18,7 +18,9 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.awaitility.Awaitility; import org.awaitility.core.ConditionFactory; @@ -96,6 +98,26 @@ public void deRegistersOnceEventSources() { assertThat(source.getStatus()).isEqualTo(Status.HEALTHY); } + @Test + public void schedulesOnTheNewExecutorWhenStartedAgainWhileRunning() { + var first = Executors.newScheduledThreadPool(1); + var second = Executors.newScheduledThreadPool(1); + var current = new AtomicReference(first); + var timerSource = new TimerEventSource("timer", false, current::get); + setUpSource(timerSource, new CapturingEventHandler()); + try { + first.shutdownNow(); + current.set(second); + + source.start(); + source.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); + + untilAsserted(() -> assertThat(eventHandler.events).hasSize(1)); + } finally { + second.shutdownNow(); + } + } + @Test public void eventNotRegisteredIfStopped() { var resourceID = ResourceID.fromResource(TestUtils.testCustomResource()); From b47c7197f6a4df1ff18e6d6c4ad6164aaccf0a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 30 Oct 2026 10:14:19 +0100 Subject: [PATCH 30/45] docs: correct the since version of the scheduled executor API to 5.7.0 Co-Authored-By: Claude Sonnet 5.5 --- .../operator/api/config/ConfigurationService.java | 8 ++++---- .../api/config/ConfigurationServiceOverrider.java | 9 +++++---- .../source/polling/PollingConfigurationBuilder.java | 2 +- .../event/source/polling/PollingEventSource.java | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 25572333a3..06f0797317 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -264,7 +264,7 @@ default int concurrentWorkflowExecutorThreads() { * limited then. * * @return the maximum number of concurrent scheduled task threads - * @since 5.6.0 + * @since 5.7.0 */ default int concurrentScheduledTaskThreads() { return DEFAULT_SCHEDULED_TASK_THREADS_NUMBER; @@ -278,7 +278,7 @@ default int concurrentScheduledTaskThreads() { * see {@link #useVirtualThreads()}. * * @return the maximum number of concurrent retry and reschedule threads - * @since 5.6.0 + * @since 5.7.0 */ default int concurrentRetryAndRescheduleThreads() { return DEFAULT_RETRY_AND_RESCHEDULE_THREADS_NUMBER; @@ -368,7 +368,7 @@ default ExecutorService getWorkflowExecutorService() { * Custom implementations are advised to do the same. * * @return the {@link ScheduledExecutorService} implementation to use to run scheduled tasks - * @since 5.6.0 + * @since 5.7.0 */ default ScheduledExecutorService getScheduledExecutorService() { return ExecutorServiceManager.newScheduledExecutorService( @@ -386,7 +386,7 @@ default ScheduledExecutorService getScheduledExecutorService() { * * @return the {@link ScheduledExecutorService} implementation to use to trigger retried and * rescheduled reconciliations - * @since 5.6.0 + * @since 5.7.0 */ default ScheduledExecutorService getRetryAndRescheduleExecutorService() { return ExecutorServiceManager.newScheduledExecutorService( diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index be7fb8e6c5..c4e514c282 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -99,7 +99,7 @@ public ConfigurationServiceOverrider withConcurrentWorkflowExecutorThreads(int t * @param threadNumber the maximum number of concurrent scheduled task threads * @return this {@link ConfigurationServiceOverrider} for chained customization * @see ConfigurationService#concurrentScheduledTaskThreads() - * @since 5.6.0 + * @since 5.7.0 */ public ConfigurationServiceOverrider withConcurrentScheduledTaskThreads(int threadNumber) { this.concurrentScheduledTaskThreads = threadNumber; @@ -113,7 +113,7 @@ public ConfigurationServiceOverrider withConcurrentScheduledTaskThreads(int thre * @param threadNumber the maximum number of concurrent retry and reschedule threads * @return this {@link ConfigurationServiceOverrider} for chained customization * @see ConfigurationService#concurrentRetryAndRescheduleThreads() - * @since 5.6.0 + * @since 5.7.0 */ public ConfigurationServiceOverrider withConcurrentRetryAndRescheduleThreads(int threadNumber) { this.concurrentRetryAndRescheduleThreads = threadNumber; @@ -160,7 +160,7 @@ public ConfigurationServiceOverrider withWorkflowExecutorService( * @param scheduledExecutorService the executor to run scheduled tasks on * @return this {@link ConfigurationServiceOverrider} for chained customization * @see ConfigurationService#getScheduledExecutorService() - * @since 5.6.0 + * @since 5.7.0 */ public ConfigurationServiceOverrider withScheduledExecutorService( ScheduledExecutorService scheduledExecutorService) { @@ -176,7 +176,7 @@ public ConfigurationServiceOverrider withScheduledExecutorService( * reconciliations on * @return this {@link ConfigurationServiceOverrider} for chained customization * @see ConfigurationService#getRetryAndRescheduleExecutorService() - * @since 5.6.0 + * @since 5.7.0 */ public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( ScheduledExecutorService retryAndRescheduleExecutorService) { @@ -191,6 +191,7 @@ public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( * * @param useVirtualThreads {@code true} to use virtual threads * @return this {@link ConfigurationServiceOverrider} for chained customization + * @since 5.7.0 */ public ConfigurationServiceOverrider withUseVirtualThreads(boolean useVirtualThreads) { this.useVirtualThreads = useVirtualThreads; diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java index 5a12fc4d50..80e8dc06df 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java @@ -51,7 +51,7 @@ public PollingConfigurationBuilder withName(String name) { * * @param executorService the executor to run the polls on * @return this builder for chained customization - * @since 5.6.0 + * @since 5.7.0 */ public PollingConfigurationBuilder withExecutorService( ScheduledExecutorService executorService) { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index de2d7a9bc1..0115e6d6fc 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -83,7 +83,7 @@ public class PollingEventSource * @param resourceClass the type of the polled resource * @param context the context this event source is created for * @param config the configuration of the polling - * @since 5.6.0 + * @since 5.7.0 */ public PollingEventSource( Class resourceClass, EventSourceContext

context, PollingConfiguration config) { @@ -96,7 +96,7 @@ public PollingEventSource( * source has to create a thread of its own to poll on instead of using the one the operator * shares between all its scheduled tasks. */ - @Deprecated(since = "5.6.0") + @Deprecated(since = "5.7.0") public PollingEventSource(Class resourceClass, PollingConfiguration config) { this(resourceClass, config, null); } From aab8fe21c76dce640adcf7762eadcd5bb82f6900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 30 Oct 2026 13:33:08 +0100 Subject: [PATCH 31/45] fix: restart polling and re-arm timers on the operator's new executors, and let in-flight work finish when stopping owned executors - PollingEventSource schedules its polling task again when started while running after the operator replaced its executors - TimerEventSource schedules the timers pending on the replaced executor again for their remaining delay - stopping a polling or timer event source that owns its executor no longer interrupts a poll or handler in flight Co-Authored-By: Claude Sonnet 5.5 --- .../source/polling/PollingEventSource.java | 26 ++++++- .../event/source/timer/TimerEventSource.java | 23 +++++- .../polling/PollingEventSourceTest.java | 59 +++++++++++++++ .../source/timer/TimerEventSourceTest.java | 71 +++++++++++++++++++ 4 files changed, 176 insertions(+), 3 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index 0115e6d6fc..b70b9ce550 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -130,11 +130,16 @@ private static ConfigurationService configurationServiceOf(EventSourceContext @Override public void start() throws OperatorException { if (pollingTask != null) { + switchToCurrentExecutor(); return; } super.start(); executorService = resolveExecutorService(); getStateAndFillCache(); + schedulePolling(); + } + + private void schedulePolling() { var task = executorService.scheduleWithFixedDelay( this::poll, period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS); @@ -147,6 +152,25 @@ public void start() throws OperatorException { } } + /** + * The operator replaces its executors when it is started again, which can happen while this event + * source keeps running, when a previous start failed after it started. The polling task is gone + * with the replaced executor then, so it is scheduled again on the new one. + */ + private void switchToCurrentExecutor() { + final var current = pollingTask; + if (ownsExecutorService || current == null) { + return; + } + final var resolved = resolveExecutorService(); + if (resolved == executorService) { + return; + } + current.cancel(false); + executorService = resolved; + schedulePolling(); + } + private void poll() { try { if (!isRunning()) { @@ -195,7 +219,7 @@ public void stop() throws OperatorException { pollingTask = null; } if (ownsExecutorService && executorService != null) { - executorService.shutdownNow(); + executorService.shutdown(); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index dd7d05e16b..bcfea40fec 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -151,13 +151,32 @@ public void start() { // the operator replaces its executors when it is started again, which can happen while this // event source keeps running, when a previous start failed after it started if (!isRunning() || !ownsExecutorService) { - executorService = resolveExecutorService(); + final var resolved = resolveExecutorService(); + final var replaced = isRunning() && resolved != executorService; + executorService = resolved; + if (replaced) { + rearmPendingTimers(); + } } if (!isRunning()) { super.start(); } } + /** + * The timers that were pending on the replaced executor are gone with it, so they are scheduled + * again on the new one for what remained of their delay. The ones that already fired are left + * alone, and the ones explicitly cancelled are not in the map anymore. + */ + private void rearmPendingTimers() { + onceTasks.forEach( + (resourceID, future) -> { + if (future.isCancelled() || !future.isDone()) { + scheduleOnce(resourceID, Math.max(0, future.getDelay(TimeUnit.MILLISECONDS))); + } + }); + } + private ScheduledExecutorService resolveExecutorService() { if (executorServiceSupplier != null) { return executorServiceSupplier.get(); @@ -171,7 +190,7 @@ public void stop() { super.stop(); onceTasks.keySet().forEach(this::cancelOnceSchedule); if (ownsExecutorService && executorService != null) { - executorService.shutdownNow(); + executorService.shutdown(); } } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java index c9be8caf58..ed5cc815a1 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java @@ -20,6 +20,10 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -113,6 +117,61 @@ void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { } } + @Test + void pollsOnTheNewSharedSchedulerWhenStartedAgainWhileRunning() { + when(resourceFetcher.fetchResources()).thenReturn(testResponseWithOneValue()); + var configurationService = new BaseConfigurationService(); + var executorServiceManager = configurationService.getExecutorServiceManager(); + var eventSource = + new PollingEventSource( + SampleExternalResource.class, + contextFor(configurationService), + new PollingConfiguration<>(null, resourceFetcher, POLL_PERIOD, null)); + eventSource.setEventHandler(mock(EventHandler.class)); + + try { + eventSource.start(); + executorServiceManager.stop(Duration.ofMillis(100)); + executorServiceManager.start(configurationService); + clearInvocations(resourceFetcher); + + eventSource.start(); + + await().untilAsserted(() -> verify(resourceFetcher, atLeast(2)).fetchResources()); + } finally { + eventSource.stop(); + executorServiceManager.stop(Duration.ofMillis(100)); + } + } + + @Test + void stoppingDoesNotInterruptAPollInFlight() throws Exception { + var polling = new CountDownLatch(1); + var release = new CountDownLatch(1); + var interrupted = new AtomicBoolean(); + var fetches = new AtomicInteger(); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + if (fetches.incrementAndGet() > 1) { + polling.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + interrupted.set(true); + } + } + return testResponseWithOneValue(); + }); + + pollingEventSource.start(); + assertThat(polling.await(5, TimeUnit.SECONDS)).isTrue(); + pollingEventSource.stop(); + release.countDown(); + + await().pollDelay(Duration.ofMillis(DEFAULT_WAIT_PERIOD)).until(() -> !interrupted.get()); + } + @Test void stopCancelsThePollingTask() throws InterruptedException { when(resourceFetcher.fetchResources()).thenReturn(testResponseWithTwoValues()); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index 769968b738..86c8973f47 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -17,9 +17,11 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.awaitility.Awaitility; @@ -118,6 +120,75 @@ public void schedulesOnTheNewExecutorWhenStartedAgainWhileRunning() { } } + @Test + public void timersPendingOnTheReplacedExecutorAreScheduledAgainOnTheNewOne() { + var first = Executors.newScheduledThreadPool(1); + var second = Executors.newScheduledThreadPool(1); + var current = new AtomicReference(first); + var timerSource = new TimerEventSource("timer", false, current::get); + setUpSource(timerSource, new CapturingEventHandler()); + try { + var resourceID = ResourceID.fromResource(TestUtils.testCustomResource()); + source.scheduleOnce(resourceID, 2 * PERIOD); + first.shutdownNow(); + current.set(second); + + source.start(); + + untilAsserted(() -> assertThat(eventHandler.events).hasSize(1)); + } finally { + second.shutdownNow(); + } + } + + @Test + public void timersThatAlreadyFiredAreNotScheduledAgainOnTheNewExecutor() { + var first = Executors.newScheduledThreadPool(1); + var second = Executors.newScheduledThreadPool(1); + var current = new AtomicReference(first); + var timerSource = new TimerEventSource("timer", false, current::get); + setUpSource(timerSource, new CapturingEventHandler()); + try { + source.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); + untilAsserted(() -> assertThat(eventHandler.events).hasSize(1)); + first.shutdownNow(); + current.set(second); + + source.start(); + + untilAsserted(PERIOD * 4, 0, () -> assertThat(eventHandler.events).hasSize(1)); + } finally { + second.shutdownNow(); + } + } + + @Test + public void stoppingDoesNotInterruptAnEventHandlerInFlight() throws Exception { + var handling = new CountDownLatch(1); + var release = new CountDownLatch(1); + var interrupted = new AtomicBoolean(); + setUpSource( + new TimerEventSource<>(), + new CapturingEventHandler() { + @Override + public void handleEvent(Event event) { + handling.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + interrupted.set(true); + } + } + }); + + source.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), 1); + assertThat(handling.await(5, TimeUnit.SECONDS)).isTrue(); + source.stop(); + release.countDown(); + + untilAsserted(PERIOD * 2, 0, () -> assertThat(interrupted).isFalse()); + } + @Test public void eventNotRegisteredIfStopped() { var resourceID = ResourceID.fromResource(TestUtils.testCustomResource()); From 556004365d050c172b5264956165e8c9bd27a6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Nov 2026 10:25:48 -0800 Subject: [PATCH 32/45] update how polling event sources are handled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../content/en/docs/documentation/eventing.md | 35 ++- .../documentation/operations/configuration.md | 15 +- .../api/config/ConfigurationService.java | 65 +---- .../config/ConfigurationServiceOverrider.java | 54 ---- .../api/config/ExecutorServiceManager.java | 23 +- .../PerResourcePollingConfiguration.java | 9 +- ...erResourcePollingConfigurationBuilder.java | 8 + .../PerResourcePollingEventSource.java | 50 +--- .../source/polling/PollingConfiguration.java | 5 +- .../polling/PollingConfigurationBuilder.java | 6 +- .../source/polling/PollingEventSource.java | 51 +--- .../ConfigurationServiceOverriderTest.java | 63 +---- .../config/ExecutorServiceManagerTest.java | 8 +- .../VirtualThreadScheduledExecutorTest.java | 9 +- .../PerResourcePollingEventSourceTest.java | 198 ++++++++++---- .../polling/PollingEventSourceTest.java | 243 ++++++++++++------ .../operator/config/loader/ConfigLoader.java | 4 - .../config/loader/ConfigLoaderTest.java | 13 - 18 files changed, 411 insertions(+), 448 deletions(-) diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index 4f776145c0..21107b3c3e 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -224,34 +224,33 @@ actually observed primary resources. #### Threading of the polling event sources -Both polling event sources schedule their polls on an executor the operator shares between all of -its polling event sources. A poll therefore only starts once one of that executor's threads is -free, so if your operator registers many polling event sources, or if fetching your external -resources is slow, size the pool accordingly with -`ConfigurationServiceOverrider.withConcurrentScheduledTaskThreads` (4 threads by default): +Each polling event source polls on a scheduled executor of its own, so a slow poll of one event +source never delays the polls of the others. Unless you provide one, the event source creates a +single daemon thread executor when it is started and shuts it down when it is stopped. A poll only +starts once the previous one is done, so a source that fetches slowly, or one with many registered +resources in the case of `PerResourcePollingEventSource`, can be given an executor with more threads +using the `withExecutorService` method of its configuration builder: ```java -Operator operator = new Operator(overrider -> overrider.withConcurrentScheduledTaskThreads(20)); +var config = + new PollingConfigurationBuilder<>(this::fetchResources, Duration.ofSeconds(5)) + .withExecutorService(Executors.newScheduledThreadPool(4)) + .build(); ``` -With virtual threads enabled, see [virtual threads](../operations/configuration/#virtual-threads), the -polls run on virtual threads, the executor only keeping one platform thread to do the timing. The -number of polls running at the same time is then not limited, so `withConcurrentScheduledTaskThreads` -doesn't apply, and a slow poll doesn't delay the others. +An executor provided that way is not managed by the operator: it is your responsibility to shut it +down. It is only ever used as is, the virtual threads flag has no effect on it. You can also share +it between multiple event sources deliberately. + +With virtual threads enabled, see [virtual threads](../operations/configuration/#virtual-threads), +the executor the event source creates runs the polls on virtual threads, only keeping one platform +thread to do the timing. The number of polls running at the same time is then not limited. Retried and rescheduled reconciliations are triggered on a separate executor, so a slow poll can never delay them. It is sized with `withConcurrentRetryAndRescheduleThreads` (2 threads by default); few threads are needed there since triggering a reconciliation only enqueues an event for one of the reconciliation threads to pick up. -Use `ConfigurationServiceOverrider.withScheduledExecutorService` to replace the polling executor -altogether. The operator still manages the lifecycle of that executor: it shuts it down when the -operator is stopped, so don't shut it down yourself. - -Use the `withExecutorService` method of the event source's own configuration builder to poll a -single event source on an executor of its own. An executor provided that way is not managed by the -operator: it is your responsibility to shut it down. - #### Inbound event sources [SimpleInboundEventSource](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/inbound/SimpleInboundEventSource.java) diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index 6b877af846..3c944fe2c9 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -51,12 +51,12 @@ Things to keep in mind: Java 25, removed that pinning. This is checked at runtime: on an older JVM, including the ones where virtual threads exist (Java 21 to 24), a warning is logged and platform threads are used instead, so the same configuration works on any Java version the operator runs on. -- The scheduled tasks (the polling event sources, and the retried and rescheduled reconciliations) - run on virtual threads too. The JDK has no scheduled executor backed by virtual threads, so the - default ones keep a single platform thread for the timing only and run each task they fire on a - virtual thread. The number of those tasks running at the same time is not limited, so - `withConcurrentScheduledTaskThreads(int)` and `withConcurrentRetryAndRescheduleThreads(int)` only - apply to platform threads. +- The scheduled tasks (the polls of the polling event sources, which each create an executor of + their own unless one is provided, and the retried and rescheduled reconciliations) run on virtual + threads too. The JDK has no scheduled executor backed by virtual threads, so the default ones keep + a single platform thread for the timing only and run each task they fire on a virtual thread. The + number of those tasks running at the same time is not limited, so + `withConcurrentRetryAndRescheduleThreads(int)` only applies to platform threads. - Virtual threads are always daemon threads, so unlike the threads of the platform thread pools they don't keep the JVM running: if the `main` method of your application returns after starting the operator, make sure something else keeps the JVM alive. @@ -351,11 +351,10 @@ All operator-level keys are prefixed with `josdk.`. |---|---|---| | `josdk.workflow.executor-threads` | `Integer` | Thread pool size for workflow execution | -#### Scheduled Tasks +#### Retry and Reschedule | Key | Type | Description | |---|---|---| -| `josdk.scheduled-tasks.concurrent-threads` | `Integer` | Thread pool size for the operator's scheduled tasks, i.e. the polling event sources | | `josdk.retry-and-reschedule.concurrent-threads` | `Integer` | Thread pool size for triggering retried and rescheduled reconciliations | #### Informer diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 06f0797317..92d1729619 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -64,14 +64,6 @@ public interface ConfigurationService { /** The default number of threads used to process dependent workflows */ int DEFAULT_WORKFLOW_EXECUTOR_THREAD_NUMBER = DEFAULT_RECONCILIATION_THREADS_NUMBER; - /** - * The default number of threads used to run the operator's scheduled tasks, i.e. the periodic - * polls of {@link io.javaoperatorsdk.operator.processing.event.source.polling.PollingEventSource} - * and {@link - * io.javaoperatorsdk.operator.processing.event.source.polling.PerResourcePollingEventSource} - */ - int DEFAULT_SCHEDULED_TASK_THREADS_NUMBER = 4; - /** * The default number of threads used to trigger the operator's retried and rescheduled * reconciliations @@ -254,22 +246,6 @@ default int concurrentWorkflowExecutorThreads() { return DEFAULT_WORKFLOW_EXECUTOR_THREAD_NUMBER; } - /** - * Number of threads the operator can spin out to run its scheduled (i.e. periodic or delayed) - * tasks with the default executor. These threads are shared by all the polling event sources of - * the operator, so this number should be raised when many, or slow, polling event sources are - * registered: a poll only starts once a thread is available, and a slow poll therefore delays the - * polls of the other event sources. Not used when virtual threads are enabled, see {@link - * #useVirtualThreads()}, since the number of scheduled tasks running at the same time is not - * limited then. - * - * @return the maximum number of concurrent scheduled task threads - * @since 5.7.0 - */ - default int concurrentScheduledTaskThreads() { - return DEFAULT_SCHEDULED_TASK_THREADS_NUMBER; - } - /** * Number of threads the operator can spin out to trigger its retried and rescheduled * reconciliations with the default executor. These threads are shared by all the controllers of @@ -315,15 +291,16 @@ default Metrics getMetrics() { *

Note that this only affects the executors created by the framework: a custom {@link * ExecutorService} provided through {@link #getExecutorService()} or {@link * #getWorkflowExecutorService()}, or a custom {@link ScheduledExecutorService} provided through - * {@link #getScheduledExecutorService()} or {@link #getRetryAndRescheduleExecutorService()}, is - * used as is. The default scheduled executors keep a single platform thread for the timing and - * run each task they fire on a virtual thread. The same goes for the {@link KubernetesClient}: - * the default client created by {@link #getKubernetesClient()} then also runs its internal - * asynchronous tasks (informer event dispatching, watch event delivery) on virtual threads, while - * a custom client, e.g. provided through {@link - * ConfigurationServiceOverrider#withKubernetesClient(KubernetesClient)}, keeps whatever task - * executor it was built with. The blocking calls a reconciler performs through the client don't - * need any of this: they already park the calling virtual thread rather than its carrier. + * {@link #getRetryAndRescheduleExecutorService()} or the polling configuration, is used as is. + * The default scheduled executors, including the ones the polling event sources create for + * themselves, keep a single platform thread for the timing and run each task they fire on a + * virtual thread. The same goes for the {@link KubernetesClient}: the default client created by + * {@link #getKubernetesClient()} then also runs its internal asynchronous tasks (informer event + * dispatching, watch event delivery) on virtual threads, while a custom client, e.g. provided + * through {@link ConfigurationServiceOverrider#withKubernetesClient(KubernetesClient)}, keeps + * whatever task executor it was built with. The blocking calls a reconciler performs through the + * client don't need any of this: they already park the calling virtual thread rather than its + * carrier. * * @return {@code true} to use virtual threads, {@code false} (default) to use platform threads * @since 5.7.0 @@ -357,33 +334,13 @@ default ExecutorService getWorkflowExecutorService() { /** * Override to provide a custom {@link ScheduledExecutorService} implementation to change how the - * operator's scheduled (i.e. periodic or delayed) tasks are run. This executor is shared by all - * the polling event sources of the operator. The retried and rescheduled reconciliations run on - * an executor of their own, see {@link #getRetryAndRescheduleExecutorService()}, so that a slow - * poll can't delay them. + * operator's retried and rescheduled reconciliations are triggered. * *

Note that the default implementation lets the executor discard the tasks that were scheduled * for later when it is shut down, so that they don't delay the termination of the operator, and * that it creates daemon threads so that a never stopped operator doesn't keep the JVM alive. * Custom implementations are advised to do the same. * - * @return the {@link ScheduledExecutorService} implementation to use to run scheduled tasks - * @since 5.7.0 - */ - default ScheduledExecutorService getScheduledExecutorService() { - return ExecutorServiceManager.newScheduledExecutorService( - concurrentScheduledTaskThreads(), useVirtualThreads(), "josdk-scheduled-task"); - } - - /** - * Override to provide a custom {@link ScheduledExecutorService} implementation to change how the - * operator's retried and rescheduled reconciliations are triggered. This executor is kept - * separate from the one the polling event sources use, see {@link - * #getScheduledExecutorService()}, so that a slow poll can't delay a retry. - * - *

The same notes as for {@link #getScheduledExecutorService()} apply to custom - * implementations. - * * @return the {@link ScheduledExecutorService} implementation to use to trigger retried and * rescheduled reconciliations * @since 5.7.0 diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index c4e514c282..e8a664098f 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -46,7 +46,6 @@ public class ConfigurationServiceOverrider { private Boolean checkCR; private Integer concurrentReconciliationThreads; private Integer concurrentWorkflowExecutorThreads; - private Integer concurrentScheduledTaskThreads; private Integer concurrentRetryAndRescheduleThreads; private Cloner cloner; private Boolean closeClientOnStop; @@ -54,7 +53,6 @@ public class ConfigurationServiceOverrider { private ExecutorService executorService; private ExecutorService workflowExecutorService; private Boolean useVirtualThreads; - private ScheduledExecutorService scheduledExecutorService; private ScheduledExecutorService retryAndRescheduleExecutorService; private LeaderElectionConfiguration leaderElectionConfiguration; private String clusterScopedEventNamespace; @@ -92,20 +90,6 @@ public ConfigurationServiceOverrider withConcurrentWorkflowExecutorThreads(int t return this; } - /** - * Sets the number of threads used to run the operator's scheduled (i.e. periodic or delayed) - * tasks, which are shared by all its polling event sources. - * - * @param threadNumber the maximum number of concurrent scheduled task threads - * @return this {@link ConfigurationServiceOverrider} for chained customization - * @see ConfigurationService#concurrentScheduledTaskThreads() - * @since 5.7.0 - */ - public ConfigurationServiceOverrider withConcurrentScheduledTaskThreads(int threadNumber) { - this.concurrentScheduledTaskThreads = threadNumber; - return this; - } - /** * Sets the number of threads used to trigger the operator's retried and rescheduled * reconciliations, which are shared by all its controllers. @@ -153,21 +137,6 @@ public ConfigurationServiceOverrider withWorkflowExecutorService( return this; } - /** - * Replaces the executor used to run the operator's scheduled (i.e. periodic or delayed) tasks, - * which are shared by all its polling event sources. - * - * @param scheduledExecutorService the executor to run scheduled tasks on - * @return this {@link ConfigurationServiceOverrider} for chained customization - * @see ConfigurationService#getScheduledExecutorService() - * @since 5.7.0 - */ - public ConfigurationServiceOverrider withScheduledExecutorService( - ScheduledExecutorService scheduledExecutorService) { - this.scheduledExecutorService = scheduledExecutorService; - return this; - } - /** * Replaces the executor used to trigger the operator's retried and rescheduled reconciliations, * which is shared by all its controllers. @@ -391,17 +360,6 @@ public int concurrentWorkflowExecutorThreads() { original.concurrentWorkflowExecutorThreads()); } - @Override - public int concurrentScheduledTaskThreads() { - return Utils.ensureValid( - overriddenValueOrDefault( - concurrentScheduledTaskThreads, - ConfigurationService::concurrentScheduledTaskThreads), - "maximum scheduled task threads", - 1, - original.concurrentScheduledTaskThreads()); - } - @Override public int concurrentRetryAndRescheduleThreads() { return Utils.ensureValid( @@ -446,18 +404,6 @@ public ExecutorService getWorkflowExecutorService() { } } - @Override - public ScheduledExecutorService getScheduledExecutorService() { - if (scheduledExecutorService != null) { - return scheduledExecutorService; - } else if (!changes( - concurrentScheduledTaskThreads, original.concurrentScheduledTaskThreads())) { - return original.getScheduledExecutorService(); - } else { - return super.getScheduledExecutorService(); - } - } - @Override public ScheduledExecutorService getRetryAndRescheduleExecutorService() { if (retryAndRescheduleExecutorService != null) { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index 0b06f8b9b7..0e7e7c12c1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -42,7 +42,6 @@ public class ExecutorServiceManager { private ExecutorService executor; private ExecutorService workflowExecutor; private ExecutorService cachingExecutorService; - private ScheduledExecutorService scheduledExecutorService; private ScheduledExecutorService retryAndRescheduleExecutorService; private boolean started; private ConfigurationService configurationService; @@ -184,22 +183,10 @@ public ExecutorService cachingExecutorService() { } /** - * The executor the operator runs its scheduled (i.e. periodic or delayed) tasks on, shared by its - * polling event sources. Note that it is only valid while the manager is started: it is shut down - * by {@link #stop(Duration)} and replaced by a fresh one on the next {@link - * #start(ConfigurationService)}, so callers should retrieve it when they start rather than hold - * on to it. - * - * @return the executor to run scheduled tasks on - */ - public ScheduledExecutorService scheduledExecutorService() { - return scheduledExecutorService; - } - - /** - * The executor the operator triggers its retried and rescheduled reconciliations on, kept - * separate from {@link #scheduledExecutorService()} so that a slow poll can't delay a retry. The - * same lifecycle caveat as for {@link #scheduledExecutorService()} applies. + * The executor the operator triggers its retried and rescheduled reconciliations on. Note that it + * is only valid while the manager is started: it is shut down by {@link #stop(Duration)} and + * replaced by a fresh one on the next {@link #start(ConfigurationService)}, so callers should + * retrieve it when they start rather than hold on to it. * * @return the executor to trigger retried and rescheduled reconciliations on */ @@ -212,7 +199,6 @@ public synchronized void start(ConfigurationService configurationService) { this.configurationService = configurationService; // used to lazy init workflow executor this.cachingExecutorService = newUnboundedExecutorService(configurationService.useVirtualThreads()); - this.scheduledExecutorService = configurationService.getScheduledExecutorService(); this.retryAndRescheduleExecutorService = configurationService.getRetryAndRescheduleExecutorService(); this.executor = new InstrumentedExecutorService(configurationService.getExecutorService()); @@ -226,7 +212,6 @@ public synchronized void stop(Duration gracefulShutdownTimeout) { shutdown(executor, gracefulShutdownTimeout), shutdown(workflowExecutor, gracefulShutdownTimeout), shutdown(cachingExecutorService, gracefulShutdownTimeout), - shutdown(scheduledExecutorService, gracefulShutdownTimeout), shutdown(retryAndRescheduleExecutorService, gracefulShutdownTimeout)); var parallelExec = Executors.newFixedThreadPool(shutdowns.size()); try { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java index 49bc8dd7ba..4f184bd94e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfiguration.java @@ -24,8 +24,9 @@ import io.javaoperatorsdk.operator.processing.ResourceIDMapper; /** - * @param executorService the executor to run the polls on, {@code null} (the default) to run them - * on the executor the operator shares between all its scheduled tasks + * @param executorService the executor to run the polls on, {@code null} (the default) to have the + * event source create one of its own, on virtual threads if enabled, which it shuts down when + * stopped. A provided executor is not shut down by the event source. */ public record PerResourcePollingConfiguration( String name, @@ -36,8 +37,8 @@ public record PerResourcePollingConfiguration( Duration defaultPollingPeriod) { /** - * @deprecated not used anymore, polls now run on the executor the operator shares between all its - * scheduled tasks unless an executor is explicitly configured + * @deprecated not used anymore, the event source creates an executor of its own unless one is + * explicitly configured */ @Deprecated(forRemoval = true) public static final int DEFAULT_EXECUTOR_THREAD_NUMBER = 1; diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfigurationBuilder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfigurationBuilder.java index 6fcf5d2678..115d3ee9f3 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfigurationBuilder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingConfigurationBuilder.java @@ -39,6 +39,14 @@ public PerResourcePollingConfigurationBuilder( this.defaultPollingPeriod = defaultPollingPeriod; } + /** + * Runs the polls on the specified executor instead of the one the event source creates by + * default. Note that an explicitly provided executor is not managed by the operator: it is the + * caller's responsibility to shut it down. It can be shared between event sources. + * + * @param executorService the executor to run the polls on + * @return this builder for chained customization + */ @SuppressWarnings("unused") public PerResourcePollingConfigurationBuilder withExecutorService( ScheduledExecutorService executorService) { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 3874836233..22d7471d6e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -30,9 +30,9 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ExecutorServiceManager; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.javaoperatorsdk.operator.processing.event.source.Cache; @@ -59,7 +59,7 @@ public class PerResourcePollingEventSource private final Set fetchedForPrimaries = ConcurrentHashMap.newKeySet(); private final ScheduledExecutorService configuredExecutorService; - private final ConfigurationService configurationService; + private final boolean useVirtualThreads; private final boolean ownsExecutorService; private final ResourceFetcher resourceFetcher; private final Predicate

registerPredicate; @@ -82,10 +82,10 @@ public PerResourcePollingEventSource( this.resourceFetcher = config.resourceFetcher(); this.registerPredicate = config.registerPredicate(); this.configuredExecutorService = config.executorService(); - this.configurationService = configurationServiceOf(context); - // when neither the configuration nor the operator provides one, the event source has to create - // an executor of its own, and is then the one responsible for shutting it down - this.ownsExecutorService = configuredExecutorService == null && configurationService == null; + var configurationService = configurationServiceOf(context); + this.useVirtualThreads = + configurationService != null && configurationService.useVirtualThreads(); + this.ownsExecutorService = configuredExecutorService == null; this.period = config.defaultPollingPeriod(); } @@ -104,7 +104,6 @@ private static ConfigurationService configurationServiceOf(EventSourceContext @Override public void start() throws OperatorException { if (isRunning()) { - switchToCurrentExecutor(); return; } executorService = resolveExecutorService(); @@ -112,28 +111,6 @@ public void start() throws OperatorException { replayRegistrationsBeforeStart(); } - /** - * The operator replaces its executors when it is started again, which can happen while this event - * source keeps running, when a previous start failed after it started. The polls scheduled on the - * replaced executor are gone then, so they are scheduled again on the new one. - */ - private void switchToCurrentExecutor() { - if (ownsExecutorService) { - return; - } - var resolved = resolveExecutorService(); - synchronized (this) { - if (resolved == executorService) { - return; - } - executorService = resolved; - lifecycleGeneration++; - scheduledFutures.values().forEach(future -> future.cancel(true)); - scheduledFutures.clear(); - } - primaryResourceCache.list().forEach(this::checkAndRegisterTask); - } - private void replayRegistrationsBeforeStart() { for (var primaryID : registrationsBeforeStart.keySet()) { var resource = registrationsBeforeStart.remove(primaryID); @@ -144,18 +121,15 @@ private void replayRegistrationsBeforeStart() { } /** - * Resolves the executor to poll on. Note that this happens on every start, and not once at - * creation time, since the operator shuts its executors down when it is stopped and creates new - * ones if it is started again. + * Resolves the executor to poll on. Note that this happens on every start, since an executor + * created by this event source is shut down when it is stopped. */ private ScheduledExecutorService resolveExecutorService() { if (configuredExecutorService != null) { return configuredExecutorService; } - if (configurationService != null) { - return configurationService.getExecutorServiceManager().scheduledExecutorService(); - } - return ExecutorUtilsInternal.daemonScheduledThreadPool(1, "josdk-polling-" + name()); + return ExecutorServiceManager.newScheduledExecutorService( + 1, useVirtualThreads, "josdk-polling-" + name()); } private Set getAndCacheResource(P primary, boolean fromGetter) { @@ -316,8 +290,8 @@ default Optional fetchDelay(Set lastFetchedResource, P primary) { @Override public void stop() throws OperatorException { super.stop(); - // the tasks have to be cancelled explicitly now that the executor can be shared with the rest - // of the operator, and the map cleared so that they are registered again on a restart + // the tasks have to be cancelled explicitly since a provided executor is not shut down, and the + // map cleared so that they are registered again on a restart synchronized (this) { lifecycleGeneration++; scheduledFutures.values().forEach(future -> future.cancel(true)); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java index 0836058cf4..6ca904eca1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfiguration.java @@ -22,8 +22,9 @@ import io.javaoperatorsdk.operator.processing.ResourceIDMapper; /** - * @param executorService the executor to run the polls on, {@code null} (the default) to run them - * on the executor the operator shares between all its scheduled tasks + * @param executorService the executor to run the polls on, {@code null} (the default) to have the + * event source create one of its own, on virtual threads if enabled, which it shuts down when + * stopped. A provided executor is not shut down by the event source. */ public record PollingConfiguration( String name, diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java index 80e8dc06df..d9e072bd9e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingConfigurationBuilder.java @@ -45,9 +45,9 @@ public PollingConfigurationBuilder withName(String name) { } /** - * Runs the polls on the specified executor instead of the one the operator shares between all its - * scheduled tasks. Note that an explicitly provided executor is not managed by the operator: it - * is the caller's responsibility to shut it down. + * Runs the polls on the specified executor instead of the one the event source creates by + * default. Note that an explicitly provided executor is not managed by the operator: it is the + * caller's responsibility to shut it down. It can be shared between event sources. * * @param executorService the executor to run the polls on * @return this builder for chained customization diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index b70b9ce550..115f86693e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -27,9 +27,9 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.javaoperatorsdk.operator.ExecutorUtilsInternal; import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ExecutorServiceManager; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.health.Status; import io.javaoperatorsdk.operator.processing.event.ResourceID; @@ -70,15 +70,16 @@ public class PollingEventSource private final Duration period; private final AtomicBoolean healthy = new AtomicBoolean(true); private final ScheduledExecutorService configuredExecutorService; - private final ConfigurationService configurationService; + private final boolean useVirtualThreads; private final boolean ownsExecutorService; private volatile ScheduledExecutorService executorService; private volatile ScheduledFuture pollingTask; /** - * Creates an event source polling on the executor the operator shares between all its scheduled - * tasks, unless the configuration provides one of its own. + * Creates an event source polling on an executor of its own: the one provided by the + * configuration, or otherwise one it creates and shuts down itself, which runs on virtual threads + * if {@link ConfigurationService#useVirtualThreads()} is enabled. * * @param resourceClass the type of the polled resource * @param context the context this event source is created for @@ -93,8 +94,8 @@ public PollingEventSource( /** * @deprecated use {@link #PollingEventSource(Class, EventSourceContext, PollingConfiguration)} * instead: without a context, and unless the configuration provides an executor, this event - * source has to create a thread of its own to poll on instead of using the one the operator - * shares between all its scheduled tasks. + * source polls on a platform thread even if the operator is configured to use virtual + * threads. */ @Deprecated(since = "5.7.0") public PollingEventSource(Class resourceClass, PollingConfiguration config) { @@ -109,10 +110,9 @@ private PollingEventSource( this.genericResourceFetcher = config.genericResourceFetcher(); this.period = config.period(); this.configuredExecutorService = config.executorService(); - this.configurationService = configurationService; - // when neither the configuration nor the operator provides one, the event source has to create - // an executor of its own, and is then the one responsible for shutting it down - this.ownsExecutorService = configuredExecutorService == null && configurationService == null; + this.useVirtualThreads = + configurationService != null && configurationService.useVirtualThreads(); + this.ownsExecutorService = configuredExecutorService == null; } /** @@ -130,7 +130,6 @@ private static ConfigurationService configurationServiceOf(EventSourceContext @Override public void start() throws OperatorException { if (pollingTask != null) { - switchToCurrentExecutor(); return; } super.start(); @@ -152,25 +151,6 @@ private void schedulePolling() { } } - /** - * The operator replaces its executors when it is started again, which can happen while this event - * source keeps running, when a previous start failed after it started. The polling task is gone - * with the replaced executor then, so it is scheduled again on the new one. - */ - private void switchToCurrentExecutor() { - final var current = pollingTask; - if (ownsExecutorService || current == null) { - return; - } - final var resolved = resolveExecutorService(); - if (resolved == executorService) { - return; - } - current.cancel(false); - executorService = resolved; - schedulePolling(); - } - private void poll() { try { if (!isRunning()) { @@ -187,18 +167,15 @@ private void poll() { } /** - * Resolves the executor to poll on. Note that this happens on every start, and not once at - * creation time, since the operator shuts its executors down when it is stopped and creates new - * ones if it is started again. + * Resolves the executor to poll on. Note that this happens on every start, since an executor + * created by this event source is shut down when it is stopped. */ private ScheduledExecutorService resolveExecutorService() { if (configuredExecutorService != null) { return configuredExecutorService; } - if (configurationService != null) { - return configurationService.getExecutorServiceManager().scheduledExecutorService(); - } - return ExecutorUtilsInternal.daemonScheduledThreadPool(1, "josdk-polling-" + name()); + return ExecutorServiceManager.newScheduledExecutorService( + 1, useVirtualThreads, "josdk-polling-" + name()); } protected synchronized void getStateAndFillCache() { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java index c5ae255d60..9c356b55f7 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverriderTest.java @@ -120,40 +120,28 @@ public R clone(R object) { } @Test - void scheduledExecutorCanBeOverridden() { - final var scheduledExecutorService = Executors.newScheduledThreadPool(1); + void retryAndRescheduleExecutorCanBeOverridden() { final var retryExecutorService = Executors.newScheduledThreadPool(1); try { final var overridden = new ConfigurationServiceOverrider(config) - .withConcurrentScheduledTaskThreads(7) - .withScheduledExecutorService(scheduledExecutorService) .withConcurrentRetryAndRescheduleThreads(5) .withRetryAndRescheduleExecutorService(retryExecutorService) .build(); - assertThat(overridden.concurrentScheduledTaskThreads()).isEqualTo(7); - assertThat(overridden.getScheduledExecutorService()).isSameAs(scheduledExecutorService); assertThat(overridden.concurrentRetryAndRescheduleThreads()).isEqualTo(5); assertThat(overridden.getRetryAndRescheduleExecutorService()).isSameAs(retryExecutorService); } finally { - scheduledExecutorService.shutdownNow(); retryExecutorService.shutdownNow(); } } @Test - void executorsOfTheOriginalConfigurationAreKeptUnlessTheirSettingsAreOverridden() { - final var scheduledExecutorService = Executors.newScheduledThreadPool(1); + void executorOfTheOriginalConfigurationIsKeptUnlessItsSettingsAreOverridden() { final var retryExecutorService = Executors.newScheduledThreadPool(1); try { final var original = new BaseConfigurationService() { - @Override - public ScheduledExecutorService getScheduledExecutorService() { - return scheduledExecutorService; - } - @Override public ScheduledExecutorService getRetryAndRescheduleExecutorService() { return retryExecutorService; @@ -163,38 +151,28 @@ public ScheduledExecutorService getRetryAndRescheduleExecutorService() { final var overridden = new ConfigurationServiceOverrider(original).withCloseClientOnStop(false); - assertThat(overridden.build().getScheduledExecutorService()) - .isSameAs(scheduledExecutorService); assertThat(overridden.build().getRetryAndRescheduleExecutorService()) .isSameAs(retryExecutorService); - final var resized = overridden.withConcurrentScheduledTaskThreads(3).build(); - final var defaultScheduled = resized.getScheduledExecutorService(); + final var resized = overridden.withConcurrentRetryAndRescheduleThreads(3).build(); + final var defaultRetry = resized.getRetryAndRescheduleExecutorService(); try { - assertThat(defaultScheduled).isNotSameAs(scheduledExecutorService); - assertThat(((ScheduledThreadPoolExecutor) defaultScheduled).getCorePoolSize()).isEqualTo(3); - assertThat(resized.getRetryAndRescheduleExecutorService()).isSameAs(retryExecutorService); + assertThat(defaultRetry).isNotSameAs(retryExecutorService); + assertThat(((ScheduledThreadPoolExecutor) defaultRetry).getCorePoolSize()).isEqualTo(3); } finally { - defaultScheduled.shutdownNow(); + defaultRetry.shutdownNow(); } } finally { - scheduledExecutorService.shutdownNow(); retryExecutorService.shutdownNow(); } } @Test - void executorsOfTheOriginalConfigurationAreKeptWhenOverridesDoNotChangeTheirSettings() { - final var scheduledExecutorService = Executors.newScheduledThreadPool(1); + void executorOfTheOriginalConfigurationIsKeptWhenOverridesDoNotChangeItsSettings() { final var retryExecutorService = Executors.newScheduledThreadPool(1); try { final var original = new BaseConfigurationService() { - @Override - public ScheduledExecutorService getScheduledExecutorService() { - return scheduledExecutorService; - } - @Override public ScheduledExecutorService getRetryAndRescheduleExecutorService() { return retryExecutorService; @@ -204,27 +182,24 @@ public ScheduledExecutorService getRetryAndRescheduleExecutorService() { final var overridden = new ConfigurationServiceOverrider(original) .withUseVirtualThreads(original.useVirtualThreads()) - .withConcurrentScheduledTaskThreads(original.concurrentScheduledTaskThreads()) .withConcurrentRetryAndRescheduleThreads( original.concurrentRetryAndRescheduleThreads()) .build(); - assertThat(overridden.getScheduledExecutorService()).isSameAs(scheduledExecutorService); assertThat(overridden.getRetryAndRescheduleExecutorService()).isSameAs(retryExecutorService); } finally { - scheduledExecutorService.shutdownNow(); retryExecutorService.shutdownNow(); } } @Test - void scheduledExecutorDefaultsToADaemonPoolOfTheConfiguredSize() { + void retryAndRescheduleExecutorDefaultsToADaemonPoolOfTheConfiguredSize() { final var overridden = (ScheduledThreadPoolExecutor) new ConfigurationServiceOverrider(config) - .withConcurrentScheduledTaskThreads(3) + .withConcurrentRetryAndRescheduleThreads(3) .build() - .getScheduledExecutorService(); + .getRetryAndRescheduleExecutorService(); try { assertThat(overridden.getCorePoolSize()).isEqualTo(3); @@ -236,22 +211,6 @@ void scheduledExecutorDefaultsToADaemonPoolOfTheConfiguredSize() { } } - @Test - void retryAndRescheduleExecutorIsSeparateFromTheScheduledTaskOne() { - final var scheduled = config.getScheduledExecutorService(); - final var retryAndReschedule = config.getRetryAndRescheduleExecutorService(); - - try { - // a slow poll on the scheduled task executor must not be able to delay a retry - assertThat(retryAndReschedule).isNotSameAs(scheduled); - assertThat(((ScheduledThreadPoolExecutor) retryAndReschedule).getCorePoolSize()) - .isEqualTo(ConfigurationService.DEFAULT_RETRY_AND_RESCHEDULE_THREADS_NUMBER); - } finally { - scheduled.shutdownNow(); - retryAndReschedule.shutdownNow(); - } - } - @Test void eventRecorderIsNotConfiguredByDefaultAndCanBeOverridden() { final var eventRecorder = diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java index ea6725896f..c06f0db26c 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java @@ -28,22 +28,17 @@ class ExecutorServiceManagerTest { private static final Duration SHUTDOWN_TIMEOUT = Duration.ofMillis(100); @Test - void stopShutsDownTheScheduledExecutorService() { + void stopShutsDownTheRetryAndRescheduleExecutorService() { ConfigurationService configurationService = new BaseConfigurationService(); var manager = configurationService.getExecutorServiceManager(); - var scheduled = manager.scheduledExecutorService(); var retryAndReschedule = manager.retryAndRescheduleExecutorService(); try { - assertThat(scheduled.isShutdown()).isFalse(); assertThat(retryAndReschedule.isShutdown()).isFalse(); - // retries are triggered on an executor of their own so that a slow poll can't delay them - assertThat(retryAndReschedule).isNotSameAs(scheduled); } finally { manager.stop(SHUTDOWN_TIMEOUT); } - assertThat(scheduled.isShutdown()).isTrue(); assertThat(retryAndReschedule.isShutdown()).isTrue(); } @@ -59,7 +54,6 @@ void canBeRestartedAfterStop() { // handing out already terminated executors assertThat(manager.reconcileExecutorService().isShutdown()).isFalse(); assertThat(manager.cachingExecutorService().isShutdown()).isFalse(); - assertThat(manager.scheduledExecutorService().isShutdown()).isFalse(); assertThat(manager.retryAndRescheduleExecutorService().isShutdown()).isFalse(); manager.stop(SHUTDOWN_TIMEOUT); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java index c2d2fb5089..4fe9555987 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java @@ -284,18 +284,13 @@ void shutdownNowInterruptsRunningTasks() throws Exception { } @Test - void theOperatorsDefaultScheduledExecutorsRunOnVirtualThreadsWhenRequested() throws Exception { + void theOperatorsDefaultRetryAndRescheduleExecutorRunsOnVirtualThreadsWhenRequested() + throws Exception { var configurationService = ConfigurationService.newOverriddenConfigurationService( new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); var manager = configurationService.getExecutorServiceManager(); try { - assertThat( - manager - .scheduledExecutorService() - .schedule(VirtualThreadsTest::onVirtualThread, 0, TimeUnit.MILLISECONDS) - .get(10, TimeUnit.SECONDS)) - .isTrue(); assertThat( manager .retryAndRescheduleExecutorService() diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 11a97308c0..4a0258c3e9 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -19,15 +19,20 @@ import java.util.Collections; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; import io.javaoperatorsdk.operator.TestUtils; import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.ResourceIDMapper; @@ -356,75 +361,166 @@ void aRegistrationRacingStopDoesNotBlockPollingAfterARestart() throws Exception } @Test - void pollsOnTheNewSharedSchedulerWhenStartedAgainWhileRunning() { - var configurationService = new BaseConfigurationService(); - var executorServiceManager = configurationService.getExecutorServiceManager(); - var controllerConfiguration = mock(ControllerConfiguration.class); - when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); - when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); - when(resourceCache.list()).thenAnswer(i -> Stream.of(testCustomResource)); - setUpSource( - new PerResourcePollingEventSource<>( - SampleExternalResource.class, - context, - new PerResourcePollingConfigurationBuilder< - SampleExternalResource, TestCustomResource, String>( - supplier, Duration.ofMillis(PERIOD)) - .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) - .build())); + void createsAnExecutorOfItsOwnAndShutsItDownOnStop() { + var pollingThreads = new CopyOnWriteArrayList(); + when(supplier.fetchResources(any())) + .thenAnswer( + invocation -> { + pollingThreads.add(Thread.currentThread()); + return Set.of(SampleExternalResource.testResource1()); + }); + setUpSource(eventSourceFor(new BaseConfigurationService(), null)); + + source.onResourceCreated(testCustomResource); + + await().until(() -> !pollingThreads.isEmpty()); + var pollingThread = pollingThreads.get(0); + assertThat(pollingThread.getName()).startsWith("josdk-polling-"); + assertThat(pollingThread.isDaemon()).isTrue(); + + source.stop(); + + await().untilAsserted(() -> assertThat(pollingThread.isAlive()).isFalse()); + } + + @Test + void usesANewExecutorWhenStartedAfterStop() { + setUpSource(eventSourceFor(new BaseConfigurationService(), null)); + source.onResourceCreated(testCustomResource); + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + source.stop(); + clearInvocations(supplier); + + source.start(); + source.onResourceCreated(testCustomResource); + + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + } + + @Test + void pollsOnTheProvidedExecutorAndLeavesItRunningOnStop() { + var pollingThreadNames = new CopyOnWriteArrayList(); + when(supplier.fetchResources(any())) + .thenAnswer( + invocation -> { + pollingThreadNames.add(Thread.currentThread().getName()); + return Set.of(SampleExternalResource.testResource1()); + }); + var provided = + Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "provided")); + setUpSource(eventSourceFor(new BaseConfigurationService(), provided)); try { source.onResourceCreated(testCustomResource); - await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + await().untilAsserted(() -> assertThat(pollingThreadNames).contains("provided")); - executorServiceManager.stop(Duration.ofMillis(100)); - executorServiceManager.start(configurationService); - clearInvocations(supplier); - source.start(); + source.stop(); - await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + assertThat(provided.isShutdown()).isFalse(); + assertThat(pollingThreadNames).containsOnly("provided"); } finally { - source.stop(); - executorServiceManager.stop(Duration.ofMillis(100)); + provided.shutdownNow(); } } @Test - void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { + void eventSourcesDoNotShareTheirExecutor() throws Exception { + var otherThreads = new CopyOnWriteArrayList(); + var blockFirst = new CountDownLatch(1); + var firstPolling = new CountDownLatch(1); + when(supplier.fetchResources(any())) + .thenAnswer( + invocation -> { + firstPolling.countDown(); + blockFirst.await(); + return Set.of(SampleExternalResource.testResource1()); + }); + @SuppressWarnings("unchecked") + PerResourcePollingEventSource.ResourceFetcher + otherSupplier = mock(PerResourcePollingEventSource.ResourceFetcher.class); + when(otherSupplier.fetchResources(any())) + .thenAnswer( + invocation -> { + otherThreads.add(Thread.currentThread()); + return Set.of(SampleExternalResource.testResource1()); + }); var configurationService = new BaseConfigurationService(); - var executorServiceManager = configurationService.getExecutorServiceManager(); - var controllerConfiguration = mock(ControllerConfiguration.class); - when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); - when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); - setUpSource( - new PerResourcePollingEventSource<>( - SampleExternalResource.class, - context, - new PerResourcePollingConfigurationBuilder< - SampleExternalResource, TestCustomResource, String>( - supplier, Duration.ofMillis(PERIOD)) - .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) - .build())); + var first = eventSourceFor(configurationService, supplier, null); + var second = eventSourceFor(configurationService, otherSupplier, null); + first.setEventHandler(mock(EventHandler.class)); + second.setEventHandler(mock(EventHandler.class)); try { - source.onResourceCreated(testCustomResource); - await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); - var firstExecutor = executorServiceManager.scheduledExecutorService(); - source.stop(); + first.start(); + second.start(); + first.onResourceCreated(testCustomResource); + assertThat(firstPolling.await(5, TimeUnit.SECONDS)).isTrue(); + second.onResourceCreated(testCustomResource); + + // a poll of the first one hanging must not delay the polls of the second one + await().until(() -> otherThreads.size() >= 2); + } finally { + blockFirst.countDown(); + first.stop(); + second.stop(); + } + } - executorServiceManager.stop(Duration.ofMillis(100)); - executorServiceManager.start(configurationService); - assertThat(firstExecutor.isShutdown()).isTrue(); - assertThat(executorServiceManager.scheduledExecutorService()).isNotSameAs(firstExecutor); + @Test + @EnabledForJreRange(min = JRE.JAVA_25) + void pollsOnVirtualThreadsWhenEnabledInTheConfiguration() { + var onVirtualThread = new CopyOnWriteArrayList(); + when(supplier.fetchResources(any())) + .thenAnswer( + invocation -> { + onVirtualThread.add(isVirtual(Thread.currentThread())); + return Set.of(SampleExternalResource.testResource1()); + }); + var configurationService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); + setUpSource(eventSourceFor(configurationService, null)); - source.start(); - clearInvocations(supplier); + try { source.onResourceCreated(testCustomResource); - await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + await().untilAsserted(() -> assertThat(onVirtualThread).contains(true)); } finally { source.stop(); - executorServiceManager.stop(Duration.ofMillis(100)); + } + } + + private PerResourcePollingEventSource + eventSourceFor( + ConfigurationService configurationService, ScheduledExecutorService executorService) { + return eventSourceFor(configurationService, supplier, executorService); + } + + private PerResourcePollingEventSource + eventSourceFor( + ConfigurationService configurationService, + PerResourcePollingEventSource.ResourceFetcher + fetcher, + ScheduledExecutorService executorService) { + var controllerConfiguration = mock(ControllerConfiguration.class); + when(controllerConfiguration.getConfigurationService()).thenReturn(configurationService); + when(context.getControllerConfiguration()).thenReturn(controllerConfiguration); + return new PerResourcePollingEventSource<>( + SampleExternalResource.class, + context, + new PerResourcePollingConfigurationBuilder< + SampleExternalResource, TestCustomResource, String>( + fetcher, Duration.ofMillis(PERIOD)) + .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) + .withExecutorService(executorService) + .build()); + } + + private static boolean isVirtual(Thread thread) { + try { + return (Boolean) Thread.class.getMethod("isVirtual").invoke(thread); + } catch (ReflectiveOperationException e) { + return false; } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java index ed5cc815a1..1cd483aceb 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java @@ -21,12 +21,16 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; @@ -85,65 +89,6 @@ void canBeRestartedAfterStop() throws InterruptedException { assertThat(pollingEventSource.getStatus()).isEqualTo(Status.HEALTHY); } - @Test - void pollsOnTheNewSharedSchedulerWhenTheOperatorIsRestarted() { - when(resourceFetcher.fetchResources()).thenReturn(testResponseWithOneValue()); - var configurationService = new BaseConfigurationService(); - var executorServiceManager = configurationService.getExecutorServiceManager(); - var eventSource = - new PollingEventSource( - SampleExternalResource.class, - contextFor(configurationService), - new PollingConfiguration<>(null, resourceFetcher, POLL_PERIOD, null)); - eventSource.setEventHandler(mock(EventHandler.class)); - - try { - eventSource.start(); - var firstExecutor = executorServiceManager.scheduledExecutorService(); - eventSource.stop(); - - executorServiceManager.stop(Duration.ofMillis(100)); - executorServiceManager.start(configurationService); - assertThat(firstExecutor.isShutdown()).isTrue(); - assertThat(executorServiceManager.scheduledExecutorService()).isNotSameAs(firstExecutor); - - eventSource.start(); - clearInvocations(resourceFetcher); - - await().untilAsserted(() -> verify(resourceFetcher, atLeast(2)).fetchResources()); - } finally { - eventSource.stop(); - executorServiceManager.stop(Duration.ofMillis(100)); - } - } - - @Test - void pollsOnTheNewSharedSchedulerWhenStartedAgainWhileRunning() { - when(resourceFetcher.fetchResources()).thenReturn(testResponseWithOneValue()); - var configurationService = new BaseConfigurationService(); - var executorServiceManager = configurationService.getExecutorServiceManager(); - var eventSource = - new PollingEventSource( - SampleExternalResource.class, - contextFor(configurationService), - new PollingConfiguration<>(null, resourceFetcher, POLL_PERIOD, null)); - eventSource.setEventHandler(mock(EventHandler.class)); - - try { - eventSource.start(); - executorServiceManager.stop(Duration.ofMillis(100)); - executorServiceManager.start(configurationService); - clearInvocations(resourceFetcher); - - eventSource.start(); - - await().untilAsserted(() -> verify(resourceFetcher, atLeast(2)).fetchResources()); - } finally { - eventSource.stop(); - executorServiceManager.stop(Duration.ofMillis(100)); - } - } - @Test void stoppingDoesNotInterruptAPollInFlight() throws Exception { var polling = new CountDownLatch(1); @@ -266,7 +211,65 @@ void updatesHealthIndicatorBasedOnExceptionsInFetcher() { } @Test - void pollsOnTheOperatorsSharedSchedulerWhenCreatedWithAContext() { + void createsAnExecutorOfItsOwnAndShutsItDownOnStop() { + var pollingThreads = new CopyOnWriteArrayList(); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + pollingThreads.add(Thread.currentThread()); + return testResponseWithOneValue(); + }); + var eventSource = eventSourceFor(new BaseConfigurationService(), null); + + eventSource.start(); + + // the initial fetch happens on the calling thread, the scheduled ones on the own executor + await() + .untilAsserted( + () -> + assertThat(pollingThreads) + .anyMatch(thread -> thread.getName().startsWith("josdk-polling-"))); + var pollingThread = + pollingThreads.stream() + .filter(thread -> thread.getName().startsWith("josdk-polling-")) + .findFirst() + .orElseThrow(); + assertThat(pollingThread.isDaemon()).isTrue(); + + eventSource.stop(); + + await().untilAsserted(() -> assertThat(pollingThread.isAlive()).isFalse()); + } + + @Test + void usesANewExecutorWhenStartedAfterStop() { + var pollingThreads = new CopyOnWriteArrayList(); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + pollingThreads.add(Thread.currentThread()); + return testResponseWithOneValue(); + }); + var eventSource = eventSourceFor(new BaseConfigurationService(), null); + + try { + eventSource.start(); + await() + .until(() -> pollingThreads.stream().anyMatch(PollingEventSourceTest::isPollingThread)); + eventSource.stop(); + pollingThreads.clear(); + + eventSource.start(); + + await() + .until(() -> pollingThreads.stream().anyMatch(PollingEventSourceTest::isPollingThread)); + } finally { + eventSource.stop(); + } + } + + @Test + void pollsOnTheProvidedExecutorAndLeavesItRunningOnStop() { var pollingThreadNames = new CopyOnWriteArrayList(); when(resourceFetcher.fetchResources()) .thenAnswer( @@ -274,32 +277,118 @@ void pollsOnTheOperatorsSharedSchedulerWhenCreatedWithAContext() { pollingThreadNames.add(Thread.currentThread().getName()); return testResponseWithOneValue(); }); + var provided = + Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "provided")); + var eventSource = eventSourceFor(new BaseConfigurationService(), provided); + + try { + eventSource.start(); + await().untilAsserted(() -> assertThat(pollingThreadNames).contains("provided")); + + eventSource.stop(); + + assertThat(provided.isShutdown()).isFalse(); + assertThat(pollingThreadNames).noneMatch(name -> name.startsWith("josdk-polling-")); + } finally { + provided.shutdownNow(); + } + } + @Test + void eventSourcesDoNotShareTheirExecutor() { + var firstThreads = new CopyOnWriteArrayList(); + var secondThreads = new CopyOnWriteArrayList(); + var blockFirst = new CountDownLatch(1); + @SuppressWarnings("unchecked") + PollingEventSource.GenericResourceFetcher slowFetcher = + mock(PollingEventSource.GenericResourceFetcher.class); + var fetches = new AtomicInteger(); + when(slowFetcher.fetchResources()) + .thenAnswer( + invocation -> { + if (fetches.incrementAndGet() > 1) { + firstThreads.add(Thread.currentThread()); + blockFirst.await(); + } + return testResponseWithOneValue(); + }); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + secondThreads.add(Thread.currentThread()); + return testResponseWithOneValue(); + }); var configurationService = new BaseConfigurationService(); - var executorServiceManager = configurationService.getExecutorServiceManager(); + var first = eventSourceFor(configurationService, slowFetcher, null); + var second = eventSourceFor(configurationService, resourceFetcher, null); + + try { + first.start(); + await().until(() -> !firstThreads.isEmpty()); + second.start(); + + // a poll of the first one hanging must not delay the polls of the second one + await().until(() -> secondThreads.size() > 2); + assertThat(secondThreads).doesNotContainAnyElementsOf(firstThreads); + } finally { + blockFirst.countDown(); + first.stop(); + second.stop(); + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_25) + void pollsOnVirtualThreadsWhenEnabledInTheConfiguration() { + var onVirtualThread = new CopyOnWriteArrayList(); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + onVirtualThread.add(isVirtual(Thread.currentThread())); + return testResponseWithOneValue(); + }); + var configurationService = + ConfigurationService.newOverriddenConfigurationService( + new BaseConfigurationService(), o -> o.withUseVirtualThreads(true)); + var eventSource = eventSourceFor(configurationService, null); + + try { + eventSource.start(); + + // the initial fetch happens on the calling thread, the scheduled ones on virtual threads + await().untilAsserted(() -> assertThat(onVirtualThread).contains(true)); + } finally { + eventSource.stop(); + } + } + + private PollingEventSource eventSourceFor( + ConfigurationService configurationService, ScheduledExecutorService executorService) { + return eventSourceFor(configurationService, resourceFetcher, executorService); + } + + private PollingEventSource eventSourceFor( + ConfigurationService configurationService, + PollingEventSource.GenericResourceFetcher fetcher, + ScheduledExecutorService executorService) { var eventSource = new PollingEventSource( SampleExternalResource.class, contextFor(configurationService), - new PollingConfiguration<>(null, resourceFetcher, POLL_PERIOD, null)); + new PollingConfiguration<>(null, fetcher, POLL_PERIOD, null, executorService)); eventSource.setEventHandler(mock(EventHandler.class)); + return eventSource; + } - try { - eventSource.start(); - - // the initial fetch happens on the calling thread, the scheduled ones on the shared pool - await() - .untilAsserted( - () -> - assertThat(pollingThreadNames) - .anyMatch(name -> name.startsWith("josdk-scheduled-task-"))); - - eventSource.stop(); + private static boolean isPollingThread(Thread thread) { + return thread.getName().startsWith("josdk-polling-"); + } - // the shared executor belongs to the operator, stopping an event source must not shut it down - assertThat(executorServiceManager.scheduledExecutorService().isShutdown()).isFalse(); - } finally { - executorServiceManager.stop(Duration.ofMillis(100)); + private static boolean isVirtual(Thread thread) { + try { + return (Boolean) Thread.class.getMethod("isVirtual").invoke(thread); + } catch (ReflectiveOperationException e) { + return false; } } diff --git a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java index 35e0bda882..56e98721a1 100644 --- a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java +++ b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java @@ -82,10 +82,6 @@ public static ConfigLoader getDefault() { "workflow.executor-threads", Integer.class, ConfigurationServiceOverrider::withConcurrentWorkflowExecutorThreads), - new ConfigBinding<>( - "scheduled-tasks.concurrent-threads", - Integer.class, - ConfigurationServiceOverrider::withConcurrentScheduledTaskThreads), new ConfigBinding<>( "retry-and-reschedule.concurrent-threads", Integer.class, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java index 13f44b036a..0cd384e936 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java @@ -86,18 +86,6 @@ void applyConfigsAppliesConcurrentWorkflowExecutorThreads() { assertThat(result.concurrentWorkflowExecutorThreads()).isEqualTo(3); } - @Test - void applyConfigsAppliesConcurrentScheduledTaskThreads() { - var loader = - new ConfigLoader(mapProvider(Map.of("josdk.scheduled-tasks.concurrent-threads", 8))); - - var base = new BaseConfigurationService(null); - var result = - ConfigurationService.newOverriddenConfigurationService(base, loader.applyConfigs()); - - assertThat(result.concurrentScheduledTaskThreads()).isEqualTo(8); - } - @Test void applyConfigsAppliesConcurrentRetryAndRescheduleThreads() { var loader = @@ -340,7 +328,6 @@ public Optional getValue(String key, Class type) { Map.entry("withClusterScopedEventNamespace", "events.cluster-scoped-namespace"), Map.entry( "withConcurrentRetryAndRescheduleThreads", "retry-and-reschedule.concurrent-threads"), - Map.entry("withConcurrentScheduledTaskThreads", "scheduled-tasks.concurrent-threads"), Map.entry("withUseVirtualThreads", "use-virtual-threads")); /** From da582be39ac990137e1158c5f61258e4d22918cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Nov 2026 14:39:19 -0800 Subject: [PATCH 33/45] fix: restore the deprecated scheduledExecutorService accessor and close races in the polling event sources - ExecutorServiceManager keeps the public scheduledExecutorService() of previous versions, deprecated, created on first use and replaced after a restart - PerResourcePollingEventSource drops the result of a fetch from a previous run instead of overwriting the cache of the current one - PollingEventSource ends start() cleanly when stopped during the initial fetch instead of failing on the shut down executor it owns Co-Authored-By: Claude Sonnet 5.5 --- .../api/config/ExecutorServiceManager.java | 24 ++++++++++ .../PerResourcePollingEventSource.java | 18 +++++-- .../source/polling/PollingEventSource.java | 19 ++++++-- .../config/ExecutorServiceManagerTest.java | 20 ++++++++ .../PerResourcePollingEventSourceTest.java | 47 +++++++++++++++++++ .../polling/PollingEventSourceTest.java | 37 +++++++++++++++ 6 files changed, 159 insertions(+), 6 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java index 0e7e7c12c1..7f90b1b60d 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManager.java @@ -42,6 +42,7 @@ public class ExecutorServiceManager { private ExecutorService executor; private ExecutorService workflowExecutor; private ExecutorService cachingExecutorService; + private ScheduledExecutorService scheduledExecutorService; private ScheduledExecutorService retryAndRescheduleExecutorService; private boolean started; private ConfigurationService configurationService; @@ -182,6 +183,27 @@ public ExecutorService cachingExecutorService() { return cachingExecutorService; } + /** + * A general purpose scheduled executor, created on first use, which is shut down by {@link + * #stop(Duration)} and replaced by a fresh one on the next use after a {@link + * #start(ConfigurationService)}. The framework itself doesn't use it: the polling event sources + * poll on executors of their own, and retries are triggered on {@link + * #retryAndRescheduleExecutorService()}. + * + * @return the executor to run scheduled tasks on + * @deprecated the framework doesn't use this executor anymore, create and manage the executor you + * need yourself, e.g. with {@link #newScheduledExecutorService(int, boolean, String)} + */ + @Deprecated(since = "5.7.0") + public synchronized ScheduledExecutorService scheduledExecutorService() { + if (scheduledExecutorService == null) { + scheduledExecutorService = + newScheduledExecutorService( + 1, configurationService.useVirtualThreads(), "josdk-scheduled-task"); + } + return scheduledExecutorService; + } + /** * The executor the operator triggers its retried and rescheduled reconciliations on. Note that it * is only valid while the manager is started: it is shut down by {@link #stop(Duration)} and @@ -212,6 +234,7 @@ public synchronized void stop(Duration gracefulShutdownTimeout) { shutdown(executor, gracefulShutdownTimeout), shutdown(workflowExecutor, gracefulShutdownTimeout), shutdown(cachingExecutorService, gracefulShutdownTimeout), + shutdown(scheduledExecutorService, gracefulShutdownTimeout), shutdown(retryAndRescheduleExecutorService, gracefulShutdownTimeout)); var parallelExec = Executors.newFixedThreadPool(shutdowns.size()); try { @@ -226,6 +249,7 @@ public synchronized void stop(Duration gracefulShutdownTimeout) { // silently leaving the manager with already terminated executors parallelExec.shutdownNow(); workflowExecutor = null; + scheduledExecutorService = null; started = false; } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 22d7471d6e..9fce4eca3b 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -134,10 +134,14 @@ private ScheduledExecutorService resolveExecutorService() { private Set getAndCacheResource(P primary, boolean fromGetter) { var values = resourceFetcher.fetchResources(primary); + cacheResources(primary, values, fromGetter); + return values; + } + + private void cacheResources(P primary, Set values, boolean fromGetter) { var primaryID = ResourceID.fromResource(primary); handleResources(primaryID, values, !fromGetter); fetchedForPrimaries.add(primaryID); - return values; } @SuppressWarnings("unchecked") @@ -238,8 +242,16 @@ public void run() { log.warn("No resource in cache for resource ID: {}", primaryID); // no new execution is scheduled in this case, an on delete event should be received shortly } else { - var actualResources = primary.map(p -> getAndCacheResource(p, false)); - scheduleNextExecution(primary.get(), actualResources.orElse(null), generation); + var values = resourceFetcher.fetchResources(primary.get()); + // a fetch ignoring the interruption of a stop can return after a restart, its result is + // not to overwrite what the polls of the new run cached in the meantime + synchronized (PerResourcePollingEventSource.this) { + if (generation != lifecycleGeneration) { + return; + } + cacheResources(primary.get(), values, false); + scheduleNextExecution(primary.get(), values, generation); + } } } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java index 115f86693e..a33e6cb44e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSource.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.Map; import java.util.Set; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -139,15 +140,27 @@ public void start() throws OperatorException { } private void schedulePolling() { - var task = - executorService.scheduleWithFixedDelay( - this::poll, period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS); + ScheduledFuture task; + try { + task = + executorService.scheduleWithFixedDelay( + this::poll, period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + // a stop during the initial fetch shut down the executor this event source owns + if (isRunning()) { + throw e; + } + return; + } pollingTask = task; // stop() clears the running flag before it looks for the task, so a stop that happened during // the initial fetch is either seen by stop (which cancels the task) or caught here if (!isRunning()) { task.cancel(false); pollingTask = null; + if (ownsExecutorService) { + executorService.shutdown(); + } } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java index c06f0db26c..5105139e02 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ExecutorServiceManagerTest.java @@ -27,6 +27,26 @@ class ExecutorServiceManagerTest { private static final Duration SHUTDOWN_TIMEOUT = Duration.ofMillis(100); + @Test + @SuppressWarnings("deprecation") + void stopShutsDownTheScheduledExecutorServiceAndItIsReplacedAfterARestart() { + ConfigurationService configurationService = new BaseConfigurationService(); + var manager = configurationService.getExecutorServiceManager(); + var scheduled = manager.scheduledExecutorService(); + + assertThat(scheduled.isShutdown()).isFalse(); + manager.stop(SHUTDOWN_TIMEOUT); + assertThat(scheduled.isShutdown()).isTrue(); + + manager.start(configurationService); + try { + assertThat(manager.scheduledExecutorService()).isNotSameAs(scheduled); + assertThat(manager.scheduledExecutorService().isShutdown()).isFalse(); + } finally { + manager.stop(SHUTDOWN_TIMEOUT); + } + } + @Test void stopShutsDownTheRetryAndRescheduleExecutorService() { ConfigurationService configurationService = new BaseConfigurationService(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 4a0258c3e9..9be00dcfa1 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -320,6 +320,53 @@ void aFetchOutlivingARestartDoesNotStartASecondPollingChain() throws Exception { .untilAsserted(() -> verify(supplier, atMost(15)).fetchResources(any())); } + @Test + void aFetchOutlivingARestartDoesNotOverwriteTheCacheOfTheNewRun() throws Exception { + var fetching = new CountDownLatch(1); + var release = new CountDownLatch(1); + var oldResource = SampleExternalResource.testResource1(); + var newResource = SampleExternalResource.testResource2(); + when(supplier.fetchResources(any())) + .thenAnswer( + invocation -> { + fetching.countDown(); + var interrupted = false; + while (release.getCount() > 0) { + try { + release.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + return Set.of(oldResource); + }) + .thenReturn(Set.of(newResource)); + + source.onResourceCreated(testCustomResource); + assertThat(fetching.await(5, TimeUnit.SECONDS)).isTrue(); + source.stop(); + source.start(); + source.onResourceCreated(testCustomResource); + await().untilAsserted(() -> verify(supplier, atLeast(2)).fetchResources(any())); + await() + .untilAsserted( + () -> + assertThat(source.getSecondaryResources(testCustomResource)) + .containsExactly(newResource)); + + release.countDown(); + + await() + .pollDelay(Duration.ofMillis(PERIOD / 3)) + .untilAsserted( + () -> + assertThat(source.getSecondaryResources(testCustomResource)) + .containsExactly(newResource)); + } + @Test void aRegistrationRacingStopDoesNotBlockPollingAfterARestart() throws Exception { var configurationService = new BaseConfigurationService(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java index 1cd483aceb..9641a7302f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PollingEventSourceTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -362,6 +363,42 @@ void pollsOnVirtualThreadsWhenEnabledInTheConfiguration() { } } + @Test + void stoppingDuringTheInitialFetchEndsStartCleanly() throws Exception { + var fetching = new CountDownLatch(1); + var release = new CountDownLatch(1); + when(resourceFetcher.fetchResources()) + .thenAnswer( + invocation -> { + fetching.countDown(); + release.await(); + return testResponseWithOneValue(); + }); + var eventSource = eventSourceFor(new BaseConfigurationService(), null); + var failure = new AtomicReference(); + var starter = + new Thread( + () -> { + try { + eventSource.start(); + } catch (Throwable t) { + failure.set(t); + } + }); + + starter.start(); + assertThat(fetching.await(5, TimeUnit.SECONDS)).isTrue(); + eventSource.stop(); + release.countDown(); + starter.join(5000); + + assertThat(starter.isAlive()).isFalse(); + assertThat(failure.get()).isNull(); + clearInvocations(resourceFetcher); + Thread.sleep(DEFAULT_WAIT_PERIOD); + verify(resourceFetcher, never()).fetchResources(); + } + private PollingEventSource eventSourceFor( ConfigurationService configurationService, ScheduledExecutorService executorService) { return eventSourceFor(configurationService, resourceFetcher, executorService); From 64363090dedd2fbbf20aab777174f00b36745574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Nov 2026 14:44:57 -0800 Subject: [PATCH 34/45] fix: synchronize on the outer class method instead of exposing the lock in the fetching task Co-Authored-By: Claude Sonnet 5.5 --- .../PerResourcePollingEventSource.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 9fce4eca3b..a5617f115c 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -144,6 +144,17 @@ private void cacheResources(P primary, Set values, boolean fromGetter) { fetchedForPrimaries.add(primaryID); } + // a fetch ignoring the interruption of a stop can return after a restart, its result is not to + // overwrite what the polls of the new run cached in the meantime + private synchronized void cacheAndScheduleNextExecution( + P primary, Set values, long generation) { + if (generation != lifecycleGeneration) { + return; + } + cacheResources(primary, values, false); + scheduleNextExecution(primary, values, generation); + } + @SuppressWarnings("unchecked") private synchronized void scheduleNextExecution( P primary, Set actualResources, long generation) { @@ -243,15 +254,7 @@ public void run() { // no new execution is scheduled in this case, an on delete event should be received shortly } else { var values = resourceFetcher.fetchResources(primary.get()); - // a fetch ignoring the interruption of a stop can return after a restart, its result is - // not to overwrite what the polls of the new run cached in the meantime - synchronized (PerResourcePollingEventSource.this) { - if (generation != lifecycleGeneration) { - return; - } - cacheResources(primary.get(), values, false); - scheduleNextExecution(primary.get(), values, generation); - } + cacheAndScheduleNextExecution(primary.get(), values, generation); } } } From 81ed9ae0303cb120fcdd9eef54d8aa41a40ee45f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Nov 2026 15:51:31 -0800 Subject: [PATCH 35/45] docs: point the virtual threads note to Operator.keepAlive for keeping the JVM alive Co-Authored-By: Claude Sonnet 5.5 --- docs/content/en/docs/documentation/operations/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index 3c944fe2c9..9b075907bf 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -59,7 +59,8 @@ Things to keep in mind: `withConcurrentRetryAndRescheduleThreads(int)` only applies to platform threads. - Virtual threads are always daemon threads, so unlike the threads of the platform thread pools they don't keep the JVM running: if the `main` method of your application returns after starting the - operator, make sure something else keeps the JVM alive. + operator and nothing else keeps the JVM alive, call `Operator.keepAlive()`, see + [Keeping the JVM Alive](#keeping-the-jvm-alive). - A custom `ExecutorService` provided through `withExecutorService(...)` or `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. - The Kubernetes client the framework creates when none is provided also switches its internal From d57d8f233246f90084d39399f75e176c21dcd70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 10:01:35 -0800 Subject: [PATCH 36/45] fix: do not schedule again the timers whose handler is still running when the executor is replaced Co-Authored-By: Claude Sonnet 5.5 --- .../event/source/timer/TimerEventSource.java | 6 ++- .../source/timer/TimerEventSourceTest.java | 43 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index bcfea40fec..31be829512 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -166,12 +166,14 @@ public void start() { /** * The timers that were pending on the replaced executor are gone with it, so they are scheduled * again on the new one for what remained of their delay. The ones that already fired are left - * alone, and the ones explicitly cancelled are not in the map anymore. + * alone, including the ones whose handler is still running, since scheduling those again would + * produce the event twice. The ones explicitly cancelled are not in the map anymore. */ private void rearmPendingTimers() { onceTasks.forEach( (resourceID, future) -> { - if (future.isCancelled() || !future.isDone()) { + if (future.isCancelled() + || (!future.isDone() && future.getDelay(TimeUnit.MILLISECONDS) > 0)) { scheduleOnce(resourceID, Math.max(0, future.getDelay(TimeUnit.MILLISECONDS))); } }); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index 86c8973f47..d4f1fd709d 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -22,6 +22,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.awaitility.Awaitility; @@ -162,6 +163,48 @@ public void timersThatAlreadyFiredAreNotScheduledAgainOnTheNewExecutor() { } } + @Test + public void timersWhoseHandlerIsStillRunningAreNotScheduledAgainOnTheNewExecutor() + throws Exception { + var first = Executors.newScheduledThreadPool(1); + var second = Executors.newScheduledThreadPool(1); + var current = new AtomicReference(first); + var handling = new CountDownLatch(1); + var release = new CountDownLatch(1); + var handled = new AtomicInteger(); + var timerSource = new TimerEventSource("timer", false, current::get); + setUpSource( + timerSource, + new CapturingEventHandler() { + @Override + public void handleEvent(Event event) { + if (handled.incrementAndGet() > 1) { + return; + } + handling.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }); + try { + source.scheduleOnce(ResourceID.fromResource(TestUtils.testCustomResource()), PERIOD); + assertThat(handling.await(5, TimeUnit.SECONDS)).isTrue(); + first.shutdown(); + current.set(second); + + source.start(); + + untilAsserted(PERIOD * 4, 0, () -> assertThat(handled).hasValue(1)); + } finally { + release.countDown(); + first.shutdownNow(); + second.shutdownNow(); + } + } + @Test public void stoppingDoesNotInterruptAnEventHandlerInFlight() throws Exception { var handling = new CountDownLatch(1); From bb1fd8a6a09d5201f567e199db7ca7aa45f96165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 11:03:08 -0800 Subject: [PATCH 37/45] refactor: deprecate the TimerEventSource constructor without an executor supplier Co-Authored-By: Claude Sonnet 5.5 --- .../processing/event/source/timer/TimerEventSource.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index 31be829512..5353cc10f8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -52,6 +52,12 @@ public TimerEventSource() { this((Supplier) null); } + /** + * @deprecated use {@link #TimerEventSource(String, boolean, Supplier)} instead, with a {@code + * null} supplier to keep the behavior of this constructor, which is to have the event source + * create, and shut down, an executor of its own + */ + @Deprecated(since = "5.7.0") public TimerEventSource(String name, boolean triggerReconcilerOnAllEvents) { this(name, triggerReconcilerOnAllEvents, null); } From c061f092ffaf9e8e84b28087c7af4b938cf403ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 11:03:08 -0800 Subject: [PATCH 38/45] docs: document that overriding the virtual threads flag replaces a custom retry executor Co-Authored-By: Claude Sonnet 5.5 --- .../en/docs/documentation/operations/configuration.md | 6 +++++- .../api/config/ConfigurationServiceOverrider.java | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index 9b075907bf..831c1aa60b 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -62,7 +62,11 @@ Things to keep in mind: operator and nothing else keeps the JVM alive, call `Operator.keepAlive()`, see [Keeping the JVM Alive](#keeping-the-jvm-alive). - A custom `ExecutorService` provided through `withExecutorService(...)` or - `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. + `withWorkflowExecutorService(...)` is always used as is, the flag has no effect on it. The same + goes for `withRetryAndRescheduleExecutorService(...)`. Note however that overriding the flag, or + `withConcurrentRetryAndRescheduleThreads(int)`, of a configuration that provides its own retry and + reschedule executor replaces that executor with the default one, so provide it again with + `withRetryAndRescheduleExecutorService(...)` in that case. - The Kubernetes client the framework creates when none is provided also switches its internal task executor (used to dispatch informer events to their handlers and to deliver watch events) to virtual threads. A client you provide through `withKubernetesClient(...)` is used as is: to diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java index e8a664098f..05112b4da7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationServiceOverrider.java @@ -94,6 +94,11 @@ public ConfigurationServiceOverrider withConcurrentWorkflowExecutorThreads(int t * Sets the number of threads used to trigger the operator's retried and rescheduled * reconciliations, which are shared by all its controllers. * + *

Note that changing the number of threads makes the overridden configuration build the + * default retry and reschedule executor itself, so a custom one provided by the configuration + * being overridden is not used anymore. Use {@link #withRetryAndRescheduleExecutorService} to + * provide one. + * * @param threadNumber the maximum number of concurrent retry and reschedule threads * @return this {@link ConfigurationServiceOverrider} for chained customization * @see ConfigurationService#concurrentRetryAndRescheduleThreads() @@ -158,6 +163,11 @@ public ConfigurationServiceOverrider withRetryAndRescheduleExecutorService( * platform threads. Only supported on Java 25 or later, see {@link * ConfigurationService#useVirtualThreads()} for the details. * + *

Note that if this changes the value of the configuration being overridden, the overridden + * configuration builds the default retry and reschedule executor itself, so a custom one provided + * by the configuration being overridden is replaced and not used as is. Use {@link + * #withRetryAndRescheduleExecutorService} to provide one along with this setting. + * * @param useVirtualThreads {@code true} to use virtual threads * @return this {@link ConfigurationServiceOverrider} for chained customization * @since 5.7.0 From 50381e04d40a6d115b1b482f650eb7ce3a013cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 11:16:43 -0800 Subject: [PATCH 39/45] refactor: do not schedule again the pending timers on a replaced executor and drop the executor supplier only constructor of TimerEventSource - the timers pending on an executor the operator replaced are not scheduled again - the constructor taking just an executor supplier is removed, the event source is mostly for internal usage Co-Authored-By: Claude Sonnet 5.5 --- .../event/source/timer/TimerEventSource.java | 48 ++++--------------- .../source/timer/TimerEventSourceTest.java | 24 +--------- 2 files changed, 10 insertions(+), 62 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java index 5353cc10f8..c244242a62 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSource.java @@ -49,15 +49,11 @@ public class TimerEventSource extends AbstractEventSource private volatile ScheduledExecutorService executorService; public TimerEventSource() { - this((Supplier) null); + super(Void.class); + this.executorServiceSupplier = null; + this.ownsExecutorService = true; } - /** - * @deprecated use {@link #TimerEventSource(String, boolean, Supplier)} instead, with a {@code - * null} supplier to keep the behavior of this constructor, which is to have the event source - * create, and shut down, an executor of its own - */ - @Deprecated(since = "5.7.0") public TimerEventSource(String name, boolean triggerReconcilerOnAllEvents) { this(name, triggerReconcilerOnAllEvents, null); } @@ -67,20 +63,12 @@ public TimerEventSource(String name, boolean triggerReconcilerOnAllEvents) { * supplier is called on every start, and not once at creation time, since the operator shuts its * executors down when it is stopped and creates new ones if it is started again. * + * @param name the name of the event source + * @param triggerReconcilerOnAllEvents whether the reconciler is triggered on all events * @param executorServiceSupplier supplies the executor to schedule on, {@code null} to have the * event source create, and shut down, an executor of its own * @since 5.7.0 */ - public TimerEventSource(Supplier executorServiceSupplier) { - super(Void.class); - this.executorServiceSupplier = executorServiceSupplier; - this.ownsExecutorService = executorServiceSupplier == null; - } - - /** - * @see #TimerEventSource(Supplier) - * @since 5.7.0 - */ public TimerEventSource( String name, boolean triggerReconcilerOnAllEvents, @@ -155,36 +143,16 @@ public void cancelOnceSchedule(ResourceID customResourceUid) { @Override public void start() { // the operator replaces its executors when it is started again, which can happen while this - // event source keeps running, when a previous start failed after it started + // event source keeps running, when a previous start failed after it started. The timers + // pending on the replaced executor are not scheduled again. if (!isRunning() || !ownsExecutorService) { - final var resolved = resolveExecutorService(); - final var replaced = isRunning() && resolved != executorService; - executorService = resolved; - if (replaced) { - rearmPendingTimers(); - } + executorService = resolveExecutorService(); } if (!isRunning()) { super.start(); } } - /** - * The timers that were pending on the replaced executor are gone with it, so they are scheduled - * again on the new one for what remained of their delay. The ones that already fired are left - * alone, including the ones whose handler is still running, since scheduling those again would - * produce the event twice. The ones explicitly cancelled are not in the map anymore. - */ - private void rearmPendingTimers() { - onceTasks.forEach( - (resourceID, future) -> { - if (future.isCancelled() - || (!future.isDone() && future.getDelay(TimeUnit.MILLISECONDS) > 0)) { - scheduleOnce(resourceID, Math.max(0, future.getDelay(TimeUnit.MILLISECONDS))); - } - }); - } - private ScheduledExecutorService resolveExecutorService() { if (executorServiceSupplier != null) { return executorServiceSupplier.get(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java index d4f1fd709d..e5ac53441d 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/timer/TimerEventSourceTest.java @@ -121,27 +121,6 @@ public void schedulesOnTheNewExecutorWhenStartedAgainWhileRunning() { } } - @Test - public void timersPendingOnTheReplacedExecutorAreScheduledAgainOnTheNewOne() { - var first = Executors.newScheduledThreadPool(1); - var second = Executors.newScheduledThreadPool(1); - var current = new AtomicReference(first); - var timerSource = new TimerEventSource("timer", false, current::get); - setUpSource(timerSource, new CapturingEventHandler()); - try { - var resourceID = ResourceID.fromResource(TestUtils.testCustomResource()); - source.scheduleOnce(resourceID, 2 * PERIOD); - first.shutdownNow(); - current.set(second); - - source.start(); - - untilAsserted(() -> assertThat(eventHandler.events).hasSize(1)); - } finally { - second.shutdownNow(); - } - } - @Test public void timersThatAlreadyFiredAreNotScheduledAgainOnTheNewExecutor() { var first = Executors.newScheduledThreadPool(1); @@ -269,7 +248,8 @@ public void schedulesOnTheProvidedExecutorAndLeavesItRunningOnStop() { Executors.newSingleThreadScheduledExecutor( ExecutorUtilsInternal.daemonThreadFactory("provided-timer-executor")); var handler = new CapturingEventHandler(); - var eventSource = new TimerEventSource(() -> providedExecutor); + var eventSource = + new TimerEventSource("timer", false, () -> providedExecutor); eventSource.setEventHandler(handler); try { From 7939ed2be1ec278b4140fd3c0434b31250694ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 11:40:23 -0800 Subject: [PATCH 40/45] refactor: register the resources of the primary cache when the per resource polling starts instead of buffering registrations - the registrations seen before start, or racing stop, are dropped, start registers every resource of the primary cache - the registration racing the end of start is not handled anymore Co-Authored-By: Claude Sonnet 5.5 --- .../PerResourcePollingEventSource.java | 38 +++---------------- .../PerResourcePollingEventSourceTest.java | 4 +- 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index a5617f115c..523869cde9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -65,8 +65,6 @@ public class PerResourcePollingEventSource private final Predicate

registerPredicate; private final Duration period; - private final Map registrationsBeforeStart = new ConcurrentHashMap<>(); - private volatile ScheduledExecutorService executorService; // incremented on every stop, to tell the polls of previous runs apart, which can outlive the @@ -108,16 +106,9 @@ public void start() throws OperatorException { } executorService = resolveExecutorService(); super.start(); - replayRegistrationsBeforeStart(); - } - - private void replayRegistrationsBeforeStart() { - for (var primaryID : registrationsBeforeStart.keySet()) { - var resource = registrationsBeforeStart.remove(primaryID); - if (resource != null) { - checkAndRegisterTask(resource); - } - } + // the primary informer is started before this event source, and the executor to poll on is + // only resolved when this one starts, so the resources seen in the meantime are registered now + primaryResourceCache.list().forEach(this::checkAndRegisterTask); } /** @@ -187,7 +178,6 @@ public void onResourceUpdated(P newResource, P oldResource) { @Override public void onResourceDeleted(P resource) { var resourceID = ResourceID.fromResource(resource); - registrationsBeforeStart.remove(resourceID); var scheduledFuture = scheduledFutures.remove(resourceID); if (scheduledFuture != null) { log.debug("Canceling scheduledFuture for resource: {}", resource); @@ -200,28 +190,13 @@ public void onResourceDeleted(P resource) { // This method is always called from the same Thread for the same resource, // since events from ResourceEventAware are propagated from the thread of the informer. This is // important because otherwise there will be a race condition related to the timerTasks. - private void checkAndRegisterTask(P resource) { - var primaryID = ResourceID.fromResource(resource); - // the primary informer is started before this event source, and the executor to poll on is - // only resolved when this one starts, so resources seen in the meantime are registered then - if (!isRunning()) { - registrationsBeforeStart.put(primaryID, resource); - if (!isRunning()) { - return; - } - replayRegistrationsBeforeStart(); - return; - } - registerTask(primaryID, resource); - } - - private synchronized void registerTask(ResourceID primaryID, P resource) { + private synchronized void checkAndRegisterTask(P resource) { // stop() clears the running flag before it takes the monitor, so a registration that raced it - // has to be held back, otherwise a task scheduled after the stop would block a restart + // is dropped, otherwise a task scheduled after the stop would block a restart if (!isRunning()) { - registrationsBeforeStart.put(primaryID, resource); return; } + var primaryID = ResourceID.fromResource(resource); if (scheduledFutures.get(primaryID) == null && (registerPredicate == null || registerPredicate.test(resource))) { var cachedResources = cachedResourcesFor(primaryID); @@ -312,7 +287,6 @@ public void stop() throws OperatorException { scheduledFutures.values().forEach(future -> future.cancel(true)); scheduledFutures.clear(); } - registrationsBeforeStart.clear(); if (ownsExecutorService && executorService != null) { executorService.shutdownNow(); } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index 9be00dcfa1..f538cbcabe 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -99,7 +100,7 @@ void pollsTheResourceAfterAwareOfIt() { } @Test - void resourcesSeenBeforeStartArePolledOnceStarted() { + void resourcesInThePrimaryCacheArePolledOnceStarted() { source.stop(); setUpSource( new PerResourcePollingEventSource<>( @@ -111,6 +112,7 @@ void resourcesSeenBeforeStartArePolledOnceStarted() { .withResourceIDMapper(r -> r.getName() + "#" + r.getValue()) .build()), false); + when(resourceCache.list()).thenAnswer(i -> Stream.of(testCustomResource)); source.onResourceCreated(testCustomResource); verify(supplier, never()).fetchResources(any()); From 7624b1104584890edb8a496970f323cedccd85c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 14:45:00 -0800 Subject: [PATCH 41/45] fix: let a resource be polled again when it is recreated after its poll found it gone from the cache - a poll that finds its primary resource gone from the cache removes its registration, so a recreated resource is registered again instead of being skipped for the stale one - onResourceDeleted is synchronized with the registration and scheduling of the polls Co-Authored-By: Claude Sonnet 5.5 --- .../polling/PerResourcePollingEventSource.java | 11 ++++++++++- .../polling/PerResourcePollingEventSourceTest.java | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java index 523869cde9..10c8cba913 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSource.java @@ -135,6 +135,14 @@ private void cacheResources(P primary, Set values, boolean fromGetter) { fetchedForPrimaries.add(primaryID); } + // the polling of a resource that is gone ends with its last task, which would otherwise stay + // registered and keep the resource from being polled again if it is recreated + private synchronized void forgetTask(ResourceID primaryID, long generation) { + if (generation == lifecycleGeneration) { + scheduledFutures.remove(primaryID); + } + } + // a fetch ignoring the interruption of a stop can return after a restart, its result is not to // overwrite what the polls of the new run cached in the meantime private synchronized void cacheAndScheduleNextExecution( @@ -176,7 +184,7 @@ public void onResourceUpdated(P newResource, P oldResource) { } @Override - public void onResourceDeleted(P resource) { + public synchronized void onResourceDeleted(P resource) { var resourceID = ResourceID.fromResource(resource); var scheduledFuture = scheduledFutures.remove(resourceID); if (scheduledFuture != null) { @@ -227,6 +235,7 @@ public void run() { if (primary.isEmpty()) { log.warn("No resource in cache for resource ID: {}", primaryID); // no new execution is scheduled in this case, an on delete event should be received shortly + forgetTask(primaryID, generation); } else { var values = resourceFetcher.fetchResources(primary.get()); cacheAndScheduleNextExecution(primary.get(), values, generation); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java index f538cbcabe..f5109532d1 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/polling/PerResourcePollingEventSourceTest.java @@ -573,6 +573,20 @@ private static boolean isVirtual(Thread thread) { } } + @Test + void aResourceGoneFromTheCacheWhenItsPollRunsIsPolledAgainWhenRecreated() { + when(resourceCache.get(any())).thenReturn(Optional.empty()); + source.onResourceCreated(testCustomResource); + await() + .pollDelay(Duration.ofMillis(2 * PERIOD)) + .untilAsserted(() -> verify(supplier, never()).fetchResources(any())); + + when(resourceCache.get(any())).thenReturn(Optional.of(testCustomResource)); + source.onResourceCreated(testCustomResource); + + await().untilAsserted(() -> verify(supplier, atLeast(1)).fetchResources(any())); + } + @Test void deleteEventCancelsTheScheduling() { when(supplier.fetchResources(any())).thenReturn(Set.of(SampleExternalResource.testResource1())); From 565d891fec9eb13355b5cfdd8bac50cba6161b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 14:45:00 -0800 Subject: [PATCH 42/45] test: remove the throwaway audit test and leftovers, fix a misleading test name - remove BoundedVirtualThreadExecutorAuditTest and the AUDIT_N override of VirtualThreadsStartupDeadlockTest - describe in the Javadoc of VirtualThreadsStartupDeadlockTest what is guaranteed on which Java version - rename the test checking that executeAndWaitForAllToComplete respects the bound of the executor Co-Authored-By: Claude Sonnet 5.5 --- .../VirtualThreadsStartupDeadlockTest.java | 18 +-- ...BoundedVirtualThreadExecutorAuditTest.java | 124 ------------------ .../api/config/VirtualThreadsTest.java | 2 +- 3 files changed, 8 insertions(+), 136 deletions(-) delete mode 100644 operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java index 914559eeb8..0262e5faff 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/VirtualThreadsStartupDeadlockTest.java @@ -36,12 +36,13 @@ import static org.mockito.Mockito.when; /** - * With virtual threads enabled on Java 21 to 24 (no JEP 491, hence only run on Java 25 or later, - * the official baseline for virtual thread support), starting as many controllers as there are - * virtual thread carriers deadlocks, because each controller is started on a virtual thread of the - * caching executor that blocks, while holding the monitors of the synchronized Controller.start / + * Starting as many controllers as there are virtual thread carriers must not deadlock. On Java 21 + * to 24 (no JEP 491) it does: each controller is started on a virtual thread of the caching + * executor that blocks, while holding the monitors of the synchronized Controller.start / * EventSourceManager.start / ControllerEventSource.start, on informer-start tasks submitted to the - * very same virtual thread executor. + * very same virtual thread executor, and the carriers are pinned by those monitors. This is why the + * virtual thread variant only runs on Java 25 or later, the official baseline for virtual thread + * support, where the monitors don't pin the carriers anymore. */ class VirtualThreadsStartupDeadlockTest { @@ -67,12 +68,7 @@ private static void assertStartsWithoutDeadlock(boolean useVirtualThreads) throw final var executorServiceManager = configService.getExecutorServiceManager(); final var controllerManager = new ControllerManager(executorServiceManager); - // default virtual thread scheduler parallelism == number of available processors, e.g. 1 or 2 - // in a typical resource-limited operator pod - final int controllers = - System.getenv("AUDIT_N") != null - ? Integer.parseInt(System.getenv("AUDIT_N")) - : Runtime.getRuntime().availableProcessors(); + final int controllers = Runtime.getRuntime().availableProcessors(); for (int i = 0; i < controllers; i++) { final var configuration = MockControllerConfiguration.forResource(ConfigMap.class); when(configuration.getConfigurationService()).thenReturn(configService); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java deleted file mode 100644 index 5e17c52ed9..0000000000 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/BoundedVirtualThreadExecutorAuditTest.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright Java Operator SDK Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.javaoperatorsdk.operator.api.config; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledForJreRange; -import org.junit.jupiter.api.condition.JRE; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; - -/** Throwaway audit tests for VirtualThreads.BoundedExecutorService. */ -class BoundedVirtualThreadExecutorAuditTest { - - @Test - void controlPlatformPoolRejectsNonPositiveConcurrency() { - assertThatIllegalArgumentException() - .isThrownBy(() -> ExecutorServiceManager.newBoundedExecutorService(0, false)); - } - - /** - * The platform variant fails fast for a non positive limit, the virtual variant creates a - * Semaphore(0) (or a negative one) whose tasks wait for a permit forever. - */ - @Test - @EnabledForJreRange(min = JRE.JAVA_25) - void virtualThreadVariantRejectsNonPositiveConcurrencyToo() { - assertThatIllegalArgumentException() - .isThrownBy(() -> ExecutorServiceManager.newBoundedExecutorService(0, true).shutdownNow()); - } - - @Test - void controlPlatformPoolShutdownNowReturnsTasksThatNeverStarted() throws Exception { - assertThat(queuedTasksReturnedByShutdownNow(false)).isEqualTo(5); - } - - /** - * ExecutorService#shutdownNow must return the tasks that never commenced execution; the bounded - * virtual executor returns an empty list, and plain Runnables handed to execute() that were still - * waiting for a permit are silently dropped (neither run nor returned). - */ - @Test - @EnabledForJreRange(min = JRE.JAVA_25) - void virtualThreadVariantShutdownNowReturnsTasksThatNeverStarted() throws Exception { - assertThat(queuedTasksReturnedByShutdownNow(true)).isEqualTo(5); - } - - @Test - @EnabledForJreRange(min = JRE.JAVA_25) - void everyTaskSubmittedWhileShuttingDownIsRunReturnedOrRejected() throws Exception { - for (int round = 0; round < 50; round++) { - final var executor = ExecutorServiceManager.newBoundedExecutorService(1, true); - final var accepted = new AtomicInteger(); - final var ran = new AtomicInteger(); - final var submitters = new CountDownLatch(4); - final var go = new CountDownLatch(1); - for (int t = 0; t < 4; t++) { - new Thread( - () -> { - try { - go.await(); - for (int i = 0; i < 200; i++) { - executor.execute(ran::incrementAndGet); - accepted.incrementAndGet(); - } - } catch (RejectedExecutionException | InterruptedException e) { - // the executor was shut down - } finally { - submitters.countDown(); - } - }) - .start(); - } - go.countDown(); - final var notStarted = executor.shutdownNow(); - assertThat(submitters.await(10, TimeUnit.SECONDS)).isTrue(); - assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); - // tasks admitted after shutdownNow returned can't exist, so none may go missing - assertThat(ran.get() + notStarted.size()).isEqualTo(accepted.get()); - } - } - - private static int queuedTasksReturnedByShutdownNow(boolean useVirtualThreads) throws Exception { - final var executor = ExecutorServiceManager.newBoundedExecutorService(1, useVirtualThreads); - final var running = new CountDownLatch(1); - final var ran = new AtomicInteger(); - executor.execute( - () -> { - running.countDown(); - try { - Thread.sleep(60_000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }); - assertThat(running.await(10, TimeUnit.SECONDS)).isTrue(); - for (int i = 0; i < 5; i++) { - executor.execute(ran::incrementAndGet); - } - Thread.sleep(200); // let the virtual threads reach the permit acquisition - final var notStarted = executor.shutdownNow(); - assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); - assertThat(ran).hasValue(0); - return notStarted.size(); - } -} diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java index 6c3b0854e3..349baf5247 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadsTest.java @@ -133,7 +133,7 @@ void boundedVirtualThreadExecutorRespectsTheConfiguredConcurrency() throws Excep @Test @EnabledForJreRange(min = JRE.JAVA_25) - void invokeAllIsBoundedTooSinceItIsUsedToStartTheEventSources() { + void executeAndWaitForAllToCompleteRespectsTheBoundOfTheExecutor() { var executor = ExecutorServiceManager.newBoundedExecutorService(MAX_CONCURRENCY, true); try { final var running = new AtomicInteger(); From 12907da022f586915b9ad391250d6231228651e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 14:45:00 -0800 Subject: [PATCH 43/45] docs: fix the polling executor example and describe the executor changes in the migration guide - the example of an executor with more threads uses PerResourcePollingConfigurationBuilder, a PollingEventSource runs a single periodic task - the migration guide describes the executors of the polling event sources, the provided executor that is not shut down anymore and the deprecations Co-Authored-By: Claude Sonnet 5.5 --- .../content/en/docs/documentation/eventing.md | 12 ++++--- .../en/docs/migration/v5-7-migration.md | 32 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/content/en/docs/documentation/eventing.md b/docs/content/en/docs/documentation/eventing.md index 21107b3c3e..03d4dcab86 100644 --- a/docs/content/en/docs/documentation/eventing.md +++ b/docs/content/en/docs/documentation/eventing.md @@ -226,14 +226,16 @@ actually observed primary resources. Each polling event source polls on a scheduled executor of its own, so a slow poll of one event source never delays the polls of the others. Unless you provide one, the event source creates a -single daemon thread executor when it is started and shuts it down when it is stopped. A poll only -starts once the previous one is done, so a source that fetches slowly, or one with many registered -resources in the case of `PerResourcePollingEventSource`, can be given an executor with more threads -using the `withExecutorService` method of its configuration builder: +single daemon thread executor when it is started and shuts it down when it is stopped. With a single +thread the polls of a source run one after the other. A `PollingEventSource` runs a single periodic +task, so more threads would not help it. A `PerResourcePollingEventSource` however polls each +registered resource on its own, so if it has many registered resources or fetches slowly, it can be +given an executor with more threads using the `withExecutorService` method of its configuration +builder: ```java var config = - new PollingConfigurationBuilder<>(this::fetchResources, Duration.ofSeconds(5)) + new PerResourcePollingConfigurationBuilder<>(this::fetchResources, Duration.ofSeconds(5)) .withExecutorService(Executors.newScheduledThreadPool(4)) .build(); ``` diff --git a/docs/content/en/docs/migration/v5-7-migration.md b/docs/content/en/docs/migration/v5-7-migration.md index 8d735c8acd..9ac3ae40b4 100644 --- a/docs/content/en/docs/migration/v5-7-migration.md +++ b/docs/content/en/docs/migration/v5-7-migration.md @@ -78,3 +78,35 @@ on the classpath next to your implementation. This is especially important if yo based client (`kubernetes-httpclient-vertx`), since Vert.x 4 and Vert.x 5 cannot be used together. If you want to keep the Vert.x 4 based client, use it in place of `kubernetes-httpclient-jdk` in the snippet above. + +## Executors of the polling and timer event sources + +The polling event sources and the internal `TimerEventSource` no longer use a `java.util.Timer`, and +`PerResourcePollingEventSource` no longer creates its executor when its configuration is built. They +run on scheduled executors, see [virtual threads](../documentation/operations/configuration/#virtual-threads) +and [Threading of the polling event sources](../documentation/eventing/#threading-of-the-polling-event-sources). +This is transparent unless your operator provides an executor to a polling event source, or relies +on one of the following: + +- A `ScheduledExecutorService` provided with `withExecutorService(...)` to a + `PerResourcePollingConfigurationBuilder` was shut down with `shutdownNow()` when the event source + was stopped. It is not anymore: **an executor you provide is yours to shut down**, like it is for + `PollingConfigurationBuilder`. If you relied on the event source for this, shut it down yourself + after stopping the operator, otherwise its threads can keep the JVM from exiting if they are not + daemon threads. +- `PerResourcePollingConfiguration.executorService()` returns `null` when no executor was provided, + it used to be a `ScheduledThreadPoolExecutor` with a single non-daemon thread created by the + configuration. The event source now creates a single daemon thread executor, or one running on + virtual threads if `useVirtualThreads` is enabled, when it starts, shuts it down when it stops, and + creates a new one if it is started again. `PerResourcePollingConfiguration.DEFAULT_EXECUTOR_THREAD_NUMBER` + is deprecated, it isn't used anymore. +- Each polling event source has an executor of its own: they don't share threads, and the number of + threads of a source is not configurable other than by providing an executor. +- `new PollingEventSource<>(Class, PollingConfiguration)` is deprecated. Use the constructor that + takes the `EventSourceContext` as well, `new PollingEventSource<>(Class, EventSourceContext, + PollingConfiguration)`, otherwise the event source cannot know about the `useVirtualThreads` + setting and polls on a platform thread. +- `TimerEventSource(String, boolean)` is deprecated, use the constructor that also takes an + executor supplier, `null` to have the event source create an executor of its own. +- `ExecutorServiceManager.scheduledExecutorService()` is deprecated. The framework doesn't use it + anymore, it creates the executor on first use, and shuts it down when the manager is stopped. From 83d9e11a46bf47409efc99144b08107a9ef1d6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 14:45:00 -0800 Subject: [PATCH 44/45] fix: set the root log level of the webpage sample back to info Co-Authored-By: Claude Sonnet 5.5 --- sample-operators/webpage/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample-operators/webpage/src/main/resources/log4j2.xml b/sample-operators/webpage/src/main/resources/log4j2.xml index 2979258355..c8c007835f 100644 --- a/sample-operators/webpage/src/main/resources/log4j2.xml +++ b/sample-operators/webpage/src/main/resources/log4j2.xml @@ -23,7 +23,7 @@ - + From 4a597ccff47fe031aa9a6326d1309be4a7ae836f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 4 Nov 2026 15:25:52 -0800 Subject: [PATCH 45/45] fix: compare the deadlines of the scheduled futures instead of sampling their remaining delays Sampling the remaining delay of each future separately could make a.compareTo(b) and b.compareTo(a) disagree for close deadlines, violating the Comparable contract. Co-Authored-By: Claude Sonnet 5.5 --- .../config/VirtualThreadScheduledExecutor.java | 5 +++++ .../VirtualThreadScheduledExecutorTest.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java index 61d7c74d66..3e3935e5ad 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutor.java @@ -301,6 +301,11 @@ public int compareTo(Delayed other) { if (other == this) { return 0; } + if (other instanceof Handle otherHandle) { + // the deadlines are compared as they are, sampling the remaining delay of each handle + // separately would make a.compareTo(b) and b.compareTo(a) disagree for close deadlines + return Long.compare(triggerTime - otherHandle.triggerTime, 0); + } return Long.compare(getDelay(TimeUnit.NANOSECONDS), other.getDelay(TimeUnit.NANOSECONDS)); } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java index 4fe9555987..a567f3e56d 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/VirtualThreadScheduledExecutorTest.java @@ -19,6 +19,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Delayed; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; @@ -147,6 +148,23 @@ void repeatsTasksAtAFixedRate() { .untilAsserted(() -> assertThat(runs).hasValueGreaterThan(3)); } + @Test + void scheduledFuturesAreOrderedConsistentlyByTheirDeadline() { + var first = (Delayed) executor.schedule(() -> {}, 10, TimeUnit.SECONDS); + var second = (Delayed) executor.schedule(() -> {}, 10, TimeUnit.SECONDS); + var later = (Delayed) executor.schedule(() -> {}, 20, TimeUnit.SECONDS); + + assertThat(first.compareTo(first)).isZero(); + assertThat(first.compareTo(later)).isNegative(); + assertThat(later.compareTo(first)).isPositive(); + assertThat(first.compareTo(second)).isLessThanOrEqualTo(0); + // comparing the futures must not depend on when it happens, i.e. on the remaining delays + for (int i = 0; i < 10_000; i++) { + assertThat(Integer.signum(first.compareTo(second))) + .isEqualTo(-Integer.signum(second.compareTo(first))); + } + } + @Test void aFixedRateTaskCatchesUpAfterAnOverrun() { var starts = new CopyOnWriteArrayList();