From dc1cd4d4960a7be9eb054ea48a10fe20cfc63532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 22 Apr 2026 13:12:44 +0200 Subject: [PATCH 1/4] improve: log diagnostic for remote deployment junit test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../ClusterDeployedOperatorExtension.java | 136 +++++++++++++++++- 1 file changed, 133 insertions(+), 3 deletions(-) diff --git a/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java b/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java index a8513d9d69..32c65c3087 100644 --- a/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java +++ b/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java @@ -32,10 +32,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.kubernetes.api.model.ContainerStatus; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; +import io.fabric8.kubernetes.client.KubernetesClientTimeoutException; public class ClusterDeployedOperatorExtension extends AbstractOperatorExtension { @@ -139,9 +143,14 @@ protected void before(ExtensionContext context) { }); kubernetesClient.resourceList(operatorDeployment).inNamespace(namespace).createOrReplace(); - kubernetesClient - .resourceList(operatorDeployment) - .waitUntilReady(operatorDeploymentTimeout.toMillis(), TimeUnit.MILLISECONDS); + try { + kubernetesClient + .resourceList(operatorDeployment) + .waitUntilReady(operatorDeploymentTimeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (KubernetesClientTimeoutException e) { + logDiagnosticInfo(kubernetesClient); + throw e; + } LOGGER.debug("Operator resources deployed."); } @@ -153,6 +162,127 @@ protected void deleteOperator() { .delete(); } + private void logDiagnosticInfo(KubernetesClient kubernetesClient) { + LOGGER.error( + "Operator deployment timed out after {} seconds in namespace: {}", + operatorDeploymentTimeout.getSeconds(), + namespace); + try { + // Log deployment status + var deployments = + kubernetesClient.apps().deployments().inNamespace(namespace).list().getItems(); + for (Deployment deployment : deployments) { + var status = deployment.getStatus(); + LOGGER.error( + "Deployment '{}': replicas={}, readyReplicas={}, availableReplicas={}," + + " unavailableReplicas={}, conditions={}", + deployment.getMetadata().getName(), + status != null ? status.getReplicas() : "null", + status != null ? status.getReadyReplicas() : "null", + status != null ? status.getAvailableReplicas() : "null", + status != null ? status.getUnavailableReplicas() : "null", + status != null ? status.getConditions() : "null"); + } + + // Log pod status and container details + var pods = kubernetesClient.pods().inNamespace(namespace).list().getItems(); + for (Pod pod : pods) { + var podStatus = pod.getStatus(); + LOGGER.error( + "Pod '{}': phase={}, reason={}, message={}", + pod.getMetadata().getName(), + podStatus != null ? podStatus.getPhase() : "null", + podStatus != null ? podStatus.getReason() : "null", + podStatus != null ? podStatus.getMessage() : "null"); + + if (podStatus != null && podStatus.getContainerStatuses() != null) { + for (ContainerStatus cs : podStatus.getContainerStatuses()) { + LOGGER.error( + " Container '{}': ready={}, restartCount={}, state={}", + cs.getName(), + cs.getReady(), + cs.getRestartCount(), + cs.getState()); + } + } + if (podStatus != null && podStatus.getInitContainerStatuses() != null) { + for (ContainerStatus cs : podStatus.getInitContainerStatuses()) { + LOGGER.error( + " InitContainer '{}': ready={}, restartCount={}, state={}", + cs.getName(), + cs.getReady(), + cs.getRestartCount(), + cs.getState()); + } + } + + // Log pod events + var events = + kubernetesClient + .v1() + .events() + .inNamespace(namespace) + .withField("involvedObject.name", pod.getMetadata().getName()) + .list() + .getItems(); + for (var event : events) { + LOGGER.error( + " Event: type={}, reason={}, message={}", + event.getType(), + event.getReason(), + event.getMessage()); + } + + // Try to get container logs + try { + String logs = + kubernetesClient + .pods() + .inNamespace(namespace) + .withName(pod.getMetadata().getName()) + .tailingLines(50) + .getLog(); + if (logs != null && !logs.isEmpty()) { + LOGGER.error(" Logs for pod '{}':\n{}", pod.getMetadata().getName(), logs); + } + } catch (Exception logEx) { + LOGGER.error( + " Could not retrieve logs for pod '{}': {}", + pod.getMetadata().getName(), + logEx.getMessage()); + } + } + + if (pods.isEmpty()) { + LOGGER.error( + "No pods found in namespace '{}'. The deployment may have failed to" + + " create pods. Check if the image exists and is pullable.", + namespace); + + // Log deployment events when no pods exist + for (Deployment deployment : deployments) { + var events = + kubernetesClient + .v1() + .events() + .inNamespace(namespace) + .withField("involvedObject.name", deployment.getMetadata().getName()) + .list() + .getItems(); + for (var event : events) { + LOGGER.error( + " Deployment event: type={}, reason={}, message={}", + event.getType(), + event.getReason(), + event.getMessage()); + } + } + } + } catch (Exception diagEx) { + LOGGER.error("Failed to collect diagnostic info: {}", diagEx.getMessage(), diagEx); + } + } + public static class Builder extends AbstractBuilder { private final List operatorDeployment; private Duration deploymentTimeout; From 39c048b655460c9ec9f31ef4071f04489e79e6a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 23 Apr 2026 17:10:14 +0200 Subject: [PATCH 2/4] logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- sample-operators/tomcat-operator/src/test/resources/log4j2.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sample-operators/tomcat-operator/src/test/resources/log4j2.xml b/sample-operators/tomcat-operator/src/test/resources/log4j2.xml index 21b0ee5480..34792a9e37 100644 --- a/sample-operators/tomcat-operator/src/test/resources/log4j2.xml +++ b/sample-operators/tomcat-operator/src/test/resources/log4j2.xml @@ -23,6 +23,9 @@ + + + From ef752702d40ce262f4c17129be9005851e7aa453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Thu, 23 Apr 2026 17:36:18 +0200 Subject: [PATCH 3/4] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../ClusterDeployedOperatorExtension.java | 14 ++++++++++---- .../junit/LocallyRunOperatorExtension.java | 18 +++++++++++++++--- .../sample/metrics/MetricsHandlingE2E.java | 1 + .../sample/MySQLSchemaOperatorE2E.java | 1 + .../operator/sample/TomcatOperatorE2E.java | 1 + .../operator/sample/WebPageOperatorE2E.java | 1 + 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java b/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java index 32c65c3087..0eb33ad62a 100644 --- a/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java +++ b/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/ClusterDeployedOperatorExtension.java @@ -23,7 +23,6 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; -import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; @@ -103,10 +102,17 @@ protected void applyCrds(ExtensionContext context) { final var crdPath = "./target/classes/META-INF/fabric8/"; final var crdSuffix = "-v1.yml"; + final var crdDir = new File(crdPath); + final var crdFiles = crdDir.listFiles((ignored, name) -> name.endsWith(crdSuffix)); + if (crdFiles == null || crdFiles.length == 0) { + LOGGER.warn( + "No CRD files found with suffix '{}' in directory: {}", + crdSuffix, + crdDir.getAbsolutePath()); + } + final var kubernetesClient = getInfrastructureKubernetesClient(); - for (var crdFile : - Objects.requireNonNull( - new File(crdPath).listFiles((ignored, name) -> name.endsWith(crdSuffix)))) { + for (var crdFile : crdFiles != null ? crdFiles : new File[0]) { try (InputStream is = new FileInputStream(crdFile)) { final var crd = kubernetesClient.load(is); crd.createOrReplace(); diff --git a/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/LocallyRunOperatorExtension.java b/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/LocallyRunOperatorExtension.java index 8d2b2fc26d..15d4ad9659 100644 --- a/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/LocallyRunOperatorExtension.java +++ b/operator-framework-junit/src/main/java/io/javaoperatorsdk/operator/junit/LocallyRunOperatorExtension.java @@ -63,8 +63,9 @@ public class LocallyRunOperatorExtension extends AbstractOperatorExtension { private static final int CRD_DELETE_TIMEOUT = 5000; private static final int CRD_DELETE_WAIT_TIMEOUT = 60000; private static final Set appliedCRDs = new HashSet<>(); - private static final boolean deleteCRDs = + private static final boolean DELETE_CRDS_DEFAULT = Boolean.parseBoolean(System.getProperty("testsuite.deleteCRDs", "true")); + private static volatile boolean deleteCRDs = DELETE_CRDS_DEFAULT; private final Operator operator; private final List reconcilers; @@ -93,7 +94,8 @@ private LocallyRunOperatorExtension( Function namespaceNameSupplier, Function perClassNamespaceNameSupplier, List additionalCrds, - Consumer beforeStartHook) { + Consumer beforeStartHook, + Boolean deleteCRDsOverride) { super( infrastructure, infrastructureTimeout, @@ -119,6 +121,9 @@ private LocallyRunOperatorExtension( this.operator = new Operator(configurationServiceOverrider); this.registeredControllers = new HashMap<>(); crdMappings = getAdditionalCRDsFromFiles(additionalCrds, getKubernetesClient()); + if (deleteCRDsOverride != null) { + deleteCRDs = deleteCRDsOverride; + } } static Map getAdditionalCRDsFromFiles( @@ -496,6 +501,7 @@ public static class Builder extends AbstractBuilder { private final List additionalCustomResourceDefinitionInstances; private final List additionalCRDs = new ArrayList<>(); private Consumer beforeStartHook; + private Boolean deleteCRDs; private KubernetesClient kubernetesClient; private KubernetesClient infrastructureKubernetesClient; @@ -586,6 +592,11 @@ public Builder withBeforeStartHook(Consumer beforeS return this; } + public Builder withDeleteCRDs(boolean deleteCRDs) { + this.deleteCRDs = deleteCRDs; + return this; + } + public LocallyRunOperatorExtension build() { return new LocallyRunOperatorExtension( reconcilers, @@ -604,7 +615,8 @@ public LocallyRunOperatorExtension build() { namespaceNameSupplier, perClassNamespaceNameSupplier, additionalCRDs, - beforeStartHook); + beforeStartHook, + deleteCRDs); } } diff --git a/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java b/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java index 34e96a5870..1cd2f4d215 100644 --- a/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java +++ b/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java @@ -74,6 +74,7 @@ class MetricsHandlingE2E { ? LocallyRunOperatorExtension.builder() .withReconciler(new MetricsHandlingReconciler1()) .withReconciler(new MetricsHandlingReconciler2()) + .withDeleteCRDs(false) .withConfigurationService( c -> c.withMetrics(MetricsHandlingSampleOperator.initOTLPMetrics(true))) .build() diff --git a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java index 80d07333f6..622c693fd5 100644 --- a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java +++ b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java @@ -74,6 +74,7 @@ boolean isLocal() { .withReconciler(new MySQLSchemaReconciler()) // configuration for schema comes from // SchemaDependentResource annotation .withInfrastructure(infrastructure) + .withDeleteCRDs(false) .withPortForward(MY_SQL_NS, "app", "mysql", 3306, SchemaDependentResource.LOCAL_PORT) .build() : ClusterDeployedOperatorExtension.builder() diff --git a/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java b/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java index f889b242db..62475a816e 100644 --- a/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java +++ b/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java @@ -63,6 +63,7 @@ boolean isLocal() { .waitForNamespaceDeletion(false) .withReconciler(new TomcatReconciler()) .withReconciler(new WebappReconciler(client)) + .withDeleteCRDs(false) .build() : ClusterDeployedOperatorExtension.builder() .waitForNamespaceDeletion(false) diff --git a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java index 979f471de0..a9285f052b 100644 --- a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java +++ b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java @@ -43,6 +43,7 @@ public WebPageOperatorE2E() throws FileNotFoundException {} ? LocallyRunOperatorExtension.builder() .waitForNamespaceDeletion(false) .withReconciler(new WebPageReconciler()) + .withDeleteCRDs(false) .build() : ClusterDeployedOperatorExtension.builder() .waitForNamespaceDeletion(false) From 8ac6a30087271b3e12235c1dc17f43a250607e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 24 Apr 2026 14:01:24 +0200 Subject: [PATCH 4/4] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/sample/metrics/MetricsHandlingE2E.java | 1 - .../javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java | 1 - .../io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java | 1 - .../io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java | 1 - 4 files changed, 4 deletions(-) diff --git a/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java b/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java index 1cd2f4d215..34e96a5870 100644 --- a/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java +++ b/sample-operators/metrics-processing/src/test/java/io/javaoperatorsdk/operator/sample/metrics/MetricsHandlingE2E.java @@ -74,7 +74,6 @@ class MetricsHandlingE2E { ? LocallyRunOperatorExtension.builder() .withReconciler(new MetricsHandlingReconciler1()) .withReconciler(new MetricsHandlingReconciler2()) - .withDeleteCRDs(false) .withConfigurationService( c -> c.withMetrics(MetricsHandlingSampleOperator.initOTLPMetrics(true))) .build() diff --git a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java index 622c693fd5..80d07333f6 100644 --- a/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java +++ b/sample-operators/mysql-schema/src/test/java/io/javaoperatorsdk/operator/sample/MySQLSchemaOperatorE2E.java @@ -74,7 +74,6 @@ boolean isLocal() { .withReconciler(new MySQLSchemaReconciler()) // configuration for schema comes from // SchemaDependentResource annotation .withInfrastructure(infrastructure) - .withDeleteCRDs(false) .withPortForward(MY_SQL_NS, "app", "mysql", 3306, SchemaDependentResource.LOCAL_PORT) .build() : ClusterDeployedOperatorExtension.builder() diff --git a/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java b/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java index 62475a816e..f889b242db 100644 --- a/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java +++ b/sample-operators/tomcat-operator/src/test/java/io/javaoperatorsdk/operator/sample/TomcatOperatorE2E.java @@ -63,7 +63,6 @@ boolean isLocal() { .waitForNamespaceDeletion(false) .withReconciler(new TomcatReconciler()) .withReconciler(new WebappReconciler(client)) - .withDeleteCRDs(false) .build() : ClusterDeployedOperatorExtension.builder() .waitForNamespaceDeletion(false) diff --git a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java index a9285f052b..979f471de0 100644 --- a/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java +++ b/sample-operators/webpage/src/test/java/io/javaoperatorsdk/operator/sample/WebPageOperatorE2E.java @@ -43,7 +43,6 @@ public WebPageOperatorE2E() throws FileNotFoundException {} ? LocallyRunOperatorExtension.builder() .waitForNamespaceDeletion(false) .withReconciler(new WebPageReconciler()) - .withDeleteCRDs(false) .build() : ClusterDeployedOperatorExtension.builder() .waitForNamespaceDeletion(false)