Remove readiness probe otel-agent sidecars

e2e/nomostest/config_sync.go

@@ -83,10 +83,11 @@ var (
 	//
 	// All paths must be relative to the test file that is running. There is probably
 	// a more elegant way to do this.
-	baseDir            = filepath.FromSlash("../..")
-	outputManifestsDir = filepath.Join(baseDir, ".output", "staging", "oss")
-	configSyncManifest = filepath.Join(outputManifestsDir, "config-sync-manifest.yaml")
-	multiConfigMaps    = filepath.Join(baseDir, "e2e", "raw-nomos", configSyncManifests, multiConfigMapsName)
+	baseDir                  = filepath.FromSlash("../..")
+	outputManifestsDir       = filepath.Join(baseDir, ".output", "staging", "oss")
+	configSyncManifest       = filepath.Join(outputManifestsDir, "config-sync-manifest.yaml")
+	admissionWebhookManifest = filepath.Join(outputManifestsDir, "admission-webhook.yaml")
+	multiConfigMaps          = filepath.Join(baseDir, "e2e", "raw-nomos", configSyncManifests, multiConfigMapsName)
 )
 
 var (
@@ -245,6 +246,34 @@ func InstallConfigSync(nt *NT) error {
 	return nil
 }
 
+// InstallConfigSyncFromManifest installs ConfigSync on the test cluster by directly
+// applying the manifest file using kubectl client-side apply
+func InstallConfigSyncFromManifest(nt *NT) error {
+	nt.T.Log("[SETUP] Installing Config Sync directly from manifest file")
+
+	nt.T.Logf("Applying Config Sync manifest directly from %s", configSyncManifest)
+
+	out, err := nt.Shell.Kubectl("apply", "--server-side=false", "-f", configSyncManifest)
+	if err != nil {
+		return fmt.Errorf("failed to apply Config Sync manifest: %v\n%s", err, out)
+	}
+
+	nt.T.Logf("Applying multi-repo configmaps from %s", multiConfigMaps)
+	out, err = nt.Shell.Kubectl("apply", "--server-side=false", "-f", multiConfigMaps)
+	if err != nil {
+		return fmt.Errorf("failed to apply multi-repo configmaps: %v\n%s", err, out)
+	}
+
+	// Apply the admission webhook manifest
+	nt.T.Logf("Applying admission webhook manifest from %s", admissionWebhookManifest)
+	out, err = nt.Shell.Kubectl("apply", "--server-side=false", "-f", admissionWebhookManifest)
+	if err != nil {
+		return fmt.Errorf("failed to apply admission webhook manifest: %v\n%s", err, out)
+	}
+
+	return nil
+}
+
 // uninstallConfigSync uninstalls ConfigSync on the test cluster
 func uninstallConfigSync(nt *NT) error {
 	nt.T.Log("[CLEANUP] Uninstalling Config Sync")

e2e/testcases/cli_test.go

@@ -1297,13 +1297,14 @@ func TestApiResourceFormatting(t *testing.T) {
 }
 
 func TestNomosMigrate(t *testing.T) {
-	nt := nomostest.New(t, nomostesting.NomosCLI, ntopts.SkipConfigSyncInstall)
+	nt := nomostest.New(t, nomostesting.NomosCLI)
 
 	nt.T.Cleanup(func() {
 		// Restore state of Config Sync installation after test
-		if err := nomostest.InstallConfigSync(nt); err != nil {
+		if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil {
 			nt.T.Fatal(err)
 		}
+		nt.Must(nt.WatchForAllSyncs())
 	})
 	nt.T.Cleanup(func() {
 		cmObj := &unstructured.Unstructured{
@@ -1451,11 +1452,11 @@ func TestNomosMigrate(t *testing.T) {
 			configmanagement.RGControllerName, configmanagement.RGControllerNamespace)
 	})
 	tg.Go(func() error {
-		return nt.Watcher.WatchForNotFound(kinds.Deployment(),
+		return nt.Watcher.WatchForCurrentStatus(kinds.Deployment(),
 			core.RootReconcilerName(configsync.RootSyncName), configsync.ControllerNamespace)
 	})
 	tg.Go(func() error {
-		return nt.Watcher.WatchForNotFound(kinds.RootSyncV1Beta1(),
+		return nt.Watcher.WatchForCurrentStatus(kinds.RootSyncV1Beta1(),
 			configsync.RootSyncName, configsync.ControllerNamespace)
 	})
 	if err := tg.Wait(); err != nil {
@@ -1464,14 +1465,14 @@ func TestNomosMigrate(t *testing.T) {
 }
 
 func TestNomosMigrateMonoRepo(t *testing.T) {
-	nt := nomostest.New(t, nomostesting.NomosCLI, ntopts.SkipConfigSyncInstall)
+	nt := nomostest.New(t, nomostesting.NomosCLI)
 
 	nt.T.Cleanup(func() {
 		// Restore state of Config Sync installation after test.
-		// This also emulates upgrading to the current version after migrating
-		if err := nomostest.InstallConfigSync(nt); err != nil {
+		if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil {
 			nt.T.Fatal(err)
 		}
+		nt.Must(nt.WatchForAllSyncs())
 	})
 	nt.T.Cleanup(func() {
 		crds := []string{
@@ -1707,13 +1708,14 @@ func TestNomosMigrateMonoRepo(t *testing.T) {
 // This test case validates the behavior of the uninstall script defined
 // at installation/uninstall_configmanagement.sh
 func TestACMUninstallScript(t *testing.T) {
-	nt := nomostest.New(t, nomostesting.NomosCLI, ntopts.SkipConfigSyncInstall)
+	nt := nomostest.New(t, nomostesting.NomosCLI)
 
 	nt.T.Cleanup(func() {
 		// Restore state of Config Sync installation after test
-		if err := nomostest.InstallConfigSync(nt); err != nil {
+		if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil {
 			nt.T.Fatal(err)
 		}
+		nt.Must(nt.WatchForAllSyncs())
 	})
 	nt.T.Cleanup(func() {
 		cmObj := &unstructured.Unstructured{
@@ -1861,11 +1863,11 @@ func TestACMUninstallScript(t *testing.T) {
 			configmanagement.RGControllerName, configmanagement.RGControllerNamespace)
 	})
 	tg.Go(func() error {
-		return nt.Watcher.WatchForNotFound(kinds.Deployment(),
+		return nt.Watcher.WatchForCurrentStatus(kinds.Deployment(),
 			core.RootReconcilerName(configsync.RootSyncName), configsync.ControllerNamespace)
 	})
 	tg.Go(func() error {
-		return nt.Watcher.WatchForNotFound(kinds.RootSyncV1Beta1(),
+		return nt.Watcher.WatchForCurrentStatus(kinds.RootSyncV1Beta1(),
 			configsync.RootSyncName, configsync.ControllerNamespace)
 	})
 	if err := tg.Wait(); err != nil {

manifests/otel-agent-cm.yaml

@@ -39,11 +39,7 @@ data:
       batch:
       resourcedetection:
         detectors: [env, gcp]
-    extensions:
-      health_check:
-        endpoint: 0.0.0.0:13133
     service:
-      extensions: [health_check]
       pipelines:
         metrics:
           receivers: [opencensus]

manifests/otel-agent-reconciler-cm.yaml

@@ -59,11 +59,7 @@ data:
       # the GCE metadata service, if available.
       resourcedetection:
         detectors: [env, gcp]
-    extensions:
-      health_check:
-        endpoint: 0.0.0.0:13133
     service:
-      extensions: [health_check]
       pipelines:
         metrics:
           receivers: [opencensus]

manifests/templates/reconciler-manager-configmap.yaml

@@ -190,16 +190,9 @@ data:
              protocol: TCP
            - containerPort: 8888  # Metrics.
              protocol: TCP
-           - containerPort: 13133 # Health check
-             protocol: TCP
            volumeMounts:
            - name: otel-agent-config-reconciler-vol
              mountPath: /conf
-           readinessProbe:
-             httpGet:
-               path: /
-               port: 13133 # Health Check extension default port.
-               scheme: HTTP
            imagePullPolicy: IfNotPresent
            # These KUBE env vars help populate OTEL_RESOURCE_ATTRIBUTES which
            # is used by the otel-agent to populate resource attributes when

manifests/templates/reconciler-manager.yaml

@@ -81,7 +81,6 @@ spec:
         ports:
         - containerPort: 55678 # Default OpenCensus receiver port.
         - containerPort: 8888  # Metrics.
-        - containerPort: 13133 # Health check
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
@@ -91,10 +90,6 @@ spec:
         volumeMounts:
         - name: otel-agent-config-vol
           mountPath: /conf
-        readinessProbe:
-          httpGet:
-            path: /
-            port: 13133 # Health Check extension default port.
         # These KUBE env vars help populate OTEL_RESOURCE_ATTRIBUTES which
         # is used by the otel-agent to populate resource attributes when
         # emiting metrics to the otel-collector. This is more efficient than

manifests/templates/resourcegroup-manifest.yaml

@@ -167,11 +167,7 @@ data:
       # the GCE metadata service, if available.
       resourcedetection:
         detectors: [env, gcp]
-    extensions:
-      health_check:
-        endpoint: 0.0.0.0:13133
     service:
-      extensions: [health_check]
       pipelines:
         metrics:
           receivers: [opencensus]
@@ -274,11 +270,6 @@ spec:
         ports:
         - containerPort: 55678
         - containerPort: 8888
-        - containerPort: 13133
-        readinessProbe:
-          httpGet:
-            path: /
-            port: 13133
         resources:
           requests:
             cpu: 10m

test/kustomization/expected.yaml

@@ -5650,11 +5650,7 @@ data:
       batch:
       resourcedetection:
         detectors: [env, gcp]
-    extensions:
-      health_check:
-        endpoint: 0.0.0.0:13133
     service:
-      extensions: [health_check]
       pipelines:
         metrics:
           receivers: [opencensus]
@@ -5708,11 +5704,7 @@ data:
       # the GCE metadata service, if available.
       resourcedetection:
         detectors: [env, gcp]
-    extensions:
-      health_check:
-        endpoint: 0.0.0.0:13133
     service:
-      extensions: [health_check]
       pipelines:
         metrics:
           receivers: [opencensus]
@@ -5902,16 +5894,9 @@ data:
               protocol: TCP
             - containerPort: 8888  # Metrics.
               protocol: TCP
-            - containerPort: 13133 # Health check
-              protocol: TCP
             volumeMounts:
             - name: otel-agent-config-reconciler-vol
               mountPath: /conf
-            readinessProbe:
-              httpGet:
-                path: /
-                port: 13133 # Health Check extension default port.
-                scheme: HTTP
             imagePullPolicy: IfNotPresent
             # These KUBE env vars help populate OTEL_RESOURCE_ATTRIBUTES which
             # is used by the otel-agent to populate resource attributes when
@@ -6034,11 +6019,7 @@ data:
       # the GCE metadata service, if available.
       resourcedetection:
         detectors: [env, gcp]
-    extensions:
-      health_check:
-        endpoint: 0.0.0.0:13133
     service:
-      extensions: [health_check]
       pipelines:
         metrics:
           receivers: [opencensus]
@@ -6369,11 +6350,6 @@ spec:
         ports:
         - containerPort: 55678
         - containerPort: 8888
-        - containerPort: 13133
-        readinessProbe:
-          httpGet:
-            path: /
-            port: 13133
         resources:
           limits:
             cpu: 1
@@ -6493,11 +6469,6 @@ spec:
         ports:
         - containerPort: 55678
         - containerPort: 8888
-        - containerPort: 13133
-        readinessProbe:
-          httpGet:
-            path: /
-            port: 13133
         resources:
           requests:
             cpu: 10m