diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index c19a4f1..84bcf27 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -21,6 +21,14 @@ jobs: builder: [pack, s2i] deployer: [knative, raw, keda] steps: + - name: Free up disk space + run: | + # Remove large packages to free up disk space on GitHub runners + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + # Clean up Docker to start fresh + docker system prune -af --volumes + df -h + - name: Clone the code uses: actions/checkout@v6 @@ -65,8 +73,21 @@ jobs: if: failure() run: | mkdir -p /tmp/k8s-artifacts - kubectl logs -n func-operator-system -l control-plane=controller-manager --tail=-1 --all-containers --prefix --timestamps > /tmp/k8s-artifacts/func-operator.log - kubectl get functions -A -o yaml > /tmp/functions.yaml + kubectl logs -n func-operator-system -l control-plane=controller-manager --tail=-1 --all-containers --prefix --timestamps > /tmp/k8s-artifacts/func-operator.log || true + for resource in functions deployments configmaps pipelineruns roles rolebindings; do + kubectl get ${resource} -A -o yaml > /tmp/k8s-artifacts/${resource}.yaml || true + done + + # Install Tekton CLI for better PipelineRun log collection + curl -LO https://github.com/tektoncd/cli/releases/download/v0.44.1/tkn_0.44.1_Linux_x86_64.tar.gz + tar xvzf tkn_0.44.1_Linux_x86_64.tar.gz -C /tmp + chmod +x /tmp/tkn + + # Collect logs from all pipelineruns using Tekton CLI + /tmp/tkn pipelinerun list -A > /tmp/k8s-artifacts/pipelinerun-list.txt 2>&1 || true + kubectl get pipelineruns -A -o json | jq -r '.items[] | "\(.metadata.namespace) \(.metadata.name)"' | while read ns name; do + /tmp/tkn pipelinerun logs $name -n $ns > /tmp/k8s-artifacts/pipelinerun-logs-${ns}-${name}.log 2>&1 || true + done - name: Upload Kubernetes artifacts if: failure() diff --git a/Dockerfile b/Dockerfile index 58fa242..975e0dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,9 @@ ARG FUNC_CLI_GH_REPO=knative/func ARG FUNC_CLI_BRANCH=main # workaround to invalidate cache when func cli repo got updated -ADD https://api.github.com/repos/${FUNC_CLI_GH_REPO}/git/refs/heads/${FUNC_CLI_BRANCH} version.json +# Use git ls-remote instead of GitHub API to avoid rate limiting (60 req/hour for unauthenticated) +# which caused merge queue failures due to multiple concurrent builds +RUN git ls-remote https://github.com/${FUNC_CLI_GH_REPO} refs/heads/${FUNC_CLI_BRANCH} > version.json WORKDIR /workspace RUN git clone --branch ${FUNC_CLI_BRANCH} --single-branch --depth 1 https://github.com/${FUNC_CLI_GH_REPO} . diff --git a/Makefile b/Makefile index 89aeeb2..5c4b0f4 100644 --- a/Makefile +++ b/Makefile @@ -119,12 +119,12 @@ test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out .PHONY: test-e2e ## Run e2e tests. -test-e2e: - go test -timeout 1h ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=1h -ginkgo.label-filter="!bundle" +test-e2e: ginkgo + $(GINKGO) -v --timeout=1h --label-filter="!bundle" --fail-fast -p ./test/e2e/ .PHONY: test-e2e-bundle ## Run bundle e2e tests. -test-e2e-bundle: operator-sdk docker-build docker-push bundle bundle-build bundle-push install-olm-in-cluster - OPERATOR_SDK=$(OPERATOR_SDK) BUNDLE_IMG=$(BUNDLE_IMG) go test -timeout 1h ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=1h -ginkgo.label-filter="bundle" +test-e2e-bundle: operator-sdk docker-build docker-push bundle bundle-build bundle-push install-olm-in-cluster ginkgo + OPERATOR_SDK=$(OPERATOR_SDK) BUNDLE_IMG=$(BUNDLE_IMG) $(GINKGO) -v --timeout=1h --label-filter="bundle" --fail-fast ./test/e2e/ .PHONY: install-olm-in-cluster install-olm-in-cluster: operator-sdk ## Install OLM in cluster if not already installed. @@ -277,10 +277,12 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint MOCKERY = $(LOCALBIN)/mockery +GINKGO = $(LOCALBIN)/ginkgo ## Tool Versions KUSTOMIZE_VERSION ?= v5.6.0 CONTROLLER_TOOLS_VERSION ?= v0.18.0 +GINKGO_VERSION ?= v2.28.1 #ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') #ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) @@ -321,6 +323,11 @@ mockery: ${MOCKERY} ## Download mockery locally if necessary. ${MOCKERY}: $(LOCALBIN) $(call go-install-tool,${MOCKERY},github.com/vektra/mockery/v3,${MOCKERY_VERSION}) +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download ginkgo locally if necessary. +$(GINKGO): $(LOCALBIN) + $(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo,$(GINKGO_VERSION)) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/test/e2e/bundle_test.go b/test/e2e/bundle_test.go index 651a262..bbf52f2 100644 --- a/test/e2e/bundle_test.go +++ b/test/e2e/bundle_test.go @@ -47,9 +47,6 @@ var _ = Describe("Bundle", Label("bundle"), Ordered, func() { testNamespaces []TestNamespace ) - SetDefaultEventuallyTimeout(5 * time.Minute) - SetDefaultEventuallyPollingInterval(time.Second) - BeforeAll(func() { bundleImage = os.Getenv("BUNDLE_IMG") Expect(bundleImage).ToNot(BeEmpty(), "BUNDLE_IMG must be given") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 31d786b..3e8bfcd 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -51,6 +52,11 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { ctx = context.Background() + // Set global timeout for Eventually assertions + // Must be set here (not in Describe blocks) to avoid race conditions in parallel execution + SetDefaultEventuallyTimeout(10 * time.Minute) + SetDefaultEventuallyPollingInterval(1 * time.Second) + // Register the Function API scheme err := functionsdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 94b2358..662d1d4 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -19,12 +19,9 @@ package e2e import ( "fmt" "os/exec" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/functions-dev/func-operator/test/utils" + . "github.com/onsi/ginkgo/v2" ) // namespace where the project is deployed in @@ -33,191 +30,6 @@ const namespace = "func-operator-system" // serviceAccountName created for the project const serviceAccountName = "func-operator-controller-manager" -// metricsServiceName is the name of the metrics service of the project -const metricsServiceName = "func-operator-controller-manager-metrics-service" - -// metricsPort is the port of the metrics service providing the managers metrics -const metricsPort = "8080" - -var _ = Describe("Manager", func() { - var controllerPodName string - - // After each test, check for failures and collect logs, events, - // and pod descriptions for debugging. - AfterEach(func() { - specReport := CurrentSpecReport() - if specReport.Failed() { - By("Fetching controller manager pod logs") - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - controllerLogs, err := utils.Run(cmd) - if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) - } - - By("Fetching Kubernetes events") - cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") - eventsOutput, err := utils.Run(cmd) - if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) - } - - By("Fetching curl-metrics logs") - cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) - metricsOutput, err := utils.Run(cmd) - if err == nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) - } - - By("Fetching controller manager pod description") - cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) - podDescription, err := utils.Run(cmd) - if err == nil { - fmt.Println("Pod description:\n", podDescription) - } else { - fmt.Println("Failed to describe controller pod") - } - } - }) - - SetDefaultEventuallyTimeout(2 * time.Minute) - SetDefaultEventuallyPollingInterval(time.Second) - - Context("Manager", func() { - It("should run successfully", func() { - By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func(g Gomega) { - // Get the name of the controller-manager pod - cmd := exec.Command("kubectl", "get", - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, - ) - - podOutput, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") - podNames := utils.GetNonEmptyLines(podOutput) - g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") - controllerPodName = podNames[0] - g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) - - // Validate the pod's status - cmd = exec.Command("kubectl", "get", - "pods", controllerPodName, "-o", "jsonpath={.status.phase}", - "-n", namespace, - ) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") - } - Eventually(verifyControllerUp).Should(Succeed()) - }) - - Context("with curl-metrics-pod", func() { - curlMetricPodName := "curl-metrics" - - AfterEach(func() { - cmd := exec.Command("kubectl", "delete", "pod", curlMetricPodName, "-n", namespace, "--ignore-not-found") - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) - }) - - It("should ensure the metrics endpoint is serving metrics", func() { - By("validating that the metrics service is available") - cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") - - By("waiting for the metrics endpoint to be ready") - verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring(metricsPort), "Metrics endpoint is not ready") - } - Eventually(verifyMetricsEndpointReady).Should(Succeed()) - - By("verifying that the controller manager is serving the metrics server") - verifyMetricsServerStarted := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), - "Metrics server not yet started") - } - Eventually(verifyMetricsServerStarted).Should(Succeed()) - - By("creating the curl-metrics pod to access the metrics endpoint") - cmd = exec.Command("kubectl", "run", curlMetricPodName, "--restart=Never", - "--namespace", namespace, - "--image=curlimages/curl:latest", - "--overrides", - fmt.Sprintf(`{ - "spec": { - "containers": [{ - "name": "curl", - "image": "curlimages/curl:latest", - "command": ["/bin/sh", "-c"], - "args": ["curl -v %s.%s.svc.cluster.local:%s/metrics"], - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"] - }, - "runAsNonRoot": true, - "runAsUser": 1000, - "seccompProfile": { - "type": "RuntimeDefault" - } - } - }], - "serviceAccount": "%s" - } - }`, metricsServiceName, namespace, metricsPort, serviceAccountName)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") - - By("waiting for the curl-metrics pod to complete.") - verifyCurlUp := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "pods", curlMetricPodName, - "-o", "jsonpath={.status.phase}", - "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") - } - Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) - - By("getting the metrics by checking curl-metrics logs") - metricsOutput := getMetricsOutput() - Expect(metricsOutput).To(ContainSubstring( - "controller_runtime_reconcile_total", - )) - }) - }) - - // +kubebuilder:scaffold:e2e-webhooks-checks - }) -}) - -// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. -func getMetricsOutput() string { - By("getting the curl-metrics logs") - cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) - metricsOutput, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") - Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) - return metricsOutput -} - // logFailedTestDetails logs function resource and controller logs on test failure func logFailedTestDetails(functionName, functionNamespace string) { specReport := CurrentSpecReport() diff --git a/test/e2e/func_deploy_test.go b/test/e2e/func_deploy_test.go index 95ab51d..8eafc14 100644 --- a/test/e2e/func_deploy_test.go +++ b/test/e2e/func_deploy_test.go @@ -126,9 +126,6 @@ func functionNotDeployed(functionName, functionNamespace string) func(g Gomega) var _ = Describe("Operator", func() { - SetDefaultEventuallyTimeout(2 * time.Minute) - SetDefaultEventuallyPollingInterval(time.Second) - Context("with a deployed function", func() { var repoURL string var repoDir string @@ -138,20 +135,20 @@ var _ = Describe("Operator", func() { // Create repository provider resources with automatic cleanup username, password, _, cleanup, err := repoProvider.CreateRandomUser() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) // Initialize repository with function code repoDir, err = utils.InitializeRepoWithFunction(repoURL, username, password, "go") Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, repoDir) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) functionNamespace, err = utils.GetTestNamespace() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanupNamespaces, functionNamespace) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) // Deploy function using func CLI out, err := utils.RunFuncDeploy(repoDir, utils.WithNamespace(functionNamespace)) @@ -159,7 +156,7 @@ var _ = Describe("Operator", func() { _, _ = fmt.Fprint(GinkgoWriter, out) // Cleanup func deployment - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace) }) @@ -198,8 +195,7 @@ var _ = Describe("Operator", func() { functionName = function.Name - // redeploy could take a bit longer therefore give a bit more time - Eventually(functionBecomesReady(functionName, functionNamespace), 6*time.Minute).Should(Succeed()) + Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) }) }) Context("with a function in a subdirectory in a monorepo", func() { @@ -212,11 +208,11 @@ var _ = Describe("Operator", func() { // Create repository provider resources with automatic cleanup username, password, _, cleanup, err := repoProvider.CreateRandomUser() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) // Initialize repository with function code repoDir, err = utils.InitializeRepoWithFunction( @@ -226,11 +222,11 @@ var _ = Describe("Operator", func() { "go", utils.WithSubDir(subPath)) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, repoDir) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) functionNamespace, err = utils.GetTestNamespace() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanupNamespaces, functionNamespace) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) functionDir := filepath.Join(repoDir, subPath) @@ -240,7 +236,7 @@ var _ = Describe("Operator", func() { _, _ = fmt.Fprint(GinkgoWriter, out) // Cleanup func deployment - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _, _ = utils.RunFunc("delete", "--path", functionDir, "--namespace", functionNamespace) }) @@ -280,8 +276,7 @@ var _ = Describe("Operator", func() { functionName = function.Name - // redeploy could take a bit longer therefore give a bit more time - Eventually(functionBecomesReady(functionName, functionNamespace), 6*time.Minute).Should(Succeed()) + Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) }) }) Context("with a not yet deployed function", func() { @@ -295,20 +290,20 @@ var _ = Describe("Operator", func() { // Create repository with function code but don't deploy username, password, _, cleanup, err := repoProvider.CreateRandomUser() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) // Initialize repository with function code repoDir, err = utils.InitializeRepoWithFunction(repoURL, username, password, "go") Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, repoDir) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) functionNamespace, err = utils.GetTestNamespace() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanupNamespaces, functionNamespace) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) }) AfterEach(func() { @@ -355,11 +350,11 @@ var _ = Describe("Operator", func() { username, password, _, cleanup, err = repoProvider.CreateRandomUser() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, true) // private repo Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) // Create access token for the user token, err = repoProvider.CreateAccessToken(username, password, "e2e-token") @@ -368,11 +363,11 @@ var _ = Describe("Operator", func() { // Initialize repository with function code repoDir, err = utils.InitializeRepoWithFunction(repoURL, username, password, "go") Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, repoDir) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) functionNamespace, err = utils.GetTestNamespace() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanupNamespaces, functionNamespace) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) // Deploy function using func CLI out, err := utils.RunFuncDeploy(repoDir, utils.WithNamespace(functionNamespace)) @@ -380,7 +375,7 @@ var _ = Describe("Operator", func() { _, _ = fmt.Fprint(GinkgoWriter, out) // Cleanup func deployment - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace) }) @@ -414,7 +409,7 @@ var _ = Describe("Operator", func() { } err := k8sClient.Create(ctx, secret) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -439,7 +434,7 @@ var _ = Describe("Operator", func() { functionName = function.Name - Eventually(functionBecomesReady(functionName, functionNamespace), 6*time.Minute).Should(Succeed()) + Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) }) It("should fail with authentication error when authSecretRef is not provided", func() { @@ -481,7 +476,7 @@ var _ = Describe("Operator", func() { } err := k8sClient.Create(ctx, secret) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _ = k8sClient.Delete(ctx, secret) }) @@ -506,7 +501,7 @@ var _ = Describe("Operator", func() { functionName = function.Name - Eventually(functionBecomesReady(functionName, functionNamespace), 6*time.Minute).Should(Succeed()) + Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) }) It("should fail with authentication error when authSecretRef is not provided", func() { diff --git a/test/e2e/func_middleware_update_test.go b/test/e2e/func_middleware_update_test.go index ce89e9d..bca973e 100644 --- a/test/e2e/func_middleware_update_test.go +++ b/test/e2e/func_middleware_update_test.go @@ -39,9 +39,6 @@ import ( var _ = Describe("Middleware Update", func() { - SetDefaultEventuallyTimeout(2 * time.Minute) - SetDefaultEventuallyPollingInterval(time.Second) - Context("with a function deployed using old func CLI", func() { var repoURL string @@ -59,15 +56,15 @@ var _ = Describe("Middleware Update", func() { // Create repository provider resources with automatic cleanup username, password, _, cleanup, err := repoProvider.CreateRandomUser() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) functionNamespace, err = utils.GetTestNamespace() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanupNamespaces, functionNamespace) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) // Initialize repository with function code using OLD func CLI version // v1.20.2 has no middleware-version label and uses instance-compatible templates @@ -79,7 +76,7 @@ var _ = Describe("Middleware Update", func() { "go", utils.WithCliVersion(oldFuncVersion)) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, repoDir) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) // Deploy function using the same OLD func CLI version out, err := utils.RunFuncDeploy(repoDir, @@ -89,7 +86,7 @@ var _ = Describe("Middleware Update", func() { _, _ = fmt.Fprint(GinkgoWriter, out) // Cleanup func deployment - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace) }) @@ -171,7 +168,7 @@ var _ = Describe("Middleware Update", func() { err = json.Unmarshal([]byte(skopeoOutput), &initialImageLabels) Expect(err).NotTo(HaveOccurred()) - initialMiddlewareVersion := initialImageLabels.Labels["middleware-version"] + initialMiddlewareVersion := initialImageLabels.Labels[funcfn.MiddlewareVersionLabelKey] _, _ = fmt.Fprintf(GinkgoWriter, "Initial middleware-version label: '%s' (expected empty for v1.20.2)\n", initialMiddlewareVersion) @@ -193,8 +190,7 @@ var _ = Describe("Middleware Update", func() { functionName = fn.Name - // Middleware update could take a bit longer therefore give more time - Eventually(functionBecomesReady(functionName, functionNamespace), 6*time.Minute).Should(Succeed()) + Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) // Verify middleware was actually updated by inspecting the new image out, err = utils.RunFunc("describe", deployedFunctionName, "-n", functionNamespace, "-o", "yaml") @@ -238,7 +234,7 @@ var _ = Describe("Middleware Update", func() { err = json.Unmarshal([]byte(skopeoOutput), &updatedImageLabels) Expect(err).NotTo(HaveOccurred()) - updatedMiddlewareVersion := updatedImageLabels.Labels["middleware-version"] + updatedMiddlewareVersion := updatedImageLabels.Labels[funcfn.MiddlewareVersionLabelKey] _, _ = fmt.Fprintf(GinkgoWriter, "Updated middleware-version label: '%s'\n", updatedMiddlewareVersion) // The operator should have set a middleware version @@ -248,7 +244,8 @@ var _ = Describe("Middleware Update", func() { }) }) - Context("when ConfigMap autoUpdateMiddleware setting changes", func() { + // this context should not run in parallel (--> Serial), as this would interfere other tests + Context("when ConfigMap autoUpdateMiddleware setting changes", Serial /* don't run in parallel */, func() { const ( operatorNamespace = "func-operator-system" controllerConfigName = "func-operator-controller-config" @@ -297,15 +294,15 @@ var _ = Describe("Middleware Update", func() { // Create repository provider resources with automatic cleanup username, password, _, cleanup, err := repoProvider.CreateRandomUser() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) _, repoURL, cleanup, err = repoProvider.CreateRandomRepo(username, false) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanup) + utils.DeferCleanupOnSuccess(cleanup) functionNamespace, err = utils.GetTestNamespace() Expect(err).NotTo(HaveOccurred()) - DeferCleanup(cleanupNamespaces, functionNamespace) + utils.DeferCleanupOnSuccess(cleanupNamespaces, functionNamespace) // Initialize repository with function code using OLD func CLI version // to ensure middleware will be outdated @@ -317,7 +314,7 @@ var _ = Describe("Middleware Update", func() { "go", utils.WithCliVersion(oldFuncVersion)) Expect(err).NotTo(HaveOccurred()) - DeferCleanup(os.RemoveAll, repoDir) + utils.DeferCleanupOnSuccess(os.RemoveAll, repoDir) // Deploy function using the same OLD func CLI version out, err := utils.RunFuncDeploy(repoDir, @@ -327,7 +324,7 @@ var _ = Describe("Middleware Update", func() { _, _ = fmt.Fprint(GinkgoWriter, out) // Cleanup func deployment - DeferCleanup(func() { + utils.DeferCleanupOnSuccess(func() { _, _ = utils.RunFunc("delete", "--path", repoDir, "--namespace", functionNamespace) }) @@ -456,7 +453,7 @@ var _ = Describe("Middleware Update", func() { } } g.Expect(false).To(BeTrue(), "MiddlewareUpToDate condition not found") - }, 5*time.Minute).Should(Succeed()) + }).Should(Succeed()) Eventually(functionBecomesReady(functionName, functionNamespace)).Should(Succeed()) }) diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go new file mode 100644 index 0000000..47e0f37 --- /dev/null +++ b/test/e2e/metrics_test.go @@ -0,0 +1,228 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/functions-dev/func-operator/test/utils" +) + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "func-operator-controller-manager-metrics-service" + +// metricsPort is the port of the metrics service providing the managers metrics +const metricsPort = "8080" + +var _ = Describe("Manager", func() { + var controllerPodName string + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + Context("Manager", func() { + // BeforeEach ensures controllerPodName is set before each test runs + BeforeEach(func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + // Validate the pod's status + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp, 2*time.Minute).Should(Succeed()) + }) + + It("should run successfully", func() { + // Controller pod validation happens in BeforeEach + Expect(controllerPodName).NotTo(BeEmpty()) + }) + + Context("with curl-metrics-pod", func() { + curlMetricPodName := "curl-metrics" + + It("should ensure the metrics endpoint is serving metrics", func() { + By("validating that the metrics service is available") + cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring(metricsPort), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady, 2*time.Minute).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted, 2*time.Minute).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", curlMetricPodName, "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -v %s.%s.svc.cluster.local:%s/metrics"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccount": "%s" + } + }`, metricsServiceName, namespace, metricsPort, serviceAccountName)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + // Ensure pod cleanup happens after test completes and debug logs are collected + // Using DeferCleanup ensures cleanup runs after AfterEach hooks (which collect debug logs on failure) + utils.DeferCleanupOnSuccess(func() { + cmd := exec.Command("kubectl", "delete", "pod", curlMetricPodName, "-n", namespace, "--ignore-not-found") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", curlMetricPodName, + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("waiting for curl-metrics logs to be available") + // Add a small delay to ensure logs are flushed after pod completion + verifyLogsAvailable := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", curlMetricPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "Logs should not be empty") + } + Eventually(verifyLogsAvailable, 30*time.Second).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + }) +}) + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} diff --git a/test/utils/func.go b/test/utils/func.go index d315848..640ef50 100644 --- a/test/utils/func.go +++ b/test/utils/func.go @@ -24,6 +24,9 @@ import ( "os/exec" "path/filepath" "runtime" + "time" + + ginkgo "github.com/onsi/ginkgo/v2" ) // RunFunc executes the func CLI with the current/latest version @@ -46,7 +49,7 @@ func RunFuncWithVersion(version string, command string, args ...string) (string, return Run(cmd) } -// RunFuncDeploy runs func deploy +// RunFuncDeploy runs func deploy with retry logic for transient network errors func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, error) { opts := &FuncDeployOptions{ // defaults @@ -78,11 +81,28 @@ func RunFuncDeploy(functionDir string, optFns ...FuncDeployOption) (string, erro args = append(args, "--deployer", opts.Deployer) } - if opts.CliVersion != "" { - return RunFuncWithVersion(opts.CliVersion, "deploy", args...) + var output string + var err error + + // Retry up to 3 times with 5s delay between attempts + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + time.Sleep(5 * time.Second) + _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "func deploy attempt %d failed: %v (retrying)\n", attempt, err) + } + + if opts.CliVersion != "" { + output, err = RunFuncWithVersion(opts.CliVersion, "deploy", args...) + } else { + output, err = RunFunc("deploy", args...) + } + + if err == nil { + return output, nil + } } - return RunFunc("deploy", args...) + return output, err } type FuncDeployOptions struct { diff --git a/test/utils/utils.go b/test/utils/utils.go index eb8de2c..3d27388 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "os/exec" + "reflect" "strings" . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck @@ -81,3 +82,16 @@ func GetTestNamespace() (string, error) { return name, nil } + +func DeferCleanupOnSuccess(args ...any) { + DeferCleanup(func() { + if !CurrentSpecReport().Failed() { + fn := reflect.ValueOf(args[0]) + in := make([]reflect.Value, len(args)-1) + for i, arg := range args[1:] { + in[i] = reflect.ValueOf(arg) + } + fn.Call(in) + } + }) +}