(ci) - fix cleanup to allow tests broken for 1.33 work

Author: camilamacedo86

test/e2e/v4/plugin_cluster_test.go

@@ -63,6 +63,9 @@ var _ = Describe("kubebuilder", func() {
 			By("clean up API objects created during the test")
 			_ = kbc.Make("undeploy")
 
+			By("clean up API objects created during the test")
+			_ = kbc.Make("uninstall")
+
 			By("removing controller image and working dir")
 			kbc.Destroy()
 		})
@@ -88,25 +91,19 @@ var _ = Describe("kubebuilder", func() {
 			GenerateV4WithoutMetrics(kbc)
 			Run(kbc, true, false, false, false, false)
 		})
-		// FIXME: This test is currently disabled because it requires to be fixed:
-		// https://github.com/kubernetes-sigs/kubebuilder/issues/4853
-		// It is not working for k8s 1.33
-		// It("should generate a runnable project with metrics protected by network policies", func() {
-		// 	 GenerateV4WithNetworkPoliciesWithoutWebhooks(kbc)
-		//	 Run(kbc, false, false, false, true, true)
-		// })
+		It("should generate a runnable project with metrics protected by network policies", func() {
+			GenerateV4WithNetworkPoliciesWithoutWebhooks(kbc)
+			Run(kbc, false, false, false, true, true)
+		})
 		It("should generate a runnable project with webhooks and metrics protected by network policies", func() {
 			GenerateV4WithNetworkPolicies(kbc)
 			Run(kbc, true, false, false, true, true)
 		})
-		// FIXME: This test is currently disabled because it requires to be fixed:
-		// https://github.com/kubernetes-sigs/kubebuilder/issues/4853
-		// It is not working for k8s 1.33
-		// It("should generate a runnable project with the manager running "+
-		//	 "as restricted and without webhooks", func() {
-		//	 GenerateV4WithoutWebhooks(kbc)
-		//	 Run(kbc, false, false, false, true, false)
-		// })
+		It("should generate a runnable project with the manager running "+
+			"as restricted and without webhooks", func() {
+			GenerateV4WithoutWebhooks(kbc)
+			Run(kbc, false, false, false, true, false)
+		})
 	})
 })
 
@@ -189,7 +186,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, isToUseHelmChart,
 	By("Checking if all flags are applied to the manager pod")
 	podOutput, err := kbc.Kubectl.Get(
 		true,
-		"pod", controllerPodName,
+		"pods", controllerPodName,
 		"-o", "jsonpath={.spec.containers[0].args}",
 	)
 	Expect(err).NotTo(HaveOccurred())
@@ -453,7 +450,6 @@ func getControllerName(kbc *utils.TestContext) string {
 		controllerPodName = podNames[0]
 		g.Expect(controllerPodName).Should(ContainSubstring("controller-manager"))
 
-		// Validate pod status
 		status, err := kbc.Kubectl.Get(
 			true,
 			"pods", controllerPodName, "-o", "jsonpath={.status.phase}")
@@ -464,9 +460,14 @@ func getControllerName(kbc *utils.TestContext) string {
 		return nil
 	}
 	defer func() {
-		out, err := kbc.Kubectl.CommandInNamespace("describe", "all")
-		Expect(err).NotTo(HaveOccurred())
-		_, _ = fmt.Fprintln(GinkgoWriter, out)
+		status, err := kbc.Kubectl.Get(
+			true,
+			"pods", controllerPodName, "-o", "jsonpath={.status.phase}")
+		if status != "Running" || err != nil {
+			out, err := kbc.Kubectl.CommandInNamespace("describe", "all")
+			Expect(err).NotTo(HaveOccurred())
+			_, _ = fmt.Fprintln(GinkgoWriter, out)
+		}
 	}()
 	Eventually(verifyControllerUp, 5*time.Minute, time.Second).Should(Succeed())
 	return controllerPodName
@@ -501,6 +502,17 @@ func getMetricsOutput(kbc *utils.TestContext) string {
 	)
 	Expect(err).NotTo(HaveOccurred(), "Controller-manager service should exist")
 
+	By("checking controller-manager logs to verify metrics server is up")
+	controllerPodName := getControllerName(kbc)
+	checkLogs := func(g Gomega) {
+		logs, curlLogsError := kbc.Kubectl.Logs(controllerPodName)
+		g.Expect(curlLogsError).NotTo(HaveOccurred(), "failed to get controller-manager logs")
+		g.Expect(logs).To(ContainSubstring("Serving metrics server"),
+			"controller logs should show that metrics server is up")
+	}
+	Eventually(checkLogs, 2*time.Minute, 5*time.Second).Should(Succeed(),
+		"controller logs should show metrics server startup within 2 minutes")
+
 	By("ensuring the service endpoint is ready")
 	checkServiceEndpoint := func(g Gomega) {
 		var output string
@@ -515,30 +527,62 @@ func getMetricsOutput(kbc *utils.TestContext) string {
 	Eventually(checkServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(),
 		"Service endpoint should be ready")
 
-	By("creating a curl pod to access the metrics endpoint")
-	cmdOpts := cmdOptsToCreateCurlPod(kbc, token)
-	_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
-	Expect(err).NotTo(HaveOccurred())
+	By("checking if curl pod to access the metrics endpoint exist")
+	podName := fmt.Sprintf("%s-curl", kbc.TestSuffix)
+	if _, err = kbc.Kubectl.Get(true, "pods", podName); err != nil && strings.Contains(err.Error(), "NotFound") {
+		By("creating a curl pod to access the metrics endpoint")
+		cmdOpts := cmdOptsToCreateCurlPod(kbc, podName, token)
+		_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
+		Expect(err).NotTo(HaveOccurred())
+	}
 
-	By("validating that the curl pod is running as expected")
+	By(fmt.Sprintf("validating that the pod %s is running as expected", podName))
 	verifyCurlUp := func(g Gomega) {
 		var status string
 		status, err = kbc.Kubectl.Get(
 			true,
-			"pods", "curl", "-o", "jsonpath={.status.phase}")
+			"pods", podName, "-o", "jsonpath={.status.phase}")
 		g.Expect(err).NotTo(HaveOccurred())
-		g.Expect(status).To(Equal("Succeeded"), fmt.Sprintf("curl pod in %s status", status))
+		g.Expect(status).To(Equal("Succeeded"), fmt.Sprintf("%s pod in %s status", podName, status))
 	}
+	defer func() {
+		status, errCurl := kbc.Kubectl.Get(
+			true,
+			"pods", podName, "-o", "jsonpath={.status.phase}")
+		if status != "Succeeded" || errCurl != nil {
+			out, errDescribe := kbc.Kubectl.CommandInNamespace("describe", "all")
+			Expect(errDescribe).NotTo(HaveOccurred())
+			_, _ = fmt.Fprintln(GinkgoWriter, out)
+
+			out, err = kbc.Kubectl.Get(true,
+				"pods", podName, "-o", "jsonpath={.status.phase}", "--namespace", kbc.Kubectl.Namespace)
+			Expect(err).NotTo(HaveOccurred())
+			_, _ = fmt.Fprintln(GinkgoWriter, out)
+		}
+	}()
 	Eventually(verifyCurlUp, 240*time.Second, time.Second).Should(Succeed())
 
+	By("validating that the correct ServiceAccount is being used")
+	saName := kbc.Kubectl.ServiceAccount
+	currentSAOutput, err := kbc.Kubectl.Get(
+		true,
+		"serviceaccount", saName,
+		"-o", "jsonpath={.metadata.name}",
+	)
+	Expect(err).NotTo(HaveOccurred(), "Failed to fetch the service account")
+	Expect(currentSAOutput).To(Equal(saName), "The ServiceAccount in use does not match the expected one")
+
 	By("validating that the metrics endpoint is serving as expected")
 	getCurlLogs := func(g Gomega) {
-		metricsOutput, err = kbc.Kubectl.Logs("curl")
+		metricsOutput, err = kbc.Kubectl.Logs("pod/" + podName)
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(metricsOutput).Should(ContainSubstring("< HTTP/1.1 200 OK"))
 	}
 	Eventually(getCurlLogs, 10*time.Second, time.Second).Should(Succeed())
+
+	By("By removing pod create for the metrics")
 	removeCurlPod(kbc)
+
 	return metricsOutput
 }
 
@@ -553,81 +597,99 @@ func metricsShouldBeUnavailable(kbc *utils.TestContext) {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(token).NotTo(BeEmpty())
 
-	By("creating a curl pod to access the metrics endpoint")
-	cmdOpts := cmdOptsToCreateCurlPod(kbc, token)
-	_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
-	Expect(err).NotTo(HaveOccurred())
+	By("checking if curl pod to access the metrics endpoint exist")
+	podName := fmt.Sprintf("%s-curl", kbc.TestSuffix)
+	if _, errCurl := kbc.Kubectl.Get(true, "pods", podName); errCurl != nil &&
+		strings.Contains(errCurl.Error(), "NotFound") {
+		By("creating a curl pod to access the metrics endpoint")
+		cmdOpts := cmdOptsToCreateCurlPod(kbc, podName, token)
+		_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
+		Expect(err).NotTo(HaveOccurred())
+	}
 
 	By("validating that the curl pod fail as expected")
 	verifyCurlUp := func(g Gomega) {
 		status, errCurl := kbc.Kubectl.Get(
 			true,
-			"pods", "curl", "-o", "jsonpath={.status.phase}")
+			"pods", podName, "-o", "jsonpath={.status.phase}")
 		g.Expect(errCurl).NotTo(HaveOccurred())
 		g.Expect(status).NotTo(Equal("Failed"),
 			fmt.Sprintf("curl pod in %s status when should fail with an error", status))
 	}
 	Eventually(verifyCurlUp, 240*time.Second, time.Second).Should(Succeed())
 
-	By("validating that the correct ServiceAccount is being used")
-	saName := kbc.Kubectl.ServiceAccount
-	currentSAOutput, err := kbc.Kubectl.Get(
-		true,
-		"serviceaccount", saName,
-		"-o", "jsonpath={.metadata.name}",
-	)
-	Expect(err).NotTo(HaveOccurred(), "Failed to fetch the service account")
-	Expect(currentSAOutput).To(Equal(saName), "The ServiceAccount in use does not match the expected one")
-
 	By("validating that the metrics endpoint is not working as expected")
 	getCurlLogs := func(g Gomega) {
-		metricsOutput, err := kbc.Kubectl.Logs("curl")
+		metricsOutput, err := kbc.Kubectl.Logs("pod/" + podName)
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(metricsOutput).Should(ContainSubstring("Could not resolve host"))
 	}
 	Eventually(getCurlLogs, 10*time.Second, time.Second).Should(Succeed())
+
+	By("By removing pod create for the metrics")
 	removeCurlPod(kbc)
 }
-
-func cmdOptsToCreateCurlPod(kbc *utils.TestContext, token string) []string {
-	//nolint:lll
-	cmdOpts := []string{
-		"run", "curl",
-		"--restart=Never",
-		"--namespace", kbc.Kubectl.Namespace,
-		"--image=curlimages/curl:latest",
-		"--overrides",
-		fmt.Sprintf(`{
-			"spec": {
-				"containers": [{
-					"name": "curl",
-					"image": "curlimages/curl:latest",
-					"command": ["/bin/sh", "-c"],
-					"args": ["curl -v -k -H 'Authorization: Bearer %s' https://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8443/metrics"],
-					"securityContext": {
-						"readOnlyRootFilesystem": true,
-						"allowPrivilegeEscalation": false,
-						"capabilities": {
-							"drop": ["ALL"]
-						},
-						"runAsNonRoot": true,
-						"runAsUser": 1000,
-						"seccompProfile": {
-							"type": "RuntimeDefault"
-						}
-					}
-				}],
-				"serviceAccountName": "%s"
-			}
-    }`, token, kbc.TestSuffix, kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount),
-	}
-	return cmdOpts
+func cmdOptsToCreateCurlPod(kbc *utils.TestContext, podName, token string) []string {
+	jobYAML := fmt.Sprintf(`{
+  "apiVersion": "batch/v1",
+  "kind": "Job",
+  "metadata": {
+    "name": "%s",
+    "namespace": "%s"
+  },
+  "spec": {
+    "template": {
+      "spec": {
+        "restartPolicy": "OnFailure",
+        "containers": [{
+          "name": "curl",
+          "image": "curlimages/curl:latest",
+          "command": ["/bin/sh", "-c"],
+          "args": ["curl -v -k -H 'Authorization: Bearer %s' https://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8443/metrics"],
+          "securityContext": {
+            "readOnlyRootFilesystem": true,
+            "allowPrivilegeEscalation": false,
+            "capabilities": {
+              "drop": ["ALL"]
+            },
+            "runAsNonRoot": true,
+            "runAsUser": 1000,
+            "seccompProfile": {
+              "type": "RuntimeDefault"
+            }
+          }
+        }],
+        "serviceAccountName": "%s"
+      }
+    }
+  }
+}`, podName, kbc.Kubectl.Namespace, token, kbc.TestSuffix, kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)
+
+	// Save the manifest into the Stdin of the Kubectl context
+	kbc.Kubectl.Stdin = strings.NewReader(jobYAML)
+
+	// Return the kubectl command options for `kubectl apply -f -`
+	return []string{"apply", "-f", "-"}
 }
 
 func removeCurlPod(kbc *utils.TestContext) {
-	By("cleaning up the curl pod")
-	_, err := kbc.Kubectl.Delete(true, "pods/curl")
-	Expect(err).NotTo(HaveOccurred())
+	podName := fmt.Sprintf("%s-curl", kbc.TestSuffix)
+
+	By(fmt.Sprintf("checking if pod %q exists before attempting cleanup", podName))
+	if _, err := kbc.Kubectl.Get(true, "pods", podName); err != nil {
+		By(fmt.Sprintf("pod %q does not exist, skipping cleanup", podName))
+		return
+	}
+
+	By("waiting for the pod to be deleted")
+	if _, err := kbc.Kubectl.Wait(true, "pod/"+podName, "--for=delete", "--timeout=40s"); err != nil {
+		By("force deleting the curl pod with 0s grace period")
+		_, err := kbc.Kubectl.Delete(true, "pods", podName, "--grace-period=0", "--force")
+		Expect(err).NotTo(HaveOccurred())
+
+		_, err = kbc.Kubectl.Wait(true, "pod/"+podName, "--for=delete", "--timeout=30s")
+		Expect(err).NotTo(HaveOccurred())
+	}
 }
 
 // serviceAccountToken provides a helper function that can provide you with a service account