test: Fix cleanup script for e2e test namespaces. (#703)

This updates the script that cleans up the e2e tests. This script was failing due to a bug in the
cleanup code, causing e2e tests to fail.

Author: hessjcg

tools/delete-test-namespaces.sh

@@ -19,22 +19,55 @@ export USE_GKE_E2E_AUTH_PLUGIN=True
 export KUBECONFIG=${KUBECONFIG:-bin/e2e-kubeconfig.yaml}
 
 mkdir -p bin/ns
+
+function remove_pod() {
+  ns="$1"
+  pod_name="$2"
+
+  # Gracefully delete in 10 seconds, then force delete.
+  if ! kubectl -n "$ns" delete pod --grace-period=10 "$pod_name" ; then
+    kubectl -n "$ns" delete pod --grace-period=0 --force "$pod_name"
+  fi
+}
+
 function remove_ns(){
   # Check that the namespace exists, return if not.
   if ! $KUBECTL get namespace "$1" ; then
     return
   fi
 
   # Tell kubernetes to delete the namespace, If it times out, force delete.
-  if ! $KUBECTL delete namespace "$1" --timeout=10s ; then
+  if ! $KUBECTL delete namespace "$1" --timeout=30s ; then
+
+    # Attempt to delete all the pods in the namespace
+    for pod_name in $(kubectl -n "$1" get pods -o json | jq -r .items[].metadata.name) ; do
+      remove_pod "$1" "$pod_name"
+    done
+
+    # Check if the namespace was deleted. If so, return
+    if ! $KUBECTL get namespace "$1" ; then
+      return
+    fi
 
     # Get the namespace, remove finalizers from the namespace spec.
+    # Force update the namespace resource, removing finalizers.
+    # This will allow Kubernetes to continue the deletion of the resource.
+
     $KUBECTL get namespace "$1" -o json | \
       jq '.spec.finalizers = []' > "bin/ns/$1.json"
 
-    # Force update the namespace resource, removing finalizers.
-    # This will allow Kubernetes to continue the deletion of the resource.
-    $KUBECTL replace --raw "/api/v1/namespaces/$1/finalize" -f "bin/ns/$1.json"
+    if ! $KUBECTL replace --raw "/api/v1/namespaces/$1/finalize" -f "bin/ns/$1.json" ; then
+      echo "Update finalizers failed. Will force delete"
+    fi
+    sleep 5
+
+    # Check if the namespace was deleted. If so, return
+    if ! $KUBECTL get namespace "$1" ; then
+      return
+    fi
+
+    # Attempt to delete the namespace again
+    $KUBECTL delete namespace "$1" --force || true # ignore failure
   fi
 
 }