Ensure failed addon job is deleted before recreating

k8s/job.go

@@ -2,7 +2,6 @@ package k8s
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/sirupsen/logrus"
 	"k8s.io/api/batch/v1"
@@ -43,7 +42,10 @@ func ApplyK8sSystemJob(jobYaml, kubeConfigPath string, k8sWrapTransport WrapTran
 				return err
 			}
 		} else { // ignoring NotFound errors
-			time.Sleep(time.Second * 5)
+			//Jobs take longer to delete than to complete, 2 x the timeout
+			if err := retryToWithTimeout(ensureJobDeleted, k8sClient, job, timeout*2); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -72,6 +74,19 @@ func ensureJobCompleted(k8sClient *kubernetes.Clientset, j interface{}) error {
 	return fmt.Errorf("Failed to get job complete status: %v", err)
 }
 
+func ensureJobDeleted(k8sClient *kubernetes.Clientset, j interface{}) error {
+	job := j.(v1.Job)
+	_, err := k8sClient.BatchV1().Jobs(job.Namespace).Get(job.Name, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			// this is the "true" return of the function
+			return nil
+		}
+		return err
+	}
+	return fmt.Errorf("[k8s] Job [%s] is not deleted", job.Name)
+}
+
 func deleteK8sJob(k8sClient *kubernetes.Clientset, name, namespace string) error {
 	deletePolicy := metav1.DeletePropagationForeground
 	return k8sClient.BatchV1().Jobs(namespace).Delete(