diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 5d4f6a12009..2a2cbe8e902 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -52,6 +52,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testConfigMapsDuringUpgrade(f, sem) testGuestbookApplicationDuringUpgrade(f, sem) testDaemonSetDuringUpgrade(f, sem) + testJobsDuringUpgrade(f, sem) }) cm.Do() }) @@ -72,6 +73,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testConfigMapsBeforeAndAfterUpgrade(f, sem) testGuestbookApplicationBeforeAndAfterUpgrade(f, sem) testDaemonSetBeforeAndAfterUpgrade(f, sem) + testJobsBeforeAndAfterUpgrade(f, sem) }) cm.Do() }) @@ -90,6 +92,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testConfigMapsDuringUpgrade(f, sem) testGuestbookApplicationDuringUpgrade(f, sem) testDaemonSetDuringUpgrade(f, sem) + testJobsDuringUpgrade(f, sem) }) cm.Do() }) @@ -112,6 +115,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testConfigMapsBeforeAndAfterUpgrade(f, sem) testGuestbookApplicationBeforeAndAfterUpgrade(f, sem) testDaemonSetBeforeAndAfterUpgrade(f, sem) + testJobsBeforeAndAfterUpgrade(f, sem) }) cm.Do() }) @@ -132,6 +136,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testConfigMapsDuringUpgrade(f, sem) testGuestbookApplicationDuringUpgrade(f, sem) testDaemonSetDuringUpgrade(f, sem) + testJobsDuringUpgrade(f, sem) }) cm.Do() }) @@ -404,3 +409,43 @@ func testDaemonSet(f *framework.Framework, sem *chaosmonkey.Semaphore, testDurin By("teardown daemonset") TestDaemonSetWithNodeAffinityTeardown(f, dsName) } + +func testJobsBeforeAndAfterUpgrade(f *framework.Framework, sem *chaosmonkey.Semaphore) { + testJobs(f, sem, false) +} + +func testJobsDuringUpgrade(f *framework.Framework, sem *chaosmonkey.Semaphore) { + testJobs(f, sem, true) +} + +func testJobs(f *framework.Framework, sem *chaosmonkey.Semaphore, testDuringDisruption bool) { + parallelism := int32(2) + completions := int32(4) + + // Setup + By("setup job") + job := TestJobsSetup("randomlySucceedOrFail", "rand-non-local", api.RestartPolicyNever, parallelism, completions) + // Validate + By("validate job before upgrade") + TestJobsValidate(f, job, completions) + + sem.Ready() + + if testDuringDisruption { + // Continuously validate + wait.Until(func() { + By("validate job during upgrade") + TestJobsValidate(f, job, completions) + }, framework.Poll, sem.StopCh) + } else { + // Block until chaosmonkey is done + By("waiting for upgrade to finish without validating job") + <-sem.StopCh + } + + // Validate after upgrade + By("validate job after upgrade") + TestJobsValidate(f, job, completions) + + // Teardown +} diff --git a/test/e2e/job.go b/test/e2e/job.go index 05bea4bd323..aa08071c028 100644 --- a/test/e2e/job.go +++ b/test/e2e/job.go @@ -79,20 +79,13 @@ var _ = framework.KubeDescribe("Job", func() { // Pods sometimes fail, but eventually succeed, after pod restarts It("should run a job to completion when tasks sometimes fail and are not locally restarted", func() { - By("Creating a job") // 50% chance of container success, local restarts. // Can't use the failOnce approach because that relies // on an emptyDir, which is not preserved across new pods. // Worst case analysis: 15 failures, each taking 1 minute to // run due to some slowness, 1 in 2^15 chance of happening, // causing test flake. Should be very rare. - job := newTestJob("randomlySucceedOrFail", "rand-non-local", api.RestartPolicyNever, parallelism, completions) - job, err := createJob(f.ClientSet, f.Namespace.Name, job) - Expect(err).NotTo(HaveOccurred()) - - By("Ensuring job reaches completions") - err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions) - Expect(err).NotTo(HaveOccurred()) + TestJobs(f, "randomlySucceedOrFail", "rand-non-local", api.RestartPolicyNever, parallelism, completions) }) It("should keep restarting failed pods", func() { @@ -339,3 +332,26 @@ func newBool(val bool) *bool { *p = val return p } + +func TestJobsSetup(behavior, name string, rPol api.RestartPolicy, parallelism, completions int32) *batch.Job { + return newTestJob(behavior, name, rPol, parallelism, completions) +} + +func TestJobsValidate(f *framework.Framework, job *batch.Job, completions int32) { + By("Creating a job") + job, err := createJob(f.ClientSet, f.Namespace.Name, job) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring job reaches completions") + err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions) + Expect(err).NotTo(HaveOccurred()) + + By("Delete the job") + err = deleteJob(f.ClientSet, f.Namespace.Name, job.ObjectMeta.Name) + Expect(err).NotTo(HaveOccurred()) +} + +func TestJobs(f *framework.Framework, behavior, name string, rPol api.RestartPolicy, parallelism, completions int32) { + job := TestJobsSetup(behavior, name, rPol, parallelism, completions) + TestJobsValidate(f, job, completions) +}