From e3840da83d269525f0156b37f6e4bfa2b1994426 Mon Sep 17 00:00:00 2001 From: shashidharatd Date: Mon, 31 Oct 2016 14:07:24 +0530 Subject: [PATCH] Add Daemonset to k8s upgrade tests --- test/e2e/cluster_upgrade.go | 47 +++++++++++ test/e2e/daemon_set.go | 153 +++++++++++++++++++++--------------- 2 files changed, 135 insertions(+), 65 deletions(-) diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index edef83a4a81..5d4f6a12009 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -51,6 +51,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testSecretsDuringUpgrade(f, sem) testConfigMapsDuringUpgrade(f, sem) testGuestbookApplicationDuringUpgrade(f, sem) + testDaemonSetDuringUpgrade(f, sem) }) cm.Do() }) @@ -70,6 +71,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testSecretsBeforeAndAfterUpgrade(f, sem) testConfigMapsBeforeAndAfterUpgrade(f, sem) testGuestbookApplicationBeforeAndAfterUpgrade(f, sem) + testDaemonSetBeforeAndAfterUpgrade(f, sem) }) cm.Do() }) @@ -87,6 +89,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testSecretsDuringUpgrade(f, sem) testConfigMapsDuringUpgrade(f, sem) testGuestbookApplicationDuringUpgrade(f, sem) + testDaemonSetDuringUpgrade(f, sem) }) cm.Do() }) @@ -108,6 +111,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testSecretsBeforeAndAfterUpgrade(f, sem) testConfigMapsBeforeAndAfterUpgrade(f, sem) testGuestbookApplicationBeforeAndAfterUpgrade(f, sem) + testDaemonSetBeforeAndAfterUpgrade(f, sem) }) cm.Do() }) @@ -127,6 +131,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { testSecretsDuringUpgrade(f, sem) testConfigMapsDuringUpgrade(f, sem) testGuestbookApplicationDuringUpgrade(f, sem) + testDaemonSetDuringUpgrade(f, sem) }) cm.Do() }) @@ -357,3 +362,45 @@ func testGuestbookApplication(f *framework.Framework, sem *chaosmonkey.Semaphore By("teardown guestbook app") GuestbookApplicationTeardown(f.ClientSet, f.Namespace.Name) } + +func testDaemonSetBeforeAndAfterUpgrade(f *framework.Framework, sem *chaosmonkey.Semaphore) { + testDaemonSet(f, sem, false) +} + +func testDaemonSetDuringUpgrade(f *framework.Framework, sem *chaosmonkey.Semaphore) { + testDaemonSet(f, sem, true) +} + +func testDaemonSet(f *framework.Framework, sem *chaosmonkey.Semaphore, testDuringDisruption bool) { + image := "gcr.io/google_containers/serve_hostname:v1.4" + dsName := "daemon-set" + // Setup + By("setup daemonset") + complexLabel, nodeSelector := TestDaemonSetWithNodeAffinitySetup(f, dsName, image) + + // Validate + By("validate daemonset before upgrade") + TestDaemonSetWithNodeAffinityValidate(f, dsName, complexLabel, nodeSelector) + + sem.Ready() + + if testDuringDisruption { + // Continuously validate + wait.Until(func() { + By("validate daemonset during upgrade") + TestDaemonSetWithNodeAffinityValidate(f, dsName, complexLabel, nodeSelector) + }, framework.Poll, sem.StopCh) + } else { + // Block until chaosmonkey is done + By("waiting for upgrade to finish without validating daemonset") + <-sem.StopCh + } + + // Validate after upgrade + By("validate daemonset after upgrade") + TestDaemonSetWithNodeAffinityValidate(f, dsName, complexLabel, nodeSelector) + + // Teardown + By("teardown daemonset") + TestDaemonSetWithNodeAffinityTeardown(f, dsName) +} diff --git a/test/e2e/daemon_set.go b/test/e2e/daemon_set.go index aa0f59aad32..69134def4ed 100644 --- a/test/e2e/daemon_set.go +++ b/test/e2e/daemon_set.go @@ -204,71 +204,7 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() { }) It("should run and stop complex daemon with node affinity", func() { - complexLabel := map[string]string{daemonsetNameLabel: dsName} - nodeSelector := map[string]string{daemonsetColorLabel: "blue"} - framework.Logf("Creating daemon with a node affinity %s", dsName) - affinity := map[string]string{ - api.AffinityAnnotationKey: fmt.Sprintf(` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "%s", - "operator": "In", - "values": ["%s"] - }] - }] - }}}`, daemonsetColorLabel, nodeSelector[daemonsetColorLabel]), - } - _, err := c.Extensions().DaemonSets(ns).Create(&extensions.DaemonSet{ - ObjectMeta: api.ObjectMeta{ - Name: dsName, - }, - Spec: extensions.DaemonSetSpec{ - Selector: &unversioned.LabelSelector{MatchLabels: complexLabel}, - Template: api.PodTemplateSpec{ - ObjectMeta: api.ObjectMeta{ - Labels: complexLabel, - Annotations: affinity, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: dsName, - Image: image, - Ports: []api.ContainerPort{{ContainerPort: 9376}}, - }, - }, - }, - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Initially, daemon pods should not be running on any nodes.") - err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, complexLabel)) - Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on no nodes") - - By("Change label of node, check that daemon pod is launched.") - nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - Expect(len(nodeList.Items)).To(BeNumerically(">", 0)) - newNode, err := setDaemonSetNodeLabels(c, nodeList.Items[0].Name, nodeSelector) - Expect(err).NotTo(HaveOccurred(), "error setting labels on node") - daemonSetLabels, _ := separateDaemonSetNodeLabels(newNode.Labels) - Expect(len(daemonSetLabels)).To(Equal(1)) - err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonPodOnNodes(f, complexLabel, []string{newNode.Name})) - Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on new nodes") - err = checkDaemonStatus(f, dsName) - Expect(err).NotTo(HaveOccurred()) - - By("remove the node selector and wait for daemons to be unscheduled") - _, err = setDaemonSetNodeLabels(c, nodeList.Items[0].Name, map[string]string{}) - Expect(err).NotTo(HaveOccurred(), "error removing labels on node") - Expect(wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, complexLabel))). - NotTo(HaveOccurred(), "error waiting for daemon pod to not be running on nodes") - - By("We should now be able to delete the daemon set.") - Expect(c.Extensions().DaemonSets(ns).Delete(dsName, nil)).NotTo(HaveOccurred()) - + testDaemonSetWithNodeAffinity(f, dsName, image) }) }) @@ -393,3 +329,90 @@ func checkDaemonStatus(f *framework.Framework, dsName string) error { } return nil } + +func TestDaemonSetWithNodeAffinitySetup(f *framework.Framework, dsName, image string) (map[string]string, map[string]string) { + ns := f.Namespace.Name + c := f.ClientSet + + err := clearDaemonSetNodeLabels(c) + Expect(err).NotTo(HaveOccurred()) + + complexLabel := map[string]string{daemonsetNameLabel: dsName} + nodeSelector := map[string]string{daemonsetColorLabel: "blue"} + framework.Logf("Creating daemon with a node affinity %s", dsName) + affinity := map[string]string{ + api.AffinityAnnotationKey: fmt.Sprintf(` + {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{ + "matchExpressions": [{ + "key": "%s", + "operator": "In", + "values": ["%s"] + }] + }] + }}}`, daemonsetColorLabel, nodeSelector[daemonsetColorLabel]), + } + _, err = c.Extensions().DaemonSets(ns).Create(&extensions.DaemonSet{ + ObjectMeta: api.ObjectMeta{ + Name: dsName, + }, + Spec: extensions.DaemonSetSpec{ + Selector: &unversioned.LabelSelector{MatchLabels: complexLabel}, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{ + Labels: complexLabel, + Annotations: affinity, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: dsName, + Image: image, + Ports: []api.ContainerPort{{ContainerPort: 9376}}, + }, + }, + }, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + return complexLabel, nodeSelector +} + +func TestDaemonSetWithNodeAffinityValidate(f *framework.Framework, dsName string, complexLabel, nodeSelector map[string]string) { + c := f.ClientSet + By("Initially, daemon pods should not be running on any nodes.") + err := wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, complexLabel)) + Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on no nodes") + + By("Change label of node, check that daemon pod is launched.") + nodeList := framework.GetReadySchedulableNodesOrDie(c) + Expect(len(nodeList.Items)).To(BeNumerically(">", 0)) + newNode, err := setDaemonSetNodeLabels(c, nodeList.Items[0].Name, nodeSelector) + Expect(err).NotTo(HaveOccurred(), "error setting labels on node") + daemonSetLabels, _ := separateDaemonSetNodeLabels(newNode.Labels) + Expect(len(daemonSetLabels)).To(Equal(1)) + err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonPodOnNodes(f, complexLabel, []string{newNode.Name})) + Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on new nodes") + err = checkDaemonStatus(f, dsName) + Expect(err).NotTo(HaveOccurred()) + + By("remove the node selector and wait for daemons to be unscheduled") + _, err = setDaemonSetNodeLabels(c, nodeList.Items[0].Name, map[string]string{}) + Expect(err).NotTo(HaveOccurred(), "error removing labels on node") + Expect(wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, complexLabel))). + NotTo(HaveOccurred(), "error waiting for daemon pod to not be running on nodes") +} + +func TestDaemonSetWithNodeAffinityTeardown(f *framework.Framework, dsName string) { + ns := f.Namespace.Name + c := f.ClientSet + By("We should now be able to delete the daemon set.") + Expect(c.Extensions().DaemonSets(ns).Delete(dsName, nil)).NotTo(HaveOccurred()) +} + +func testDaemonSetWithNodeAffinity(f *framework.Framework, dsName, image string) { + complexLabel, nodeSelector := TestDaemonSetWithNodeAffinitySetup(f, dsName, image) + TestDaemonSetWithNodeAffinityValidate(f, dsName, complexLabel, nodeSelector) + TestDaemonSetWithNodeAffinityTeardown(f, dsName) +}