diff --git a/test/e2e/apps/daemon_set.go b/test/e2e/apps/daemon_set.go index 1c06b29522b..00fd86cb620 100644 --- a/test/e2e/apps/daemon_set.go +++ b/test/e2e/apps/daemon_set.go @@ -52,6 +52,7 @@ import ( extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/pkg/controller/daemon" "k8s.io/kubernetes/test/e2e/framework" + e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2eresource "k8s.io/kubernetes/test/e2e/framework/resource" ) @@ -169,7 +170,7 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { ginkgo.By("Check that daemon pods launch on every node of the cluster.") err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, ds)) framework.ExpectNoError(err, "error waiting for daemon pod to start") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By("Stop a daemon pod, check that the daemon pod is revived.") @@ -207,9 +208,9 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { framework.ExpectNoError(err, "error setting labels on node") daemonSetLabels, _ := separateDaemonSetNodeLabels(newNode.Labels) framework.ExpectEqual(len(daemonSetLabels), 1) - err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkDaemonPodOnNodes(f, ds, []string{newNode.Name})) + err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, e2edaemonset.CheckDaemonPodOnNodes(f, ds, []string{newNode.Name})) framework.ExpectNoError(err, "error waiting for daemon pods to be running on new nodes") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By("Update the node label to green, and wait for daemons to be unscheduled") @@ -226,9 +227,9 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { framework.ExpectNoError(err, "error patching daemon set") daemonSetLabels, _ = separateDaemonSetNodeLabels(greenNode.Labels) framework.ExpectEqual(len(daemonSetLabels), 1) - err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkDaemonPodOnNodes(f, ds, []string{greenNode.Name})) + err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, e2edaemonset.CheckDaemonPodOnNodes(f, ds, []string{greenNode.Name})) framework.ExpectNoError(err, "error waiting for daemon pods to be running on new nodes") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) }) @@ -270,9 +271,9 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { framework.ExpectNoError(err, "error setting labels on node") daemonSetLabels, _ := separateDaemonSetNodeLabels(newNode.Labels) framework.ExpectEqual(len(daemonSetLabels), 1) - err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkDaemonPodOnNodes(f, ds, []string{newNode.Name})) + err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, e2edaemonset.CheckDaemonPodOnNodes(f, ds, []string{newNode.Name})) framework.ExpectNoError(err, "error waiting for daemon pods to be running on new nodes") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By("Remove the node label and wait for daemons to be unscheduled") @@ -297,7 +298,7 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { ginkgo.By("Check that daemon pods launch on every node of the cluster.") err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, ds)) framework.ExpectNoError(err, "error waiting for daemon pod to start") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By("Set a daemon pod's phase to 'Failed', check that the daemon pod is revived.") @@ -745,7 +746,7 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { } // Make sure every daemon pod on the node has been updated - nodeNames := schedulableNodes(c, ds) + nodeNames := e2edaemonset.SchedulableNodes(c, ds) for _, node := range nodeNames { switch { case @@ -831,7 +832,7 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { ginkgo.By("Check that daemon pods launch on every node of the cluster.") err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, testDaemonset)) framework.ExpectNoError(err, "error waiting for daemon pod to start") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By("listing all DeamonSets") @@ -879,7 +880,7 @@ var _ = SIGDescribe("Daemon set [Serial]", func() { ginkgo.By("Check that daemon pods launch on every node of the cluster.") err = wait.PollImmediate(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, testDaemonset)) framework.ExpectNoError(err, "error waiting for daemon pod to start") - err = checkDaemonStatus(f, dsName) + err = e2edaemonset.CheckDaemonStatus(f, dsName) framework.ExpectNoError(err) ginkgo.By("Getting /status") @@ -1008,58 +1009,13 @@ func getDaemonSetImagePatch(containerName, containerImage string) string { } func newDaemonSet(dsName, image string, label map[string]string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: dsName, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: label, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: label, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "app", - Image: image, - Ports: []v1.ContainerPort{{ContainerPort: 9376}}, - }, - }, - }, - }, - }, - } + ds := newDaemonSetWithLabel(dsName, image, label) + ds.ObjectMeta.Labels = nil + return ds } func newDaemonSetWithLabel(dsName, image string, label map[string]string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: dsName, - Labels: label, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: label, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: label, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "app", - Image: image, - Ports: []v1.ContainerPort{{ContainerPort: 9376}}, - }, - }, - }, - }, - }, - } + return e2edaemonset.NewDaemonSet(dsName, image, label, nil, nil, []v1.ContainerPort{{ContainerPort: 9376}}) } func listDaemonPods(c clientset.Interface, ns string, label map[string]string) *v1.PodList { @@ -1158,66 +1114,12 @@ func setDaemonSetNodeLabels(c clientset.Interface, nodeName string, labels map[s return newNode, nil } -func checkDaemonPodOnNodes(f *framework.Framework, ds *appsv1.DaemonSet, nodeNames []string) func() (bool, error) { - return func() (bool, error) { - podList, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - framework.Logf("could not get the pod list: %v", err) - return false, nil - } - pods := podList.Items - - nodesToPodCount := make(map[string]int) - for _, pod := range pods { - if !metav1.IsControlledBy(&pod, ds) { - continue - } - if pod.DeletionTimestamp != nil { - continue - } - if podutil.IsPodAvailable(&pod, ds.Spec.MinReadySeconds, metav1.Now()) { - nodesToPodCount[pod.Spec.NodeName]++ - } - } - framework.Logf("Number of nodes with available pods: %d", len(nodesToPodCount)) - - // Ensure that exactly 1 pod is running on all nodes in nodeNames. - for _, nodeName := range nodeNames { - if nodesToPodCount[nodeName] != 1 { - framework.Logf("Node %s is running more than one daemon pod", nodeName) - return false, nil - } - } - - framework.Logf("Number of running nodes: %d, number of available pods: %d", len(nodeNames), len(nodesToPodCount)) - // Ensure that sizes of the lists are the same. We've verified that every element of nodeNames is in - // nodesToPodCount, so verifying the lengths are equal ensures that there aren't pods running on any - // other nodes. - return len(nodesToPodCount) == len(nodeNames), nil - } -} - func checkRunningOnAllNodes(f *framework.Framework, ds *appsv1.DaemonSet) func() (bool, error) { return func() (bool, error) { - nodeNames := schedulableNodes(f.ClientSet, ds) - return checkDaemonPodOnNodes(f, ds, nodeNames)() + return e2edaemonset.CheckRunningOnAllNodes(f, ds) } } -func schedulableNodes(c clientset.Interface, ds *appsv1.DaemonSet) []string { - nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - framework.ExpectNoError(err) - nodeNames := make([]string, 0) - for _, node := range nodeList.Items { - if !canScheduleOnNode(node, ds) { - framework.Logf("DaemonSet pods can't tolerate node %s with taints %+v, skip checking this node", node.Name, node.Spec.Taints) - continue - } - nodeNames = append(nodeNames, node.Name) - } - return nodeNames -} - func checkAtLeastOneNewPod(c clientset.Interface, ns string, label map[string]string, newImage string) func() (bool, error) { return func() (bool, error) { pods := listDaemonPods(c, ns, label) @@ -1230,27 +1132,8 @@ func checkAtLeastOneNewPod(c clientset.Interface, ns string, label map[string]st } } -// canScheduleOnNode checks if a given DaemonSet can schedule pods on the given node -func canScheduleOnNode(node v1.Node, ds *appsv1.DaemonSet) bool { - newPod := daemon.NewPod(ds, node.Name) - fitsNodeName, fitsNodeAffinity, fitsTaints := daemon.Predicates(newPod, &node, node.Spec.Taints) - return fitsNodeName && fitsNodeAffinity && fitsTaints -} - func checkRunningOnNoNodes(f *framework.Framework, ds *appsv1.DaemonSet) func() (bool, error) { - return checkDaemonPodOnNodes(f, ds, make([]string, 0)) -} - -func checkDaemonStatus(f *framework.Framework, dsName string) error { - ds, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Get(context.TODO(), dsName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("could not get daemon set from v1") - } - desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady - if desired != scheduled && desired != ready { - return fmt.Errorf("error in daemon status. DesiredScheduled: %d, CurrentScheduled: %d, Ready: %d", desired, scheduled, ready) - } - return nil + return e2edaemonset.CheckDaemonPodOnNodes(f, ds, make([]string, 0)) } func checkDaemonPodsImageAndAvailability(c clientset.Interface, ds *appsv1.DaemonSet, image string, maxUnavailable int) func() (bool, error) { @@ -1286,7 +1169,7 @@ func checkDaemonPodsImageAndAvailability(c clientset.Interface, ds *appsv1.Daemo return false, fmt.Errorf("number of unavailable pods: %d is greater than maxUnavailable: %d", unavailablePods, maxUnavailable) } // Make sure every daemon pod on the node has been updated - nodeNames := schedulableNodes(c, ds) + nodeNames := e2edaemonset.SchedulableNodes(c, ds) for _, node := range nodeNames { if nodesToUpdatedPodCount[node] == 0 { return false, nil diff --git a/test/e2e/framework/daemonset/fixtures.go b/test/e2e/framework/daemonset/fixtures.go new file mode 100644 index 00000000000..1d765cf5991 --- /dev/null +++ b/test/e2e/framework/daemonset/fixtures.go @@ -0,0 +1,140 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package daemonset + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/controller/daemon" + "k8s.io/kubernetes/test/e2e/framework" +) + +func NewDaemonSet(dsName, image string, labels map[string]string, volumes []v1.Volume, mounts []v1.VolumeMount, ports []v1.ContainerPort, args ...string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsName, + Labels: labels, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "app", + Image: image, + Args: args, + Ports: ports, + VolumeMounts: mounts, + SecurityContext: &v1.SecurityContext{}, + }, + }, + SecurityContext: &v1.PodSecurityContext{}, + Volumes: volumes, + }, + }, + }, + } +} + +func CheckRunningOnAllNodes(f *framework.Framework, ds *appsv1.DaemonSet) (bool, error) { + nodeNames := SchedulableNodes(f.ClientSet, ds) + return CheckDaemonPodOnNodes(f, ds, nodeNames)() +} + +func SchedulableNodes(c clientset.Interface, ds *appsv1.DaemonSet) []string { + nodeList, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + framework.ExpectNoError(err) + nodeNames := make([]string, 0) + for _, node := range nodeList.Items { + if !canScheduleOnNode(node, ds) { + framework.Logf("DaemonSet pods can't tolerate node %s with taints %+v, skip checking this node", node.Name, node.Spec.Taints) + continue + } + nodeNames = append(nodeNames, node.Name) + } + return nodeNames +} + +// canScheduleOnNode checks if a given DaemonSet can schedule pods on the given node +func canScheduleOnNode(node v1.Node, ds *appsv1.DaemonSet) bool { + newPod := daemon.NewPod(ds, node.Name) + fitsNodeName, fitsNodeAffinity, fitsTaints := daemon.Predicates(newPod, &node, node.Spec.Taints) + return fitsNodeName && fitsNodeAffinity && fitsTaints +} + +func CheckDaemonPodOnNodes(f *framework.Framework, ds *appsv1.DaemonSet, nodeNames []string) func() (bool, error) { + return func() (bool, error) { + podList, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + framework.Logf("could not get the pod list: %v", err) + return false, nil + } + pods := podList.Items + + nodesToPodCount := make(map[string]int) + for _, pod := range pods { + if !metav1.IsControlledBy(&pod, ds) { + continue + } + if pod.DeletionTimestamp != nil { + continue + } + if podutil.IsPodAvailable(&pod, ds.Spec.MinReadySeconds, metav1.Now()) { + nodesToPodCount[pod.Spec.NodeName]++ + } + } + framework.Logf("Number of nodes with available pods: %d", len(nodesToPodCount)) + + // Ensure that exactly 1 pod is running on all nodes in nodeNames. + for _, nodeName := range nodeNames { + if nodesToPodCount[nodeName] != 1 { + framework.Logf("Node %s is running %d daemon pod, expected 1", nodeName, nodesToPodCount[nodeName]) + return false, nil + } + } + + framework.Logf("Number of running nodes: %d, number of available pods: %d", len(nodeNames), len(nodesToPodCount)) + // Ensure that sizes of the lists are the same. We've verified that every element of nodeNames is in + // nodesToPodCount, so verifying the lengths are equal ensures that there aren't pods running on any + // other nodes. + return len(nodesToPodCount) == len(nodeNames), nil + } +} + +func CheckDaemonStatus(f *framework.Framework, dsName string) error { + ds, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Get(context.TODO(), dsName, metav1.GetOptions{}) + if err != nil { + return err + } + desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady + if desired != scheduled && desired != ready { + return fmt.Errorf("error in daemon status. DesiredScheduled: %d, CurrentScheduled: %d, Ready: %d", desired, scheduled, ready) + } + return nil +} diff --git a/test/e2e/network/networking_perf.go b/test/e2e/network/networking_perf.go index 5f0888607ce..d66de56871c 100644 --- a/test/e2e/network/networking_perf.go +++ b/test/e2e/network/networking_perf.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" e2enode "k8s.io/kubernetes/test/e2e/framework/node" "k8s.io/kubernetes/test/e2e/network/common" @@ -133,34 +134,8 @@ func iperf2ServerService(client clientset.Interface, namespace string) (*v1.Serv func iperf2ClientDaemonSet(client clientset.Interface, namespace string) (*appsv1.DaemonSet, error) { one := int64(1) labels := map[string]string{labelKey: clientLabelValue} - spec := &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "iperf2-clients", - Labels: labels, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "iperf2-client", - Image: imageutils.GetE2EImage(imageutils.Agnhost), - Command: []string{"/agnhost"}, - Args: []string{"pause"}, - }, - }, - TerminationGracePeriodSeconds: &one, - }, - }, - }, - Status: appsv1.DaemonSetStatus{}, - } + spec := e2edaemonset.NewDaemonSet("iperf2-clients", imageutils.GetE2EImage(imageutils.Agnhost), labels, nil, nil, nil) + spec.Spec.Template.Spec.TerminationGracePeriodSeconds = &one ds, err := client.AppsV1().DaemonSets(namespace).Create(context.TODO(), spec, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/upgrades/apps/daemonsets.go b/test/e2e/upgrades/apps/daemonsets.go index 063ad4cd28f..a74af4a9713 100644 --- a/test/e2e/upgrades/apps/daemonsets.go +++ b/test/e2e/upgrades/apps/daemonsets.go @@ -23,10 +23,9 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/test/e2e/framework" + e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" "k8s.io/kubernetes/test/e2e/upgrades" ) @@ -47,35 +46,9 @@ func (t *DaemonSetUpgradeTest) Setup(f *framework.Framework) { ns := f.Namespace - t.daemonSet = &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: daemonSetName, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labelSet, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labelSet, - }, - Spec: v1.PodSpec{ - Tolerations: []v1.Toleration{ - {Operator: v1.TolerationOpExists}, - }, - Containers: []v1.Container{ - { - Name: daemonSetName, - Image: image, - Args: []string{"serve-hostname"}, - Ports: []v1.ContainerPort{{ContainerPort: 9376}}, - SecurityContext: &v1.SecurityContext{}, - }, - }, - }, - }, - }, + t.daemonSet = e2edaemonset.NewDaemonSet(daemonSetName, image, labelSet, nil, nil, []v1.ContainerPort{{ContainerPort: 9376}}, "serve-hostname") + t.daemonSet.Spec.Template.Spec.Tolerations = []v1.Toleration{ + {Operator: v1.TolerationOpExists}, } ginkgo.By("Creating a DaemonSet") @@ -86,7 +59,7 @@ func (t *DaemonSetUpgradeTest) Setup(f *framework.Framework) { ginkgo.By("Waiting for DaemonSet pods to become ready") err = wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) { - return checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels) + return e2edaemonset.CheckRunningOnAllNodes(f, t.daemonSet) }) framework.ExpectNoError(err) @@ -111,7 +84,7 @@ func (t *DaemonSetUpgradeTest) Teardown(f *framework.Framework) { func (t *DaemonSetUpgradeTest) validateRunningDaemonSet(f *framework.Framework) { ginkgo.By("confirming the DaemonSet pods are running on all expected nodes") - res, err := checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels) + res, err := e2edaemonset.CheckRunningOnAllNodes(f, t.daemonSet) framework.ExpectNoError(err) if !res { framework.Failf("expected DaemonSet pod to be running on all nodes, it was not") @@ -119,72 +92,6 @@ func (t *DaemonSetUpgradeTest) validateRunningDaemonSet(f *framework.Framework) // DaemonSet resource itself should be good ginkgo.By("confirming the DaemonSet resource is in a good state") - res, err = checkDaemonStatus(f, t.daemonSet.Namespace, t.daemonSet.Name) + err = e2edaemonset.CheckDaemonStatus(f, t.daemonSet.Name) framework.ExpectNoError(err) - if !res { - framework.Failf("expected DaemonSet to be in a good state, it was not") - } -} - -func checkRunningOnAllNodes(f *framework.Framework, namespace string, selector map[string]string) (bool, error) { - nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return false, err - } - - nodeNames := make([]string, 0) - for _, node := range nodeList.Items { - if len(node.Spec.Taints) != 0 { - framework.Logf("Ignore taints %v on Node %v for DaemonSet Pod.", node.Spec.Taints, node.Name) - } - // DaemonSet Pods are expected to run on all the nodes in e2e. - nodeNames = append(nodeNames, node.Name) - } - - return checkDaemonPodOnNodes(f, namespace, selector, nodeNames) -} - -func checkDaemonPodOnNodes(f *framework.Framework, namespace string, labelSet map[string]string, nodeNames []string) (bool, error) { - selector := labels.Set(labelSet).AsSelector() - options := metav1.ListOptions{LabelSelector: selector.String()} - podList, err := f.ClientSet.CoreV1().Pods(namespace).List(context.TODO(), options) - if err != nil { - return false, err - } - pods := podList.Items - - nodesToPodCount := make(map[string]int) - for _, pod := range pods { - if controller.IsPodActive(&pod) { - framework.Logf("Pod name: %v\t Node Name: %v", pod.Name, pod.Spec.NodeName) - nodesToPodCount[pod.Spec.NodeName]++ - } - } - framework.Logf("nodesToPodCount: %v", nodesToPodCount) - - // Ensure that exactly 1 pod is running on all nodes in nodeNames. - for _, nodeName := range nodeNames { - if nodesToPodCount[nodeName] != 1 { - return false, nil - } - } - - // Ensure that sizes of the lists are the same. We've verified that every element of nodeNames is in - // nodesToPodCount, so verifying the lengths are equal ensures that there aren't pods running on any - // other nodes. - return len(nodesToPodCount) == len(nodeNames), nil -} - -func checkDaemonStatus(f *framework.Framework, namespace string, dsName string) (bool, error) { - ds, err := f.ClientSet.AppsV1().DaemonSets(namespace).Get(context.TODO(), dsName, metav1.GetOptions{}) - if err != nil { - return false, err - } - - desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady - if desired != scheduled && desired != ready { - return false, nil - } - - return true, nil } diff --git a/test/e2e/upgrades/network/kube_proxy_migration.go b/test/e2e/upgrades/network/kube_proxy_migration.go index 85fa4fb22af..05fe1f4fb6b 100644 --- a/test/e2e/upgrades/network/kube_proxy_migration.go +++ b/test/e2e/upgrades/network/kube_proxy_migration.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2enode "k8s.io/kubernetes/test/e2e/framework/node" "k8s.io/kubernetes/test/e2e/upgrades" @@ -68,7 +69,7 @@ func (t *KubeProxyUpgradeTest) Test(f *framework.Framework, done <-chan struct{} framework.ExpectNoError(err) ginkgo.By("Waiting for kube-proxy DaemonSet running and ready") - err = waitForKubeProxyDaemonSetRunning(c) + err = waitForKubeProxyDaemonSetRunning(f, c) framework.ExpectNoError(err) } @@ -86,7 +87,7 @@ func (KubeProxyDowngradeTest) Name() string { return "[sig-network] kube-proxy-d // Setup verifies kube-proxy DaemonSet is running before upgrade. func (t *KubeProxyDowngradeTest) Setup(f *framework.Framework) { ginkgo.By("Waiting for kube-proxy DaemonSet running and ready") - err := waitForKubeProxyDaemonSetRunning(f.ClientSet) + err := waitForKubeProxyDaemonSetRunning(f, f.ClientSet) framework.ExpectNoError(err) } @@ -170,7 +171,7 @@ func waitForKubeProxyStaticPodsDisappear(c clientset.Interface) error { return nil } -func waitForKubeProxyDaemonSetRunning(c clientset.Interface) error { +func waitForKubeProxyDaemonSetRunning(f *framework.Framework, c clientset.Interface) error { framework.Logf("Waiting up to %v for kube-proxy DaemonSet running", defaultTestTimeout) condition := func() (bool, error) { @@ -185,19 +186,7 @@ func waitForKubeProxyDaemonSetRunning(c clientset.Interface) error { return false, nil } - nodes, err := e2enode.GetReadySchedulableNodes(c) - if err != nil { - framework.Logf("Failed to get nodes: %v", err) - return false, nil - } - - numberSchedulableNodes := len(nodes.Items) - numberkubeProxyPods := int(daemonSets.Items[0].Status.NumberAvailable) - if numberkubeProxyPods != numberSchedulableNodes { - framework.Logf("Expect %v kube-proxy DaemonSet pods running, got %v", numberSchedulableNodes, numberkubeProxyPods) - return false, nil - } - return true, nil + return e2edaemonset.CheckRunningOnAllNodes(f, &daemonSets.Items[0]) } if err := wait.PollImmediate(5*time.Second, defaultTestTimeout, condition); err != nil { diff --git a/test/e2e/windows/device_plugin.go b/test/e2e/windows/device_plugin.go index c8dd1a285da..76390530608 100644 --- a/test/e2e/windows/device_plugin.go +++ b/test/e2e/windows/device_plugin.go @@ -20,12 +20,12 @@ import ( "context" "time" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/daemonset" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" imageutils "k8s.io/kubernetes/test/utils/image" @@ -57,60 +57,37 @@ var _ = SIGDescribe("[Feature:GPUDevicePlugin] Device Plugin", func() { labels := map[string]string{ daemonsetNameLabel: dsName, } - ds := &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: dsName, - Namespace: "kube-system", + volumes := []v1.Volume{ + { + Name: mountName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: mountPath, + }, + }, }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: v1.PodSpec{ - PriorityClassName: "system-node-critical", - Tolerations: []v1.Toleration{ - { - Key: "CriticalAddonsOnly", - Operator: "Exists", - }, - }, - Containers: []v1.Container{ - { - Name: "hostdev", - Image: image, - VolumeMounts: []v1.VolumeMount{ - { - Name: mountName, - MountPath: mountPath, - }, - }, - Env: []v1.EnvVar{ - { - Name: "DIRECTX_GPU_MATCH_NAME", - Value: " ", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: mountName, - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: mountPath, - }, - }, - }, - }, - NodeSelector: map[string]string{ - "kubernetes.io/os": "windows", - }, - }, - }, + } + mounts := []v1.VolumeMount{ + { + Name: mountName, + MountPath: mountPath, + }, + } + ds := daemonset.NewDaemonSet(dsName, image, labels, volumes, mounts, nil) + ds.Spec.Template.Spec.PriorityClassName = "system-node-critical" + ds.Spec.Template.Spec.Tolerations = []v1.Toleration{ + { + Key: "CriticalAddonsOnly", + Operator: "Exists", + }, + } + ds.Spec.Template.Spec.NodeSelector = map[string]string{ + "kubernetes.io/os": "windows", + } + ds.Spec.Template.Spec.Containers[0].Env = []v1.EnvVar{ + { + Name: "DIRECTX_GPU_MATCH_NAME", + Value: " ", }, }