From 0b37152f1745ca8eb85eed222c4cfde9f7031bb7 Mon Sep 17 00:00:00 2001 From: Jiatong Wang Date: Sun, 7 Jul 2019 23:45:10 -0700 Subject: [PATCH] Refactor and Move node related methods to framework/node package Signed-off-by: Jiatong Wang --- test/e2e/apps/daemon_restart.go | 3 +- .../autoscaling/cluster_size_autoscaling.go | 5 +- test/e2e/cloud/nodes.go | 5 +- test/e2e/framework/node/BUILD | 3 + test/e2e/framework/node/resource.go | 152 +++++++++++++++++- test/e2e/framework/node/wait.go | 11 +- test/e2e/framework/service_util.go | 50 ------ test/e2e/framework/util.go | 30 +--- test/e2e/network/service.go | 8 +- test/e2e/scalability/BUILD | 1 + test/e2e/scalability/density.go | 7 +- .../equivalence_cache_predicates.go | 4 +- test/e2e/scheduling/predicates.go | 9 +- test/e2e/scheduling/preemption.go | 7 +- test/e2e/scheduling/priorities.go | 8 +- test/e2e/storage/vsphere/BUILD | 1 + .../vsphere/vsphere_volume_vsan_policy.go | 4 +- test/e2e/windows/BUILD | 1 + test/e2e/windows/service.go | 4 +- 19 files changed, 211 insertions(+), 102 deletions(-) diff --git a/test/e2e/apps/daemon_restart.go b/test/e2e/apps/daemon_restart.go index 2c6c1657635..7d0cf8a36b2 100644 --- a/test/e2e/apps/daemon_restart.go +++ b/test/e2e/apps/daemon_restart.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -298,7 +299,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() { ginkgo.It("Kubelet should not restart containers across restart", func() { - nodeIPs, err := framework.GetNodePublicIps(f.ClientSet) + nodeIPs, err := e2enode.GetPublicIps(f.ClientSet) framework.ExpectNoError(err) preRestarts, badNodes := getContainerRestarts(f.ClientSet, ns, labelSelector) if preRestarts != 0 { diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go index 5c25f68bb0e..0850aa55eb0 100644 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ b/test/e2e/autoscaling/cluster_size_autoscaling.go @@ -28,7 +28,7 @@ import ( "strings" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -1226,7 +1226,8 @@ func deleteNodePool(name string) { func getPoolNodes(f *framework.Framework, poolName string) []*v1.Node { nodes := make([]*v1.Node, 0, 1) - nodeList := framework.GetReadyNodesIncludingTaintedOrDie(f.ClientSet) + nodeList, err := e2enode.GetReadyNodesIncludingTaintedOrDie(f.ClientSet) + framework.ExpectNoError(err) for _, node := range nodeList.Items { if node.Labels[gkeNodepoolNameKey] == poolName { nodes = append(nodes, &node) diff --git a/test/e2e/cloud/nodes.go b/test/e2e/cloud/nodes.go index 22ea815022b..ff38b1ce075 100644 --- a/test/e2e/cloud/nodes.go +++ b/test/e2e/cloud/nodes.go @@ -47,10 +47,11 @@ var _ = SIGDescribe("[Feature:CloudProvider][Disruptive] Nodes", func() { nodeDeleteCandidates := framework.GetReadySchedulableNodesOrDie(c) nodeToDelete := nodeDeleteCandidates.Items[0] - origNodes := framework.GetReadyNodesIncludingTaintedOrDie(c) + origNodes, err := e2enode.GetReadyNodesIncludingTaintedOrDie(c) + framework.ExpectNoError(err) e2elog.Logf("Original number of ready nodes: %d", len(origNodes.Items)) - err := framework.DeleteNodeOnCloudProvider(&nodeToDelete) + err = framework.DeleteNodeOnCloudProvider(&nodeToDelete) if err != nil { e2elog.Failf("failed to delete node %q, err: %q", nodeToDelete.Name, err) } diff --git a/test/e2e/framework/node/BUILD b/test/e2e/framework/node/BUILD index 04e936df9b6..226dcc60279 100644 --- a/test/e2e/framework/node/BUILD +++ b/test/e2e/framework/node/BUILD @@ -10,10 +10,13 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/controller/nodelifecycle:go_default_library", + "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/util/system:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index 4ab55313dac..39b8611744c 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -21,13 +21,17 @@ import ( "net" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" nodectlr "k8s.io/kubernetes/pkg/controller/nodelifecycle" + "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" + "k8s.io/kubernetes/pkg/util/system" e2elog "k8s.io/kubernetes/test/e2e/framework/log" testutils "k8s.io/kubernetes/test/utils" ) @@ -48,6 +52,14 @@ const ( proxyTimeout = 2 * time.Minute ) +// PodNode is a pod-node pair indicating which node a given pod is running on +type PodNode struct { + // Pod represents pod name + Pod string + // Node represents node name + Node string +} + // FirstAddress returns the first address of the given type of each node. // TODO: Use return type string instead of []string func FirstAddress(nodelist *v1.NodeList, addrType v1.NodeAddressType) []string { @@ -315,3 +327,141 @@ func CollectAddresses(nodes *v1.NodeList, addressType v1.NodeAddressType) []stri } return ips } + +// PickIP picks one public node IP +func PickIP(c clientset.Interface) (string, error) { + publicIps, err := GetPublicIps(c) + if err != nil { + return "", fmt.Errorf("get node public IPs error: %s", err) + } + if len(publicIps) == 0 { + return "", fmt.Errorf("got unexpected number (%d) of public IPs", len(publicIps)) + } + ip := publicIps[0] + return ip, nil +} + +// GetPublicIps returns a public IP list of nodes. +func GetPublicIps(c clientset.Interface) ([]string, error) { + nodes, err := GetReadySchedulableNodesOrDie(c) + if err != nil { + return nil, fmt.Errorf("get schedulable and ready nodes error: %s", err) + } + ips := CollectAddresses(nodes, v1.NodeExternalIP) + if len(ips) == 0 { + // If ExternalIP isn't set, assume the test programs can reach the InternalIP + ips = CollectAddresses(nodes, v1.NodeInternalIP) + } + return ips, nil +} + +// GetReadySchedulableNodesOrDie addresses the common use case of getting nodes you can do work on. +// 1) Needs to be schedulable. +// 2) Needs to be ready. +// If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely. +// TODO: remove references in framework/util.go. +func GetReadySchedulableNodesOrDie(c clientset.Interface) (nodes *v1.NodeList, err error) { + nodes, err = waitListSchedulableNodesOrDie(c) + if err != nil { + return nil, fmt.Errorf("listing schedulable nodes error: %s", err) + } + // previous tests may have cause failures of some nodes. Let's skip + // 'Not Ready' nodes, just in case (there is no need to fail the test). + Filter(nodes, func(node v1.Node) bool { + return isNodeSchedulable(&node) && isNodeUntainted(&node) + }) + return nodes, nil +} + +// GetReadyNodesIncludingTaintedOrDie returns all ready nodes, even those which are tainted. +// There are cases when we care about tainted nodes +// E.g. in tests related to nodes with gpu we care about nodes despite +// presence of nvidia.com/gpu=present:NoSchedule taint +func GetReadyNodesIncludingTaintedOrDie(c clientset.Interface) (nodes *v1.NodeList, err error) { + nodes, err = waitListSchedulableNodesOrDie(c) + if err != nil { + return nil, fmt.Errorf("listing schedulable nodes error: %s", err) + } + Filter(nodes, func(node v1.Node) bool { + return isNodeSchedulable(&node) + }) + return nodes, nil +} + +// GetMasterAndWorkerNodesOrDie will return a list masters and schedulable worker nodes +func GetMasterAndWorkerNodesOrDie(c clientset.Interface) (sets.String, *v1.NodeList, error) { + nodes := &v1.NodeList{} + masters := sets.NewString() + all, err := c.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("get nodes error: %s", err) + } + for _, n := range all.Items { + if system.IsMasterNode(n.Name) { + masters.Insert(n.Name) + } else if isNodeSchedulable(&n) && isNodeUntainted(&n) { + nodes.Items = append(nodes.Items, n) + } + } + return masters, nodes, nil +} + +// Test whether a fake pod can be scheduled on "node", given its current taints. +func isNodeUntainted(node *v1.Node) bool { + fakePod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-not-scheduled", + Namespace: "fake-not-scheduled", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fake-not-scheduled", + Image: "fake-not-scheduled", + }, + }, + }, + } + nodeInfo := schedulernodeinfo.NewNodeInfo() + nodeInfo.SetNode(node) + fit, _, err := predicates.PodToleratesNodeTaints(fakePod, nil, nodeInfo) + if err != nil { + e2elog.Failf("Can't test predicates for node %s: %v", node.Name, err) + return false + } + return fit +} + +// Node is schedulable if: +// 1) doesn't have "unschedulable" field set +// 2) it's Ready condition is set to true +// 3) doesn't have NetworkUnavailable condition set to true +func isNodeSchedulable(node *v1.Node) bool { + nodeReady := IsConditionSetAsExpected(node, v1.NodeReady, true) + networkReady := IsConditionUnset(node, v1.NodeNetworkUnavailable) || + IsConditionSetAsExpectedSilent(node, v1.NodeNetworkUnavailable, false) + return !node.Spec.Unschedulable && nodeReady && networkReady +} + +// PodNodePairs return podNode pairs for all pods in a namespace +func PodNodePairs(c clientset.Interface, ns string) ([]PodNode, error) { + var result []PodNode + + podList, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{}) + if err != nil { + return result, err + } + + for _, pod := range podList.Items { + result = append(result, PodNode{ + Pod: pod.Name, + Node: pod.Spec.NodeName, + }) + } + + return result, nil +} diff --git a/test/e2e/framework/node/wait.go b/test/e2e/framework/node/wait.go index ec370ce2b3c..bbc0b624e6a 100644 --- a/test/e2e/framework/node/wait.go +++ b/test/e2e/framework/node/wait.go @@ -21,7 +21,7 @@ import ( "regexp" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" @@ -197,3 +197,12 @@ func waitListSchedulableNodes(c clientset.Interface) (*v1.NodeList, error) { } return nodes, nil } + +// waitListSchedulableNodesOrDie is a wrapper around listing nodes supporting retries. +func waitListSchedulableNodesOrDie(c clientset.Interface) (*v1.NodeList, error) { + nodes, err := waitListSchedulableNodes(c) + if err != nil { + return nil, fmt.Errorf("error: %s. Non-retryable failure or timed out while listing nodes for e2e cluster", err) + } + return nodes, nil +} diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index f6817d2079b..fb312389cd2 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -116,14 +116,6 @@ type ServiceTestJig struct { Labels map[string]string } -// PodNode is a pod-node pair indicating which node a given pod is running on -type PodNode struct { - // Pod represents pod name - Pod string - // Node represents node name - Node string -} - // NewServiceTestJig allocates and inits a new ServiceTestJig. func NewServiceTestJig(client clientset.Interface, name string) *ServiceTestJig { j := &ServiceTestJig{} @@ -325,48 +317,6 @@ func (j *ServiceTestJig) CreateLoadBalancerService(namespace, serviceName string return svc } -// GetNodePublicIps returns a public IP list of nodes. -func GetNodePublicIps(c clientset.Interface) ([]string, error) { - nodes := GetReadySchedulableNodesOrDie(c) - - ips := e2enode.CollectAddresses(nodes, v1.NodeExternalIP) - if len(ips) == 0 { - // If ExternalIP isn't set, assume the test programs can reach the InternalIP - ips = e2enode.CollectAddresses(nodes, v1.NodeInternalIP) - } - return ips, nil -} - -// PickNodeIP picks one public node IP -func PickNodeIP(c clientset.Interface) string { - publicIps, err := GetNodePublicIps(c) - ExpectNoError(err) - if len(publicIps) == 0 { - e2elog.Failf("got unexpected number (%d) of public IPs", len(publicIps)) - } - ip := publicIps[0] - return ip -} - -// PodNodePairs return PodNode pairs for all pods in a namespace -func PodNodePairs(c clientset.Interface, ns string) ([]PodNode, error) { - var result []PodNode - - podList, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{}) - if err != nil { - return result, err - } - - for _, pod := range podList.Items { - result = append(result, PodNode{ - Pod: pod.Name, - Node: pod.Spec.NodeName, - }) - } - - return result, nil -} - // GetEndpointNodes returns a map of nodenames:external-ip on which the // endpoints of the given Service are running. func (j *ServiceTestJig) GetEndpointNodes(svc *v1.Service) map[string][]string { diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index b8a5b92ef52..d73f7052327 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -79,7 +79,6 @@ import ( "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" - "k8s.io/kubernetes/pkg/util/system" taintutils "k8s.io/kubernetes/pkg/util/taints" "k8s.io/kubernetes/test/e2e/framework/ginkgowrapper" e2elog "k8s.io/kubernetes/test/e2e/framework/log" @@ -1943,6 +1942,7 @@ func isNodeUntainted(node *v1.Node) bool { // 1) Needs to be schedulable. // 2) Needs to be ready. // If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely. +// TODO: remove this function here when references point to e2enode. func GetReadySchedulableNodesOrDie(c clientset.Interface) (nodes *v1.NodeList) { nodes = waitListSchedulableNodesOrDie(c) // previous tests may have cause failures of some nodes. Let's skip @@ -1953,18 +1953,6 @@ func GetReadySchedulableNodesOrDie(c clientset.Interface) (nodes *v1.NodeList) { return nodes } -// GetReadyNodesIncludingTaintedOrDie returns all ready nodes, even those which are tainted. -// There are cases when we care about tainted nodes -// E.g. in tests related to nodes with gpu we care about nodes despite -// presence of nvidia.com/gpu=present:NoSchedule taint -func GetReadyNodesIncludingTaintedOrDie(c clientset.Interface) (nodes *v1.NodeList) { - nodes = waitListSchedulableNodesOrDie(c) - e2enode.Filter(nodes, func(node v1.Node) bool { - return isNodeSchedulable(&node) - }) - return nodes -} - // WaitForAllNodesSchedulable waits up to timeout for all // (but TestContext.AllowedNotReadyNodes) to become scheduable. func WaitForAllNodesSchedulable(c clientset.Interface, timeout time.Duration) error { @@ -3046,22 +3034,6 @@ func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int { return len(scheduledPods) } -// GetMasterAndWorkerNodesOrDie will return a list masters and schedulable worker nodes -func GetMasterAndWorkerNodesOrDie(c clientset.Interface) (sets.String, *v1.NodeList) { - nodes := &v1.NodeList{} - masters := sets.NewString() - all, err := c.CoreV1().Nodes().List(metav1.ListOptions{}) - ExpectNoError(err) - for _, n := range all.Items { - if system.IsMasterNode(n.Name) { - masters.Insert(n.Name) - } else if isNodeSchedulable(&n) && isNodeUntainted(&n) { - nodes.Items = append(nodes.Items, n) - } - } - return masters, nodes -} - // ListNamespaceEvents lists the events in the given namespace. func ListNamespaceEvents(c clientset.Interface, ns string) error { ls, err := c.CoreV1().Events(ns).List(metav1.ListOptions{}) diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 2c510feb822..7fbda94134b 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -498,7 +498,8 @@ var _ = SIGDescribe("Services", func() { ns := f.Namespace.Name jig := framework.NewServiceTestJig(cs, serviceName) - nodeIP := framework.PickNodeIP(jig.Client) // for later + nodeIP, err := e2enode.PickIP(jig.Client) // for later + framework.ExpectNoError(err) ginkgo.By("creating service " + serviceName + " with type=NodePort in namespace " + ns) service := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) { @@ -554,7 +555,8 @@ var _ = SIGDescribe("Services", func() { e2elog.Logf("namespace for UDP test: %s", ns2) jig := framework.NewServiceTestJig(cs, serviceName) - nodeIP := framework.PickNodeIP(jig.Client) // for later + nodeIP, err := e2enode.PickIP(jig.Client) // for later + framework.ExpectNoError(err) // Test TCP and UDP Services. Services with the same name in different // namespaces should get different node ports and load balancers. @@ -2416,7 +2418,7 @@ func execAffinityTestForLBServiceWithOptionalTransition(f *framework.Framework, svc = jig.WaitForLoadBalancerOrFail(ns, serviceName, framework.LoadBalancerCreateTimeoutDefault) jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer) defer func() { - podNodePairs, err := framework.PodNodePairs(cs, ns) + podNodePairs, err := e2enode.PodNodePairs(cs, ns) e2elog.Logf("[pod,node] pairs: %+v; err: %v", podNodePairs, err) framework.StopServeHostnameService(cs, ns, serviceName) lb := cloudprovider.DefaultLoadBalancerName(svc) diff --git a/test/e2e/scalability/BUILD b/test/e2e/scalability/BUILD index ff0926a3445..628eca3b504 100644 --- a/test/e2e/scalability/BUILD +++ b/test/e2e/scalability/BUILD @@ -40,6 +40,7 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/metrics:go_default_library", + "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/e2e/framework/timer:go_default_library", "//test/utils:go_default_library", diff --git a/test/e2e/scalability/density.go b/test/e2e/scalability/density.go index 06a925f8bfa..27ba55161d5 100644 --- a/test/e2e/scalability/density.go +++ b/test/e2e/scalability/density.go @@ -53,6 +53,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/framework/timer" testutils "k8s.io/kubernetes/test/utils" @@ -502,6 +503,7 @@ var _ = SIGDescribe("Density", func() { f.NamespaceDeletionTimeout = time.Hour ginkgo.BeforeEach(func() { + var err error c = f.ClientSet ns = f.Namespace.Name testPhaseDurations = timer.NewTestPhaseTimer() @@ -518,7 +520,8 @@ var _ = SIGDescribe("Density", func() { }, }) - _, nodes = framework.GetMasterAndWorkerNodesOrDie(c) + _, nodes, err = e2enode.GetMasterAndWorkerNodesOrDie(c) + framework.ExpectNoError(err) nodeCount = len(nodes.Items) gomega.Expect(nodeCount).NotTo(gomega.BeZero()) @@ -529,7 +532,7 @@ var _ = SIGDescribe("Density", func() { // Terminating a namespace (deleting the remaining objects from it - which // generally means events) can affect the current run. Thus we wait for all // terminating namespace to be finally deleted before starting this test. - err := framework.CheckTestingNSDeletedExcept(c, ns) + err = framework.CheckTestingNSDeletedExcept(c, ns) framework.ExpectNoError(err) uuid = string(utiluuid.NewUUID()) diff --git a/test/e2e/scheduling/equivalence_cache_predicates.go b/test/e2e/scheduling/equivalence_cache_predicates.go index d5bebe04ce5..39d29fee866 100644 --- a/test/e2e/scheduling/equivalence_cache_predicates.go +++ b/test/e2e/scheduling/equivalence_cache_predicates.go @@ -48,6 +48,7 @@ var _ = framework.KubeDescribe("EquivalenceCache [Serial]", func() { var masterNodes sets.String var systemPodsNo int var ns string + var err error f := framework.NewDefaultFramework("equivalence-cache") ginkgo.BeforeEach(func() { @@ -55,7 +56,8 @@ var _ = framework.KubeDescribe("EquivalenceCache [Serial]", func() { ns = f.Namespace.Name e2enode.WaitForTotalHealthy(cs, time.Minute) - masterNodes, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs) + masterNodes, nodeList, err = e2enode.GetMasterAndWorkerNodesOrDie(cs) + framework.ExpectNoError(err) framework.ExpectNoError(framework.CheckTestingNSDeletedExcept(cs, ns)) diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 09234069abc..e02667bb527 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -31,6 +31,7 @@ import ( "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" @@ -82,11 +83,13 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { cs = f.ClientSet ns = f.Namespace.Name nodeList = &v1.NodeList{} + var err error framework.AllNodesReady(cs, time.Minute) - masterNodes, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs) + masterNodes, nodeList, err = e2enode.GetMasterAndWorkerNodesOrDie(cs) + framework.ExpectNoError(err) - err := framework.CheckTestingNSDeletedExcept(cs, ns) + err = framework.CheckTestingNSDeletedExcept(cs, ns) framework.ExpectNoError(err) for _, node := range nodeList.Items { diff --git a/test/e2e/scheduling/preemption.go b/test/e2e/scheduling/preemption.go index 8648890e799..f2fdeea5f7f 100644 --- a/test/e2e/scheduling/preemption.go +++ b/test/e2e/scheduling/preemption.go @@ -41,6 +41,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + // ensure libs have a chance to initialize _ "github.com/stretchr/testify/assert" ) @@ -76,15 +77,17 @@ var _ = SIGDescribe("SchedulerPreemption [Serial]", func() { cs = f.ClientSet ns = f.Namespace.Name nodeList = &v1.NodeList{} + var err error for _, pair := range priorityPairs { _, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(&schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: pair.name}, Value: pair.value}) framework.ExpectEqual(err == nil || errors.IsAlreadyExists(err), true) } e2enode.WaitForTotalHealthy(cs, time.Minute) - masterNodes, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs) + masterNodes, nodeList, err = e2enode.GetMasterAndWorkerNodesOrDie(cs) + framework.ExpectNoError(err) - err := framework.CheckTestingNSDeletedExcept(cs, ns) + err = framework.CheckTestingNSDeletedExcept(cs, ns) framework.ExpectNoError(err) }) diff --git a/test/e2e/scheduling/priorities.go b/test/e2e/scheduling/priorities.go index 20006791dee..8280f79e51c 100644 --- a/test/e2e/scheduling/priorities.go +++ b/test/e2e/scheduling/priorities.go @@ -27,7 +27,7 @@ import ( // ensure libs have a chance to initialize _ "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -77,11 +77,13 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { cs = f.ClientSet ns = f.Namespace.Name nodeList = &v1.NodeList{} + var err error e2enode.WaitForTotalHealthy(cs, time.Minute) - _, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs) + _, nodeList, err = e2enode.GetMasterAndWorkerNodesOrDie(cs) + framework.ExpectNoError(err) - err := framework.CheckTestingNSDeletedExcept(cs, ns) + err = framework.CheckTestingNSDeletedExcept(cs, ns) framework.ExpectNoError(err) err = e2epod.WaitForPodsRunningReady(cs, metav1.NamespaceSystem, int32(systemPodsNo), 0, framework.PodReadyBeforeTimeout, map[string]string{}) framework.ExpectNoError(err) diff --git a/test/e2e/storage/vsphere/BUILD b/test/e2e/storage/vsphere/BUILD index 3082275ccd4..9d765df7dc0 100644 --- a/test/e2e/storage/vsphere/BUILD +++ b/test/e2e/storage/vsphere/BUILD @@ -54,6 +54,7 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e/framework/deployment:go_default_library", "//test/e2e/framework/log:go_default_library", + "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/e2e/framework/ssh:go_default_library", "//test/e2e/framework/statefulset:go_default_library", diff --git a/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go index 99ea427fe97..5883756c973 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go +++ b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go @@ -30,6 +30,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" "k8s.io/kubernetes/test/e2e/storage/utils" ) @@ -110,7 +111,8 @@ var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsp if !(len(nodeList.Items) > 0) { e2elog.Failf("Unable to find ready and schedulable Node") } - masternodes, _ := framework.GetMasterAndWorkerNodesOrDie(client) + masternodes, _, err := e2enode.GetMasterAndWorkerNodesOrDie(client) + framework.ExpectNoError(err) gomega.Expect(masternodes).NotTo(gomega.BeEmpty()) masterNode = masternodes.List()[0] }) diff --git a/test/e2e/windows/BUILD b/test/e2e/windows/BUILD index 711d8d7a0f5..6ca4c612784 100644 --- a/test/e2e/windows/BUILD +++ b/test/e2e/windows/BUILD @@ -33,6 +33,7 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/metrics:go_default_library", + "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", diff --git a/test/e2e/windows/service.go b/test/e2e/windows/service.go index ca865d1bb91..c13018edc75 100644 --- a/test/e2e/windows/service.go +++ b/test/e2e/windows/service.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" "github.com/onsi/ginkgo" ) @@ -41,7 +42,8 @@ var _ = SIGDescribe("Services", func() { ns := f.Namespace.Name jig := framework.NewServiceTestJig(cs, serviceName) - nodeIP := framework.PickNodeIP(jig.Client) + nodeIP, err := e2enode.PickIP(jig.Client) + framework.ExpectNoError(err) ginkgo.By("creating service " + serviceName + " with type=NodePort in namespace " + ns) service := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) {