diff --git a/test/e2e/framework/ingress/BUILD b/test/e2e/framework/ingress/BUILD index 528efea8812..78709438297 100644 --- a/test/e2e/framework/ingress/BUILD +++ b/test/e2e/framework/ingress/BUILD @@ -11,6 +11,7 @@ go_library( "//staging/src/k8s.io/api/networking/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", @@ -18,7 +19,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", - "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/service:go_default_library", "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/manifest:go_default_library", diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index 88bc504d93c..43cadf3b838 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -43,6 +43,7 @@ import ( networkingv1beta1 "k8s.io/api/networking/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" utilnet "k8s.io/apimachinery/pkg/util/net" @@ -50,7 +51,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" "k8s.io/kubernetes/test/e2e/framework/testfiles" "k8s.io/kubernetes/test/e2e/manifest" @@ -106,6 +106,14 @@ const ( // names of GCP resources such as forwarding rules, url maps, target proxies, etc // that it created for the corresponding ingress. StatusPrefix = "ingress.kubernetes.io" + + // poll is how often to Poll pods, nodes and claims. + poll = 2 * time.Second + + // singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent + // transient failures from failing tests. + // TODO: client should not apply this timeout to Watch calls. Increased from 30s until that is fixed. + singleCallTimeout = 5 * time.Minute ) // TestLogger is an interface for log. @@ -786,13 +794,70 @@ func (j *TestJig) VerifyURL(route, host string, iterations int, interval time.Du func (j *TestJig) pollServiceNodePort(ns, name string, port int) error { // TODO: Curl all nodes? - u, err := e2enode.GetPortURL(j.Client, ns, name, port) + u, err := getPortURL(j.Client, ns, name, port) if err != nil { return err } return PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false) } +// getSvcNodePort returns the node port for the given service:port. +func getSvcNodePort(client clientset.Interface, ns, name string, svcPort int) (int, error) { + svc, err := client.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) + if err != nil { + return 0, err + } + for _, p := range svc.Spec.Ports { + if p.Port == int32(svcPort) { + if p.NodePort != 0 { + return int(p.NodePort), nil + } + } + } + return 0, fmt.Errorf( + "no node port found for service %v, port %v", name, svcPort) +} + +// getPortURL returns the url to a nodeport Service. +func getPortURL(client clientset.Interface, ns, name string, svcPort int) (string, error) { + nodePort, err := getSvcNodePort(client, ns, name, svcPort) + if err != nil { + return "", err + } + // This list of nodes must not include the master, which is marked + // unschedulable, since the master doesn't run kube-proxy. Without + // kube-proxy NodePorts won't work. + var nodes *v1.NodeList + if wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) { + nodes, err = client.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ + "spec.unschedulable": "false", + }.AsSelector().String()}) + if err != nil { + if testutils.IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + return true, nil + }) != nil { + return "", err + } + if len(nodes.Items) == 0 { + return "", fmt.Errorf("Unable to list nodes in cluster") + } + for _, node := range nodes.Items { + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeExternalIP { + if address.Address != "" { + host := net.JoinHostPort(address.Address, fmt.Sprint(nodePort)) + return fmt.Sprintf("http://%s", host), nil + } + } + } + } + return "", fmt.Errorf("failed to find external address for service %v", name) +} + // GetIngressNodePorts returns related backend services' nodePorts. // Current GCE ingress controller allows traffic to the default HTTP backend // by default, so retrieve its nodePort if includeDefaultBackend is true. diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index 548727bf0a7..5ad15c5d169 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -24,17 +24,14 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" nodectlr "k8s.io/kubernetes/pkg/controller/nodelifecycle" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" e2elog "k8s.io/kubernetes/test/e2e/framework/log" "k8s.io/kubernetes/test/e2e/system" - testutils "k8s.io/kubernetes/test/utils" ) const ( @@ -189,63 +186,6 @@ func TotalReady(c clientset.Interface) (int, error) { return len(nodes.Items), nil } -// getSvcNodePort returns the node port for the given service:port. -func getSvcNodePort(client clientset.Interface, ns, name string, svcPort int) (int, error) { - svc, err := client.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) - if err != nil { - return 0, err - } - for _, p := range svc.Spec.Ports { - if p.Port == int32(svcPort) { - if p.NodePort != 0 { - return int(p.NodePort), nil - } - } - } - return 0, fmt.Errorf( - "No node port found for service %v, port %v", name, svcPort) -} - -// GetPortURL returns the url to a nodeport Service. -func GetPortURL(client clientset.Interface, ns, name string, svcPort int) (string, error) { - nodePort, err := getSvcNodePort(client, ns, name, svcPort) - if err != nil { - return "", err - } - // This list of nodes must not include the master, which is marked - // unschedulable, since the master doesn't run kube-proxy. Without - // kube-proxy NodePorts won't work. - var nodes *v1.NodeList - if wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) { - nodes, err = client.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ - "spec.unschedulable": "false", - }.AsSelector().String()}) - if err != nil { - if testutils.IsRetryableAPIError(err) { - return false, nil - } - return false, err - } - return true, nil - }) != nil { - return "", err - } - if len(nodes.Items) == 0 { - return "", fmt.Errorf("Unable to list nodes in cluster") - } - for _, node := range nodes.Items { - for _, address := range node.Status.Addresses { - if address.Type == v1.NodeExternalIP { - if address.Address != "" { - host := net.JoinHostPort(address.Address, fmt.Sprint(nodePort)) - return fmt.Sprintf("http://%s", host), nil - } - } - } - } - return "", fmt.Errorf("Failed to find external address for service %v", name) -} - // GetExternalIP returns node external IP concatenated with port 22 for ssh // e.g. 1.2.3.4:22 func GetExternalIP(node *v1.Node) (string, error) {