From ca4c349096e15b15a1a5a65ced769d489f617772 Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Mon, 28 Oct 2019 18:04:53 +0000 Subject: [PATCH] Move functions from e2e/framework/util.go - SimpleGET: Moved to ingress sub package of e2e framework - PollURL: Moved to ingress sub package of e2e framework - ProxyMode: Moved to service e2e test package - ListNamespaceEvents: Moved to e2e_node test package - NewE2ETestNodePreparer: Removed since 59533f0cd1b6a31f8ccd040ecba5055e7cf3fdbe --- test/e2e/framework/ingress/ingress_utils.go | 56 ++++++++++-- test/e2e/framework/util.go | 96 --------------------- test/e2e/network/ingress.go | 6 +- test/e2e/network/service.go | 32 ++++++- test/e2e_node/BUILD | 2 +- test/e2e_node/util.go | 17 +++- 6 files changed, 101 insertions(+), 108 deletions(-) diff --git a/test/e2e/framework/ingress/ingress_utils.go b/test/e2e/framework/ingress/ingress_utils.go index f435d0e6956..1c178c346b6 100644 --- a/test/e2e/framework/ingress/ingress_utils.go +++ b/test/e2e/framework/ingress/ingress_utils.go @@ -25,6 +25,7 @@ import ( "crypto/x509/pkix" "encoding/pem" "fmt" + "io/ioutil" "math/big" "net" "net/http" @@ -156,6 +157,51 @@ type NegStatus struct { Zones []string `json:"zones,omitempty"` } +// SimpleGET executes a get on the given url, returns error if non-200 returned. +func SimpleGET(c *http.Client, url, host string) (string, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + req.Host = host + res, err := c.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + rawBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + body := string(rawBody) + if res.StatusCode != http.StatusOK { + err = fmt.Errorf( + "GET returned http error %v", res.StatusCode) + } + return body, err +} + +// PollURL polls till the url responds with a healthy http code. If +// expectUnreachable is true, it breaks on first non-healthy http code instead. +func PollURL(route, host string, timeout time.Duration, interval time.Duration, httpClient *http.Client, expectUnreachable bool) error { + var lastBody string + pollErr := wait.PollImmediate(interval, timeout, func() (bool, error) { + var err error + lastBody, err = SimpleGET(httpClient, route, host) + if err != nil { + framework.Logf("host %v path %v: %v unreachable", host, route, err) + return expectUnreachable, nil + } + framework.Logf("host %v path %v: reached", host, route) + return !expectUnreachable, nil + }) + if pollErr != nil { + return fmt.Errorf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", + timeout, route, host, lastBody, pollErr) + } + return nil +} + // CreateIngressComformanceTests generates an slice of sequential test cases: // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname, // ingress https with a modified URLMap @@ -210,7 +256,7 @@ func CreateIngressComformanceTests(jig *TestJig, ns string, annotations map[stri }) ginkgo.By("Checking that " + pathToFail + " is not exposed by polling for failure") route := fmt.Sprintf("http://%v%v", jig.Address, pathToFail) - framework.ExpectNoError(framework.PollURL(route, updateURLMapHost, e2eservice.LoadBalancerCleanupTimeout, jig.PollInterval, &http.Client{Timeout: IngressReqTimeout}, true)) + framework.ExpectNoError(PollURL(route, updateURLMapHost, e2eservice.LoadBalancerCleanupTimeout, jig.PollInterval, &http.Client{Timeout: IngressReqTimeout}, true)) }, fmt.Sprintf("Waiting for path updates to reflect in L7"), }, @@ -657,7 +703,7 @@ func (j *TestJig) pollIngressWithCert(ing *networkingv1beta1.Ingress, address st } route := fmt.Sprintf("%v://%v%v", proto, address, p.Path) j.Logger.Infof("Testing route %v host %v with simple GET", route, rules.Host) - if err := framework.PollURL(route, rules.Host, timeout, j.PollInterval, timeoutClient, false); err != nil { + if err := PollURL(route, rules.Host, timeout, j.PollInterval, timeoutClient, false); err != nil { return err } } @@ -727,7 +773,7 @@ func (j *TestJig) WaitForIngressWithCert(waitForNodePort bool, knownHosts []stri // given url returns a non-healthy http code even once. func (j *TestJig) VerifyURL(route, host string, iterations int, interval time.Duration, httpClient *http.Client) error { for i := 0; i < iterations; i++ { - b, err := framework.SimpleGET(httpClient, route, host) + b, err := SimpleGET(httpClient, route, host) if err != nil { framework.Logf(b) return err @@ -744,7 +790,7 @@ func (j *TestJig) pollServiceNodePort(ns, name string, port int) error { if err != nil { return err } - return framework.PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false) + return PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false) } // GetIngressNodePorts returns related backend services' nodePorts. @@ -816,7 +862,7 @@ func (j *TestJig) GetDistinctResponseFromIngress() (sets.String, error) { for i := 0; i < 100; i++ { url := fmt.Sprintf("http://%v", address) - res, err := framework.SimpleGET(timeoutClient, url, "") + res, err := SimpleGET(timeoutClient, url, "") if err != nil { j.Logger.Errorf("Failed to GET %q. Got responses: %q: %v", url, res, err) return responses, err diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index cd8827c8f23..fe54821051c 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -307,36 +307,6 @@ func NodeOSDistroIs(supportedNodeOsDistros ...string) bool { return false } -// ProxyMode returns a proxyMode of a kube-proxy. -func ProxyMode(f *Framework) (string, error) { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-proxy-mode-detector", - Namespace: f.Namespace.Name, - }, - Spec: v1.PodSpec{ - HostNetwork: true, - Containers: []v1.Container{ - { - Name: "detector", - Image: AgnHostImage, - Args: []string{"pause"}, - }, - }, - }, - } - f.PodClient().CreateSync(pod) - defer f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, DefaultPodDeletionTimeout) - - cmd := "curl -q -s --connect-timeout 1 http://localhost:10249/proxyMode" - stdout, err := RunHostCmd(pod.Namespace, pod.Name, cmd) - if err != nil { - return "", err - } - Logf("ProxyMode: %s", stdout) - return stdout, nil -} - // WaitForDaemonSets for all daemonsets in the given namespace to be ready // (defined as all but 'allowedNotReadyNodes' pods associated with that // daemonset are ready). @@ -2390,18 +2360,6 @@ func WaitForStableCluster(c clientset.Interface, masterNodes sets.String) int { return len(scheduledPods) } -// ListNamespaceEvents lists the events in the given namespace. -func ListNamespaceEvents(c clientset.Interface, ns string) error { - ls, err := c.CoreV1().Events(ns).List(metav1.ListOptions{}) - if err != nil { - return err - } - for _, event := range ls.Items { - klog.Infof("Event(%#v): type: '%v' reason: '%v' %v", event.InvolvedObject, event.Type, event.Reason, event.Message) - } - return nil -} - // E2ETestNodePreparer implements testutils.TestNodePreparer interface, which is used // to create/modify Nodes before running a test. type E2ETestNodePreparer struct { @@ -2413,15 +2371,6 @@ type E2ETestNodePreparer struct { nodeToAppliedStrategy map[string]testutils.PrepareNodeStrategy } -// NewE2ETestNodePreparer returns a new instance of E2ETestNodePreparer. -func NewE2ETestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy) testutils.TestNodePreparer { - return &E2ETestNodePreparer{ - client: client, - countToStrategy: countToStrategy, - nodeToAppliedStrategy: make(map[string]testutils.PrepareNodeStrategy), - } -} - // PrepareNodes prepares nodes in the cluster. func (p *E2ETestNodePreparer) PrepareNodes() error { nodes, err := e2enode.GetReadySchedulableNodes(p.client) @@ -2523,51 +2472,6 @@ func GetAllMasterAddresses(c clientset.Interface) []string { return ips.List() } -// SimpleGET executes a get on the given url, returns error if non-200 returned. -func SimpleGET(c *http.Client, url, host string) (string, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", err - } - req.Host = host - res, err := c.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - rawBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - body := string(rawBody) - if res.StatusCode != http.StatusOK { - err = fmt.Errorf( - "GET returned http error %v", res.StatusCode) - } - return body, err -} - -// PollURL polls till the url responds with a healthy http code. If -// expectUnreachable is true, it breaks on first non-healthy http code instead. -func PollURL(route, host string, timeout time.Duration, interval time.Duration, httpClient *http.Client, expectUnreachable bool) error { - var lastBody string - pollErr := wait.PollImmediate(interval, timeout, func() (bool, error) { - var err error - lastBody, err = SimpleGET(httpClient, route, host) - if err != nil { - Logf("host %v path %v: %v unreachable", host, route, err) - return expectUnreachable, nil - } - Logf("host %v path %v: reached", host, route) - return !expectUnreachable, nil - }) - if pollErr != nil { - return fmt.Errorf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", - timeout, route, host, lastBody, pollErr) - } - return nil -} - // DescribeIng describes information of ingress by running kubectl describe ing. func DescribeIng(ns string) { Logf("\nOutput of kubectl describe ing:\n") diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index 404fff472a0..e9ae7b3c3bd 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -831,10 +831,10 @@ func executeStaticIPHttpsOnlyTest(f *framework.Framework, jig *ingress.TestJig, ginkgo.By("waiting for Ingress to come up with ip: " + ip) httpClient := ingress.BuildInsecureClient(ingress.IngressReqTimeout) - framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%s/", ip), "", e2eservice.LoadBalancerPollTimeout, jig.PollInterval, httpClient, false)) + framework.ExpectNoError(ingress.PollURL(fmt.Sprintf("https://%s/", ip), "", e2eservice.LoadBalancerPollTimeout, jig.PollInterval, httpClient, false)) ginkgo.By("should reject HTTP traffic") - framework.ExpectNoError(framework.PollURL(fmt.Sprintf("http://%s/", ip), "", e2eservice.LoadBalancerPollTimeout, jig.PollInterval, httpClient, true)) + framework.ExpectNoError(ingress.PollURL(fmt.Sprintf("http://%s/", ip), "", e2eservice.LoadBalancerPollTimeout, jig.PollInterval, httpClient, true)) } func executeBacksideBacksideHTTPSTest(f *framework.Framework, jig *ingress.TestJig, staticIPName string) { @@ -855,7 +855,7 @@ func executeBacksideBacksideHTTPSTest(f *framework.Framework, jig *ingress.TestJ ginkgo.By(fmt.Sprintf("Polling on address %s and verify the backend is serving HTTPS", ingIP)) timeoutClient := &http.Client{Timeout: ingress.IngressReqTimeout} err = wait.PollImmediate(e2eservice.LoadBalancerPollInterval, e2eservice.LoadBalancerPollTimeout, func() (bool, error) { - resp, err := framework.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "") + resp, err := ingress.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "") if err != nil { framework.Logf("SimpleGET failed: %v", err) return false, nil diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 8736c9d6b11..b390327029a 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -262,7 +262,7 @@ var _ = SIGDescribe("Services", func() { // This behavior is not supported if Kube-proxy is in "userspace" mode. // So we check the kube-proxy mode and skip this test if that's the case. - if proxyMode, err := framework.ProxyMode(f); err == nil { + if proxyMode, err := proxyMode(f); err == nil { if proxyMode == "userspace" { framework.Skipf("The test doesn't work with kube-proxy in userspace mode") } @@ -2618,3 +2618,33 @@ func checkReachabilityFromPod(expectToBeReachable bool, timeout time.Duration, n }) framework.ExpectNoError(err) } + +// proxyMode returns a proxyMode of a kube-proxy. +func proxyMode(f *framework.Framework) (string, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-proxy-mode-detector", + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + HostNetwork: true, + Containers: []v1.Container{ + { + Name: "detector", + Image: framework.AgnHostImage, + Args: []string{"pause"}, + }, + }, + }, + } + f.PodClient().CreateSync(pod) + defer f.PodClient().DeleteSync(pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) + + cmd := "curl -q -s --connect-timeout 1 http://localhost:10249/proxyMode" + stdout, err := framework.RunHostCmd(pod.Namespace, pod.Name, cmd) + if err != nil { + return "", err + } + framework.Logf("proxyMode: %s", stdout) + return stdout, nil +} diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 92f21edf97f..ebe67665456 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -38,6 +38,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/component-base/featuregate:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", @@ -67,7 +68,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/perf:go_default_library", diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index e6562937877..8d93022513a 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/component-base/featuregate" internalapi "k8s.io/cri-api/pkg/apis" @@ -332,15 +333,27 @@ func newKubeletConfigMap(name string, internalKC *kubeletconfig.KubeletConfigura return cmap } +// listNamespaceEvents lists the events in the given namespace. +func listNamespaceEvents(c clientset.Interface, ns string) error { + ls, err := c.CoreV1().Events(ns).List(metav1.ListOptions{}) + if err != nil { + return err + } + for _, event := range ls.Items { + klog.Infof("Event(%#v): type: '%v' reason: '%v' %v", event.InvolvedObject, event.Type, event.Reason, event.Message) + } + return nil +} + func logPodEvents(f *framework.Framework) { framework.Logf("Summary of pod events during the test:") - err := framework.ListNamespaceEvents(f.ClientSet, f.Namespace.Name) + err := listNamespaceEvents(f.ClientSet, f.Namespace.Name) framework.ExpectNoError(err) } func logNodeEvents(f *framework.Framework) { framework.Logf("Summary of node events during the test:") - err := framework.ListNamespaceEvents(f.ClientSet, "") + err := listNamespaceEvents(f.ClientSet, "") framework.ExpectNoError(err) }