From 2cd45b2c541182bb3ed4aedb94b30201471126ef Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Tue, 13 Aug 2019 23:25:47 +0000 Subject: [PATCH] Move ProxyRequest to kubelet e2e fw In e2e test framework, ProxyRequest() is called in kubelet e2e test framework only. In addition, the function works for kubelet as tests which call the function do. So this moves the function into kubelet e2e test framework to make the dependency simple. --- test/e2e/framework/kubelet/BUILD | 2 +- test/e2e/framework/kubelet/kubelet_pods.go | 3 +- test/e2e/framework/kubelet/stats.go | 33 ++++++++++++++++++++-- test/e2e/framework/node/BUILD | 1 - test/e2e/framework/node/resource.go | 28 ------------------ test/e2e/lifecycle/BUILD | 1 + test/e2e/lifecycle/kubelet_security.go | 5 ++-- test/e2e/node/pre_stop.go | 4 +-- 8 files changed, 39 insertions(+), 38 deletions(-) diff --git a/test/e2e/framework/kubelet/BUILD b/test/e2e/framework/kubelet/BUILD index ed8151cbb2a..c48e9d3aa09 100644 --- a/test/e2e/framework/kubelet/BUILD +++ b/test/e2e/framework/kubelet/BUILD @@ -17,9 +17,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/metrics:go_default_library", - "//test/e2e/framework/node:go_default_library", ], ) diff --git a/test/e2e/framework/kubelet/kubelet_pods.go b/test/e2e/framework/kubelet/kubelet_pods.go index 012cf25170a..79f2973d88d 100644 --- a/test/e2e/framework/kubelet/kubelet_pods.go +++ b/test/e2e/framework/kubelet/kubelet_pods.go @@ -21,7 +21,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/master/ports" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" ) // GetKubeletPods retrieves the list of pods on the kubelet. @@ -38,7 +37,7 @@ func GetKubeletRunningPods(c clientset.Interface, node string) (*v1.PodList, err func getKubeletPods(c clientset.Interface, node, resource string) (*v1.PodList, error) { result := &v1.PodList{} - client, err := e2enode.ProxyRequest(c, node, resource, ports.KubeletPort) + client, err := ProxyRequest(c, node, resource, ports.KubeletPort) if err != nil { return &v1.PodList{}, err } diff --git a/test/e2e/framework/kubelet/stats.go b/test/e2e/framework/kubelet/stats.go index b1d1f28789f..0f7461f0971 100644 --- a/test/e2e/framework/kubelet/stats.go +++ b/test/e2e/framework/kubelet/stats.go @@ -32,12 +32,17 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" kubeletstatsv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" dockermetrics "k8s.io/kubernetes/pkg/kubelet/dockershim/metrics" "k8s.io/kubernetes/pkg/master/ports" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" +) + +const ( + // timeout for proxy requests. + proxyTimeout = 2 * time.Minute ) // ContainerResourceUsage is a structure for gathering container resource usage. @@ -82,6 +87,30 @@ type RuntimeOperationErrorRate struct { TimeoutRate float64 } +// ProxyRequest performs a get on a node proxy endpoint given the nodename and rest client. +func ProxyRequest(c clientset.Interface, node, endpoint string, port int) (restclient.Result, error) { + // proxy tends to hang in some cases when Node is not ready. Add an artificial timeout for this call. + // This will leak a goroutine if proxy hangs. #22165 + var result restclient.Result + finished := make(chan struct{}) + go func() { + result = c.CoreV1().RESTClient().Get(). + Resource("nodes"). + SubResource("proxy"). + Name(fmt.Sprintf("%v:%v", node, port)). + Suffix(endpoint). + Do() + + finished <- struct{}{} + }() + select { + case <-finished: + return result, nil + case <-time.After(proxyTimeout): + return restclient.Result{}, nil + } +} + // NewRuntimeOperationMonitor returns a new RuntimeOperationMonitor. func NewRuntimeOperationMonitor(c clientset.Interface) *RuntimeOperationMonitor { m := &RuntimeOperationMonitor{ @@ -368,7 +397,7 @@ func formatResourceUsageStats(nodeName string, containerStats ResourceUsagePerCo // GetKubeletHeapStats returns stats of kubelet heap. func GetKubeletHeapStats(c clientset.Interface, nodeName string) (string, error) { - client, err := e2enode.ProxyRequest(c, nodeName, "debug/pprof/heap", ports.KubeletPort) + client, err := ProxyRequest(c, nodeName, "debug/pprof/heap", ports.KubeletPort) if err != nil { return "", err } diff --git a/test/e2e/framework/node/BUILD b/test/e2e/framework/node/BUILD index 226dcc60279..09080ea1c64 100644 --- a/test/e2e/framework/node/BUILD +++ b/test/e2e/framework/node/BUILD @@ -19,7 +19,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/utils:go_default_library", ], diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index cd9d22d24f7..c15111b7980 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" nodectlr "k8s.io/kubernetes/pkg/controller/nodelifecycle" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" @@ -47,9 +46,6 @@ const ( // ssh port sshPort = "22" - - // timeout for proxy requests. - proxyTimeout = 2 * time.Minute ) // PodNode is a pod-node pair indicating which node a given pod is running on @@ -250,30 +246,6 @@ func GetPortURL(client clientset.Interface, ns, name string, svcPort int) (strin return "", fmt.Errorf("Failed to find external address for service %v", name) } -// ProxyRequest performs a get on a node proxy endpoint given the nodename and rest client. -func ProxyRequest(c clientset.Interface, node, endpoint string, port int) (restclient.Result, error) { - // proxy tends to hang in some cases when Node is not ready. Add an artificial timeout for this call. - // This will leak a goroutine if proxy hangs. #22165 - var result restclient.Result - finished := make(chan struct{}) - go func() { - result = c.CoreV1().RESTClient().Get(). - Resource("nodes"). - SubResource("proxy"). - Name(fmt.Sprintf("%v:%v", node, port)). - Suffix(endpoint). - Do() - - finished <- struct{}{} - }() - select { - case <-finished: - return result, nil - case <-time.After(proxyTimeout): - return restclient.Result{}, nil - } -} - // GetExternalIP returns node external IP concatenated with port 22 for ssh // e.g. 1.2.3.4:22 func GetExternalIP(node *v1.Node) (string, error) { diff --git a/test/e2e/lifecycle/BUILD b/test/e2e/lifecycle/BUILD index 6fe3eff259a..dcb3cdbc0df 100644 --- a/test/e2e/lifecycle/BUILD +++ b/test/e2e/lifecycle/BUILD @@ -35,6 +35,7 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e/framework/config:go_default_library", "//test/e2e/framework/ginkgowrapper:go_default_library", + "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/lifecycle:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/node:go_default_library", diff --git a/test/e2e/lifecycle/kubelet_security.go b/test/e2e/lifecycle/kubelet_security.go index 3ac07ae1cfa..3f29cd94335 100644 --- a/test/e2e/lifecycle/kubelet_security.go +++ b/test/e2e/lifecycle/kubelet_security.go @@ -28,6 +28,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2enode "k8s.io/kubernetes/test/e2e/framework/node" ) @@ -47,7 +48,7 @@ var _ = SIGDescribe("Ports Security Check [Feature:KubeletSecurity]", func() { // make sure kubelet readonly (10255) and cadvisor (4194) ports are disabled via API server proxy ginkgo.It(fmt.Sprintf("should not be able to proxy to the readonly kubelet port %v using proxy subresource", ports.KubeletReadOnlyPort), func() { - result, err := e2enode.ProxyRequest(f.ClientSet, nodeName, "pods/", ports.KubeletReadOnlyPort) + result, err := e2ekubelet.ProxyRequest(f.ClientSet, nodeName, "pods/", ports.KubeletReadOnlyPort) framework.ExpectNoError(err) var statusCode int @@ -55,7 +56,7 @@ var _ = SIGDescribe("Ports Security Check [Feature:KubeletSecurity]", func() { framework.ExpectNotEqual(statusCode, http.StatusOK) }) ginkgo.It("should not be able to proxy to cadvisor port 4194 using proxy subresource", func() { - result, err := e2enode.ProxyRequest(f.ClientSet, nodeName, "containers/", 4194) + result, err := e2ekubelet.ProxyRequest(f.ClientSet, nodeName, "containers/", 4194) framework.ExpectNoError(err) var statusCode int diff --git a/test/e2e/node/pre_stop.go b/test/e2e/node/pre_stop.go index 6088b7ae6e4..7760de0bc11 100644 --- a/test/e2e/node/pre_stop.go +++ b/test/e2e/node/pre_stop.go @@ -30,8 +30,8 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/test/e2e/framework" + e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" imageutils "k8s.io/kubernetes/test/utils/image" @@ -209,7 +209,7 @@ var _ = SIGDescribe("PreStop", func() { ginkgo.By("verifying the pod running state after graceful termination") result := &v1.PodList{} err = wait.Poll(time.Second*5, time.Second*60, func() (bool, error) { - client, err := e2enode.ProxyRequest(f.ClientSet, pod.Spec.NodeName, "pods", ports.KubeletPort) + client, err := e2ekubelet.ProxyRequest(f.ClientSet, pod.Spec.NodeName, "pods", ports.KubeletPort) framework.ExpectNoError(err, "failed to get the pods of the node") err = client.Into(result) framework.ExpectNoError(err, "failed to parse the pods of the node")