diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 4b18a8f9c1a..acdf33bfc07 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -9,6 +9,7 @@ go_library( "flake_reporting_util.go", "framework.go", "google_compute.go", + "kubeletconfig.go", "log.go", "log_size_monitoring.go", "nodes_util.go", @@ -64,6 +65,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", + "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//test/e2e/framework/auth:go_default_library", "//test/e2e/framework/ginkgowrapper:go_default_library", "//test/e2e/framework/kubectl:go_default_library", diff --git a/test/e2e/framework/kubeletconfig.go b/test/e2e/framework/kubeletconfig.go new file mode 100644 index 00000000000..01067062e43 --- /dev/null +++ b/test/e2e/framework/kubeletconfig.go @@ -0,0 +1,131 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "regexp" + "strconv" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" +) + +// GetCurrentKubeletConfig fetches the current Kubelet Config for the given node +func GetCurrentKubeletConfig(nodeName, namespace string, useProxy bool) (*kubeletconfig.KubeletConfiguration, error) { + resp := pollConfigz(5*time.Minute, 5*time.Second, nodeName, namespace, useProxy) + if resp == nil { + return nil, fmt.Errorf("failed to fetch /configz from %q", nodeName) + } + kubeCfg, err := decodeConfigz(resp) + if err != nil { + return nil, err + } + return kubeCfg, nil +} + +// returns a status 200 response from the /configz endpoint or nil if fails +func pollConfigz(timeout time.Duration, pollInterval time.Duration, nodeName, namespace string, useProxy bool) *http.Response { + endpoint := "" + if useProxy { + // start local proxy, so we can send graceful deletion over query string, rather than body parameter + Logf("Opening proxy to cluster") + tk := e2ekubectl.NewTestKubeconfig(TestContext.CertDir, TestContext.Host, TestContext.KubeConfig, TestContext.KubeContext, TestContext.KubectlPath, namespace) + cmd := tk.KubectlCmd("proxy", "-p", "0") + stdout, stderr, err := StartCmdAndStreamOutput(cmd) + ExpectNoError(err) + defer stdout.Close() + defer stderr.Close() + defer TryKill(cmd) + + buf := make([]byte, 128) + var n int + n, err = stdout.Read(buf) + ExpectNoError(err) + output := string(buf[:n]) + proxyRegexp := regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)") + match := proxyRegexp.FindStringSubmatch(output) + ExpectEqual(len(match), 2) + port, err := strconv.Atoi(match[1]) + ExpectNoError(err) + Logf("http requesting node kubelet /configz") + endpoint = fmt.Sprintf("http://127.0.0.1:%d/api/v1/nodes/%s/proxy/configz", port, nodeName) + } else { + endpoint = fmt.Sprintf("http://127.0.0.1:8080/api/v1/nodes/%s/proxy/configz", TestContext.NodeName) + } + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + req, err := http.NewRequest("GET", endpoint, nil) + ExpectNoError(err) + req.Header.Add("Accept", "application/json") + + var resp *http.Response + wait.PollImmediate(pollInterval, timeout, func() (bool, error) { + resp, err = client.Do(req) + if err != nil { + Logf("Failed to get /configz, retrying. Error: %v", err) + return false, nil + } + if resp.StatusCode != 200 { + Logf("/configz response status not 200, retrying. Response was: %+v", resp) + return false, nil + } + + return true, nil + }) + return resp +} + +// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type). +func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) { + // This hack because /configz reports the following structure: + // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} + type configzWrapper struct { + ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + } + + configz := configzWrapper{} + kubeCfg := kubeletconfig.KubeletConfiguration{} + + contentsBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(contentsBytes, &configz) + if err != nil { + return nil, err + } + + err = scheme.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil) + if err != nil { + return nil, err + } + + return &kubeCfg, nil +} diff --git a/test/e2e/windows/BUILD b/test/e2e/windows/BUILD index 4bc0683042e..d8cde73d6c7 100644 --- a/test/e2e/windows/BUILD +++ b/test/e2e/windows/BUILD @@ -21,7 +21,6 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e/windows", visibility = ["//visibility:public"], deps = [ - "//pkg/kubelet/apis/config:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -31,11 +30,8 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//test/e2e/framework:go_default_library", - "//test/e2e/framework/kubectl:go_default_library", "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/node:go_default_library", diff --git a/test/e2e/windows/memory_limits.go b/test/e2e/windows/memory_limits.go index 46df390cc08..ed43945d4e8 100644 --- a/test/e2e/windows/memory_limits.go +++ b/test/e2e/windows/memory_limits.go @@ -18,12 +18,7 @@ package windows import ( "context" - "crypto/tls" - "encoding/json" "fmt" - "io/ioutil" - "net/http" - "regexp" "strconv" "time" @@ -32,11 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/kubernetes/scheme" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework" - e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" imageutils "k8s.io/kubernetes/test/utils/image" @@ -197,7 +188,7 @@ func getNodeMemory(f *framework.Framework) nodeMemory { nodeName := nodeList.Items[0].ObjectMeta.Name - kubeletConfig, err := getCurrentKubeletConfig(nodeName, f.Namespace.Name) + kubeletConfig, err := framework.GetCurrentKubeletConfig(nodeName, f.Namespace.Name, true) framework.ExpectNoError(err) systemReserve, err := resource.ParseQuantity(kubeletConfig.SystemReserved["memory"]) @@ -250,91 +241,3 @@ func getTotalAllocatableMemory(f *framework.Framework) *resource.Quantity { return totalAllocatable } - -// getCurrentKubeletConfig modified from test/e2e_node/util.go -func getCurrentKubeletConfig(nodeName, namespace string) (*kubeletconfig.KubeletConfiguration, error) { - - resp := pollConfigz(5*time.Minute, 5*time.Second, nodeName, namespace) - kubeCfg, err := decodeConfigz(resp) - if err != nil { - return nil, err - } - return kubeCfg, nil -} - -// Causes the test to fail, or returns a status 200 response from the /configz endpoint -func pollConfigz(timeout time.Duration, pollInterval time.Duration, nodeName, namespace string) *http.Response { - // start local proxy, so we can send graceful deletion over query string, rather than body parameter - ginkgo.By("Opening proxy to cluster") - tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, namespace) - cmd := tk.KubectlCmd("proxy", "-p", "0") - stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd) - framework.ExpectNoError(err) - defer stdout.Close() - defer stderr.Close() - defer framework.TryKill(cmd) - buf := make([]byte, 128) - var n int - n, err = stdout.Read(buf) - framework.ExpectNoError(err) - output := string(buf[:n]) - proxyRegexp := regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)") - match := proxyRegexp.FindStringSubmatch(output) - framework.ExpectEqual(len(match), 2) - port, err := strconv.Atoi(match[1]) - framework.ExpectNoError(err) - ginkgo.By("http requesting node kubelet /configz") - endpoint := fmt.Sprintf("http://127.0.0.1:%d/api/v1/nodes/%s/proxy/configz", port, nodeName) - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - req, err := http.NewRequest("GET", endpoint, nil) - framework.ExpectNoError(err) - req.Header.Add("Accept", "application/json") - - var resp *http.Response - gomega.Eventually(func() bool { - resp, err = client.Do(req) - if err != nil { - framework.Logf("Failed to get /configz, retrying. Error: %v", err) - return false - } - if resp.StatusCode != 200 { - framework.Logf("/configz response status not 200, retrying. Response was: %+v", resp) - return false - } - - return true - }, timeout, pollInterval).Should(gomega.Equal(true)) - return resp -} - -// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type). -func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) { - // This hack because /configz reports the following structure: - // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} - type configzWrapper struct { - ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` - } - - configz := configzWrapper{} - kubeCfg := kubeletconfig.KubeletConfiguration{} - - contentsBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - err = json.Unmarshal(contentsBytes, &configz) - if err != nil { - return nil, err - } - - err = scheme.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil) - if err != nil { - return nil, err - } - - return &kubeCfg, nil -} diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 29d740a2a97..eacd75b08d6 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -25,7 +25,6 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/kubelet/apis/config:go_default_library", - "//pkg/kubelet/apis/config/scheme:go_default_library", "//pkg/kubelet/apis/podresources:go_default_library", "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index f02ad16201b..c57a417c86b 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -40,7 +40,6 @@ import ( kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" - kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" "k8s.io/kubernetes/pkg/kubelet/apis/podresources" kubeletpodresourcesv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" kubeletstatsv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" @@ -123,12 +122,8 @@ func getNodeDevices() (*kubeletpodresourcesv1alpha1.ListPodResourcesResponse, er // Returns the current KubeletConfiguration func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) { - resp := pollConfigz(5*time.Minute, 5*time.Second) - kubeCfg, err := decodeConfigz(resp) - if err != nil { - return nil, err - } - return kubeCfg, nil + // namespace only relevant if useProxy==true, so we don't bother + return framework.GetCurrentKubeletConfig(framework.TestContext.NodeName, "", false) } // Must be called within a Context. Allows the function to modify the KubeletConfiguration during the BeforeEach of the context. @@ -255,64 +250,6 @@ func setNodeConfigSource(f *framework.Framework, source *v1.NodeConfigSource) er return nil } -// Causes the test to fail, or returns a status 200 response from the /configz endpoint -func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response { - endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/nodes/%s/proxy/configz", framework.TestContext.NodeName) - client := &http.Client{} - req, err := http.NewRequest("GET", endpoint, nil) - framework.ExpectNoError(err) - req.Header.Add("Accept", "application/json") - - var resp *http.Response - gomega.Eventually(func() bool { - resp, err = client.Do(req) - if err != nil { - klog.Errorf("Failed to get /configz, retrying. Error: %v", err) - return false - } - if resp.StatusCode != 200 { - klog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp) - return false - } - return true - }, timeout, pollInterval).Should(gomega.Equal(true)) - return resp -} - -// Decodes the http response from /configz and returns a kubeletconfig.KubeletConfiguration (internal type). -func decodeConfigz(resp *http.Response) (*kubeletconfig.KubeletConfiguration, error) { - // This hack because /configz reports the following structure: - // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} - type configzWrapper struct { - ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` - } - - scheme, _, err := kubeletscheme.NewSchemeAndCodecs() - if err != nil { - return nil, err - } - - configz := configzWrapper{} - kubeCfg := kubeletconfig.KubeletConfiguration{} - - contentsBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - err = json.Unmarshal(contentsBytes, &configz) - if err != nil { - return nil, err - } - - err = scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil) - if err != nil { - return nil, err - } - - return &kubeCfg, nil -} - // creates a configmap containing kubeCfg in kube-system namespace func createConfigMap(f *framework.Framework, internalKC *kubeletconfig.KubeletConfiguration) (*v1.ConfigMap, error) { cmap := newKubeletConfigMap("testcfg", internalKC)