diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 12fbe879f9b..d4a999342ae 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -27,6 +27,8 @@ go_library( "//pkg/api:go_default_library", "//pkg/api/errors:go_default_library", "//pkg/api/unversioned:go_default_library", + "//pkg/apis/componentconfig:go_default_library", + "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/kubelet/api/v1alpha1/stats:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/dockertools:go_default_library", @@ -77,8 +79,6 @@ go_test( "//pkg/api/errors:go_default_library", "//pkg/api/resource:go_default_library", "//pkg/api/unversioned:go_default_library", - "//pkg/apis/componentconfig:go_default_library", - "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/client/cache:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/kubelet/api/v1alpha1/stats:go_default_library", diff --git a/test/e2e_node/dynamic_kubelet_configuration_test.go b/test/e2e_node/dynamic_kubelet_configuration_test.go index 4e70e8a65e2..a088a8682b4 100644 --- a/test/e2e_node/dynamic_kubelet_configuration_test.go +++ b/test/e2e_node/dynamic_kubelet_configuration_test.go @@ -17,20 +17,13 @@ limitations under the License. package e2e_node import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" "time" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/apis/componentconfig" - "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" + "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) // This test is marked [Disruptive] because the Kubelet temporarily goes down as part of of this test. @@ -39,14 +32,9 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube Context("When a configmap called `kubelet-` is added to the `kube-system` namespace", func() { It("The Kubelet on that node should restart to take up the new config", func() { - const ( - restartGap = 40 * time.Second - ) - // Get the current KubeletConfiguration (known to be valid) by // querying the configz endpoint for the current node. - resp := pollConfigz(2*time.Minute, 5*time.Second) - kubeCfg, err := decodeConfigz(resp) + kubeCfg, err := getCurrentKubeletConfig() framework.ExpectNoError(err) glog.Infof("KubeletConfiguration - Initial values: %+v", *kubeCfg) @@ -60,138 +48,15 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube kubeCfg.FileCheckFrequency.Duration = newFileCheckFrequency // Use the new config to create a new kube- configmap in `kube-system` namespace. - _, err = createConfigMap(f, kubeCfg) + // Note: setKubeletConfiguration will return an error if the Kubelet does not present the + // modified configuration via /configz when it comes back up. + err = setKubeletConfiguration(f, kubeCfg) framework.ExpectNoError(err) - // Give the Kubelet time to see that there is new config and restart. If we don't do this, - // the Kubelet will still have the old config when we poll, and the test will fail. - time.Sleep(restartGap) - - // Use configz to get the new config. - resp = pollConfigz(2*time.Minute, 5*time.Second) - kubeCfg, err = decodeConfigz(resp) - framework.ExpectNoError(err) - glog.Infof("KubeletConfiguration - After modification of FileCheckFrequency: %+v", *kubeCfg) - - // We expect to see the new value in the new config. - Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(newFileCheckFrequency)) - // Change the config back to what it originally was. kubeCfg.FileCheckFrequency.Duration = oldFileCheckFrequency - _, err = updateConfigMap(f, kubeCfg) + err = setKubeletConfiguration(f, kubeCfg) framework.ExpectNoError(err) - - // Give the Kubelet time to see that there is new config and restart. If we don't do this, - // the Kubelet will still have the old config when we poll, and the test will fail. - time.Sleep(restartGap) - - // User configz to get the new config. - resp = pollConfigz(2*time.Minute, 5*time.Second) - kubeCfg, err = decodeConfigz(resp) - framework.ExpectNoError(err) - glog.Infof("KubeletConfiguration - After restoration of FileCheckFrequency: %+v", *kubeCfg) - - // We expect to see the original value restored in the new config. - Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(oldFileCheckFrequency)) }) }) }) - -// This function either causes the test to fail, or it returns a status 200 response. -func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response { - endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName) - client := &http.Client{} - req, err := http.NewRequest("GET", endpoint, nil) - framework.ExpectNoError(err) - req.Header.Add("Accept", "application/json") - - var resp *http.Response - Eventually(func() bool { - resp, err = client.Do(req) - if err != nil { - glog.Errorf("Failed to get /configz, retrying. Error: %v", err) - return false - } - if resp.StatusCode != 200 { - glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp) - return false - } - return true - }, timeout, pollInterval).Should(Equal(true)) - return resp -} - -// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type). -func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) { - // This hack because /configz reports the following structure: - // {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}} - type configzWrapper struct { - ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"` - } - - configz := configzWrapper{} - kubeCfg := componentconfig.KubeletConfiguration{} - - contentsBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - err = json.Unmarshal(contentsBytes, &configz) - if err != nil { - return nil, err - } - - api.Scheme.Default(&configz.ComponentConfig) - - err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil) - if err != nil { - return nil, err - } - - return &kubeCfg, nil -} - -// Uses KubeletConfiguration to create a `kubelet-` ConfigMap in the "kube-system" namespace. -func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) { - kubeCfgExt := v1alpha1.KubeletConfiguration{} - api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil) - - bytes, err := json.Marshal(kubeCfgExt) - framework.ExpectNoError(err) - - cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(&api.ConfigMap{ - ObjectMeta: api.ObjectMeta{ - Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName), - }, - Data: map[string]string{ - "kubelet.config": string(bytes), - }, - }) - if err != nil { - return nil, err - } - return cmap, nil -} - -// Similar to createConfigMap, except this updates an existing ConfigMap. -func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) { - kubeCfgExt := v1alpha1.KubeletConfiguration{} - api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil) - - bytes, err := json.Marshal(kubeCfgExt) - framework.ExpectNoError(err) - - cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(&api.ConfigMap{ - ObjectMeta: api.ObjectMeta{ - Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName), - }, - Data: map[string]string{ - "kubelet.config": string(bytes), - }, - }) - if err != nil { - return nil, err - } - return cmap, nil -} diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index b1babcd48bb..352f197554c 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -22,9 +22,21 @@ import ( "fmt" "io/ioutil" "net/http" + "reflect" "strings" + "time" + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/api" + k8serr "k8s.io/kubernetes/pkg/api/errors" + "k8s.io/kubernetes/pkg/apis/componentconfig" + v1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats" + // utilconfig "k8s.io/kubernetes/pkg/util/config" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/gomega" ) // TODO(random-liu): Get this automatically from kubelet flag. @@ -60,3 +72,163 @@ func getNodeSummary() (*stats.Summary, error) { } return &summary, nil } + +// Returns the current KubeletConfiguration +func getCurrentKubeletConfig() (*componentconfig.KubeletConfiguration, error) { + resp := pollConfigz(5*time.Minute, 5*time.Second) + kubeCfg, err := decodeConfigz(resp) + if err != nil { + return nil, err + } + return kubeCfg, nil +} + +// Queries the API server for a Kubelet configuration for the node described by framework.TestContext.NodeName +func getCurrentKubeletConfigMap(f *framework.Framework) (*api.ConfigMap, error) { + return f.ClientSet.Core().ConfigMaps("kube-system").Get(fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName)) +} + +// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart +// with the new configuration. Returns an error if the configuration after waiting 40 seconds +// doesn't match what you attempted to set, or if the dynamic configuration feature is disabled. +func setKubeletConfiguration(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) error { + const ( + restartGap = 30 * time.Second + ) + + // Make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure + cfgz, err := getCurrentKubeletConfig() + if err != nil { + return fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err) + } + if !strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true") { + return fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" + + "Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" + + "For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.") + } + + // Check whether a configmap for KubeletConfiguration already exists + _, err = getCurrentKubeletConfigMap(f) + + if k8serr.IsNotFound(err) { + _, err := createConfigMap(f, kubeCfg) + if err != nil { + return err + } + } else if err != nil { + return err + } else { + // The configmap exists, update it instead of creating it. + _, err := updateConfigMap(f, kubeCfg) + if err != nil { + return err + } + } + + // Wait for the Kubelet to restart. + time.Sleep(restartGap) + + // Retrieve the new config and compare it to the one we attempted to set + newKubeCfg, err := getCurrentKubeletConfig() + if err != nil { + return err + } + + // Return an error if the desired config is not in use by now + if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) { + return fmt.Errorf("either the Kubelet did not restart or it did not present the modified configuration via /configz after restarting.") + } + return nil +} + +// Causes the test to fail, or returns a status 200 response from the /configz endpoint +func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response { + endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName) + client := &http.Client{} + req, err := http.NewRequest("GET", endpoint, nil) + framework.ExpectNoError(err) + req.Header.Add("Accept", "application/json") + + var resp *http.Response + Eventually(func() bool { + resp, err = client.Do(req) + if err != nil { + glog.Errorf("Failed to get /configz, retrying. Error: %v", err) + return false + } + if resp.StatusCode != 200 { + glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp) + return false + } + return true + }, timeout, pollInterval).Should(Equal(true)) + return resp +} + +// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type). +func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) { + // This hack because /configz reports the following structure: + // {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}} + type configzWrapper struct { + ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"` + } + + configz := configzWrapper{} + kubeCfg := componentconfig.KubeletConfiguration{} + + contentsBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(contentsBytes, &configz) + if err != nil { + return nil, err + } + + err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil) + if err != nil { + return nil, err + } + + return &kubeCfg, nil +} + +// Constructs a Kubelet ConfigMap targeting the current node running the node e2e tests +func makeKubeletConfigMap(nodeName string, kubeCfg *componentconfig.KubeletConfiguration) *api.ConfigMap { + kubeCfgExt := v1alpha1.KubeletConfiguration{} + api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil) + + bytes, err := json.Marshal(kubeCfgExt) + framework.ExpectNoError(err) + + cmap := &api.ConfigMap{ + ObjectMeta: api.ObjectMeta{ + Name: fmt.Sprintf("kubelet-%s", nodeName), + }, + Data: map[string]string{ + "kubelet.config": string(bytes), + }, + } + return cmap +} + +// Uses KubeletConfiguration to create a `kubelet-` ConfigMap in the "kube-system" namespace. +func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) { + cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg) + cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(cmap) + if err != nil { + return nil, err + } + return cmap, nil +} + +// Similar to createConfigMap, except this updates an existing ConfigMap. +func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) { + cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg) + cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(cmap) + if err != nil { + return nil, err + } + return cmap, nil +}