github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-11-03 23:40:03 +00:00
Code Issues Projects Releases Wiki Activity

Add dynamic kubelet configuration utilities to node e2e tests

Browse Source 
Also modify dynamic kubelet configuration test to rely on new utility functions
This commit is contained in:
Michael Taufen
2016-09-12 15:11:12 -07:00
parent 85fe0f1aad
commit 5190a7d72d
3 changed files with 180 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/e2e_node/BUILD
View File

@@ -27,6 +27,8 @@ go_library(
"//pkg/api:go_default_library",
"//pkg/api/errors:go_default_library",
"//pkg/api/unversioned:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/dockertools:go_default_library",
@@ -77,8 +79,6 @@ go_test(
"//pkg/api/errors:go_default_library",
"//pkg/api/resource:go_default_library",
"//pkg/api/unversioned:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/client/cache:go_default_library",
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",

147
test/e2e_node/dynamic_kubelet_configuration_test.go
View File

@@ -17,20 +17,13 @@ limitations under the License.
package e2e_node
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
// This test is marked [Disruptive] because the Kubelet temporarily goes down as part of of this test.
@@ -39,14 +32,9 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube
Context("When a configmap called `kubelet-<node-name>` is added to the `kube-system` namespace", func() {
It("The Kubelet on that node should restart to take up the new config", func() {
const (
restartGap = 40 * time.Second
)
// Get the current KubeletConfiguration (known to be valid) by
// querying the configz endpoint for the current node.
resp := pollConfigz(2*time.Minute, 5*time.Second)
kubeCfg, err := decodeConfigz(resp)
kubeCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
glog.Infof("KubeletConfiguration - Initial values: %+v", *kubeCfg)
@@ -60,138 +48,15 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube
kubeCfg.FileCheckFrequency.Duration = newFileCheckFrequency
// Use the new config to create a new kube-<node-name> configmap in `kube-system` namespace.
_, err = createConfigMap(f, kubeCfg)
// Note: setKubeletConfiguration will return an error if the Kubelet does not present the
// modified configuration via /configz when it comes back up.
err = setKubeletConfiguration(f, kubeCfg)
framework.ExpectNoError(err)
// Give the Kubelet time to see that there is new config and restart. If we don't do this,
// the Kubelet will still have the old config when we poll, and the test will fail.
time.Sleep(restartGap)
// Use configz to get the new config.
resp = pollConfigz(2*time.Minute, 5*time.Second)
kubeCfg, err = decodeConfigz(resp)
framework.ExpectNoError(err)
glog.Infof("KubeletConfiguration - After modification of FileCheckFrequency: %+v", *kubeCfg)
// We expect to see the new value in the new config.
Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(newFileCheckFrequency))
// Change the config back to what it originally was.
kubeCfg.FileCheckFrequency.Duration = oldFileCheckFrequency
_, err = updateConfigMap(f, kubeCfg)
err = setKubeletConfiguration(f, kubeCfg)
framework.ExpectNoError(err)
// Give the Kubelet time to see that there is new config and restart. If we don't do this,
// the Kubelet will still have the old config when we poll, and the test will fail.
time.Sleep(restartGap)
// User configz to get the new config.
resp = pollConfigz(2*time.Minute, 5*time.Second)
kubeCfg, err = decodeConfigz(resp)
framework.ExpectNoError(err)
glog.Infof("KubeletConfiguration - After restoration of FileCheckFrequency: %+v", *kubeCfg)
// We expect to see the original value restored in the new config.
Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(oldFileCheckFrequency))
})
})
})
// This function either causes the test to fail, or it returns a status 200 response.
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
client := &http.Client{}
req, err := http.NewRequest("GET", endpoint, nil)
framework.ExpectNoError(err)
req.Header.Add("Accept", "application/json")
var resp *http.Response
Eventually(func() bool {
resp, err = client.Do(req)
if err != nil {
glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
return false
}
if resp.StatusCode != 200 {
glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
return false
}
return true
}, timeout, pollInterval).Should(Equal(true))
return resp
}
// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type).
func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) {
// This hack because /configz reports the following structure:
// {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}}
type configzWrapper struct {
ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"`
}
configz := configzWrapper{}
kubeCfg := componentconfig.KubeletConfiguration{}
contentsBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(contentsBytes, &configz)
if err != nil {
return nil, err
}
api.Scheme.Default(&configz.ComponentConfig)
err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
if err != nil {
return nil, err
}
return &kubeCfg, nil
}
// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
kubeCfgExt := v1alpha1.KubeletConfiguration{}
api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
bytes, err := json.Marshal(kubeCfgExt)
framework.ExpectNoError(err)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(&api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName),
},
Data: map[string]string{
"kubelet.config": string(bytes),
},
})
if err != nil {
return nil, err
}
return cmap, nil
}
// Similar to createConfigMap, except this updates an existing ConfigMap.
func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
kubeCfgExt := v1alpha1.KubeletConfiguration{}
api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
bytes, err := json.Marshal(kubeCfgExt)
framework.ExpectNoError(err)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(&api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName),
},
Data: map[string]string{
"kubelet.config": string(bytes),
},
})
if err != nil {
return nil, err
}
return cmap, nil
}

172
test/e2e_node/util.go
View File

@@ -22,9 +22,21 @@ import (
"fmt"
"io/ioutil"
"net/http"
"reflect"
"strings"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
k8serr "k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/apis/componentconfig"
v1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
// utilconfig "k8s.io/kubernetes/pkg/util/config"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/gomega"
)
// TODO(random-liu): Get this automatically from kubelet flag.
@@ -60,3 +72,163 @@ func getNodeSummary() (*stats.Summary, error) {
}
return &summary, nil
}
// Returns the current KubeletConfiguration
func getCurrentKubeletConfig() (*componentconfig.KubeletConfiguration, error) {
resp := pollConfigz(5*time.Minute, 5*time.Second)
kubeCfg, err := decodeConfigz(resp)
if err != nil {
return nil, err
}
return kubeCfg, nil
}
// Queries the API server for a Kubelet configuration for the node described by framework.TestContext.NodeName
func getCurrentKubeletConfigMap(f *framework.Framework) (*api.ConfigMap, error) {
return f.ClientSet.Core().ConfigMaps("kube-system").Get(fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName))
}
// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
// with the new configuration. Returns an error if the configuration after waiting 40 seconds
// doesn't match what you attempted to set, or if the dynamic configuration feature is disabled.
func setKubeletConfiguration(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) error {
const (
restartGap = 30 * time.Second
)
// Make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
cfgz, err := getCurrentKubeletConfig()
if err != nil {
return fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
}
if !strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true") {
return fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
}
// Check whether a configmap for KubeletConfiguration already exists
_, err = getCurrentKubeletConfigMap(f)
if k8serr.IsNotFound(err) {
_, err := createConfigMap(f, kubeCfg)
if err != nil {
return err
}
} else if err != nil {
return err
} else {
// The configmap exists, update it instead of creating it.
_, err := updateConfigMap(f, kubeCfg)
if err != nil {
return err
}
}
// Wait for the Kubelet to restart.
time.Sleep(restartGap)
// Retrieve the new config and compare it to the one we attempted to set
newKubeCfg, err := getCurrentKubeletConfig()
if err != nil {
return err
}
// Return an error if the desired config is not in use by now
if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
return fmt.Errorf("either the Kubelet did not restart or it did not present the modified configuration via /configz after restarting.")
}
return nil
}
// Causes the test to fail, or returns a status 200 response from the /configz endpoint
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
client := &http.Client{}
req, err := http.NewRequest("GET", endpoint, nil)
framework.ExpectNoError(err)
req.Header.Add("Accept", "application/json")
var resp *http.Response
Eventually(func() bool {
resp, err = client.Do(req)
if err != nil {
glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
return false
}
if resp.StatusCode != 200 {
glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
return false
}
return true
}, timeout, pollInterval).Should(Equal(true))
return resp
}
// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type).
func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) {
// This hack because /configz reports the following structure:
// {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}}
type configzWrapper struct {
ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"`
}
configz := configzWrapper{}
kubeCfg := componentconfig.KubeletConfiguration{}
contentsBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(contentsBytes, &configz)
if err != nil {
return nil, err
}
err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
if err != nil {
return nil, err
}
return &kubeCfg, nil
}
// Constructs a Kubelet ConfigMap targeting the current node running the node e2e tests
func makeKubeletConfigMap(nodeName string, kubeCfg *componentconfig.KubeletConfiguration) *api.ConfigMap {
kubeCfgExt := v1alpha1.KubeletConfiguration{}
api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
bytes, err := json.Marshal(kubeCfgExt)
framework.ExpectNoError(err)
cmap := &api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("kubelet-%s", nodeName),
},
Data: map[string]string{
"kubelet.config": string(bytes),
},
}
return cmap
}
// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(cmap)
if err != nil {
return nil, err
}
return cmap, nil
}
// Similar to createConfigMap, except this updates an existing ConfigMap.
func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(cmap)
if err != nil {
return nil, err
}
return cmap, nil
}