diff --git a/test/e2e_node/jenkins/jenkins-ci-ubuntu.properties b/test/e2e_node/jenkins/jenkins-ci-ubuntu.properties index 1e0c76fa14d..14e30d3d7e4 100644 --- a/test/e2e_node/jenkins/jenkins-ci-ubuntu.properties +++ b/test/e2e_node/jenkins/jenkins-ci-ubuntu.properties @@ -6,7 +6,8 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-ubuntu-node CLEANUP=true GINKGO_FLAGS='--skip="\[Flaky\]|\[Serial\]"' -KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' +TEST_ARGS='--feature-gates=KubeletConfigFile=true --generate-kubelet-config-file=true' +KUBELET_ARGS='' TIMEOUT=1h # Use the system spec defined in test/e2e_node/system/specs/gke.yaml. SYSTEM_SPEC_NAME=gke diff --git a/test/e2e_node/jenkins/jenkins-ci.properties b/test/e2e_node/jenkins/jenkins-ci.properties index 9c563b6a050..148f0cb8580 100644 --- a/test/e2e_node/jenkins/jenkins-ci.properties +++ b/test/e2e_node/jenkins/jenkins-ci.properties @@ -4,5 +4,6 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--skip="\[Flaky\]|\[Serial\]"' -KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' +TEST_ARGS='--feature-gates=KubeletConfigFile=true --generate-kubelet-config-file=true' +KUBELET_ARGS='' TIMEOUT=1h diff --git a/test/e2e_node/jenkins/jenkins-flaky.properties b/test/e2e_node/jenkins/jenkins-flaky.properties index 824c1309dcf..4689ebc90e4 100644 --- a/test/e2e_node/jenkins/jenkins-flaky.properties +++ b/test/e2e_node/jenkins/jenkins-flaky.properties @@ -4,8 +4,8 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--focus="\[Flaky\]"' -TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,LocalStorageCapacityIsolation=true,PodPriority=true' -KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' +TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,LocalStorageCapacityIsolation=true,PodPriority=true,KubeletConfigFile=true --generate-kubelet-config-file=true' +KUBELET_ARGS='' PARALLELISM=1 TIMEOUT=3h diff --git a/test/e2e_node/jenkins/jenkins-pull.properties b/test/e2e_node/jenkins/jenkins-pull.properties index 884e45884f1..3db3dbc7d75 100644 --- a/test/e2e_node/jenkins/jenkins-pull.properties +++ b/test/e2e_node/jenkins/jenkins-pull.properties @@ -4,5 +4,6 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-pr-node-e2e CLEANUP=true GINKGO_FLAGS='--skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2' -KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' +TEST_ARGS='--feature-gates=KubeletConfigFile=true --generate-kubelet-config-file=true' +KUBELET_ARGS='' diff --git a/test/e2e_node/jenkins/jenkins-serial-ubuntu.properties b/test/e2e_node/jenkins/jenkins-serial-ubuntu.properties index 5333bb8b037..7043d308f6f 100644 --- a/test/e2e_node/jenkins/jenkins-serial-ubuntu.properties +++ b/test/e2e_node/jenkins/jenkins-serial-ubuntu.properties @@ -6,8 +6,8 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-ubuntu-node-serial CLEANUP=true GINKGO_FLAGS='--focus="\[Serial\]" --skip="\[Flaky\]|\[Benchmark\]"' -TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' -KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' +TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,KubeletConfigFile=true --generate-kubelet-config-file=true' +KUBELET_ARGS='' PARALLELISM=1 TIMEOUT=3h # Use the system spec defined at test/e2e_node/system/specs/gke.yaml. diff --git a/test/e2e_node/jenkins/jenkins-serial.properties b/test/e2e_node/jenkins/jenkins-serial.properties index 31bded6deb2..9cc243ce193 100644 --- a/test/e2e_node/jenkins/jenkins-serial.properties +++ b/test/e2e_node/jenkins/jenkins-serial.properties @@ -4,7 +4,7 @@ GCE_ZONE=us-west1-b GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--focus="\[Serial\]" --skip="\[Flaky\]|\[Benchmark\]"' -TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' -KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' +TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,KubeletConfigFile=true --generate-kubelet-config-file=true' +KUBELET_ARGS='' PARALLELISM=1 TIMEOUT=3h diff --git a/test/e2e_node/services/BUILD b/test/e2e_node/services/BUILD index b0852a5828a..e0ab73e37d7 100644 --- a/test/e2e_node/services/BUILD +++ b/test/e2e_node/services/BUILD @@ -22,9 +22,13 @@ go_library( deps = [ "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", + "//cmd/kubelet/app/options:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/controller/namespace:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e_node/builder:go_default_library", "//vendor/github.com/coreos/etcd/etcdserver:go_default_library", @@ -33,8 +37,12 @@ go_library( "//vendor/github.com/coreos/etcd/pkg/types:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/kardianos/osext:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 42be2558c87..aa803859a39 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -25,11 +25,20 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/golang/glog" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilflag "k8s.io/apiserver/pkg/util/flag" + "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/builder" ) @@ -62,11 +71,14 @@ func (a *args) Set(value string) error { var kubeletArgs args var kubeletContainerized bool var hyperkubeImage string +var genKubeletConfigFile bool func init() { flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.") flag.BoolVar(&kubeletContainerized, "kubelet-containerized", false, "Run kubelet in a docker container") flag.StringVar(&hyperkubeImage, "hyperkube-image", "", "Docker image with containerized kubelet") + flag.BoolVar(&genKubeletConfigFile, "generate-kubelet-config-file", false, "The test runner will generate a Kubelet config file containing test defaults instead of passing default flags to the Kubelet. "+ + "If you use this test framework feature, ensure that the KubeletConfigFile feature gate is enabled.") } // RunKubelet starts kubelet and waits for termination signal. Once receives the @@ -112,6 +124,12 @@ func (e *E2EServices) startKubelet() (*server, error) { return nil, err } + // KubeletConfiguration file path + kubeletConfigPath, err := kubeletConfigCWDPath() + if err != nil { + return nil, err + } + // Create pod manifest path manifestPath, err := createPodManifestDirectory() if err != nil { @@ -122,6 +140,58 @@ func (e *E2EServices) startKubelet() (*server, error) { if err != nil { return nil, err } + + // PLEASE NOTE: If you set new KubeletConfiguration values or stop setting values here, + // you must also update the flag names in kubeletConfigFlags! + kubeletConfigFlags := []string{} + + // set up the default kubeletconfiguration + kc, err := options.NewKubeletConfiguration() + if err != nil { + return nil, err + } + + kc.CgroupRoot = "/" + kubeletConfigFlags = append(kubeletConfigFlags, "cgroup-root") + + kc.VolumeStatsAggPeriod = metav1.Duration{Duration: 10 * time.Second} // Aggregate volumes frequently so tests don't need to wait as long + kubeletConfigFlags = append(kubeletConfigFlags, "volume-stats-agg-period") + + kc.AllowPrivileged = true + kubeletConfigFlags = append(kubeletConfigFlags, "allow-privileged") + + kc.SerializeImagePulls = false + kubeletConfigFlags = append(kubeletConfigFlags, "serialize-image-pulls") + + kc.PodManifestPath = manifestPath + kubeletConfigFlags = append(kubeletConfigFlags, "pod-manifest-path") + + kc.FileCheckFrequency = metav1.Duration{Duration: 10 * time.Second} // Check file frequently so tests won't wait too long + kubeletConfigFlags = append(kubeletConfigFlags, "file-check-frequency") + + // Assign a fixed CIDR to the node because there is no node controller. + // Note: this MUST be in sync with with the IP in + // - cluster/gce/config-test.sh and + // - test/e2e_node/conformance/run_test.sh. + kc.PodCIDR = "10.100.0.0/24" + kubeletConfigFlags = append(kubeletConfigFlags, "pod-cidr") + + kc.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 30 * time.Second} + kubeletConfigFlags = append(kubeletConfigFlags, "eviction-pressure-transition-period") + + kc.EvictionHard = map[string]string{ + "memory.available": "250Mi", + "nodefs.available": "10%", + "nodefs.inodesFree": "5%", + } + kubeletConfigFlags = append(kubeletConfigFlags, "eviction-hard") + + kc.EvictionMinimumReclaim = map[string]string{ + "nodefs.available": "5%", + "nodefs.inodesFree": "5%", + } + kubeletConfigFlags = append(kubeletConfigFlags, "eviction-minimum-reclaim") + var killCommand, restartCommand *exec.Cmd var isSystemd bool // Apply default kubelet flags. @@ -151,12 +221,22 @@ func (e *E2EServices) startKubelet() (*server, error) { "-v", "/var/lib/kubelet:/var/lib/kubelet:rw,rslave", "-v", "/var/log:/var/log", "-v", manifestPath+":"+manifestPath+":rw", - hyperkubeImage, "/hyperkube", "kubelet", - "--containerized", ) + + // if we will generate a kubelet config file, we need to mount that path into the container too + if genKubeletConfigFile { + cmdArgs = append(cmdArgs, "-v", filepath.Dir(kubeletConfigPath)+":"+filepath.Dir(kubeletConfigPath)+":ro") + } + + cmdArgs = append(cmdArgs, hyperkubeImage, "/hyperkube", "kubelet", "--containerized") kubeconfigPath = "/etc/kubernetes/kubeconfig" } else { - cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--slice=runtime.slice", "--remain-after-exit", builder.GetKubeletServerBin()) + cmdArgs = append(cmdArgs, + systemdRun, + "--unit="+unitName, + "--slice=runtime.slice", + "--remain-after-exit", + builder.GetKubeletServerBin()) } killCommand = exec.Command("systemctl", "kill", unitName) @@ -165,41 +245,24 @@ func (e *E2EServices) startKubelet() (*server, error) { Name: "kubelet.log", JournalctlCommand: []string{"-u", unitName}, } - cmdArgs = append(cmdArgs, - "--kubelet-cgroups=/kubelet.slice", - "--cgroup-root=/", - ) + + kc.KubeletCgroups = "/kubelet.slice" + kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups") } else { cmdArgs = append(cmdArgs, builder.GetKubeletServerBin()) - cmdArgs = append(cmdArgs, - // TODO(random-liu): Get rid of this docker specific thing. - "--runtime-cgroups=/docker-daemon", - "--kubelet-cgroups=/kubelet", - "--cgroup-root=/", - "--system-cgroups=/system", - ) + // TODO(random-liu): Get rid of this docker specific thing. + cmdArgs = append(cmdArgs, "--runtime-cgroups=/docker-daemon") + + kc.KubeletCgroups = "/kubelet" + kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups") + + kc.SystemCgroups = "/system" + kubeletConfigFlags = append(kubeletConfigFlags, "system-cgroups") } cmdArgs = append(cmdArgs, "--kubeconfig", kubeconfigPath, - "--address", "0.0.0.0", - "--port", kubeletPort, - "--read-only-port", kubeletReadOnlyPort, "--root-dir", KubeletRootDirectory, - "--volume-stats-agg-period", "10s", // Aggregate volumes frequently so tests don't need to wait as long - "--allow-privileged", "true", - "--serialize-image-pulls", "false", - "--pod-manifest-path", manifestPath, - "--file-check-frequency", "10s", // Check file frequently so tests won't wait too long "--docker-disable-shared-pid=false", - // Assign a fixed CIDR to the node because there is no node controller. - // - // Note: this MUST be in sync with with the IP in - // - cluster/gce/config-test.sh and - // - test/e2e_node/conformance/run_test.sh. - "--pod-cidr", "10.100.0.0/24", - "--eviction-pressure-transition-period", "30s", - "--eviction-hard", "memory.available<250Mi,nodefs.available<10%,nodefs.inodesFree<5%", // The hard eviction thresholds. - "--eviction-minimum-reclaim", "nodefs.available=5%,nodefs.inodesFree=5%", // The minimum reclaimed resources after eviction. "--v", LOG_VERBOSITY_LEVEL, "--logtostderr", ) @@ -207,6 +270,7 @@ func (e *E2EServices) startKubelet() (*server, error) { // by kubelet-flags. if framework.TestContext.FeatureGates != "" { cmdArgs = append(cmdArgs, "--feature-gates", framework.TestContext.FeatureGates) + utilflag.NewMapStringBool(&kc.FeatureGates).Set(framework.TestContext.FeatureGates) } if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { @@ -239,6 +303,18 @@ func (e *E2EServices) startKubelet() (*server, error) { cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName) } + // Write config file or flags, depending on whether --generate-kubelet-config-file was provided + if genKubeletConfigFile { + if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil { + return nil, err + } + // add the flag to load config from a file + cmdArgs = append(cmdArgs, "--config", kubeletConfigPath) + } else { + // generate command line flags from the default config, since --generate-kubelet-config-file was not provided + addKubeletConfigFlags(&cmdArgs, kc, kubeletConfigFlags) + } + // Override the default kubelet flags. cmdArgs = append(cmdArgs, kubeletArgs...) @@ -260,6 +336,61 @@ func (e *E2EServices) startKubelet() (*server, error) { return server, server.start() } +// addKubeletConfigFlags adds the flags we care about from the provided kubelet configuration object +func addKubeletConfigFlags(cmdArgs *[]string, kc *kubeletconfig.KubeletConfiguration, flags []string) { + fs := pflag.NewFlagSet("kubelet", pflag.ExitOnError) + options.AddKubeletConfigFlags(fs, kc) + for _, name := range flags { + *cmdArgs = append(*cmdArgs, "--"+name, fs.Lookup(name).Value.String()) + } +} + +// writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename +func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error { + // extract the KubeletConfiguration and convert to versioned + versioned := &v1alpha1.KubeletConfiguration{} + scheme, _, err := scheme.NewSchemeAndCodecs() + if err != nil { + return err + } + if err := scheme.Convert(internal, versioned, nil); err != nil { + return err + } + // encode + encoder, err := newKubeletConfigJSONEncoder() + if err != nil { + return err + } + data, err := runtime.Encode(encoder, versioned) + if err != nil { + return err + } + // create the directory, if it does not exist + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + // write the file + if err := ioutil.WriteFile(path, data, 0755); err != nil { + return err + } + return nil +} + +func newKubeletConfigJSONEncoder() (runtime.Encoder, error) { + _, kubeletCodecs, err := scheme.NewSchemeAndCodecs() + if err != nil { + return nil, err + } + + mediaType := "application/json" + info, ok := runtime.SerializerInfoForMediaType(kubeletCodecs.SupportedMediaTypes(), mediaType) + if !ok { + return nil, fmt.Errorf("unsupported media type %q", mediaType) + } + return kubeletCodecs.EncoderForVersion(info.Serializer, v1alpha1.SchemeGroupVersion), nil +} + // createPodManifestDirectory creates pod manifest directory. func createPodManifestDirectory() (string, error) { cwd, err := os.Getwd() @@ -316,6 +447,15 @@ func kubeconfigCWDPath() (string, error) { return filepath.Join(cwd, "kubeconfig"), nil } +func kubeletConfigCWDPath() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current working directory: %v", err) + } + // DO NOT name this file "kubelet" - you will overwrite the the kubelet binary and be very confused :) + return filepath.Join(cwd, "kubelet-config"), nil +} + // like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig // returns a fully-qualified path to the kubeconfig file func createKubeconfigCWD() (string, error) {