e2e_node: allow customizing the base kubeletconfig

This commit forces Kubelet Configuration files to always be generated
and when possible will use the kubeletconfig file that has been provided
by the test orchestrator

hack/make-rules/test-e2e-node.sh

@@ -49,6 +49,7 @@ extra_envs=${EXTRA_ENVS:-}
 runtime_config=${RUNTIME_CONFIG:-}
 ssh_user=${SSH_USER:-"${USER}"}
 ssh_key=${SSH_KEY:-}
+kubelet_config_file=${KUBELET_CONFIG_FILE:-""}
 
 # Parse the flags to pass to ginkgo
 ginkgoflags=""
@@ -164,6 +165,8 @@ if [ "${remote}" = true ] ; then
   echo "Ginkgo Flags: ${ginkgoflags}"
   echo "Instance Metadata: ${metadata}"
   echo "Image Config File: ${image_config_file}"
+  echo "Kubelet Config File: ${kubelet_config_file}"
+
   # Invoke the runner
   go run test/e2e_node/runner/remote/run_remote.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
     --zone="${zone}" --project="${project}" --gubernator="${gubernator}" \
@@ -174,7 +177,7 @@ if [ "${remote}" = true ] ; then
     --image-config-file="${image_config_file}" --system-spec-name="${system_spec_name}" \
     --runtime-config="${runtime_config}" --preemptible-instances="${preemptible_instances}" \
     --ssh-user="${ssh_user}" --ssh-key="${ssh_key}" --image-config-dir="${image_config_dir}" \
-    --extra-envs="${extra_envs}" --test-suite="${test_suite}" \
+    --extra-envs="${extra_envs}" --kubelet-config-file="${kubelet_config_file}"  --test-suite="${test_suite}" \
     "${timeout_arg}" \
     2>&1 | tee -i "${artifacts}/build-log.txt"
   exit $?

test/e2e_node/remote/remote.go

@@ -65,11 +65,7 @@ func copyKubeletConfigIfExists(kubeletConfigFile, dstDir string) error {
 	defer destination.Close()
 
 	_, err = io.Copy(destination, source)
-	if err != nil {
+	return err
-		return err
-	}
-
-	return os.Chmod(dst, 0x644)
 }
 
 // CreateTestArchive creates the archive package for the node e2e test.

test/e2e_node/runner/remote/run_remote.go

@@ -405,7 +405,7 @@ func callGubernator(gubernator bool) {
 }
 
 func (a *Archive) getArchive() (string, error) {
-	a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName) })
+	a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName, *kubeletConfigFile) })
 	return a.path, a.err
 }
 

test/e2e_node/services/kubelet.go

@@ -26,7 +26,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/spf13/pflag"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -34,9 +33,12 @@ import (
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
+	"k8s.io/kubernetes/pkg/cluster/ports"
 	"k8s.io/kubernetes/pkg/features"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles"
 	kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
+	utilfs "k8s.io/kubernetes/pkg/util/filesystem"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e_node/builder"
 	"k8s.io/kubernetes/test/e2e_node/remote"
@@ -62,11 +64,9 @@ func (a *args) Set(value string) error {
 
 // kubeletArgs is the override kubelet args specified by the test runner.
 var kubeletArgs args
-var genKubeletConfigFile bool
 
 func init() {
 	flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.")
-	flag.BoolVar(&genKubeletConfigFile, "generate-kubelet-config-file", true, "The test runner will generate a Kubelet config file containing test defaults instead of passing default flags to the Kubelet.")
 }
 
 // RunKubelet starts kubelet and waits for termination signal. Once receives the
@@ -94,6 +94,35 @@ const (
 	kubeletHealthCheckURL = "http://127.0.0.1:" + kubeletReadOnlyPort + "/healthz"
 )
 
+// TODO(endocrimes): Refactor to take a path to the kubeletconfig
+func baseKubeConfiguration() (*kubeletconfig.KubeletConfiguration, error) {
+	cwd, _ := os.Getwd()
+	cfgPath, err := filepath.Abs(filepath.Join(cwd, "kubeletconfig.yaml"))
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = os.Stat(cfgPath)
+	if err != nil {
+		// If the kubeletconfig exists, but for some reason we can't read it, then
+		// return an error to avoid silently skipping it.
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+
+		// If the kubeletconfig file doesn't exist, then use a default configuration
+		// as the base.
+		return options.NewKubeletConfiguration()
+	}
+
+	loader, err := configfiles.NewFsLoader(&utilfs.DefaultFs{}, cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return loader.Load()
+}
+
 // startKubelet starts the Kubelet in a separate process or returns an error
 // if the Kubelet fails to start.
 func (e *E2EServices) startKubelet() (*server, error) {
@@ -127,53 +156,54 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		return nil, err
 	}
 
-	// PLEASE NOTE: If you set new KubeletConfiguration values or stop setting values here,
+	kc, err := baseKubeConfiguration()
-	// you must also update the flag names in kubeletConfigFlags!
-	kubeletConfigFlags := []string{}
-
-	// set up the default kubeletconfiguration
-	kc, err := options.NewKubeletConfiguration()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to load base kubelet configuration: %v", err)
 	}
 
+	// Apply overrides to allow access to the Kubelet API from the test suite.
+	// These are insecure and should generally not be used outside of test infra.
+
+	// --anonymous-auth
+	kc.Authentication.Anonymous.Enabled = true
+	// --authentication-token-webhook
+	kc.Authentication.Webhook.Enabled = false
+	// --authorization-mode
+	kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow
+	// --read-only-port
+	kc.ReadOnlyPort = ports.KubeletReadOnlyPort
+
+	// Setup general overrides for the kubelet.
+	// TODO(endocrimes): Move the following to a `default` configuration file
+
 	kc.CgroupRoot = "/"
-	kubeletConfigFlags = append(kubeletConfigFlags, "cgroup-root")
 
 	kc.VolumeStatsAggPeriod = metav1.Duration{Duration: 10 * time.Second} // Aggregate volumes frequently so tests don't need to wait as long
-	kubeletConfigFlags = append(kubeletConfigFlags, "volume-stats-agg-period")
 
 	kc.SerializeImagePulls = false
-	kubeletConfigFlags = append(kubeletConfigFlags, "serialize-image-pulls")
 
 	kc.StaticPodPath = podPath
-	kubeletConfigFlags = append(kubeletConfigFlags, "pod-manifest-path")
 
 	kc.FileCheckFrequency = metav1.Duration{Duration: 10 * time.Second} // Check file frequently so tests won't wait too long
-	kubeletConfigFlags = append(kubeletConfigFlags, "file-check-frequency")
 
 	// Assign a fixed CIDR to the node because there is no node controller.
 	// Note: this MUST be in sync with the IP in
 	// - cluster/gce/config-test.sh and
 	// - test/e2e_node/conformance/run_test.sh.
 	kc.PodCIDR = "10.100.0.0/24"
-	kubeletConfigFlags = append(kubeletConfigFlags, "pod-cidr")
 
 	kc.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 30 * time.Second}
-	kubeletConfigFlags = append(kubeletConfigFlags, "eviction-pressure-transition-period")
 
 	kc.EvictionHard = map[string]string{
 		"memory.available":  "250Mi",
 		"nodefs.available":  "10%",
 		"nodefs.inodesFree": "5%",
 	}
-	kubeletConfigFlags = append(kubeletConfigFlags, "eviction-hard")
 
 	kc.EvictionMinimumReclaim = map[string]string{
 		"nodefs.available":  "5%",
 		"nodefs.inodesFree": "5%",
 	}
-	kubeletConfigFlags = append(kubeletConfigFlags, "eviction-minimum-reclaim")
 
 	var killCommand, restartCommand *exec.Cmd
 	var isSystemd bool
@@ -204,17 +234,14 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		restartCommand = exec.Command("systemctl", "restart", unitName)
 
 		kc.KubeletCgroups = "/kubelet.slice"
-		kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups")
 	} else {
 		cmdArgs = append(cmdArgs, builder.GetKubeletServerBin())
 		// TODO(random-liu): Get rid of this docker specific thing.
 		cmdArgs = append(cmdArgs, "--runtime-cgroups=/docker-daemon")
 
 		kc.KubeletCgroups = "/kubelet"
-		kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups")
 
 		kc.SystemCgroups = "/system"
-		kubeletConfigFlags = append(kubeletConfigFlags, "system-cgroups")
 	}
 	cmdArgs = append(cmdArgs,
 		"--kubeconfig", kubeconfigPath,
@@ -277,17 +304,11 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint)
 	}
 
-	// Write config file or flags, depending on whether --generate-kubelet-config-file was provided
+	if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil {
-	if genKubeletConfigFile {
+		return nil, err
-		if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil {
-			return nil, err
-		}
-		// add the flag to load config from a file
-		cmdArgs = append(cmdArgs, "--config", kubeletConfigPath)
-	} else {
-		// generate command line flags from the default config, since --generate-kubelet-config-file was not provided
-		addKubeletConfigFlags(&cmdArgs, kc, kubeletConfigFlags)
 	}
+	// add the flag to load config from a file
+	cmdArgs = append(cmdArgs, "--config", kubeletConfigPath)
 
 	// Override the default kubelet flags.
 	cmdArgs = append(cmdArgs, kubeletArgs...)
@@ -311,15 +332,6 @@ func (e *E2EServices) startKubelet() (*server, error) {
 	return server, server.start()
 }
 
-// addKubeletConfigFlags adds the flags we care about from the provided kubelet configuration object
-func addKubeletConfigFlags(cmdArgs *[]string, kc *kubeletconfig.KubeletConfiguration, flags []string) {
-	fs := pflag.NewFlagSet("kubelet", pflag.ExitOnError)
-	options.AddKubeletConfigFlags(fs, kc)
-	for _, name := range flags {
-		*cmdArgs = append(*cmdArgs, fmt.Sprintf("--%s=%s", name, fs.Lookup(name).Value.String()))
-	}
-}
-
 // writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename
 func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error {
 	data, err := kubeletconfigcodec.EncodeKubeletConfig(internal, kubeletconfigv1beta1.SchemeGroupVersion)