diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index 24b25c35920..7ee66c14ef8 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -49,6 +49,7 @@ extra_envs=${EXTRA_ENVS:-} runtime_config=${RUNTIME_CONFIG:-} ssh_user=${SSH_USER:-"${USER}"} ssh_key=${SSH_KEY:-} +kubelet_config_file=${KUBELET_CONFIG_FILE:-""} # Parse the flags to pass to ginkgo ginkgoflags="" @@ -164,6 +165,8 @@ if [ "${remote}" = true ] ; then echo "Ginkgo Flags: ${ginkgoflags}" echo "Instance Metadata: ${metadata}" echo "Image Config File: ${image_config_file}" + echo "Kubelet Config File: ${kubelet_config_file}" + # Invoke the runner go run test/e2e_node/runner/remote/run_remote.go --logtostderr --vmodule=*=4 --ssh-env="gce" \ --zone="${zone}" --project="${project}" --gubernator="${gubernator}" \ @@ -174,7 +177,7 @@ if [ "${remote}" = true ] ; then --image-config-file="${image_config_file}" --system-spec-name="${system_spec_name}" \ --runtime-config="${runtime_config}" --preemptible-instances="${preemptible_instances}" \ --ssh-user="${ssh_user}" --ssh-key="${ssh_key}" --image-config-dir="${image_config_dir}" \ - --extra-envs="${extra_envs}" --test-suite="${test_suite}" \ + --extra-envs="${extra_envs}" --kubelet-config-file="${kubelet_config_file}" --test-suite="${test_suite}" \ "${timeout_arg}" \ 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index 86bafbd14a7..b6aa5496551 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -65,11 +65,7 @@ func copyKubeletConfigIfExists(kubeletConfigFile, dstDir string) error { defer destination.Close() _, err = io.Copy(destination, source) - if err != nil { - return err - } - - return os.Chmod(dst, 0x644) + return err } // CreateTestArchive creates the archive package for the node e2e test. diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index b13e0c18589..5f3285120ed 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -405,7 +405,7 @@ func callGubernator(gubernator bool) { } func (a *Archive) getArchive() (string, error) { - a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName) }) + a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName, *kubeletConfigFile) }) return a.path, a.err } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 19344dce658..34d63089520 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -26,7 +26,6 @@ import ( "strings" "time" - "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" cliflag "k8s.io/component-base/cli/flag" @@ -34,9 +33,12 @@ import ( kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/cmd/kubelet/app/options" + "k8s.io/kubernetes/pkg/cluster/ports" "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles" kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" + utilfs "k8s.io/kubernetes/pkg/util/filesystem" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/builder" "k8s.io/kubernetes/test/e2e_node/remote" @@ -62,11 +64,9 @@ func (a *args) Set(value string) error { // kubeletArgs is the override kubelet args specified by the test runner. var kubeletArgs args -var genKubeletConfigFile bool func init() { flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.") - flag.BoolVar(&genKubeletConfigFile, "generate-kubelet-config-file", true, "The test runner will generate a Kubelet config file containing test defaults instead of passing default flags to the Kubelet.") } // RunKubelet starts kubelet and waits for termination signal. Once receives the @@ -94,6 +94,35 @@ const ( kubeletHealthCheckURL = "http://127.0.0.1:" + kubeletReadOnlyPort + "/healthz" ) +// TODO(endocrimes): Refactor to take a path to the kubeletconfig +func baseKubeConfiguration() (*kubeletconfig.KubeletConfiguration, error) { + cwd, _ := os.Getwd() + cfgPath, err := filepath.Abs(filepath.Join(cwd, "kubeletconfig.yaml")) + if err != nil { + return nil, err + } + + _, err = os.Stat(cfgPath) + if err != nil { + // If the kubeletconfig exists, but for some reason we can't read it, then + // return an error to avoid silently skipping it. + if !os.IsNotExist(err) { + return nil, err + } + + // If the kubeletconfig file doesn't exist, then use a default configuration + // as the base. + return options.NewKubeletConfiguration() + } + + loader, err := configfiles.NewFsLoader(&utilfs.DefaultFs{}, cfgPath) + if err != nil { + return nil, err + } + + return loader.Load() +} + // startKubelet starts the Kubelet in a separate process or returns an error // if the Kubelet fails to start. func (e *E2EServices) startKubelet() (*server, error) { @@ -127,53 +156,54 @@ func (e *E2EServices) startKubelet() (*server, error) { return nil, err } - // PLEASE NOTE: If you set new KubeletConfiguration values or stop setting values here, - // you must also update the flag names in kubeletConfigFlags! - kubeletConfigFlags := []string{} - - // set up the default kubeletconfiguration - kc, err := options.NewKubeletConfiguration() + kc, err := baseKubeConfiguration() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load base kubelet configuration: %v", err) } + // Apply overrides to allow access to the Kubelet API from the test suite. + // These are insecure and should generally not be used outside of test infra. + + // --anonymous-auth + kc.Authentication.Anonymous.Enabled = true + // --authentication-token-webhook + kc.Authentication.Webhook.Enabled = false + // --authorization-mode + kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow + // --read-only-port + kc.ReadOnlyPort = ports.KubeletReadOnlyPort + + // Setup general overrides for the kubelet. + // TODO(endocrimes): Move the following to a `default` configuration file + kc.CgroupRoot = "/" - kubeletConfigFlags = append(kubeletConfigFlags, "cgroup-root") kc.VolumeStatsAggPeriod = metav1.Duration{Duration: 10 * time.Second} // Aggregate volumes frequently so tests don't need to wait as long - kubeletConfigFlags = append(kubeletConfigFlags, "volume-stats-agg-period") kc.SerializeImagePulls = false - kubeletConfigFlags = append(kubeletConfigFlags, "serialize-image-pulls") kc.StaticPodPath = podPath - kubeletConfigFlags = append(kubeletConfigFlags, "pod-manifest-path") kc.FileCheckFrequency = metav1.Duration{Duration: 10 * time.Second} // Check file frequently so tests won't wait too long - kubeletConfigFlags = append(kubeletConfigFlags, "file-check-frequency") // Assign a fixed CIDR to the node because there is no node controller. // Note: this MUST be in sync with the IP in // - cluster/gce/config-test.sh and // - test/e2e_node/conformance/run_test.sh. kc.PodCIDR = "10.100.0.0/24" - kubeletConfigFlags = append(kubeletConfigFlags, "pod-cidr") kc.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 30 * time.Second} - kubeletConfigFlags = append(kubeletConfigFlags, "eviction-pressure-transition-period") kc.EvictionHard = map[string]string{ "memory.available": "250Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%", } - kubeletConfigFlags = append(kubeletConfigFlags, "eviction-hard") kc.EvictionMinimumReclaim = map[string]string{ "nodefs.available": "5%", "nodefs.inodesFree": "5%", } - kubeletConfigFlags = append(kubeletConfigFlags, "eviction-minimum-reclaim") var killCommand, restartCommand *exec.Cmd var isSystemd bool @@ -204,17 +234,14 @@ func (e *E2EServices) startKubelet() (*server, error) { restartCommand = exec.Command("systemctl", "restart", unitName) kc.KubeletCgroups = "/kubelet.slice" - kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups") } else { cmdArgs = append(cmdArgs, builder.GetKubeletServerBin()) // TODO(random-liu): Get rid of this docker specific thing. cmdArgs = append(cmdArgs, "--runtime-cgroups=/docker-daemon") kc.KubeletCgroups = "/kubelet" - kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups") kc.SystemCgroups = "/system" - kubeletConfigFlags = append(kubeletConfigFlags, "system-cgroups") } cmdArgs = append(cmdArgs, "--kubeconfig", kubeconfigPath, @@ -277,17 +304,11 @@ func (e *E2EServices) startKubelet() (*server, error) { cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint) } - // Write config file or flags, depending on whether --generate-kubelet-config-file was provided - if genKubeletConfigFile { - if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil { - return nil, err - } - // add the flag to load config from a file - cmdArgs = append(cmdArgs, "--config", kubeletConfigPath) - } else { - // generate command line flags from the default config, since --generate-kubelet-config-file was not provided - addKubeletConfigFlags(&cmdArgs, kc, kubeletConfigFlags) + if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil { + return nil, err } + // add the flag to load config from a file + cmdArgs = append(cmdArgs, "--config", kubeletConfigPath) // Override the default kubelet flags. cmdArgs = append(cmdArgs, kubeletArgs...) @@ -311,15 +332,6 @@ func (e *E2EServices) startKubelet() (*server, error) { return server, server.start() } -// addKubeletConfigFlags adds the flags we care about from the provided kubelet configuration object -func addKubeletConfigFlags(cmdArgs *[]string, kc *kubeletconfig.KubeletConfiguration, flags []string) { - fs := pflag.NewFlagSet("kubelet", pflag.ExitOnError) - options.AddKubeletConfigFlags(fs, kc) - for _, name := range flags { - *cmdArgs = append(*cmdArgs, fmt.Sprintf("--%s=%s", name, fs.Lookup(name).Value.String())) - } -} - // writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error { data, err := kubeletconfigcodec.EncodeKubeletConfig(internal, kubeletconfigv1beta1.SchemeGroupVersion)