diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index a95f50c7dc1..533f040b4ef 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -99,13 +99,14 @@ var ( // Test suite authors can use framework/viper to make all command line // parameters also configurable via a configuration file. type TestContextType struct { - KubeConfig string - KubeContext string - KubeAPIContentType string - KubeletRootDir string - CertDir string - Host string - BearerToken string `datapolicy:"token"` + KubeConfig string + KubeContext string + KubeAPIContentType string + KubeletRootDir string + KubeletConfigDropinDir string + CertDir string + Host string + BearerToken string `datapolicy:"token"` // TODO: Deprecating this over time... instead just use gobindata_util.go , see #23987. RepoRoot string // ListImages will list off all images that are used then quit diff --git a/test/e2e/nodefeature/nodefeature.go b/test/e2e/nodefeature/nodefeature.go index ed0b3f091b3..528cf852214 100644 --- a/test/e2e/nodefeature/nodefeature.go +++ b/test/e2e/nodefeature/nodefeature.go @@ -37,6 +37,7 @@ var ( GracefulNodeShutdownBasedOnPodPriority = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdownBasedOnPodPriority")) HostAccess = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("HostAccess")) ImageID = framework.WithNodeFeature(framework.ValidNodeFeatures.Add(" ImageID")) + KubeletConfigDropInDir = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("KubeletConfigDropInDir")) LSCIQuotaMonitoring = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("LSCIQuotaMonitoring")) NodeAllocatable = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeAllocatable")) NodeProblemDetector = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeProblemDetector")) diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index dbe0a809af3..cd559b1b97c 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -90,6 +90,7 @@ func registerNodeFlags(flags *flag.FlagSet) { framework.TestContext.NodeE2E = true flags.StringVar(&framework.TestContext.BearerToken, "bearer-token", "", "The bearer token to authenticate with. If not specified, it would be a random token. Currently this token is only used in node e2e tests.") flags.StringVar(&framework.TestContext.NodeName, "node-name", "", "Name of the node to run tests on.") + flags.StringVar(&framework.TestContext.KubeletConfigDropinDir, "config-dir", "", "Path to a directory containing drop-in configurations for the kubelet.") // TODO(random-liu): Move kubelet start logic out of the test. // TODO(random-liu): Move log fetch logic out of the test. // There are different ways to start kubelet (systemd, initd, docker, manually started etc.) @@ -200,6 +201,14 @@ func TestE2eNode(t *testing.T) { // We're not running in a special mode so lets run tests. gomega.RegisterFailHandler(ginkgo.Fail) + // Initialize the KubeletConfigDropinDir again if the test doesn't run in run-kubelet-mode. + if framework.TestContext.KubeletConfigDropinDir == "" { + var err error + framework.TestContext.KubeletConfigDropinDir, err = services.KubeletConfigDirCWDDir() + if err != nil { + klog.Errorf("failed to create kubelet config directory: %v", err) + } + } reportDir := framework.TestContext.ReportDir if reportDir != "" { // Create the directory if it doesn't already exist diff --git a/test/e2e_node/kubelet_config_dir_test.go b/test/e2e_node/kubelet_config_dir_test.go new file mode 100644 index 00000000000..54c23607887 --- /dev/null +++ b/test/e2e_node/kubelet_config_dir_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/nodefeature" +) + +var _ = SIGDescribe("Kubelet Config", framework.WithSlow(), framework.WithSerial(), framework.WithDisruptive(), nodefeature.KubeletConfigDropInDir, func() { + f := framework.NewDefaultFramework("kubelet-config-drop-in-dir-test") + ginkgo.Context("when merging drop-in configs", func() { + var oldcfg *kubeletconfig.KubeletConfiguration + ginkgo.BeforeEach(func(ctx context.Context) { + var err error + oldcfg, err = getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + }) + ginkgo.AfterEach(func(ctx context.Context) { + files, err := filepath.Glob(filepath.Join(framework.TestContext.KubeletConfigDropinDir, "*"+".conf")) + framework.ExpectNoError(err) + for _, file := range files { + err := os.Remove(file) + framework.ExpectNoError(err) + } + updateKubeletConfig(ctx, f, oldcfg, true) + }) + ginkgo.It("should merge kubelet configs correctly", func(ctx context.Context) { + // Get the initial kubelet configuration + initialConfig, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + + ginkgo.By("Stopping the kubelet") + restartKubelet := stopKubelet() + + // wait until the kubelet health check will fail + gomega.Eventually(ctx, func() bool { + return kubeletHealthCheck(kubeletHealthCheckURL) + }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse()) + + configDir := framework.TestContext.KubeletConfigDropinDir + + contents := []byte(`apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 10255 +readOnlyPort: 10257 +clusterDNS: +- 192.168.1.10 +systemReserved: + memory: 1Gi`) + framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "10-kubelet.conf"), contents, 0755)) + contents = []byte(`apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDNS: +- 192.168.1.1 +- 192.168.1.5 +- 192.168.1.8 +port: 8080 +cpuManagerReconcilePeriod: 0s +systemReserved: + memory: 2Gi`) + framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "20-kubelet.conf"), contents, 0755)) + ginkgo.By("Restarting the kubelet") + restartKubelet() + // wait until the kubelet health check will succeed + gomega.Eventually(ctx, func() bool { + return kubeletHealthCheck(kubeletHealthCheckURL) + }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue()) + + mergedConfig, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + + // Replace specific fields in the initial configuration with expectedConfig values + initialConfig.Port = int32(8080) // not overridden by second file, should be retained. + initialConfig.ReadOnlyPort = int32(10257) // overridden by second file. + initialConfig.SystemReserved = map[string]string{ // overridden by map in second file. + "memory": "2Gi", + } + initialConfig.ClusterDNS = []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"} // overridden by slice in second file. + // This value was explicitly set in the drop-in, make sure it is retained + initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: time.Duration(0)} + // Meanwhile, this value was not explicitly set, but could have been overridden by a "default" of 0 for the type. + // Ensure the true default persists. + initialConfig.CPUCFSQuotaPeriod = metav1.Duration{Duration: time.Duration(100000000)} + // Compare the expected config with the merged config + gomega.Expect(initialConfig).To(gomega.BeComparableTo(mergedConfig), "Merged kubelet config does not match the expected configuration.") + }) + }) + +}) diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 1b35e1a35ab..069db99b410 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -174,6 +174,12 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error return nil, err } + // KubeletDropInConfiguration directory path + framework.TestContext.KubeletConfigDropinDir, err = KubeletConfigDirCWDDir() + if err != nil { + return nil, err + } + // Create pod directory podPath, err := createPodDirectory() if err != nil { @@ -243,6 +249,8 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error unitName = fmt.Sprintf("kubelet-%s.service", unitTimestamp) cmdArgs = append(cmdArgs, systemdRun, + // Set the environment variable to enable kubelet config drop-in directory. + "-E", "KUBELET_CONFIG_DROPIN_DIR_ALPHA=yes", "-p", "Delegate=true", "-p", logLocation+framework.TestContext.ReportDir+"/kubelet.log", "--unit="+unitName, @@ -282,6 +290,9 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error kc.FeatureGates = featureGates } + // Add the KubeletDropinConfigDirectory flag if set. + cmdArgs = append(cmdArgs, "--config-dir", framework.TestContext.KubeletConfigDropinDir) + // Keep hostname override for convenience. if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override. cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName) @@ -295,7 +306,7 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint) } - if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil { + if err := WriteKubeletConfigFile(kc, kubeletConfigPath); err != nil { return nil, err } // add the flag to load config from a file @@ -324,8 +335,8 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error return server, server.start() } -// writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename -func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error { +// WriteKubeletConfigFile writes the kubelet config file based on the args and returns the filename +func WriteKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error { data, err := kubeletconfigcodec.EncodeKubeletConfig(internal, kubeletconfigv1beta1.SchemeGroupVersion) if err != nil { return err @@ -408,6 +419,18 @@ func kubeletConfigCWDPath() (string, error) { return filepath.Join(cwd, "kubelet-config"), nil } +func KubeletConfigDirCWDDir() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current working directory: %w", err) + } + dir := filepath.Join(cwd, "kubelet.conf.d") + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + return dir, nil +} + // like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig // returns a fully-qualified path to the kubeconfig file func createKubeconfigCWD() (string, error) {