diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 0328e0ec0cf..71ea243361c 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -86,6 +86,10 @@ type KubeletFlags struct { // Omit this flag to use the combination of built-in default configuration values and flags. KubeletConfigFile string + // kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify + // additional configs to overwrite what is provided by default and in the KubeletConfigFile flag + KubeletDropinConfigDirectory string + // WindowsService should be set to true if kubelet is running as a service on Windows. // Its corresponding flag only gets registered in Windows builds. WindowsService bool @@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { f.addOSFlags(fs) fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.") + fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']") fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.") fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+ diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 64dbb4cd969..78b2d2c1394 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "io/fs" "math" "net" "net/http" @@ -33,6 +34,7 @@ import ( "time" "github.com/coreos/go-systemd/v22/daemon" + "github.com/imdario/mergo" "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc/codes" @@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`, } // load kubelet config file, if provided - if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 { - kubeletConfig, err = loadConfigFile(configFile) + if len(kubeletFlags.KubeletConfigFile) > 0 { + kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile) if err != nil { - return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile) + return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err) } + } + // Merge the kubelet configurations if --config-dir is set + if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 { + _, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA") + if !ok { + return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory) + } + if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil { + return fmt.Errorf("failed to merge kubelet configs: %w", err) + } + } + + if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 { // We must enforce flag precedence by re-parsing the command line into the new object. // This is necessary to preserve backwards-compatibility across binary upgrades. // See issue #56171 for more details. @@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`, return cmd } +// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration. +// The drop-in configurations are processed in lexical order based on the file names. This means that the +// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes. +// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf", +// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied, +// potentially overriding the previous values. +func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error { + const dropinFileExtension = ".conf" + + // Walk through the drop-in directory and update the configuration for each file + err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension { + dropinConfig, err := loadConfigFile(path) + if err != nil { + return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err) + } + + // Merge dropinConfig with kubeletConfig + if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err) + } + } + return nil + }) + + if err != nil { + return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err) + } + + return nil +} + // newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered // on it. func newFlagSetWithGlobals() *pflag.FlagSet { diff --git a/cmd/kubelet/app/server_test.go b/cmd/kubelet/app/server_test.go index 1db214ab954..0a4fda291a2 100644 --- a/cmd/kubelet/app/server_test.go +++ b/cmd/kubelet/app/server_test.go @@ -17,7 +17,14 @@ limitations under the License. package app import ( + "os" + "path/filepath" + "reflect" "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kubernetes/cmd/kubelet/app/options" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" ) func TestValueOfAllocatableResources(t *testing.T) { @@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) { } } } + +func TestMergeKubeletConfigurations(t *testing.T) { + testCases := []struct { + kubeletConfig string + dropin1 string + dropin2 string + overwrittenConfigFields map[string]interface{} + cliArgs []string + name string + }{ + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9080 +readOnlyPort: 10257 +`, + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 8080 +readOnlyPort: 10255 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(8080), + "ReadOnlyPort": int32(10255), + }, + name: "kubelet.conf.d overrides kubelet.conf", + }, + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +readOnlyPort: 10256 +kubeReserved: + memory: 70Mi +`, + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +readOnlyPort: 10255 +kubeReserved: + memory: 150Mi + cpu: 200m +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +readOnlyPort: 10257 +kubeReserved: + memory: 100Mi +`, + overwrittenConfigFields: map[string]interface{}{ + "ReadOnlyPort": int32(10257), + "KubeReserved": map[string]string{ + "cpu": "200m", + "memory": "100Mi", + }, + }, + name: "kubelet.conf.d overrides kubelet.conf with subfield override", + }, + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +clusterDNS: + - 192.168.1.3 + - 192.168.1.4 +`, + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +systemReserved: + memory: 1Gi +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 8080 +readOnlyPort: 10255 +systemReserved: + memory: 2Gi +clusterDNS: + - 192.168.1.1 + - 192.168.1.5 + - 192.168.1.8 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(8080), + "ReadOnlyPort": int32(10255), + "SystemReserved": map[string]string{ + "memory": "2Gi", + }, + "ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"}, + }, + name: "kubelet.conf.d overrides kubelet.conf with slices/lists", + }, + { + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 8080 +readOnlyPort: 10255 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(8081), + "ReadOnlyPort": int32(10256), + }, + cliArgs: []string{ + "--port=8081", + "--read-only-port=10256", + }, + name: "cli args override kubelet.conf.d", + }, + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +clusterDNS: + - 192.168.1.3 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(9090), + "ClusterDNS": []string{"192.168.1.2"}, + }, + cliArgs: []string{ + "--port=9090", + "--cluster-dns=192.168.1.2", + }, + name: "cli args override kubelet.conf", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + // Prepare a temporary directory for testing + tempDir := t.TempDir() + + kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{} + kubeletFlags := &options.KubeletFlags{} + + if len(test.kubeletConfig) > 0 { + // Create the Kubeletconfig + kubeletConfFile := filepath.Join(tempDir, "kubelet.conf") + err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644) + require.NoError(t, err, "failed to create config from a yaml file") + kubeletFlags.KubeletConfigFile = kubeletConfFile + } + if len(test.dropin1) > 0 || len(test.dropin2) > 0 { + // Create kubelet.conf.d directory and drop-in configuration files + kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d") + err := os.Mkdir(kubeletConfDir, 0755) + require.NoError(t, err, "Failed to create kubelet.conf.d directory") + + err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644) + require.NoError(t, err, "failed to create config from a yaml file") + + err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644) + require.NoError(t, err, "failed to create config from a yaml file") + + // Merge the kubelet configurations + err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir) + require.NoError(t, err, "failed to merge kubelet drop-in configs") + } + + // Use kubelet config flag precedence + err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs) + require.NoError(t, err, "failed to set the kubelet config flag precedence") + + // Verify the merged configuration fields + for fieldName, expectedValue := range test.overwrittenConfigFields { + value := reflect.ValueOf(kubeletConfig).Elem() + field := value.FieldByName(fieldName) + require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName) + } + }) + } +} diff --git a/go.mod b/go.mod index cc321d19c64..7a2142ae605 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.3.0 + github.com/imdario/mergo v0.3.6 github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5 github.com/libopenstorage/openstorage v1.0.0 github.com/lithammer/dedent v1.1.0 @@ -183,7 +184,6 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect