From 06a81d13959e0e960eb726e98bc1f444d7c36f12 Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Mon, 17 Jul 2023 15:19:34 -0400 Subject: [PATCH] cmd/kubelet: implement drop-in configuration directory for kubelet This implements a drop-in configuration directory for the kubelet by introducing a "--config-dir" flag. Users can provide individual kubelet config snippets in separate files, formatted similarly to kubelet.conf. The kubelet will process the files in alphanumeric order, appending configurations if subfield(s) doesn't exist, overwriting them if they do, and handling lists by overwriting instead of merging. Co-authored-by: Yu Qi Zhang --- cmd/kubelet/app/options/options.go | 5 + cmd/kubelet/app/server.go | 56 +++++++- cmd/kubelet/app/server_test.go | 198 +++++++++++++++++++++++++++++ go.mod | 2 +- 4 files changed, 257 insertions(+), 4 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 0328e0ec0cf..71ea243361c 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -86,6 +86,10 @@ type KubeletFlags struct { // Omit this flag to use the combination of built-in default configuration values and flags. KubeletConfigFile string + // kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify + // additional configs to overwrite what is provided by default and in the KubeletConfigFile flag + KubeletDropinConfigDirectory string + // WindowsService should be set to true if kubelet is running as a service on Windows. // Its corresponding flag only gets registered in Windows builds. WindowsService bool @@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { f.addOSFlags(fs) fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.") + fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']") fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.") fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+ diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 64dbb4cd969..78b2d2c1394 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "io/fs" "math" "net" "net/http" @@ -33,6 +34,7 @@ import ( "time" "github.com/coreos/go-systemd/v22/daemon" + "github.com/imdario/mergo" "github.com/spf13/cobra" "github.com/spf13/pflag" "google.golang.org/grpc/codes" @@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`, } // load kubelet config file, if provided - if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 { - kubeletConfig, err = loadConfigFile(configFile) + if len(kubeletFlags.KubeletConfigFile) > 0 { + kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile) if err != nil { - return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile) + return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err) } + } + // Merge the kubelet configurations if --config-dir is set + if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 { + _, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA") + if !ok { + return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory) + } + if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil { + return fmt.Errorf("failed to merge kubelet configs: %w", err) + } + } + + if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 { // We must enforce flag precedence by re-parsing the command line into the new object. // This is necessary to preserve backwards-compatibility across binary upgrades. // See issue #56171 for more details. @@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`, return cmd } +// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration. +// The drop-in configurations are processed in lexical order based on the file names. This means that the +// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes. +// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf", +// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied, +// potentially overriding the previous values. +func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error { + const dropinFileExtension = ".conf" + + // Walk through the drop-in directory and update the configuration for each file + err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension { + dropinConfig, err := loadConfigFile(path) + if err != nil { + return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err) + } + + // Merge dropinConfig with kubeletConfig + if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err) + } + } + return nil + }) + + if err != nil { + return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err) + } + + return nil +} + // newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered // on it. func newFlagSetWithGlobals() *pflag.FlagSet { diff --git a/cmd/kubelet/app/server_test.go b/cmd/kubelet/app/server_test.go index 1db214ab954..0a4fda291a2 100644 --- a/cmd/kubelet/app/server_test.go +++ b/cmd/kubelet/app/server_test.go @@ -17,7 +17,14 @@ limitations under the License. package app import ( + "os" + "path/filepath" + "reflect" "testing" + + "github.com/stretchr/testify/require" + "k8s.io/kubernetes/cmd/kubelet/app/options" + kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" ) func TestValueOfAllocatableResources(t *testing.T) { @@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) { } } } + +func TestMergeKubeletConfigurations(t *testing.T) { + testCases := []struct { + kubeletConfig string + dropin1 string + dropin2 string + overwrittenConfigFields map[string]interface{} + cliArgs []string + name string + }{ + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9080 +readOnlyPort: 10257 +`, + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 8080 +readOnlyPort: 10255 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(8080), + "ReadOnlyPort": int32(10255), + }, + name: "kubelet.conf.d overrides kubelet.conf", + }, + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +readOnlyPort: 10256 +kubeReserved: + memory: 70Mi +`, + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +readOnlyPort: 10255 +kubeReserved: + memory: 150Mi + cpu: 200m +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +readOnlyPort: 10257 +kubeReserved: + memory: 100Mi +`, + overwrittenConfigFields: map[string]interface{}{ + "ReadOnlyPort": int32(10257), + "KubeReserved": map[string]string{ + "cpu": "200m", + "memory": "100Mi", + }, + }, + name: "kubelet.conf.d overrides kubelet.conf with subfield override", + }, + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +clusterDNS: + - 192.168.1.3 + - 192.168.1.4 +`, + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +systemReserved: + memory: 1Gi +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 8080 +readOnlyPort: 10255 +systemReserved: + memory: 2Gi +clusterDNS: + - 192.168.1.1 + - 192.168.1.5 + - 192.168.1.8 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(8080), + "ReadOnlyPort": int32(10255), + "SystemReserved": map[string]string{ + "memory": "2Gi", + }, + "ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"}, + }, + name: "kubelet.conf.d overrides kubelet.conf with slices/lists", + }, + { + dropin1: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +`, + dropin2: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 8080 +readOnlyPort: 10255 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(8081), + "ReadOnlyPort": int32(10256), + }, + cliArgs: []string{ + "--port=8081", + "--read-only-port=10256", + }, + name: "cli args override kubelet.conf.d", + }, + { + kubeletConfig: ` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +port: 9090 +clusterDNS: + - 192.168.1.3 +`, + overwrittenConfigFields: map[string]interface{}{ + "Port": int32(9090), + "ClusterDNS": []string{"192.168.1.2"}, + }, + cliArgs: []string{ + "--port=9090", + "--cluster-dns=192.168.1.2", + }, + name: "cli args override kubelet.conf", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + // Prepare a temporary directory for testing + tempDir := t.TempDir() + + kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{} + kubeletFlags := &options.KubeletFlags{} + + if len(test.kubeletConfig) > 0 { + // Create the Kubeletconfig + kubeletConfFile := filepath.Join(tempDir, "kubelet.conf") + err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644) + require.NoError(t, err, "failed to create config from a yaml file") + kubeletFlags.KubeletConfigFile = kubeletConfFile + } + if len(test.dropin1) > 0 || len(test.dropin2) > 0 { + // Create kubelet.conf.d directory and drop-in configuration files + kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d") + err := os.Mkdir(kubeletConfDir, 0755) + require.NoError(t, err, "Failed to create kubelet.conf.d directory") + + err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644) + require.NoError(t, err, "failed to create config from a yaml file") + + err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644) + require.NoError(t, err, "failed to create config from a yaml file") + + // Merge the kubelet configurations + err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir) + require.NoError(t, err, "failed to merge kubelet drop-in configs") + } + + // Use kubelet config flag precedence + err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs) + require.NoError(t, err, "failed to set the kubelet config flag precedence") + + // Verify the merged configuration fields + for fieldName, expectedValue := range test.overwrittenConfigFields { + value := reflect.ValueOf(kubeletConfig).Elem() + field := value.FieldByName(fieldName) + require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName) + } + }) + } +} diff --git a/go.mod b/go.mod index cc321d19c64..7a2142ae605 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.3.0 + github.com/imdario/mergo v0.3.6 github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5 github.com/libopenstorage/openstorage v1.0.0 github.com/lithammer/dedent v1.1.0 @@ -183,7 +184,6 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect