Merge pull request #119390 from sohankunkerkar/add-dropin

cmd/kubelet: implement drop-in configuration directory for kubelet

cmd/kubelet/app/options/options.go

@@ -86,6 +86,10 @@ type KubeletFlags struct {
 	// Omit this flag to use the combination of built-in default configuration values and flags.
 	KubeletConfigFile string
 
+	// kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify
+	// additional configs to overwrite what is provided by default and in the KubeletConfigFile flag
+	KubeletDropinConfigDirectory string
+
 	// WindowsService should be set to true if kubelet is running as a service on Windows.
 	// Its corresponding flag only gets registered in Windows builds.
 	WindowsService bool
@@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
 	f.addOSFlags(fs)
 
 	fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
+	fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']")
 	fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")
 
 	fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+

cmd/kubelet/app/server.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"math"
 	"net"
 	"net/http"
@@ -33,6 +34,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-systemd/v22/daemon"
+	"github.com/imdario/mergo"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"google.golang.org/grpc/codes"
@@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`,
 			}
 
 			// load kubelet config file, if provided
-			if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
+			if len(kubeletFlags.KubeletConfigFile) > 0 {
-				kubeletConfig, err = loadConfigFile(configFile)
+				kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile)
 				if err != nil {
-					return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile)
+					return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err)
 				}
+			}
+			// Merge the kubelet configurations if --config-dir is set
+			if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
+				_, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA")
+				if !ok {
+					return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory)
+				}
+				if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil {
+					return fmt.Errorf("failed to merge kubelet configs: %w", err)
+				}
+			}
+
+			if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
 				// We must enforce flag precedence by re-parsing the command line into the new object.
 				// This is necessary to preserve backwards-compatibility across binary upgrades.
 				// See issue #56171 for more details.
@@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`,
 	return cmd
 }
 
+// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration.
+// The drop-in configurations are processed in lexical order based on the file names. This means that the
+// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes.
+// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf",
+// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied,
+// potentially overriding the previous values.
+func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error {
+	const dropinFileExtension = ".conf"
+
+	// Walk through the drop-in directory and update the configuration for each file
+	err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension {
+			dropinConfig, err := loadConfigFile(path)
+			if err != nil {
+				return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err)
+			}
+
+			// Merge dropinConfig with kubeletConfig
+			if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil {
+				return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err)
+	}
+
+	return nil
+}
+
 // newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered
 // on it.
 func newFlagSetWithGlobals() *pflag.FlagSet {

cmd/kubelet/app/server_test.go

@@ -17,7 +17,14 @@ limitations under the License.
 package app
 
 import (
+	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/kubernetes/cmd/kubelet/app/options"
+	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
 )
 
 func TestValueOfAllocatableResources(t *testing.T) {
@@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) {
 		}
 	}
 }
+
+func TestMergeKubeletConfigurations(t *testing.T) {
+	testCases := []struct {
+		kubeletConfig           string
+		dropin1                 string
+		dropin2                 string
+		overwrittenConfigFields map[string]interface{}
+		cliArgs                 []string
+		name                    string
+	}{
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9080
+readOnlyPort: 10257
+`,
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 8080
+readOnlyPort: 10255
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":         int32(8080),
+				"ReadOnlyPort": int32(10255),
+			},
+			name: "kubelet.conf.d overrides kubelet.conf",
+		},
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+readOnlyPort: 10256
+kubeReserved:
+	memory: 70Mi
+`,
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+readOnlyPort: 10255
+kubeReserved:
+  memory: 150Mi
+  cpu: 200m
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+readOnlyPort: 10257
+kubeReserved:
+  memory: 100Mi
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"ReadOnlyPort": int32(10257),
+				"KubeReserved": map[string]string{
+					"cpu":    "200m",
+					"memory": "100Mi",
+				},
+			},
+			name: "kubelet.conf.d overrides kubelet.conf with subfield override",
+		},
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+clusterDNS:
+	- 192.168.1.3
+	- 192.168.1.4
+`,
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+systemReserved:
+  memory: 1Gi
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 8080
+readOnlyPort: 10255
+systemReserved:
+  memory: 2Gi
+clusterDNS:
+  - 192.168.1.1
+  - 192.168.1.5
+  - 192.168.1.8
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":         int32(8080),
+				"ReadOnlyPort": int32(10255),
+				"SystemReserved": map[string]string{
+					"memory": "2Gi",
+				},
+				"ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"},
+			},
+			name: "kubelet.conf.d overrides kubelet.conf with slices/lists",
+		},
+		{
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 8080
+readOnlyPort: 10255
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":         int32(8081),
+				"ReadOnlyPort": int32(10256),
+			},
+			cliArgs: []string{
+				"--port=8081",
+				"--read-only-port=10256",
+			},
+			name: "cli args override kubelet.conf.d",
+		},
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+clusterDNS:
+	- 192.168.1.3
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":       int32(9090),
+				"ClusterDNS": []string{"192.168.1.2"},
+			},
+			cliArgs: []string{
+				"--port=9090",
+				"--cluster-dns=192.168.1.2",
+			},
+			name: "cli args override kubelet.conf",
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			// Prepare a temporary directory for testing
+			tempDir := t.TempDir()
+
+			kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{}
+			kubeletFlags := &options.KubeletFlags{}
+
+			if len(test.kubeletConfig) > 0 {
+				// Create the Kubeletconfig
+				kubeletConfFile := filepath.Join(tempDir, "kubelet.conf")
+				err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644)
+				require.NoError(t, err, "failed to create config from a yaml file")
+				kubeletFlags.KubeletConfigFile = kubeletConfFile
+			}
+			if len(test.dropin1) > 0 || len(test.dropin2) > 0 {
+				// Create kubelet.conf.d directory and drop-in configuration files
+				kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d")
+				err := os.Mkdir(kubeletConfDir, 0755)
+				require.NoError(t, err, "Failed to create kubelet.conf.d directory")
+
+				err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644)
+				require.NoError(t, err, "failed to create config from a yaml file")
+
+				err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644)
+				require.NoError(t, err, "failed to create config from a yaml file")
+
+				// Merge the kubelet configurations
+				err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir)
+				require.NoError(t, err, "failed to merge kubelet drop-in configs")
+			}
+
+			// Use kubelet config flag precedence
+			err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs)
+			require.NoError(t, err, "failed to set the kubelet config flag precedence")
+
+			// Verify the merged configuration fields
+			for fieldName, expectedValue := range test.overwrittenConfigFields {
+				value := reflect.ValueOf(kubeletConfig).Elem()
+				field := value.FieldByName(fieldName)
+				require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName)
+			}
+		})
+	}
+}

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/gofuzz v1.2.0
 	github.com/google/uuid v1.3.0
+	github.com/imdario/mergo v0.3.6
 	github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5
 	github.com/libopenstorage/openstorage v1.0.0
 	github.com/lithammer/dedent v1.1.0
@@ -183,7 +184,6 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect