github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-06 02:34:03 +00:00
Code Issues Projects Releases Wiki Activity

cmd/kubelet: implement drop-in configuration directory for kubelet

Browse Source 
This implements a drop-in configuration directory for the kubelet
by introducing a "--config-dir" flag. Users can provide individual
kubelet config snippets in separate files, formatted similarly to
kubelet.conf. The kubelet will process the files in alphanumeric order,
appending configurations if subfield(s) doesn't exist, overwriting them if
they do, and handling lists by overwriting instead of merging.

Co-authored-by: Yu Qi Zhang <jerzhang@redhat.com>
This commit is contained in:
Sohan Kunkerkar 2023-07-17 15:19:34 -04:00
parent a9b3ca34b5
commit 06a81d1395
4 changed files with 257 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/kubelet/app/options/options.go
View File

@ -86,6 +86,10 @@ type KubeletFlags struct {
// Omit this flag to use the combination of built-in default configuration values and flags.
KubeletConfigFile string
// kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify
// additional configs to overwrite what is provided by default and in the KubeletConfigFile flag
KubeletDropinConfigDirectory string
// WindowsService should be set to true if kubelet is running as a service on Windows.
// Its corresponding flag only gets registered in Windows builds.
WindowsService bool
@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
f.addOSFlags(fs)
fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']")
fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")
fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+

56
cmd/kubelet/app/server.go
View File

@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"math"
"net"
"net/http"
@ -33,6 +34,7 @@ import (
"time"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/imdario/mergo"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"google.golang.org/grpc/codes"
@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`,
}
// load kubelet config file, if provided
if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
kubeletConfig, err = loadConfigFile(configFile)
if len(kubeletFlags.KubeletConfigFile) > 0 {
kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile)
if err != nil {
return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile)
return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err)
}
}
// Merge the kubelet configurations if --config-dir is set
if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
_, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA")
if !ok {
return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory)
}
if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil {
return fmt.Errorf("failed to merge kubelet configs: %w", err)
}
}
if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
// We must enforce flag precedence by re-parsing the command line into the new object.
// This is necessary to preserve backwards-compatibility across binary upgrades.
// See issue #56171 for more details.
@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`,
return cmd
}
// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration.
// The drop-in configurations are processed in lexical order based on the file names. This means that the
// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes.
// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf",
// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied,
// potentially overriding the previous values.
func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error {
const dropinFileExtension = ".conf"
// Walk through the drop-in directory and update the configuration for each file
err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error {
if err != nil {
return err
}
if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension {
dropinConfig, err := loadConfigFile(path)
if err != nil {
return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err)
}
// Merge dropinConfig with kubeletConfig
if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil {
return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err)
}
}
return nil
})
if err != nil {
return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err)
}
return nil
}
// newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered
// on it.
func newFlagSetWithGlobals() *pflag.FlagSet {

198
cmd/kubelet/app/server_test.go
View File

@ -17,7 +17,14 @@ limitations under the License.
package app
import (
"os"
"path/filepath"
"reflect"
"testing"
"github.com/stretchr/testify/require"
"k8s.io/kubernetes/cmd/kubelet/app/options"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
)
func TestValueOfAllocatableResources(t *testing.T) {
@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) {
}
}
}
func TestMergeKubeletConfigurations(t *testing.T) {
testCases := []struct {
kubeletConfig string
dropin1 string
dropin2 string
overwrittenConfigFields map[string]interface{}
cliArgs []string
name string
}{
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9080
readOnlyPort: 10257
`,
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 8080
readOnlyPort: 10255
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(8080),
"ReadOnlyPort": int32(10255),
},
name: "kubelet.conf.d overrides kubelet.conf",
},
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 10256
kubeReserved:
memory: 70Mi
`,
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 10255
kubeReserved:
memory: 150Mi
cpu: 200m
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 10257
kubeReserved:
memory: 100Mi
`,
overwrittenConfigFields: map[string]interface{}{
"ReadOnlyPort": int32(10257),
"KubeReserved": map[string]string{
"cpu": "200m",
"memory": "100Mi",
},
},
name: "kubelet.conf.d overrides kubelet.conf with subfield override",
},
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
clusterDNS:
- 192.168.1.3
- 192.168.1.4
`,
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
systemReserved:
memory: 1Gi
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 8080
readOnlyPort: 10255
systemReserved:
memory: 2Gi
clusterDNS:
- 192.168.1.1
- 192.168.1.5
- 192.168.1.8
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(8080),
"ReadOnlyPort": int32(10255),
"SystemReserved": map[string]string{
"memory": "2Gi",
},
"ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"},
},
name: "kubelet.conf.d overrides kubelet.conf with slices/lists",
},
{
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 8080
readOnlyPort: 10255
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(8081),
"ReadOnlyPort": int32(10256),
},
cliArgs: []string{
"--port=8081",
"--read-only-port=10256",
},
name: "cli args override kubelet.conf.d",
},
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
clusterDNS:
- 192.168.1.3
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(9090),
"ClusterDNS": []string{"192.168.1.2"},
},
cliArgs: []string{
"--port=9090",
"--cluster-dns=192.168.1.2",
},
name: "cli args override kubelet.conf",
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
// Prepare a temporary directory for testing
tempDir := t.TempDir()
kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{}
kubeletFlags := &options.KubeletFlags{}
if len(test.kubeletConfig) > 0 {
// Create the Kubeletconfig
kubeletConfFile := filepath.Join(tempDir, "kubelet.conf")
err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644)
require.NoError(t, err, "failed to create config from a yaml file")
kubeletFlags.KubeletConfigFile = kubeletConfFile
}
if len(test.dropin1) > 0 || len(test.dropin2) > 0 {
// Create kubelet.conf.d directory and drop-in configuration files
kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d")
err := os.Mkdir(kubeletConfDir, 0755)
require.NoError(t, err, "Failed to create kubelet.conf.d directory")
err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644)
require.NoError(t, err, "failed to create config from a yaml file")
err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644)
require.NoError(t, err, "failed to create config from a yaml file")
// Merge the kubelet configurations
err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir)
require.NoError(t, err, "failed to merge kubelet drop-in configs")
}
// Use kubelet config flag precedence
err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs)
require.NoError(t, err, "failed to set the kubelet config flag precedence")
// Verify the merged configuration fields
for fieldName, expectedValue := range test.overwrittenConfigFields {
value := reflect.ValueOf(kubeletConfig).Elem()
field := value.FieldByName(fieldName)
require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName)
}
})
}
}

2
go.mod
View File

@ -43,6 +43,7 @@ require (
github.com/google/go-cmp v0.5.9
github.com/google/gofuzz v1.2.0
github.com/google/uuid v1.3.0
github.com/imdario/mergo v0.3.6
github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5
github.com/libopenstorage/openstorage v1.0.0
github.com/lithammer/dedent v1.1.0
@ -183,7 +184,6 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect