github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-19 00:31:00 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #119390 from sohankunkerkar/add-dropin

Browse Source 
cmd/kubelet: implement drop-in configuration directory for kubelet
This commit is contained in:
Kubernetes Prow Robot 2023-07-18 19:44:02 -07:00 committed by GitHub
commit 8c1dc65da9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 257 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/kubelet/app/options/options.go
View File

@ -86,6 +86,10 @@ type KubeletFlags struct {
// Omit this flag to use the combination of built-in default configuration values and flags.
KubeletConfigFile string
// kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify
// additional configs to overwrite what is provided by default and in the KubeletConfigFile flag
KubeletDropinConfigDirectory string
// WindowsService should be set to true if kubelet is running as a service on Windows.
// Its corresponding flag only gets registered in Windows builds.
WindowsService bool
@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
f.addOSFlags(fs)
fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']")
fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")
fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+

56
cmd/kubelet/app/server.go
View File

@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"math"
"net"
"net/http"
@ -33,6 +34,7 @@ import (
"time"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/imdario/mergo"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"google.golang.org/grpc/codes"
@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`,
}
// load kubelet config file, if provided
if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
kubeletConfig, err = loadConfigFile(configFile)
if len(kubeletFlags.KubeletConfigFile) > 0 {
kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile)
if err != nil {
return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile)
return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err)
}
}
// Merge the kubelet configurations if --config-dir is set
if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
_, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA")
if !ok {
return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory)
}
if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil {
return fmt.Errorf("failed to merge kubelet configs: %w", err)
}
}
if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
// We must enforce flag precedence by re-parsing the command line into the new object.
// This is necessary to preserve backwards-compatibility across binary upgrades.
// See issue #56171 for more details.
@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`,
return cmd
}
// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration.
// The drop-in configurations are processed in lexical order based on the file names. This means that the
// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes.
// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf",
// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied,
// potentially overriding the previous values.
func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error {
const dropinFileExtension = ".conf"
// Walk through the drop-in directory and update the configuration for each file
err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error {
if err != nil {
return err
}
if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension {
dropinConfig, err := loadConfigFile(path)
if err != nil {
return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err)
}
// Merge dropinConfig with kubeletConfig
if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil {
return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err)
}
}
return nil
})
if err != nil {
return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err)
}
return nil
}
// newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered
// on it.
func newFlagSetWithGlobals() *pflag.FlagSet {

198
cmd/kubelet/app/server_test.go
View File

@ -17,7 +17,14 @@ limitations under the License.
package app
import (
"os"
"path/filepath"
"reflect"
"testing"
"github.com/stretchr/testify/require"
"k8s.io/kubernetes/cmd/kubelet/app/options"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
)
func TestValueOfAllocatableResources(t *testing.T) {
@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) {
}
}
}
func TestMergeKubeletConfigurations(t *testing.T) {
testCases := []struct {
kubeletConfig string
dropin1 string
dropin2 string
overwrittenConfigFields map[string]interface{}
cliArgs []string
name string
}{
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9080
readOnlyPort: 10257
`,
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 8080
readOnlyPort: 10255
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(8080),
"ReadOnlyPort": int32(10255),
},
name: "kubelet.conf.d overrides kubelet.conf",
},
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 10256
kubeReserved:
memory: 70Mi
`,
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 10255
kubeReserved:
memory: 150Mi
cpu: 200m
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 10257
kubeReserved:
memory: 100Mi
`,
overwrittenConfigFields: map[string]interface{}{
"ReadOnlyPort": int32(10257),
"KubeReserved": map[string]string{
"cpu": "200m",
"memory": "100Mi",
},
},
name: "kubelet.conf.d overrides kubelet.conf with subfield override",
},
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
clusterDNS:
- 192.168.1.3
- 192.168.1.4
`,
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
systemReserved:
memory: 1Gi
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 8080
readOnlyPort: 10255
systemReserved:
memory: 2Gi
clusterDNS:
- 192.168.1.1
- 192.168.1.5
- 192.168.1.8
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(8080),
"ReadOnlyPort": int32(10255),
"SystemReserved": map[string]string{
"memory": "2Gi",
},
"ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"},
},
name: "kubelet.conf.d overrides kubelet.conf with slices/lists",
},
{
dropin1: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
`,
dropin2: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 8080
readOnlyPort: 10255
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(8081),
"ReadOnlyPort": int32(10256),
},
cliArgs: []string{
"--port=8081",
"--read-only-port=10256",
},
name: "cli args override kubelet.conf.d",
},
{
kubeletConfig: `
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 9090
clusterDNS:
- 192.168.1.3
`,
overwrittenConfigFields: map[string]interface{}{
"Port": int32(9090),
"ClusterDNS": []string{"192.168.1.2"},
},
cliArgs: []string{
"--port=9090",
"--cluster-dns=192.168.1.2",
},
name: "cli args override kubelet.conf",
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
// Prepare a temporary directory for testing
tempDir := t.TempDir()
kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{}
kubeletFlags := &options.KubeletFlags{}
if len(test.kubeletConfig) > 0 {
// Create the Kubeletconfig
kubeletConfFile := filepath.Join(tempDir, "kubelet.conf")
err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644)
require.NoError(t, err, "failed to create config from a yaml file")
kubeletFlags.KubeletConfigFile = kubeletConfFile
}
if len(test.dropin1) > 0 || len(test.dropin2) > 0 {
// Create kubelet.conf.d directory and drop-in configuration files
kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d")
err := os.Mkdir(kubeletConfDir, 0755)
require.NoError(t, err, "Failed to create kubelet.conf.d directory")
err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644)
require.NoError(t, err, "failed to create config from a yaml file")
err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644)
require.NoError(t, err, "failed to create config from a yaml file")
// Merge the kubelet configurations
err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir)
require.NoError(t, err, "failed to merge kubelet drop-in configs")
}
// Use kubelet config flag precedence
err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs)
require.NoError(t, err, "failed to set the kubelet config flag precedence")
// Verify the merged configuration fields
for fieldName, expectedValue := range test.overwrittenConfigFields {
value := reflect.ValueOf(kubeletConfig).Elem()
field := value.FieldByName(fieldName)
require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName)
}
})
}
}

2
go.mod
View File

@ -43,6 +43,7 @@ require (
github.com/google/go-cmp v0.5.9
github.com/google/gofuzz v1.2.0
github.com/google/uuid v1.3.0
github.com/imdario/mergo v0.3.6
github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5
github.com/libopenstorage/openstorage v1.0.0
github.com/lithammer/dedent v1.1.0
@ -183,7 +184,6 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect