github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-06 10:43:56 +00:00
Code Issues Projects Releases Wiki Activity

test/e2e_node: add e2e test for Kubeletconfig drop-in dir

Browse Source 
Signed-off-by: Sohan Kunkerkar <sohank2602@gmail.com>
Co-authored-by: Peter Hunt <pehunt@redhat.com>
This commit is contained in:
Sohan Kunkerkar 2023-10-12 12:10:36 -04:00
parent ee5578be52
commit ad7b9b56f5
5 changed files with 157 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
test/e2e/framework/test_context.go
View File

@ -99,13 +99,14 @@ var (
// Test suite authors can use framework/viper to make all command line
// parameters also configurable via a configuration file.
type TestContextType struct {
KubeConfig string
KubeContext string
KubeAPIContentType string
KubeletRootDir string
CertDir string
Host string
BearerToken string `datapolicy:"token"`
KubeConfig string
KubeContext string
KubeAPIContentType string
KubeletRootDir string
KubeletConfigDropinDir string
CertDir string
Host string
BearerToken string `datapolicy:"token"`
// TODO: Deprecating this over time... instead just use gobindata_util.go , see #23987.
RepoRoot string
// ListImages will list off all images that are used then quit

1
test/e2e/nodefeature/nodefeature.go
View File

@ -37,6 +37,7 @@ var (
GracefulNodeShutdownBasedOnPodPriority = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdownBasedOnPodPriority"))
HostAccess = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("HostAccess"))
ImageID = framework.WithNodeFeature(framework.ValidNodeFeatures.Add(" ImageID"))
KubeletConfigDropInDir = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("KubeletConfigDropInDir"))
LSCIQuotaMonitoring = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("LSCIQuotaMonitoring"))
NodeAllocatable = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeAllocatable"))
NodeProblemDetector = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeProblemDetector"))

9
test/e2e_node/e2e_node_suite_test.go
View File

@ -90,6 +90,7 @@ func registerNodeFlags(flags *flag.FlagSet) {
framework.TestContext.NodeE2E = true
flags.StringVar(&framework.TestContext.BearerToken, "bearer-token", "", "The bearer token to authenticate with. If not specified, it would be a random token. Currently this token is only used in node e2e tests.")
flags.StringVar(&framework.TestContext.NodeName, "node-name", "", "Name of the node to run tests on.")
flags.StringVar(&framework.TestContext.KubeletConfigDropinDir, "config-dir", "", "Path to a directory containing drop-in configurations for the kubelet.")
// TODO(random-liu): Move kubelet start logic out of the test.
// TODO(random-liu): Move log fetch logic out of the test.
// There are different ways to start kubelet (systemd, initd, docker, manually started etc.)
@ -200,6 +201,14 @@ func TestE2eNode(t *testing.T) {
// We're not running in a special mode so lets run tests.
gomega.RegisterFailHandler(ginkgo.Fail)
// Initialize the KubeletConfigDropinDir again if the test doesn't run in run-kubelet-mode.
if framework.TestContext.KubeletConfigDropinDir == "" {
var err error
framework.TestContext.KubeletConfigDropinDir, err = services.KubeletConfigDirCWDDir()
if err != nil {
klog.Errorf("failed to create kubelet config directory: %v", err)
}
}
reportDir := framework.TestContext.ReportDir
if reportDir != "" {
// Create the directory if it doesn't already exist

113
test/e2e_node/kubelet_config_dir_test.go Normal file
View File

@ -0,0 +1,113 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2enode
import (
"context"
"os"
"path/filepath"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/nodefeature"
)
var _ = SIGDescribe("Kubelet Config", framework.WithSlow(), framework.WithSerial(), framework.WithDisruptive(), nodefeature.KubeletConfigDropInDir, func() {
f := framework.NewDefaultFramework("kubelet-config-drop-in-dir-test")
ginkgo.Context("when merging drop-in configs", func() {
var oldcfg *kubeletconfig.KubeletConfiguration
ginkgo.BeforeEach(func(ctx context.Context) {
var err error
oldcfg, err = getCurrentKubeletConfig(ctx)
framework.ExpectNoError(err)
})
ginkgo.AfterEach(func(ctx context.Context) {
files, err := filepath.Glob(filepath.Join(framework.TestContext.KubeletConfigDropinDir, "*"+".conf"))
framework.ExpectNoError(err)
for _, file := range files {
err := os.Remove(file)
framework.ExpectNoError(err)
}
updateKubeletConfig(ctx, f, oldcfg, true)
})
ginkgo.It("should merge kubelet configs correctly", func(ctx context.Context) {
// Get the initial kubelet configuration
initialConfig, err := getCurrentKubeletConfig(ctx)
framework.ExpectNoError(err)
ginkgo.By("Stopping the kubelet")
restartKubelet := stopKubelet()
// wait until the kubelet health check will fail
gomega.Eventually(ctx, func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse())
configDir := framework.TestContext.KubeletConfigDropinDir
contents := []byte(`apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 10255
readOnlyPort: 10257
clusterDNS:
- 192.168.1.10
systemReserved:
memory: 1Gi`)
framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "10-kubelet.conf"), contents, 0755))
contents = []byte(`apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
clusterDNS:
- 192.168.1.1
- 192.168.1.5
- 192.168.1.8
port: 8080
cpuManagerReconcilePeriod: 0s
systemReserved:
memory: 2Gi`)
framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "20-kubelet.conf"), contents, 0755))
ginkgo.By("Restarting the kubelet")
restartKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(ctx, func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue())
mergedConfig, err := getCurrentKubeletConfig(ctx)
framework.ExpectNoError(err)
// Replace specific fields in the initial configuration with expectedConfig values
initialConfig.Port = int32(8080) // not overridden by second file, should be retained.
initialConfig.ReadOnlyPort = int32(10257) // overridden by second file.
initialConfig.SystemReserved = map[string]string{ // overridden by map in second file.
"memory": "2Gi",
}
initialConfig.ClusterDNS = []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"} // overridden by slice in second file.
// This value was explicitly set in the drop-in, make sure it is retained
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: time.Duration(0)}
// Meanwhile, this value was not explicitly set, but could have been overridden by a "default" of 0 for the type.
// Ensure the true default persists.
initialConfig.CPUCFSQuotaPeriod = metav1.Duration{Duration: time.Duration(100000000)}
// Compare the expected config with the merged config
gomega.Expect(initialConfig).To(gomega.BeComparableTo(mergedConfig), "Merged kubelet config does not match the expected configuration.")
})
})
})

29
test/e2e_node/services/kubelet.go
View File

@ -174,6 +174,12 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
return nil, err
}
// KubeletDropInConfiguration directory path
framework.TestContext.KubeletConfigDropinDir, err = KubeletConfigDirCWDDir()
if err != nil {
return nil, err
}
// Create pod directory
podPath, err := createPodDirectory()
if err != nil {
@ -243,6 +249,8 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
unitName = fmt.Sprintf("kubelet-%s.service", unitTimestamp)
cmdArgs = append(cmdArgs,
systemdRun,
// Set the environment variable to enable kubelet config drop-in directory.
"-E", "KUBELET_CONFIG_DROPIN_DIR_ALPHA=yes",
"-p", "Delegate=true",
"-p", logLocation+framework.TestContext.ReportDir+"/kubelet.log",
"--unit="+unitName,
@ -282,6 +290,9 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
kc.FeatureGates = featureGates
}
// Add the KubeletDropinConfigDirectory flag if set.
cmdArgs = append(cmdArgs, "--config-dir", framework.TestContext.KubeletConfigDropinDir)
// Keep hostname override for convenience.
if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override.
cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName)
@ -295,7 +306,7 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint)
}
if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil {
if err := WriteKubeletConfigFile(kc, kubeletConfigPath); err != nil {
return nil, err
}
// add the flag to load config from a file
@ -324,8 +335,8 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
return server, server.start()
}
// writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename
func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error {
// WriteKubeletConfigFile writes the kubelet config file based on the args and returns the filename
func WriteKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error {
data, err := kubeletconfigcodec.EncodeKubeletConfig(internal, kubeletconfigv1beta1.SchemeGroupVersion)
if err != nil {
return err
@ -408,6 +419,18 @@ func kubeletConfigCWDPath() (string, error) {
return filepath.Join(cwd, "kubelet-config"), nil
}
func KubeletConfigDirCWDDir() (string, error) {
cwd, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("failed to get current working directory: %w", err)
}
dir := filepath.Join(cwd, "kubelet.conf.d")
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
return dir, nil
}
// like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig
// returns a fully-qualified path to the kubeconfig file
func createKubeconfigCWD() (string, error) {