diff --git a/pkg/kubelet/apis/config/validation/validation.go b/pkg/kubelet/apis/config/validation/validation.go index 7a89f87ddd7..e38eb028fe1 100644 --- a/pkg/kubelet/apis/config/validation/validation.go +++ b/pkg/kubelet/apis/config/validation/validation.go @@ -68,7 +68,7 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error allErrors = append(allErrors, fmt.Errorf("invalid configuration: healthzPort (--healthz-port) %v must be between 1 and 65535, inclusive", kc.HealthzPort)) } if !localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && kc.CPUCFSQuotaPeriod != defaultCFSQuota { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod %v requires feature gate CustomCPUCFSQuotaPeriod", kc.CPUCFSQuotaPeriod)) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) %v requires feature gate CustomCPUCFSQuotaPeriod", kc.CPUCFSQuotaPeriod)) } if localFeatureGate.Enabled(features.CPUCFSQuotaPeriod) && utilvalidation.IsInRange(int(kc.CPUCFSQuotaPeriod.Duration), int(1*time.Microsecond), int(time.Second)) != nil { allErrors = append(allErrors, fmt.Errorf("invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) %v must be between 1usec and 1sec, inclusive", kc.CPUCFSQuotaPeriod)) @@ -95,7 +95,7 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error allErrors = append(allErrors, fmt.Errorf("invalid configuration: kubeAPIQPS (--kube-api-qps) %v must not be a negative number", kc.KubeAPIQPS)) } if kc.NodeStatusMaxImages < -1 { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: nodeStatusMaxImages (--node-status-max-images) must be -1 or greater")) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: nodeStatusMaxImages (--node-status-max-images) %v must be -1 or greater", kc.NodeStatusMaxImages)) } if kc.MaxOpenFiles < 0 { allErrors = append(allErrors, fmt.Errorf("invalid configuration: maxOpenFiles (--max-open-files) %v must not be a negative number", kc.MaxOpenFiles)) @@ -143,28 +143,31 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error case kubeletconfig.RestrictedTopologyManagerPolicy: case kubeletconfig.SingleNumaNodeTopologyManagerPolicy: default: - allErrors = append(allErrors, fmt.Errorf("invalid configuration: topologyManagerPolicy non-allowable value: %v", kc.TopologyManagerPolicy)) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: topologyManagerPolicy (--topology-manager-policy) %q must be one of: %q", kc.TopologyManagerPolicy, []string{kubeletconfig.NoneTopologyManagerPolicy, kubeletconfig.BestEffortTopologyManagerPolicy, kubeletconfig.RestrictedTopologyManagerPolicy, kubeletconfig.SingleNumaNodeTopologyManagerPolicy})) } if kc.TopologyManagerScope != kubeletconfig.ContainerTopologyManagerScope && !localFeatureGate.Enabled(features.TopologyManager) { allErrors = append(allErrors, fmt.Errorf("invalid configuration: topologyManagerScope %v requires feature gate TopologyManager", kc.TopologyManagerScope)) } - if kc.TopologyManagerScope != kubeletconfig.ContainerTopologyManagerScope && kc.TopologyManagerScope != kubeletconfig.PodTopologyManagerScope { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: topologyManagerScope non-allowable value: %v", kc.TopologyManagerScope)) + switch kc.TopologyManagerScope { + case kubeletconfig.ContainerTopologyManagerScope: + case kubeletconfig.PodTopologyManagerScope: + default: + allErrors = append(allErrors, fmt.Errorf("invalid configuration: topologyManagerScope (--topology-manager-scope) %q must be one of: %q, or %q", kc.TopologyManagerScope, kubeletconfig.ContainerTopologyManagerScope, kubeletconfig.PodTopologyManagerScope)) } if localFeatureGate.Enabled(features.GracefulNodeShutdown) { - if kc.ShutdownGracePeriod.Duration < 0 || kc.ShutdownGracePeriodCriticalPods.Duration < 0 || kc.ShutdownGracePeriodCriticalPods.Duration > kc.ShutdownGracePeriod.Duration { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: ShutdownGracePeriod %v must be >= 0, ShutdownGracePeriodCriticalPods %v must be >= 0, and ShutdownGracePeriodCriticalPods %v must be <= ShutdownGracePeriod %v", kc.ShutdownGracePeriod, kc.ShutdownGracePeriodCriticalPods, kc.ShutdownGracePeriodCriticalPods, kc.ShutdownGracePeriod)) + if kc.ShutdownGracePeriodCriticalPods.Duration > kc.ShutdownGracePeriod.Duration { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: shutdownGracePeriodCriticalPods %v must be <= shutdownGracePeriod %v", kc.ShutdownGracePeriodCriticalPods, kc.ShutdownGracePeriod)) } - if kc.ShutdownGracePeriod.Duration > 0 && kc.ShutdownGracePeriod.Duration < time.Duration(time.Second) { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: ShutdownGracePeriod %v must be either zero or otherwise >= 1 sec", kc.ShutdownGracePeriod)) + if kc.ShutdownGracePeriod.Duration < 0 || (kc.ShutdownGracePeriod.Duration > 0 && kc.ShutdownGracePeriod.Duration < time.Second) { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: shutdownGracePeriod %v must be either zero or otherwise >= 1 sec", kc.ShutdownGracePeriod)) } - if kc.ShutdownGracePeriodCriticalPods.Duration > 0 && kc.ShutdownGracePeriodCriticalPods.Duration < time.Duration(time.Second) { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: ShutdownGracePeriodCriticalPods %v must be either zero or otherwise >= 1 sec", kc.ShutdownGracePeriodCriticalPods)) + if kc.ShutdownGracePeriodCriticalPods.Duration < 0 || (kc.ShutdownGracePeriodCriticalPods.Duration > 0 && kc.ShutdownGracePeriodCriticalPods.Duration < time.Second) { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: shutdownGracePeriodCriticalPods %v must be either zero or otherwise >= 1 sec", kc.ShutdownGracePeriodCriticalPods)) } } if (kc.ShutdownGracePeriod.Duration > 0 || kc.ShutdownGracePeriodCriticalPods.Duration > 0) && !localFeatureGate.Enabled(features.GracefulNodeShutdown) { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: Specifying ShutdownGracePeriod or ShutdownGracePeriodCriticalPods requires feature gate GracefulNodeShutdown")) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: specifying shutdownGracePeriod or shutdownGracePeriodCriticalPods requires feature gate GracefulNodeShutdown")) } if localFeatureGate.Enabled(features.GracefulNodeShutdownBasedOnPodPriority) { if len(kc.ShutdownGracePeriodByPodPriority) != 0 && (kc.ShutdownGracePeriod.Duration > 0 || kc.ShutdownGracePeriodCriticalPods.Duration > 0) { @@ -177,12 +180,16 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error } } if localFeatureGate.Enabled(features.NodeSwap) { - if kc.MemorySwap.SwapBehavior != "" && kc.MemorySwap.SwapBehavior != kubetypes.LimitedSwap && kc.MemorySwap.SwapBehavior != kubetypes.UnlimitedSwap { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: MemorySwap.SwapBehavior %v must be one of: LimitedSwap, UnlimitedSwap", kc.MemorySwap.SwapBehavior)) + switch kc.MemorySwap.SwapBehavior { + case "": + case kubetypes.LimitedSwap: + case kubetypes.UnlimitedSwap: + default: + allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q, or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.UnlimitedSwap)) } } if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: MemorySwap.SwapBehavior cannot be set when NodeSwap feature flag is disabled")) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior cannot be set when NodeSwap feature flag is disabled")) } for _, val := range kc.EnforceNodeAllocatable { @@ -190,15 +197,15 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error case kubetypes.NodeAllocatableEnforcementKey: case kubetypes.SystemReservedEnforcementKey: if kc.SystemReservedCgroup == "" { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: systemReservedCgroup (--system-reserved-cgroup) must be specified when 'system-reserved' contained in enforceNodeAllocatable (--enforce-node-allocatable)")) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: systemReservedCgroup (--system-reserved-cgroup) must be specified when %q contained in enforceNodeAllocatable (--enforce-node-allocatable)", kubetypes.SystemReservedEnforcementKey)) } case kubetypes.KubeReservedEnforcementKey: if kc.KubeReservedCgroup == "" { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: kubeReservedCgroup (--kube-reserved-cgroup) must be specified when 'kube-reserved' contained in enforceNodeAllocatable (--enforce-node-allocatable)")) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: kubeReservedCgroup (--kube-reserved-cgroup) must be specified when %q contained in enforceNodeAllocatable (--enforce-node-allocatable)", kubetypes.KubeReservedEnforcementKey)) } case kubetypes.NodeAllocatableNoneKey: if len(kc.EnforceNodeAllocatable) > 1 { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: enforceNodeAllocatable (--enforce-node-allocatable) may not contain additional enforcements when '%s' is specified", kubetypes.NodeAllocatableNoneKey)) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: enforceNodeAllocatable (--enforce-node-allocatable) may not contain additional enforcements when %q is specified", kubetypes.NodeAllocatableNoneKey)) } default: allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for enforceNodeAllocatable (--enforce-node-allocatable). Valid options are %q, %q, %q, or %q", @@ -216,10 +223,10 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error if kc.ReservedSystemCPUs != "" { // --reserved-cpus does not support --system-reserved-cgroup or --kube-reserved-cgroup if kc.SystemReservedCgroup != "" || kc.KubeReservedCgroup != "" { - allErrors = append(allErrors, fmt.Errorf("can't use reservedSystemCPUs (--reserved-cpus) with systemReservedCgroup (--system-reserved-cgroup) or kubeReservedCgroup (--kube-reserved-cgroup)")) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: can't use reservedSystemCPUs (--reserved-cpus) with systemReservedCgroup (--system-reserved-cgroup) or kubeReservedCgroup (--kube-reserved-cgroup)")) } if _, err := cpuset.Parse(kc.ReservedSystemCPUs); err != nil { - allErrors = append(allErrors, fmt.Errorf("unable to parse reservedSystemCPUs (--reserved-cpus), error: %v", err)) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: unable to parse reservedSystemCPUs (--reserved-cpus) %v, error: %w", kc.ReservedSystemCPUs, err)) } } @@ -238,7 +245,7 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error allErrors = append(allErrors, fmt.Errorf("invalid configuration: memoryThrottlingFactor is required when MemoryQoS feature flag is enabled")) } if kc.MemoryThrottlingFactor != nil && (*kc.MemoryThrottlingFactor <= 0 || *kc.MemoryThrottlingFactor > 1.0) { - allErrors = append(allErrors, fmt.Errorf("invalid configuration: memoryThrottlingFactor %v must be greater than 0 and less than or equal to 1.0", kc.MemoryThrottlingFactor)) + allErrors = append(allErrors, fmt.Errorf("invalid configuration: memoryThrottlingFactor %v must be greater than 0 and less than or equal to 1.0", *kc.MemoryThrottlingFactor)) } return utilerrors.NewAggregate(allErrors) diff --git a/pkg/kubelet/apis/config/validation/validation_reserved_memory.go b/pkg/kubelet/apis/config/validation/validation_reserved_memory.go index ba4d9a289ff..abc133ce876 100644 --- a/pkg/kubelet/apis/config/validation/validation_reserved_memory.go +++ b/pkg/kubelet/apis/config/validation/validation_reserved_memory.go @@ -41,17 +41,17 @@ func validateReservedMemoryConfiguration(kc *kubeletconfig.KubeletConfiguration) for resourceName, q := range reservedMemory.Limits { if !reservedMemorySupportedLimit(resourceName) { - errors = append(errors, fmt.Errorf("the limit type %q for NUMA node %d is not supported, only %v is accepted", resourceName, numaNode, []v1.ResourceName{v1.ResourceMemory, v1.ResourceHugePagesPrefix + ""})) + errors = append(errors, fmt.Errorf("invalid configuration: the limit type %q for NUMA node %d is not supported, only %v is accepted", resourceName, numaNode, []v1.ResourceName{v1.ResourceMemory, v1.ResourceHugePagesPrefix + ""})) } // validates that the limit has non-zero value if q.IsZero() { - errors = append(errors, fmt.Errorf("reserved memory may not be zero for NUMA node %d and resource %q", numaNode, resourceName)) + errors = append(errors, fmt.Errorf("invalid configuration: reserved memory may not be zero for NUMA node %d and resource %q", numaNode, resourceName)) } // validates that no duplication for NUMA node and limit type occurred if _, ok := numaTypeDuplicates[numaNode][resourceName]; ok { - errors = append(errors, fmt.Errorf("the reserved memory has a duplicate value for NUMA node %d and resource %q", numaNode, resourceName)) + errors = append(errors, fmt.Errorf("invalid configuration: the reserved memory has a duplicate value for NUMA node %d and resource %q", numaNode, resourceName)) } numaTypeDuplicates[numaNode][resourceName] = true } diff --git a/pkg/kubelet/apis/config/validation/validation_reserved_memory_test.go b/pkg/kubelet/apis/config/validation/validation_reserved_memory_test.go index 1910ffaa1eb..d1850375c23 100644 --- a/pkg/kubelet/apis/config/validation/validation_reserved_memory_test.go +++ b/pkg/kubelet/apis/config/validation/validation_reserved_memory_test.go @@ -68,7 +68,7 @@ func TestValidateReservedMemoryConfiguration(t *testing.T) { }, }, }, - expectedError: fmt.Errorf("the reserved memory has a duplicate value for NUMA node %d and resource %q", 0, v1.ResourceMemory), + expectedError: fmt.Errorf("invalid configuration: the reserved memory has a duplicate value for NUMA node %d and resource %q", 0, v1.ResourceMemory), }, { description: "The reserved memory has unsupported limit type", @@ -82,7 +82,7 @@ func TestValidateReservedMemoryConfiguration(t *testing.T) { }, }, }, - expectedError: fmt.Errorf("the limit type %q for NUMA node %d is not supported, only [memory hugepages-] is accepted", "blabla", 0), + expectedError: fmt.Errorf("invalid configuration: the limit type %q for NUMA node %d is not supported, only [memory hugepages-] is accepted", "blabla", 0), }, { description: "The reserved memory has limit type with zero value", @@ -96,7 +96,7 @@ func TestValidateReservedMemoryConfiguration(t *testing.T) { }, }, }, - expectedError: fmt.Errorf("reserved memory may not be zero for NUMA node %d and resource %q", 0, v1.ResourceMemory), + expectedError: fmt.Errorf("invalid configuration: reserved memory may not be zero for NUMA node %d and resource %q", 0, v1.ResourceMemory), }, } diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index 2128cbbe423..37e4d0b0c7b 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -14,22 +14,23 @@ See the License for the specific language governing permissions and limitations under the License. */ -package validation +package validation_test import ( + "strings" "testing" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilerrors "k8s.io/apimachinery/pkg/util/errors" componentbaseconfig "k8s.io/component-base/config" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/apis/config/validation" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" utilpointer "k8s.io/utils/pointer" ) -func TestValidateKubeletConfiguration(t *testing.T) { - successCase1 := &kubeletconfig.KubeletConfiguration{ +var ( + successConfig = kubeletconfig.KubeletConfiguration{ CgroupsPerQOS: true, EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved"}, SystemReservedCgroup: "/system.slice", @@ -70,200 +71,440 @@ func TestValidateKubeletConfiguration(t *testing.T) { Format: "text", }, } - if allErrors := ValidateKubeletConfiguration(successCase1); allErrors != nil { - t.Errorf("expect no errors, got %v", allErrors) - } +) - successCase2 := &kubeletconfig.KubeletConfiguration{ - CgroupsPerQOS: true, - EnforceNodeAllocatable: []string{"pods"}, - SystemReservedCgroup: "", - KubeReservedCgroup: "", - SystemCgroups: "", - CgroupRoot: "", - EventBurst: 10, - EventRecordQPS: 5, - HealthzPort: 10248, - ImageGCHighThresholdPercent: 85, - ImageGCLowThresholdPercent: 80, - IPTablesDropBit: 15, - IPTablesMasqueradeBit: 14, - KubeAPIBurst: 10, - KubeAPIQPS: 5, - MaxOpenFiles: 1000000, - MaxPods: 110, - OOMScoreAdj: -999, - PodsPerCore: 100, - Port: 65535, - ReadOnlyPort: 0, - RegistryBurst: 10, - RegistryPullQPS: 5, - HairpinMode: kubeletconfig.PromiscuousBridge, - NodeLeaseDurationSeconds: 1, - CPUCFSQuotaPeriod: metav1.Duration{Duration: 50 * time.Millisecond}, - ReservedSystemCPUs: "0-3", - TopologyManagerScope: kubeletconfig.ContainerTopologyManagerScope, - TopologyManagerPolicy: kubeletconfig.NoneTopologyManagerPolicy, - ShutdownGracePeriod: metav1.Duration{Duration: 10 * time.Minute}, - ShutdownGracePeriodCriticalPods: metav1.Duration{Duration: 0}, - MemoryThrottlingFactor: utilpointer.Float64Ptr(0.9), - FeatureGates: map[string]bool{ - "CustomCPUCFSQuotaPeriod": true, - "MemoryQoS": true, - "GracefulNodeShutdownBasedOnPodPriority": true, - }, - Logging: componentbaseconfig.LoggingConfiguration{ - Format: "text", - }, - } - if allErrors := ValidateKubeletConfiguration(successCase2); allErrors != nil { - t.Errorf("expect no errors, got %v", allErrors) - } - - successCase3 := &kubeletconfig.KubeletConfiguration{ - CgroupsPerQOS: true, - EnforceNodeAllocatable: []string{"pods"}, - SystemReservedCgroup: "", - KubeReservedCgroup: "", - SystemCgroups: "", - CgroupRoot: "", - EventBurst: 10, - EventRecordQPS: 5, - HealthzPort: 10248, - ImageGCHighThresholdPercent: 85, - ImageGCLowThresholdPercent: 80, - IPTablesDropBit: 15, - IPTablesMasqueradeBit: 14, - KubeAPIBurst: 10, - KubeAPIQPS: 5, - MaxOpenFiles: 1000000, - MaxPods: 110, - OOMScoreAdj: -999, - PodsPerCore: 100, - Port: 65535, - ReadOnlyPort: 0, - RegistryBurst: 10, - RegistryPullQPS: 5, - HairpinMode: kubeletconfig.PromiscuousBridge, - NodeLeaseDurationSeconds: 1, - CPUCFSQuotaPeriod: metav1.Duration{Duration: 50 * time.Millisecond}, - ReservedSystemCPUs: "0-3", - TopologyManagerScope: kubeletconfig.ContainerTopologyManagerScope, - TopologyManagerPolicy: kubeletconfig.NoneTopologyManagerPolicy, - ShutdownGracePeriod: metav1.Duration{Duration: 0}, - ShutdownGracePeriodCriticalPods: metav1.Duration{Duration: 0}, - ShutdownGracePeriodByPodPriority: []kubeletconfig.ShutdownGracePeriodByPodPriority{ - { - Priority: 0, - ShutdownGracePeriodSeconds: 10, +func TestValidateKubeletConfiguration(t *testing.T) { + cases := []struct { + name string + configure func(config *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration + errMsg string + }{ + { + name: "Success", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + return conf }, }, - MemorySwap: kubeletconfig.MemorySwapConfiguration{SwapBehavior: kubetypes.UnlimitedSwap}, - MemoryThrottlingFactor: utilpointer.Float64Ptr(0.5), - FeatureGates: map[string]bool{ - "CustomCPUCFSQuotaPeriod": true, - "GracefulNodeShutdown": true, - "GracefulNodeShutdownBasedOnPodPriority": true, - "NodeSwap": true, - "MemoryQoS": true, - }, - Logging: componentbaseconfig.LoggingConfiguration{ - Format: "text", - }, - } - if allErrors := ValidateKubeletConfiguration(successCase3); allErrors != nil { - t.Errorf("expect no errors, got %v", allErrors) - } - - errorCase1 := &kubeletconfig.KubeletConfiguration{ - CgroupsPerQOS: false, - EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved", "illegal-key"}, - SystemCgroups: "/", - CgroupRoot: "", - EventBurst: -10, - EventRecordQPS: -10, - HealthzPort: -10, - ImageGCHighThresholdPercent: 101, - ImageGCLowThresholdPercent: 101, - IPTablesDropBit: -10, - IPTablesMasqueradeBit: -10, - KubeAPIBurst: -10, - KubeAPIQPS: -10, - MaxOpenFiles: -10, - MaxPods: -10, - OOMScoreAdj: -1001, - PodsPerCore: -10, - Port: 0, - ReadOnlyPort: -10, - RegistryBurst: -10, - RegistryPullQPS: -10, - HairpinMode: "foo", - NodeLeaseDurationSeconds: -1, - CPUCFSQuotaPeriod: metav1.Duration{Duration: 100 * time.Millisecond}, - ShutdownGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - ShutdownGracePeriodCriticalPods: metav1.Duration{Duration: 60 * time.Second}, - ShutdownGracePeriodByPodPriority: []kubeletconfig.ShutdownGracePeriodByPodPriority{ - { - Priority: 0, - ShutdownGracePeriodSeconds: 10, + { + name: "invalid NodeLeaseDurationSeconds", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.NodeLeaseDurationSeconds = 0 + return conf }, + errMsg: "invalid configuration: nodeLeaseDurationSeconds must be greater than 0", }, - Logging: componentbaseconfig.LoggingConfiguration{ - Format: "", + { + name: "specify EnforceNodeAllocatable without enabling CgroupsPerQOS", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.CgroupsPerQOS = false + conf.EnforceNodeAllocatable = []string{"pods"} + return conf + }, + errMsg: "invalid configuration: enforceNodeAllocatable (--enforce-node-allocatable) is not supported unless cgroupsPerQOS (--cgroups-per-qos) is set to true", + }, + { + name: "specify SystemCgroups without CgroupRoot", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.SystemCgroups = "/" + conf.CgroupRoot = "" + return conf + }, + errMsg: "invalid configuration: systemCgroups (--system-cgroups) was specified and cgroupRoot (--cgroup-root) was not specified", + }, + { + name: "invalid EventBurst", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.EventBurst = -1 + return conf + }, + errMsg: "invalid configuration: eventBurst (--event-burst) -1 must not be a negative number", + }, + { + name: "invalid EventRecordQPS", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.EventRecordQPS = -1 + return conf + }, + errMsg: "invalid configuration: eventRecordQPS (--event-qps) -1 must not be a negative number", + }, + { + name: "invalid HealthzPort", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.HealthzPort = 65536 + return conf + }, + errMsg: "invalid configuration: healthzPort (--healthz-port) 65536 must be between 1 and 65535, inclusive", + }, + { + name: "specify CPUCFSQuotaPeriod without enabling CPUCFSQuotaPeriod", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"CustomCPUCFSQuotaPeriod": false} + conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 200 * time.Millisecond} + return conf + }, + errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {200ms} requires feature gate CustomCPUCFSQuotaPeriod", + }, + { + name: "invalid CPUCFSQuotaPeriod", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"CustomCPUCFSQuotaPeriod": true} + conf.CPUCFSQuotaPeriod = metav1.Duration{Duration: 2 * time.Second} + return conf + }, + errMsg: "invalid configuration: cpuCFSQuotaPeriod (--cpu-cfs-quota-period) {2s} must be between 1usec and 1sec, inclusive", + }, + { + name: "invalid ImageGCHighThresholdPercent", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ImageGCHighThresholdPercent = 101 + return conf + }, + errMsg: "invalid configuration: imageGCHighThresholdPercent (--image-gc-high-threshold) 101 must be between 0 and 100, inclusive", + }, + { + name: "invalid ImageGCLowThresholdPercent", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ImageGCLowThresholdPercent = -1 + return conf + }, + errMsg: "invalid configuration: imageGCLowThresholdPercent (--image-gc-low-threshold) -1 must be between 0 and 100, inclusive", + }, + { + name: "ImageGCLowThresholdPercent is equal to ImageGCHighThresholdPercent", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ImageGCHighThresholdPercent = 0 + conf.ImageGCLowThresholdPercent = 0 + return conf + }, + errMsg: "invalid configuration: imageGCLowThresholdPercent (--image-gc-low-threshold) 0 must be less than imageGCHighThresholdPercent (--image-gc-high-threshold) 0", + }, + { + name: "ImageGCLowThresholdPercent is greater than ImageGCHighThresholdPercent", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ImageGCHighThresholdPercent = 0 + conf.ImageGCLowThresholdPercent = 1 + return conf + }, + errMsg: "invalid configuration: imageGCLowThresholdPercent (--image-gc-low-threshold) 1 must be less than imageGCHighThresholdPercent (--image-gc-high-threshold) 0", + }, + { + name: "invalid IPTablesDropBit", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.IPTablesDropBit = 32 + return conf + }, + errMsg: "invalid configuration: iptablesDropBit (--iptables-drop-bit) 32 must be between 0 and 31, inclusive", + }, + { + name: "invalid IPTablesMasqueradeBit", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.IPTablesMasqueradeBit = 32 + return conf + }, + errMsg: "invalid configuration: iptablesMasqueradeBit (--iptables-masquerade-bit) 32 must be between 0 and 31, inclusive", + }, + { + name: "invalid KubeAPIBurst", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.KubeAPIBurst = -1 + return conf + }, + errMsg: "invalid configuration: kubeAPIBurst (--kube-api-burst) -1 must not be a negative number", + }, + { + name: "invalid KubeAPIQPS", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.KubeAPIQPS = -1 + return conf + }, + errMsg: "invalid configuration: kubeAPIQPS (--kube-api-qps) -1 must not be a negative number", + }, + { + name: "invalid NodeStatusMaxImages", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.NodeStatusMaxImages = -2 + return conf + }, + errMsg: "invalid configuration: nodeStatusMaxImages (--node-status-max-images) -2 must be -1 or greater", + }, + { + name: "invalid MaxOpenFiles", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.MaxOpenFiles = -1 + return conf + }, + errMsg: "invalid configuration: maxOpenFiles (--max-open-files) -1 must not be a negative number", + }, + { + name: "invalid MaxPods", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.MaxPods = -1 + return conf + }, + errMsg: "invalid configuration: maxPods (--max-pods) -1 must not be a negative number", + }, + { + name: "invalid OOMScoreAdj", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.OOMScoreAdj = 1001 + return conf + }, + errMsg: "invalid configuration: oomScoreAdj (--oom-score-adj) 1001 must be between -1000 and 1000, inclusive", + }, + { + name: "invalid PodsPerCore", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.PodsPerCore = -1 + return conf + }, + errMsg: "invalid configuration: podsPerCore (--pods-per-core) -1 must not be a negative number", + }, + { + name: "invalid Port", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.Port = 65536 + return conf + }, + errMsg: "invalid configuration: port (--port) 65536 must be between 1 and 65535, inclusive", + }, + { + name: "invalid ReadOnlyPort", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ReadOnlyPort = 65536 + return conf + }, + errMsg: "invalid configuration: readOnlyPort (--read-only-port) 65536 must be between 0 and 65535, inclusive", + }, + { + name: "invalid RegistryBurst", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.RegistryBurst = -1 + return conf + }, + errMsg: "invalid configuration: registryBurst (--registry-burst) -1 must not be a negative number", + }, + { + name: "invalid RegistryPullQPS", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.RegistryPullQPS = -1 + return conf + }, + errMsg: "invalid configuration: registryPullQPS (--registry-qps) -1 must not be a negative number", + }, + { + name: "specify ServerTLSBootstrap without enabling RotateKubeletServerCertificate", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"RotateKubeletServerCertificate": false} + conf.ServerTLSBootstrap = true + return conf + }, + errMsg: "invalid configuration: serverTLSBootstrap true requires feature gate RotateKubeletServerCertificate", + }, + { + name: "use SingleNumaNodeTopologyManagerPolicy without enabling TopologyManager", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"TopologyManager": false} + conf.TopologyManagerPolicy = kubeletconfig.SingleNumaNodeTopologyManagerPolicy + return conf + }, + errMsg: "invalid configuration: topologyManagerPolicy single-numa-node requires feature gate TopologyManager", + }, + { + name: "invalid TopologyManagerPolicy", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.TopologyManagerPolicy = "invalid-policy" + return conf + }, + errMsg: "invalid configuration: topologyManagerPolicy (--topology-manager-policy) \"invalid-policy\" must be one of: [\"none\" \"best-effort\" \"restricted\" \"single-numa-node\"]", + }, + { + name: "use PodTopologyManagerScope without enabling TopologyManager", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"TopologyManager": false} + conf.TopologyManagerScope = kubeletconfig.PodTopologyManagerScope + return conf + }, + errMsg: "invalid configuration: topologyManagerScope pod requires feature gate TopologyManager", + }, + { + name: "invalid TopologyManagerScope", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.TopologyManagerScope = "invalid-scope" + return conf + }, + errMsg: "invalid configuration: topologyManagerScope (--topology-manager-scope) \"invalid-scope\" must be one of: \"container\", or \"pod\"", + }, + { + name: "ShutdownGracePeriodCriticalPods is greater than ShutdownGracePeriod", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"GracefulNodeShutdown": true} + conf.ShutdownGracePeriodCriticalPods = metav1.Duration{Duration: 2 * time.Second} + conf.ShutdownGracePeriod = metav1.Duration{Duration: 1 * time.Second} + return conf + }, + errMsg: "invalid configuration: shutdownGracePeriodCriticalPods {2s} must be <= shutdownGracePeriod {1s}", + }, + { + name: "ShutdownGracePeriod is less than 1 sec", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"GracefulNodeShutdown": true} + conf.ShutdownGracePeriod = metav1.Duration{Duration: 1 * time.Millisecond} + return conf + }, + errMsg: "invalid configuration: shutdownGracePeriod {1ms} must be either zero or otherwise >= 1 sec", + }, + { + name: "ShutdownGracePeriodCriticalPods is less than 1 sec", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"GracefulNodeShutdown": true} + conf.ShutdownGracePeriodCriticalPods = metav1.Duration{Duration: 1 * time.Millisecond} + return conf + }, + errMsg: "invalid configuration: shutdownGracePeriodCriticalPods {1ms} must be either zero or otherwise >= 1 sec", + }, + { + name: "specify ShutdownGracePeriod without enabling GracefulNodeShutdown", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"GracefulNodeShutdown": false} + conf.ShutdownGracePeriod = metav1.Duration{Duration: 1 * time.Second} + return conf + }, + errMsg: "invalid configuration: specifying shutdownGracePeriod or shutdownGracePeriodCriticalPods requires feature gate GracefulNodeShutdown", + }, + { + name: "specify ShutdownGracePeriodCriticalPods without enabling GracefulNodeShutdown", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"GracefulNodeShutdown": false} + conf.ShutdownGracePeriodCriticalPods = metav1.Duration{Duration: 1 * time.Second} + return conf + }, + errMsg: "invalid configuration: specifying shutdownGracePeriod or shutdownGracePeriodCriticalPods requires feature gate GracefulNodeShutdown", + }, + { + name: "invalid MemorySwap.SwapBehavior", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"NodeSwap": true} + conf.MemorySwap.SwapBehavior = "invalid-behavior" + return conf + }, + errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\", or \"UnlimitedSwap\"", + }, + { + name: "specify MemorySwap.SwapBehavior without enabling NodeSwap", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"NodeSwap": false} + conf.MemorySwap.SwapBehavior = kubetypes.LimitedSwap + return conf + }, + errMsg: "invalid configuration: memorySwap.swapBehavior cannot be set when NodeSwap feature flag is disabled", + }, + { + name: "specify SystemReservedEnforcementKey without specifying SystemReservedCgroup", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.EnforceNodeAllocatable = []string{kubetypes.SystemReservedEnforcementKey} + conf.SystemReservedCgroup = "" + return conf + }, + errMsg: "invalid configuration: systemReservedCgroup (--system-reserved-cgroup) must be specified when \"system-reserved\" contained in enforceNodeAllocatable (--enforce-node-allocatable)", + }, + { + name: "specify KubeReservedEnforcementKey without specifying KubeReservedCgroup", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.EnforceNodeAllocatable = []string{kubetypes.KubeReservedEnforcementKey} + conf.KubeReservedCgroup = "" + return conf + }, + errMsg: "invalid configuration: kubeReservedCgroup (--kube-reserved-cgroup) must be specified when \"kube-reserved\" contained in enforceNodeAllocatable (--enforce-node-allocatable)", + }, + { + name: "specify NodeAllocatableNoneKey with additional enforcements", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.EnforceNodeAllocatable = []string{kubetypes.NodeAllocatableNoneKey, kubetypes.KubeReservedEnforcementKey} + return conf + }, + errMsg: "invalid configuration: enforceNodeAllocatable (--enforce-node-allocatable) may not contain additional enforcements when \"none\" is specified", + }, + { + name: "invalid EnforceNodeAllocatable", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.EnforceNodeAllocatable = []string{"invalid-enforce-node-allocatable"} + return conf + }, + errMsg: "invalid configuration: option \"invalid-enforce-node-allocatable\" specified for enforceNodeAllocatable (--enforce-node-allocatable). Valid options are \"pods\", \"system-reserved\", \"kube-reserved\", or \"none\"", + }, + { + name: "invalid HairpinMode", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.HairpinMode = "invalid-hair-pin-mode" + return conf + }, + errMsg: "invalid configuration: option \"invalid-hair-pin-mode\" specified for hairpinMode (--hairpin-mode). Valid options are \"none\", \"hairpin-veth\" or \"promiscuous-bridge\"", + }, + { + name: "specify ReservedSystemCPUs with SystemReservedCgroup", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ReservedSystemCPUs = "0-3" + conf.SystemReservedCgroup = "/system.slice" + return conf + }, + errMsg: "invalid configuration: can't use reservedSystemCPUs (--reserved-cpus) with systemReservedCgroup (--system-reserved-cgroup) or kubeReservedCgroup (--kube-reserved-cgroup)", + }, + { + name: "specify ReservedSystemCPUs with KubeReservedCgroup", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ReservedSystemCPUs = "0-3" + conf.KubeReservedCgroup = "/system.slice" + return conf + }, + errMsg: "invalid configuration: can't use reservedSystemCPUs (--reserved-cpus) with systemReservedCgroup (--system-reserved-cgroup) or kubeReservedCgroup (--kube-reserved-cgroup)", + }, + { + name: "invalid ReservedSystemCPUs", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.ReservedSystemCPUs = "invalid-reserved-system-cpus" + return conf + }, + errMsg: "invalid configuration: unable to parse reservedSystemCPUs (--reserved-cpus) invalid-reserved-system-cpus, error:", + }, + { + name: "enable MemoryQoS without specifying MemoryThrottlingFactor", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"MemoryQoS": true} + conf.MemoryThrottlingFactor = nil + return conf + }, + errMsg: "invalid configuration: memoryThrottlingFactor is required when MemoryQoS feature flag is enabled", + }, + { + name: "invalid MemoryThrottlingFactor", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.MemoryThrottlingFactor = utilpointer.Float64(1.1) + return conf + }, + errMsg: "invalid configuration: memoryThrottlingFactor 1.1 must be greater than 0 and less than or equal to 1.0", }, - MemorySwap: kubeletconfig.MemorySwapConfiguration{SwapBehavior: kubetypes.UnlimitedSwap}, - } - const numErrsErrorCase1 = 31 - if allErrors := ValidateKubeletConfiguration(errorCase1); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrsErrorCase1 { - t.Errorf("expect %d errors, got %v", numErrsErrorCase1, len(allErrors.(utilerrors.Aggregate).Errors())) } - errorCase2 := &kubeletconfig.KubeletConfiguration{ - CgroupsPerQOS: true, - EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved"}, - SystemReservedCgroup: "/system.slice", - KubeReservedCgroup: "/kubelet.service", - SystemCgroups: "", - CgroupRoot: "", - EventBurst: 10, - EventRecordQPS: 5, - HealthzPort: 10248, - ImageGCHighThresholdPercent: 85, - ImageGCLowThresholdPercent: 80, - IPTablesDropBit: 15, - IPTablesMasqueradeBit: 14, - KubeAPIBurst: 10, - KubeAPIQPS: 5, - MaxOpenFiles: 1000000, - MaxPods: 110, - OOMScoreAdj: -999, - PodsPerCore: 100, - Port: 65535, - ReadOnlyPort: 0, - RegistryBurst: 10, - RegistryPullQPS: 5, - HairpinMode: kubeletconfig.PromiscuousBridge, - NodeLeaseDurationSeconds: 1, - CPUCFSQuotaPeriod: metav1.Duration{Duration: 50 * time.Millisecond}, - ReservedSystemCPUs: "0-3", - TopologyManagerScope: "invalid", - TopologyManagerPolicy: "invalid", - ShutdownGracePeriod: metav1.Duration{Duration: 40 * time.Second}, - ShutdownGracePeriodCriticalPods: metav1.Duration{Duration: 10 * time.Second}, - MemorySwap: kubeletconfig.MemorySwapConfiguration{SwapBehavior: "invalid"}, - MemoryThrottlingFactor: utilpointer.Float64Ptr(1.1), - FeatureGates: map[string]bool{ - "CustomCPUCFSQuotaPeriod": true, - "GracefulNodeShutdown": true, - "NodeSwap": true, - "MemoryQoS": true, - }, - Logging: componentbaseconfig.LoggingConfiguration{ - Format: "text", - }, - } - const numErrsErrorCase2 = 5 - if allErrors := ValidateKubeletConfiguration(errorCase2); len(allErrors.(utilerrors.Aggregate).Errors()) != numErrsErrorCase2 { - t.Errorf("expect %d errors, got %v", numErrsErrorCase2, len(allErrors.(utilerrors.Aggregate).Errors())) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + errs := validation.ValidateKubeletConfiguration(tc.configure(successConfig.DeepCopy())) + + if len(tc.errMsg) == 0 { + if errs != nil { + t.Errorf("unexpected error: %s", errs) + } + + return + } + + if errs == nil { + t.Errorf("expected error: %s", tc.errMsg) + return + } + + if got := errs.Error(); !strings.Contains(got, tc.errMsg) { + t.Errorf("unexpected error: %s expected to contain %s", got, tc.errMsg) + } + }) } }