diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 233f87be0d7..6d16fc9fda7 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -172,6 +172,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,C API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,CronJobControllerConfiguration,ConcurrentCronJobSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,DaemonSetControllerConfiguration,ConcurrentDaemonSetSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,DeploymentControllerConfiguration,ConcurrentDeploymentSyncs +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,DeviceTaintEvictionControllerConfiguration,ConcurrentSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,EndpointControllerConfiguration,ConcurrentEndpointSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,EndpointControllerConfiguration,EndpointUpdatesBatchPeriod API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,EndpointSliceControllerConfiguration,ConcurrentServiceEndpointSyncs @@ -199,6 +200,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,DaemonSetController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,DeploymentController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,DeprecatedController +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,DeviceTaintEvictionController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,EndpointController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,EndpointSliceController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,EndpointSliceMirroringController diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 676e8ea03ed..54a9df182fc 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -276,7 +276,7 @@ func newDeviceTaintEvictionController(ctx context.Context, controllerContext Con controllerName, ) return newControllerLoop(func(ctx context.Context) { - if err := deviceTaintEvictionController.Run(ctx); err != nil { + if err := deviceTaintEvictionController.Run(ctx, int(controllerContext.ComponentConfig.DeviceTaintEvictionController.ConcurrentSyncs)); err != nil { klog.FromContext(ctx).Error(err, "Device taint processing leading to Pod eviction failed and is now paused") } <-ctx.Done() diff --git a/cmd/kube-controller-manager/app/options/devicetaintevictioncontroller.go b/cmd/kube-controller-manager/app/options/devicetaintevictioncontroller.go new file mode 100644 index 00000000000..64bd263ed4d --- /dev/null +++ b/cmd/kube-controller-manager/app/options/devicetaintevictioncontroller.go @@ -0,0 +1,63 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + + "github.com/spf13/pflag" + + devicetaintevictionconfig "k8s.io/kubernetes/pkg/controller/devicetainteviction/config" +) + +// DeviceTaintEvictionControllerOptions holds the DeviceTaintEvictionController options. +type DeviceTaintEvictionControllerOptions struct { + *devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration +} + +// AddFlags adds flags related to DeviceTaintEvictionController for controller manager to the specified FlagSet. +func (o *DeviceTaintEvictionControllerOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.Int32Var(&o.ConcurrentSyncs, "concurrent-device-taint-eviction-syncs", o.ConcurrentSyncs, "The number of operations (evicting pods, updating DeviceTaintRule status) allowed to run concurrently. Greater number = more responsive, but more CPU (and network) load") +} + +// ApplyTo fills up DeviceTaintEvictionController config with options. +func (o *DeviceTaintEvictionControllerOptions) ApplyTo(cfg *devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration) error { + if o == nil { + return nil + } + + cfg.ConcurrentSyncs = o.ConcurrentSyncs + + return nil +} + +// Validate checks validation of DeviceTaintEvictionControllerOptions. +func (o *DeviceTaintEvictionControllerOptions) Validate() []error { + if o == nil { + return nil + } + + var errs []error + if o.ConcurrentSyncs <= 0 { + errs = append(errs, fmt.Errorf("concurrent-device-taint-eviction-syncs must be greater than zero, got %d", o.ConcurrentSyncs)) + } + return errs +} diff --git a/cmd/kube-controller-manager/app/options/devicetaintevictioncontroller_test.go b/cmd/kube-controller-manager/app/options/devicetaintevictioncontroller_test.go new file mode 100644 index 00000000000..a804ad7675c --- /dev/null +++ b/cmd/kube-controller-manager/app/options/devicetaintevictioncontroller_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "reflect" + "strings" + "testing" + + "github.com/spf13/pflag" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + devicetaintevictionconfig "k8s.io/kubernetes/pkg/controller/devicetainteviction/config" +) + +func TestDeviceTaintEvictionControllerOptions_AddFlags(t *testing.T) { + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + opts := &DeviceTaintEvictionControllerOptions{ + &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 50, + }, + } + + opts.AddFlags(fs) + + // Test that the flag was added + flag := fs.Lookup("concurrent-device-taint-eviction-syncs") + if flag == nil { + t.Error("concurrent-device-taint-eviction-syncs flag was not added") + return + } + + // Test that the flag has the correct default value + if flag.DefValue != "50" { + t.Errorf("expected default value 50, got %s", flag.DefValue) + } + + // Test flag parsing + args := []string{"--concurrent-device-taint-eviction-syncs=25"} + if err := fs.Parse(args); err != nil { + t.Errorf("failed to parse flags: %v", err) + } + + if opts.ConcurrentSyncs != 25 { + t.Errorf("expected ConcurrentSyncs to be 25, got %d", opts.ConcurrentSyncs) + } +} + +func TestDeviceTaintEvictionControllerOptions_AddFlags_Nil(t *testing.T) { + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + var opts *DeviceTaintEvictionControllerOptions + + // Should not panic when options is nil + opts.AddFlags(fs) + + // Flag should not be added + flag := fs.Lookup("concurrent-device-taint-eviction-syncs") + if flag != nil { + t.Error("concurrent-device-taint-eviction-syncs flag should not be added when options is nil") + } +} + +func TestDeviceTaintEvictionControllerOptions_ApplyTo(t *testing.T) { + opts := &DeviceTaintEvictionControllerOptions{ + &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 75, + }, + } + + cfg := &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{} + + err := opts.ApplyTo(cfg) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if cfg.ConcurrentSyncs != 75 { + t.Errorf("expected ConcurrentSyncs to be 75, got %d", cfg.ConcurrentSyncs) + } +} + +func TestDeviceTaintEvictionControllerOptions_ApplyTo_Nil(t *testing.T) { + var opts *DeviceTaintEvictionControllerOptions + cfg := &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 50, + } + + err := opts.ApplyTo(cfg) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + // Configuration should remain unchanged + if cfg.ConcurrentSyncs != 50 { + t.Errorf("expected ConcurrentSyncs to remain 50, got %d", cfg.ConcurrentSyncs) + } +} + +func TestDeviceTaintEvictionControllerOptions_Validate(t *testing.T) { + testCases := []struct { + name string + concurrentSyncs int32 + expectErrors bool + expectedErrorSubString string + }{ + { + name: "valid concurrent syncs", + concurrentSyncs: 50, + expectErrors: false, + }, + { + name: "valid minimum concurrent syncs", + concurrentSyncs: 1, + expectErrors: false, + }, + { + name: "invalid zero concurrent syncs", + concurrentSyncs: 0, + expectErrors: true, + expectedErrorSubString: "concurrent-device-taint-eviction-syncs must be greater than zero, got 0", + }, + { + name: "invalid negative concurrent syncs", + concurrentSyncs: -5, + expectErrors: true, + expectedErrorSubString: "concurrent-device-taint-eviction-syncs must be greater than zero, got -5", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts := &DeviceTaintEvictionControllerOptions{ + &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: tc.concurrentSyncs, + }, + } + + errs := opts.Validate() + + if tc.expectErrors && len(errs) == 0 { + t.Error("expected validation errors, but got none") + } + + if !tc.expectErrors && len(errs) > 0 { + t.Errorf("expected no validation errors, but got: %v", errs) + } + + if tc.expectErrors && len(errs) > 0 { + gotErr := utilerrors.NewAggregate(errs).Error() + if !strings.Contains(gotErr, tc.expectedErrorSubString) { + t.Errorf("expected error to contain %q, but got %q", tc.expectedErrorSubString, gotErr) + } + } + }) + } +} + +func TestDeviceTaintEvictionControllerOptions_Validate_Nil(t *testing.T) { + var opts *DeviceTaintEvictionControllerOptions + + errs := opts.Validate() + if len(errs) != 0 { + t.Errorf("expected no validation errors for nil options, but got: %v", errs) + } +} + +func TestDeviceTaintEvictionControllerOptions_Integration(t *testing.T) { + // Test the complete workflow: create options, set flags, apply to config + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + opts := &DeviceTaintEvictionControllerOptions{ + &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 50, + }, + } + + // Add flags + opts.AddFlags(fs) + + // Parse flags with custom value + args := []string{"--concurrent-device-taint-eviction-syncs=100"} + if err := fs.Parse(args); err != nil { + t.Fatalf("failed to parse flags: %v", err) + } + + // Validate + errs := opts.Validate() + if len(errs) > 0 { + t.Fatalf("validation failed: %v", errs) + } + + // Apply to config + cfg := &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{} + if err := opts.ApplyTo(cfg); err != nil { + t.Fatalf("failed to apply options: %v", err) + } + + // Verify final configuration + expected := &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 100, + } + + if !reflect.DeepEqual(cfg, expected) { + t.Errorf("expected config %+v, got %+v", expected, cfg) + } +} diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index c599ff58bd6..b6767c067a5 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -77,6 +77,7 @@ type KubeControllerManagerOptions struct { CSRSigningController *CSRSigningControllerOptions DaemonSetController *DaemonSetControllerOptions DeploymentController *DeploymentControllerOptions + DeviceTaintEvictionController *DeviceTaintEvictionControllerOptions StatefulSetController *StatefulSetControllerOptions DeprecatedFlags *DeprecatedControllerOptions EndpointController *EndpointControllerOptions @@ -151,6 +152,9 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { DeploymentController: &DeploymentControllerOptions{ &componentConfig.DeploymentController, }, + DeviceTaintEvictionController: &DeviceTaintEvictionControllerOptions{ + &componentConfig.DeviceTaintEvictionController, + }, StatefulSetController: &StatefulSetControllerOptions{ &componentConfig.StatefulSetController, }, @@ -272,6 +276,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy s.AttachDetachController.AddFlags(fss.FlagSet(names.PersistentVolumeAttachDetachController)) s.CSRSigningController.AddFlags(fss.FlagSet(names.CertificateSigningRequestSigningController)) s.DeploymentController.AddFlags(fss.FlagSet(names.DeploymentController)) + s.DeviceTaintEvictionController.AddFlags(fss.FlagSet(names.DeviceTaintEvictionController)) s.StatefulSetController.AddFlags(fss.FlagSet(names.StatefulSetController)) s.DaemonSetController.AddFlags(fss.FlagSet(names.DaemonSetController)) s.DeprecatedFlags.AddFlags(fss.FlagSet("deprecated")) @@ -341,6 +346,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config, a if err := s.DeploymentController.ApplyTo(&c.ComponentConfig.DeploymentController); err != nil { return err } + if err := s.DeviceTaintEvictionController.ApplyTo(&c.ComponentConfig.DeviceTaintEvictionController); err != nil { + return err + } if err := s.StatefulSetController.ApplyTo(&c.ComponentConfig.StatefulSetController); err != nil { return err } @@ -440,6 +448,7 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable errs = append(errs, s.CSRSigningController.Validate()...) errs = append(errs, s.DaemonSetController.Validate()...) errs = append(errs, s.DeploymentController.Validate()...) + errs = append(errs, s.DeviceTaintEvictionController.Validate()...) errs = append(errs, s.StatefulSetController.Validate()...) errs = append(errs, s.DeprecatedFlags.Validate()...) errs = append(errs, s.EndpointController.Validate()...) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 4773fd925b3..d9311700832 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -56,6 +56,7 @@ import ( cronjobconfig "k8s.io/kubernetes/pkg/controller/cronjob/config" daemonconfig "k8s.io/kubernetes/pkg/controller/daemon/config" deploymentconfig "k8s.io/kubernetes/pkg/controller/deployment/config" + devicetaintevictionconfig "k8s.io/kubernetes/pkg/controller/devicetainteviction/config" endpointconfig "k8s.io/kubernetes/pkg/controller/endpoint/config" endpointsliceconfig "k8s.io/kubernetes/pkg/controller/endpointslice/config" endpointslicemirroringconfig "k8s.io/kubernetes/pkg/controller/endpointslicemirroring/config" @@ -98,6 +99,7 @@ var args = []string{ "--cluster-signing-legacy-unknown-cert-file=/cluster-signing-legacy-unknown/cert-file", "--cluster-signing-legacy-unknown-key-file=/cluster-signing-legacy-unknown/key-file", "--concurrent-deployment-syncs=10", + "--concurrent-device-taint-eviction-syncs=10", "--concurrent-daemonset-syncs=10", "--concurrent-horizontal-pod-autoscaler-syncs=10", "--concurrent-statefulset-syncs=15", @@ -273,6 +275,11 @@ func TestAddFlags(t *testing.T) { ConcurrentDeploymentSyncs: 10, }, }, + DeviceTaintEvictionController: &DeviceTaintEvictionControllerOptions{ + &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 10, + }, + }, StatefulSetController: &StatefulSetControllerOptions{ &statefulsetconfig.StatefulSetControllerConfiguration{ ConcurrentStatefulSetSyncs: 15, @@ -624,6 +631,9 @@ func TestApplyTo(t *testing.T) { DeploymentController: deploymentconfig.DeploymentControllerConfiguration{ ConcurrentDeploymentSyncs: 10, }, + DeviceTaintEvictionController: devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 10, + }, StatefulSetController: statefulsetconfig.StatefulSetControllerConfiguration{ ConcurrentStatefulSetSyncs: 15, }, @@ -1262,6 +1272,15 @@ func TestValidateControllersOptions(t *testing.T) { }, }, }, + { + name: "DeviceTaintEvictionControllerOptions", + expectErrors: false, + options: &DeviceTaintEvictionControllerOptions{ + &devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration{ + ConcurrentSyncs: 10, + }, + }, + }, { name: "DeprecatedControllerOptions", expectErrors: false, diff --git a/pkg/controller/apis/config/types.go b/pkg/controller/apis/config/types.go index 880a4fd4928..3be7f98a5c4 100644 --- a/pkg/controller/apis/config/types.go +++ b/pkg/controller/apis/config/types.go @@ -25,6 +25,7 @@ import ( cronjobconfig "k8s.io/kubernetes/pkg/controller/cronjob/config" daemonconfig "k8s.io/kubernetes/pkg/controller/daemon/config" deploymentconfig "k8s.io/kubernetes/pkg/controller/deployment/config" + devicetaintevictionconfig "k8s.io/kubernetes/pkg/controller/devicetainteviction/config" endpointconfig "k8s.io/kubernetes/pkg/controller/endpoint/config" endpointsliceconfig "k8s.io/kubernetes/pkg/controller/endpointslice/config" endpointslicemirroringconfig "k8s.io/kubernetes/pkg/controller/endpointslicemirroring/config" @@ -62,6 +63,9 @@ type KubeControllerManagerConfiguration struct { // AttachDetachControllerConfiguration holds configuration for // AttachDetachController related features. AttachDetachController attachdetachconfig.AttachDetachControllerConfiguration + // CronJobControllerConfiguration holds configuration for CronJobController + // related features. + CronJobController cronjobconfig.CronJobControllerConfiguration // CSRSigningControllerConfiguration holds configuration for // CSRSigningController related features. CSRSigningController csrsigningconfig.CSRSigningControllerConfiguration @@ -71,9 +75,8 @@ type KubeControllerManagerConfiguration struct { // DeploymentControllerConfiguration holds configuration for // DeploymentController related features. DeploymentController deploymentconfig.DeploymentControllerConfiguration - // StatefulSetControllerConfiguration holds configuration for - // StatefulSetController related features. - StatefulSetController statefulsetconfig.StatefulSetControllerConfiguration + // DeviceTaintEvictionControllerConfiguration contains elements configuring the device taint eviction controller. + DeviceTaintEvictionController devicetaintevictionconfig.DeviceTaintEvictionControllerConfiguration // DeprecatedControllerConfiguration holds configuration for some deprecated // features. DeprecatedController DeprecatedControllerConfiguration @@ -96,9 +99,6 @@ type KubeControllerManagerConfiguration struct { HPAController poautosclerconfig.HPAControllerConfiguration // JobControllerConfiguration holds configuration for JobController related features. JobController jobconfig.JobControllerConfiguration - // CronJobControllerConfiguration holds configuration for CronJobController - // related features. - CronJobController cronjobconfig.CronJobControllerConfiguration // LegacySATokenCleanerConfiguration holds configuration for LegacySATokenCleaner related features. LegacySATokenCleaner serviceaccountconfig.LegacySATokenCleanerConfiguration // NamespaceControllerConfiguration holds configuration for NamespaceController @@ -130,6 +130,9 @@ type KubeControllerManagerConfiguration struct { // ServiceControllerConfiguration holds configuration for ServiceController // related features. ServiceController serviceconfig.ServiceControllerConfiguration + // StatefulSetControllerConfiguration holds configuration for + // StatefulSetController related features. + StatefulSetController statefulsetconfig.StatefulSetControllerConfiguration // TTLAfterFinishedControllerConfiguration holds configuration for // TTLAfterFinishedController related features. TTLAfterFinishedController ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration diff --git a/pkg/controller/apis/config/v1alpha1/defaults.go b/pkg/controller/apis/config/v1alpha1/defaults.go index c5a6ccef62f..d0d4fd160ae 100644 --- a/pkg/controller/apis/config/v1alpha1/defaults.go +++ b/pkg/controller/apis/config/v1alpha1/defaults.go @@ -25,6 +25,7 @@ import ( cronjobconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/cronjob/config/v1alpha1" daemonconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/daemon/config/v1alpha1" deploymentconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/deployment/config/v1alpha1" + devicetaintevictionconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/devicetainteviction/config/v1alpha1" endpointconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/endpoint/config/v1alpha1" endpointsliceconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/endpointslice/config/v1alpha1" endpointslicemirroringconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/endpointslicemirroring/config/v1alpha1" @@ -71,6 +72,8 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *kubectrlmgrconfigv1alph daemonconfigv1alpha1.RecommendedDefaultDaemonSetControllerConfiguration(&obj.DaemonSetController) // Use the default RecommendedDefaultDeploymentControllerConfiguration options deploymentconfigv1alpha1.RecommendedDefaultDeploymentControllerConfiguration(&obj.DeploymentController) + // Use the default RecommendedDefaultDeviceTaintEvictionControllerConfiguration options + devicetaintevictionconfigv1alpha1.RecommendedDefaultDeviceTaintEvictionControllerConfiguration(&obj.DeviceTaintEvictionController) // Use the default RecommendedDefaultStatefulSetControllerConfiguration options statefulsetconfigv1alpha1.RecommendedDefaultStatefulSetControllerConfiguration(&obj.StatefulSetController) // Use the default RecommendedDefaultEndpointControllerConfiguration options diff --git a/pkg/controller/devicetainteviction/config/doc.go b/pkg/controller/devicetainteviction/config/doc.go new file mode 100644 index 00000000000..e605045a401 --- /dev/null +++ b/pkg/controller/devicetainteviction/config/doc.go @@ -0,0 +1,19 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +package config diff --git a/pkg/controller/devicetainteviction/config/types.go b/pkg/controller/devicetainteviction/config/types.go new file mode 100644 index 00000000000..8bc0544ae0e --- /dev/null +++ b/pkg/controller/devicetainteviction/config/types.go @@ -0,0 +1,26 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// DeviceTaintEvictionControllerConfiguration contains elements configuring the device taint eviction controller. +type DeviceTaintEvictionControllerConfiguration struct { + // ConcurrentSyncs is the number of operations (deleting a pod, updating a ResourcClaim status, etc.) + // that will be done concurrently. Larger number = processing, but more CPU (and network) load. + // + // The default is 10. + ConcurrentSyncs int32 +} diff --git a/pkg/controller/devicetainteviction/config/v1alpha1/conversion.go b/pkg/controller/devicetainteviction/config/v1alpha1/conversion.go new file mode 100644 index 00000000000..6b441c434c1 --- /dev/null +++ b/pkg/controller/devicetainteviction/config/v1alpha1/conversion.go @@ -0,0 +1,40 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/kube-controller-manager/config/v1alpha1" + "k8s.io/kubernetes/pkg/controller/devicetainteviction/config" +) + +// Important! The public back-and-forth conversion functions for the types in this package +// with DeviceTaintEvictionControllerConfiguration types need to be manually exposed like this in order for +// other packages that reference this package to be able to call these conversion functions +// in an autogenerated manner. +// TODO: Fix the bug in conversion-gen so it automatically discovers these Convert_* functions +// in autogenerated code as well. + +// Convert_v1alpha1_DeviceTaintEvictionControllerConfiguration_To_config_DeviceTaintEvictionControllerConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_DeviceTaintEvictionControllerConfiguration_To_config_DeviceTaintEvictionControllerConfiguration(in *v1alpha1.DeviceTaintEvictionControllerConfiguration, out *config.DeviceTaintEvictionControllerConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_DeviceTaintEvictionControllerConfiguration_To_config_DeviceTaintEvictionControllerConfiguration(in, out, s) +} + +// Convert_config_DeviceTaintEvictionControllerConfiguration_To_v1alpha1_DeviceTaintEvictionControllerConfiguration is an autogenerated conversion function. +func Convert_config_DeviceTaintEvictionControllerConfiguration_To_v1alpha1_DeviceTaintEvictionControllerConfiguration(in *config.DeviceTaintEvictionControllerConfiguration, out *v1alpha1.DeviceTaintEvictionControllerConfiguration, s conversion.Scope) error { + return autoConvert_config_DeviceTaintEvictionControllerConfiguration_To_v1alpha1_DeviceTaintEvictionControllerConfiguration(in, out, s) +} diff --git a/pkg/controller/devicetainteviction/config/v1alpha1/defaults.go b/pkg/controller/devicetainteviction/config/v1alpha1/defaults.go new file mode 100644 index 00000000000..e9295e561f7 --- /dev/null +++ b/pkg/controller/devicetainteviction/config/v1alpha1/defaults.go @@ -0,0 +1,41 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1" +) + +// RecommendedDefaultDeviceTaintEvictionControllerConfiguration defaults a pointer to a +// DeviceTaintEvictionControllerConfiguration struct. This will set the recommended default +// values, but they may be subject to change between API versions. This function +// is intentionally not registered in the scheme as a "normal" `SetDefaults_Foo` +// function to allow consumers of this type to set whatever defaults for their +// embedded configs. Forcing consumers to use these defaults would be problematic +// as defaulting in the scheme is done as part of the conversion, and there would +// be no easy way to opt-out. Instead, if you want to use this defaulting method +// run it in your wrapper struct of this type in its `SetDefaults_` method. +func RecommendedDefaultDeviceTaintEvictionControllerConfiguration(obj *kubectrlmgrconfigv1alpha1.DeviceTaintEvictionControllerConfiguration) { + if obj.ConcurrentSyncs == 0 { + // This is a compromise between getting work done and not overwhelming the apiserver + // and pod informers. Integration testing with 100 workers modified pods so quickly + // that a watch in the integration test couldn't keep up: + // cacher.go:855] cacher (pods): 100 objects queued in incoming channel. + // cache_watcher.go:203] Forcing pods watcher close due to unresponsiveness: key: "/pods/", labels: "", fields: "". len(c.input) = 10, len(c.result) = 10, graceful = false + obj.ConcurrentSyncs = 8 + } +} diff --git a/pkg/controller/devicetainteviction/config/v1alpha1/defaults_test.go b/pkg/controller/devicetainteviction/config/v1alpha1/defaults_test.go new file mode 100644 index 00000000000..7f07f608407 --- /dev/null +++ b/pkg/controller/devicetainteviction/config/v1alpha1/defaults_test.go @@ -0,0 +1,31 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1" +) + +func TestRecommendedDefaultDeviceTaintEvictionControllerConfiguration(t *testing.T) { + config := new(kubectrlmgrconfigv1alpha1.DeviceTaintEvictionControllerConfiguration) + RecommendedDefaultDeviceTaintEvictionControllerConfiguration(config) + if config.ConcurrentSyncs != 8 { + t.Errorf("incorrect default value, expected 8 but got %v", config.ConcurrentSyncs) + } +} diff --git a/pkg/controller/devicetainteviction/config/v1alpha1/doc.go b/pkg/controller/devicetainteviction/config/v1alpha1/doc.go new file mode 100644 index 00000000000..67b68078375 --- /dev/null +++ b/pkg/controller/devicetainteviction/config/v1alpha1/doc.go @@ -0,0 +1,21 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +k8s:conversion-gen=k8s.io/kubernetes/pkg/controller/devicetainteviction/config +// +k8s:conversion-gen-external-types=k8s.io/kube-controller-manager/config/v1alpha1 + +package v1alpha1 diff --git a/pkg/controller/devicetainteviction/config/v1alpha1/register.go b/pkg/controller/devicetainteviction/config/v1alpha1/register.go new file mode 100644 index 00000000000..85bbf3effe4 --- /dev/null +++ b/pkg/controller/devicetainteviction/config/v1alpha1/register.go @@ -0,0 +1,31 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + // SchemeBuilder is the scheme builder with scheme init functions to run for this API package + SchemeBuilder runtime.SchemeBuilder + // localSchemeBuilder extends the SchemeBuilder instance with the external types. In this package, + // defaulting and conversion init funcs are registered as well. + localSchemeBuilder = &SchemeBuilder + // AddToScheme is a global function that registers this API group & version to a scheme + AddToScheme = localSchemeBuilder.AddToScheme +) diff --git a/pkg/controller/devicetainteviction/device_taint_eviction.go b/pkg/controller/devicetainteviction/device_taint_eviction.go index bff689e108e..e3cbe35736d 100644 --- a/pkg/controller/devicetainteviction/device_taint_eviction.go +++ b/pkg/controller/devicetainteviction/device_taint_eviction.go @@ -57,15 +57,6 @@ import ( utilpod "k8s.io/kubernetes/pkg/util/pod" ) -const ( - // This is a compromise between getting work done and not overwhelming the apiserver - // and pod informers. Integration testing with 100 workers modified pods so quickly - // that a watch in the integration test couldn't keep up: - // cacher.go:855] cacher (pods): 100 objects queued in incoming channel. - // cache_watcher.go:203] Forcing pods watcher close due to unresponsiveness: key: "/pods/", labels: "", fields: "". len(c.input) = 10, len(c.result) = 10, graceful = false - numWorkers = 10 -) - // Controller listens to Taint changes of DRA devices and Toleration changes of ResourceClaims, // then deletes Pods which use ResourceClaims that don't tolerate a NoExecute taint. // Pods which have already reached a final state (aka terminated) don't need to be deleted. @@ -383,7 +374,7 @@ func New(c clientset.Interface, podInformer coreinformers.PodInformer, claimInfo // Run starts the controller which will run until the context is done. // An error is returned for startup problems. -func (tc *Controller) Run(ctx context.Context) error { +func (tc *Controller) Run(ctx context.Context, numWorkers int) error { defer utilruntime.HandleCrash() logger := klog.FromContext(ctx) logger.Info("Starting", "controller", tc.name) diff --git a/pkg/controller/devicetainteviction/device_taint_eviction_test.go b/pkg/controller/devicetainteviction/device_taint_eviction_test.go index fdf616d0d75..69834b36a1d 100644 --- a/pkg/controller/devicetainteviction/device_taint_eviction_test.go +++ b/pkg/controller/devicetainteviction/device_taint_eviction_test.go @@ -1509,7 +1509,7 @@ func TestEviction(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - assert.NoError(tCtx, controller.Run(tCtx), "eviction controller failed") + assert.NoError(tCtx, controller.Run(tCtx, 10 /* workers */), "eviction controller failed") }() // Eventually the controller should have synced it's informers. @@ -1659,7 +1659,7 @@ func testCancelEviction(tCtx ktesting.TContext, deletePod bool) { wg.Add(1) go func() { defer wg.Done() - assert.NoError(tCtx, controller.Run(tCtx), "eviction controller failed") + assert.NoError(tCtx, controller.Run(tCtx, 10 /* workers */), "eviction controller failed") }() // Eventually the pod gets scheduled for eviction. @@ -1772,7 +1772,7 @@ func TestParallelPodDeletion(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - assert.NoError(tCtx, controller.Run(tCtx), "eviction controller failed") + assert.NoError(tCtx, controller.Run(tCtx, 10 /* workers */), "eviction controller failed") }() // Eventually the pod gets deleted, in this test by us. @@ -1846,7 +1846,7 @@ func TestRetry(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - assert.NoError(tCtx, controller.Run(tCtx), "eviction controller failed") + assert.NoError(tCtx, controller.Run(tCtx, 10 /* workers */), "eviction controller failed") }() // Eventually the pod gets deleted and the event is recorded.