From 18ba2643dba3c5d7445bf1eda9a5a4c3320fe180 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Tue, 26 Nov 2019 11:48:19 -0800 Subject: [PATCH] add applyto tests for controller configs --- cmd/kube-controller-manager/app/options/BUILD | 1 + .../app/options/hpacontroller.go | 2 + .../app/options/options_test.go | 342 +++++++++++++----- 3 files changed, 263 insertions(+), 82 deletions(-) diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index 2b6aafdca56..d32192aff41 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -97,6 +97,7 @@ go_test( embed = [":go_default_library"], deps = [ "//cmd/controller-manager/app/options:go_default_library", + "//cmd/kube-controller-manager/app/config:go_default_library", "//pkg/controller/apis/config:go_default_library", "//pkg/controller/certificates/signer/config:go_default_library", "//pkg/controller/daemon/config:go_default_library", diff --git a/cmd/kube-controller-manager/app/options/hpacontroller.go b/cmd/kube-controller-manager/app/options/hpacontroller.go index 41a928150a0..5a34eec508c 100644 --- a/cmd/kube-controller-manager/app/options/hpacontroller.go +++ b/cmd/kube-controller-manager/app/options/hpacontroller.go @@ -58,6 +58,8 @@ func (o *HPAControllerOptions) ApplyTo(cfg *poautosclerconfig.HPAControllerConfi cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod cfg.HorizontalPodAutoscalerInitialReadinessDelay = o.HorizontalPodAutoscalerInitialReadinessDelay + cfg.HorizontalPodAutoscalerUpscaleForbiddenWindow = o.HorizontalPodAutoscalerUpscaleForbiddenWindow + cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow return nil } diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 3e215249731..dc43ee87e65 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -30,6 +30,7 @@ import ( apiserveroptions "k8s.io/apiserver/pkg/server/options" componentbaseconfig "k8s.io/component-base/config" cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" + kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config" daemonconfig "k8s.io/kubernetes/pkg/controller/daemon/config" @@ -54,6 +55,89 @@ import ( persistentvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/config" ) +var args = []string{ + "--address=192.168.4.10", + "--allocate-node-cidrs=true", + "--attach-detach-reconcile-sync-period=30s", + "--cidr-allocator-type=CloudAllocator", + "--cloud-config=/cloud-config", + "--cloud-provider=gce", + "--cluster-cidr=1.2.3.4/24", + "--cluster-name=k8s", + "--cluster-signing-cert-file=/cluster-signing-cert", + "--cluster-signing-key-file=/cluster-signing-key", + "--concurrent-deployment-syncs=10", + "--concurrent-statefulset-syncs=15", + "--concurrent-endpoint-syncs=10", + "--concurrent-service-endpoint-syncs=10", + "--concurrent-gc-syncs=30", + "--concurrent-namespace-syncs=20", + "--concurrent-replicaset-syncs=10", + "--concurrent-resource-quota-syncs=10", + "--concurrent-service-syncs=2", + "--concurrent-serviceaccount-token-syncs=10", + "--concurrent_rc_syncs=10", + "--configure-cloud-routes=false", + "--contention-profiling=true", + "--controller-start-interval=2m", + "--controllers=foo,bar", + "--deployment-controller-sync-period=45s", + "--disable-attach-detach-reconcile-sync=true", + "--enable-dynamic-provisioning=false", + "--enable-garbage-collector=false", + "--enable-hostpath-provisioner=true", + "--enable-taint-manager=false", + "--experimental-cluster-signing-duration=10h", + "--flex-volume-plugin-dir=/flex-volume-plugin", + "--horizontal-pod-autoscaler-downscale-delay=2m", + "--horizontal-pod-autoscaler-sync-period=45s", + "--horizontal-pod-autoscaler-upscale-delay=1m", + "--horizontal-pod-autoscaler-downscale-stabilization=3m", + "--horizontal-pod-autoscaler-cpu-initialization-period=90s", + "--horizontal-pod-autoscaler-initial-readiness-delay=50s", + "--http2-max-streams-per-connection=47", + "--kube-api-burst=100", + "--kube-api-content-type=application/json", + "--kube-api-qps=50.0", + "--kubeconfig=/kubeconfig", + "--large-cluster-size-threshold=100", + "--leader-elect=false", + "--leader-elect-lease-duration=30s", + "--leader-elect-renew-deadline=15s", + "--leader-elect-resource-lock=configmap", + "--leader-elect-retry-period=5s", + "--master=192.168.4.20", + "--max-endpoints-per-slice=200", + "--min-resync-period=8h", + "--namespace-sync-period=10m", + "--node-cidr-mask-size=48", + "--node-cidr-mask-size-ipv4=48", + "--node-cidr-mask-size-ipv6=108", + "--node-eviction-rate=0.2", + "--node-monitor-grace-period=30s", + "--node-monitor-period=10s", + "--node-startup-grace-period=30s", + "--pod-eviction-timeout=2m", + "--port=10000", + "--profiling=false", + "--pv-recycler-increment-timeout-nfs=45", + "--pv-recycler-minimum-timeout-hostpath=45", + "--pv-recycler-minimum-timeout-nfs=200", + "--pv-recycler-timeout-increment-hostpath=45", + "--pvclaimbinder-sync-period=30s", + "--resource-quota-sync-period=10m", + "--route-reconciliation-period=30s", + "--secondary-node-eviction-rate=0.05", + "--service-account-private-key-file=/service-account-private-key", + "--terminated-pod-gc-threshold=12000", + "--unhealthy-zone-threshold=0.6", + "--use-service-account-credentials=true", + "--cert-dir=/a/b/c", + "--bind-address=192.168.4.21", + "--secure-port=10001", + "--concurrent-ttl-after-finished-syncs=8", +} + func TestAddFlags(t *testing.T) { fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) s, _ := NewKubeControllerManagerOptions() @@ -61,88 +145,6 @@ func TestAddFlags(t *testing.T) { fs.AddFlagSet(f) } - args := []string{ - "--address=192.168.4.10", - "--allocate-node-cidrs=true", - "--attach-detach-reconcile-sync-period=30s", - "--cidr-allocator-type=CloudAllocator", - "--cloud-config=/cloud-config", - "--cloud-provider=gce", - "--cluster-cidr=1.2.3.4/24", - "--cluster-name=k8s", - "--cluster-signing-cert-file=/cluster-signing-cert", - "--cluster-signing-key-file=/cluster-signing-key", - "--concurrent-deployment-syncs=10", - "--concurrent-statefulset-syncs=15", - "--concurrent-endpoint-syncs=10", - "--concurrent-service-endpoint-syncs=10", - "--concurrent-gc-syncs=30", - "--concurrent-namespace-syncs=20", - "--concurrent-replicaset-syncs=10", - "--concurrent-resource-quota-syncs=10", - "--concurrent-service-syncs=2", - "--concurrent-serviceaccount-token-syncs=10", - "--concurrent_rc_syncs=10", - "--configure-cloud-routes=false", - "--contention-profiling=true", - "--controller-start-interval=2m", - "--controllers=foo,bar", - "--deployment-controller-sync-period=45s", - "--disable-attach-detach-reconcile-sync=true", - "--enable-dynamic-provisioning=false", - "--enable-garbage-collector=false", - "--enable-hostpath-provisioner=true", - "--enable-taint-manager=false", - "--experimental-cluster-signing-duration=10h", - "--flex-volume-plugin-dir=/flex-volume-plugin", - "--horizontal-pod-autoscaler-downscale-delay=2m", - "--horizontal-pod-autoscaler-sync-period=45s", - "--horizontal-pod-autoscaler-upscale-delay=1m", - "--horizontal-pod-autoscaler-downscale-stabilization=3m", - "--horizontal-pod-autoscaler-cpu-initialization-period=90s", - "--horizontal-pod-autoscaler-initial-readiness-delay=50s", - "--http2-max-streams-per-connection=47", - "--kube-api-burst=100", - "--kube-api-content-type=application/json", - "--kube-api-qps=50.0", - "--kubeconfig=/kubeconfig", - "--large-cluster-size-threshold=100", - "--leader-elect=false", - "--leader-elect-lease-duration=30s", - "--leader-elect-renew-deadline=15s", - "--leader-elect-resource-lock=configmap", - "--leader-elect-retry-period=5s", - "--master=192.168.4.20", - "--max-endpoints-per-slice=200", - "--min-resync-period=8h", - "--namespace-sync-period=10m", - "--node-cidr-mask-size=48", - "--node-cidr-mask-size-ipv4=48", - "--node-cidr-mask-size-ipv6=108", - "--node-eviction-rate=0.2", - "--node-monitor-grace-period=30s", - "--node-monitor-period=10s", - "--node-startup-grace-period=30s", - "--pod-eviction-timeout=2m", - "--port=10000", - "--profiling=false", - "--pv-recycler-increment-timeout-nfs=45", - "--pv-recycler-minimum-timeout-hostpath=45", - "--pv-recycler-minimum-timeout-nfs=200", - "--pv-recycler-timeout-increment-hostpath=45", - "--pvclaimbinder-sync-period=30s", - "--resource-quota-sync-period=10m", - "--route-reconciliation-period=30s", - "--secondary-node-eviction-rate=0.05", - "--service-account-private-key-file=/service-account-private-key", - "--terminated-pod-gc-threshold=12000", - "--unhealthy-zone-threshold=0.6", - "--use-service-account-credentials=true", - "--cert-dir=/a/b/c", - "--bind-address=192.168.4.21", - "--secure-port=10001", - "--concurrent-ttl-after-finished-syncs=8", - } fs.Parse(args) // Sort GCIgnoredResources because it's built from a map, which means the // insertion order is random. @@ -390,6 +392,182 @@ func TestAddFlags(t *testing.T) { } } +func TestApplyTo(t *testing.T) { + fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) + s, _ := NewKubeControllerManagerOptions() + // flag set to parse the args that are required to start the kube controller manager + for _, f := range s.Flags([]string{""}, []string{""}).FlagSets { + fs.AddFlagSet(f) + } + + fs.Parse(args) + // Sort GCIgnoredResources because it's built from a map, which means the + // insertion order is random. + sort.Sort(sortedGCIgnoredResources(s.GarbageCollectorController.GCIgnoredResources)) + + expected := &kubecontrollerconfig.Config{ + ComponentConfig: kubectrlmgrconfig.KubeControllerManagerConfiguration{ + Generic: kubectrlmgrconfig.GenericControllerManagerConfiguration{ + Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config + MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, + ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ + ContentType: "application/json", + QPS: 50.0, + Burst: 100, + }, + ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + ResourceName: "kube-controller-manager", + ResourceNamespace: "kube-system", + }, + Controllers: []string{"foo", "bar"}, + Debugging: componentbaseconfig.DebuggingConfiguration{ + EnableProfiling: false, + EnableContentionProfiling: true, + }, + }, + KubeCloudShared: kubectrlmgrconfig.KubeCloudSharedConfiguration{ + UseServiceAccountCredentials: true, + RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, + ClusterName: "k8s", + ClusterCIDR: "1.2.3.4/24", + AllocateNodeCIDRs: true, + CIDRAllocatorType: "CloudAllocator", + ConfigureCloudRoutes: false, + CloudProvider: kubectrlmgrconfig.CloudProviderConfiguration{ + Name: "gce", + CloudConfigFile: "/cloud-config", + }, + }, + ServiceController: serviceconfig.ServiceControllerConfiguration{ + ConcurrentServiceSyncs: 2, + }, + AttachDetachController: attachdetachconfig.AttachDetachControllerConfiguration{ + ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, + DisableAttachDetachReconcilerSync: true, + }, + CSRSigningController: csrsigningconfig.CSRSigningControllerConfiguration{ + ClusterSigningCertFile: "/cluster-signing-cert", + ClusterSigningKeyFile: "/cluster-signing-key", + ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, + }, + DaemonSetController: daemonconfig.DaemonSetControllerConfiguration{ + ConcurrentDaemonSetSyncs: 2, + }, + DeploymentController: deploymentconfig.DeploymentControllerConfiguration{ + ConcurrentDeploymentSyncs: 10, + DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + }, + StatefulSetController: statefulsetconfig.StatefulSetControllerConfiguration{ + ConcurrentStatefulSetSyncs: 15, + }, + DeprecatedController: kubectrlmgrconfig.DeprecatedControllerConfiguration{ + DeletingPodsQPS: 0.1, + RegisterRetryCount: 10, + }, + EndpointController: endpointconfig.EndpointControllerConfiguration{ + ConcurrentEndpointSyncs: 10, + }, + EndpointSliceController: endpointsliceconfig.EndpointSliceControllerConfiguration{ + ConcurrentServiceEndpointSyncs: 10, + MaxEndpointsPerSlice: 200, + }, + GarbageCollectorController: garbagecollectorconfig.GarbageCollectorControllerConfiguration{ + ConcurrentGCSyncs: 30, + GCIgnoredResources: []garbagecollectorconfig.GroupResource{ + {Group: "", Resource: "events"}, + }, + EnableGarbageCollector: false, + }, + HPAController: poautosclerconfig.HPAControllerConfiguration{ + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, + HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute}, + HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second}, + HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second}, + HorizontalPodAutoscalerTolerance: 0.1, + HorizontalPodAutoscalerUseRESTClients: true, + }, + JobController: jobconfig.JobControllerConfiguration{ + ConcurrentJobSyncs: 5, + }, + NamespaceController: namespaceconfig.NamespaceControllerConfiguration{ + NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, + ConcurrentNamespaceSyncs: 20, + }, + NodeIPAMController: nodeipamconfig.NodeIPAMControllerConfiguration{ + NodeCIDRMaskSize: 48, + NodeCIDRMaskSizeIPv4: 48, + NodeCIDRMaskSizeIPv6: 108, + }, + NodeLifecycleController: nodelifecycleconfig.NodeLifecycleControllerConfiguration{ + EnableTaintManager: false, + NodeEvictionRate: 0.2, + SecondaryNodeEvictionRate: 0.05, + NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, + PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, + LargeClusterSizeThreshold: 100, + UnhealthyZoneThreshold: 0.6, + }, + PersistentVolumeBinderController: persistentvolumeconfig.PersistentVolumeBinderControllerConfiguration{ + PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + VolumeConfiguration: persistentvolumeconfig.VolumeConfiguration{ + EnableDynamicProvisioning: false, + EnableHostPathProvisioning: true, + FlexVolumePluginDir: "/flex-volume-plugin", + PersistentVolumeRecyclerConfiguration: persistentvolumeconfig.PersistentVolumeRecyclerConfiguration{ + MaximumRetry: 3, + MinimumTimeoutNFS: 200, + IncrementTimeoutNFS: 45, + MinimumTimeoutHostPath: 45, + IncrementTimeoutHostPath: 45, + }, + }, + }, + PodGCController: podgcconfig.PodGCControllerConfiguration{ + TerminatedPodGCThreshold: 12000, + }, + ReplicaSetController: replicasetconfig.ReplicaSetControllerConfiguration{ + ConcurrentRSSyncs: 10, + }, + ReplicationController: replicationconfig.ReplicationControllerConfiguration{ + ConcurrentRCSyncs: 10, + }, + ResourceQuotaController: resourcequotaconfig.ResourceQuotaControllerConfiguration{ + ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, + ConcurrentResourceQuotaSyncs: 10, + }, + SAController: serviceaccountconfig.SAControllerConfiguration{ + ServiceAccountKeyFile: "/service-account-private-key", + ConcurrentSATokenSyncs: 10, + }, + TTLAfterFinishedController: ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration{ + ConcurrentTTLSyncs: 8, + }, + }, + } + + // Sort GCIgnoredResources because it's built from a map, which means the + // insertion order is random. + sort.Sort(sortedGCIgnoredResources(expected.ComponentConfig.GarbageCollectorController.GCIgnoredResources)) + + c := &kubecontrollerconfig.Config{} + s.ApplyTo(c) + + if !reflect.DeepEqual(expected.ComponentConfig, c.ComponentConfig) { + t.Errorf("Got different configuration than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected.ComponentConfig, c.ComponentConfig)) + } +} + type sortedGCIgnoredResources []garbagecollectorconfig.GroupResource func (r sortedGCIgnoredResources) Len() int {