From 05c6eaf0d16558306b0bde5d868b749e1774b96d Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Tue, 9 Mar 2021 10:56:03 -0500 Subject: [PATCH 1/5] promote ServiceLBNodePortControl to beta Signed-off-by: Andrew Sy Kim --- pkg/features/kube_features.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 86f59e6fa4b..5ef89dab848 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -600,6 +600,7 @@ const ( // owner: @andrewsykim @uablrek // kep: http://kep.k8s.io/1864 // alpha: v1.20 + // beta: v1.21 // // Allows control if NodePorts shall be created for services with "type: LoadBalancer" by defining the spec.AllocateLoadBalancerNodePorts field (bool) ServiceLBNodePortControl featuregate.Feature = "ServiceLBNodePortControl" @@ -833,7 +834,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS ExecProbeTimeout: {Default: true, PreRelease: featuregate.GA}, // lock to default and remove after v1.22 based on KEP #1972 update KubeletCredentialProviders: {Default: false, PreRelease: featuregate.Alpha}, GracefulNodeShutdown: {Default: true, PreRelease: featuregate.Beta}, - ServiceLBNodePortControl: {Default: false, PreRelease: featuregate.Alpha}, + ServiceLBNodePortControl: {Default: true, PreRelease: featuregate.Beta}, MixedProtocolLBService: {Default: false, PreRelease: featuregate.Alpha}, VolumeCapacityPriority: {Default: false, PreRelease: featuregate.Alpha}, PreferNominatedNode: {Default: true, PreRelease: featuregate.Beta}, From 24592ca98903cfbf215a79f3a37f6a75f9d3e43a Mon Sep 17 00:00:00 2001 From: Hanlin Shi Date: Fri, 19 Mar 2021 19:33:56 +0000 Subject: [PATCH 2/5] Update the related tests 1. add AllocateLoadBalancerNodePorts fields in specs for validation test cases 2. update fuzzer 3. in resource quota e2e, allocate node port for loadbalancer type service and exceed the node port quota Signed-off-by: Hanlin Shi --- pkg/api/service/testing/make.go | 11 ++++ pkg/apis/core/fuzzer/fuzzer.go | 4 ++ pkg/apis/core/validation/validation_test.go | 60 +++++++++++++++++- .../core/service/storage/rest_test.go | 61 +++++++++++-------- test/e2e/apimachinery/resource_quota.go | 11 +++- 5 files changed, 117 insertions(+), 30 deletions(-) diff --git a/pkg/api/service/testing/make.go b/pkg/api/service/testing/make.go index bb4d2b8c761..0af907e1846 100644 --- a/pkg/api/service/testing/make.go +++ b/pkg/api/service/testing/make.go @@ -19,6 +19,7 @@ package testing import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + utilpointer "k8s.io/utils/pointer" api "k8s.io/kubernetes/pkg/apis/core" ) @@ -88,6 +89,16 @@ func SetTypeExternalName(svc *api.Service) { svc.Spec.ClusterIPs = nil } +// SetTypeExternalNameTrue sets the allocate LB node port to true. +func SetAllocateLBNodePortTrue(svc *api.Service) { + svc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) +} + +// SetTypeExternalNameFalse sets the allocate LB node port to false. +func SetAllocateLBNodePortFalse(svc *api.Service) { + svc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false) +} + // SetPorts sets the service ports list. func SetPorts(ports ...api.ServicePort) Tweak { return func(svc *api.Service) { diff --git a/pkg/apis/core/fuzzer/fuzzer.go b/pkg/apis/core/fuzzer/fuzzer.go index 815d01791c1..fcb9d8a042a 100644 --- a/pkg/apis/core/fuzzer/fuzzer.go +++ b/pkg/apis/core/fuzzer/fuzzer.go @@ -31,6 +31,7 @@ import ( runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/kubernetes/pkg/apis/core" + utilpointer "k8s.io/utils/pointer" ) // Funcs returns the fuzzer functions for the core group. @@ -518,6 +519,9 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { case core.ServiceAffinityNone: ss.SessionAffinityConfig = nil } + if ss.AllocateLoadBalancerNodePorts == nil { + ss.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) + } }, func(s *core.NodeStatus, c fuzz.Continue) { c.FuzzNoCustom(s) diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 7dc13795955..6f5b6571cc8 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -10880,6 +10880,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid load balancer protocol UDP 1", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports[0].Protocol = "UDP" }, numErrs: 0, @@ -10888,6 +10889,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid load balancer protocol UDP 2", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports[0] = core.ServicePort{Name: "q", Port: 12345, Protocol: "UDP", TargetPort: intstr.FromInt(12345)} }, numErrs: 0, @@ -10896,6 +10898,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "load balancer with mix protocol", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "UDP", TargetPort: intstr.FromInt(12345)}) }, numErrs: 0, @@ -10949,6 +10952,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid type - loadbalancer", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) }, numErrs: 0, }, @@ -10956,6 +10960,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid type loadbalancer 2 ports", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)}) }, numErrs: 0, @@ -10964,6 +10969,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid external load balancer 2 ports", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)}) }, numErrs: 0, @@ -11021,9 +11027,10 @@ func TestValidateServiceCreate(t *testing.T) { numErrs: 0, }, { - name: "valid type - loadbalancer", + name: "valid type - loadbalancer with allocateLoadBalancerNodePorts=true", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) }, numErrs: 0, }, @@ -11031,6 +11038,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid type loadbalancer 2 ports", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)}) }, numErrs: 0, @@ -11039,6 +11047,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid type loadbalancer with NodePort", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", NodePort: 12345, TargetPort: intstr.FromInt(12345)}) }, numErrs: 0, @@ -11088,6 +11097,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid type=LoadBalancer", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "q", Port: 12345, Protocol: "TCP", TargetPort: intstr.FromInt(12345)}) }, numErrs: 0, @@ -11098,6 +11108,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid port type=LoadBalancer", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.Ports = append(s.Spec.Ports, core.ServicePort{Name: "kubelet", Port: 10250, Protocol: "TCP", TargetPort: intstr.FromInt(12345)}) }, numErrs: 1, @@ -11106,6 +11117,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid LoadBalancer source range annotation", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Annotations[core.AnnotationLoadBalancerSourceRangesKey] = "1.2.3.4/8, 5.6.7.8/16" }, numErrs: 0, @@ -11114,6 +11126,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "empty LoadBalancer source range annotation", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Annotations[core.AnnotationLoadBalancerSourceRangesKey] = "" }, numErrs: 0, @@ -11129,6 +11142,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid LoadBalancer source range annotation (invalid CIDR)", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Annotations[core.AnnotationLoadBalancerSourceRangesKey] = "1.2.3.4/33" }, numErrs: 1, @@ -11144,6 +11158,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid LoadBalancer source range", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.LoadBalancerSourceRanges = []string{"1.2.3.4/8", "5.6.7.8/16"} }, numErrs: 0, @@ -11152,6 +11167,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "empty LoadBalancer source range", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.LoadBalancerSourceRanges = []string{" "} }, numErrs: 1, @@ -11160,6 +11176,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid LoadBalancer source range", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.LoadBalancerSourceRanges = []string{"foo.bar"} }, numErrs: 1, @@ -11214,6 +11231,7 @@ func TestValidateServiceCreate(t *testing.T) { s.Spec.ClusterIP = "None" s.Spec.ClusterIPs = []string{"None"} s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) }, numErrs: 1, }, @@ -11232,6 +11250,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid externalTraffic field", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.ExternalTrafficPolicy = "invalid" }, numErrs: 1, @@ -11256,6 +11275,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "nagative healthCheckNodePort field", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeLocal s.Spec.HealthCheckNodePort = -1 }, @@ -11265,6 +11285,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "nagative healthCheckNodePort field", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeLocal s.Spec.HealthCheckNodePort = 31100 }, @@ -11288,6 +11309,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "sessionAffinityConfig can't be set when session affinity is None", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.SessionAffinity = core.ServiceAffinityNone s.Spec.SessionAffinityConfig = &core.SessionAffinityConfig{ ClientIP: &core.ClientIPConfig{ @@ -11740,6 +11762,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "valid LoadBalancerClass when type is LoadBalancer", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class") }, numErrs: 0, @@ -11748,6 +11771,7 @@ func TestValidateServiceCreate(t *testing.T) { name: "invalid LoadBalancerClass", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.LoadBalancerClass = utilpointer.StringPtr("Bad/LoadBalancerClass") }, numErrs: 1, @@ -11787,6 +11811,7 @@ func TestValidateServiceExternalTrafficFieldsCombination(t *testing.T) { name: "valid loadBalancer service with externalTrafficPolicy and healthCheckNodePort set", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeLocal s.Spec.HealthCheckNodePort = 34567 }, @@ -11811,6 +11836,7 @@ func TestValidateServiceExternalTrafficFieldsCombination(t *testing.T) { name: "cannot set healthCheckNodePort field on loadBalancer service with externalTrafficPolicy!=Local", tweakSvc: func(s *core.Service) { s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) s.Spec.ExternalTrafficPolicy = core.ServiceExternalTrafficPolicyTypeCluster s.Spec.HealthCheckNodePort = 34567 }, @@ -13330,6 +13356,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "change type", tweakSvc: func(oldSvc, newSvc *core.Service) { newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) }, numErrs: 0, }, @@ -13351,7 +13378,9 @@ func TestValidateServiceUpdate(t *testing.T) { name: "add loadBalancerSourceRanges", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerSourceRanges = []string{"10.0.0.0/8"} }, numErrs: 0, @@ -13360,8 +13389,10 @@ func TestValidateServiceUpdate(t *testing.T) { name: "update loadBalancerSourceRanges", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerSourceRanges = []string{"10.0.0.0/8"} newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerSourceRanges = []string{"10.100.0.0/16"} }, numErrs: 0, @@ -13372,6 +13403,7 @@ func TestValidateServiceUpdate(t *testing.T) { newSvc.Spec.ClusterIP = "None" newSvc.Spec.ClusterIPs = []string{"None"} newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) }, numErrs: 1, }, @@ -13472,6 +13504,7 @@ func TestValidateServiceUpdate(t *testing.T) { tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeClusterIP newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.ClusterIP = "1.2.3.4" oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} @@ -13486,6 +13519,7 @@ func TestValidateServiceUpdate(t *testing.T) { tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeClusterIP newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.ClusterIP = "" oldSvc.Spec.ClusterIPs = nil @@ -13556,6 +13590,7 @@ func TestValidateServiceUpdate(t *testing.T) { tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeNodePort newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.ClusterIP = "1.2.3.4" oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} @@ -13570,6 +13605,7 @@ func TestValidateServiceUpdate(t *testing.T) { tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeNodePort newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.ClusterIP = "" oldSvc.Spec.ClusterIPs = nil @@ -13583,7 +13619,9 @@ func TestValidateServiceUpdate(t *testing.T) { name: "Service with LoadBalancer type cannot change its set ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.ClusterIP = "1.2.3.4" oldSvc.Spec.ClusterIPs = []string{"1.2.3.4"} @@ -13597,7 +13635,9 @@ func TestValidateServiceUpdate(t *testing.T) { name: "Service with LoadBalancer type can change its empty ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.ClusterIP = "" oldSvc.Spec.ClusterIPs = nil @@ -13611,6 +13651,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "Service with LoadBalancer type cannot change its set ClusterIP when changing type to ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "1.2.3.4" @@ -13625,6 +13666,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "Service with LoadBalancer type can change its empty ClusterIP when changing type to ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeClusterIP oldSvc.Spec.ClusterIP = "" @@ -13639,6 +13681,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "Service with LoadBalancer type cannot change its set ClusterIP when changing type to NodePort", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "1.2.3.4" @@ -13653,6 +13696,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "Service with LoadBalancer type can change its empty ClusterIP when changing type to NodePort", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.Type = core.ServiceTypeNodePort oldSvc.Spec.ClusterIP = "" @@ -14144,9 +14188,11 @@ func TestValidateServiceUpdate(t *testing.T) { name: "update LoadBalancer type of service without change LoadBalancerClass", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old") newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old") }, numErrs: 0, @@ -14155,9 +14201,11 @@ func TestValidateServiceUpdate(t *testing.T) { name: "invalid: change LoadBalancerClass when update service", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old") newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-new") }, numErrs: 1, @@ -14166,9 +14214,11 @@ func TestValidateServiceUpdate(t *testing.T) { name: "invalid: unset LoadBalancerClass when update service", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-old") newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = nil }, numErrs: 1, @@ -14177,9 +14227,11 @@ func TestValidateServiceUpdate(t *testing.T) { name: "invalid: set LoadBalancerClass when update service", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = nil newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-new") }, numErrs: 1, @@ -14190,6 +14242,7 @@ func TestValidateServiceUpdate(t *testing.T) { oldSvc.Spec.Type = core.ServiceTypeClusterIP newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class") }, numErrs: 0, @@ -14200,6 +14253,7 @@ func TestValidateServiceUpdate(t *testing.T) { oldSvc.Spec.Type = core.ServiceTypeClusterIP newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = nil }, numErrs: 0, @@ -14210,6 +14264,7 @@ func TestValidateServiceUpdate(t *testing.T) { oldSvc.Spec.Type = core.ServiceTypeClusterIP newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) newSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("Bad/LoadBalancerclass") }, numErrs: 2, @@ -14248,6 +14303,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "invalid: set LoadBalancerClass when update from LoadBalancer service to non LoadBalancer type of service", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class") newSvc.Spec.Type = core.ServiceTypeClusterIP @@ -14259,6 +14315,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "invalid: set LoadBalancerClass when update from LoadBalancer service to non LoadBalancer type of service", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class") newSvc.Spec.Type = core.ServiceTypeExternalName @@ -14270,6 +14327,7 @@ func TestValidateServiceUpdate(t *testing.T) { name: "invalid: set LoadBalancerClass when update from LoadBalancer service to non LoadBalancer type of service", tweakSvc: func(oldSvc, newSvc *core.Service) { oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) oldSvc.Spec.LoadBalancerClass = utilpointer.StringPtr("test.com/test-load-balancer-class") newSvc.Spec.Type = core.ServiceTypeNodePort diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go index fcd41b5ec68..233d87264ae 100644 --- a/pkg/registry/core/service/storage/rest_test.go +++ b/pkg/registry/core/service/storage/rest_test.go @@ -736,7 +736,7 @@ func TestServiceRegistryLoadBalancerService(t *testing.T) { ctx := genericapirequest.NewDefaultContext() storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol}) defer server.Terminate(t) - svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer) + svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer, svctest.SetAllocateLBNodePortTrue) _, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if err != nil { t.Errorf("Failed to create service: %#v", err) @@ -755,12 +755,6 @@ func TestServiceRegistryLoadBalancerService(t *testing.T) { } func TestAllocateLoadBalancerNodePorts(t *testing.T) { - setAlloc := func(val bool) svctest.Tweak { - return func(s *api.Service) { - s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(val) - } - } - testcases := []struct { name string svc *api.Service @@ -769,12 +763,12 @@ func TestAllocateLoadBalancerNodePorts(t *testing.T) { expectError bool }{{ name: "allocate false, gate on", - svc: svctest.MakeService("alloc-false", svctest.SetTypeLoadBalancer, setAlloc(false)), + svc: svctest.MakeService("alloc-false", svctest.SetTypeLoadBalancer, svctest.SetAllocateLBNodePortFalse), expectNodePorts: false, allocateNodePortGate: true, }, { name: "allocate true, gate on", - svc: svctest.MakeService("alloc-true", svctest.SetTypeLoadBalancer, setAlloc(true)), + svc: svctest.MakeService("alloc-true", svctest.SetTypeLoadBalancer, svctest.SetAllocateLBNodePortTrue), expectNodePorts: true, allocateNodePortGate: true, }, { @@ -784,12 +778,12 @@ func TestAllocateLoadBalancerNodePorts(t *testing.T) { allocateNodePortGate: false, }, { name: "allocate false, gate off", - svc: svctest.MakeService("alloc-false", svctest.SetTypeLoadBalancer, setAlloc(false)), + svc: svctest.MakeService("alloc-false", svctest.SetTypeLoadBalancer, svctest.SetAllocateLBNodePortFalse), expectNodePorts: true, allocateNodePortGate: false, }, { name: "allocate true, gate off", - svc: svctest.MakeService("alloc-true", svctest.SetTypeLoadBalancer, setAlloc(true)), + svc: svctest.MakeService("alloc-true", svctest.SetTypeLoadBalancer, svctest.SetAllocateLBNodePortTrue), expectNodePorts: true, allocateNodePortGate: false, }} @@ -948,6 +942,7 @@ func TestServiceRegistryUpdateLoadBalancerService(t *testing.T) { // Modify to be loadbalancer. svc2 := obj.(*api.Service).DeepCopy() svc2.Spec.Type = api.ServiceTypeLoadBalancer + svc2.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) if _, _, err := storage.Update(ctx, svc2.Name, rest.DefaultUpdatedObjectInfo(svc2), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}); err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -970,7 +965,9 @@ func TestServiceRegistryUpdateMultiPortLoadBalancerService(t *testing.T) { svctest.SetTypeLoadBalancer, svctest.SetPorts( svctest.MakeServicePort("p", 6502, intstr.FromInt(6502), api.ProtocolTCP), - svctest.MakeServicePort("q", 8086, intstr.FromInt(8086), api.ProtocolTCP))) + svctest.MakeServicePort("q", 8086, intstr.FromInt(8086), api.ProtocolTCP)), + svctest.SetAllocateLBNodePortTrue, + ) obj, err := storage.Create(ctx, svc1, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) @@ -1324,7 +1321,7 @@ func TestServiceRegistryIPLoadBalancer(t *testing.T) { storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol}) defer server.Terminate(t) - svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer) + svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer, svctest.SetAllocateLBNodePortTrue) ctx := genericapirequest.NewDefaultContext() createdSvc, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if createdSvc == nil || err != nil { @@ -1353,9 +1350,13 @@ func TestServiceRegistryExternalTrafficHealthCheckNodePortAllocation(t *testing. ctx := genericapirequest.NewDefaultContext() storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol}) defer server.Terminate(t) - svc := svctest.MakeService("external-lb-esipp", svctest.SetTypeLoadBalancer, func(s *api.Service) { - s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal - }) + svc := svctest.MakeService("external-lb-esipp", + svctest.SetTypeLoadBalancer, + svctest.SetAllocateLBNodePortTrue, + func(s *api.Service) { + s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal + }, + ) obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if obj == nil || err != nil { t.Errorf("Unexpected failure creating service %v", err) @@ -1377,13 +1378,17 @@ func TestServiceRegistryExternalTrafficHealthCheckNodePortUserAllocation(t *test ctx := genericapirequest.NewDefaultContext() storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol}) defer server.Terminate(t) - svc := svctest.MakeService("external-lb-esipp", svctest.SetTypeLoadBalancer, func(s *api.Service) { - // hard-code NodePort to make sure it doesn't conflict with the healthport. - // TODO: remove this once http://issue.k8s.io/93922 fixes auto-allocation conflicting with user-specified health check ports - s.Spec.Ports[0].NodePort = 30500 - s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal - s.Spec.HealthCheckNodePort = 30501 - }) + svc := svctest.MakeService("external-lb-esipp", + svctest.SetTypeLoadBalancer, + svctest.SetAllocateLBNodePortTrue, + func(s *api.Service) { + // hard-code NodePort to make sure it doesn't conflict with the healthport. + // TODO: remove this once http://issue.k8s.io/93922 fixes auto-allocation conflicting with user-specified health check ports + s.Spec.Ports[0].NodePort = 30500 + s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal + s.Spec.HealthCheckNodePort = 30501 + }, + ) obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if obj == nil || err != nil { t.Fatalf("Unexpected failure creating service :%v", err) @@ -1423,9 +1428,13 @@ func TestServiceRegistryExternalTrafficGlobal(t *testing.T) { ctx := genericapirequest.NewDefaultContext() storage, server := NewTestREST(t, []api.IPFamily{api.IPv4Protocol}) defer server.Terminate(t) - svc := svctest.MakeService("external-lb-esipp", svctest.SetTypeLoadBalancer, func(s *api.Service) { - s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeCluster - }) + svc := svctest.MakeService("external-lb-esipp", + svctest.SetTypeLoadBalancer, + svctest.SetAllocateLBNodePortTrue, + func(s *api.Service) { + s.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeCluster + }, + ) obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) if obj == nil || err != nil { t.Errorf("Unexpected failure creating service %v", err) diff --git a/test/e2e/apimachinery/resource_quota.go b/test/e2e/apimachinery/resource_quota.go index 938e4d22b22..c561e962c71 100644 --- a/test/e2e/apimachinery/resource_quota.go +++ b/test/e2e/apimachinery/resource_quota.go @@ -36,7 +36,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/utils/crd" imageutils "k8s.io/kubernetes/test/utils/image" - "k8s.io/utils/pointer" "github.com/onsi/ginkgo" ) @@ -111,7 +110,7 @@ var _ = SIGDescribe("ResourceQuota", func() { framework.ExpectNoError(err) ginkgo.By("Not allowing a LoadBalancer Service with NodePort to be created that exceeds remaining quota") - loadbalancer := newTestServiceForQuota("test-service-lb", v1.ServiceTypeLoadBalancer, false) + loadbalancer := newTestServiceForQuota("test-service-lb", v1.ServiceTypeLoadBalancer, true) _, err = f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(context.TODO(), loadbalancer, metav1.CreateOptions{}) framework.ExpectError(err) @@ -1737,6 +1736,12 @@ func newTestReplicaSetForQuota(name, image string, replicas int32) *appsv1.Repli // newTestServiceForQuota returns a simple service func newTestServiceForQuota(name string, serviceType v1.ServiceType, allocateLoadBalancerNodePorts bool) *v1.Service { + var allocateNPs *bool + // Only set allocateLoadBalancerNodePorts when service type is LB + if serviceType == v1.ServiceTypeLoadBalancer { + allocateNPs = &allocateLoadBalancerNodePorts + } + return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -1747,7 +1752,7 @@ func newTestServiceForQuota(name string, serviceType v1.ServiceType, allocateLoa Port: 80, TargetPort: intstr.FromInt(80), }}, - AllocateLoadBalancerNodePorts: pointer.BoolPtr(allocateLoadBalancerNodePorts), + AllocateLoadBalancerNodePorts: allocateNPs, }, } } From 79b6df96fc6a47c94f35f2cb9e6bbe9d8eddf43b Mon Sep 17 00:00:00 2001 From: Hanlin Shi Date: Thu, 8 Apr 2021 19:29:40 +0000 Subject: [PATCH 3/5] Add tests for LB type service 1. create LB type svc with nodeport allocation set to false 1. create LB type svc with nodeport allocation unset 3. update LB type svc's nodeport allocation field Signed-off-by: Hanlin Shi --- pkg/apis/core/validation/validation_test.go | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 6f5b6571cc8..2975b8ac712 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -10956,6 +10956,21 @@ func TestValidateServiceCreate(t *testing.T) { }, numErrs: 0, }, + { + name: "valid type - loadbalancer with allocateLoadBalancerNodePorts=false", + tweakSvc: func(s *core.Service) { + s.Spec.Type = core.ServiceTypeLoadBalancer + s.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false) + }, + numErrs: 0, + }, + { + name: "invalid type - missing AllocateLoadBalancerNodePorts for loadbalancer type", + tweakSvc: func(s *core.Service) { + s.Spec.Type = core.ServiceTypeLoadBalancer + }, + numErrs: 1, + }, { name: "valid type loadbalancer 2 ports", tweakSvc: func(s *core.Service) { @@ -13529,6 +13544,26 @@ func TestValidateServiceUpdate(t *testing.T) { }, numErrs: 0, }, + { + name: "Service with LoadBalancer type can change its AllocateLoadBalancerNodePorts from true to false", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) + newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false) + }, + numErrs: 0, + }, + { + name: "Service with LoadBalancer type can change its AllocateLoadBalancerNodePorts from false to true", + tweakSvc: func(oldSvc, newSvc *core.Service) { + oldSvc.Spec.Type = core.ServiceTypeLoadBalancer + oldSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(false) + newSvc.Spec.Type = core.ServiceTypeLoadBalancer + newSvc.Spec.AllocateLoadBalancerNodePorts = utilpointer.BoolPtr(true) + }, + numErrs: 0, + }, { name: "Service with NodePort type cannot change its set ClusterIP", tweakSvc: func(oldSvc, newSvc *core.Service) { From c8bc4202454f0acd791943609c7a814a8089e8a7 Mon Sep 17 00:00:00 2001 From: Hanlin Shi Date: Fri, 9 Apr 2021 00:00:45 +0000 Subject: [PATCH 4/5] Fix the beta release version. Signed-off-by: Hanlin Shi --- pkg/features/kube_features.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5ef89dab848..fb65ec4232a 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -600,7 +600,7 @@ const ( // owner: @andrewsykim @uablrek // kep: http://kep.k8s.io/1864 // alpha: v1.20 - // beta: v1.21 + // beta: v1.22 // // Allows control if NodePorts shall be created for services with "type: LoadBalancer" by defining the spec.AllocateLoadBalancerNodePorts field (bool) ServiceLBNodePortControl featuregate.Feature = "ServiceLBNodePortControl" From c96c809539f399acee92f9073418b61f5dd5df0f Mon Sep 17 00:00:00 2001 From: Hanlin Shi Date: Wed, 30 Jun 2021 11:57:44 -0700 Subject: [PATCH 5/5] Add integration test for LB node port control Signed-off-by: Hanlin Shi --- test/integration/quota/quota_test.go | 179 +++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/test/integration/quota/quota_test.go b/test/integration/quota/quota_test.go index 9917a5344b9..dec7a6faa2f 100644 --- a/test/integration/quota/quota_test.go +++ b/test/integration/quota/quota_test.go @@ -30,19 +30,23 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/admission/plugin/resourcequota" resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" "k8s.io/apiserver/pkg/quota/v1/generic" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" watchtools "k8s.io/client-go/tools/watch" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller" replicationcontroller "k8s.io/kubernetes/pkg/controller/replication" resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota" + "k8s.io/kubernetes/pkg/features" quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" "k8s.io/kubernetes/test/integration/framework" ) @@ -371,3 +375,178 @@ func TestQuotaLimitedResourceDenial(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestQuotaLimitService(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, true)() + type testCase struct { + description string + svc *v1.Service + success bool + } + // Set up an API server + h := &framework.APIServerHolder{Initialized: make(chan struct{})} + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + <-h.Initialized + h.M.GenericAPIServer.Handler.ServeHTTP(w, req) + })) + + admissionCh := make(chan struct{}) + clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) + + // stop creation of a pod resource unless there is a quota + config := &resourcequotaapi.Configuration{ + LimitedResources: []resourcequotaapi.LimitedResource{ + { + Resource: "pods", + MatchContains: []string{"pods"}, + }, + }, + } + qca := quotainstall.NewQuotaConfigurationForAdmission() + admission, err := resourcequota.NewResourceQuota(config, 5, admissionCh) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + admission.SetExternalKubeClientSet(clientset) + externalInformers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc()) + admission.SetExternalKubeInformerFactory(externalInformers) + admission.SetQuotaConfiguration(qca) + defer close(admissionCh) + + controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() + controlPlaneConfig.GenericConfig.AdmissionControl = admission + _, _, closeFn := framework.RunAnAPIServerUsingServer(controlPlaneConfig, s, h) + defer closeFn() + + ns := framework.CreateTestingNamespace("quota", s, t) + defer framework.DeleteTestingNamespace(ns, s, t) + + controllerCh := make(chan struct{}) + defer close(controllerCh) + + informers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc()) + rm := replicationcontroller.NewReplicationManager( + informers.Core().V1().Pods(), + informers.Core().V1().ReplicationControllers(), + clientset, + replicationcontroller.BurstReplicas, + ) + rm.SetEventRecorder(&record.FakeRecorder{}) + go rm.Run(3, controllerCh) + + discoveryFunc := clientset.Discovery().ServerPreferredNamespacedResources + listerFuncForResource := generic.ListerFuncForResourceFunc(informers.ForResource) + qc := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource) + informersStarted := make(chan struct{}) + resourceQuotaControllerOptions := &resourcequotacontroller.ControllerOptions{ + QuotaClient: clientset.CoreV1(), + ResourceQuotaInformer: informers.Core().V1().ResourceQuotas(), + ResyncPeriod: controller.NoResyncPeriodFunc, + InformerFactory: informers, + ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc, + DiscoveryFunc: discoveryFunc, + IgnoredResourcesFunc: qc.IgnoredResources, + InformersStarted: informersStarted, + Registry: generic.NewRegistry(qc.Evaluators()), + } + resourceQuotaController, err := resourcequotacontroller.NewController(resourceQuotaControllerOptions) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + go resourceQuotaController.Run(2, controllerCh) + + // Periodically the quota controller to detect new resource types + go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, controllerCh) + + externalInformers.Start(controllerCh) + informers.Start(controllerCh) + close(informersStarted) + + // now create a covering quota + // note: limited resource does a matchContains, so we now have "pods" matching "pods" and "count/pods" + quota := &v1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "quota", + Namespace: ns.Name, + }, + Spec: v1.ResourceQuotaSpec{ + Hard: v1.ResourceList{ + v1.ResourceServices: resource.MustParse("4"), + v1.ResourceServicesNodePorts: resource.MustParse("2"), + v1.ResourceServicesLoadBalancers: resource.MustParse("2"), + }, + }, + } + waitForQuota(t, quota, clientset) + + tests := []testCase{ + { + description: "node port service should be created successfully", + svc: newService("np-svc", v1.ServiceTypeNodePort, true), + success: true, + }, + { + description: "first LB type service that allocates node port should be created successfully", + svc: newService("lb-svc-withnp1", v1.ServiceTypeLoadBalancer, true), + success: true, + }, + { + description: "second LB type service that allocates node port creation should fail as node port quota is exceeded", + svc: newService("lb-svc-withnp2", v1.ServiceTypeLoadBalancer, true), + success: false, + }, + { + description: "first LB type service that doesn't allocates node port should be created successfully", + svc: newService("lb-svc-wonp1", v1.ServiceTypeLoadBalancer, false), + success: true, + }, + { + description: "second LB type service that doesn't allocates node port creation should fail as loadbalancer quota is exceeded", + svc: newService("lb-svc-wonp2", v1.ServiceTypeLoadBalancer, false), + success: false, + }, + { + description: "forth service creation should be successful", + svc: newService("clusterip-svc1", v1.ServiceTypeClusterIP, false), + success: true, + }, + { + description: "fifth service creation should fail as service quota is exceeded", + svc: newService("clusterip-svc2", v1.ServiceTypeClusterIP, false), + success: false, + }, + } + + for _, test := range tests { + t.Log(test.description) + _, err := clientset.CoreV1().Services(ns.Name).Create(context.TODO(), test.svc, metav1.CreateOptions{}) + if (err == nil) != test.success { + if err != nil { + t.Fatalf("Error creating test service: %v, svc: %+v", err, test.svc) + } else { + t.Fatalf("Expect service creation to fail, but service %s is created", test.svc.Name) + } + } + } +} + +func newService(name string, svcType v1.ServiceType, allocateNodePort bool) *v1.Service { + var allocateNPs *bool + // Only set allocateLoadBalancerNodePorts when service type is LB + if svcType == v1.ServiceTypeLoadBalancer { + allocateNPs = &allocateNodePort + } + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.ServiceSpec{ + Type: svcType, + AllocateLoadBalancerNodePorts: allocateNPs, + Ports: []v1.ServicePort{{ + Port: int32(80), + TargetPort: intstr.FromInt(80), + }}, + }, + } +}