From c96c809539f399acee92f9073418b61f5dd5df0f Mon Sep 17 00:00:00 2001 From: Hanlin Shi Date: Wed, 30 Jun 2021 11:57:44 -0700 Subject: [PATCH] Add integration test for LB node port control Signed-off-by: Hanlin Shi --- test/integration/quota/quota_test.go | 179 +++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/test/integration/quota/quota_test.go b/test/integration/quota/quota_test.go index 9917a5344b9..dec7a6faa2f 100644 --- a/test/integration/quota/quota_test.go +++ b/test/integration/quota/quota_test.go @@ -30,19 +30,23 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/admission/plugin/resourcequota" resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota" "k8s.io/apiserver/pkg/quota/v1/generic" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" watchtools "k8s.io/client-go/tools/watch" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller" replicationcontroller "k8s.io/kubernetes/pkg/controller/replication" resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota" + "k8s.io/kubernetes/pkg/features" quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" "k8s.io/kubernetes/test/integration/framework" ) @@ -371,3 +375,178 @@ func TestQuotaLimitedResourceDenial(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestQuotaLimitService(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, true)() + type testCase struct { + description string + svc *v1.Service + success bool + } + // Set up an API server + h := &framework.APIServerHolder{Initialized: make(chan struct{})} + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + <-h.Initialized + h.M.GenericAPIServer.Handler.ServeHTTP(w, req) + })) + + admissionCh := make(chan struct{}) + clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) + + // stop creation of a pod resource unless there is a quota + config := &resourcequotaapi.Configuration{ + LimitedResources: []resourcequotaapi.LimitedResource{ + { + Resource: "pods", + MatchContains: []string{"pods"}, + }, + }, + } + qca := quotainstall.NewQuotaConfigurationForAdmission() + admission, err := resourcequota.NewResourceQuota(config, 5, admissionCh) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + admission.SetExternalKubeClientSet(clientset) + externalInformers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc()) + admission.SetExternalKubeInformerFactory(externalInformers) + admission.SetQuotaConfiguration(qca) + defer close(admissionCh) + + controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() + controlPlaneConfig.GenericConfig.AdmissionControl = admission + _, _, closeFn := framework.RunAnAPIServerUsingServer(controlPlaneConfig, s, h) + defer closeFn() + + ns := framework.CreateTestingNamespace("quota", s, t) + defer framework.DeleteTestingNamespace(ns, s, t) + + controllerCh := make(chan struct{}) + defer close(controllerCh) + + informers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc()) + rm := replicationcontroller.NewReplicationManager( + informers.Core().V1().Pods(), + informers.Core().V1().ReplicationControllers(), + clientset, + replicationcontroller.BurstReplicas, + ) + rm.SetEventRecorder(&record.FakeRecorder{}) + go rm.Run(3, controllerCh) + + discoveryFunc := clientset.Discovery().ServerPreferredNamespacedResources + listerFuncForResource := generic.ListerFuncForResourceFunc(informers.ForResource) + qc := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource) + informersStarted := make(chan struct{}) + resourceQuotaControllerOptions := &resourcequotacontroller.ControllerOptions{ + QuotaClient: clientset.CoreV1(), + ResourceQuotaInformer: informers.Core().V1().ResourceQuotas(), + ResyncPeriod: controller.NoResyncPeriodFunc, + InformerFactory: informers, + ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc, + DiscoveryFunc: discoveryFunc, + IgnoredResourcesFunc: qc.IgnoredResources, + InformersStarted: informersStarted, + Registry: generic.NewRegistry(qc.Evaluators()), + } + resourceQuotaController, err := resourcequotacontroller.NewController(resourceQuotaControllerOptions) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + go resourceQuotaController.Run(2, controllerCh) + + // Periodically the quota controller to detect new resource types + go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, controllerCh) + + externalInformers.Start(controllerCh) + informers.Start(controllerCh) + close(informersStarted) + + // now create a covering quota + // note: limited resource does a matchContains, so we now have "pods" matching "pods" and "count/pods" + quota := &v1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "quota", + Namespace: ns.Name, + }, + Spec: v1.ResourceQuotaSpec{ + Hard: v1.ResourceList{ + v1.ResourceServices: resource.MustParse("4"), + v1.ResourceServicesNodePorts: resource.MustParse("2"), + v1.ResourceServicesLoadBalancers: resource.MustParse("2"), + }, + }, + } + waitForQuota(t, quota, clientset) + + tests := []testCase{ + { + description: "node port service should be created successfully", + svc: newService("np-svc", v1.ServiceTypeNodePort, true), + success: true, + }, + { + description: "first LB type service that allocates node port should be created successfully", + svc: newService("lb-svc-withnp1", v1.ServiceTypeLoadBalancer, true), + success: true, + }, + { + description: "second LB type service that allocates node port creation should fail as node port quota is exceeded", + svc: newService("lb-svc-withnp2", v1.ServiceTypeLoadBalancer, true), + success: false, + }, + { + description: "first LB type service that doesn't allocates node port should be created successfully", + svc: newService("lb-svc-wonp1", v1.ServiceTypeLoadBalancer, false), + success: true, + }, + { + description: "second LB type service that doesn't allocates node port creation should fail as loadbalancer quota is exceeded", + svc: newService("lb-svc-wonp2", v1.ServiceTypeLoadBalancer, false), + success: false, + }, + { + description: "forth service creation should be successful", + svc: newService("clusterip-svc1", v1.ServiceTypeClusterIP, false), + success: true, + }, + { + description: "fifth service creation should fail as service quota is exceeded", + svc: newService("clusterip-svc2", v1.ServiceTypeClusterIP, false), + success: false, + }, + } + + for _, test := range tests { + t.Log(test.description) + _, err := clientset.CoreV1().Services(ns.Name).Create(context.TODO(), test.svc, metav1.CreateOptions{}) + if (err == nil) != test.success { + if err != nil { + t.Fatalf("Error creating test service: %v, svc: %+v", err, test.svc) + } else { + t.Fatalf("Expect service creation to fail, but service %s is created", test.svc.Name) + } + } + } +} + +func newService(name string, svcType v1.ServiceType, allocateNodePort bool) *v1.Service { + var allocateNPs *bool + // Only set allocateLoadBalancerNodePorts when service type is LB + if svcType == v1.ServiceTypeLoadBalancer { + allocateNPs = &allocateNodePort + } + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.ServiceSpec{ + Type: svcType, + AllocateLoadBalancerNodePorts: allocateNPs, + Ports: []v1.ServicePort{{ + Port: int32(80), + TargetPort: intstr.FromInt(80), + }}, + }, + } +}