Add integration test for LB node port control

Signed-off-by: Hanlin Shi <shihanlin9@gmail.com>

test/integration/quota/quota_test.go

@@ -30,19 +30,23 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
 	resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota"
 	"k8s.io/apiserver/pkg/quota/v1/generic"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	watchtools "k8s.io/client-go/tools/watch"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/pkg/controller"
 	replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
 	resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota"
+	"k8s.io/kubernetes/pkg/features"
 	quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
 	"k8s.io/kubernetes/test/integration/framework"
 )
@@ -371,3 +375,178 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
 		t.Fatalf("unexpected error: %v", err)
 	}
 }
+
+func TestQuotaLimitService(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, true)()
+	type testCase struct {
+		description string
+		svc         *v1.Service
+		success     bool
+	}
+	// Set up an API server
+	h := &framework.APIServerHolder{Initialized: make(chan struct{})}
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		<-h.Initialized
+		h.M.GenericAPIServer.Handler.ServeHTTP(w, req)
+	}))
+
+	admissionCh := make(chan struct{})
+	clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}})
+
+	// stop creation of a pod resource unless there is a quota
+	config := &resourcequotaapi.Configuration{
+		LimitedResources: []resourcequotaapi.LimitedResource{
+			{
+				Resource:      "pods",
+				MatchContains: []string{"pods"},
+			},
+		},
+	}
+	qca := quotainstall.NewQuotaConfigurationForAdmission()
+	admission, err := resourcequota.NewResourceQuota(config, 5, admissionCh)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	admission.SetExternalKubeClientSet(clientset)
+	externalInformers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc())
+	admission.SetExternalKubeInformerFactory(externalInformers)
+	admission.SetQuotaConfiguration(qca)
+	defer close(admissionCh)
+
+	controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
+	controlPlaneConfig.GenericConfig.AdmissionControl = admission
+	_, _, closeFn := framework.RunAnAPIServerUsingServer(controlPlaneConfig, s, h)
+	defer closeFn()
+
+	ns := framework.CreateTestingNamespace("quota", s, t)
+	defer framework.DeleteTestingNamespace(ns, s, t)
+
+	controllerCh := make(chan struct{})
+	defer close(controllerCh)
+
+	informers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc())
+	rm := replicationcontroller.NewReplicationManager(
+		informers.Core().V1().Pods(),
+		informers.Core().V1().ReplicationControllers(),
+		clientset,
+		replicationcontroller.BurstReplicas,
+	)
+	rm.SetEventRecorder(&record.FakeRecorder{})
+	go rm.Run(3, controllerCh)
+
+	discoveryFunc := clientset.Discovery().ServerPreferredNamespacedResources
+	listerFuncForResource := generic.ListerFuncForResourceFunc(informers.ForResource)
+	qc := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
+	informersStarted := make(chan struct{})
+	resourceQuotaControllerOptions := &resourcequotacontroller.ControllerOptions{
+		QuotaClient:               clientset.CoreV1(),
+		ResourceQuotaInformer:     informers.Core().V1().ResourceQuotas(),
+		ResyncPeriod:              controller.NoResyncPeriodFunc,
+		InformerFactory:           informers,
+		ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc,
+		DiscoveryFunc:             discoveryFunc,
+		IgnoredResourcesFunc:      qc.IgnoredResources,
+		InformersStarted:          informersStarted,
+		Registry:                  generic.NewRegistry(qc.Evaluators()),
+	}
+	resourceQuotaController, err := resourcequotacontroller.NewController(resourceQuotaControllerOptions)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	go resourceQuotaController.Run(2, controllerCh)
+
+	// Periodically the quota controller to detect new resource types
+	go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, controllerCh)
+
+	externalInformers.Start(controllerCh)
+	informers.Start(controllerCh)
+	close(informersStarted)
+
+	// now create a covering quota
+	// note: limited resource does a matchContains, so we now have "pods" matching "pods" and "count/pods"
+	quota := &v1.ResourceQuota{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "quota",
+			Namespace: ns.Name,
+		},
+		Spec: v1.ResourceQuotaSpec{
+			Hard: v1.ResourceList{
+				v1.ResourceServices:              resource.MustParse("4"),
+				v1.ResourceServicesNodePorts:     resource.MustParse("2"),
+				v1.ResourceServicesLoadBalancers: resource.MustParse("2"),
+			},
+		},
+	}
+	waitForQuota(t, quota, clientset)
+
+	tests := []testCase{
+		{
+			description: "node port service should be created successfully",
+			svc:         newService("np-svc", v1.ServiceTypeNodePort, true),
+			success:     true,
+		},
+		{
+			description: "first LB type service that allocates node port should be created successfully",
+			svc:         newService("lb-svc-withnp1", v1.ServiceTypeLoadBalancer, true),
+			success:     true,
+		},
+		{
+			description: "second LB type service that allocates node port creation should fail as node port quota is exceeded",
+			svc:         newService("lb-svc-withnp2", v1.ServiceTypeLoadBalancer, true),
+			success:     false,
+		},
+		{
+			description: "first LB type service that doesn't allocates node port should be created successfully",
+			svc:         newService("lb-svc-wonp1", v1.ServiceTypeLoadBalancer, false),
+			success:     true,
+		},
+		{
+			description: "second LB type service that doesn't allocates node port creation should fail as loadbalancer quota is exceeded",
+			svc:         newService("lb-svc-wonp2", v1.ServiceTypeLoadBalancer, false),
+			success:     false,
+		},
+		{
+			description: "forth service creation should be successful",
+			svc:         newService("clusterip-svc1", v1.ServiceTypeClusterIP, false),
+			success:     true,
+		},
+		{
+			description: "fifth service creation should fail as service quota is exceeded",
+			svc:         newService("clusterip-svc2", v1.ServiceTypeClusterIP, false),
+			success:     false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Log(test.description)
+		_, err := clientset.CoreV1().Services(ns.Name).Create(context.TODO(), test.svc, metav1.CreateOptions{})
+		if (err == nil) != test.success {
+			if err != nil {
+				t.Fatalf("Error creating test service: %v, svc: %+v", err, test.svc)
+			} else {
+				t.Fatalf("Expect service creation to fail, but service %s is created", test.svc.Name)
+			}
+		}
+	}
+}
+
+func newService(name string, svcType v1.ServiceType, allocateNodePort bool) *v1.Service {
+	var allocateNPs *bool
+	// Only set allocateLoadBalancerNodePorts when service type is LB
+	if svcType == v1.ServiceTypeLoadBalancer {
+		allocateNPs = &allocateNodePort
+	}
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: v1.ServiceSpec{
+			Type:                          svcType,
+			AllocateLoadBalancerNodePorts: allocateNPs,
+			Ports: []v1.ServicePort{{
+				Port:       int32(80),
+				TargetPort: intstr.FromInt(80),
+			}},
+		},
+	}
+}