mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Add integration test for LB node port control
Signed-off-by: Hanlin Shi <shihanlin9@gmail.com>
This commit is contained in:
parent
c8bc420245
commit
c96c809539
@ -30,19 +30,23 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/fields"
|
"k8s.io/apimachinery/pkg/fields"
|
||||||
"k8s.io/apimachinery/pkg/labels"
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
"k8s.io/apimachinery/pkg/util/intstr"
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
"k8s.io/apimachinery/pkg/watch"
|
"k8s.io/apimachinery/pkg/watch"
|
||||||
"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
|
"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
|
||||||
resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota"
|
resourcequotaapi "k8s.io/apiserver/pkg/admission/plugin/resourcequota/apis/resourcequota"
|
||||||
"k8s.io/apiserver/pkg/quota/v1/generic"
|
"k8s.io/apiserver/pkg/quota/v1/generic"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
"k8s.io/client-go/informers"
|
"k8s.io/client-go/informers"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
restclient "k8s.io/client-go/rest"
|
restclient "k8s.io/client-go/rest"
|
||||||
"k8s.io/client-go/tools/record"
|
"k8s.io/client-go/tools/record"
|
||||||
watchtools "k8s.io/client-go/tools/watch"
|
watchtools "k8s.io/client-go/tools/watch"
|
||||||
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
"k8s.io/kubernetes/pkg/controller"
|
"k8s.io/kubernetes/pkg/controller"
|
||||||
replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
|
replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
|
||||||
resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota"
|
resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota"
|
||||||
|
"k8s.io/kubernetes/pkg/features"
|
||||||
quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
|
quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
|
||||||
"k8s.io/kubernetes/test/integration/framework"
|
"k8s.io/kubernetes/test/integration/framework"
|
||||||
)
|
)
|
||||||
@ -371,3 +375,178 @@ func TestQuotaLimitedResourceDenial(t *testing.T) {
|
|||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestQuotaLimitService(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLBNodePortControl, true)()
|
||||||
|
type testCase struct {
|
||||||
|
description string
|
||||||
|
svc *v1.Service
|
||||||
|
success bool
|
||||||
|
}
|
||||||
|
// Set up an API server
|
||||||
|
h := &framework.APIServerHolder{Initialized: make(chan struct{})}
|
||||||
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
<-h.Initialized
|
||||||
|
h.M.GenericAPIServer.Handler.ServeHTTP(w, req)
|
||||||
|
}))
|
||||||
|
|
||||||
|
admissionCh := make(chan struct{})
|
||||||
|
clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}})
|
||||||
|
|
||||||
|
// stop creation of a pod resource unless there is a quota
|
||||||
|
config := &resourcequotaapi.Configuration{
|
||||||
|
LimitedResources: []resourcequotaapi.LimitedResource{
|
||||||
|
{
|
||||||
|
Resource: "pods",
|
||||||
|
MatchContains: []string{"pods"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
qca := quotainstall.NewQuotaConfigurationForAdmission()
|
||||||
|
admission, err := resourcequota.NewResourceQuota(config, 5, admissionCh)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
admission.SetExternalKubeClientSet(clientset)
|
||||||
|
externalInformers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc())
|
||||||
|
admission.SetExternalKubeInformerFactory(externalInformers)
|
||||||
|
admission.SetQuotaConfiguration(qca)
|
||||||
|
defer close(admissionCh)
|
||||||
|
|
||||||
|
controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
|
||||||
|
controlPlaneConfig.GenericConfig.AdmissionControl = admission
|
||||||
|
_, _, closeFn := framework.RunAnAPIServerUsingServer(controlPlaneConfig, s, h)
|
||||||
|
defer closeFn()
|
||||||
|
|
||||||
|
ns := framework.CreateTestingNamespace("quota", s, t)
|
||||||
|
defer framework.DeleteTestingNamespace(ns, s, t)
|
||||||
|
|
||||||
|
controllerCh := make(chan struct{})
|
||||||
|
defer close(controllerCh)
|
||||||
|
|
||||||
|
informers := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc())
|
||||||
|
rm := replicationcontroller.NewReplicationManager(
|
||||||
|
informers.Core().V1().Pods(),
|
||||||
|
informers.Core().V1().ReplicationControllers(),
|
||||||
|
clientset,
|
||||||
|
replicationcontroller.BurstReplicas,
|
||||||
|
)
|
||||||
|
rm.SetEventRecorder(&record.FakeRecorder{})
|
||||||
|
go rm.Run(3, controllerCh)
|
||||||
|
|
||||||
|
discoveryFunc := clientset.Discovery().ServerPreferredNamespacedResources
|
||||||
|
listerFuncForResource := generic.ListerFuncForResourceFunc(informers.ForResource)
|
||||||
|
qc := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
|
||||||
|
informersStarted := make(chan struct{})
|
||||||
|
resourceQuotaControllerOptions := &resourcequotacontroller.ControllerOptions{
|
||||||
|
QuotaClient: clientset.CoreV1(),
|
||||||
|
ResourceQuotaInformer: informers.Core().V1().ResourceQuotas(),
|
||||||
|
ResyncPeriod: controller.NoResyncPeriodFunc,
|
||||||
|
InformerFactory: informers,
|
||||||
|
ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc,
|
||||||
|
DiscoveryFunc: discoveryFunc,
|
||||||
|
IgnoredResourcesFunc: qc.IgnoredResources,
|
||||||
|
InformersStarted: informersStarted,
|
||||||
|
Registry: generic.NewRegistry(qc.Evaluators()),
|
||||||
|
}
|
||||||
|
resourceQuotaController, err := resourcequotacontroller.NewController(resourceQuotaControllerOptions)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected err: %v", err)
|
||||||
|
}
|
||||||
|
go resourceQuotaController.Run(2, controllerCh)
|
||||||
|
|
||||||
|
// Periodically the quota controller to detect new resource types
|
||||||
|
go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, controllerCh)
|
||||||
|
|
||||||
|
externalInformers.Start(controllerCh)
|
||||||
|
informers.Start(controllerCh)
|
||||||
|
close(informersStarted)
|
||||||
|
|
||||||
|
// now create a covering quota
|
||||||
|
// note: limited resource does a matchContains, so we now have "pods" matching "pods" and "count/pods"
|
||||||
|
quota := &v1.ResourceQuota{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "quota",
|
||||||
|
Namespace: ns.Name,
|
||||||
|
},
|
||||||
|
Spec: v1.ResourceQuotaSpec{
|
||||||
|
Hard: v1.ResourceList{
|
||||||
|
v1.ResourceServices: resource.MustParse("4"),
|
||||||
|
v1.ResourceServicesNodePorts: resource.MustParse("2"),
|
||||||
|
v1.ResourceServicesLoadBalancers: resource.MustParse("2"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
waitForQuota(t, quota, clientset)
|
||||||
|
|
||||||
|
tests := []testCase{
|
||||||
|
{
|
||||||
|
description: "node port service should be created successfully",
|
||||||
|
svc: newService("np-svc", v1.ServiceTypeNodePort, true),
|
||||||
|
success: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "first LB type service that allocates node port should be created successfully",
|
||||||
|
svc: newService("lb-svc-withnp1", v1.ServiceTypeLoadBalancer, true),
|
||||||
|
success: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "second LB type service that allocates node port creation should fail as node port quota is exceeded",
|
||||||
|
svc: newService("lb-svc-withnp2", v1.ServiceTypeLoadBalancer, true),
|
||||||
|
success: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "first LB type service that doesn't allocates node port should be created successfully",
|
||||||
|
svc: newService("lb-svc-wonp1", v1.ServiceTypeLoadBalancer, false),
|
||||||
|
success: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "second LB type service that doesn't allocates node port creation should fail as loadbalancer quota is exceeded",
|
||||||
|
svc: newService("lb-svc-wonp2", v1.ServiceTypeLoadBalancer, false),
|
||||||
|
success: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "forth service creation should be successful",
|
||||||
|
svc: newService("clusterip-svc1", v1.ServiceTypeClusterIP, false),
|
||||||
|
success: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "fifth service creation should fail as service quota is exceeded",
|
||||||
|
svc: newService("clusterip-svc2", v1.ServiceTypeClusterIP, false),
|
||||||
|
success: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Log(test.description)
|
||||||
|
_, err := clientset.CoreV1().Services(ns.Name).Create(context.TODO(), test.svc, metav1.CreateOptions{})
|
||||||
|
if (err == nil) != test.success {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Error creating test service: %v, svc: %+v", err, test.svc)
|
||||||
|
} else {
|
||||||
|
t.Fatalf("Expect service creation to fail, but service %s is created", test.svc.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newService(name string, svcType v1.ServiceType, allocateNodePort bool) *v1.Service {
|
||||||
|
var allocateNPs *bool
|
||||||
|
// Only set allocateLoadBalancerNodePorts when service type is LB
|
||||||
|
if svcType == v1.ServiceTypeLoadBalancer {
|
||||||
|
allocateNPs = &allocateNodePort
|
||||||
|
}
|
||||||
|
return &v1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: name,
|
||||||
|
},
|
||||||
|
Spec: v1.ServiceSpec{
|
||||||
|
Type: svcType,
|
||||||
|
AllocateLoadBalancerNodePorts: allocateNPs,
|
||||||
|
Ports: []v1.ServicePort{{
|
||||||
|
Port: int32(80),
|
||||||
|
TargetPort: intstr.FromInt(80),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user