mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-20 18:31:15 +00:00
Merge pull request #110503 from aojea/iptables_rules
kube-proxy iptables test number of generated iptables rules
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
832c4d8cb7
438
pkg/proxy/iptables/number_generated_rules_test.go
Normal file
438
pkg/proxy/iptables/number_generated_rules_test.go
Normal file
@ -0,0 +1,438 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package iptables
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
discovery "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
|
||||
netutils "k8s.io/utils/net"
|
||||
utilpointer "k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
// kube-proxy generates iptables rules to forward traffic from Services to Endpoints
|
||||
// kube-proxy uses iptables-restore to configure the rules atomically, however,
|
||||
// this has the downside that large number of rules take a long time to be processed,
|
||||
// causing disruption.
|
||||
// There are different parameters than influence the number of rules generated:
|
||||
// - ServiceType
|
||||
// - Number of Services
|
||||
// - Number of Endpoints per Service
|
||||
// This test will fail when the number of rules change, so the person
|
||||
// that is modifying the code can have feedback about the performance impact
|
||||
// on their changes. It also runs multiple number of rules test cases to check
|
||||
// if the number of rules grows linearly.
|
||||
func TestNumberIptablesRules(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
epsFunc func(eps *discovery.EndpointSlice)
|
||||
svcFunc func(svc *v1.Service)
|
||||
services int
|
||||
epPerService int
|
||||
expectedFilterRules int
|
||||
expectedNatRules int
|
||||
}{
|
||||
{
|
||||
name: "0 Services 0 EndpointsPerService - ClusterIP",
|
||||
services: 0,
|
||||
epPerService: 0,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 5,
|
||||
},
|
||||
{
|
||||
name: "1 Services 0 EndpointPerService - ClusterIP",
|
||||
services: 1,
|
||||
epPerService: 0,
|
||||
expectedFilterRules: 4,
|
||||
expectedNatRules: 5,
|
||||
},
|
||||
{
|
||||
name: "1 Services 1 EndpointPerService - ClusterIP",
|
||||
services: 1,
|
||||
epPerService: 1,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 10,
|
||||
},
|
||||
{
|
||||
name: "1 Services 2 EndpointPerService - ClusterIP",
|
||||
services: 1,
|
||||
epPerService: 2,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 13,
|
||||
},
|
||||
{
|
||||
name: "1 Services 10 EndpointPerService - ClusterIP",
|
||||
services: 1,
|
||||
epPerService: 10,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 37,
|
||||
},
|
||||
{
|
||||
name: "10 Services 0 EndpointsPerService - ClusterIP",
|
||||
services: 10,
|
||||
epPerService: 0,
|
||||
expectedFilterRules: 13,
|
||||
expectedNatRules: 5,
|
||||
},
|
||||
{
|
||||
name: "10 Services 1 EndpointPerService - ClusterIP",
|
||||
services: 10,
|
||||
epPerService: 1,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 55,
|
||||
},
|
||||
{
|
||||
name: "10 Services 2 EndpointPerService - ClusterIP",
|
||||
services: 10,
|
||||
epPerService: 2,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 85,
|
||||
},
|
||||
{
|
||||
name: "10 Services 10 EndpointPerService - ClusterIP",
|
||||
services: 10,
|
||||
epPerService: 10,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 325,
|
||||
},
|
||||
|
||||
{
|
||||
name: "0 Services 0 EndpointsPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 0,
|
||||
epPerService: 0,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 5,
|
||||
},
|
||||
{
|
||||
name: "1 Services 0 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 1,
|
||||
epPerService: 0,
|
||||
expectedFilterRules: 7,
|
||||
expectedNatRules: 5,
|
||||
},
|
||||
{
|
||||
name: "1 Services 1 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 1,
|
||||
epPerService: 1,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 17,
|
||||
},
|
||||
{
|
||||
name: "1 Services 2 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 1,
|
||||
epPerService: 2,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 20,
|
||||
},
|
||||
{
|
||||
name: "1 Services 10 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 1,
|
||||
epPerService: 10,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 44,
|
||||
},
|
||||
{
|
||||
name: "10 Services 0 EndpointsPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 10,
|
||||
epPerService: 0,
|
||||
expectedFilterRules: 43,
|
||||
expectedNatRules: 5,
|
||||
},
|
||||
{
|
||||
name: "10 Services 1 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 10,
|
||||
epPerService: 1,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 125,
|
||||
},
|
||||
{
|
||||
name: "10 Services 2 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 10,
|
||||
epPerService: 2,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 155,
|
||||
},
|
||||
{
|
||||
name: "10 Services 10 EndpointPerService - LoadBalancer",
|
||||
svcFunc: func(svc *v1.Service) {
|
||||
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||
IP: "1.2.3.4",
|
||||
}}
|
||||
},
|
||||
services: 10,
|
||||
epPerService: 10,
|
||||
expectedFilterRules: 3,
|
||||
expectedNatRules: 395,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ipt := iptablestest.NewFake()
|
||||
fp := NewFakeProxier(ipt)
|
||||
|
||||
svcs, eps := generateServiceEndpoints(test.services, test.epPerService, test.epsFunc, test.svcFunc)
|
||||
|
||||
makeServiceMap(fp, svcs...)
|
||||
populateEndpointSlices(fp, eps...)
|
||||
|
||||
now := time.Now()
|
||||
fp.syncProxyRules()
|
||||
t.Logf("time to sync rule: %v", time.Since(now))
|
||||
t.Logf("iptables data size: %d bytes", fp.iptablesData.Len())
|
||||
|
||||
if fp.filterRules.Lines() != test.expectedFilterRules {
|
||||
t.Errorf("expected number of Filter rules: %d, got: %d", test.expectedFilterRules, fp.filterRules.Lines())
|
||||
}
|
||||
|
||||
if fp.natRules.Lines() != test.expectedNatRules {
|
||||
t.Errorf("expected number of NAT rules: %d, got: %d", test.expectedNatRules, fp.natRules.Lines())
|
||||
}
|
||||
|
||||
// print generated iptables data
|
||||
// t.Logf("Generated rules:\n %s", fp.iptablesData.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_generateServiceEndpoints(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
services int
|
||||
epPerService int
|
||||
svcType v1.ServiceType
|
||||
}{
|
||||
{
|
||||
name: "Generate 10 Services with 10 Endpoints per Service and LoadBalancer Type",
|
||||
services: 10,
|
||||
epPerService: 10,
|
||||
svcType: v1.ServiceTypeLoadBalancer,
|
||||
},
|
||||
{
|
||||
name: "Generate 10 Services with 20 Endpoints per Service and NodePort Type",
|
||||
services: 10,
|
||||
epPerService: 20,
|
||||
svcType: v1.ServiceTypeNodePort,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
// test the function to mutate services
|
||||
svcFunc := func(svc *v1.Service) {
|
||||
svc.Spec.Type = test.svcType
|
||||
}
|
||||
// test the function to mutate endpoint slices
|
||||
epsFunc := func(eps *discovery.EndpointSlice) {
|
||||
for i := range eps.Endpoints {
|
||||
nodeName := fmt.Sprintf("node-%d", i)
|
||||
eps.Endpoints[i].NodeName = &nodeName
|
||||
}
|
||||
}
|
||||
|
||||
svcs, eps := generateServiceEndpoints(test.services, test.epPerService, epsFunc, svcFunc)
|
||||
|
||||
if len(svcs) != test.services {
|
||||
t.Fatalf("expected %d service, received %d", test.services, len(svcs))
|
||||
}
|
||||
if len(eps) != test.services {
|
||||
t.Fatalf("expected %d endpoint slice , received %d", test.services, len(eps))
|
||||
}
|
||||
|
||||
for i := 0; i < test.services; i++ {
|
||||
if svcs[i].Spec.Type != test.svcType {
|
||||
t.Fatalf("expected Service Type %s, got %s", test.svcType, svcs[i].Spec.Type)
|
||||
}
|
||||
if eps[i].ObjectMeta.Labels[discovery.LabelServiceName] != svcs[i].Name {
|
||||
t.Fatalf("endpoint slice reference %s instead of Service %s", eps[i].ObjectMeta.Labels[discovery.LabelServiceName], svcs[i].Name)
|
||||
}
|
||||
if len(eps[i].Endpoints) != test.epPerService {
|
||||
t.Fatalf("expected %d endpoints per slice , received %d", test.epPerService, len(eps[i].Endpoints))
|
||||
}
|
||||
for j := 0; j < test.epPerService; j++ {
|
||||
nodeName := fmt.Sprintf("node-%d", j)
|
||||
if *eps[i].Endpoints[j].NodeName != nodeName {
|
||||
t.Errorf("Endpoint %d on EndpointSlice %d expected Nodename %s, got %s", j, i, nodeName, *eps[i].Endpoints[j].NodeName)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// generateServiceEndpoints generate Services with the Type specified and it creates N Endpoints per Service
|
||||
func generateServiceEndpoints(nServices, nEndpoints int, epsFunc func(eps *discovery.EndpointSlice), svcFunc func(svc *v1.Service)) ([]*v1.Service, []*discovery.EndpointSlice) {
|
||||
services := make([]*v1.Service, nServices)
|
||||
endpointSlices := make([]*discovery.EndpointSlice, nServices)
|
||||
|
||||
// base parameters
|
||||
basePort := 80
|
||||
base := netutils.BigForIP(netutils.ParseIPSloppy("10.0.0.1"))
|
||||
|
||||
// generate a base endpoint slice object
|
||||
baseEp := netutils.BigForIP(netutils.ParseIPSloppy("172.16.0.1"))
|
||||
epPort := 8080
|
||||
|
||||
tcpProtocol := v1.ProtocolTCP
|
||||
|
||||
eps := &discovery.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ep",
|
||||
Namespace: "namespace",
|
||||
},
|
||||
AddressType: discovery.AddressTypeIPv4,
|
||||
Endpoints: []discovery.Endpoint{},
|
||||
Ports: []discovery.EndpointPort{{
|
||||
Name: utilpointer.StringPtr(fmt.Sprintf("%d", epPort)),
|
||||
Port: utilpointer.Int32(int32(epPort)),
|
||||
Protocol: &tcpProtocol,
|
||||
}},
|
||||
}
|
||||
|
||||
for j := 0; j < nEndpoints; j++ {
|
||||
ipEp := netutils.AddIPOffset(baseEp, j)
|
||||
eps.Endpoints = append(eps.Endpoints, discovery.Endpoint{
|
||||
Addresses: []string{ipEp.String()},
|
||||
})
|
||||
}
|
||||
|
||||
if epsFunc != nil {
|
||||
epsFunc(eps)
|
||||
}
|
||||
|
||||
// generate a base service object
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "svc",
|
||||
Namespace: "namespace",
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Type: v1.ServiceTypeClusterIP,
|
||||
},
|
||||
}
|
||||
|
||||
if svcFunc != nil {
|
||||
svcFunc(svc)
|
||||
}
|
||||
|
||||
// Create the Services and associate and endpoint slice object to each one
|
||||
for i := 0; i < nServices; i++ {
|
||||
ip := netutils.AddIPOffset(base, i)
|
||||
services[i] = svc.DeepCopy()
|
||||
services[i].Name = fmt.Sprintf("svc%d", i)
|
||||
services[i].Spec.ClusterIP = ip.String()
|
||||
services[i].Spec.Ports = []v1.ServicePort{
|
||||
{
|
||||
Name: fmt.Sprintf("%d", epPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
Port: int32(basePort + i),
|
||||
TargetPort: intstr.FromInt(epPort),
|
||||
},
|
||||
}
|
||||
|
||||
if svc.Spec.Type == v1.ServiceTypeNodePort || svc.Spec.Type == v1.ServiceTypeLoadBalancer {
|
||||
services[i].Spec.Ports[0].NodePort = int32(30000 + i)
|
||||
|
||||
}
|
||||
if svc.Spec.Type == v1.ServiceTypeLoadBalancer {
|
||||
services[i].Spec.HealthCheckNodePort = int32(32000 + nServices + i)
|
||||
}
|
||||
|
||||
endpointSlices[i] = eps.DeepCopy()
|
||||
endpointSlices[i].Name = services[i].Name
|
||||
endpointSlices[i].ObjectMeta.Labels = map[string]string{
|
||||
discovery.LabelServiceName: services[i].Name,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return services, endpointSlices
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user