mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 10:51:29 +00:00
Merge pull request #110503 from aojea/iptables_rules
kube-proxy iptables test number of generated iptables rules
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
832c4d8cb7
438
pkg/proxy/iptables/number_generated_rules_test.go
Normal file
438
pkg/proxy/iptables/number_generated_rules_test.go
Normal file
@ -0,0 +1,438 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2022 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package iptables
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
discovery "k8s.io/api/discovery/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/intstr"
|
||||||
|
iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
|
||||||
|
netutils "k8s.io/utils/net"
|
||||||
|
utilpointer "k8s.io/utils/pointer"
|
||||||
|
)
|
||||||
|
|
||||||
|
// kube-proxy generates iptables rules to forward traffic from Services to Endpoints
|
||||||
|
// kube-proxy uses iptables-restore to configure the rules atomically, however,
|
||||||
|
// this has the downside that large number of rules take a long time to be processed,
|
||||||
|
// causing disruption.
|
||||||
|
// There are different parameters than influence the number of rules generated:
|
||||||
|
// - ServiceType
|
||||||
|
// - Number of Services
|
||||||
|
// - Number of Endpoints per Service
|
||||||
|
// This test will fail when the number of rules change, so the person
|
||||||
|
// that is modifying the code can have feedback about the performance impact
|
||||||
|
// on their changes. It also runs multiple number of rules test cases to check
|
||||||
|
// if the number of rules grows linearly.
|
||||||
|
func TestNumberIptablesRules(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
epsFunc func(eps *discovery.EndpointSlice)
|
||||||
|
svcFunc func(svc *v1.Service)
|
||||||
|
services int
|
||||||
|
epPerService int
|
||||||
|
expectedFilterRules int
|
||||||
|
expectedNatRules int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "0 Services 0 EndpointsPerService - ClusterIP",
|
||||||
|
services: 0,
|
||||||
|
epPerService: 0,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 0 EndpointPerService - ClusterIP",
|
||||||
|
services: 1,
|
||||||
|
epPerService: 0,
|
||||||
|
expectedFilterRules: 4,
|
||||||
|
expectedNatRules: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 1 EndpointPerService - ClusterIP",
|
||||||
|
services: 1,
|
||||||
|
epPerService: 1,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 10,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 2 EndpointPerService - ClusterIP",
|
||||||
|
services: 1,
|
||||||
|
epPerService: 2,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 13,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 10 EndpointPerService - ClusterIP",
|
||||||
|
services: 1,
|
||||||
|
epPerService: 10,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 37,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 0 EndpointsPerService - ClusterIP",
|
||||||
|
services: 10,
|
||||||
|
epPerService: 0,
|
||||||
|
expectedFilterRules: 13,
|
||||||
|
expectedNatRules: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 1 EndpointPerService - ClusterIP",
|
||||||
|
services: 10,
|
||||||
|
epPerService: 1,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 55,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 2 EndpointPerService - ClusterIP",
|
||||||
|
services: 10,
|
||||||
|
epPerService: 2,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 85,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 10 EndpointPerService - ClusterIP",
|
||||||
|
services: 10,
|
||||||
|
epPerService: 10,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 325,
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: "0 Services 0 EndpointsPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 0,
|
||||||
|
epPerService: 0,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 0 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 1,
|
||||||
|
epPerService: 0,
|
||||||
|
expectedFilterRules: 7,
|
||||||
|
expectedNatRules: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 1 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 1,
|
||||||
|
epPerService: 1,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 17,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 2 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 1,
|
||||||
|
epPerService: 2,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 20,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "1 Services 10 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 1,
|
||||||
|
epPerService: 10,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 44,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 0 EndpointsPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 10,
|
||||||
|
epPerService: 0,
|
||||||
|
expectedFilterRules: 43,
|
||||||
|
expectedNatRules: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 1 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 10,
|
||||||
|
epPerService: 1,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 125,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 2 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 10,
|
||||||
|
epPerService: 2,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 155,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "10 Services 10 EndpointPerService - LoadBalancer",
|
||||||
|
svcFunc: func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = v1.ServiceTypeLoadBalancer
|
||||||
|
svc.Spec.ExternalIPs = []string{"1.2.3.4"}
|
||||||
|
svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
|
||||||
|
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
|
||||||
|
IP: "1.2.3.4",
|
||||||
|
}}
|
||||||
|
},
|
||||||
|
services: 10,
|
||||||
|
epPerService: 10,
|
||||||
|
expectedFilterRules: 3,
|
||||||
|
expectedNatRules: 395,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range testCases {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
ipt := iptablestest.NewFake()
|
||||||
|
fp := NewFakeProxier(ipt)
|
||||||
|
|
||||||
|
svcs, eps := generateServiceEndpoints(test.services, test.epPerService, test.epsFunc, test.svcFunc)
|
||||||
|
|
||||||
|
makeServiceMap(fp, svcs...)
|
||||||
|
populateEndpointSlices(fp, eps...)
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
fp.syncProxyRules()
|
||||||
|
t.Logf("time to sync rule: %v", time.Since(now))
|
||||||
|
t.Logf("iptables data size: %d bytes", fp.iptablesData.Len())
|
||||||
|
|
||||||
|
if fp.filterRules.Lines() != test.expectedFilterRules {
|
||||||
|
t.Errorf("expected number of Filter rules: %d, got: %d", test.expectedFilterRules, fp.filterRules.Lines())
|
||||||
|
}
|
||||||
|
|
||||||
|
if fp.natRules.Lines() != test.expectedNatRules {
|
||||||
|
t.Errorf("expected number of NAT rules: %d, got: %d", test.expectedNatRules, fp.natRules.Lines())
|
||||||
|
}
|
||||||
|
|
||||||
|
// print generated iptables data
|
||||||
|
// t.Logf("Generated rules:\n %s", fp.iptablesData.String())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_generateServiceEndpoints(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
services int
|
||||||
|
epPerService int
|
||||||
|
svcType v1.ServiceType
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Generate 10 Services with 10 Endpoints per Service and LoadBalancer Type",
|
||||||
|
services: 10,
|
||||||
|
epPerService: 10,
|
||||||
|
svcType: v1.ServiceTypeLoadBalancer,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Generate 10 Services with 20 Endpoints per Service and NodePort Type",
|
||||||
|
services: 10,
|
||||||
|
epPerService: 20,
|
||||||
|
svcType: v1.ServiceTypeNodePort,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range testCases {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
// test the function to mutate services
|
||||||
|
svcFunc := func(svc *v1.Service) {
|
||||||
|
svc.Spec.Type = test.svcType
|
||||||
|
}
|
||||||
|
// test the function to mutate endpoint slices
|
||||||
|
epsFunc := func(eps *discovery.EndpointSlice) {
|
||||||
|
for i := range eps.Endpoints {
|
||||||
|
nodeName := fmt.Sprintf("node-%d", i)
|
||||||
|
eps.Endpoints[i].NodeName = &nodeName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
svcs, eps := generateServiceEndpoints(test.services, test.epPerService, epsFunc, svcFunc)
|
||||||
|
|
||||||
|
if len(svcs) != test.services {
|
||||||
|
t.Fatalf("expected %d service, received %d", test.services, len(svcs))
|
||||||
|
}
|
||||||
|
if len(eps) != test.services {
|
||||||
|
t.Fatalf("expected %d endpoint slice , received %d", test.services, len(eps))
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < test.services; i++ {
|
||||||
|
if svcs[i].Spec.Type != test.svcType {
|
||||||
|
t.Fatalf("expected Service Type %s, got %s", test.svcType, svcs[i].Spec.Type)
|
||||||
|
}
|
||||||
|
if eps[i].ObjectMeta.Labels[discovery.LabelServiceName] != svcs[i].Name {
|
||||||
|
t.Fatalf("endpoint slice reference %s instead of Service %s", eps[i].ObjectMeta.Labels[discovery.LabelServiceName], svcs[i].Name)
|
||||||
|
}
|
||||||
|
if len(eps[i].Endpoints) != test.epPerService {
|
||||||
|
t.Fatalf("expected %d endpoints per slice , received %d", test.epPerService, len(eps[i].Endpoints))
|
||||||
|
}
|
||||||
|
for j := 0; j < test.epPerService; j++ {
|
||||||
|
nodeName := fmt.Sprintf("node-%d", j)
|
||||||
|
if *eps[i].Endpoints[j].NodeName != nodeName {
|
||||||
|
t.Errorf("Endpoint %d on EndpointSlice %d expected Nodename %s, got %s", j, i, nodeName, *eps[i].Endpoints[j].NodeName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// generateServiceEndpoints generate Services with the Type specified and it creates N Endpoints per Service
|
||||||
|
func generateServiceEndpoints(nServices, nEndpoints int, epsFunc func(eps *discovery.EndpointSlice), svcFunc func(svc *v1.Service)) ([]*v1.Service, []*discovery.EndpointSlice) {
|
||||||
|
services := make([]*v1.Service, nServices)
|
||||||
|
endpointSlices := make([]*discovery.EndpointSlice, nServices)
|
||||||
|
|
||||||
|
// base parameters
|
||||||
|
basePort := 80
|
||||||
|
base := netutils.BigForIP(netutils.ParseIPSloppy("10.0.0.1"))
|
||||||
|
|
||||||
|
// generate a base endpoint slice object
|
||||||
|
baseEp := netutils.BigForIP(netutils.ParseIPSloppy("172.16.0.1"))
|
||||||
|
epPort := 8080
|
||||||
|
|
||||||
|
tcpProtocol := v1.ProtocolTCP
|
||||||
|
|
||||||
|
eps := &discovery.EndpointSlice{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "ep",
|
||||||
|
Namespace: "namespace",
|
||||||
|
},
|
||||||
|
AddressType: discovery.AddressTypeIPv4,
|
||||||
|
Endpoints: []discovery.Endpoint{},
|
||||||
|
Ports: []discovery.EndpointPort{{
|
||||||
|
Name: utilpointer.StringPtr(fmt.Sprintf("%d", epPort)),
|
||||||
|
Port: utilpointer.Int32(int32(epPort)),
|
||||||
|
Protocol: &tcpProtocol,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for j := 0; j < nEndpoints; j++ {
|
||||||
|
ipEp := netutils.AddIPOffset(baseEp, j)
|
||||||
|
eps.Endpoints = append(eps.Endpoints, discovery.Endpoint{
|
||||||
|
Addresses: []string{ipEp.String()},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if epsFunc != nil {
|
||||||
|
epsFunc(eps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// generate a base service object
|
||||||
|
svc := &v1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "svc",
|
||||||
|
Namespace: "namespace",
|
||||||
|
},
|
||||||
|
Spec: v1.ServiceSpec{
|
||||||
|
Type: v1.ServiceTypeClusterIP,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if svcFunc != nil {
|
||||||
|
svcFunc(svc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the Services and associate and endpoint slice object to each one
|
||||||
|
for i := 0; i < nServices; i++ {
|
||||||
|
ip := netutils.AddIPOffset(base, i)
|
||||||
|
services[i] = svc.DeepCopy()
|
||||||
|
services[i].Name = fmt.Sprintf("svc%d", i)
|
||||||
|
services[i].Spec.ClusterIP = ip.String()
|
||||||
|
services[i].Spec.Ports = []v1.ServicePort{
|
||||||
|
{
|
||||||
|
Name: fmt.Sprintf("%d", epPort),
|
||||||
|
Protocol: v1.ProtocolTCP,
|
||||||
|
Port: int32(basePort + i),
|
||||||
|
TargetPort: intstr.FromInt(epPort),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if svc.Spec.Type == v1.ServiceTypeNodePort || svc.Spec.Type == v1.ServiceTypeLoadBalancer {
|
||||||
|
services[i].Spec.Ports[0].NodePort = int32(30000 + i)
|
||||||
|
|
||||||
|
}
|
||||||
|
if svc.Spec.Type == v1.ServiceTypeLoadBalancer {
|
||||||
|
services[i].Spec.HealthCheckNodePort = int32(32000 + nServices + i)
|
||||||
|
}
|
||||||
|
|
||||||
|
endpointSlices[i] = eps.DeepCopy()
|
||||||
|
endpointSlices[i].Name = services[i].Name
|
||||||
|
endpointSlices[i].ObjectMeta.Labels = map[string]string{
|
||||||
|
discovery.LabelServiceName: services[i].Name,
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
return services, endpointSlices
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user