From 068963fc0622f2fc4007fe8da0936abcbde20202 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Wed, 11 Mar 2020 15:41:45 -0700 Subject: [PATCH] add testing --- pkg/proxy/iptables/BUILD | 3 + pkg/proxy/iptables/proxier_test.go | 77 +++++++++++++++++++++++++ pkg/proxy/ipvs/BUILD | 3 + pkg/proxy/ipvs/proxier_test.go | 90 ++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+) diff --git a/pkg/proxy/iptables/BUILD b/pkg/proxy/iptables/BUILD index 24f842e3d2d..1773b637e91 100644 --- a/pkg/proxy/iptables/BUILD +++ b/pkg/proxy/iptables/BUILD @@ -39,6 +39,7 @@ go_test( srcs = ["proxier_test.go"], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/healthcheck:go_default_library", "//pkg/proxy/util:go_default_library", @@ -53,6 +54,8 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index f1fc7af1418..ba414dc7350 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -34,6 +34,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/healthcheck" utilproxy "k8s.io/kubernetes/pkg/proxy/util" @@ -832,6 +835,80 @@ func TestExternalIPsReject(t *testing.T) { } } +func TestOnlyLocalExternalIPs(t *testing.T) { + // TODO(freehan): remove this in k8s 1.19 + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExternalPolicyForExternalIP, true)() + + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt, false) + svcIP := "10.20.30.41" + svcPort := 80 + svcExternalIPs := "50.60.70.81" + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + svc.Spec.ClusterIP = svcIP + svc.Spec.ExternalIPs = []string{svcExternalIPs} + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + TargetPort: intstr.FromInt(svcPort), + }} + }), + ) + makeEndpointsMap(fp) + epIP1 := "10.180.0.1" + epIP2 := "10.180.2.1" + epStrLocal := fmt.Sprintf("%s:%d", epIP1, svcPort) + epStrNonLocal := fmt.Sprintf("%s:%d", epIP2, svcPort) + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: epIP1, + NodeName: nil, + }, { + IP: epIP2, + NodeName: utilpointer.StringPtr(testHostname), + }}, + Ports: []v1.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + }}, + }} + }), + ) + + fp.syncProxyRules() + + proto := strings.ToLower(string(v1.ProtocolTCP)) + lbChain := string(serviceLBChainName(svcPortName.String(), proto)) + + nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStrLocal)) + localEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(v1.ProtocolTCP)), epStrNonLocal)) + + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, lbChain, svcExternalIPs, svcPort) { + errorf(fmt.Sprintf("Failed to find jump to xlb chain %v", lbChain), kubeSvcRules, t) + } + + lbRules := ipt.GetRules(lbChain) + if hasJump(lbRules, nonLocalEpChain, "", 0) { + errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) + } + if !hasJump(lbRules, localEpChain, "", 0) { + errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrNonLocal), lbRules, t) + } +} + func TestNodePortReject(t *testing.T) { ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt, false) diff --git a/pkg/proxy/ipvs/BUILD b/pkg/proxy/ipvs/BUILD index 67c6d4c1597..9da47dbd4f7 100644 --- a/pkg/proxy/ipvs/BUILD +++ b/pkg/proxy/ipvs/BUILD @@ -15,6 +15,7 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/healthcheck:go_default_library", "//pkg/proxy/ipvs/testing:go_default_library", @@ -34,6 +35,8 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 8e3f36bf751..53a056ab6ad 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -33,6 +33,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/proxy/healthcheck" netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing" @@ -1256,6 +1259,92 @@ func TestExternalIPs(t *testing.T) { } } +func TestOnlyLocalExternalIPs(t *testing.T) { + // TODO(freehan): remove this in k8s 1.19 + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExternalPolicyForExternalIP, true)() + + ipt := iptablestest.NewFake() + ipvs := ipvstest.NewFake() + ipset := ipsettest.NewFake(testIPSetVersion) + fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, false) + svcIP := "10.20.30.41" + svcPort := 80 + svcExternalIPs := sets.NewString("50.60.70.81", "2012::51", "127.0.0.1") + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = svcIP + svc.Spec.ExternalIPs = svcExternalIPs.UnsortedList() + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + TargetPort: intstr.FromInt(svcPort), + }} + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + }), + ) + epIP := "10.180.0.1" + epIP1 := "10.180.1.1" + thisHostname := testHostname + otherHostname := "other-hostname" + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: epIP, + NodeName: utilpointer.StringPtr(thisHostname), + }, + { + IP: epIP1, + NodeName: utilpointer.StringPtr(otherHostname), + }, + }, + Ports: []v1.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + }}, + }} + }), + ) + + fp.syncProxyRules() + + // check ipvs service and destinations + services, err := ipvs.GetVirtualServers() + if err != nil { + t.Errorf("Failed to get ipvs services, err: %v", err) + } + if len(services) != 4 { + t.Errorf("Expect 4 ipvs services, got %d", len(services)) + } + found := false + for _, svc := range services { + if svcExternalIPs.Has(svc.Address.String()) && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) { + found = true + destinations, _ := ipvs.GetRealServers(svc) + if len(destinations) != 1 { + t.Errorf("Expect only 1 local endpoint. but got %v", len(destinations)) + } + for _, dest := range destinations { + if dest.Address.String() != epIP || dest.Port != uint16(svcPort) { + t.Errorf("service Endpoint mismatch ipvs service destination") + } + } + break + } + } + if !found { + t.Errorf("Expect external ip type service, got none") + } +} + func TestLoadBalancer(t *testing.T) { ipt, fp := buildFakeProxier() svcIP := "10.20.30.41" @@ -1432,6 +1521,7 @@ func TestOnlyLocalNodePorts(t *testing.T) { } checkIptables(t, ipt, epIpt) } + func TestLoadBalanceSourceRanges(t *testing.T) { ipt, fp := buildFakeProxier()