From 089e0cd9effabde401d30867927ae13eb8987dd6 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Tue, 30 Jul 2019 22:04:52 -0400 Subject: [PATCH] proxy/ipvs: refactor TestNodePort to use test tables Signed-off-by: Andrew Sy Kim --- pkg/proxy/ipvs/proxier_test.go | 502 ++++++++++++++++++++------------- 1 file changed, 302 insertions(+), 200 deletions(-) diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 50a716428c1..4b54222f989 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -395,214 +395,316 @@ func TestGetNodeIPs(t *testing.T) { } } -func TestNodePortUDP(t *testing.T) { - nodeIP := net.ParseIP("100.101.102.103") - ipt := iptablestest.NewFake() - ipvs := ipvstest.NewFake() - ipset := ipsettest.NewFake(testIPSetVersion) - fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{nodeIP}, nil) - - svcIP := "10.20.30.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolUDP, - NodePort: int32(svcNodePort), - }} - }), - ) - epIP := "10.180.0.1" - makeEndpointsMap(fp, - makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) { - ept.Subsets = []v1.EndpointSubset{{ - Addresses: []v1.EndpointAddress{{ - IP: epIP, - }}, - Ports: []v1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - }}, - }} - }), - ) - - fp.nodePortAddresses = []string{"0.0.0.0/0"} - fp.syncProxyRules() - - // Check ipvs service and destinations - epVS := &netlinktest.ExpectedVirtualServer{ - VSNum: 2, IP: nodeIP.String(), Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP), - RS: []netlinktest.ExpectedRealServer{{ - IP: epIP, Port: uint16(svcPort), - }}} - checkIPVS(t, fp, epVS) - - // check ipSet rules - epIPSet := netlinktest.ExpectedIPSet{ - kubeNodePortSetUDP: {{ - Port: svcNodePort, - Protocol: strings.ToLower(string(v1.ProtocolUDP)), - SetType: utilipset.BitmapPort, - }}, - } - checkIPSet(t, fp, epIPSet) - - // Check iptables chain and rules - epIpt := netlinktest.ExpectedIptablesChain{ - string(KubeNodePortChain): {{ - JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetUDP, - }}, - string(kubeServicesChain): {{ - JumpChain: string(KubeNodePortChain), MatchSet: "", - }}, - } - checkIptables(t, ipt, epIpt) -} - func TestNodePort(t *testing.T) { - ipt := iptablestest.NewFake() - ipvs := ipvstest.NewFake() - ipset := ipsettest.NewFake(testIPSetVersion) - nodeIPv4 := net.ParseIP("100.101.102.103") - nodeIPv6 := net.ParseIP("2001:db8::1:1") - nodeIPs := sets.NewString(nodeIPv4.String(), nodeIPv6.String()) - fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{nodeIPv4, nodeIPv6}, nil) - svcIP := "10.20.30.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) - epIPv4 := "10.180.0.1" - epIPv6 := "1002:ab8::2:10" - epIPs := sets.NewString(epIPv4, epIPv6) - makeEndpointsMap(fp, - makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) { - ept.Subsets = []v1.EndpointSubset{{ - Addresses: []v1.EndpointAddress{{ - IP: epIPv4, - }, { - IP: epIPv6, + tests := []struct { + name string + services []*v1.Service + endpoints []*v1.Endpoints + nodeIPs []net.IP + nodePortAddresses []string + expectedIPVS *ipvstest.FakeIPVS + expectedIPSets netlinktest.ExpectedIPSet + expectedIptablesChains netlinktest.ExpectedIptablesChain + }{ + { + name: "1 service with node port, has 2 endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "10.20.30.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{ + makeTestEndpoints("ns1", "svc1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "10.180.0.1", + }, { + IP: "1002:ab8::2:10", + }}, + Ports: []v1.EndpointPort{{ + Name: "p80", + Port: int32(80), + }}, + }} + }), + }, + nodeIPs: []net.IP{ + net.ParseIP("100.101.102.103"), + net.ParseIP("2001:db8::1:1"), + }, + nodePortAddresses: []string{}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "10.20.30.41", + Port: 80, + Protocol: "TCP", + }: { + Address: net.ParseIP("10.20.30.41"), + Protocol: "TCP", + Port: uint16(80), + Scheduler: "rr", + }, + { + IP: "100.101.102.103", + Port: 3001, + Protocol: "TCP", + }: { + Address: net.ParseIP("100.101.102.103"), + Protocol: "TCP", + Port: uint16(3001), + Scheduler: "rr", + }, + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "TCP", + }: { + Address: net.ParseIP("2001:db8::1:1"), + Protocol: "TCP", + Port: uint16(3001), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "10.20.30.41", + Port: 80, + Protocol: "TCP", + }: { + { + Address: net.ParseIP("10.180.0.1"), + Port: uint16(80), + Weight: 1, + }, + { + Address: net.ParseIP("1002:ab8::2:10"), + Port: uint16(80), + Weight: 1, + }, + }, + { + IP: "100.101.102.103", + Port: 3001, + Protocol: "TCP", + }: { + { + Address: net.ParseIP("10.180.0.1"), + Port: uint16(80), + Weight: 1, + }, + { + Address: net.ParseIP("1002:ab8::2:10"), + Port: uint16(80), + Weight: 1, + }, + }, + { + IP: "2001:db8::1:1", + Port: 3001, + Protocol: "TCP", + }: { + { + Address: net.ParseIP("10.180.0.1"), + Port: uint16(80), + Weight: 1, + }, + { + Address: net.ParseIP("1002:ab8::2:10"), + Port: uint16(80), + Weight: 1, + }, + }, + }, + }, + }, + { + name: "1 UDP service with node port, has endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "10.20.30.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolUDP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{ + makeTestEndpoints("ns1", "svc1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: "10.180.0.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p80", + Port: int32(80), + }}, + }} + }), + }, + nodeIPs: []net.IP{ + net.ParseIP("100.101.102.103"), + }, + nodePortAddresses: []string{"0.0.0.0/0"}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "10.20.30.41", + Port: 80, + Protocol: "UDP", + }: { + Address: net.ParseIP("10.20.30.41"), + Protocol: "UDP", + Port: uint16(80), + Scheduler: "rr", + }, + { + IP: "100.101.102.103", + Port: 3001, + Protocol: "UDP", + }: { + Address: net.ParseIP("100.101.102.103"), + Protocol: "UDP", + Port: uint16(3001), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "10.20.30.41", + Port: 80, + Protocol: "UDP", + }: { + { + Address: net.ParseIP("10.180.0.1"), + Port: uint16(80), + Weight: 1, + }, + }, + { + IP: "100.101.102.103", + Port: 3001, + Protocol: "UDP", + }: { + { + Address: net.ParseIP("10.180.0.1"), + Port: uint16(80), + Weight: 1, + }, + }, + }, + }, + expectedIPSets: netlinktest.ExpectedIPSet{ + kubeNodePortSetUDP: {{ + Port: 3001, + Protocol: strings.ToLower(string(v1.ProtocolUDP)), + SetType: utilipset.BitmapPort, }}, - Ports: []v1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), + }, + expectedIptablesChains: netlinktest.ExpectedIptablesChain{ + string(KubeNodePortChain): {{ + JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetUDP, }}, - }} - }), - ) - - fp.nodePortAddresses = []string{"0.0.0.0/0"} - - fp.syncProxyRules() - - // Check ipvs service and destinations - services, err := ipvs.GetVirtualServers() - if err != nil { - t.Errorf("Failed to get ipvs services, err: %v", err) + string(kubeServicesChain): {{ + JumpChain: string(KubeNodePortChain), MatchSet: "", + }}, + }, + }, + { + name: "service has node port but no endpoints", + services: []*v1.Service{ + makeTestService("ns1", "svc1", func(svc *v1.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = "10.20.30.41" + svc.Spec.Ports = []v1.ServicePort{{ + Name: "p80", + Port: int32(80), + Protocol: v1.ProtocolTCP, + NodePort: int32(3001), + }} + }), + }, + endpoints: []*v1.Endpoints{}, + nodeIPs: []net.IP{ + net.ParseIP("100.101.102.103"), + }, + nodePortAddresses: []string{}, + expectedIPVS: &ipvstest.FakeIPVS{ + Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "10.20.30.41", + Port: 80, + Protocol: "TCP", + }: { + Address: net.ParseIP("10.20.30.41"), + Protocol: "TCP", + Port: uint16(80), + Scheduler: "rr", + }, + { + IP: "100.101.102.103", + Port: 3001, + Protocol: "TCP", + }: { + Address: net.ParseIP("100.101.102.103"), + Protocol: "TCP", + Port: uint16(3001), + Scheduler: "rr", + }, + }, + Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "10.20.30.41", + Port: 80, + Protocol: "TCP", + }: {}, // no real servers corresponding to no endpoints + { + IP: "100.101.102.103", + Port: 3001, + Protocol: "TCP", + }: {}, // no real servers corresponding to no endpoints + }, + }, + }, } - if len(services) != 3 { - t.Errorf("Expect 3 ipvs services, got %d", len(services)) - } - found := false - for _, svc := range services { - if nodeIPs.Has(svc.Address.String()) && svc.Port == uint16(svcNodePort) && svc.Protocol == string(v1.ProtocolTCP) { - found = true - destinations, err := ipvs.GetRealServers(svc) - if err != nil { - t.Errorf("Failed to get ipvs destinations, err: %v", err) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ipt := iptablestest.NewFake() + ipvs := ipvstest.NewFake() + ipset := ipsettest.NewFake(testIPSetVersion) + fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil) + fp.nodePortAddresses = test.nodePortAddresses + + makeServiceMap(fp, test.services...) + makeEndpointsMap(fp, test.endpoints...) + + fp.syncProxyRules() + + if !reflect.DeepEqual(ipvs, test.expectedIPVS) { + t.Logf("actual ipvs state: %v", ipvs) + t.Logf("expected ipvs state: %v", test.expectedIPVS) + t.Errorf("unexpected IPVS state") } - for _, dest := range destinations { - if !epIPs.Has(dest.Address.String()) || dest.Port != uint16(svcPort) { - t.Errorf("service Endpoint mismatch ipvs service destination") - } + + if test.expectedIPSets != nil { + checkIPSet(t, fp, test.expectedIPSets) } - break - } - } - if !found { - t.Errorf("Expect node port type service, got none") - } -} -func TestNodePortNoEndpoint(t *testing.T) { - ipt := iptablestest.NewFake() - ipvs := ipvstest.NewFake() - ipset := ipsettest.NewFake(testIPSetVersion) - nodeIP := net.ParseIP("100.101.102.103") - fp := NewFakeProxier(ipt, ipvs, ipset, []net.IP{nodeIP}, nil) - svcIP := "10.20.30.41" - svcPort := 80 - svcNodePort := 3001 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = "NodePort" - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) - makeEndpointsMap(fp) - - fp.nodePortAddresses = []string{"0.0.0.0/0"} - - fp.syncProxyRules() - - // Check ipvs service and destinations - services, err := ipvs.GetVirtualServers() - if err != nil { - t.Errorf("Failed to get ipvs services, err: %v", err) - } - if len(services) != 2 { - t.Errorf("Expect 2 ipvs services, got %d", len(services)) - } - found := false - for _, svc := range services { - if svc.Address.Equal(nodeIP) && svc.Port == uint16(svcNodePort) && svc.Protocol == string(v1.ProtocolTCP) { - found = true - destinations, _ := ipvs.GetRealServers(svc) - if len(destinations) != 0 { - t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations)) + if test.expectedIptablesChains != nil { + checkIptables(t, ipt, test.expectedIptablesChains) } - break - } - } - if !found { - t.Errorf("Expect node port type service, got none") + }) } }