From b4a37509124eb358c0334d0ac59664e74359f8e3 Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Mon, 11 Apr 2022 18:06:52 +0800 Subject: [PATCH] [e2e][azure] Make internalStaticIP flexible Now, internalStaticIP is hard-coded to "10.240.11.11". Such IP works for aks-engine cluster but not for CAPZ ones (node-subnet 10.1.0.0/16) Signed-off-by: Zhecheng Li --- test/e2e/framework/node/resource.go | 36 +++++++++++++++++++++++++++++ test/e2e/network/loadbalancer.go | 20 ++++++++++++---- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index a7444199a6d..f4662628d1d 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -377,6 +377,42 @@ func GetRandomReadySchedulableNode(c clientset.Interface) (*v1.Node, error) { return &nodes.Items[rand.Intn(len(nodes.Items))], nil } +// GetSubnetPrefix gets first 2 number of an IP in the node subnet. [IPv4] +// It assumes that the subnet mask is /16. +func GetSubnetPrefix(c clientset.Interface) ([]string, error) { + node, err := GetReadySchedulableWorkerNode(c) + if err != nil { + return nil, fmt.Errorf("error getting a ready schedulable worker Node, err: %v", err) + } + internalIP, err := GetInternalIP(node) + if err != nil { + return nil, fmt.Errorf("error getting Node internal IP, err: %v", err) + } + splitted := strings.Split(internalIP, ".") + if len(splitted) == 4 { + return splitted[:2], nil + } + return nil, fmt.Errorf("invalid IP address format: %s", internalIP) +} + +// GetReadySchedulableWorkerNode gets a single worker node which is available for +// running pods on. If there are no such available nodes it will return an error. +func GetReadySchedulableWorkerNode(c clientset.Interface) (*v1.Node, error) { + nodes, err := GetReadySchedulableNodes(c) + if err != nil { + return nil, err + } + for i := range nodes.Items { + node := nodes.Items[i] + _, isMaster := node.Labels["node-role.kubernetes.io/master"] + _, isControlPlane := node.Labels["node-role.kubernetes.io/control-plane"] + if !isMaster && !isControlPlane { + return &node, nil + } + } + return nil, fmt.Errorf("there are currently no ready, schedulable worker nodes in the cluster") +} + // GetReadyNodesIncludingTainted returns all ready nodes, even those which are tainted. // There are cases when we care about tainted nodes // E.g. in tests related to nodes with gpu we care about nodes despite diff --git a/test/e2e/network/loadbalancer.go b/test/e2e/network/loadbalancer.go index 10bfa1140b0..cf3ca5d9c3a 100644 --- a/test/e2e/network/loadbalancer.go +++ b/test/e2e/network/loadbalancer.go @@ -57,9 +57,13 @@ var _ = common.SIGDescribe("LoadBalancers", func() { var cs clientset.Interface serviceLBNames := []string{} + var subnetPrefix []string + var err error ginkgo.BeforeEach(func() { cs = f.ClientSet + subnetPrefix, err = e2enode.GetSubnetPrefix(cs) + framework.ExpectNoError(err) }) ginkgo.AfterEach(func() { @@ -590,7 +594,7 @@ var _ = common.SIGDescribe("LoadBalancers", func() { isInternalEndpoint := func(lbIngress *v1.LoadBalancerIngress) bool { ingressEndpoint := e2eservice.GetIngressPoint(lbIngress) // Needs update for providers using hostname as endpoint. - return strings.HasPrefix(ingressEndpoint, "10.") + return strings.HasPrefix(ingressEndpoint, subnetPrefix[0]+".") } ginkgo.By("creating a service with type LoadBalancer and cloud specific Internal-LB annotation enabled") @@ -669,7 +673,9 @@ var _ = common.SIGDescribe("LoadBalancers", func() { // will be removed when GCP supports similar functionality. if framework.ProviderIs("azure") { ginkgo.By("switching back to interal type LoadBalancer, with static IP specified.") - internalStaticIP := "10.240.11.11" + // For a cluster created with CAPZ, node-subnet may not be "10.240.0.0/16", e.g. "10.1.0.0/16". + internalStaticIP := fmt.Sprintf("%s.%s.11.11", subnetPrefix[0], subnetPrefix[1]) + svc, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.LoadBalancerIP = internalStaticIP enableILB(svc) @@ -987,6 +993,8 @@ var _ = common.SIGDescribe("LoadBalancers ESIPP [Slow]", func() { var cs clientset.Interface serviceLBNames := []string{} + var subnetPrefix []string + var err error ginkgo.BeforeEach(func() { // requires cloud load-balancer support - this feature currently supported only on GCE/GKE @@ -994,6 +1002,8 @@ var _ = common.SIGDescribe("LoadBalancers ESIPP [Slow]", func() { cs = f.ClientSet loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs) + subnetPrefix, err = e2enode.GetSubnetPrefix(cs) + framework.ExpectNoError(err) }) ginkgo.AfterEach(func() { @@ -1052,7 +1062,7 @@ var _ = common.SIGDescribe("LoadBalancers ESIPP [Slow]", func() { framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP) ginkgo.By("checking if Source IP is preserved") - if strings.HasPrefix(clientIP, "10.") { + if strings.HasPrefix(clientIP, subnetPrefix[0]+".") { framework.Failf("Source IP was NOT preserved") } }) @@ -1334,7 +1344,7 @@ var _ = common.SIGDescribe("LoadBalancers ESIPP [Slow]", func() { if err != nil { return false, nil } - if strings.HasPrefix(clientIP, "10.") { + if strings.HasPrefix(clientIP, subnetPrefix[0]+".") { return true, nil } return false, nil @@ -1363,7 +1373,7 @@ var _ = common.SIGDescribe("LoadBalancers ESIPP [Slow]", func() { return false, nil } ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP)) - if !strings.HasPrefix(clientIP, "10.") { + if !strings.HasPrefix(clientIP, subnetPrefix[0]+".") { return true, nil } return false, nil