From ad1f2063298e8f0913812dccd6bd6b6219fbb850 Mon Sep 17 00:00:00 2001 From: Ashley Gau Date: Mon, 26 Feb 2018 14:16:26 -0800 Subject: [PATCH] isolate logic to be shared with internal lb tests into separate file --- .../gce/gce_loadbalancer_external_test.go | 216 ++++++------------ .../gce/gce_loadbalancer_test_utils.go | 144 ++++++++++++ 2 files changed, 214 insertions(+), 146 deletions(-) create mode 100644 pkg/cloudprovider/providers/gce/gce_loadbalancer_test_utils.go diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go index 587de352404..07ec73a7888 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go @@ -18,90 +18,26 @@ package gce import ( "fmt" - "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" computealpha "google.golang.org/api/compute/v0.alpha" - compute "google.golang.org/api/compute/v1" "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" - "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/mock" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" ) -const ( - projectID = "test-project" - region = "us-central1" - zoneName = "us-central1-b" - nodeName = "test-node-1" - clusterName = "Test Cluster Name" - clusterID = "test-cluster-id" - serviceName = "" -) - -var apiService = &v1.Service{ - Spec: v1.ServiceSpec{ - SessionAffinity: v1.ServiceAffinityClientIP, - Type: v1.ServiceTypeClusterIP, - Ports: []v1.ServicePort{{Protocol: v1.ProtocolTCP, Port: int32(123)}}, - }, -} - -type fakeRoundTripper struct{} - -func (*fakeRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, fmt.Errorf("err: test used fake http client") -} - -func fakeGCECloud() (*GCECloud, error) { - c := &http.Client{Transport: &fakeRoundTripper{}} - - service, err := compute.New(c) - if err != nil { - return nil, err - } - - // Used in disk unit tests - fakeManager := newFakeManager(projectID, region) - zonesWithNodes := createNodeZones([]string{zoneName}) - - alphaFeatureGate, err := NewAlphaFeatureGate([]string{}) - if err != nil { - return nil, err - } - - gce := &GCECloud{ - region: region, - service: service, - manager: fakeManager, - managedZones: []string{zoneName}, - projectID: projectID, - networkProjectID: projectID, - AlphaFeatureGate: alphaFeatureGate, - nodeZones: zonesWithNodes, - nodeInformerSynced: func() bool { return true }, - } - - cloud := cloud.NewMockGCE(&gceProjectRouter{gce}) - cloud.MockTargetPools.AddInstanceHook = mock.AddInstanceHook - cloud.MockTargetPools.RemoveInstanceHook = mock.RemoveInstanceHook - cloud.MockForwardingRules.InsertHook = mock.InsertFwdRuleHook - cloud.MockAddresses.InsertHook = mock.InsertAddressHook - cloud.MockAlphaAddresses.InsertHook = mock.InsertAlphaAddressHook - - gce.c = cloud - - return gce, nil -} +const serviceName = "" func TestEnsureStaticIP(t *testing.T) { - gce, err := fakeGCECloud() + projectID := "test-project" + region := "us-central1" + zoneName := "us-central1-b" + + gce, err := fakeGCECloud(projectID, region, zoneName) require.NoError(t, err) ipName := "some-static-ip" @@ -121,7 +57,11 @@ func TestEnsureStaticIP(t *testing.T) { } func TestEnsureStaticIPWithTier(t *testing.T) { - s, err := fakeGCECloud() + projectID := "test-project" + region := "us-central1" + zoneName := "us-central1-b" + + s, err := fakeGCECloud(projectID, region, zoneName) require.NoError(t, err) for desc, tc := range map[string]struct { @@ -156,6 +96,7 @@ func TestEnsureStaticIPWithTier(t *testing.T) { func TestVerifyRequestedIP(t *testing.T) { lbRef := "test-lb" + region := "us-central1" for desc, tc := range map[string]struct { requestedIP string @@ -196,7 +137,7 @@ func TestVerifyRequestedIP(t *testing.T) { }, } { t.Run(desc, func(t *testing.T) { - s, err := fakeGCECloud() + s, err := fakeGCECloud("test-project", region, "us-central1a") require.NoError(t, err) for _, addr := range tc.addrList { @@ -214,6 +155,9 @@ func TestCreateForwardingRuleWithTier(t *testing.T) { ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}} target := "test-target-pool" svcName := "foo-svc" + projectID := "test-project" + region := "us-central1" + baseLinkUrl := "https://www.googleapis.com/compute/%v/projects/%v/regions/%v/forwardingRules/%v" for desc, tc := range map[string]struct { @@ -248,7 +192,7 @@ func TestCreateForwardingRuleWithTier(t *testing.T) { }, } { t.Run(desc, func(t *testing.T) { - s, err := fakeGCECloud() + s, err := fakeGCECloud(projectID, region, "us-central1a") require.NoError(t, err) lbName := tc.expectedRule.Name @@ -266,8 +210,9 @@ func TestCreateForwardingRuleWithTier(t *testing.T) { func TestDeleteAddressWithWrongTier(t *testing.T) { lbRef := "test-lb" + region := "us-central1" - s, err := fakeGCECloud() + s, err := fakeGCECloud("test-region", region, "us-central1a") require.NoError(t, err) // Enable the cloud.NetworkTiers feature @@ -324,56 +269,8 @@ func TestDeleteAddressWithWrongTier(t *testing.T) { } } -func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error) { - nodes := []*v1.Node{} - - for _, name := range nodeNames { - // Inserting the same node name twice causes an error - here we check if - // the instance exists already before insertion. - // TestUpdateExternalLoadBalancer inserts a new node, and relies on an older - // node to already have been inserted. - instance, _ := gce.getInstanceByName(name) - - if instance == nil { - err := gce.InsertInstance( - projectID, - zoneName, - &compute.Instance{ - Name: name, - Tags: &compute.Tags{ - Items: []string{name}, - }, - }, - ) - if err != nil { - return nodes, err - } - } - - nodes = append( - nodes, - &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - kubeletapis.LabelHostname: name, - }, - }, - Status: v1.NodeStatus{ - NodeInfo: v1.NodeSystemInfo{ - KubeProxyVersion: "v1.7.2", - }, - }, - }, - ) - - } - - return nodes, nil -} - -func createExternalLoadBalancer(gce *GCECloud) (*v1.LoadBalancerStatus, error) { - nodes, err := createAndInsertNodes(gce, []string{nodeName}) +func createExternalLoadBalancer(gce *GCECloud, nodeNames []string, clusterName, clusterID, zoneName string) (*v1.LoadBalancerStatus, error) { + nodes, err := createAndInsertNodes(gce, nodeNames, zoneName) if err != nil { return nil, err } @@ -381,21 +278,28 @@ func createExternalLoadBalancer(gce *GCECloud) (*v1.LoadBalancerStatus, error) { return gce.ensureExternalLoadBalancer( clusterName, clusterID, - apiService, + fakeApiService, nil, nodes, ) } func TestEnsureExternalLoadBalancer(t *testing.T) { - gce, err := fakeGCECloud() + projectID := "test-project" + region := "us-central1" + zoneName := "us-central1-b" + clusterName := "Test Cluster Name" + clusterID := "test-cluster-id" + nodeName := "test-node-1" + + gce, err := fakeGCECloud(projectID, region, zoneName) require.NoError(t, err) - status, err := createExternalLoadBalancer(gce) + status, err := createExternalLoadBalancer(gce, []string{nodeName}, clusterName, clusterID, zoneName) assert.NoError(t, err) assert.NotEmpty(t, status.Ingress) - lbName := cloudprovider.GetLoadBalancerName(apiService) + lbName := cloudprovider.GetLoadBalancerName(fakeApiService) hcName := MakeNodesHealthCheckName(clusterID) // Check that Firewalls are created for the LoadBalancer and the HealthCheck @@ -432,21 +336,29 @@ func TestEnsureExternalLoadBalancer(t *testing.T) { } func TestUpdateExternalLoadBalancer(t *testing.T) { - gce, err := fakeGCECloud() + projectID := "test-project" + region := "us-central1" + zoneName := "us-central1-b" + clusterName := "Test Cluster Name" + clusterID := "test-cluster-1" + + nodeName := "test-node-1" + + gce, err := fakeGCECloud(projectID, region, zoneName) require.NoError(t, err) - _, err = createExternalLoadBalancer(gce) + _, err = createExternalLoadBalancer(gce, []string{nodeName}, clusterName, clusterID, zoneName) assert.NoError(t, err) newNodeName := "test-node-2" - newNodes, err := createAndInsertNodes(gce, []string{nodeName, newNodeName}) + newNodes, err := createAndInsertNodes(gce, []string{nodeName, newNodeName}, zoneName) assert.NoError(t, err) // Add the new node, then check that it is properly added to the TargetPool - err = gce.updateExternalLoadBalancer(clusterName, apiService, newNodes) + err = gce.updateExternalLoadBalancer("", fakeApiService, newNodes) assert.NoError(t, err) - lbName := cloudprovider.GetLoadBalancerName(apiService) + lbName := cloudprovider.GetLoadBalancerName(fakeApiService) pool, err := gce.GetTargetPool(lbName, region) require.NoError(t, err) @@ -464,13 +376,13 @@ func TestUpdateExternalLoadBalancer(t *testing.T) { fmt.Sprintf("/zones/%s/instances/%s", zoneName, newNodeName), ) - newNodes, err = createAndInsertNodes(gce, []string{nodeName}) + newNodes, err = createAndInsertNodes(gce, []string{nodeName}, zoneName) assert.NoError(t, err) // Remove the new node by calling updateExternalLoadBalancer with a list // only containing the old node, and test that the TargetPool no longer // contains the new node. - err = gce.updateExternalLoadBalancer(clusterName, apiService, newNodes) + err = gce.updateExternalLoadBalancer(clusterName, fakeApiService, newNodes) assert.NoError(t, err) pool, err = gce.GetTargetPool(lbName, region) @@ -484,16 +396,22 @@ func TestUpdateExternalLoadBalancer(t *testing.T) { } func TestEnsureExternalLoadBalancerDeleted(t *testing.T) { - gce, err := fakeGCECloud() + projectID := "test-project" + region := "us-central1" + zoneName := "us-central1-b" + clusterName := "Test Cluster Name" + clusterID := "test-cluster-id" + + gce, err := fakeGCECloud(projectID, region, zoneName) require.NoError(t, err) - _, err = createExternalLoadBalancer(gce) + _, err = createExternalLoadBalancer(gce, []string{"test-node-1"}, clusterName, clusterID, zoneName) assert.NoError(t, err) - err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, apiService) + err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, fakeApiService) assert.NoError(t, err) - lbName := cloudprovider.GetLoadBalancerName(apiService) + lbName := cloudprovider.GetLoadBalancerName(fakeApiService) hcName := MakeNodesHealthCheckName(clusterID) // Check that Firewalls are deleted for the LoadBalancer and the HealthCheck @@ -525,20 +443,26 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) { } func TestLoadBalancerWrongTierResourceDeletion(t *testing.T) { - gce, err := fakeGCECloud() + projectID := "test-project" + region := "us-central1" + zoneName := "us-central1-b" + clusterName := "Test Cluster Name" + clusterID := "test-cluster-id" + + gce, err := fakeGCECloud(projectID, region, zoneName) require.NoError(t, err) // Enable the cloud.NetworkTiers feature gce.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true - apiService.Annotations = map[string]string{NetworkTierAnnotationKey: "Premium"} + fakeApiService.Annotations = map[string]string{NetworkTierAnnotationKey: "Premium"} // cloud.NetworkTier defaults to Premium - desiredTier, err := gce.getServiceNetworkTier(apiService) + desiredTier, err := gce.getServiceNetworkTier(fakeApiService) require.NoError(t, err) assert.Equal(t, cloud.NetworkTierPremium, desiredTier) - lbName := cloudprovider.GetLoadBalancerName(apiService) - serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} + lbName := cloudprovider.GetLoadBalancerName(fakeApiService) + serviceName := types.NamespacedName{Namespace: fakeApiService.Namespace, Name: fakeApiService.Name} // create ForwardingRule and Address with the wrong tier err = createForwardingRule( @@ -548,7 +472,7 @@ func TestLoadBalancerWrongTierResourceDeletion(t *testing.T) { region, "", gce.targetPoolURL(lbName), - apiService.Spec.Ports, + fakeApiService.Spec.Ports, cloud.NetworkTierStandard, ) require.NoError(t, err) @@ -562,7 +486,7 @@ func TestLoadBalancerWrongTierResourceDeletion(t *testing.T) { err = gce.ReserveAlphaRegionAddress(addressObj, region) require.NoError(t, err) - _, err = createExternalLoadBalancer(gce) + _, err = createExternalLoadBalancer(gce, []string{"test-node-1"}, clusterName, clusterID, zoneName) require.NoError(t, err) // Expect forwarding rule tier to not be Standard diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_test_utils.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_test_utils.go new file mode 100644 index 00000000000..8e57e35ad12 --- /dev/null +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_test_utils.go @@ -0,0 +1,144 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This file contains shared functions and variables to set up for tests for +ExternalLoadBalancer and InternalLoadBalancers. It currently cannot live in a +separate package from GCE because then it would cause a circular import. +*/ + +package gce + +import ( + "fmt" + "net/http" + + compute "google.golang.org/api/compute/v1" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/mock" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" +) + +var fakeApiService = &v1.Service{ + Spec: v1.ServiceSpec{ + SessionAffinity: v1.ServiceAffinityClientIP, + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{{Protocol: v1.ProtocolTCP, Port: int32(123)}}, + }, +} + +type fakeRoundTripper struct{} + +func (*fakeRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, fmt.Errorf("err: test used fake http client") +} + +func fakeGCECloud(projectID, region, zoneName string) (*GCECloud, error) { + client := &http.Client{Transport: &fakeRoundTripper{}} + + service, err := compute.New(client) + if err != nil { + return nil, err + } + + // Used in disk unit tests + fakeManager := newFakeManager(projectID, region) + zonesWithNodes := createNodeZones([]string{zoneName}) + + alphaFeatureGate, err := NewAlphaFeatureGate([]string{}) + if err != nil { + return nil, err + } + + gce := &GCECloud{ + region: region, + service: service, + manager: fakeManager, + managedZones: []string{zoneName}, + projectID: projectID, + networkProjectID: projectID, + AlphaFeatureGate: alphaFeatureGate, + nodeZones: zonesWithNodes, + nodeInformerSynced: func() bool { return true }, + } + + c := cloud.NewMockGCE(&gceProjectRouter{gce}) + c.MockTargetPools.AddInstanceHook = mock.AddInstanceHook + c.MockTargetPools.RemoveInstanceHook = mock.RemoveInstanceHook + c.MockForwardingRules.InsertHook = mock.InsertFwdRuleHook + c.MockAddresses.InsertHook = mock.InsertAddressHook + c.MockAlphaAddresses.InsertHook = mock.InsertAlphaAddressHook + + keyGA := meta.GlobalKey("key-ga") + c.MockZones.Objects[*keyGA] = &cloud.MockZonesObj{ + Obj: &compute.Zone{Name: zoneName, Region: gce.getRegionLink(region)}, + } + + gce.c = c + + return gce, nil +} + +func createAndInsertNodes(gce *GCECloud, nodeNames []string, zoneName string) ([]*v1.Node, error) { + nodes := []*v1.Node{} + + for _, name := range nodeNames { + // Inserting the same node name twice causes an error - here we check if + // the instance exists already before insertion. + // TestUpdateExternalLoadBalancer inserts a new node, and relies on an older + // node to already have been inserted. + instance, _ := gce.getInstanceByName(name) + + if instance == nil { + err := gce.InsertInstance( + gce.ProjectID(), + zoneName, + &compute.Instance{ + Name: name, + Tags: &compute.Tags{ + Items: []string{name}, + }, + }, + ) + if err != nil { + return nodes, err + } + } + + nodes = append( + nodes, + &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + kubeletapis.LabelHostname: name, + kubeletapis.LabelZoneFailureDomain: zoneName, + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + KubeProxyVersion: "v1.7.2", + }, + }, + }, + ) + + } + + return nodes, nil +}