From e1daee5b371f704df61b4aaf3da889dfdf466b89 Mon Sep 17 00:00:00 2001 From: bprashanth Date: Tue, 6 Dec 2016 15:16:24 -0800 Subject: [PATCH] Delete regional static-ip instead of global for type=lb --- test/e2e/BUILD | 1 - test/e2e/framework/BUILD | 2 + test/e2e/{ => framework}/google_compute.go | 19 +++++---- test/e2e/framework/util.go | 47 ++++++++++++++++++---- test/e2e/service.go | 6 +-- 5 files changed, 54 insertions(+), 21 deletions(-) rename test/e2e/{ => framework}/google_compute.go (86%) diff --git a/test/e2e/BUILD b/test/e2e/BUILD index bab200f59b1..0990873610d 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -57,7 +57,6 @@ go_library( "generated_clientset.go", "gke_local_ssd.go", "gke_node_pools.go", - "google_compute.go", "ha_master.go", "horizontal_pod_autoscaling.go", "ingress.go", diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 19801c35cf6..d1a11c6a5b0 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -17,6 +17,7 @@ go_library( "exec_util.go", "federation_util.go", "framework.go", + "google_compute.go", "kubelet_stats.go", "log_size_monitoring.go", "metrics_util.go", @@ -99,6 +100,7 @@ go_library( "//vendor:github.com/spf13/viper", "//vendor:golang.org/x/crypto/ssh", "//vendor:golang.org/x/net/websocket", + "//vendor:google.golang.org/api/googleapi", "//vendor:gopkg.in/yaml.v2", "//vendor:k8s.io/client-go/kubernetes", "//vendor:k8s.io/client-go/pkg/util/sets", diff --git a/test/e2e/google_compute.go b/test/e2e/framework/google_compute.go similarity index 86% rename from test/e2e/google_compute.go rename to test/e2e/framework/google_compute.go index 19f067fab62..5cf336e26a7 100644 --- a/test/e2e/google_compute.go +++ b/test/e2e/framework/google_compute.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "fmt" @@ -26,13 +26,12 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" - "k8s.io/kubernetes/test/e2e/framework" ) // TODO: These should really just use the GCE API client library or at least use // better formatted output from the --format flag. -func createGCEStaticIP(name string) (string, error) { +func CreateGCEStaticIP(name string) (string, error) { // gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1" // abshah@abhidesk:~/go/src/code.google.com/p/google-api-go-client/compute/v1$ gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1" // Created [https://www.googleapis.com/compute/v1/projects/abshah-kubernetes-001/regions/us-central1/addresses/test-static-ip]. @@ -41,14 +40,14 @@ func createGCEStaticIP(name string) (string, error) { var outputBytes []byte var err error - region, err := gce.GetGCERegion(framework.TestContext.CloudConfig.Zone) + region, err := gce.GetGCERegion(TestContext.CloudConfig.Zone) if err != nil { return "", fmt.Errorf("failed to convert zone to region: %v", err) } - glog.Infof("Creating static IP with name %q in project %q in region %q", name, framework.TestContext.CloudConfig.ProjectID, region) + glog.Infof("Creating static IP with name %q in project %q in region %q", name, TestContext.CloudConfig.ProjectID, region) for attempts := 0; attempts < 4; attempts++ { outputBytes, err = exec.Command("gcloud", "compute", "addresses", "create", - name, "--project", framework.TestContext.CloudConfig.ProjectID, + name, "--project", TestContext.CloudConfig.ProjectID, "--region", region, "-q").CombinedOutput() if err == nil { break @@ -75,20 +74,20 @@ func createGCEStaticIP(name string) (string, error) { } } -func deleteGCEStaticIP(name string) error { +func DeleteGCEStaticIP(name string) error { // gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1" // abshah@abhidesk:~/go/src/code.google.com/p/google-api-go-client/compute/v1$ gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1" // Created [https://www.googleapis.com/compute/v1/projects/abshah-kubernetes-001/regions/us-central1/addresses/test-static-ip]. // NAME REGION ADDRESS STATUS // test-static-ip us-central1 104.197.143.7 RESERVED - region, err := gce.GetGCERegion(framework.TestContext.CloudConfig.Zone) + region, err := gce.GetGCERegion(TestContext.CloudConfig.Zone) if err != nil { return fmt.Errorf("failed to convert zone to region: %v", err) } - glog.Infof("Deleting static IP with name %q in project %q in region %q", name, framework.TestContext.CloudConfig.ProjectID, region) + glog.Infof("Deleting static IP with name %q in project %q in region %q", name, TestContext.CloudConfig.ProjectID, region) outputBytes, err := exec.Command("gcloud", "compute", "addresses", "delete", - name, "--project", framework.TestContext.CloudConfig.ProjectID, + name, "--project", TestContext.CloudConfig.ProjectID, "--region", region, "-q").CombinedOutput() if err != nil { // Ditch the error, since the stderr in the output is what actually contains diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 335b775cd2d..55c6bbc3309 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -41,6 +41,7 @@ import ( "time" "github.com/golang/glog" + "google.golang.org/api/googleapi" "k8s.io/kubernetes/federation/client/clientset_generated/federation_release_1_5" "k8s.io/kubernetes/pkg/api" apierrs "k8s.io/kubernetes/pkg/api/errors" @@ -4945,17 +4946,49 @@ func (p *E2ETestNodePreparer) CleanupNodes() error { return encounteredError } -func CleanupGCEResources(loadBalancerName string) (err error) { +// CleanupGCEResources cleans up GCE Service Type=LoadBalancer resources with +// the given name. The name is usually the UUID of the Service prefixed with an +// alpha-numeric character ('a') to work around cloudprovider rules. +func CleanupGCEResources(loadBalancerName string) (retErr error) { gceCloud, ok := TestContext.CloudConfig.Provider.(*gcecloud.GCECloud) if !ok { return fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", TestContext.CloudConfig.Provider) } - gceCloud.DeleteFirewall(loadBalancerName) - gceCloud.DeleteForwardingRule(loadBalancerName) - gceCloud.DeleteGlobalStaticIP(loadBalancerName) - hc, _ := gceCloud.GetHttpHealthCheck(loadBalancerName) - gceCloud.DeleteTargetPool(loadBalancerName, hc) - return nil + if err := gceCloud.DeleteFirewall(loadBalancerName); err != nil && + !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { + retErr = err + } + if err := gceCloud.DeleteForwardingRule(loadBalancerName); err != nil && + !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { + retErr = fmt.Errorf("%v\n%v", retErr, err) + + } + if err := gceCloud.DeleteGlobalStaticIP(loadBalancerName); err != nil && + !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { + retErr = fmt.Errorf("%v\n%v", retErr, err) + } + // This function shells out to gcloud, so we can't compare for NotFound errors. + // TODO: Invoke cloudprovider method directly instead. + if err := DeleteGCEStaticIP(loadBalancerName); err != nil { + Logf("%v", err) + } + hc, getErr := gceCloud.GetHttpHealthCheck(loadBalancerName) + if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) { + retErr = fmt.Errorf("%v\n%v", retErr, getErr) + return + } + if err := gceCloud.DeleteTargetPool(loadBalancerName, hc); err != nil && + !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { + retErr = fmt.Errorf("%v\n%v", retErr, err) + } + return +} + +// IsHTTPErrorCode returns true if the error is a google api +// error matching the corresponding HTTP error code. +func IsGoogleAPIHTTPErrorCode(err error, code int) bool { + apiErr, ok := err.(*googleapi.Error) + return ok && apiErr.Code == code } // getMaster populates the externalIP, internalIP and hostname fields of the master. diff --git a/test/e2e/service.go b/test/e2e/service.go index 7847ac57ec4..c53f4e1c2d8 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -580,12 +580,12 @@ var _ = framework.KubeDescribe("Services", func() { if framework.ProviderIs("gce", "gke") { By("creating a static load balancer IP") staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunId) - requestedIP, err = createGCEStaticIP(staticIPName) + requestedIP, err = framework.CreateGCEStaticIP(staticIPName) Expect(err).NotTo(HaveOccurred()) defer func() { if staticIPName != "" { // Release GCE static IP - this is not kube-managed and will not be automatically released. - if err := deleteGCEStaticIP(staticIPName); err != nil { + if err := framework.DeleteGCEStaticIP(staticIPName); err != nil { framework.Logf("failed to release static IP %s: %v", staticIPName, err) } } @@ -632,7 +632,7 @@ var _ = framework.KubeDescribe("Services", func() { if staticIPName != "" { // Deleting it after it is attached "demotes" it to an // ephemeral IP, which can be auto-released. - if err := deleteGCEStaticIP(staticIPName); err != nil { + if err := framework.DeleteGCEStaticIP(staticIPName); err != nil { framework.Failf("failed to release static IP %s: %v", staticIPName, err) } staticIPName = ""