diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index a9bd6c4ec62..6fd8b325a09 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -372,6 +372,19 @@ func CleanupGCEIngressController(gceController *GCEIngressController) { } } +func (cont *GCEIngressController) ListGlobalForwardingRules() []*compute.ForwardingRule { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + fwdList := []*compute.ForwardingRule{} + l, err := gceCloud.ListGlobalForwardingRules() + Expect(err).NotTo(HaveOccurred()) + for _, fwd := range l { + if cont.isOwned(fwd.Name) { + fwdList = append(fwdList, fwd) + } + } + return fwdList +} + func (cont *GCEIngressController) deleteForwardingRule(del bool) string { msg := "" fwList := []compute.ForwardingRule{} @@ -394,6 +407,13 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string { return msg } +func (cont *GCEIngressController) GetGlobalAddress(ipName string) *compute.Address { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + ip, err := gceCloud.GetGlobalAddress(ipName) + Expect(err).NotTo(HaveOccurred()) + return ip +} + func (cont *GCEIngressController) deleteAddresses(del bool) string { msg := "" ipList := []compute.Address{} @@ -414,6 +434,32 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string { return msg } +func (cont *GCEIngressController) ListTargetHttpProxies() []*compute.TargetHttpProxy { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + tpList := []*compute.TargetHttpProxy{} + l, err := gceCloud.ListTargetHttpProxies() + Expect(err).NotTo(HaveOccurred()) + for _, tp := range l { + if cont.isOwned(tp.Name) { + tpList = append(tpList, tp) + } + } + return tpList +} + +func (cont *GCEIngressController) ListTargetHttpsProxies() []*compute.TargetHttpsProxy { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + tpsList := []*compute.TargetHttpsProxy{} + l, err := gceCloud.ListTargetHttpsProxies() + Expect(err).NotTo(HaveOccurred()) + for _, tps := range l { + if cont.isOwned(tps.Name) { + tpsList = append(tpsList, tps) + } + } + return tpsList +} + func (cont *GCEIngressController) deleteTargetProxy(del bool) string { msg := "" tpList := []compute.TargetHttpProxy{} @@ -449,6 +495,19 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string { return msg } +func (cont *GCEIngressController) ListUrlMaps() []*compute.UrlMap { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + umList := []*compute.UrlMap{} + l, err := gceCloud.ListUrlMaps() + Expect(err).NotTo(HaveOccurred()) + for _, um := range l { + if cont.isOwned(um.Name) { + umList = append(umList, um) + } + } + return umList +} + func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) { gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) umList, err := gceCloud.ListUrlMaps() @@ -478,6 +537,19 @@ func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) { return msg } +func (cont *GCEIngressController) ListGlobalBackendServices() []*compute.BackendService { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + beList := []*compute.BackendService{} + l, err := gceCloud.ListGlobalBackendServices() + Expect(err).NotTo(HaveOccurred()) + for _, be := range l { + if cont.isOwned(be.Name) { + beList = append(beList, be) + } + } + return beList +} + func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) { gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) beList, err := gceCloud.ListGlobalBackendServices() @@ -537,6 +609,19 @@ func (cont *GCEIngressController) deleteHTTPHealthCheck(del bool) (msg string) { return msg } +func (cont *GCEIngressController) ListSslCertificates() []*compute.SslCertificate { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + sslList := []*compute.SslCertificate{} + l, err := gceCloud.ListSslCertificates() + Expect(err).NotTo(HaveOccurred()) + for _, ssl := range l { + if cont.isOwned(ssl.Name) { + sslList = append(sslList, ssl) + } + } + return sslList +} + func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) { gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) sslList, err := gceCloud.ListSslCertificates() @@ -565,6 +650,19 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) { return msg } +func (cont *GCEIngressController) ListInstanceGroups() []*compute.InstanceGroup { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + igList := []*compute.InstanceGroup{} + l, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone) + Expect(err).NotTo(HaveOccurred()) + for _, ig := range l { + if cont.isOwned(ig.Name) { + igList = append(igList, ig) + } + } + return igList +} + func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) { gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) // TODO: E2E cloudprovider has only 1 zone, but the cluster can have many. @@ -658,6 +756,12 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri return canDeleteWithTimestamp(resourceName, creationTimestamp) } +// isOwned returns true if the resourceName ends in a suffix matching this +// controller UID. +func (cont *GCEIngressController) isOwned(resourceName string) bool { + return cont.canDelete(resourceName, "", false) +} + // canDeleteNEG returns true if either the name contains this controller's UID, // or the creationTimestamp exceeds the maxAge and del is set to true. func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool { diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index f24f85fcc3c..b96d683cd12 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -40,6 +40,15 @@ func EtcdUpgrade(target_storage, target_version string) error { } } +func IngressUpgrade() error { + switch TestContext.Provider { + case "gce": + return ingressUpgradeGCE() + default: + return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider) + } +} + func MasterUpgrade(v string) error { switch TestContext.Provider { case "gce": @@ -64,6 +73,15 @@ func etcdUpgradeGCE(target_storage, target_version string) error { return err } +func ingressUpgradeGCE() error { + // Flip glbc image from latest release image to HEAD to simulate an upgrade. + // Kubelet should restart glbc automatically. + sshResult, err := NodeExec(GetMasterHost(), "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/e2e-ingress-gce\\/ingress-gce-e2e-glbc-amd64:latest/' /etc/kubernetes/manifests/glbc.manifest") + // TODO(rramkumar): Ensure glbc pod is in "Running" state before proceeding. + LogSSHResult(sshResult) + return err +} + // TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error { return masterUpgradeGCE(v, enableKubeProxyDaemonSet) diff --git a/test/e2e/lifecycle/cluster_upgrade.go b/test/e2e/lifecycle/cluster_upgrade.go index 6986be69c7d..243febacd11 100644 --- a/test/e2e/lifecycle/cluster_upgrade.go +++ b/test/e2e/lifecycle/cluster_upgrade.go @@ -72,6 +72,11 @@ var kubeProxyDowngradeTests = []upgrades.Test{ &upgrades.IngressUpgradeTest{}, } +// Upgrade ingress with custom image. +var ingressUpgradeTests = []upgrades.Test{ + &upgrades.IngressUpgradeTest{}, +} + var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() { f := framework.NewDefaultFramework("cluster-upgrade") @@ -201,6 +206,31 @@ var _ = SIGDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() { }) }) +var _ = SIGDescribe("ingress Upgrade [Feature:IngressUpgrade]", func() { + f := framework.NewDefaultFramework("ingress-upgrade") + + // Create the frameworks here because we can only create them + // in a "Describe". + testFrameworks := createUpgradeFrameworks(ingressUpgradeTests) + Describe("ingress upgrade", func() { + It("should maintain a functioning ingress", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "") + framework.ExpectNoError(err) + + testSuite := &junit.TestSuite{Name: "ingress upgrade"} + ingressTest := &junit.TestCase{Name: "[sig-networking] ingress-upgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, ingressTest) + + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, ingressTest) + framework.ExpectNoError(framework.IngressUpgrade()) + } + runUpgradeSuite(f, ingressUpgradeTests, testFrameworks, testSuite, upgCtx, upgrades.IngressUpgrade, upgradeFunc) + }) + }) +}) + var _ = Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func() { f := framework.NewDefaultFramework("stateful-upgrade") diff --git a/test/e2e/testing-manifests/ingress/static-ip-2/ing.yaml b/test/e2e/testing-manifests/ingress/static-ip-2/ing.yaml new file mode 100644 index 00000000000..097854434e2 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/static-ip-2/ing.yaml @@ -0,0 +1,11 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: static-ip + # This annotation is added by the test upon allocating a staticip. + # annotations: + # kubernetes.io/ingress.global-static-ip-name: "staticip" +spec: + backend: + serviceName: echoheaders-https + servicePort: 80 diff --git a/test/e2e/testing-manifests/ingress/static-ip-2/rc.yaml b/test/e2e/testing-manifests/ingress/static-ip-2/rc.yaml new file mode 100644 index 00000000000..abf9b036edd --- /dev/null +++ b/test/e2e/testing-manifests/ingress/static-ip-2/rc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: echoheaders-https +spec: + replicas: 2 + template: + metadata: + labels: + app: echoheaders-https + spec: + containers: + - name: echoheaders-https + image: gcr.io/google_containers/echoserver:1.6 + ports: + - containerPort: 8080 diff --git a/test/e2e/testing-manifests/ingress/static-ip-2/svc.yaml b/test/e2e/testing-manifests/ingress/static-ip-2/svc.yaml new file mode 100644 index 00000000000..b022aa17fce --- /dev/null +++ b/test/e2e/testing-manifests/ingress/static-ip-2/svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: echoheaders-https + labels: + app: echoheaders-https +spec: + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: echoheaders-https diff --git a/test/e2e/testing-manifests/ingress/static-ip/ing.yaml b/test/e2e/testing-manifests/ingress/static-ip/ing.yaml index 342c9dcafbf..a75d42db2e6 100644 --- a/test/e2e/testing-manifests/ingress/static-ip/ing.yaml +++ b/test/e2e/testing-manifests/ingress/static-ip/ing.yaml @@ -13,4 +13,3 @@ spec: backend: serviceName: echoheaders-https servicePort: 80 - diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index fecb075d19f..a3db3528724 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -29,9 +29,11 @@ go_library( "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/utils/image:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/gstruct:go_default_library", + "//vendor/google.golang.org/api/compute/v1:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", diff --git a/test/e2e/upgrades/ingress.go b/test/e2e/upgrades/ingress.go index d2d8c71369a..40a142fff44 100644 --- a/test/e2e/upgrades/ingress.go +++ b/test/e2e/upgrades/ingress.go @@ -17,12 +17,17 @@ limitations under the License. package upgrades import ( + "encoding/json" "fmt" "net/http" "path/filepath" + "reflect" + "github.com/davecgh/go-spew/spew" . "github.com/onsi/ginkgo" + compute "google.golang.org/api/compute/v1" + extensions "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" ) @@ -30,16 +35,34 @@ import ( // IngressUpgradeTest adapts the Ingress e2e for upgrade testing type IngressUpgradeTest struct { gceController *framework.GCEIngressController + // holds GCP resources pre-upgrade + resourceStore *GCPResourceStore jig *framework.IngressTestJig httpClient *http.Client ip string ipName string } +// GCPResourceStore keeps track of the GCP resources spun up by an ingress. +// Note: Fields are exported so that we can utilize reflection. +type GCPResourceStore struct { + Fw *compute.Firewall + FwdList []*compute.ForwardingRule + UmList []*compute.UrlMap + TpList []*compute.TargetHttpProxy + TpsList []*compute.TargetHttpsProxy + SslList []*compute.SslCertificate + BeList []*compute.BackendService + Ip *compute.Address + IgList []*compute.InstanceGroup +} + func (IngressUpgradeTest) Name() string { return "ingress-upgrade" } // Setup creates a GLBC, allocates an ip, and an ingress resource, // then waits for a successful connectivity check to the ip. +// Also keeps track of all load balancer resources for cross-checking +// during an IngressUpgrade. func (t *IngressUpgradeTest) Setup(f *framework.Framework) { framework.SkipUnlessProviderIs("gce", "gke") @@ -66,13 +89,18 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) { // Create a working basic Ingress By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip)) - jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{ + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{ "kubernetes.io/ingress.global-static-ip-name": t.ipName, "kubernetes.io/ingress.allow-http": "false", }, map[string]string{}) + jig.AddHTTPS("tls-secret", "ingress.test.com") By("waiting for Ingress to come up with ip: " + t.ip) framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false)) + + By("keeping track of GCP resources created by Ingress") + t.resourceStore = &GCPResourceStore{} + t.populateGCPResourceStore(t.resourceStore) } // Test waits for the upgrade to complete, and then verifies @@ -85,6 +113,8 @@ func (t *IngressUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, // while it's down will leak cloud resources, because the ingress // controller doesn't checkpoint to disk. t.verify(f, done, true) + case IngressUpgrade: + t.verify(f, done, true) default: // Currently ingress gets disrupted across node upgrade, because endpoints // get killed and we don't have any guarantees that 2 nodes don't overlap @@ -99,6 +129,7 @@ func (t *IngressUpgradeTest) Teardown(f *framework.Framework) { if CurrentGinkgoTestDescription().Failed { framework.DescribeIng(t.gceController.Ns) } + if t.jig.Ingress != nil { By("Deleting ingress") t.jig.TryDeleteIngress() @@ -122,4 +153,80 @@ func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{} } By("hitting the Ingress IP " + t.ip) framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false)) + + // We want to manually trigger a sync because then we can easily verify + // a correct sync completed after update. + By("updating ingress spec to manually trigger a sync") + t.jig.Update(func(ing *extensions.Ingress) { + ing.Spec.TLS[0].Hosts = append(ing.Spec.TLS[0].Hosts, "ingress.test.com") + ing.Spec.Rules = append( + ing.Spec.Rules, + extensions.IngressRule{ + Host: "ingress.test.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/test", + // Note: Dependant on using "static-ip-2" manifest. + Backend: *(ing.Spec.Backend), + }, + }, + }, + }, + }) + }) + // WaitForIngress() tests that all paths are pinged, which is how we know + // everything is synced with the cloud. + t.jig.WaitForIngress(false) + By("comparing GCP resources post-upgrade") + postUpgradeResourceStore := &GCPResourceStore{} + t.populateGCPResourceStore(postUpgradeResourceStore) + framework.ExpectNoError(compareGCPResourceStores(t.resourceStore, postUpgradeResourceStore, func(v1 reflect.Value, v2 reflect.Value) error { + i1 := v1.Interface() + i2 := v2.Interface() + // Skip verifying the UrlMap since we did that via WaitForIngress() + if !reflect.DeepEqual(i1, i2) && (v1.Type() != reflect.TypeOf([]*compute.UrlMap{})) { + return spew.Errorf("resources after ingress upgrade were different:\n Pre-Upgrade: %#v\n Post-Upgrade: %#v", i1, i2) + } + return nil + })) +} + +func (t *IngressUpgradeTest) populateGCPResourceStore(resourceStore *GCPResourceStore) { + cont := t.gceController + resourceStore.Fw = cont.GetFirewallRule() + resourceStore.FwdList = cont.ListGlobalForwardingRules() + resourceStore.UmList = cont.ListUrlMaps() + resourceStore.TpList = cont.ListTargetHttpProxies() + resourceStore.TpsList = cont.ListTargetHttpsProxies() + resourceStore.SslList = cont.ListSslCertificates() + resourceStore.BeList = cont.ListGlobalBackendServices() + resourceStore.Ip = cont.GetGlobalAddress(t.ipName) + resourceStore.IgList = cont.ListInstanceGroups() +} + +func compareGCPResourceStores(rs1 *GCPResourceStore, rs2 *GCPResourceStore, compare func(v1 reflect.Value, v2 reflect.Value) error) error { + // Before we do a comparison, remove the ServerResponse field from the + // Compute API structs. This is needed because two objects could be the same + // but their ServerResponse will be different if they were populated through + // separate API calls. + rs1Json, _ := json.Marshal(rs1) + rs2Json, _ := json.Marshal(rs2) + rs1New := &GCPResourceStore{} + rs2New := &GCPResourceStore{} + json.Unmarshal(rs1Json, rs1New) + json.Unmarshal(rs2Json, rs2New) + + // Iterate through struct fields and perform equality checks on the fields. + // We do this rather than performing a deep equal on the struct itself because + // it is easier to log which field, if any, is not the same. + rs1V := reflect.ValueOf(*rs1New) + rs2V := reflect.ValueOf(*rs2New) + for i := 0; i < rs1V.NumField(); i++ { + if err := compare(rs1V.Field(i), rs2V.Field(i)); err != nil { + return err + } + } + return nil } diff --git a/test/e2e/upgrades/upgrade.go b/test/e2e/upgrades/upgrade.go index 510999a543e..11eeadb0fa1 100644 --- a/test/e2e/upgrades/upgrade.go +++ b/test/e2e/upgrades/upgrade.go @@ -40,6 +40,9 @@ const ( // EtcdUpgrade indicates that only etcd is being upgraded (or migrated // between storage versions). EtcdUpgrade + + // IngressUpgrade indicates that only ingress is being upgraded. + IngressUpgrade ) // Test is an interface for upgrade tests.