Merge pull request #59903 from MrHohn/e2e-ingress-app-protocol

Automatic merge from submit-queue (batch tested with PRs 60475, 60514, 60506, 59903, 60319). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[e2e ingress-gce] Add test for backside re-encryption

**What this PR does / why we need it**:
Add a basic e2e testcase for ingress backside re-encryption. echoheaders image with HTTPS support was added by https://github.com/kubernetes/ingress-nginx/pull/2091.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #NONE 

**Special notes for your reviewer**:
/assign @nicksardo @rramkumar1 
cc @nikhiljindal 

**Release note**:

```release-note
NONE
```

test/e2e/framework/ingress_utils.go

@@ -45,6 +45,7 @@ import (
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -80,6 +81,9 @@ const (
 	// Pre-shared-cert annotation defined in ingress repository.
 	IngressPreSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
 
+	// ServiceApplicationProtocolKey annotation defined in ingress repository.
+	ServiceApplicationProtocolKey = "service.alpha.kubernetes.io/app-protocols"
+
 	// all cloud resources created by the ingress controller start with this
 	// prefix.
 	k8sPrefix = "k8s-"
@@ -1519,3 +1523,111 @@ func (cont *NginxIngressController) Init() {
 	ExpectNoError(err)
 	Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
 }
+
+func GenerateReencryptionIngressSpec() *extensions.Ingress {
+	return &extensions.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "echoheaders-reencryption",
+		},
+		Spec: extensions.IngressSpec{
+			Backend: &extensions.IngressBackend{
+				ServiceName: "echoheaders-reencryption",
+				ServicePort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: 443,
+				},
+			},
+		},
+	}
+}
+
+func GenerateReencryptionServiceSpec() *v1.Service {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "echoheaders-reencryption",
+			Annotations: map[string]string{
+				ServiceApplicationProtocolKey: `{"my-https-port":"HTTPS"}`,
+			},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{{
+				Name:       "my-https-port",
+				Protocol:   v1.ProtocolTCP,
+				Port:       443,
+				TargetPort: intstr.FromString("echo-443"),
+			}},
+			Selector: map[string]string{
+				"app": "echoheaders-reencryption",
+			},
+			Type: v1.ServiceTypeNodePort,
+		},
+	}
+}
+
+func GenerateReencryptionDeploymentSpec() *extensions.Deployment {
+	return &extensions.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "echoheaders-reencryption",
+		},
+		Spec: extensions.DeploymentSpec{
+			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+				"app": "echoheaders-reencryption",
+			}},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": "echoheaders-reencryption",
+					},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  "echoheaders-reencryption",
+							Image: "k8s.gcr.io/echoserver:1.9",
+							Ports: []v1.ContainerPort{{
+								ContainerPort: 8443,
+								Name:          "echo-443",
+							}},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func CreateReencryptionIngress(cs clientset.Interface, namespace string) (*extensions.Deployment, *v1.Service, *extensions.Ingress, error) {
+	deployCreated, err := cs.ExtensionsV1beta1().Deployments(namespace).Create(GenerateReencryptionDeploymentSpec())
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	svcCreated, err := cs.CoreV1().Services(namespace).Create(GenerateReencryptionServiceSpec())
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	ingCreated, err := cs.ExtensionsV1beta1().Ingresses(namespace).Create(GenerateReencryptionIngressSpec())
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	return deployCreated, svcCreated, ingCreated, nil
+}
+
+func CleanupReencryptionIngress(cs clientset.Interface, deploy *extensions.Deployment, svc *v1.Service, ing *extensions.Ingress) []error {
+	var errs []error
+	if ing != nil {
+		if err := cs.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil); err != nil {
+			errs = append(errs, fmt.Errorf("error while deleting ingress %s/%s: %v", ing.Namespace, ing.Name, err))
+		}
+	}
+	if svc != nil {
+		if err := cs.CoreV1().Services(svc.Namespace).Delete(svc.Name, nil); err != nil {
+			errs = append(errs, fmt.Errorf("error while deleting service %s/%s: %v", svc.Namespace, svc.Name, err))
+		}
+	}
+	if deploy != nil {
+		if err := cs.ExtensionsV1beta1().Deployments(deploy.Namespace).Delete(deploy.Name, nil); err != nil {
+			errs = append(errs, fmt.Errorf("error while deleting deployment %s/%s: %v", deploy.Namespace, deploy.Name, err))
+		}
+	}
+	return errs
+}

test/e2e/network/ingress.go

@@ -18,6 +18,7 @@ package network
 
 import (
 	"fmt"
+	"net/http"
 	"path/filepath"
 	"strings"
 	"time"
@@ -363,6 +364,38 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
 		})
 
+		It("should create ingress with backside re-encryption", func() {
+			By("Creating a set of ingress, service and deployment that have backside re-encryption configured")
+			deployCreated, svcCreated, ingCreated, err := framework.CreateReencryptionIngress(f.ClientSet, f.Namespace.Name)
+			defer func() {
+				By("Cleaning up re-encryption ingress, service and deployment")
+				if errs := framework.CleanupReencryptionIngress(f.ClientSet, deployCreated, svcCreated, ingCreated); len(errs) > 0 {
+					framework.Failf("Failed to cleanup re-encryption ingress: %v", errs)
+				}
+			}()
+			Expect(err).NotTo(HaveOccurred(), "Failed to create re-encryption ingress")
+
+			By(fmt.Sprintf("Waiting for ingress %s to come up", ingCreated.Name))
+			ingIP, err := jig.WaitForIngressAddress(f.ClientSet, f.Namespace.Name, ingCreated.Name, framework.LoadBalancerPollTimeout)
+			Expect(err).NotTo(HaveOccurred(), "Failed to wait for ingress IP")
+
+			By(fmt.Sprintf("Polling on address %s and verify the backend is serving HTTPS", ingIP))
+			timeoutClient := &http.Client{Timeout: framework.IngressReqTimeout}
+			err = wait.PollImmediate(framework.LoadBalancerPollInterval, framework.LoadBalancerPollTimeout, func() (bool, error) {
+				resp, err := framework.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "")
+				if err != nil {
+					framework.Logf("SimpleGET failed: %v", err)
+					return false, nil
+				}
+				if !strings.Contains(resp, "request_scheme=https") {
+					return false, fmt.Errorf("request wasn't served by HTTPS, response body: %s", resp)
+				}
+				framework.Logf("Poll succeeded, request was served by HTTPS")
+				return true, nil
+			})
+			Expect(err).NotTo(HaveOccurred(), "Failed to verify backside re-encryption ingress")
+		})
+
 		It("multicluster ingress should get instance group annotation", func() {
 			name := "echomap"
 			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "http"), ns, map[string]string{