[GCE Ingress e2e] Add test for pre-shared certificate

This commit is contained in:
Zihong Zheng 2018-01-26 16:35:05 -08:00
parent 89cbdc0d6f
commit 38290535d6
6 changed files with 171 additions and 45 deletions

View File

@ -26,7 +26,6 @@ import (
"encoding/json"
"encoding/pem"
"fmt"
"io"
"math/big"
"net"
"net/http"
@ -64,11 +63,20 @@ const (
// Ingress class annotation defined in ingress repository.
// TODO: All these annotations should be reused from
// ingress-gce/pkg/annotations instead of duplicating them here.
IngressClass = "kubernetes.io/ingress.class"
IngressClassKey = "kubernetes.io/ingress.class"
// Ingress class annotation value for multi cluster ingress.
MulticlusterIngressClassValue = "gce-multi-cluster"
// Static IP annotation defined in ingress repository.
IngressStaticIPKey = "kubernetes.io/ingress.global-static-ip-name"
// Allow HTTP annotation defined in ingress repository.
IngressAllowHTTPKey = "kubernetes.io/ingress.allow-http"
// Pre-shared-cert annotation defined in ingress repository.
IngressPreSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
// all cloud resources created by the ingress controller start with this
// prefix.
k8sPrefix = "k8s-"
@ -210,15 +218,15 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations m
}
}
// generateRSACerts generates a basic self signed certificate using a key length
// GenerateRSACerts generates a basic self signed certificate using a key length
// of rsaBits, valid for validFor time.
func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
func GenerateRSACerts(host string, isCA bool) ([]byte, []byte, error) {
if len(host) == 0 {
return fmt.Errorf("Require a non-empty host for client hello")
return nil, nil, fmt.Errorf("Require a non-empty host for client hello")
}
priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
if err != nil {
return fmt.Errorf("Failed to generate key: %v", err)
return nil, nil, fmt.Errorf("Failed to generate key: %v", err)
}
notBefore := time.Now()
notAfter := notBefore.Add(validFor)
@ -227,7 +235,7 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return fmt.Errorf("failed to generate serial number: %s", err)
return nil, nil, fmt.Errorf("failed to generate serial number: %s", err)
}
template := x509.Certificate{
SerialNumber: serialNumber,
@ -257,17 +265,18 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
template.KeyUsage |= x509.KeyUsageCertSign
}
var keyOut, certOut bytes.Buffer
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return fmt.Errorf("Failed to create certificate: %s", err)
return nil, nil, fmt.Errorf("Failed to create certificate: %s", err)
}
if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
return fmt.Errorf("Failed creating cert: %v", err)
if err := pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
return nil, nil, fmt.Errorf("Failed creating cert: %v", err)
}
if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
return fmt.Errorf("Failed creating keay: %v", err)
if err := pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
return nil, nil, fmt.Errorf("Failed creating keay: %v", err)
}
return nil
return certOut.Bytes(), keyOut.Bytes(), nil
}
// buildTransportWithCA creates a transport for use in executing HTTPS requests with
@ -297,16 +306,13 @@ func BuildInsecureClient(timeout time.Duration) *http.Client {
// If a secret with the same name already pathExists in the namespace of the
// Ingress, it's updated.
func createIngressTLSSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) {
var k, c bytes.Buffer
tls := ing.Spec.TLS[0]
host = strings.Join(tls.Hosts, ",")
Logf("Generating RSA cert for host %v", host)
if err = generateRSACerts(host, true, &k, &c); err != nil {
cert, key, err := GenerateRSACerts(host, true)
if err != nil {
return
}
cert := c.Bytes()
key := k.Bytes()
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: tls.SecretName,
@ -1046,7 +1052,7 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml"))
ExpectNoError(err)
j.Ingress.Namespace = ns
j.Ingress.Annotations = map[string]string{IngressClass: j.Class}
j.Ingress.Annotations = map[string]string{IngressClassKey: j.Class}
for k, v := range ingAnnotations {
j.Ingress.Annotations[k] = v
}
@ -1145,6 +1151,31 @@ func WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout ti
return address, err
}
func (j *IngressTestJig) PollIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) {
// Check that all rules respond to a simple GET.
knownHostsSet := sets.NewString(knownHosts...)
for _, rules := range j.Ingress.Spec.Rules {
timeoutClient := &http.Client{Timeout: IngressReqTimeout}
proto := "http"
if knownHostsSet.Has(rules.Host) {
var err error
// Create transport with cert to verify if the server uses the correct one.
timeoutClient.Transport, err = buildTransportWithCA(rules.Host, cert)
ExpectNoError(err)
proto = "https"
}
for _, p := range rules.IngressRuleValue.HTTP.Paths {
if waitForNodePort {
j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))
}
route := fmt.Sprintf("%v://%v%v", proto, j.Address, p.Path)
Logf("Testing route %v host %v with simple GET", route, rules.Host)
ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false))
}
}
Logf("Finished polling on all rules on ingress %q", j.Ingress.Name)
}
// WaitForIngress waits till the ingress acquires an IP, then waits for its
// hosts/urls to respond to a protocol check (either http or https). If
// waitForNodePort is true, the NodePort of the Service is verified before
@ -1158,28 +1189,31 @@ func (j *IngressTestJig) WaitForIngress(waitForNodePort bool) {
}
j.Address = address
Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name)
timeoutClient := &http.Client{Timeout: IngressReqTimeout}
// Check that all rules respond to a simple GET.
for _, rules := range j.Ingress.Spec.Rules {
proto := "http"
if len(j.Ingress.Spec.TLS) > 0 {
knownHosts := sets.NewString(j.Ingress.Spec.TLS[0].Hosts...)
if knownHosts.Has(rules.Host) {
timeoutClient.Transport, err = buildTransportWithCA(rules.Host, j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName))
ExpectNoError(err)
proto = "https"
}
}
for _, p := range rules.IngressRuleValue.HTTP.Paths {
if waitForNodePort {
j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))
}
route := fmt.Sprintf("%v://%v%v", proto, address, p.Path)
Logf("Testing route %v host %v with simple GET", route, rules.Host)
ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false))
}
var knownHosts []string
var cert []byte
if len(j.Ingress.Spec.TLS) > 0 {
knownHosts = j.Ingress.Spec.TLS[0].Hosts
cert = j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName)
}
j.PollIngressWithCert(waitForNodePort, knownHosts, cert)
}
// WaitForIngress waits till the ingress acquires an IP, then waits for its
// hosts/urls to respond to a protocol check (either http or https). If
// waitForNodePort is true, the NodePort of the Service is verified before
// verifying the Ingress. NodePort is currently a requirement for cloudprovider
// Ingress. Hostnames and certificate need to be explicitly passed in.
func (j *IngressTestJig) WaitForIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) {
// Wait for the loadbalancer IP.
address, err := WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, LoadBalancerPollTimeout)
if err != nil {
Failf("Ingress failed to acquire an IP address within %v", LoadBalancerPollTimeout)
}
j.Address = address
Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name)
j.PollIngressWithCert(waitForNodePort, knownHosts, cert)
}
// VerifyURL polls for the given iterations, in intervals, and fails if the

View File

@ -22,8 +22,11 @@ import (
"strings"
"time"
compute "google.golang.org/api/compute/v1"
extensions "k8s.io/api/extensions/v1beta1"
rbacv1beta1 "k8s.io/api/rbac/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/intstr"
@ -34,7 +37,6 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
compute "google.golang.org/api/compute/v1"
)
const (
@ -123,8 +125,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns, ip))
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": ns,
"kubernetes.io/ingress.allow-http": "false",
framework.IngressStaticIPKey: ns,
framework.IngressAllowHTTPKey: "false",
}, map[string]string{})
By("waiting for Ingress to come up with ip: " + ip)
@ -317,10 +319,52 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
Expect(hcAfterSync.HttpHealthCheck.RequestPath).To(Equal(hcToChange.HttpHealthCheck.RequestPath))
})
It("should create ingress with pre-shared certificate", func() {
preSharedCertName := "test-pre-shared-cert"
By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName))
testHostname := "test.ingress.com"
cert, key, err := framework.GenerateRSACerts(testHostname, true)
Expect(err).NotTo(HaveOccurred())
gceCloud, err := framework.GetGCECloud()
Expect(err).NotTo(HaveOccurred())
defer func() {
// We would not be able to delete the cert until ingress controller
// cleans up the target proxy that references it.
By("Deleting ingress before deleting ssl certificate")
jig.TryDeleteIngress()
By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName))
err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) {
if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) {
framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err)
return false, nil
}
return true, nil
})
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err))
}()
_, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{
Name: preSharedCertName,
Certificate: string(cert),
PrivateKey: string(key),
Description: "pre-shared cert for ingress testing",
})
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err))
By("Creating an ingress referencing the pre-shared certificate")
// Create an ingress referencing this cert using pre-shared-cert annotation.
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), ns, map[string]string{
framework.IngressPreSharedCertKey: preSharedCertName,
framework.IngressAllowHTTPKey: "false",
}, map[string]string{})
By("Test that ingress works with the pre-shared certificate")
jig.WaitForIngressWithCert(true, []string{testHostname}, cert)
})
It("multicluster ingress should get instance group annotation", func() {
name := "echomap"
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "http"), ns, map[string]string{
framework.IngressClass: framework.MulticlusterIngressClassValue,
framework.IngressClassKey: framework.MulticlusterIngressClassValue,
}, map[string]string{})
By(fmt.Sprintf("waiting for Ingress %s to come up", name))
@ -343,6 +387,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
// TODO: Implement a multizone e2e that verifies traffic reaches each
// zone based on pod labels.
})
Describe("GCE [Slow] [Feature:NEG]", func() {
var gceController *framework.GCEIngressController

View File

@ -0,0 +1,16 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: pre-shared-cert
# Below annotation will be added upon test:
# annotations:
# ingress.gcp.kubernetes.io/pre-shared-cert: "test-pre-shared-cert"
spec:
rules:
- host: test.ingress.com
http:
paths:
- path: /test
backend:
serviceName: echoheaders-https
servicePort: 80

View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: ReplicationController
metadata:
name: echoheaders-https
spec:
replicas: 2
template:
metadata:
labels:
app: echoheaders-https
spec:
containers:
- name: echoheaders-https
image: gcr.io/google_containers/echoserver:1.6
ports:
- containerPort: 8080

View File

@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: echoheaders-https
labels:
app: echoheaders-https
spec:
type: NodePort
ports:
- port: 80
targetPort: 8080
protocol: TCP
name: http
selector:
app: echoheaders-https

View File

@ -94,8 +94,8 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
// Create a working basic Ingress
By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip))
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": t.ipName,
"kubernetes.io/ingress.allow-http": "false",
framework.IngressStaticIPKey: t.ipName,
framework.IngressAllowHTTPKey: "false",
}, map[string]string{})
t.jig.AddHTTPS("tls-secret", "ingress.test.com")