Merge pull request #42020 from nicksardo/ingress_upgrade

Automatic merge from submit-queue (batch tested with PRs 41644, 42020, 41753, 42206, 42212)

Ingress-glbc upgrade tests

Basically #41676 but with some fixes and added comments.  @bprashanth has been away this week and it's desirable to have this in before code freeze.
This commit is contained in:
Kubernetes Submit Queue 2017-03-01 15:30:32 -08:00 committed by GitHub
commit effbad9ad0
7 changed files with 409 additions and 283 deletions

View File

@ -48,7 +48,6 @@ go_library(
"ha_master.go",
"horizontal_pod_autoscaling.go",
"ingress.go",
"ingress_utils.go",
"initial_resources.go",
"job.go",
"kibana_logging.go",
@ -179,7 +178,6 @@ go_library(
"//vendor:golang.org/x/net/context",
"//vendor:golang.org/x/net/websocket",
"//vendor:golang.org/x/oauth2/google",
"//vendor:google.golang.org/api/compute/v1",
"//vendor:google.golang.org/api/googleapi",
"//vendor:google.golang.org/api/logging/v2beta1",
"//vendor:k8s.io/apimachinery/pkg/api/errors",

View File

@ -39,6 +39,7 @@ var upgradeTests = []upgrades.Test{
&upgrades.HPAUpgradeTest{},
&upgrades.PersistentVolumeUpgradeTest{},
&upgrades.DaemonSetUpgradeTest{},
&upgrades.IngressUpgradeTest{},
}
var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {

View File

@ -18,6 +18,7 @@ go_library(
"framework.go",
"get-kubemark-resource-usage.go",
"google_compute.go",
"ingress_utils.go",
"jobs_util.go",
"kubelet_stats.go",
"log_size_monitoring.go",
@ -71,6 +72,7 @@ go_library(
"//pkg/master/ports:go_default_library",
"//pkg/metrics:go_default_library",
"//pkg/ssh:go_default_library",
"//pkg/util:go_default_library",
"//pkg/util/exec:go_default_library",
"//pkg/util/labels:go_default_library",
"//pkg/util/system:go_default_library",

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package framework
import (
"bytes"
@ -31,16 +31,12 @@ import (
"math/big"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
apierrs "k8s.io/apimachinery/pkg/api/errors"
@ -51,11 +47,13 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/util"
utilexec "k8s.io/kubernetes/pkg/util/exec"
"k8s.io/kubernetes/test/e2e/framework"
testutils "k8s.io/kubernetes/test/utils"
. "github.com/onsi/ginkgo"
@ -80,59 +78,84 @@ const (
// Name of the default http backend service
defaultBackendName = "default-http-backend"
// IP src range from which the GCE L7 performs health checks.
// GCEL7SrcRange is the IP src range from which the GCE L7 performs health checks.
GCEL7SrcRange = "130.211.0.0/22"
// Cloud resources created by the ingress controller older than this
// are automatically purged to prevent running out of quota.
// TODO(37335): write soak tests and bump this up to a week.
maxAge = -48 * time.Hour
// IngressManifestPath is the parent path to yaml test manifests.
IngressManifestPath = "test/e2e/testing-manifests/ingress"
// IngressReqTimeout is the timeout on a single http request.
IngressReqTimeout = 10 * time.Second
// healthz port used to verify glbc restarted correctly on the master.
glbcHealthzPort = 8086
// General cloud resource poll timeout (eg: create static ip, firewall etc)
cloudResourcePollTimeout = 5 * time.Minute
// Name of the config-map and key the ingress controller stores its uid in.
uidConfigMap = "ingress-uid"
uidKey = "uid"
// GCE only allows names < 64 characters, and the loadbalancer controller inserts
// a single character of padding.
nameLenLimit = 62
)
type testJig struct {
client clientset.Interface
rootCAs map[string][]byte
address string
ing *extensions.Ingress
// IngressTestJig holds the relevant state and parameters of the ingress test.
type IngressTestJig struct {
Client clientset.Interface
RootCAs map[string][]byte
Address string
Ingress *extensions.Ingress
// class is the value of the annotation keyed under
// `kubernetes.io/ingress.class`. It's added to all ingresses created by
// this jig.
class string
Class string
// The interval used to poll urls
pollInterval time.Duration
PollInterval time.Duration
}
type conformanceTests struct {
entryLog string
execute func()
exitLog string
// IngressConformanceTests contains a closure with an entry and exit log line.
type IngressConformanceTests struct {
EntryLog string
Execute func()
ExitLog string
}
func createComformanceTests(jig *testJig, ns string) []conformanceTests {
manifestPath := filepath.Join(ingressManifestPath, "http")
// These constants match the manifests used in ingressManifestPath
// CreateIngressComformanceTests generates an slice of sequential test cases:
// a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
// ingress https with a modified URLMap
func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConformanceTests {
manifestPath := filepath.Join(IngressManifestPath, "http")
// These constants match the manifests used in IngressManifestPath
tlsHost := "foo.bar.com"
tlsSecretName := "foo"
updatedTLSHost := "foobar.com"
updateURLMapHost := "bar.baz.com"
updateURLMapPath := "/testurl"
// Platform agnostic list of tests that must be satisfied by all controllers
return []conformanceTests{
return []IngressConformanceTests{
{
fmt.Sprintf("should create a basic HTTP ingress"),
func() { jig.createIngress(manifestPath, ns, map[string]string{}) },
func() { jig.CreateIngress(manifestPath, ns, map[string]string{}) },
fmt.Sprintf("waiting for urls on basic HTTP ingress"),
},
{
fmt.Sprintf("should terminate TLS for host %v", tlsHost),
func() { jig.addHTTPS(tlsSecretName, tlsHost) },
func() { jig.AddHTTPS(tlsSecretName, tlsHost) },
fmt.Sprintf("waiting for HTTPS updates to reflect in ingress"),
},
{
fmt.Sprintf("should update SSL certificate with modified hostname %v", updatedTLSHost),
func() {
jig.update(func(ing *extensions.Ingress) {
jig.Update(func(ing *extensions.Ingress) {
newRules := []extensions.IngressRule{}
for _, rule := range ing.Spec.Rules {
if rule.Host != tlsHost {
@ -146,7 +169,7 @@ func createComformanceTests(jig *testJig, ns string) []conformanceTests {
}
ing.Spec.Rules = newRules
})
jig.addHTTPS(tlsSecretName, updatedTLSHost)
jig.AddHTTPS(tlsSecretName, updatedTLSHost)
},
fmt.Sprintf("Waiting for updated certificates to accept requests for host %v", updatedTLSHost),
},
@ -154,7 +177,7 @@ func createComformanceTests(jig *testJig, ns string) []conformanceTests {
fmt.Sprintf("should update url map for host %v to expose a single url: %v", updateURLMapHost, updateURLMapPath),
func() {
var pathToFail string
jig.update(func(ing *extensions.Ingress) {
jig.Update(func(ing *extensions.Ingress) {
newRules := []extensions.IngressRule{}
for _, rule := range ing.Spec.Rules {
if rule.Host != updateURLMapHost {
@ -180,8 +203,8 @@ func createComformanceTests(jig *testJig, ns string) []conformanceTests {
ing.Spec.Rules = newRules
})
By("Checking that " + pathToFail + " is not exposed by polling for failure")
route := fmt.Sprintf("http://%v%v", jig.address, pathToFail)
framework.ExpectNoError(framework.PollURL(route, updateURLMapHost, framework.LoadBalancerCleanupTimeout, jig.pollInterval, &http.Client{Timeout: reqTimeout}, true))
route := fmt.Sprintf("http://%v%v", jig.Address, pathToFail)
ExpectNoError(PollURL(route, updateURLMapHost, LoadBalancerCleanupTimeout, jig.PollInterval, &http.Client{Timeout: IngressReqTimeout}, true))
},
fmt.Sprintf("Waiting for path updates to reflect in L7"),
},
@ -248,13 +271,13 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
return nil
}
// buildTransport creates a transport for use in executing HTTPS requests with
// buildTransportWithCA creates a transport for use in executing HTTPS requests with
// the given certs. Note that the given rootCA must be configured with isCA=true.
func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) {
func buildTransportWithCA(serverName string, rootCA []byte) (*http.Transport, error) {
pool := x509.NewCertPool()
ok := pool.AppendCertsFromPEM(rootCA)
if !ok {
return nil, fmt.Errorf("Unable to load serverCA.")
return nil, fmt.Errorf("Unable to load serverCA")
}
return utilnet.SetTransportDefaults(&http.Transport{
TLSClientConfig: &tls.Config{
@ -265,20 +288,20 @@ func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) {
}), nil
}
// buildInsecureClient returns an insecure http client. Can be used for "curl -k".
func buildInsecureClient(timeout time.Duration) *http.Client {
// BuildInsecureClient returns an insecure http client. Can be used for "curl -k".
func BuildInsecureClient(timeout time.Duration) *http.Client {
t := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
return &http.Client{Timeout: timeout, Transport: utilnet.SetTransportDefaults(t)}
}
// createSecret creates a secret containing TLS certificates for the given Ingress.
// If a secret with the same name already exists in the namespace of the
// createIngressTLSSecret creates a secret containing TLS certificates for the given Ingress.
// If a secret with the same name already pathExists in the namespace of the
// Ingress, it's updated.
func createSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) {
func createIngressTLSSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) {
var k, c bytes.Buffer
tls := ing.Spec.TLS[0]
host = strings.Join(tls.Hosts, ",")
framework.Logf("Generating RSA cert for host %v", host)
Logf("Generating RSA cert for host %v", host)
if err = generateRSACerts(host, true, &k, &c); err != nil {
return
@ -297,20 +320,22 @@ func createSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host
var s *v1.Secret
if s, err = kubeClient.Core().Secrets(ing.Namespace).Get(tls.SecretName, metav1.GetOptions{}); err == nil {
// TODO: Retry the update. We don't really expect anything to conflict though.
framework.Logf("Updating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name)
Logf("Updating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name)
s.Data = secret.Data
_, err = kubeClient.Core().Secrets(ing.Namespace).Update(s)
} else {
framework.Logf("Creating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name)
Logf("Creating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name)
_, err = kubeClient.Core().Secrets(ing.Namespace).Create(secret)
}
return host, cert, key, err
}
func cleanupGCE(gceController *GCEIngressController) {
pollErr := wait.Poll(5*time.Second, framework.LoadBalancerCleanupTimeout, func() (bool, error) {
// CleanupGCEIngressController calls the GCEIngressController.Cleanup(false)
// followed with deleting the static ip, and then a final GCEIngressController.Cleanup(true)
func CleanupGCEIngressController(gceController *GCEIngressController) {
pollErr := wait.Poll(5*time.Second, LoadBalancerCleanupTimeout, func() (bool, error) {
if err := gceController.Cleanup(false); err != nil {
framework.Logf("Still waiting for glbc to cleanup:\n%v", err)
Logf("Still waiting for glbc to cleanup:\n%v", err)
return false, nil
}
return true, nil
@ -320,9 +345,9 @@ func cleanupGCE(gceController *GCEIngressController) {
// controller. Delete this IP only after the controller has had a chance
// to cleanup or it might interfere with the controller, causing it to
// throw out confusing events.
if ipErr := wait.Poll(5*time.Second, framework.LoadBalancerCleanupTimeout, func() (bool, error) {
if ipErr := wait.Poll(5*time.Second, LoadBalancerCleanupTimeout, func() (bool, error) {
if err := gceController.deleteStaticIPs(); err != nil {
framework.Logf("Failed to delete static-ip: %v\n", err)
Logf("Failed to delete static-ip: %v\n", err)
return false, nil
}
return true, nil
@ -342,7 +367,7 @@ func cleanupGCE(gceController *GCEIngressController) {
// Fail if the controller didn't cleanup
if pollErr != nil {
framework.Failf("L7 controller failed to delete all cloud resources on time. %v", pollErr)
Failf("L7 controller failed to delete all cloud resources on time. %v", pollErr)
}
}
@ -350,7 +375,7 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
msg := ""
fwList := []compute.ForwardingRule{}
for _, regex := range []string{fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter), fmt.Sprintf("%vfws-.*%v.*", k8sPrefix, clusterDelimiter)} {
gcloudList("forwarding-rules", regex, cont.cloud.ProjectID, &fwList)
gcloudComputeResourceList("forwarding-rules", regex, cont.Cloud.ProjectID, &fwList)
if len(fwList) == 0 {
continue
}
@ -360,7 +385,7 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
}
msg += fmt.Sprintf("%v (forwarding rule)\n", f.Name)
if del {
gcloudDelete("forwarding-rules", f.Name, cont.cloud.ProjectID, "--global")
GcloudComputeResourceDelete("forwarding-rules", f.Name, cont.Cloud.ProjectID, "--global")
}
}
}
@ -371,7 +396,7 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string {
msg := ""
ipList := []compute.Address{}
regex := fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter)
gcloudList("addresses", regex, cont.cloud.ProjectID, &ipList)
gcloudComputeResourceList("addresses", regex, cont.Cloud.ProjectID, &ipList)
if len(ipList) != 0 {
for _, ip := range ipList {
if !cont.canDelete(ip.Name, ip.CreationTimestamp, del) {
@ -379,7 +404,7 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string {
}
msg += fmt.Sprintf("%v (static-ip)\n", ip.Name)
if del {
gcloudDelete("addresses", ip.Name, cont.cloud.ProjectID, "--global")
GcloudComputeResourceDelete("addresses", ip.Name, cont.Cloud.ProjectID, "--global")
}
}
}
@ -390,7 +415,7 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
msg := ""
tpList := []compute.TargetHttpProxy{}
regex := fmt.Sprintf("%vtp-.*%v.*", k8sPrefix, clusterDelimiter)
gcloudList("target-http-proxies", regex, cont.cloud.ProjectID, &tpList)
gcloudComputeResourceList("target-http-proxies", regex, cont.Cloud.ProjectID, &tpList)
if len(tpList) != 0 {
for _, t := range tpList {
if !cont.canDelete(t.Name, t.CreationTimestamp, del) {
@ -398,13 +423,13 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
}
msg += fmt.Sprintf("%v (target-http-proxy)\n", t.Name)
if del {
gcloudDelete("target-http-proxies", t.Name, cont.cloud.ProjectID)
GcloudComputeResourceDelete("target-http-proxies", t.Name, cont.Cloud.ProjectID)
}
}
}
tpsList := []compute.TargetHttpsProxy{}
regex = fmt.Sprintf("%vtps-.*%v.*", k8sPrefix, clusterDelimiter)
gcloudList("target-https-proxies", regex, cont.cloud.ProjectID, &tpsList)
gcloudComputeResourceList("target-https-proxies", regex, cont.Cloud.ProjectID, &tpsList)
if len(tpsList) != 0 {
for _, t := range tpsList {
if !cont.canDelete(t.Name, t.CreationTimestamp, del) {
@ -412,15 +437,15 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
}
msg += fmt.Sprintf("%v (target-https-proxy)\n", t.Name)
if del {
gcloudDelete("target-https-proxies", t.Name, cont.cloud.ProjectID)
GcloudComputeResourceDelete("target-https-proxies", t.Name, cont.Cloud.ProjectID)
}
}
}
return msg
}
func (cont *GCEIngressController) deleteUrlMap(del bool) (msg string) {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
umList, err := gceCloud.ListUrlMaps()
if err != nil {
if cont.isHTTPErrorCode(err, http.StatusNotFound) {
@ -447,7 +472,7 @@ func (cont *GCEIngressController) deleteUrlMap(del bool) (msg string) {
}
func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
beList, err := gceCloud.ListBackendServices()
if err != nil {
if cont.isHTTPErrorCode(err, http.StatusNotFound) {
@ -456,7 +481,7 @@ func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
return fmt.Sprintf("Failed to list backend services: %v", err)
}
if len(beList.Items) == 0 {
framework.Logf("No backend services found")
Logf("No backend services found")
return msg
}
for _, be := range beList.Items {
@ -474,8 +499,8 @@ func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
return msg
}
func (cont *GCEIngressController) deleteHttpHealthCheck(del bool) (msg string) {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
func (cont *GCEIngressController) deleteHTTPHealthCheck(del bool) (msg string) {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
hcList, err := gceCloud.ListHttpHealthChecks()
if err != nil {
if cont.isHTTPErrorCode(err, http.StatusNotFound) {
@ -502,7 +527,7 @@ func (cont *GCEIngressController) deleteHttpHealthCheck(del bool) (msg string) {
}
func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
sslList, err := gceCloud.ListSslCertificates()
if err != nil {
if cont.isHTTPErrorCode(err, http.StatusNotFound) {
@ -528,10 +553,10 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
}
func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
// We need to poll on all IGs across all zones.
igList, err := gceCloud.ListInstanceGroups(cont.cloud.Zone)
igList, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone)
if err != nil {
if cont.isHTTPErrorCode(err, http.StatusNotFound) {
return msg
@ -547,7 +572,7 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
}
msg += fmt.Sprintf("%v (instance-group)\n", ig.Name)
if del {
if err := gceCloud.DeleteInstanceGroup(ig.Name, cont.cloud.Zone); err != nil &&
if err := gceCloud.DeleteInstanceGroup(ig.Name, cont.Cloud.Zone); err != nil &&
!cont.isHTTPErrorCode(err, http.StatusNotFound) {
msg += fmt.Sprintf("Failed to delete instance group %v\n", ig.Name)
}
@ -574,23 +599,25 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
}
createdTime, err := time.Parse(time.RFC3339, creationTimestamp)
if err != nil {
framework.Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err)
Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err)
return false
}
if createdTime.Before(time.Now().Add(maxAge)) {
framework.Logf("%v created on %v IS too old", resourceName, creationTimestamp)
Logf("%v created on %v IS too old", resourceName, creationTimestamp)
return true
}
return false
}
func (cont *GCEIngressController) getFirewallRuleName() string {
// GetFirewallRuleName returns the name of the firewall used for the GCEIngressController.
func (cont *GCEIngressController) GetFirewallRuleName() string {
return fmt.Sprintf("%vfw-l7%v%v", k8sPrefix, clusterDelimiter, cont.UID)
}
func (cont *GCEIngressController) getFirewallRule() *compute.Firewall {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
fwName := cont.getFirewallRuleName()
// GetFirewallRule returns the firewall used by the GCEIngressController.
func (cont *GCEIngressController) GetFirewallRule() *compute.Firewall {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
fwName := cont.GetFirewallRuleName()
fw, err := gceCloud.GetFirewall(fwName)
Expect(err).NotTo(HaveOccurred())
return fw
@ -599,7 +626,7 @@ func (cont *GCEIngressController) getFirewallRule() *compute.Firewall {
func (cont *GCEIngressController) deleteFirewallRule(del bool) (msg string) {
fwList := []compute.Firewall{}
regex := fmt.Sprintf("%vfw-l7%v.*", k8sPrefix, clusterDelimiter)
gcloudList("firewall-rules", regex, cont.cloud.ProjectID, &fwList)
gcloudComputeResourceList("firewall-rules", regex, cont.Cloud.ProjectID, &fwList)
if len(fwList) != 0 {
for _, f := range fwList {
if !cont.canDelete(f.Name, f.CreationTimestamp, del) {
@ -607,7 +634,7 @@ func (cont *GCEIngressController) deleteFirewallRule(del bool) (msg string) {
}
msg += fmt.Sprintf("%v (firewall rule)\n", f.Name)
if del {
gcloudDelete("firewall-rules", f.Name, cont.cloud.ProjectID)
GcloudComputeResourceDelete("firewall-rules", f.Name, cont.Cloud.ProjectID)
}
}
}
@ -631,9 +658,9 @@ func (cont *GCEIngressController) Cleanup(del bool) error {
errMsg += cont.deleteAddresses(del)
errMsg += cont.deleteTargetProxy(del)
errMsg += cont.deleteUrlMap(del)
errMsg += cont.deleteURLMap(del)
errMsg += cont.deleteBackendService(del)
errMsg += cont.deleteHttpHealthCheck(del)
errMsg += cont.deleteHTTPHealthCheck(del)
errMsg += cont.deleteInstanceGroup(del)
errMsg += cont.deleteFirewallRule(del)
@ -648,63 +675,64 @@ func (cont *GCEIngressController) Cleanup(del bool) error {
return fmt.Errorf(errMsg)
}
func (cont *GCEIngressController) init() {
// Init initializes the GCEIngressController with an UID
func (cont *GCEIngressController) Init() {
uid, err := cont.getL7AddonUID()
Expect(err).NotTo(HaveOccurred())
cont.UID = uid
// There's a name limit imposed by GCE. The controller will truncate.
testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.ns, cont.UID)
testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.Ns, cont.UID)
if len(testName) > nameLenLimit {
framework.Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit)
Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit)
} else {
framework.Logf("Detected cluster UID %v", cont.UID)
Logf("Detected cluster UID %v", cont.UID)
}
}
// createStaticIP allocates a random static ip with the given name. Returns a string
// CreateStaticIP allocates a random static ip with the given name. Returns a string
// representation of the ip. Caller is expected to manage cleanup of the ip by
// invoking deleteStaticIPs.
func (cont *GCEIngressController) createStaticIP(name string) string {
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
func (cont *GCEIngressController) CreateStaticIP(name string) string {
gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
ip, err := gceCloud.ReserveGlobalStaticIP(name, "")
if err != nil {
if delErr := gceCloud.DeleteGlobalStaticIP(name); delErr != nil {
if cont.isHTTPErrorCode(delErr, http.StatusNotFound) {
framework.Logf("Static ip with name %v was not allocated, nothing to delete", name)
Logf("Static ip with name %v was not allocated, nothing to delete", name)
} else {
framework.Logf("Failed to delete static ip %v: %v", name, delErr)
Logf("Failed to delete static ip %v: %v", name, delErr)
}
}
framework.Failf("Failed to allocated static ip %v: %v", name, err)
Failf("Failed to allocated static ip %v: %v", name, err)
}
cont.staticIPName = ip.Name
framework.Logf("Reserved static ip %v: %v", cont.staticIPName, ip.Address)
Logf("Reserved static ip %v: %v", cont.staticIPName, ip.Address)
return ip.Address
}
// deleteStaticIPs deletes all static-ips allocated through calls to
// createStaticIP.
// deleteStaticIPs delets all static-ips allocated through calls to
// CreateStaticIP.
func (cont *GCEIngressController) deleteStaticIPs() error {
if cont.staticIPName != "" {
if err := gcloudDelete("addresses", cont.staticIPName, cont.cloud.ProjectID, "--global"); err == nil {
if err := GcloudComputeResourceDelete("addresses", cont.staticIPName, cont.Cloud.ProjectID, "--global"); err == nil {
cont.staticIPName = ""
} else {
return err
}
} else {
e2eIPs := []compute.Address{}
gcloudList("addresses", "e2e-.*", cont.cloud.ProjectID, &e2eIPs)
gcloudComputeResourceList("addresses", "e2e-.*", cont.Cloud.ProjectID, &e2eIPs)
ips := []string{}
for _, ip := range e2eIPs {
ips = append(ips, ip.Name)
}
framework.Logf("None of the remaining %d static-ips were created by this e2e: %v", len(ips), strings.Join(ips, ", "))
Logf("None of the remaining %d static-ips were created by this e2e: %v", len(ips), strings.Join(ips, ", "))
}
return nil
}
// gcloudList unmarshals json output of gcloud into given out interface.
func gcloudList(resource, regex, project string, out interface{}) {
// gcloudComputeResourceList unmarshals json output of gcloud into given out interface.
func gcloudComputeResourceList(resource, regex, project string, out interface{}) {
// gcloud prints a message to stderr if it has an available update
// so we only look at stdout.
command := []string{
@ -724,204 +752,210 @@ func gcloudList(resource, regex, project string, out interface{}) {
errMsg = fmt.Sprintf("%v, stderr %v", errMsg, string(osExitErr.Stderr))
}
}
framework.Logf("Error running gcloud command 'gcloud %s': err: %v, output: %v, status: %d, msg: %v", strings.Join(command, " "), err, string(output), errCode, errMsg)
Logf("Error running gcloud command 'gcloud %s': err: %v, output: %v, status: %d, msg: %v", strings.Join(command, " "), err, string(output), errCode, errMsg)
}
if err := json.Unmarshal([]byte(output), out); err != nil {
framework.Logf("Error unmarshalling gcloud output for %v: %v, output: %v", resource, err, string(output))
Logf("Error unmarshalling gcloud output for %v: %v, output: %v", resource, err, string(output))
}
}
func gcloudDelete(resource, name, project string, args ...string) error {
framework.Logf("Deleting %v: %v", resource, name)
// GcloudComputeResourceDelete deletes the specified compute resource by name and project.
func GcloudComputeResourceDelete(resource, name, project string, args ...string) error {
Logf("Deleting %v: %v", resource, name)
argList := append([]string{"compute", resource, "delete", name, fmt.Sprintf("--project=%v", project), "-q"}, args...)
output, err := exec.Command("gcloud", argList...).CombinedOutput()
if err != nil {
framework.Logf("Error deleting %v, output: %v\nerror: %+v", resource, string(output), err)
Logf("Error deleting %v, output: %v\nerror: %+v", resource, string(output), err)
}
return err
}
func gcloudCreate(resource, name, project string, args ...string) error {
framework.Logf("Creating %v in project %v: %v", resource, project, name)
// GcloudComputeResourceCreate creates a compute resource with a name and arguments.
func GcloudComputeResourceCreate(resource, name, project string, args ...string) error {
Logf("Creating %v in project %v: %v", resource, project, name)
argsList := append([]string{"compute", resource, "create", name, fmt.Sprintf("--project=%v", project)}, args...)
framework.Logf("Running command: gcloud %+v", strings.Join(argsList, " "))
Logf("Running command: gcloud %+v", strings.Join(argsList, " "))
output, err := exec.Command("gcloud", argsList...).CombinedOutput()
if err != nil {
framework.Logf("Error creating %v, output: %v\nerror: %+v", resource, string(output), err)
Logf("Error creating %v, output: %v\nerror: %+v", resource, string(output), err)
}
return err
}
// createIngress creates the Ingress and associated service/rc.
// CreateIngress creates the Ingress and associated service/rc.
// Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath
// Optional: secret.yaml, ingAnnotations
// If ingAnnotations is specified it will overwrite any annotations in ing.yaml
func (j *testJig) createIngress(manifestPath, ns string, ingAnnotations map[string]string) {
func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string) {
mkpath := func(file string) string {
return filepath.Join(framework.TestContext.RepoRoot, manifestPath, file)
return filepath.Join(TestContext.RepoRoot, manifestPath, file)
}
framework.Logf("creating replication controller")
framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", ns))
Logf("creating replication controller")
RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", ns))
framework.Logf("creating service")
framework.RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns))
Logf("creating service")
RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns))
if exists(mkpath("secret.yaml")) {
framework.Logf("creating secret")
framework.RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns))
if exists, _ := util.FileExists(mkpath("secret.yaml")); exists {
Logf("creating secret")
RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns))
}
j.ing = ingFromManifest(mkpath("ing.yaml"))
j.ing.Namespace = ns
j.ing.Annotations = map[string]string{ingressClass: j.class}
j.Ingress = createIngressFromManifest(mkpath("ing.yaml"))
j.Ingress.Namespace = ns
j.Ingress.Annotations = map[string]string{ingressClass: j.Class}
for k, v := range ingAnnotations {
j.ing.Annotations[k] = v
j.Ingress.Annotations[k] = v
}
framework.Logf(fmt.Sprintf("creating" + j.ing.Name + " ingress"))
Logf(fmt.Sprintf("creating" + j.Ingress.Name + " ingress"))
var err error
j.ing, err = j.client.Extensions().Ingresses(ns).Create(j.ing)
framework.ExpectNoError(err)
j.Ingress, err = j.Client.Extensions().Ingresses(ns).Create(j.Ingress)
ExpectNoError(err)
}
func (j *testJig) update(update func(ing *extensions.Ingress)) {
// Update retrieves the ingress, performs the passed function, and then updates it.
func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) {
var err error
ns, name := j.ing.Namespace, j.ing.Name
ns, name := j.Ingress.Namespace, j.Ingress.Name
for i := 0; i < 3; i++ {
j.ing, err = j.client.Extensions().Ingresses(ns).Get(name, metav1.GetOptions{})
j.Ingress, err = j.Client.Extensions().Ingresses(ns).Get(name, metav1.GetOptions{})
if err != nil {
framework.Failf("failed to get ingress %q: %v", name, err)
Failf("failed to get ingress %q: %v", name, err)
}
update(j.ing)
j.ing, err = j.client.Extensions().Ingresses(ns).Update(j.ing)
update(j.Ingress)
j.Ingress, err = j.Client.Extensions().Ingresses(ns).Update(j.Ingress)
if err == nil {
framework.DescribeIng(j.ing.Namespace)
DescribeIng(j.Ingress.Namespace)
return
}
if !apierrs.IsConflict(err) && !apierrs.IsServerTimeout(err) {
framework.Failf("failed to update ingress %q: %v", name, err)
Failf("failed to update ingress %q: %v", name, err)
}
}
framework.Failf("too many retries updating ingress %q", name)
Failf("too many retries updating ingress %q", name)
}
func (j *testJig) addHTTPS(secretName string, hosts ...string) {
j.ing.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}}
// TODO: Just create the secret in getRootCAs once we're watching secrets in
// AddHTTPS updates the ingress to use this secret for these hosts.
func (j *IngressTestJig) AddHTTPS(secretName string, hosts ...string) {
j.Ingress.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}}
// TODO: Just create the secret in GetRootCAs once we're watching secrets in
// the ingress controller.
_, cert, _, err := createSecret(j.client, j.ing)
framework.ExpectNoError(err)
framework.Logf("Updating ingress %v to use secret %v for TLS termination", j.ing.Name, secretName)
j.update(func(ing *extensions.Ingress) {
_, cert, _, err := createIngressTLSSecret(j.Client, j.Ingress)
ExpectNoError(err)
Logf("Updating ingress %v to use secret %v for TLS termination", j.Ingress.Name, secretName)
j.Update(func(ing *extensions.Ingress) {
ing.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}}
})
j.rootCAs[secretName] = cert
j.RootCAs[secretName] = cert
}
func (j *testJig) getRootCA(secretName string) (rootCA []byte) {
// GetRootCA returns a rootCA from the ingress test jig.
func (j *IngressTestJig) GetRootCA(secretName string) (rootCA []byte) {
var ok bool
rootCA, ok = j.rootCAs[secretName]
rootCA, ok = j.RootCAs[secretName]
if !ok {
framework.Failf("Failed to retrieve rootCAs, no recorded secret by name %v", secretName)
Failf("Failed to retrieve rootCAs, no recorded secret by name %v", secretName)
}
return
}
func (j *testJig) deleteIngress() {
framework.ExpectNoError(j.client.Extensions().Ingresses(j.ing.Namespace).Delete(j.ing.Name, nil))
// DeleteIngress deletes the ingress resource
func (j *IngressTestJig) DeleteIngress() {
ExpectNoError(j.Client.Extensions().Ingresses(j.Ingress.Namespace).Delete(j.Ingress.Name, nil))
}
// waitForIngress waits till the ingress acquires an IP, then waits for its
// WaitForIngress waits till the ingress acquires an IP, then waits for its
// hosts/urls to respond to a protocol check (either http or https). If
// waitForNodePort is true, the NodePort of the Service is verified before
// verifying the Ingress. NodePort is currently a requirement for cloudprovider
// Ingress.
func (j *testJig) waitForIngress(waitForNodePort bool) {
func (j *IngressTestJig) WaitForIngress(waitForNodePort bool) {
// Wait for the loadbalancer IP.
address, err := framework.WaitForIngressAddress(j.client, j.ing.Namespace, j.ing.Name, framework.LoadBalancerPollTimeout)
address, err := WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, LoadBalancerPollTimeout)
if err != nil {
framework.Failf("Ingress failed to acquire an IP address within %v", framework.LoadBalancerPollTimeout)
Failf("Ingress failed to acquire an IP address within %v", LoadBalancerPollTimeout)
}
j.address = address
framework.Logf("Found address %v for ingress %v", j.address, j.ing.Name)
timeoutClient := &http.Client{Timeout: reqTimeout}
j.Address = address
Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name)
timeoutClient := &http.Client{Timeout: IngressReqTimeout}
// Check that all rules respond to a simple GET.
for _, rules := range j.ing.Spec.Rules {
for _, rules := range j.Ingress.Spec.Rules {
proto := "http"
if len(j.ing.Spec.TLS) > 0 {
knownHosts := sets.NewString(j.ing.Spec.TLS[0].Hosts...)
if len(j.Ingress.Spec.TLS) > 0 {
knownHosts := sets.NewString(j.Ingress.Spec.TLS[0].Hosts...)
if knownHosts.Has(rules.Host) {
timeoutClient.Transport, err = buildTransport(rules.Host, j.getRootCA(j.ing.Spec.TLS[0].SecretName))
framework.ExpectNoError(err)
timeoutClient.Transport, err = buildTransportWithCA(rules.Host, j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName))
ExpectNoError(err)
proto = "https"
}
}
for _, p := range rules.IngressRuleValue.HTTP.Paths {
if waitForNodePort {
j.curlServiceNodePort(j.ing.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))
j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))
}
route := fmt.Sprintf("%v://%v%v", proto, address, p.Path)
framework.Logf("Testing route %v host %v with simple GET", route, rules.Host)
framework.ExpectNoError(framework.PollURL(route, rules.Host, framework.LoadBalancerPollTimeout, j.pollInterval, timeoutClient, false))
Logf("Testing route %v host %v with simple GET", route, rules.Host)
ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false))
}
}
}
// verifyURL polls for the given iterations, in intervals, and fails if the
// VerifyURL polls for the given iterations, in intervals, and fails if the
// given url returns a non-healthy http code even once.
func (j *testJig) verifyURL(route, host string, iterations int, interval time.Duration, httpClient *http.Client) error {
func (j *IngressTestJig) VerifyURL(route, host string, iterations int, interval time.Duration, httpClient *http.Client) error {
for i := 0; i < iterations; i++ {
b, err := framework.SimpleGET(httpClient, route, host)
b, err := SimpleGET(httpClient, route, host)
if err != nil {
framework.Logf(b)
Logf(b)
return err
}
framework.Logf("Verfied %v with host %v %d times, sleeping for %v", route, host, i, interval)
Logf("Verfied %v with host %v %d times, sleeping for %v", route, host, i, interval)
time.Sleep(interval)
}
return nil
}
func (j *testJig) curlServiceNodePort(ns, name string, port int) {
func (j *IngressTestJig) pollServiceNodePort(ns, name string, port int) {
// TODO: Curl all nodes?
u, err := framework.GetNodePortURL(j.client, ns, name, port)
framework.ExpectNoError(err)
framework.ExpectNoError(framework.PollURL(u, "", 30*time.Second, j.pollInterval, &http.Client{Timeout: reqTimeout}, false))
u, err := GetNodePortURL(j.Client, ns, name, port)
ExpectNoError(err)
ExpectNoError(PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false))
}
// getIngressNodePorts returns all related backend services' nodePorts.
// GetIngressNodePorts returns all related backend services' nodePorts.
// Current GCE ingress controller allows traffic to the default HTTP backend
// by default, so retrieve its nodePort as well.
func (j *testJig) getIngressNodePorts() []string {
func (j *IngressTestJig) GetIngressNodePorts() []string {
nodePorts := []string{}
defaultSvc, err := j.client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{})
defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort)))
backendSvcs := []string{}
if j.ing.Spec.Backend != nil {
backendSvcs = append(backendSvcs, j.ing.Spec.Backend.ServiceName)
if j.Ingress.Spec.Backend != nil {
backendSvcs = append(backendSvcs, j.Ingress.Spec.Backend.ServiceName)
}
for _, rule := range j.ing.Spec.Rules {
for _, rule := range j.Ingress.Spec.Rules {
for _, ingPath := range rule.HTTP.Paths {
backendSvcs = append(backendSvcs, ingPath.Backend.ServiceName)
}
}
for _, svcName := range backendSvcs {
svc, err := j.client.Core().Services(j.ing.Namespace).Get(svcName, metav1.GetOptions{})
svc, err := j.Client.Core().Services(j.Ingress.Namespace).Get(svcName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
nodePorts = append(nodePorts, strconv.Itoa(int(svc.Spec.Ports[0].NodePort)))
}
return nodePorts
}
// constructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
func (j *testJig) constructFirewallForIngress(gceController *GCEIngressController) *compute.Firewall {
nodeTags := framework.GetNodeTags(j.client, gceController.cloud)
nodePorts := j.getIngressNodePorts()
// ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController) *compute.Firewall {
nodeTags := GetNodeTags(j.Client, gceController.Cloud)
nodePorts := j.GetIngressNodePorts()
fw := compute.Firewall{}
fw.Name = gceController.getFirewallRuleName()
fw.Name = gceController.GetFirewallRuleName()
fw.SourceRanges = []string{GCEL7SrcRange}
fw.TargetTags = nodeTags.Items
fw.Allowed = []*compute.FirewallAllowed{
@ -933,23 +967,23 @@ func (j *testJig) constructFirewallForIngress(gceController *GCEIngressControlle
return &fw
}
// ingFromManifest reads a .json/yaml file and returns the rc in it.
func ingFromManifest(fileName string) *extensions.Ingress {
// createIngressFromManifest reads a .json/yaml file and returns the rc in it.
func createIngressFromManifest(fileName string) *extensions.Ingress {
var ing extensions.Ingress
framework.Logf("Parsing ingress from %v", fileName)
Logf("Parsing ingress from %v", fileName)
data, err := ioutil.ReadFile(fileName)
framework.ExpectNoError(err)
ExpectNoError(err)
json, err := utilyaml.ToJSON(data)
framework.ExpectNoError(err)
ExpectNoError(err)
framework.ExpectNoError(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &ing))
ExpectNoError(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &ing))
return &ing
}
func (cont *GCEIngressController) getL7AddonUID() (string, error) {
framework.Logf("Retrieving UID from config map: %v/%v", metav1.NamespaceSystem, uidConfigMap)
cm, err := cont.c.Core().ConfigMaps(metav1.NamespaceSystem).Get(uidConfigMap, metav1.GetOptions{})
Logf("Retrieving UID from config map: %v/%v", metav1.NamespaceSystem, uidConfigMap)
cm, err := cont.Client.Core().ConfigMaps(metav1.NamespaceSystem).Get(uidConfigMap, metav1.GetOptions{})
if err != nil {
return "", err
}
@ -959,64 +993,54 @@ func (cont *GCEIngressController) getL7AddonUID() (string, error) {
return "", fmt.Errorf("Could not find cluster UID for L7 addon pod")
}
func exists(path string) bool {
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
framework.Failf("Failed to os.Stat path %v", path)
return false
}
// GCEIngressController manages implementation details of Ingress on GCE/GKE.
type GCEIngressController struct {
ns string
Ns string
rcPath string
UID string
staticIPName string
rc *v1.ReplicationController
svc *v1.Service
c clientset.Interface
cloud framework.CloudConfig
Client clientset.Interface
Cloud CloudConfig
}
func newTestJig(c clientset.Interface) *testJig {
return &testJig{client: c, rootCAs: map[string][]byte{}, pollInterval: framework.LoadBalancerPollInterval}
// NewIngressTestJig instantiates struct with client
func NewIngressTestJig(c clientset.Interface) *IngressTestJig {
return &IngressTestJig{Client: c, RootCAs: map[string][]byte{}, PollInterval: LoadBalancerPollInterval}
}
// NginxIngressController manages implementation details of Ingress on Nginx.
type NginxIngressController struct {
ns string
Ns string
rc *v1.ReplicationController
pod *v1.Pod
c clientset.Interface
Client clientset.Interface
externalIP string
}
func (cont *NginxIngressController) init() {
// Init initializes the NginxIngressController
func (cont *NginxIngressController) Init() {
mkpath := func(file string) string {
return filepath.Join(framework.TestContext.RepoRoot, ingressManifestPath, "nginx", file)
return filepath.Join(TestContext.RepoRoot, IngressManifestPath, "nginx", file)
}
framework.Logf("initializing nginx ingress controller")
framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.ns))
Logf("initializing nginx ingress controller")
RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.Ns))
rc, err := cont.c.Core().ReplicationControllers(cont.ns).Get("nginx-ingress-controller", metav1.GetOptions{})
framework.ExpectNoError(err)
rc, err := cont.Client.Core().ReplicationControllers(cont.Ns).Get("nginx-ingress-controller", metav1.GetOptions{})
ExpectNoError(err)
cont.rc = rc
framework.Logf("waiting for pods with label %v", rc.Spec.Selector)
Logf("waiting for pods with label %v", rc.Spec.Selector)
sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector))
framework.ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.c, cont.ns, sel))
pods, err := cont.c.Core().Pods(cont.ns).List(metav1.ListOptions{LabelSelector: sel.String()})
framework.ExpectNoError(err)
ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.Client, cont.Ns, sel))
pods, err := cont.Client.Core().Pods(cont.Ns).List(metav1.ListOptions{LabelSelector: sel.String()})
ExpectNoError(err)
if len(pods.Items) == 0 {
framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel)
Failf("Failed to find nginx ingress controller pods with selector %v", sel)
}
cont.pod = &pods.Items[0]
cont.externalIP, err = framework.GetHostExternalAddress(cont.c, cont.pod)
framework.ExpectNoError(err)
framework.Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
cont.externalIP, err = GetHostExternalAddress(cont.Client, cont.pod)
ExpectNoError(err)
Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
}

View File

@ -30,48 +30,26 @@ import (
. "github.com/onsi/gomega"
)
const (
// parent path to yaml test manifests.
ingressManifestPath = "test/e2e/testing-manifests/ingress"
// timeout on a single http request.
reqTimeout = 10 * time.Second
// healthz port used to verify glbc restarted correctly on the master.
glbcHealthzPort = 8086
// General cloud resource poll timeout (eg: create static ip, firewall etc)
cloudResourcePollTimeout = 5 * time.Minute
// Name of the config-map and key the ingress controller stores its uid in.
uidConfigMap = "ingress-uid"
uidKey = "uid"
// GCE only allows names < 64 characters, and the loadbalancer controller inserts
// a single character of padding.
nameLenLimit = 62
)
var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
defer GinkgoRecover()
var (
ns string
jig *testJig
conformanceTests []conformanceTests
jig *framework.IngressTestJig
conformanceTests []framework.IngressConformanceTests
)
f := framework.NewDefaultFramework("ingress")
BeforeEach(func() {
f.BeforeEach()
jig = newTestJig(f.ClientSet)
jig = framework.NewIngressTestJig(f.ClientSet)
ns = f.Namespace.Name
// this test wants powerful permissions. Since the namespace names are unique, we can leave this
// lying around so we don't have to race any caches
framework.BindClusterRole(jig.client.Rbac(), "cluster-admin", f.Namespace.Name,
framework.BindClusterRole(jig.Client.Rbac(), "cluster-admin", f.Namespace.Name,
rbacv1beta1.Subject{Kind: rbacv1beta1.ServiceAccountKind, Namespace: f.Namespace.Name, Name: "default"})
err := framework.WaitForAuthorizationUpdate(jig.client.AuthorizationV1beta1(),
err := framework.WaitForAuthorizationUpdate(jig.Client.AuthorizationV1beta1(),
serviceaccount.MakeUsername(f.Namespace.Name, "default"),
"", "create", schema.GroupResource{Resource: "pods"}, true)
framework.ExpectNoError(err)
@ -85,18 +63,18 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
// Slow by design ~10m for each "It" block dominated by loadbalancer setup time
// TODO: write similar tests for nginx, haproxy and AWS Ingress.
framework.KubeDescribe("GCE [Slow] [Feature:Ingress]", func() {
var gceController *GCEIngressController
var gceController *framework.GCEIngressController
// Platform specific setup
BeforeEach(func() {
framework.SkipUnlessProviderIs("gce", "gke")
By("Initializing gce controller")
gceController = &GCEIngressController{
ns: ns,
c: jig.client,
cloud: framework.TestContext.CloudConfig,
gceController = &framework.GCEIngressController{
Ns: ns,
Client: jig.Client,
Cloud: framework.TestContext.CloudConfig,
}
gceController.init()
gceController.Init()
})
// Platform specific cleanup
@ -104,51 +82,51 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
if CurrentGinkgoTestDescription().Failed {
framework.DescribeIng(ns)
}
if jig.ing == nil {
if jig.Ingress == nil {
By("No ingress created, no cleanup necessary")
return
}
By("Deleting ingress")
jig.deleteIngress()
jig.DeleteIngress()
By("Cleaning up cloud resources")
cleanupGCE(gceController)
framework.CleanupGCEIngressController(gceController)
})
It("should conform to Ingress spec", func() {
conformanceTests = createComformanceTests(jig, ns)
conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
for _, t := range conformanceTests {
By(t.entryLog)
t.execute()
By(t.exitLog)
jig.waitForIngress(true)
By(t.EntryLog)
t.Execute()
By(t.ExitLog)
jig.WaitForIngress(true)
}
})
It("should create ingress with given static-ip", func() {
// ip released when the rest of lb resources are deleted in cleanupGCE
ip := gceController.createStaticIP(ns)
// ip released when the rest of lb resources are deleted in CleanupGCEIngressController
ip := gceController.CreateStaticIP(ns)
By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns, ip))
jig.createIngress(filepath.Join(ingressManifestPath, "static-ip"), ns, map[string]string{
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": ns,
"kubernetes.io/ingress.allow-http": "false",
})
By("waiting for Ingress to come up with ip: " + ip)
httpClient := buildInsecureClient(reqTimeout)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.pollInterval, httpClient, false))
httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, httpClient, false))
By("should reject HTTP traffic")
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("http://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.pollInterval, httpClient, true))
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("http://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, httpClient, true))
By("should have correct firewall rule for ingress")
fw := gceController.getFirewallRule()
expFw := jig.constructFirewallForIngress(gceController)
fw := gceController.GetFirewallRule()
expFw := jig.ConstructFirewallForIngress(gceController)
// Passed the last argument as `true` to verify the backend ports is a subset
// of the allowed ports in firewall rule, given there may be other existing
// ingress resources and backends we are not aware of.
Expect(framework.VerifyFirewallRule(fw, expFw, gceController.cloud.Network, true)).NotTo(HaveOccurred())
Expect(framework.VerifyFirewallRule(fw, expFw, gceController.Cloud.Network, true)).NotTo(HaveOccurred())
// TODO: uncomment the restart test once we have a way to synchronize
// and know that the controller has resumed watching. If we delete
@ -167,51 +145,51 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
// Time: borderline 5m, slow by design
framework.KubeDescribe("Nginx", func() {
var nginxController *NginxIngressController
var nginxController *framework.NginxIngressController
BeforeEach(func() {
framework.SkipUnlessProviderIs("gce", "gke")
By("Initializing nginx controller")
jig.class = "nginx"
nginxController = &NginxIngressController{ns: ns, c: jig.client}
jig.Class = "nginx"
nginxController = &framework.NginxIngressController{Ns: ns, Client: jig.Client}
// TODO: This test may fail on other platforms. We can simply skip it
// but we want to allow easy testing where a user might've hand
// configured firewalls.
if framework.ProviderIs("gce", "gke") {
framework.ExpectNoError(gcloudCreate("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID, "--allow", "tcp:80,tcp:443", "--network", framework.TestContext.CloudConfig.Network))
framework.ExpectNoError(framework.GcloudComputeResourceCreate("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID, "--allow", "tcp:80,tcp:443", "--network", framework.TestContext.CloudConfig.Network))
} else {
framework.Logf("WARNING: Not running on GCE/GKE, cannot create firewall rules for :80, :443. Assuming traffic can reach the external ips of all nodes in cluster on those ports.")
}
nginxController.init()
nginxController.Init()
})
AfterEach(func() {
if framework.ProviderIs("gce", "gke") {
framework.ExpectNoError(gcloudDelete("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID))
framework.ExpectNoError(framework.GcloudComputeResourceDelete("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID))
}
if CurrentGinkgoTestDescription().Failed {
framework.DescribeIng(ns)
}
if jig.ing == nil {
if jig.Ingress == nil {
By("No ingress created, no cleanup necessary")
return
}
By("Deleting ingress")
jig.deleteIngress()
jig.DeleteIngress()
})
It("should conform to Ingress spec", func() {
// Poll more frequently to reduce e2e completion time.
// This test runs in presubmit.
jig.pollInterval = 5 * time.Second
conformanceTests = createComformanceTests(jig, ns)
jig.PollInterval = 5 * time.Second
conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
for _, t := range conformanceTests {
By(t.entryLog)
t.execute()
By(t.exitLog)
jig.waitForIngress(false)
By(t.EntryLog)
t.Execute()
By(t.ExitLog)
jig.WaitForIngress(false)
}
})
})

View File

@ -14,6 +14,7 @@ go_library(
"daemonsets.go",
"deployments.go",
"horizontal_pod_autoscalers.go",
"ingress.go",
"job.go",
"persistent_volumes.go",
"secrets.go",

View File

@ -0,0 +1,122 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrades
import (
"fmt"
"net/http"
"path/filepath"
. "github.com/onsi/ginkgo"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"
)
// IngressUpgradeTest adapts the Ingress e2e for upgrade testing
type IngressUpgradeTest struct {
gceController *framework.GCEIngressController
jig *framework.IngressTestJig
httpClient *http.Client
ip string
}
// Setup creates a GLBC, allocates an ip, and an ingress resource,
// then waits for a successful connectivity check to the ip.
func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
framework.SkipUnlessProviderIs("gce", "gke")
// jig handles all Kubernetes testing logic
jig := framework.NewIngressTestJig(f.ClientSet)
ns, err := f.CreateNamespace("ingress-upgrade", nil)
framework.ExpectNoError(err)
// gceController handles all cloud testing logic
gceController := &framework.GCEIngressController{
Ns: ns.Name,
Client: jig.Client,
Cloud: framework.TestContext.CloudConfig,
}
gceController.Init()
t.gceController = gceController
t.jig = jig
t.httpClient = framework.BuildInsecureClient(framework.IngressReqTimeout)
// Allocate a static-ip for the Ingress, this IP is cleaned up via CleanupGCEIngressController
t.ip = t.gceController.CreateStaticIP(ns.Name)
// Create a working basic Ingress
By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns.Name, t.ip))
jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
"kubernetes.io/ingress.global-static-ip-name": ns.Name,
"kubernetes.io/ingress.allow-http": "false",
})
By("waiting for Ingress to come up with ip: " + t.ip)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))
}
// Test waits for the upgrade to complete, and then verifies
// with a connectvity check to the loadbalancer ip.
func (t *IngressUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
switch upgrade {
case MasterUpgrade:
// Restarting the ingress controller shouldn't disrupt a steady state
// Ingress. Restarting the ingress controller and deleting ingresses
// while it's down will leak cloud resources, because the ingress
// controller doesn't checkpoint to disk.
t.verify(f, done, true)
default:
// Currently ingress gets disrupted across node upgrade, because endpoints
// get killed and we don't have any guarantees that 2 nodes don't overlap
// their upgrades (even on cloud platforms like GCE, because VM level
// rolling upgrades are not Kubernetes aware).
t.verify(f, done, false)
}
}
// Teardown cleans up any remaining resources.
func (t *IngressUpgradeTest) Teardown(f *framework.Framework) {
if CurrentGinkgoTestDescription().Failed {
framework.DescribeIng(t.gceController.Ns)
}
if t.jig.Ingress == nil {
By("No ingress created, no cleanup necessary")
return
}
By("Deleting ingress")
t.jig.DeleteIngress()
By("Cleaning up cloud resources")
framework.CleanupGCEIngressController(t.gceController)
}
func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{}, testDuringDisruption bool) {
if testDuringDisruption {
By("continuously hitting the Ingress IP")
wait.Until(func() {
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false))
}, t.jig.PollInterval, done)
} else {
By("waiting for upgrade to finish without checking if Ingress remains up")
<-done
}
By("hitting the Ingress IP " + t.ip)
framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false))
}