diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index 246b5ef945b..bb52e3702bb 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -150,6 +150,8 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}" --node-instance-group="${NODE_INSTANCE_GROUP:-}" \ --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \ --network="${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-e2e}}" \ + --node-tag="${NODE_TAG:-}" \ + --master-tag="${MASTER_TAG:-}" \ --federated-kube-context="${FEDERATION_KUBE_CONTEXT:-e2e-federation}" \ ${KUBE_CONTAINER_RUNTIME:+"--container-runtime=${KUBE_CONTAINER_RUNTIME}"} \ ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \ diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index b504324bb95..991aa53aa75 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -452,6 +452,7 @@ manifest-url-header masquerade-all master-os-distro master-service-namespace +master-tag max-concurrency max-connection-bytes-per-sec maximum-dead-containers @@ -512,6 +513,7 @@ node-schedulable-timeout node-startup-grace-period node-status-update-frequency node-sync-period +node-tag no-headers no-headers non-masquerade-cidr diff --git a/test/e2e/firewall.go b/test/e2e/firewall.go index e0066e07edf..8586cccdc61 100644 --- a/test/e2e/firewall.go +++ b/test/e2e/firewall.go @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/cloudprovider" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/test/e2e/framework" @@ -59,6 +60,8 @@ var _ = framework.KubeDescribe("Firewall rule", func() { framework.Logf("Got cluster ID: %v", clusterID) jig := framework.NewServiceTestJig(cs, serviceName) + nodeList := jig.GetNodes(framework.MaxNodesForEndpointsTests) + Expect(nodeList).NotTo(BeNil()) nodesNames := jig.GetNodesNames(framework.MaxNodesForEndpointsTests) if len(nodesNames) <= 0 { framework.Failf("Expect at least 1 node, got: %v", nodesNames) @@ -84,14 +87,13 @@ var _ = framework.KubeDescribe("Firewall rule", func() { svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP By("Checking if service's firewall rule is correct") - nodeTags := framework.GetInstanceTags(cloudConfig, nodesNames[0]) - lbFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items) + lbFw := framework.ConstructFirewallForLBService(svc, cloudConfig.NodeTag) fw, err := gceCloud.GetFirewall(lbFw.Name) Expect(err).NotTo(HaveOccurred()) Expect(framework.VerifyFirewallRule(fw, lbFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) By("Checking if service's nodes health check firewall rule is correct") - nodesHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, true) + nodesHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, cloudConfig.NodeTag, true) fw, err = gceCloud.GetFirewall(nodesHCFw.Name) Expect(err).NotTo(HaveOccurred()) Expect(framework.VerifyFirewallRule(fw, nodesHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) @@ -107,7 +109,7 @@ var _ = framework.KubeDescribe("Firewall rule", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the correct local traffic health check firewall rule to be created") - localHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, false) + localHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, cloudConfig.NodeTag, false) fw, err = framework.WaitForFirewallRule(gceCloud, localHCFw.Name, true, framework.LoadBalancerCreateTimeoutDefault) Expect(err).NotTo(HaveOccurred()) Expect(framework.VerifyFirewallRule(fw, localHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) @@ -132,11 +134,17 @@ var _ = framework.KubeDescribe("Firewall rule", func() { // that's much harder to do in the current e2e framework. By(fmt.Sprintf("Removing tags from one of the nodes: %v", nodesNames[0])) nodesSet.Delete(nodesNames[0]) - removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], []string{}) + // Instance could run in a different zone in multi-zone test. Figure out which zone + // it is in before proceeding. + zone := cloudConfig.Zone + if zoneInLabel, ok := nodeList.Items[0].Labels[kubeletapis.LabelZoneFailureDomain]; ok { + zone = zoneInLabel + } + removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], zone, []string{}) defer func() { By("Adding tags back to the node and wait till the traffic is recovered") nodesSet.Insert(nodesNames[0]) - framework.SetInstanceTags(cloudConfig, nodesNames[0], removedTags) + framework.SetInstanceTags(cloudConfig, nodesNames[0], zone, removedTags) // Make sure traffic is recovered before exit Expect(framework.TestHitNodesFromOutside(svcExternalIP, framework.FirewallTestHttpPort, framework.FirewallTimeoutDefault, nodesSet)).NotTo(HaveOccurred()) }() @@ -146,19 +154,13 @@ var _ = framework.KubeDescribe("Firewall rule", func() { }) It("should have correct firewall rules for e2e cluster", func() { - By("Gathering firewall related information") - masterTags := framework.GetInstanceTags(cloudConfig, cloudConfig.MasterName) - Expect(len(masterTags.Items)).Should(Equal(1)) - nodes := framework.GetReadySchedulableNodesOrDie(cs) if len(nodes.Items) <= 0 { framework.Failf("Expect at least 1 node, got: %v", len(nodes.Items)) } - nodeTags := framework.GetInstanceTags(cloudConfig, nodes.Items[0].Name) - Expect(len(nodeTags.Items)).Should(Equal(1)) By("Checking if e2e firewall rules are correct") - for _, expFw := range framework.GetE2eFirewalls(cloudConfig.MasterName, masterTags.Items[0], nodeTags.Items[0], cloudConfig.Network) { + for _, expFw := range framework.GetE2eFirewalls(cloudConfig.MasterName, cloudConfig.MasterTag, cloudConfig.NodeTag, cloudConfig.Network) { fw, err := gceCloud.GetFirewall(expFw.Name) Expect(err).NotTo(HaveOccurred()) Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) diff --git a/test/e2e/framework/firewall_util.go b/test/e2e/framework/firewall_util.go index f7f65ce6178..9de9dd89f5f 100644 --- a/test/e2e/framework/firewall_util.go +++ b/test/e2e/framework/firewall_util.go @@ -51,13 +51,13 @@ func MakeFirewallNameForLBService(name string) string { } // ConstructFirewallForLBService returns the expected GCE firewall rule for a loadbalancer type service -func ConstructFirewallForLBService(svc *v1.Service, nodesTags []string) *compute.Firewall { +func ConstructFirewallForLBService(svc *v1.Service, nodeTag string) *compute.Firewall { if svc.Spec.Type != v1.ServiceTypeLoadBalancer { Failf("can not construct firewall rule for non-loadbalancer type service") } fw := compute.Firewall{} fw.Name = MakeFirewallNameForLBService(cloudprovider.GetLoadBalancerName(svc)) - fw.TargetTags = nodesTags + fw.TargetTags = []string{nodeTag} if svc.Spec.LoadBalancerSourceRanges == nil { fw.SourceRanges = []string{"0.0.0.0/0"} } else { @@ -77,13 +77,13 @@ func MakeHealthCheckFirewallNameForLBService(clusterID, name string, isNodesHeal } // ConstructHealthCheckFirewallForLBService returns the expected GCE firewall rule for a loadbalancer type service -func ConstructHealthCheckFirewallForLBService(clusterID string, svc *v1.Service, nodesTags []string, isNodesHealthCheck bool) *compute.Firewall { +func ConstructHealthCheckFirewallForLBService(clusterID string, svc *v1.Service, nodeTag string, isNodesHealthCheck bool) *compute.Firewall { if svc.Spec.Type != v1.ServiceTypeLoadBalancer { Failf("can not construct firewall rule for non-loadbalancer type service") } fw := compute.Firewall{} fw.Name = MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), isNodesHealthCheck) - fw.TargetTags = nodesTags + fw.TargetTags = []string{nodeTag} fw.SourceRanges = gcecloud.LoadBalancerSrcRanges() healthCheckPort := gcecloud.GetNodesHealthCheckPort() if !isNodesHealthCheck { @@ -98,14 +98,6 @@ func ConstructHealthCheckFirewallForLBService(clusterID string, svc *v1.Service, return &fw } -// GetNodeTags gets tags from one of the Kubernetes nodes -func GetNodeTags(c clientset.Interface, cloudConfig CloudConfig) *compute.Tags { - nodes := GetReadySchedulableNodesOrDie(c) - Expect(len(nodes.Items) > 0).Should(BeTrue()) - nodeTags := GetInstanceTags(cloudConfig, nodes.Items[0].Name) - return nodeTags -} - // GetInstanceTags gets tags from GCE instance with given name. func GetInstanceTags(cloudConfig CloudConfig, instanceName string) *compute.Tags { gceCloud := cloudConfig.Provider.(*gcecloud.GCECloud) @@ -118,12 +110,12 @@ func GetInstanceTags(cloudConfig CloudConfig, instanceName string) *compute.Tags } // SetInstanceTags sets tags on GCE instance with given name. -func SetInstanceTags(cloudConfig CloudConfig, instanceName string, tags []string) []string { +func SetInstanceTags(cloudConfig CloudConfig, instanceName, zone string, tags []string) []string { gceCloud := cloudConfig.Provider.(*gcecloud.GCECloud) // Re-get instance everytime because we need the latest fingerprint for updating metadata resTags := GetInstanceTags(cloudConfig, instanceName) _, err := gceCloud.GetComputeService().Instances.SetTags( - cloudConfig.ProjectID, cloudConfig.Zone, instanceName, + cloudConfig.ProjectID, zone, instanceName, &compute.Tags{Fingerprint: resTags.Fingerprint, Items: tags}).Do() if err != nil { Failf("failed to set instance tags: %v", err) diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index b12b02cce2c..640ab91b522 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -974,14 +974,13 @@ func (j *IngressTestJig) GetIngressNodePorts() []string { } // ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource -func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController) *compute.Firewall { - nodeTags := GetNodeTags(j.Client, gceController.Cloud) +func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController, nodeTag string) *compute.Firewall { nodePorts := j.GetIngressNodePorts() fw := compute.Firewall{} fw.Name = gceController.GetFirewallRuleName() fw.SourceRanges = gcecloud.LoadBalancerSrcRanges() - fw.TargetTags = nodeTags.Items + fw.TargetTags = []string{nodeTag} fw.Allowed = []*compute.FirewallAllowed{ { IPProtocol: "tcp", diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 07ed74b2822..5945867e395 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -143,6 +143,8 @@ type CloudConfig struct { ClusterTag string Network string ConfigFile string // for azure and openstack + NodeTag string + MasterTag string Provider cloudprovider.Interface } @@ -210,6 +212,8 @@ func RegisterClusterFlags() { flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws. If there is more than one group: comma separated list of groups.") flag.StringVar(&cloudConfig.Network, "network", "e2e", "The cloud provider network for this e2e cluster.") flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster") + flag.StringVar(&cloudConfig.NodeTag, "node-tag", "", "Network tags used on node instances. Valid only for gce, gke") + flag.StringVar(&cloudConfig.MasterTag, "master-tag", "", "Network tags used on master instances. Valid only for gce, gke") flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources. Only required if provider is aws.") flag.StringVar(&cloudConfig.ConfigFile, "cloud-config-file", "", "Cloud config file. Only required if provider is azure.") diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index d745f93e28d..b4d3e34943d 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -36,6 +36,7 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { ns string jig *framework.IngressTestJig conformanceTests []framework.IngressConformanceTests + cloudConfig framework.CloudConfig ) f := framework.NewDefaultFramework("ingress") @@ -43,6 +44,7 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { f.BeforeEach() jig = framework.NewIngressTestJig(f.ClientSet) ns = f.Namespace.Name + cloudConfig = framework.TestContext.CloudConfig // this test wants powerful permissions. Since the namespace names are unique, we can leave this // lying around so we don't have to race any caches @@ -122,7 +124,7 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { By("should have correct firewall rule for ingress") fw := gceController.GetFirewallRule() - expFw := jig.ConstructFirewallForIngress(gceController) + expFw := jig.ConstructFirewallForIngress(gceController, cloudConfig.NodeTag) // Passed the last argument as `true` to verify the backend ports is a subset // of the allowed ports in firewall rule, given there may be other existing // ingress resources and backends we are not aware of.