From 7fb9d9950ad3c29d0e8eb565980ba1121f6195e9 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Mon, 17 Apr 2017 22:47:02 -0700 Subject: [PATCH 1/2] fed: Refactor e2e retrieval of cluster config --- test/e2e_federation/apiserver.go | 2 +- test/e2e_federation/framework/BUILD | 1 - test/e2e_federation/framework/cluster.go | 78 +++++++++------------- test/e2e_federation/framework/framework.go | 4 +- 4 files changed, 36 insertions(+), 49 deletions(-) diff --git a/test/e2e_federation/apiserver.go b/test/e2e_federation/apiserver.go index f6fbcbf1359..1a232234d67 100644 --- a/test/e2e_federation/apiserver.go +++ b/test/e2e_federation/apiserver.go @@ -57,7 +57,7 @@ var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func framework.Logf("Checking that %d clusters are Ready", len(contexts)) for _, context := range contexts { - fedframework.ClusterIsReadyOrFail(f, &context) + fedframework.ClusterIsReadyOrFail(f, context.Name) } framework.Logf("%d clusters are Ready", len(contexts)) diff --git a/test/e2e_federation/framework/BUILD b/test/e2e_federation/framework/BUILD index 2ed09bcf92a..ad3061d75de 100644 --- a/test/e2e_federation/framework/BUILD +++ b/test/e2e_federation/framework/BUILD @@ -35,7 +35,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", ], ) diff --git a/test/e2e_federation/framework/cluster.go b/test/e2e_federation/framework/cluster.go index d5dba98dd3b..0099111f3a9 100644 --- a/test/e2e_federation/framework/cluster.go +++ b/test/e2e_federation/framework/cluster.go @@ -20,13 +20,12 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" - fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" "k8s.io/kubernetes/pkg/api/v1" kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" @@ -35,15 +34,7 @@ import ( . "github.com/onsi/gomega" ) -const ( - KubeAPIQPS float32 = 20.0 - KubeAPIBurst = 30 - - userAgentName = "federation-e2e" - - federatedNamespaceTimeout = 5 * time.Minute - federatedClustersWaitTimeout = 1 * time.Minute -) +const federatedClustersWaitTimeout = 1 * time.Minute // ClusterSlice is a slice of clusters type ClusterSlice []*Cluster @@ -54,7 +45,8 @@ type Cluster struct { *kubeclientset.Clientset } -func getRegisteredClusters(f *Framework) ClusterSlice { +// registeredClustersFromConfig configures clientsets for registered clusters from the e2e kubeconfig +func registeredClustersFromConfig(f *Framework) ClusterSlice { contexts := f.GetUnderlyingFederatedContexts() By("Obtaining a list of all the clusters") @@ -62,7 +54,8 @@ func getRegisteredClusters(f *Framework) ClusterSlice { framework.Logf("Checking that %d clusters are Ready", len(contexts)) for _, context := range contexts { - ClusterIsReadyOrFail(f, &context) + ClusterIsReadyOrFail(f, context.Name) + } framework.Logf("%d clusters are Ready", len(contexts)) @@ -70,12 +63,14 @@ func getRegisteredClusters(f *Framework) ClusterSlice { for i, c := range clusterList.Items { framework.Logf("Creating a clientset for the cluster %s", c.Name) Expect(framework.TestContext.KubeConfig).ToNot(Equal(""), "KubeConfig must be specified to load clusters' client config") + config := restConfigFromContext(c, i) clusters = append(clusters, &Cluster{ Name: c.Name, - Clientset: createClientsetForCluster(c, i, userAgentName), + Clientset: clientsetFromConfig(f, config, c.Spec.ServerAddressByClientCIDRs[0].ServerAddress), }) + } - waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name, federatedNamespaceTimeout) + waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name) return clusters } @@ -100,32 +95,34 @@ func waitForAllRegisteredClusters(f *Framework, clusterCount int) *federationapi return clusterList } -func createClientsetForCluster(c federationapi.Cluster, i int, userAgentName string) *kubeclientset.Clientset { +func restConfigFromContext(c federationapi.Cluster, i int) *restclient.Config { kubecfg, err := clientcmd.LoadFromFile(framework.TestContext.KubeConfig) framework.ExpectNoError(err, "error loading KubeConfig: %v", err) - cfgOverride := &clientcmd.ConfigOverrides{ - ClusterInfo: clientcmdapi.Cluster{ - Server: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress, - }, - } - ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, cfgOverride, clientcmd.NewDefaultClientConfigLoadingRules()) + ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, &clientcmd.ConfigOverrides{}, clientcmd.NewDefaultClientConfigLoadingRules()) cfg, err := ccfg.ClientConfig() framework.ExpectNoError(err, "Error creating client config in cluster #%d (%q)", i, c.Name) + return cfg +} - cfg.QPS = KubeAPIQPS - cfg.Burst = KubeAPIBurst - return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, userAgentName)) +func clientsetFromConfig(f *Framework, cfg *restclient.Config, host string) *kubeclientset.Clientset { + cfg.Host = host + cfg.QPS = f.Framework.Options.ClientQPS + cfg.Burst = f.Framework.Options.ClientBurst + return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, "federation-e2e")) } // waitForNamespaceInFederatedClusters waits for the federated namespace to be created in federated clusters -func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, timeout time.Duration) { +func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string) { for _, c := range clusters { name := c.Name - err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { + By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q", nsName, name)) + err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) { _, err := c.Clientset.Core().Namespaces().Get(nsName, metav1.GetOptions{}) - if err != nil { - By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q, err: %v", nsName, name, err)) + if errors.IsNotFound(err) { + return false, nil + } else if err != nil { + framework.Logf("An error occurred waiting for namespace %q to be created in cluster %q: %v", nsName, name, err) return false, nil } By(fmt.Sprintf("Namespace %q exists in cluster %q", nsName, name)) @@ -135,22 +132,11 @@ func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, t } } -// ClusterIsReadyOrFail checks whether the federated cluster of the provided context is ready -func ClusterIsReadyOrFail(f *Framework, context *E2EContext) { - c, err := f.FederationClientset.Federation().Clusters().Get(context.Name, metav1.GetOptions{}) - framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err)) - if c.ObjectMeta.Name != context.Name { - framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context) - } - err = isReady(context.Name, f.FederationClientset) - framework.ExpectNoError(err, fmt.Sprintf("unexpected error in verifying if cluster %s is ready: %+v", context.Name, err)) - framework.Logf("Cluster %s is Ready", context.Name) -} - -// Verify that the cluster is marked ready. -func isReady(clusterName string, clientset *fedclientset.Clientset) error { - return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) { - c, err := clientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{}) +// ClusterIsReadyOrFail checks whether the named cluster is ready +func ClusterIsReadyOrFail(f *Framework, clusterName string) { + By(fmt.Sprintf("Checking readiness of cluster %q", clusterName)) + err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) { + c, err := f.FederationClientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{}) if err != nil { return false, err } @@ -161,4 +147,6 @@ func isReady(clusterName string, clientset *fedclientset.Clientset) error { } return false, nil }) + framework.ExpectNoError(err, fmt.Sprintf("Unexpected error in verifying if cluster %q is ready: %+v", clusterName, err)) + framework.Logf("Cluster %s is Ready", clusterName) } diff --git a/test/e2e_federation/framework/framework.go b/test/e2e_federation/framework/framework.go index 689662fd34d..33dda0c1e47 100644 --- a/test/e2e_federation/framework/framework.go +++ b/test/e2e_federation/framework/framework.go @@ -228,11 +228,11 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { } func (f *Framework) GetRegisteredClusters() ClusterSlice { - return getRegisteredClusters(f) + return registeredClustersFromConfig(f) } func (f *Framework) GetClusterClients() []kubeclientset.Interface { - clusters := getRegisteredClusters(f) + clusters := f.GetRegisteredClusters() var clusterClients []kubeclientset.Interface for _, c := range clusters { clusterClients = append(clusterClients, c.Clientset) From 9a9d897d9409ae2bae7685e4a88dd2297d65434a Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Mon, 17 Apr 2017 23:17:27 -0700 Subject: [PATCH 2/2] fed: Add option to source e2e cluster config from host cluster Add the option to configure e2e access to member clusters from the same secrets in the host cluster used by the federation control plane. The default behavior will continue to be sourcing this configuration from the e2e kubeconfig. The optional behavior can be enabled by passing --federation-config-from-cluster=true as an argument to ginkgo. --- hack/verify-flags/known-flags.txt | 1 + test/e2e/framework/test_context.go | 3 + test/e2e_federation/framework/BUILD | 1 + test/e2e_federation/framework/cluster.go | 81 ++++++++++++++++++++++ test/e2e_federation/framework/framework.go | 6 +- 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 93f6781b97c..97d96fe740d 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -268,6 +268,7 @@ feature-gates federated-api-burst federated-api-qps federated-kube-context +federation-config-from-cluster federation-name federation-system-namespace federation-upgrade-target diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 53da0d3cda8..15085aa83e6 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -92,6 +92,8 @@ type TestContextType struct { FederatedKubeContext string // Federation control plane version to upgrade to while doing upgrade tests FederationUpgradeTarget string + // Whether configuration for accessing federation member clusters should be sourced from the host cluster + FederationConfigFromCluster bool // Viper-only parameters. These will in time replace all flags. @@ -179,6 +181,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver") flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.") + flag.BoolVar(&TestContext.FederationConfigFromCluster, "federation-config-from-cluster", false, "whether to source configuration for member clusters from the hosting cluster.") flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") diff --git a/test/e2e_federation/framework/BUILD b/test/e2e_federation/framework/BUILD index ad3061d75de..7b8c6012677 100644 --- a/test/e2e_federation/framework/BUILD +++ b/test/e2e_federation/framework/BUILD @@ -21,6 +21,7 @@ go_library( "//federation/client/clientset_generated/federation_clientset:go_default_library", "//federation/pkg/federatedtypes:go_default_library", "//federation/pkg/federatedtypes/crudtester:go_default_library", + "//federation/pkg/federation-controller/util:go_default_library", "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/api/validation:go_default_library", diff --git a/test/e2e_federation/framework/cluster.go b/test/e2e_federation/framework/cluster.go index 0099111f3a9..c93af924bb3 100644 --- a/test/e2e_federation/framework/cluster.go +++ b/test/e2e_federation/framework/cluster.go @@ -26,6 +26,7 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" + "k8s.io/kubernetes/federation/pkg/federation-controller/util" "k8s.io/kubernetes/pkg/api/v1" kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" @@ -150,3 +151,83 @@ func ClusterIsReadyOrFail(f *Framework, clusterName string) { framework.ExpectNoError(err, fmt.Sprintf("Unexpected error in verifying if cluster %q is ready: %+v", clusterName, err)) framework.Logf("Cluster %s is Ready", clusterName) } + +// Cache the cluster config to avoid having to retrieve it for each test +type clusterConfig struct { + name string + host string + config []byte +} + +var cachedClusterConfigs []*clusterConfig + +// registeredClustersFromSecrets configures clientsets for cluster access from secrets in the host cluster +func registeredClustersFromSecrets(f *Framework) ClusterSlice { + if cachedClusterConfigs == nil { + cachedClusterConfigs = clusterConfigFromSecrets(f) + } + + clusters := ClusterSlice{} + for _, clusterConf := range cachedClusterConfigs { + restConfig := restConfigForCluster(clusterConf) + clientset := clientsetFromConfig(f, restConfig, clusterConf.host) + clusters = append(clusters, &Cluster{ + Name: clusterConf.name, + Clientset: clientset, + }) + } + + waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name) + + return clusters +} + +// clusterConfigFromSecrets retrieves cluster configuration from +// secrets in the host cluster +func clusterConfigFromSecrets(f *Framework) []*clusterConfig { + By("Obtaining a list of registered clusters") + clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{}) + framework.ExpectNoError(err, fmt.Sprintf("Error retrieving list of federated clusters: %+v", err)) + if len(clusterList.Items) == 0 { + framework.Failf("No registered clusters found") + } + + clusterConfigs := []*clusterConfig{} + for _, c := range clusterList.Items { + ClusterIsReadyOrFail(f, c.Name) + config := clusterConfigFromSecret(f, c.Name, c.Spec.SecretRef.Name) + clusterConfigs = append(clusterConfigs, &clusterConfig{ + name: c.Name, + host: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress, + config: config, + }) + } + + return clusterConfigs +} + +// clusterConfigFromSecret retrieves configuration for a accessing a +// cluster from a secret in the host cluster +func clusterConfigFromSecret(f *Framework, clusterName string, secretName string) []byte { + By(fmt.Sprintf("Loading configuration for cluster %q", clusterName)) + namespace := framework.FederationSystemNamespace() + secret, err := f.Framework.ClientSet.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + framework.ExpectNoError(err, fmt.Sprintf("Error loading config secret \"%s/%s\" for cluster %q: %+v", namespace, secretName, clusterName, err)) + + config, ok := secret.Data[util.KubeconfigSecretDataKey] + if !ok || len(config) == 0 { + framework.Failf("Secret \"%s/%s\" for cluster %q has no value for key %q", namespace, secretName, clusterName, util.KubeconfigSecretDataKey) + } + + return config +} + +// restConfigForCluster creates a rest client config for the given cluster config +func restConfigForCluster(clusterConf *clusterConfig) *restclient.Config { + cfg, err := clientcmd.Load(clusterConf.config) + framework.ExpectNoError(err, fmt.Sprintf("Error loading configuration for cluster %q: %+v", clusterConf.name, err)) + + restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig() + framework.ExpectNoError(err, fmt.Sprintf("Error creating client for cluster %q: %+v", clusterConf.name, err)) + return restConfig +} diff --git a/test/e2e_federation/framework/framework.go b/test/e2e_federation/framework/framework.go index 33dda0c1e47..bc0123d5328 100644 --- a/test/e2e_federation/framework/framework.go +++ b/test/e2e_federation/framework/framework.go @@ -228,7 +228,11 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { } func (f *Framework) GetRegisteredClusters() ClusterSlice { - return registeredClustersFromConfig(f) + if framework.TestContext.FederationConfigFromCluster { + return registeredClustersFromSecrets(f) + } else { + return registeredClustersFromConfig(f) + } } func (f *Framework) GetClusterClients() []kubeclientset.Interface {