diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 93f6781b97c..97d96fe740d 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -268,6 +268,7 @@ feature-gates federated-api-burst federated-api-qps federated-kube-context +federation-config-from-cluster federation-name federation-system-namespace federation-upgrade-target diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 53da0d3cda8..15085aa83e6 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -92,6 +92,8 @@ type TestContextType struct { FederatedKubeContext string // Federation control plane version to upgrade to while doing upgrade tests FederationUpgradeTarget string + // Whether configuration for accessing federation member clusters should be sourced from the host cluster + FederationConfigFromCluster bool // Viper-only parameters. These will in time replace all flags. @@ -179,6 +181,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver") flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.") + flag.BoolVar(&TestContext.FederationConfigFromCluster, "federation-config-from-cluster", false, "whether to source configuration for member clusters from the hosting cluster.") flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") diff --git a/test/e2e_federation/apiserver.go b/test/e2e_federation/apiserver.go index f6fbcbf1359..1a232234d67 100644 --- a/test/e2e_federation/apiserver.go +++ b/test/e2e_federation/apiserver.go @@ -57,7 +57,7 @@ var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func framework.Logf("Checking that %d clusters are Ready", len(contexts)) for _, context := range contexts { - fedframework.ClusterIsReadyOrFail(f, &context) + fedframework.ClusterIsReadyOrFail(f, context.Name) } framework.Logf("%d clusters are Ready", len(contexts)) diff --git a/test/e2e_federation/framework/BUILD b/test/e2e_federation/framework/BUILD index 2ed09bcf92a..7b8c6012677 100644 --- a/test/e2e_federation/framework/BUILD +++ b/test/e2e_federation/framework/BUILD @@ -21,6 +21,7 @@ go_library( "//federation/client/clientset_generated/federation_clientset:go_default_library", "//federation/pkg/federatedtypes:go_default_library", "//federation/pkg/federatedtypes/crudtester:go_default_library", + "//federation/pkg/federation-controller/util:go_default_library", "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/api/validation:go_default_library", @@ -35,7 +36,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", ], ) diff --git a/test/e2e_federation/framework/cluster.go b/test/e2e_federation/framework/cluster.go index d5dba98dd3b..c93af924bb3 100644 --- a/test/e2e_federation/framework/cluster.go +++ b/test/e2e_federation/framework/cluster.go @@ -20,13 +20,13 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1" - fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" + "k8s.io/kubernetes/federation/pkg/federation-controller/util" "k8s.io/kubernetes/pkg/api/v1" kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" @@ -35,15 +35,7 @@ import ( . "github.com/onsi/gomega" ) -const ( - KubeAPIQPS float32 = 20.0 - KubeAPIBurst = 30 - - userAgentName = "federation-e2e" - - federatedNamespaceTimeout = 5 * time.Minute - federatedClustersWaitTimeout = 1 * time.Minute -) +const federatedClustersWaitTimeout = 1 * time.Minute // ClusterSlice is a slice of clusters type ClusterSlice []*Cluster @@ -54,7 +46,8 @@ type Cluster struct { *kubeclientset.Clientset } -func getRegisteredClusters(f *Framework) ClusterSlice { +// registeredClustersFromConfig configures clientsets for registered clusters from the e2e kubeconfig +func registeredClustersFromConfig(f *Framework) ClusterSlice { contexts := f.GetUnderlyingFederatedContexts() By("Obtaining a list of all the clusters") @@ -62,7 +55,8 @@ func getRegisteredClusters(f *Framework) ClusterSlice { framework.Logf("Checking that %d clusters are Ready", len(contexts)) for _, context := range contexts { - ClusterIsReadyOrFail(f, &context) + ClusterIsReadyOrFail(f, context.Name) + } framework.Logf("%d clusters are Ready", len(contexts)) @@ -70,12 +64,14 @@ func getRegisteredClusters(f *Framework) ClusterSlice { for i, c := range clusterList.Items { framework.Logf("Creating a clientset for the cluster %s", c.Name) Expect(framework.TestContext.KubeConfig).ToNot(Equal(""), "KubeConfig must be specified to load clusters' client config") + config := restConfigFromContext(c, i) clusters = append(clusters, &Cluster{ Name: c.Name, - Clientset: createClientsetForCluster(c, i, userAgentName), + Clientset: clientsetFromConfig(f, config, c.Spec.ServerAddressByClientCIDRs[0].ServerAddress), }) + } - waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name, federatedNamespaceTimeout) + waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name) return clusters } @@ -100,32 +96,34 @@ func waitForAllRegisteredClusters(f *Framework, clusterCount int) *federationapi return clusterList } -func createClientsetForCluster(c federationapi.Cluster, i int, userAgentName string) *kubeclientset.Clientset { +func restConfigFromContext(c federationapi.Cluster, i int) *restclient.Config { kubecfg, err := clientcmd.LoadFromFile(framework.TestContext.KubeConfig) framework.ExpectNoError(err, "error loading KubeConfig: %v", err) - cfgOverride := &clientcmd.ConfigOverrides{ - ClusterInfo: clientcmdapi.Cluster{ - Server: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress, - }, - } - ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, cfgOverride, clientcmd.NewDefaultClientConfigLoadingRules()) + ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, &clientcmd.ConfigOverrides{}, clientcmd.NewDefaultClientConfigLoadingRules()) cfg, err := ccfg.ClientConfig() framework.ExpectNoError(err, "Error creating client config in cluster #%d (%q)", i, c.Name) + return cfg +} - cfg.QPS = KubeAPIQPS - cfg.Burst = KubeAPIBurst - return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, userAgentName)) +func clientsetFromConfig(f *Framework, cfg *restclient.Config, host string) *kubeclientset.Clientset { + cfg.Host = host + cfg.QPS = f.Framework.Options.ClientQPS + cfg.Burst = f.Framework.Options.ClientBurst + return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, "federation-e2e")) } // waitForNamespaceInFederatedClusters waits for the federated namespace to be created in federated clusters -func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, timeout time.Duration) { +func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string) { for _, c := range clusters { name := c.Name - err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { + By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q", nsName, name)) + err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) { _, err := c.Clientset.Core().Namespaces().Get(nsName, metav1.GetOptions{}) - if err != nil { - By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q, err: %v", nsName, name, err)) + if errors.IsNotFound(err) { + return false, nil + } else if err != nil { + framework.Logf("An error occurred waiting for namespace %q to be created in cluster %q: %v", nsName, name, err) return false, nil } By(fmt.Sprintf("Namespace %q exists in cluster %q", nsName, name)) @@ -135,22 +133,11 @@ func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, t } } -// ClusterIsReadyOrFail checks whether the federated cluster of the provided context is ready -func ClusterIsReadyOrFail(f *Framework, context *E2EContext) { - c, err := f.FederationClientset.Federation().Clusters().Get(context.Name, metav1.GetOptions{}) - framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err)) - if c.ObjectMeta.Name != context.Name { - framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context) - } - err = isReady(context.Name, f.FederationClientset) - framework.ExpectNoError(err, fmt.Sprintf("unexpected error in verifying if cluster %s is ready: %+v", context.Name, err)) - framework.Logf("Cluster %s is Ready", context.Name) -} - -// Verify that the cluster is marked ready. -func isReady(clusterName string, clientset *fedclientset.Clientset) error { - return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) { - c, err := clientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{}) +// ClusterIsReadyOrFail checks whether the named cluster is ready +func ClusterIsReadyOrFail(f *Framework, clusterName string) { + By(fmt.Sprintf("Checking readiness of cluster %q", clusterName)) + err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) { + c, err := f.FederationClientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{}) if err != nil { return false, err } @@ -161,4 +148,86 @@ func isReady(clusterName string, clientset *fedclientset.Clientset) error { } return false, nil }) + framework.ExpectNoError(err, fmt.Sprintf("Unexpected error in verifying if cluster %q is ready: %+v", clusterName, err)) + framework.Logf("Cluster %s is Ready", clusterName) +} + +// Cache the cluster config to avoid having to retrieve it for each test +type clusterConfig struct { + name string + host string + config []byte +} + +var cachedClusterConfigs []*clusterConfig + +// registeredClustersFromSecrets configures clientsets for cluster access from secrets in the host cluster +func registeredClustersFromSecrets(f *Framework) ClusterSlice { + if cachedClusterConfigs == nil { + cachedClusterConfigs = clusterConfigFromSecrets(f) + } + + clusters := ClusterSlice{} + for _, clusterConf := range cachedClusterConfigs { + restConfig := restConfigForCluster(clusterConf) + clientset := clientsetFromConfig(f, restConfig, clusterConf.host) + clusters = append(clusters, &Cluster{ + Name: clusterConf.name, + Clientset: clientset, + }) + } + + waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name) + + return clusters +} + +// clusterConfigFromSecrets retrieves cluster configuration from +// secrets in the host cluster +func clusterConfigFromSecrets(f *Framework) []*clusterConfig { + By("Obtaining a list of registered clusters") + clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{}) + framework.ExpectNoError(err, fmt.Sprintf("Error retrieving list of federated clusters: %+v", err)) + if len(clusterList.Items) == 0 { + framework.Failf("No registered clusters found") + } + + clusterConfigs := []*clusterConfig{} + for _, c := range clusterList.Items { + ClusterIsReadyOrFail(f, c.Name) + config := clusterConfigFromSecret(f, c.Name, c.Spec.SecretRef.Name) + clusterConfigs = append(clusterConfigs, &clusterConfig{ + name: c.Name, + host: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress, + config: config, + }) + } + + return clusterConfigs +} + +// clusterConfigFromSecret retrieves configuration for a accessing a +// cluster from a secret in the host cluster +func clusterConfigFromSecret(f *Framework, clusterName string, secretName string) []byte { + By(fmt.Sprintf("Loading configuration for cluster %q", clusterName)) + namespace := framework.FederationSystemNamespace() + secret, err := f.Framework.ClientSet.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + framework.ExpectNoError(err, fmt.Sprintf("Error loading config secret \"%s/%s\" for cluster %q: %+v", namespace, secretName, clusterName, err)) + + config, ok := secret.Data[util.KubeconfigSecretDataKey] + if !ok || len(config) == 0 { + framework.Failf("Secret \"%s/%s\" for cluster %q has no value for key %q", namespace, secretName, clusterName, util.KubeconfigSecretDataKey) + } + + return config +} + +// restConfigForCluster creates a rest client config for the given cluster config +func restConfigForCluster(clusterConf *clusterConfig) *restclient.Config { + cfg, err := clientcmd.Load(clusterConf.config) + framework.ExpectNoError(err, fmt.Sprintf("Error loading configuration for cluster %q: %+v", clusterConf.name, err)) + + restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig() + framework.ExpectNoError(err, fmt.Sprintf("Error creating client for cluster %q: %+v", clusterConf.name, err)) + return restConfig } diff --git a/test/e2e_federation/framework/framework.go b/test/e2e_federation/framework/framework.go index 689662fd34d..bc0123d5328 100644 --- a/test/e2e_federation/framework/framework.go +++ b/test/e2e_federation/framework/framework.go @@ -228,11 +228,15 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { } func (f *Framework) GetRegisteredClusters() ClusterSlice { - return getRegisteredClusters(f) + if framework.TestContext.FederationConfigFromCluster { + return registeredClustersFromSecrets(f) + } else { + return registeredClustersFromConfig(f) + } } func (f *Framework) GetClusterClients() []kubeclientset.Interface { - clusters := getRegisteredClusters(f) + clusters := f.GetRegisteredClusters() var clusterClients []kubeclientset.Interface for _, c := range clusters { clusterClients = append(clusterClients, c.Clientset)