fed: Add option to source e2e cluster config from host cluster

Add the option to configure e2e access to member clusters from the
same secrets in the host cluster used by the federation control plane.
The default behavior will continue to be sourcing this configuration
from the e2e kubeconfig.  The optional behavior can be enabled by
passing --federation-config-from-cluster=true as an argument to
ginkgo.
This commit is contained in:
Maru Newby 2017-04-17 23:17:27 -07:00
parent 7fb9d9950a
commit 9a9d897d94
5 changed files with 91 additions and 1 deletions

View File

@ -268,6 +268,7 @@ feature-gates
federated-api-burst
federated-api-qps
federated-kube-context
federation-config-from-cluster
federation-name
federation-system-namespace
federation-upgrade-target

View File

@ -92,6 +92,8 @@ type TestContextType struct {
FederatedKubeContext string
// Federation control plane version to upgrade to while doing upgrade tests
FederationUpgradeTarget string
// Whether configuration for accessing federation member clusters should be sourced from the host cluster
FederationConfigFromCluster bool
// Viper-only parameters. These will in time replace all flags.
@ -179,6 +181,7 @@ func RegisterClusterFlags() {
flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.")
flag.BoolVar(&TestContext.FederationConfigFromCluster, "federation-config-from-cluster", false, "whether to source configuration for member clusters from the hosting cluster.")
flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")

View File

@ -21,6 +21,7 @@ go_library(
"//federation/client/clientset_generated/federation_clientset:go_default_library",
"//federation/pkg/federatedtypes:go_default_library",
"//federation/pkg/federatedtypes/crudtester:go_default_library",
"//federation/pkg/federation-controller/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/api/validation:go_default_library",

View File

@ -26,6 +26,7 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
"k8s.io/kubernetes/federation/pkg/federation-controller/util"
"k8s.io/kubernetes/pkg/api/v1"
kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework"
@ -150,3 +151,83 @@ func ClusterIsReadyOrFail(f *Framework, clusterName string) {
framework.ExpectNoError(err, fmt.Sprintf("Unexpected error in verifying if cluster %q is ready: %+v", clusterName, err))
framework.Logf("Cluster %s is Ready", clusterName)
}
// Cache the cluster config to avoid having to retrieve it for each test
type clusterConfig struct {
name string
host string
config []byte
}
var cachedClusterConfigs []*clusterConfig
// registeredClustersFromSecrets configures clientsets for cluster access from secrets in the host cluster
func registeredClustersFromSecrets(f *Framework) ClusterSlice {
if cachedClusterConfigs == nil {
cachedClusterConfigs = clusterConfigFromSecrets(f)
}
clusters := ClusterSlice{}
for _, clusterConf := range cachedClusterConfigs {
restConfig := restConfigForCluster(clusterConf)
clientset := clientsetFromConfig(f, restConfig, clusterConf.host)
clusters = append(clusters, &Cluster{
Name: clusterConf.name,
Clientset: clientset,
})
}
waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name)
return clusters
}
// clusterConfigFromSecrets retrieves cluster configuration from
// secrets in the host cluster
func clusterConfigFromSecrets(f *Framework) []*clusterConfig {
By("Obtaining a list of registered clusters")
clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Error retrieving list of federated clusters: %+v", err))
if len(clusterList.Items) == 0 {
framework.Failf("No registered clusters found")
}
clusterConfigs := []*clusterConfig{}
for _, c := range clusterList.Items {
ClusterIsReadyOrFail(f, c.Name)
config := clusterConfigFromSecret(f, c.Name, c.Spec.SecretRef.Name)
clusterConfigs = append(clusterConfigs, &clusterConfig{
name: c.Name,
host: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress,
config: config,
})
}
return clusterConfigs
}
// clusterConfigFromSecret retrieves configuration for a accessing a
// cluster from a secret in the host cluster
func clusterConfigFromSecret(f *Framework, clusterName string, secretName string) []byte {
By(fmt.Sprintf("Loading configuration for cluster %q", clusterName))
namespace := framework.FederationSystemNamespace()
secret, err := f.Framework.ClientSet.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Error loading config secret \"%s/%s\" for cluster %q: %+v", namespace, secretName, clusterName, err))
config, ok := secret.Data[util.KubeconfigSecretDataKey]
if !ok || len(config) == 0 {
framework.Failf("Secret \"%s/%s\" for cluster %q has no value for key %q", namespace, secretName, clusterName, util.KubeconfigSecretDataKey)
}
return config
}
// restConfigForCluster creates a rest client config for the given cluster config
func restConfigForCluster(clusterConf *clusterConfig) *restclient.Config {
cfg, err := clientcmd.Load(clusterConf.config)
framework.ExpectNoError(err, fmt.Sprintf("Error loading configuration for cluster %q: %+v", clusterConf.name, err))
restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
framework.ExpectNoError(err, fmt.Sprintf("Error creating client for cluster %q: %+v", clusterConf.name, err))
return restConfig
}

View File

@ -228,7 +228,11 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
}
func (f *Framework) GetRegisteredClusters() ClusterSlice {
return registeredClustersFromConfig(f)
if framework.TestContext.FederationConfigFromCluster {
return registeredClustersFromSecrets(f)
} else {
return registeredClustersFromConfig(f)
}
}
func (f *Framework) GetClusterClients() []kubeclientset.Interface {