Merge pull request #25125 from colhom/federation-e2e

e2e tests for federated-apiserver

test/e2e/cleanup/cleanup.go

@@ -17,11 +17,12 @@ limitations under the License.
 package main
 
 import (
-	flag "github.com/spf13/pflag"
 	"log"
 	"os"
 	"strings"
 
+	flag "github.com/spf13/pflag"
+
 	"k8s.io/kubernetes/test/e2e"
 )
 

test/e2e/deployment.go

@@ -20,7 +20,10 @@ import (
 	"fmt"
 	"time"
 
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/annotations"
 	"k8s.io/kubernetes/pkg/api/errors"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/apis/extensions"
@@ -35,10 +38,6 @@ import (
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/watch"
 	"k8s.io/kubernetes/test/e2e/framework"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"k8s.io/kubernetes/pkg/api/annotations"
 )
 
 const (

test/e2e/federation-apiserver.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	federationapi "k8s.io/kubernetes/federation/apis/federation"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// Create/delete cluster api objects
+var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func() {
+	f := framework.NewDefaultFederatedFramework("federated-cluster")
+	It("should allow creation of cluster api objects", func() {
+		framework.SkipUnlessFederated()
+
+		contexts := f.GetUnderlyingFederatedContexts()
+
+		for _, context := range contexts {
+			framework.Logf("Creating cluster object: %s (%s)", context.Name, context.Cluster.Cluster.Server)
+			cluster := federationapi.Cluster{
+				ObjectMeta: api.ObjectMeta{
+					Name: context.Name,
+				},
+				Spec: federationapi.ClusterSpec{
+					ServerAddressByClientCIDRs: []federationapi.ServerAddressByClientCIDR{
+						{
+							ClientCIDR:    "0.0.0.0/0",
+							ServerAddress: context.Cluster.Cluster.Server,
+						},
+					},
+					//TODO(colhom): add SecretRef when #26132 lands
+				},
+			}
+			_, err := f.FederationClient.Clusters().Create(&cluster)
+			framework.ExpectNoError(err, fmt.Sprintf("creating cluster: %+v", err))
+		}
+
+		for _, context := range contexts {
+			c, err := f.FederationClient.Clusters().Get(context.Name)
+			framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err))
+			if c.ObjectMeta.Name != context.Name {
+				framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context)
+			}
+		}
+	})
+})

test/e2e/framework/framework.go

@@ -19,11 +19,13 @@ package framework
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"reflect"
 	"strings"
 	"sync"
 	"time"
 
+	unversionedfederation "k8s.io/kubernetes/federation/client/clientset_generated/federation_internalclientset/typed/federation/unversioned"
 	"k8s.io/kubernetes/pkg/api"
 	apierrs "k8s.io/kubernetes/pkg/api/errors"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_2"
@@ -39,6 +41,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	yaml "gopkg.in/yaml.v2"
 )
 
 const (
@@ -54,6 +57,8 @@ type Framework struct {
 	Clientset_1_2 *release_1_2.Clientset
 	Clientset_1_3 *release_1_3.Clientset
 
+	FederationClient *unversionedfederation.FederationClient
+
 	Namespace                *api.Namespace   // Every test has at least one namespace
 	namespacesToDelete       []*api.Namespace // Some tests have more than one.
 	NamespaceDeletionTimeout time.Duration
@@ -75,6 +80,9 @@ type Framework struct {
 
 	// configuration for framework's client
 	options FrameworkOptions
+
+	// will this framework exercise a federated cluster as well
+	federated bool
 }
 
 type TestDataSummary interface {
@@ -97,6 +105,12 @@ func NewDefaultFramework(baseName string) *Framework {
 	return NewFramework(baseName, options, nil)
 }
 
+func NewDefaultFederatedFramework(baseName string) *Framework {
+	f := NewDefaultFramework(baseName)
+	f.federated = true
+	return f
+}
+
 func NewFramework(baseName string, options FrameworkOptions, client *client.Client) *Framework {
 	f := &Framework{
 		BaseName:                 baseName,
@@ -130,9 +144,17 @@ func (f *Framework) BeforeEach() {
 		Expect(err).NotTo(HaveOccurred())
 		f.Client = c
 	}
+
 	f.Clientset_1_2 = adapter_1_2.FromUnversionedClient(f.Client)
 	f.Clientset_1_3 = adapter_1_3.FromUnversionedClient(f.Client)
 
+	if f.federated && f.FederationClient == nil {
+		By("Creating a federated kubernetes client")
+		var err error
+		f.FederationClient, err = LoadFederationClient()
+		Expect(err).NotTo(HaveOccurred())
+	}
+
 	By("Building a namespace api object")
 	namespace, err := f.CreateNamespace(f.BaseName, map[string]string{
 		"e2e-framework": f.BaseName,
@@ -206,6 +228,18 @@ func (f *Framework) AfterEach() {
 		f.Client = nil
 	}()
 
+	if f.federated {
+		defer func() {
+			if f.FederationClient == nil {
+				Logf("Warning: framework is marked federated, but has no FederationClient")
+				return
+			}
+			if err := f.FederationClient.Clusters().DeleteCollection(nil, api.ListOptions{}); err != nil {
+				Logf("Error: failed to delete Clusters: %+v", err)
+			}
+		}()
+	}
+
 	// Print events if the test failed.
 	if CurrentGinkgoTestDescription().Failed && TestContext.DumpLogsOnFailure {
 		DumpAllNamespaceInfo(f.Client, f.Namespace.Name)
@@ -465,6 +499,97 @@ func (f *Framework) CreatePodsPerNodeForSimpleApp(appName string, podSpec func(n
 	return labels
 }
 
+type KubeUser struct {
+	Name string `yaml:"name"`
+	User struct {
+		Username string `yaml:"username"`
+		Password string `yaml:"password"`
+		Token    string `yaml:"token"`
+	} `yaml:"user"`
+}
+
+type KubeCluster struct {
+	Name    string `yaml:"name"`
+	Cluster struct {
+		CertificateAuthorityData string `yaml:"certificate-authority-data"`
+		Server                   string `yaml:"server"`
+	} `yaml:"cluster"`
+}
+
+type KubeConfig struct {
+	Contexts []struct {
+		Name    string `yaml:"name"`
+		Context struct {
+			Cluster string `yaml:"cluster"`
+			User    string
+		} `yaml:"context"`
+	} `yaml:"contexts"`
+
+	Clusters []KubeCluster `yaml:"clusters"`
+
+	Users []KubeUser `yaml:"users"`
+}
+
+func (kc *KubeConfig) findUser(name string) *KubeUser {
+	for _, user := range kc.Users {
+		if user.Name == name {
+			return &user
+		}
+	}
+	return nil
+}
+
+func (kc *KubeConfig) findCluster(name string) *KubeCluster {
+	for _, cluster := range kc.Clusters {
+		if cluster.Name == name {
+			return &cluster
+		}
+	}
+	return nil
+}
+
+type E2EContext struct {
+	Name    string       `yaml:"name"`
+	Cluster *KubeCluster `yaml:"cluster"`
+	User    *KubeUser    `yaml:"user"`
+}
+
+func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
+	if !f.federated {
+		Failf("geUnderlyingFederatedContexts called on non-federated framework")
+	}
+
+	kubeconfig := KubeConfig{}
+	configBytes, err := ioutil.ReadFile(TestContext.KubeConfig)
+	ExpectNoError(err)
+	err = yaml.Unmarshal(configBytes, &kubeconfig)
+	ExpectNoError(err)
+
+	e2eContexts := []E2EContext{}
+	for _, context := range kubeconfig.Contexts {
+		if strings.HasPrefix(context.Name, "federation-e2e") {
+
+			user := kubeconfig.findUser(context.Context.User)
+			if user == nil {
+				Failf("Could not find user for context %+v", context)
+			}
+
+			cluster := kubeconfig.findCluster(context.Context.Cluster)
+			if cluster == nil {
+				Failf("Could not find cluster for context %+v", context)
+			}
+
+			e2eContexts = append(e2eContexts, E2EContext{
+				Name:    context.Name,
+				Cluster: cluster,
+				User:    user,
+			})
+		}
+	}
+
+	return e2eContexts
+}
+
 func kubectlExecWithRetry(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
 	for numRetries := 0; numRetries < maxKubectlExecRetries; numRetries++ {
 		if numRetries > 0 {

test/e2e/framework/test_context.go

@@ -77,10 +77,7 @@ type CloudConfig struct {
 }
 
 var TestContext TestContextType
-
-func SetTestContext(t TestContextType) {
-	TestContext = t
-}
+var federatedKubeContext string
 
 func RegisterFlags() {
 	// Turn on verbose by default to get spec names
@@ -95,6 +92,8 @@ func RegisterFlags() {
 	flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
 	flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
 	flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "", "ContentType used to communicate with apiserver")
+	flag.StringVar(&federatedKubeContext, "federated-kube-context", "federated-cluster", "kubeconfig context for federated-cluster.")
+
 	flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
 	flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")
 	flag.StringVar(&TestContext.Host, "host", "", "The host, or apiserver, to connect to")

test/e2e/framework/util.go

@@ -36,6 +36,7 @@ import (
 	"sync"
 	"time"
 
+	unversionedfederation "k8s.io/kubernetes/federation/client/clientset_generated/federation_internalclientset/typed/federation/unversioned"
 	"k8s.io/kubernetes/pkg/api"
 	apierrs "k8s.io/kubernetes/pkg/api/errors"
 	"k8s.io/kubernetes/pkg/api/resource"
@@ -325,6 +326,28 @@ func SkipUnlessServerVersionGTE(v semver.Version, c discovery.ServerVersionInter
 	}
 }
 
+// Detects whether the federation namespace exists in the underlying cluster
+func SkipUnlessFederated() {
+	c, err := LoadClient()
+	if err != nil {
+		Failf("Unable to load client: %v", err)
+	}
+
+	federationNS := os.Getenv("FEDERATION_NAMESPACE")
+	if federationNS == "" {
+		federationNS = "federation-e2e"
+	}
+
+	_, err = c.Namespaces().Get(federationNS)
+	if err != nil {
+		if apierrs.IsNotFound(err) {
+			Skipf("Could not find federation namespace %s: skipping federated test", federationNS)
+		} else {
+			Failf("Unexpected error getting namespace: %v", err)
+		}
+	}
+}
+
 // ProvidersWithSSH are those providers where each node is accessible with SSH
 var ProvidersWithSSH = []string{"gce", "gke", "aws"}
 
@@ -1529,22 +1552,42 @@ func ServiceResponding(c *client.Client, ns, name string) error {
 	})
 }
 
-func LoadConfig() (*restclient.Config, error) {
-	switch {
-	case TestContext.KubeConfig != "":
-		Logf(">>> TestContext.KubeConfig: %s\n", TestContext.KubeConfig)
-		c, err := clientcmd.LoadFromFile(TestContext.KubeConfig)
-		if err != nil {
-			return nil, fmt.Errorf("error loading KubeConfig: %v", err.Error())
-		}
-		if TestContext.KubeContext != "" {
-			Logf(">>> TestContext.KubeContext: %s\n", TestContext.KubeContext)
-			c.CurrentContext = TestContext.KubeContext
-		}
-		return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: TestContext.Host}}).ClientConfig()
-	default:
+func restclientConfig(kubeContext string) (*clientcmdapi.Config, error) {
+	Logf(">>> kubeConfig: %s\n", TestContext.KubeConfig)
+	if TestContext.KubeConfig == "" {
 		return nil, fmt.Errorf("KubeConfig must be specified to load client config")
 	}
+	c, err := clientcmd.LoadFromFile(TestContext.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error loading KubeConfig: %v", err.Error())
+	}
+	if kubeContext != "" {
+		Logf(">>> kubeContext: %s\n", kubeContext)
+		c.CurrentContext = kubeContext
+	}
+	return c, nil
+}
+
+func LoadConfig() (*restclient.Config, error) {
+	c, err := restclientConfig(TestContext.KubeContext)
+	if err != nil {
+		return nil, err
+	}
+
+	return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: TestContext.Host}}).ClientConfig()
+}
+
+func LoadFederatedConfig() (*restclient.Config, error) {
+	c, err := restclientConfig(federatedKubeContext)
+	if err != nil {
+		return nil, err
+	}
+	cfg, err := clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if cfg != nil {
+		//TODO(colhom): this is only here because https://github.com/kubernetes/kubernetes/issues/25422
+		cfg.NegotiatedSerializer = api.Codecs
+	}
+	return cfg, err
 }
 
 func loadClientFromConfig(config *restclient.Config) (*client.Client, error) {
@@ -1558,6 +1601,25 @@ func loadClientFromConfig(config *restclient.Config) (*client.Client, error) {
 	return c, nil
 }
 
+func loadFederationClientFromConfig(config *restclient.Config) (*unversionedfederation.FederationClient, error) {
+	c, err := unversionedfederation.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating client: %v", err.Error())
+	}
+	if c.Client.Timeout == 0 {
+		c.Client.Timeout = SingleCallTimeout
+	}
+	return c, nil
+}
+
+func LoadFederationClient() (*unversionedfederation.FederationClient, error) {
+	config, err := LoadFederatedConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error creating client: %v", err.Error())
+	}
+	return loadFederationClientFromConfig(config)
+}
+
 func LoadClient() (*client.Client, error) {
 	config, err := LoadConfig()
 	if err != nil {

test/images/clusterapi-tester/main.go

@@ -22,11 +22,12 @@ import (
 	"log"
 
 	"fmt"
+	"net/http"
+
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/fields"
 	"k8s.io/kubernetes/pkg/labels"
-	"net/http"
 )
 
 func main() {