e2e-kubeadm-refactor

test/e2e_kubeadm/BUILD

@@ -9,24 +9,28 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_test(
     name = "go_default_test",
     srcs = [
+        "bootstrap_token_test.go",
+        "cluster_info_test.go",
+        "controlplane_nodes_test.go",
         "e2e_kubeadm_suite_test.go",
-        "kubeadm_test.go",
+        "kubeadm_config_test.go",
     ],
     out = "e2e_kubeadm.test",
     embed = [":go_default_library"],
     tags = ["e2e"],
     deps = [
+        "//staging/src/k8s.io/api/authorization/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
-        "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
     ],
 )
 
@@ -50,11 +54,22 @@ filegroup(
 
 go_library(
     name = "go_default_library",
-    srcs = ["matchers.go"],
+    srcs = [
+        "bootstrap_signer.go",
+        "const.go",
+        "framework.go",
+        "util.go",
+    ],
     importpath = "k8s.io/kubernetes/test/e2e_kubeadm",
     visibility = ["//visibility:public"],
     deps = [
+        "//staging/src/k8s.io/api/authorization/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/onsi/gomega/gstruct:go_default_library",
     ],

test/e2e_kubeadm/bootstrap_signer.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const (
+	bootstrapTokensSignerRoleName = "system:controller:bootstrap-signer"
+)
+
+// Define container for all the test specification aimed at verifying
+// that kubeadm creates the bootstrap signer
+var _ = KubeadmDescribe("bootstrap signer", func() {
+
+	// Get an instance of the k8s test framework
+	f := framework.NewDefaultFramework("bootstrap token")
+
+	// Tests in this container are not expected to create new objects in the cluster
+	// so we are disabling the creation of a namespace in order to get a faster execution
+	f.SkipNamespaceCreation = true
+
+	It("should be active", func() {
+		//NB. this is technically implemented a part of the control-plane phase
+		//    and more specifically if the controller manager is properly configured,
+		//    the bootstrapsigner controller is activated and the system:controller:bootstrap-signer
+		//    group will be automatically created
+		ExpectRole(f.ClientSet, kubeSystemNamespace, bootstrapTokensSignerRoleName)
+	})
+})

test/e2e_kubeadm/bootstrap_token_test.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	bootstrapTokensGroup                             = "system:bootstrappers:kubeadm:default-node-token"
+	bootstrapTokensAllowPostCSRClusterRoleBinding    = "kubeadm:kubelet-bootstrap"
+	bootstrapTokensAllowPostCSRClusterRoleName       = "system:node-bootstrapper"
+	bootstrapTokensCSRAutoApprovalClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap"
+	bootstrapTokensCSRAutoApprovalClusterRoleName    = "system:certificates.k8s.io:certificatesigningrequests:nodeclient"
+)
+
+// Define container for all the test specification aimed at verifying
+// that kubeadm creates the bootstrap token, the system:bootstrappers:kubeadm:default-node-token group
+// and that all the related RBAC rules are in place
+var _ = KubeadmDescribe("bootstrap token", func() {
+
+	// Get an instance of the k8s test framework
+	f := framework.NewDefaultFramework("bootstrap token")
+
+	// Tests in this container are not expected to create new objects in the cluster
+	// so we are disabling the creation of a namespace in order to get a faster execution
+	f.SkipNamespaceCreation = true
+
+	It("should exist and be properly configured", func() {
+		secrets, err := f.ClientSet.CoreV1().
+			Secrets(kubeSystemNamespace).
+			List(metav1.ListOptions{})
+		framework.ExpectNoError(err, "error reading Secrets")
+
+		tokenNum := 0
+		for _, s := range secrets.Items {
+			if s.Type == corev1.SecretTypeBootstrapToken {
+				//TODO: might be we want to further check tokens  (auth-extra-groups, usage etc)
+				tokenNum++
+			}
+		}
+		Expect(tokenNum).Should(BeNumerically(">", 0), "At least one bootstrap token should exist")
+	})
+
+	It("should be allowed to post CSR for kubelet certificates on joining nodes", func() {
+		ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet,
+			bootstrapTokensAllowPostCSRClusterRoleBinding,
+			rbacv1.GroupKind, bootstrapTokensGroup,
+			bootstrapTokensAllowPostCSRClusterRoleName,
+		)
+		//TODO: check if possible to verify "allowed to post CSR" using subject asses review as well
+	})
+
+	It("should be allowed to auto approve CSR for kubelet certificates on joining nodes", func() {
+		ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet,
+			bootstrapTokensCSRAutoApprovalClusterRoleBinding,
+			rbacv1.GroupKind, bootstrapTokensGroup,
+			bootstrapTokensCSRAutoApprovalClusterRoleName,
+		)
+		//TODO: check if possible to verify "allowed to auto approve CSR" using subject asses review as well
+	})
+})

test/e2e_kubeadm/cluster_info_test.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	authv1 "k8s.io/api/authorization/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	clusterInfoConfigMapName   = "cluster-info"
+	clusterInfoRoleName        = "kubeadm:bootstrap-signer-clusterinfo"
+	clusterInfoRoleBindingName = clusterInfoRoleName
+)
+
+var (
+	clusterInfoConfigMapResource = &authv1.ResourceAttributes{
+		Namespace: kubePublicNamespace,
+		Name:      clusterInfoConfigMapName,
+		Resource:  "configmaps",
+		Verb:      "get",
+	}
+)
+
+// Define container for all the test specification aimed at verifying
+// that kubeadm creates the cluster-info ConfigMap, that it is properly configured
+// and that all the related RBAC rules are in place
+var _ = KubeadmDescribe("cluster-info ConfigMap", func() {
+
+	// Get an instance of the k8s test framework
+	f := framework.NewDefaultFramework("cluster-info")
+
+	// Tests in this container are not expected to create new objects in the cluster
+	// so we are disabling the creation of a namespace in order to get a faster execution
+	f.SkipNamespaceCreation = true
+
+	It("should exist and be properly configured", func() {
+		// Nb. this is technically implemented a part of the bootstrap-token phase
+		cm := GetConfigMap(f.ClientSet, kubePublicNamespace, clusterInfoConfigMapName)
+
+		Expect(cm.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix)))
+		Expect(cm.Data).To(HaveKey(bootstrapapi.KubeConfigKey))
+
+		//TODO: What else? server?
+	})
+
+	It("should have related Role and RoleBinding", func() {
+		// Nb. this is technically implemented a part of the bootstrap-token phase
+		ExpectRole(f.ClientSet, kubePublicNamespace, clusterInfoRoleName)
+		ExpectRoleBinding(f.ClientSet, kubePublicNamespace, clusterInfoRoleBindingName)
+	})
+
+	It("should be accessible for anonymous", func() {
+		ExpectSubjectHasAccessToResource(f.ClientSet,
+			rbacv1.UserKind, anonymousUser,
+			clusterInfoConfigMapResource,
+		)
+	})
+})

test/e2e_kubeadm/const.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,22 +16,11 @@ limitations under the License.
 
 package e2e_kubeadm
 
-import (
+const (
-	"github.com/onsi/gomega"
+	kubePublicNamespace = "kube-public"
-	"github.com/onsi/gomega/gstruct"
+	kubeSystemNamespace = "kube-system"
-	corev1 "k8s.io/api/core/v1"
+
+	anonymousUser = "system:anonymous"
+
+	nodesGroup = "system:nodes"
 )
-
-func subject(name, kind string) gomega.OmegaMatcher {
-	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
-		"Name": gomega.Equal(name),
-		"Kind": gomega.Equal(kind),
-	})
-}
-
-func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher {
-	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
-		"Key":    gomega.Equal(key),
-		"Effect": gomega.Equal(effect),
-	})
-}

test/e2e_kubeadm/controlplane_nodes_test.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	controlPlaneTaint = "node-role.kubernetes.io/master"
+)
+
+// Define container for all the test specification aimed at verifying
+// that kubeadm configures the control-plane node as expected
+var _ = KubeadmDescribe("control-plane node", func() {
+
+	// Get an instance of the k8s test framework
+	f := framework.NewDefaultFramework("control-plane node")
+
+	// Tests in this container are not expected to create new objects in the cluster
+	// so we are disabling the creation of a namespace in order to get a faster execution
+	f.SkipNamespaceCreation = true
+
+	// Important! please note that this test can't be run on single-node clusters
+	// in case you can skip this test with SKIP=multi-node
+	It("should be labelled and tainted [multi-node]", func() {
+		// get all control-plane nodes (and this implicitly checks that node are properly labeled)
+		controlPlanes := getControlPlaneNodes(f.ClientSet)
+
+		// checks if there is at least one control-plane node
+		Expect(controlPlanes.Items).NotTo(BeEmpty(), "at least one node with label %s should exist. if you are running test on a single-node cluster, you can skip this test with SKIP=multi-node", controlPlaneTaint)
+
+		// checks that the control-plane nodes have the expected taint
+		for _, cp := range controlPlanes.Items {
+			framework.ExpectNodeHasTaint(f.ClientSet, cp.GetName(), &corev1.Taint{Key: controlPlaneTaint, Effect: corev1.TaintEffectNoSchedule})
+		}
+	})
+})
+
+func getControlPlaneNodes(c clientset.Interface) *corev1.NodeList {
+	selector := labels.Set{controlPlaneTaint: ""}.AsSelector()
+	masters, err := c.CoreV1().Nodes().
+		List(metav1.ListOptions{LabelSelector: selector.String()})
+	framework.ExpectNoError(err, "error reading control-plane nodes")
+
+	return masters
+}

test/e2e_kubeadm/framework.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import "k8s.io/kubernetes/test/e2e/framework"
+
+func KubeadmDescribe(text string, body func()) bool {
+	return framework.KubeDescribe("[sig-cluster-lifecycle] [area-kubeadm] "+text, body)
+}

test/e2e_kubeadm/kubeadm_config_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	yaml "gopkg.in/yaml.v2"
+	authv1 "k8s.io/api/authorization/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	kubeadmConfigName                             = "kubeadm-config"
+	kubeadmConfigRoleName                         = "kubeadm:nodes-kubeadm-config"
+	kubeadmConfigRoleBindingName                  = kubeadmConfigRoleName
+	kubeadmConfigClusterConfigurationConfigMapKey = "ClusterConfiguration"
+	kubeadmConfigClusterStatusConfigMapKey        = "ClusterStatus"
+)
+
+var (
+	kubeadmConfigConfigMapResource = &authv1.ResourceAttributes{
+		Namespace: kubeSystemNamespace,
+		Name:      kubeadmConfigName,
+		Resource:  "configmaps",
+		Verb:      "get",
+	}
+)
+
+// Define container for all the test specification aimed at verifying
+// that kubeadm creates the cluster-info ConfigMap, that it is properly configured
+// and that all the related RBAC rules are in place
+var _ = KubeadmDescribe("kubeadm-config ConfigMap", func() {
+
+	// Get an instance of the k8s test framework
+	f := framework.NewDefaultFramework("kubeadm-config")
+
+	// Tests in this container are not expected to create new objects in the cluster
+	// so we are disabling the creation of a namespace in order to get a faster execution
+	f.SkipNamespaceCreation = true
+
+	It("should exist and be properly configured", func() {
+		cm := GetConfigMap(f.ClientSet, kubeSystemNamespace, kubeadmConfigName)
+
+		Expect(cm.Data).To(HaveKey(kubeadmConfigClusterConfigurationConfigMapKey))
+		Expect(cm.Data).To(HaveKey(kubeadmConfigClusterStatusConfigMapKey))
+
+		m := unmarshalYaml(cm.Data[kubeadmConfigClusterStatusConfigMapKey])
+		if _, ok := m["apiEndpoints"]; ok {
+			d := m["apiEndpoints"].(map[interface{}]interface{})
+			// get all control-plane nodes
+			controlPlanes := getControlPlaneNodes(f.ClientSet)
+
+			// checks that all the control-plane nodes are in the apiEndpoints list
+			for _, cp := range controlPlanes.Items {
+				if _, ok := d[cp.Name]; !ok {
+					framework.Failf("failed to get apiEndpoints for control-plane %s in %s", cp.Name, kubeadmConfigClusterStatusConfigMapKey)
+				}
+			}
+		} else {
+			framework.Failf("failed to get apiEndpoints from %s", kubeadmConfigClusterStatusConfigMapKey)
+		}
+	})
+
+	It("should have related Role and RoleBinding", func() {
+		ExpectRole(f.ClientSet, kubeSystemNamespace, kubeadmConfigRoleName)
+		ExpectRoleBinding(f.ClientSet, kubeSystemNamespace, kubeadmConfigRoleBindingName)
+	})
+
+	It("should be accessible for bootstrap tokens", func() {
+		ExpectSubjectHasAccessToResource(f.ClientSet,
+			rbacv1.GroupKind, bootstrapTokensGroup,
+			kubeadmConfigConfigMapResource,
+		)
+	})
+
+	It("should be accessible for for nodes", func() {
+		ExpectSubjectHasAccessToResource(f.ClientSet,
+			rbacv1.GroupKind, nodesGroup,
+			kubeadmConfigConfigMapResource,
+		)
+	})
+})
+
+func getClusterConfiguration(c clientset.Interface) map[interface{}]interface{} {
+	cm := GetConfigMap(c, kubeSystemNamespace, kubeadmConfigName)
+
+	Expect(cm.Data).To(HaveKey(kubeadmConfigClusterConfigurationConfigMapKey))
+
+	return unmarshalYaml(cm.Data[kubeadmConfigClusterConfigurationConfigMapKey])
+}
+
+func unmarshalYaml(data string) map[interface{}]interface{} {
+	m := make(map[interface{}]interface{})
+	err := yaml.Unmarshal([]byte(data), &m)
+	if err != nil {
+		framework.Failf("error parsing %s ConfigMap: %v", kubeadmConfigName, err)
+	}
+	return m
+}

test/e2e_kubeadm/kubeadm_test.go

@@ -1,136 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e_kubeadm
-
-import (
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
-	"k8s.io/kubernetes/test/e2e/framework"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-const (
-	masterTaint                  = "node-role.kubernetes.io/master"
-	kubeadmConfigNamespace       = "kube-system"
-	kubeadmConfigName            = "kubeadm-config"
-	clusterInfoNamespace         = "kube-public"
-	clusterInfoName              = "cluster-info"
-	bootstrapSignerRoleNamespace = "kube-system"
-	bootstrapSignerRoleName      = "system:controller:bootstrap-signer"
-)
-
-var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() {
-	f := framework.NewDefaultFramework("kubeadm")
-
-	Describe("kubeadm master", func() {
-		It("should be labelled and tainted", func() {
-			selector := labels.Set{masterTaint: ""}.AsSelector()
-			master, err := f.ClientSet.CoreV1().Nodes().
-				List(metav1.ListOptions{LabelSelector: selector.String()})
-			framework.ExpectNoError(err, "couldn't find a master node")
-			Expect(master.Items).NotTo(BeEmpty())
-			for _, master := range master.Items {
-				Expect(master.Spec.Taints).To(
-					ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)),
-				)
-			}
-		})
-	})
-
-	Describe("kubeadm-config config map", func() {
-		It("should exist", func() {
-			_, err := f.ClientSet.CoreV1().
-				ConfigMaps(kubeadmConfigNamespace).
-				Get(kubeadmConfigName, metav1.GetOptions{})
-			framework.ExpectNoError(err)
-		})
-	})
-
-	Describe("cluster-info", func() {
-		It("should have expected keys", func() {
-			clientInfo, err := f.ClientSet.CoreV1().
-				ConfigMaps(clusterInfoNamespace).
-				Get(clusterInfoName, metav1.GetOptions{})
-			framework.ExpectNoError(err, "couldn't find config map")
-
-			Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix)))
-			Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey))
-		})
-
-		It("should be public", func() {
-			cfg, err := framework.LoadConfig()
-			framework.ExpectNoError(err, "couldn't get config")
-			cfg = rest.AnonymousClientConfig(cfg)
-			client, err := kubernetes.NewForConfig(cfg)
-			framework.ExpectNoError(err, "couldn't create client")
-
-			_, err = client.CoreV1().ConfigMaps(clusterInfoNamespace).
-				Get(clusterInfoName, metav1.GetOptions{})
-			framework.ExpectNoError(err, "couldn't anonymously access config")
-		})
-	})
-
-	Describe("bootstrap signer RBAC role", func() {
-		It("should exist", func() {
-			_, err := f.ClientSet.RbacV1().
-				Roles(bootstrapSignerRoleNamespace).
-				Get(bootstrapSignerRoleName, metav1.GetOptions{})
-			framework.ExpectNoError(err, "doesn't exist")
-		})
-	})
-
-	Describe("kubeadm:kubelet-bootstrap cluster role binding", func() {
-		It("should exist", func() {
-			binding, err := f.ClientSet.RbacV1().
-				ClusterRoleBindings().
-				Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{})
-			framework.ExpectNoError(err, "couldn't get clusterrolebinding")
-			Expect(binding.Subjects).To(
-				ContainElement(subject(
-					"system:bootstrappers:kubeadm:default-node-token",
-					rbacv1.GroupKind,
-				)),
-			)
-			Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper"))
-		})
-	})
-
-	Describe("autoapproval for new bootstrap token", func() {
-		It("should create a clusterrolebinding", func() {
-			binding, err := f.ClientSet.RbacV1().
-				ClusterRoleBindings().
-				Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{})
-			framework.ExpectNoError(err, "couldn't get clusterrolebinding")
-			Expect(binding.Subjects).To(
-				ContainElement(subject(
-					"system:bootstrappers:kubeadm:default-node-token",
-					rbacv1.GroupKind,
-				)),
-			)
-			Expect(binding.RoleRef.Name).To(
-				Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"),
-			)
-		})
-	})
-})

test/e2e_kubeadm/runner/local/run_local.go

@@ -99,7 +99,8 @@ func getBazelGinkgo() (string, error) {
 
 func execCommand(binary string, args ...string) error {
 	fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " "))
-	cmd := exec.Command(binary, args...)
+	cmd := exec.Command("sh", "-c", strings.Join(append([]string{binary}, args...), " "))
+
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	return cmd.Run()

test/e2e_kubeadm/util.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	authv1 "k8s.io/api/authorization/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gstruct"
+)
+
+// ConfigMaps utils
+
+func GetConfigMap(c clientset.Interface, namespace, name string) *corev1.ConfigMap {
+	r, err := c.CoreV1().
+		ConfigMaps(namespace).
+		Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "error getting ConfigMap %q from namespace %q", name, namespace)
+	return r
+}
+
+// RBAC utils
+
+func ExpectRole(c clientset.Interface, namespace, name string) {
+	_, err := c.RbacV1().
+		Roles(namespace).
+		Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "error getting Role %q from namespace %q", name, namespace)
+}
+
+func ExpectRoleBinding(c clientset.Interface, namespace, name string) {
+	_, err := c.RbacV1().
+		RoleBindings(namespace).
+		Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "error getting RoleBinding %q from namespace %q", name, namespace)
+}
+
+func ExpectClusterRole(c clientset.Interface, name string) {
+	_, err := c.RbacV1().
+		ClusterRoles().
+		Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "error getting ClusterRole %q", name)
+}
+
+func ExpectClusterRoleBinding(c clientset.Interface, name string) {
+	_, err := c.RbacV1().
+		ClusterRoleBindings().
+		Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "error getting ClusterRoleBindings %q", name)
+}
+
+func ExpectClusterRoleBindingWithSubjectAndRole(c clientset.Interface, name, subjectKind, subject, role string) {
+	binding, err := c.RbacV1().
+		ClusterRoleBindings().
+		Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "error getting ClusterRoleBindings %q", name)
+	Expect(binding.Subjects).To(
+		ContainElement(subjectMatcher(
+			subject,
+			subjectKind,
+		)),
+		"ClusterRole %q does not have %s %q as subject", name, subjectKind, subject,
+	)
+	Expect(binding.RoleRef.Name).To(
+		Equal(role),
+		"ClusterRole %q does not have %q as role", name, role,
+	)
+}
+
+func ExpectSubjectHasAccessToResource(c clientset.Interface, subjectKind, subject string, resource *authv1.ResourceAttributes) {
+	var sar *authv1.SubjectAccessReview
+	switch subjectKind {
+	case rbacv1.GroupKind:
+		sar = &authv1.SubjectAccessReview{
+			Spec: authv1.SubjectAccessReviewSpec{
+				Groups:             []string{subject},
+				ResourceAttributes: resource,
+			},
+		}
+	case rbacv1.UserKind:
+		fallthrough
+	case rbacv1.ServiceAccountKind:
+		sar = &authv1.SubjectAccessReview{
+			Spec: authv1.SubjectAccessReviewSpec{
+				User:               subject,
+				ResourceAttributes: resource,
+			},
+		}
+	default:
+		framework.Failf("invalid subjectKind %s", subjectKind)
+	}
+
+	s, err := c.AuthorizationV1().SubjectAccessReviews().Create(sar)
+	framework.ExpectNoError(err, "error getting SubjectAccessReview for %s %s to resource %+v", subjectKind, subject, *sar.Spec.ResourceAttributes)
+
+	Expect(s.Status.Allowed).Should(BeTrue(), "%s %s has no access to resource %+v", subjectKind, subject, *sar.Spec.ResourceAttributes)
+}
+
+// matchers
+
+func subjectMatcher(name, kind string) OmegaMatcher {
+	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+		"Name": Equal(name),
+		"Kind": Equal(kind),
+	})
+}