From cb306ae0b3afa1e5b34803744196656ac227e5d4 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Mon, 25 Mar 2019 15:28:30 +0100 Subject: [PATCH] e2e-kubeadm-refactor --- test/e2e_kubeadm/BUILD | 21 ++- test/e2e_kubeadm/bootstrap_signer.go | 47 +++++++ test/e2e_kubeadm/bootstrap_token_test.go | 82 ++++++++++++ test/e2e_kubeadm/cluster_info_test.go | 78 +++++++++++ test/e2e_kubeadm/{matchers.go => const.go} | 27 ++-- test/e2e_kubeadm/controlplane_nodes_test.go | 68 ++++++++++ test/e2e_kubeadm/framework.go | 23 ++++ test/e2e_kubeadm/kubeadm_config_test.go | 117 +++++++++++++++++ test/e2e_kubeadm/kubeadm_test.go | 136 -------------------- test/e2e_kubeadm/runner/local/run_local.go | 3 +- test/e2e_kubeadm/util.go | 125 ++++++++++++++++++ 11 files changed, 568 insertions(+), 159 deletions(-) create mode 100644 test/e2e_kubeadm/bootstrap_signer.go create mode 100644 test/e2e_kubeadm/bootstrap_token_test.go create mode 100644 test/e2e_kubeadm/cluster_info_test.go rename test/e2e_kubeadm/{matchers.go => const.go} (51%) create mode 100644 test/e2e_kubeadm/controlplane_nodes_test.go create mode 100644 test/e2e_kubeadm/framework.go create mode 100644 test/e2e_kubeadm/kubeadm_config_test.go delete mode 100644 test/e2e_kubeadm/kubeadm_test.go create mode 100644 test/e2e_kubeadm/util.go diff --git a/test/e2e_kubeadm/BUILD b/test/e2e_kubeadm/BUILD index bf37bd72c2a..cf51c1bd533 100644 --- a/test/e2e_kubeadm/BUILD +++ b/test/e2e_kubeadm/BUILD @@ -9,24 +9,28 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_test( name = "go_default_test", srcs = [ + "bootstrap_token_test.go", + "cluster_info_test.go", + "controlplane_nodes_test.go", "e2e_kubeadm_suite_test.go", - "kubeadm_test.go", + "kubeadm_config_test.go", ], out = "e2e_kubeadm.test", embed = [":go_default_library"], tags = ["e2e"], deps = [ + "//staging/src/k8s.io/api/authorization/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//test/e2e/framework:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/gopkg.in/yaml.v2:go_default_library", ], ) @@ -50,11 +54,22 @@ filegroup( go_library( name = "go_default_library", - srcs = ["matchers.go"], + srcs = [ + "bootstrap_signer.go", + "const.go", + "framework.go", + "util.go", + ], importpath = "k8s.io/kubernetes/test/e2e_kubeadm", visibility = ["//visibility:public"], deps = [ + "//staging/src/k8s.io/api/authorization/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//test/e2e/framework:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/onsi/gomega/gstruct:go_default_library", ], diff --git a/test/e2e_kubeadm/bootstrap_signer.go b/test/e2e_kubeadm/bootstrap_signer.go new file mode 100644 index 00000000000..94cd60c6506 --- /dev/null +++ b/test/e2e_kubeadm/bootstrap_signer.go @@ -0,0 +1,47 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" +) + +const ( + bootstrapTokensSignerRoleName = "system:controller:bootstrap-signer" +) + +// Define container for all the test specification aimed at verifying +// that kubeadm creates the bootstrap signer +var _ = KubeadmDescribe("bootstrap signer", func() { + + // Get an instance of the k8s test framework + f := framework.NewDefaultFramework("bootstrap token") + + // Tests in this container are not expected to create new objects in the cluster + // so we are disabling the creation of a namespace in order to get a faster execution + f.SkipNamespaceCreation = true + + It("should be active", func() { + //NB. this is technically implemented a part of the control-plane phase + // and more specifically if the controller manager is properly configured, + // the bootstrapsigner controller is activated and the system:controller:bootstrap-signer + // group will be automatically created + ExpectRole(f.ClientSet, kubeSystemNamespace, bootstrapTokensSignerRoleName) + }) +}) diff --git a/test/e2e_kubeadm/bootstrap_token_test.go b/test/e2e_kubeadm/bootstrap_token_test.go new file mode 100644 index 00000000000..bf2f5f0dc43 --- /dev/null +++ b/test/e2e_kubeadm/bootstrap_token_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + bootstrapTokensGroup = "system:bootstrappers:kubeadm:default-node-token" + bootstrapTokensAllowPostCSRClusterRoleBinding = "kubeadm:kubelet-bootstrap" + bootstrapTokensAllowPostCSRClusterRoleName = "system:node-bootstrapper" + bootstrapTokensCSRAutoApprovalClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap" + bootstrapTokensCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" +) + +// Define container for all the test specification aimed at verifying +// that kubeadm creates the bootstrap token, the system:bootstrappers:kubeadm:default-node-token group +// and that all the related RBAC rules are in place +var _ = KubeadmDescribe("bootstrap token", func() { + + // Get an instance of the k8s test framework + f := framework.NewDefaultFramework("bootstrap token") + + // Tests in this container are not expected to create new objects in the cluster + // so we are disabling the creation of a namespace in order to get a faster execution + f.SkipNamespaceCreation = true + + It("should exist and be properly configured", func() { + secrets, err := f.ClientSet.CoreV1(). + Secrets(kubeSystemNamespace). + List(metav1.ListOptions{}) + framework.ExpectNoError(err, "error reading Secrets") + + tokenNum := 0 + for _, s := range secrets.Items { + if s.Type == corev1.SecretTypeBootstrapToken { + //TODO: might be we want to further check tokens (auth-extra-groups, usage etc) + tokenNum++ + } + } + Expect(tokenNum).Should(BeNumerically(">", 0), "At least one bootstrap token should exist") + }) + + It("should be allowed to post CSR for kubelet certificates on joining nodes", func() { + ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet, + bootstrapTokensAllowPostCSRClusterRoleBinding, + rbacv1.GroupKind, bootstrapTokensGroup, + bootstrapTokensAllowPostCSRClusterRoleName, + ) + //TODO: check if possible to verify "allowed to post CSR" using subject asses review as well + }) + + It("should be allowed to auto approve CSR for kubelet certificates on joining nodes", func() { + ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet, + bootstrapTokensCSRAutoApprovalClusterRoleBinding, + rbacv1.GroupKind, bootstrapTokensGroup, + bootstrapTokensCSRAutoApprovalClusterRoleName, + ) + //TODO: check if possible to verify "allowed to auto approve CSR" using subject asses review as well + }) +}) diff --git a/test/e2e_kubeadm/cluster_info_test.go b/test/e2e_kubeadm/cluster_info_test.go new file mode 100644 index 00000000000..c81cee9f1c8 --- /dev/null +++ b/test/e2e_kubeadm/cluster_info_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + authv1 "k8s.io/api/authorization/v1" + rbacv1 "k8s.io/api/rbac/v1" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + clusterInfoConfigMapName = "cluster-info" + clusterInfoRoleName = "kubeadm:bootstrap-signer-clusterinfo" + clusterInfoRoleBindingName = clusterInfoRoleName +) + +var ( + clusterInfoConfigMapResource = &authv1.ResourceAttributes{ + Namespace: kubePublicNamespace, + Name: clusterInfoConfigMapName, + Resource: "configmaps", + Verb: "get", + } +) + +// Define container for all the test specification aimed at verifying +// that kubeadm creates the cluster-info ConfigMap, that it is properly configured +// and that all the related RBAC rules are in place +var _ = KubeadmDescribe("cluster-info ConfigMap", func() { + + // Get an instance of the k8s test framework + f := framework.NewDefaultFramework("cluster-info") + + // Tests in this container are not expected to create new objects in the cluster + // so we are disabling the creation of a namespace in order to get a faster execution + f.SkipNamespaceCreation = true + + It("should exist and be properly configured", func() { + // Nb. this is technically implemented a part of the bootstrap-token phase + cm := GetConfigMap(f.ClientSet, kubePublicNamespace, clusterInfoConfigMapName) + + Expect(cm.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix))) + Expect(cm.Data).To(HaveKey(bootstrapapi.KubeConfigKey)) + + //TODO: What else? server? + }) + + It("should have related Role and RoleBinding", func() { + // Nb. this is technically implemented a part of the bootstrap-token phase + ExpectRole(f.ClientSet, kubePublicNamespace, clusterInfoRoleName) + ExpectRoleBinding(f.ClientSet, kubePublicNamespace, clusterInfoRoleBindingName) + }) + + It("should be accessible for anonymous", func() { + ExpectSubjectHasAccessToResource(f.ClientSet, + rbacv1.UserKind, anonymousUser, + clusterInfoConfigMapResource, + ) + }) +}) diff --git a/test/e2e_kubeadm/matchers.go b/test/e2e_kubeadm/const.go similarity index 51% rename from test/e2e_kubeadm/matchers.go rename to test/e2e_kubeadm/const.go index 8af786616b7..f1798f5dde4 100644 --- a/test/e2e_kubeadm/matchers.go +++ b/test/e2e_kubeadm/const.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,22 +16,11 @@ limitations under the License. package e2e_kubeadm -import ( - "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - corev1 "k8s.io/api/core/v1" +const ( + kubePublicNamespace = "kube-public" + kubeSystemNamespace = "kube-system" + + anonymousUser = "system:anonymous" + + nodesGroup = "system:nodes" ) - -func subject(name, kind string) gomega.OmegaMatcher { - return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Name": gomega.Equal(name), - "Kind": gomega.Equal(kind), - }) -} - -func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher { - return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ - "Key": gomega.Equal(key), - "Effect": gomega.Equal(effect), - }) -} diff --git a/test/e2e_kubeadm/controlplane_nodes_test.go b/test/e2e_kubeadm/controlplane_nodes_test.go new file mode 100644 index 00000000000..84b04b622f1 --- /dev/null +++ b/test/e2e_kubeadm/controlplane_nodes_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + controlPlaneTaint = "node-role.kubernetes.io/master" +) + +// Define container for all the test specification aimed at verifying +// that kubeadm configures the control-plane node as expected +var _ = KubeadmDescribe("control-plane node", func() { + + // Get an instance of the k8s test framework + f := framework.NewDefaultFramework("control-plane node") + + // Tests in this container are not expected to create new objects in the cluster + // so we are disabling the creation of a namespace in order to get a faster execution + f.SkipNamespaceCreation = true + + // Important! please note that this test can't be run on single-node clusters + // in case you can skip this test with SKIP=multi-node + It("should be labelled and tainted [multi-node]", func() { + // get all control-plane nodes (and this implicitly checks that node are properly labeled) + controlPlanes := getControlPlaneNodes(f.ClientSet) + + // checks if there is at least one control-plane node + Expect(controlPlanes.Items).NotTo(BeEmpty(), "at least one node with label %s should exist. if you are running test on a single-node cluster, you can skip this test with SKIP=multi-node", controlPlaneTaint) + + // checks that the control-plane nodes have the expected taint + for _, cp := range controlPlanes.Items { + framework.ExpectNodeHasTaint(f.ClientSet, cp.GetName(), &corev1.Taint{Key: controlPlaneTaint, Effect: corev1.TaintEffectNoSchedule}) + } + }) +}) + +func getControlPlaneNodes(c clientset.Interface) *corev1.NodeList { + selector := labels.Set{controlPlaneTaint: ""}.AsSelector() + masters, err := c.CoreV1().Nodes(). + List(metav1.ListOptions{LabelSelector: selector.String()}) + framework.ExpectNoError(err, "error reading control-plane nodes") + + return masters +} diff --git a/test/e2e_kubeadm/framework.go b/test/e2e_kubeadm/framework.go new file mode 100644 index 00000000000..73f97057be0 --- /dev/null +++ b/test/e2e_kubeadm/framework.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import "k8s.io/kubernetes/test/e2e/framework" + +func KubeadmDescribe(text string, body func()) bool { + return framework.KubeDescribe("[sig-cluster-lifecycle] [area-kubeadm] "+text, body) +} diff --git a/test/e2e_kubeadm/kubeadm_config_test.go b/test/e2e_kubeadm/kubeadm_config_test.go new file mode 100644 index 00000000000..b1b40711776 --- /dev/null +++ b/test/e2e_kubeadm/kubeadm_config_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + yaml "gopkg.in/yaml.v2" + authv1 "k8s.io/api/authorization/v1" + rbacv1 "k8s.io/api/rbac/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + kubeadmConfigName = "kubeadm-config" + kubeadmConfigRoleName = "kubeadm:nodes-kubeadm-config" + kubeadmConfigRoleBindingName = kubeadmConfigRoleName + kubeadmConfigClusterConfigurationConfigMapKey = "ClusterConfiguration" + kubeadmConfigClusterStatusConfigMapKey = "ClusterStatus" +) + +var ( + kubeadmConfigConfigMapResource = &authv1.ResourceAttributes{ + Namespace: kubeSystemNamespace, + Name: kubeadmConfigName, + Resource: "configmaps", + Verb: "get", + } +) + +// Define container for all the test specification aimed at verifying +// that kubeadm creates the cluster-info ConfigMap, that it is properly configured +// and that all the related RBAC rules are in place +var _ = KubeadmDescribe("kubeadm-config ConfigMap", func() { + + // Get an instance of the k8s test framework + f := framework.NewDefaultFramework("kubeadm-config") + + // Tests in this container are not expected to create new objects in the cluster + // so we are disabling the creation of a namespace in order to get a faster execution + f.SkipNamespaceCreation = true + + It("should exist and be properly configured", func() { + cm := GetConfigMap(f.ClientSet, kubeSystemNamespace, kubeadmConfigName) + + Expect(cm.Data).To(HaveKey(kubeadmConfigClusterConfigurationConfigMapKey)) + Expect(cm.Data).To(HaveKey(kubeadmConfigClusterStatusConfigMapKey)) + + m := unmarshalYaml(cm.Data[kubeadmConfigClusterStatusConfigMapKey]) + if _, ok := m["apiEndpoints"]; ok { + d := m["apiEndpoints"].(map[interface{}]interface{}) + // get all control-plane nodes + controlPlanes := getControlPlaneNodes(f.ClientSet) + + // checks that all the control-plane nodes are in the apiEndpoints list + for _, cp := range controlPlanes.Items { + if _, ok := d[cp.Name]; !ok { + framework.Failf("failed to get apiEndpoints for control-plane %s in %s", cp.Name, kubeadmConfigClusterStatusConfigMapKey) + } + } + } else { + framework.Failf("failed to get apiEndpoints from %s", kubeadmConfigClusterStatusConfigMapKey) + } + }) + + It("should have related Role and RoleBinding", func() { + ExpectRole(f.ClientSet, kubeSystemNamespace, kubeadmConfigRoleName) + ExpectRoleBinding(f.ClientSet, kubeSystemNamespace, kubeadmConfigRoleBindingName) + }) + + It("should be accessible for bootstrap tokens", func() { + ExpectSubjectHasAccessToResource(f.ClientSet, + rbacv1.GroupKind, bootstrapTokensGroup, + kubeadmConfigConfigMapResource, + ) + }) + + It("should be accessible for for nodes", func() { + ExpectSubjectHasAccessToResource(f.ClientSet, + rbacv1.GroupKind, nodesGroup, + kubeadmConfigConfigMapResource, + ) + }) +}) + +func getClusterConfiguration(c clientset.Interface) map[interface{}]interface{} { + cm := GetConfigMap(c, kubeSystemNamespace, kubeadmConfigName) + + Expect(cm.Data).To(HaveKey(kubeadmConfigClusterConfigurationConfigMapKey)) + + return unmarshalYaml(cm.Data[kubeadmConfigClusterConfigurationConfigMapKey]) +} + +func unmarshalYaml(data string) map[interface{}]interface{} { + m := make(map[interface{}]interface{}) + err := yaml.Unmarshal([]byte(data), &m) + if err != nil { + framework.Failf("error parsing %s ConfigMap: %v", kubeadmConfigName, err) + } + return m +} diff --git a/test/e2e_kubeadm/kubeadm_test.go b/test/e2e_kubeadm/kubeadm_test.go deleted file mode 100644 index 0628c54935a..00000000000 --- a/test/e2e_kubeadm/kubeadm_test.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e_kubeadm - -import ( - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - "k8s.io/kubernetes/test/e2e/framework" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -const ( - masterTaint = "node-role.kubernetes.io/master" - kubeadmConfigNamespace = "kube-system" - kubeadmConfigName = "kubeadm-config" - clusterInfoNamespace = "kube-public" - clusterInfoName = "cluster-info" - bootstrapSignerRoleNamespace = "kube-system" - bootstrapSignerRoleName = "system:controller:bootstrap-signer" -) - -var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() { - f := framework.NewDefaultFramework("kubeadm") - - Describe("kubeadm master", func() { - It("should be labelled and tainted", func() { - selector := labels.Set{masterTaint: ""}.AsSelector() - master, err := f.ClientSet.CoreV1().Nodes(). - List(metav1.ListOptions{LabelSelector: selector.String()}) - framework.ExpectNoError(err, "couldn't find a master node") - Expect(master.Items).NotTo(BeEmpty()) - for _, master := range master.Items { - Expect(master.Spec.Taints).To( - ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)), - ) - } - }) - }) - - Describe("kubeadm-config config map", func() { - It("should exist", func() { - _, err := f.ClientSet.CoreV1(). - ConfigMaps(kubeadmConfigNamespace). - Get(kubeadmConfigName, metav1.GetOptions{}) - framework.ExpectNoError(err) - }) - }) - - Describe("cluster-info", func() { - It("should have expected keys", func() { - clientInfo, err := f.ClientSet.CoreV1(). - ConfigMaps(clusterInfoNamespace). - Get(clusterInfoName, metav1.GetOptions{}) - framework.ExpectNoError(err, "couldn't find config map") - - Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix))) - Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey)) - }) - - It("should be public", func() { - cfg, err := framework.LoadConfig() - framework.ExpectNoError(err, "couldn't get config") - cfg = rest.AnonymousClientConfig(cfg) - client, err := kubernetes.NewForConfig(cfg) - framework.ExpectNoError(err, "couldn't create client") - - _, err = client.CoreV1().ConfigMaps(clusterInfoNamespace). - Get(clusterInfoName, metav1.GetOptions{}) - framework.ExpectNoError(err, "couldn't anonymously access config") - }) - }) - - Describe("bootstrap signer RBAC role", func() { - It("should exist", func() { - _, err := f.ClientSet.RbacV1(). - Roles(bootstrapSignerRoleNamespace). - Get(bootstrapSignerRoleName, metav1.GetOptions{}) - framework.ExpectNoError(err, "doesn't exist") - }) - }) - - Describe("kubeadm:kubelet-bootstrap cluster role binding", func() { - It("should exist", func() { - binding, err := f.ClientSet.RbacV1(). - ClusterRoleBindings(). - Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{}) - framework.ExpectNoError(err, "couldn't get clusterrolebinding") - Expect(binding.Subjects).To( - ContainElement(subject( - "system:bootstrappers:kubeadm:default-node-token", - rbacv1.GroupKind, - )), - ) - Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper")) - }) - }) - - Describe("autoapproval for new bootstrap token", func() { - It("should create a clusterrolebinding", func() { - binding, err := f.ClientSet.RbacV1(). - ClusterRoleBindings(). - Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{}) - framework.ExpectNoError(err, "couldn't get clusterrolebinding") - Expect(binding.Subjects).To( - ContainElement(subject( - "system:bootstrappers:kubeadm:default-node-token", - rbacv1.GroupKind, - )), - ) - Expect(binding.RoleRef.Name).To( - Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"), - ) - }) - }) -}) diff --git a/test/e2e_kubeadm/runner/local/run_local.go b/test/e2e_kubeadm/runner/local/run_local.go index 4adcba10948..b57689b0087 100644 --- a/test/e2e_kubeadm/runner/local/run_local.go +++ b/test/e2e_kubeadm/runner/local/run_local.go @@ -99,7 +99,8 @@ func getBazelGinkgo() (string, error) { func execCommand(binary string, args ...string) error { fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " ")) - cmd := exec.Command(binary, args...) + cmd := exec.Command("sh", "-c", strings.Join(append([]string{binary}, args...), " ")) + cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() diff --git a/test/e2e_kubeadm/util.go b/test/e2e_kubeadm/util.go new file mode 100644 index 00000000000..65078e27b2d --- /dev/null +++ b/test/e2e_kubeadm/util.go @@ -0,0 +1,125 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + authv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" +) + +// ConfigMaps utils + +func GetConfigMap(c clientset.Interface, namespace, name string) *corev1.ConfigMap { + r, err := c.CoreV1(). + ConfigMaps(namespace). + Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting ConfigMap %q from namespace %q", name, namespace) + return r +} + +// RBAC utils + +func ExpectRole(c clientset.Interface, namespace, name string) { + _, err := c.RbacV1(). + Roles(namespace). + Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting Role %q from namespace %q", name, namespace) +} + +func ExpectRoleBinding(c clientset.Interface, namespace, name string) { + _, err := c.RbacV1(). + RoleBindings(namespace). + Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting RoleBinding %q from namespace %q", name, namespace) +} + +func ExpectClusterRole(c clientset.Interface, name string) { + _, err := c.RbacV1(). + ClusterRoles(). + Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting ClusterRole %q", name) +} + +func ExpectClusterRoleBinding(c clientset.Interface, name string) { + _, err := c.RbacV1(). + ClusterRoleBindings(). + Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting ClusterRoleBindings %q", name) +} + +func ExpectClusterRoleBindingWithSubjectAndRole(c clientset.Interface, name, subjectKind, subject, role string) { + binding, err := c.RbacV1(). + ClusterRoleBindings(). + Get(name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error getting ClusterRoleBindings %q", name) + Expect(binding.Subjects).To( + ContainElement(subjectMatcher( + subject, + subjectKind, + )), + "ClusterRole %q does not have %s %q as subject", name, subjectKind, subject, + ) + Expect(binding.RoleRef.Name).To( + Equal(role), + "ClusterRole %q does not have %q as role", name, role, + ) +} + +func ExpectSubjectHasAccessToResource(c clientset.Interface, subjectKind, subject string, resource *authv1.ResourceAttributes) { + var sar *authv1.SubjectAccessReview + switch subjectKind { + case rbacv1.GroupKind: + sar = &authv1.SubjectAccessReview{ + Spec: authv1.SubjectAccessReviewSpec{ + Groups: []string{subject}, + ResourceAttributes: resource, + }, + } + case rbacv1.UserKind: + fallthrough + case rbacv1.ServiceAccountKind: + sar = &authv1.SubjectAccessReview{ + Spec: authv1.SubjectAccessReviewSpec{ + User: subject, + ResourceAttributes: resource, + }, + } + default: + framework.Failf("invalid subjectKind %s", subjectKind) + } + + s, err := c.AuthorizationV1().SubjectAccessReviews().Create(sar) + framework.ExpectNoError(err, "error getting SubjectAccessReview for %s %s to resource %+v", subjectKind, subject, *sar.Spec.ResourceAttributes) + + Expect(s.Status.Allowed).Should(BeTrue(), "%s %s has no access to resource %+v", subjectKind, subject, *sar.Spec.ResourceAttributes) +} + +// matchers + +func subjectMatcher(name, kind string) OmegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Name": Equal(name), + "Kind": Equal(kind), + }) +}