From 04748160a65434b72dfa49076426efc0192ba614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Thu, 10 Aug 2017 06:45:19 +0300 Subject: [PATCH] kubeadm: Move all node bootstrap token related code in one phase package --- cmd/kubeadm/app/cmd/init.go | 27 ++- cmd/kubeadm/app/cmd/token.go | 2 +- cmd/kubeadm/app/constants/constants.go | 8 + cmd/kubeadm/app/phases/addons/addons.go | 11 +- .../app/phases/apiconfig/clusterroles.go | 209 +++--------------- .../bootstraptoken/clusterinfo/clusterinfo.go | 105 +++++++++ .../clusterinfo/clusterinfo_test.go} | 52 +---- .../bootstraptoken/node/tlsbootstrap.go | 106 +++++++++ .../node/token.go} | 51 +---- .../phases/bootstraptoken/node/token_test.go | 59 +++++ cmd/kubeadm/app/util/apiclient/idempotency.go | 98 ++++++++ 11 files changed, 449 insertions(+), 279 deletions(-) create mode 100644 cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go rename cmd/kubeadm/app/phases/{token/bootstrap_test.go => bootstraptoken/clusterinfo/clusterinfo_test.go} (66%) create mode 100644 cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go rename cmd/kubeadm/app/phases/{token/bootstrap.go => bootstraptoken/node/token.go} (71%) create mode 100644 cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go create mode 100644 cmd/kubeadm/app/util/apiclient/idempotency.go diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 1cc3ae834fb..6eaa449638f 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -36,11 +36,12 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" addonsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons" apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig" + clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" + nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token" uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -247,25 +248,39 @@ func (i *Init) Run(out io.Writer) error { return err } + // PHASE 4: Mark the master with the right label/taint if err := markmasterphase.MarkMaster(client, i.cfg.NodeName); err != nil { return err } - // PHASE 4: Set up the bootstrap tokens + // PHASE 5: Set up the node bootstrap tokens if !i.skipTokenPrint { fmt.Printf("[token] Using token: %s\n", i.cfg.Token) } + // Create the default node bootstrap token tokenDescription := "The default bootstrap token generated by 'kubeadm init'." - if err := tokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, tokenDescription); err != nil { + if err := nodebootstraptokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, tokenDescription); err != nil { + return err + } + // Create RBAC rules that makes the bootstrap tokens able to post CSRs + if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil { + return err + } + // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically + if err := nodebootstraptokenphase.AutoApproveNodeBootstrapTokens(client, k8sVersion); err != nil { return err } - if err := tokenphase.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { + // Create the cluster-info ConfigMap with the associated RBAC rules + if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { + return err + } + if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil { return err } - // PHASE 5: Install and deploy all addons, and configure things as necessary + // PHASE 6: Install and deploy all addons, and configure things as necessary // Upload currently used configuration to the cluster if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { @@ -285,7 +300,7 @@ func (i *Init) Run(out io.Writer) error { return err } - // Is deployment type self-hosted? + // PHASE 7: Make the control plane self-hosted if feature gate is enabled if features.Enabled(i.cfg.FeatureFlags, features.SelfHosting) { // Temporary control plane is up, now we create our self hosted control // plane components and remove the static manifests: diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go index deb7f7a607a..730356605bb 100644 --- a/cmd/kubeadm/app/cmd/token.go +++ b/cmd/kubeadm/app/cmd/token.go @@ -34,7 +34,7 @@ import ( clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token" + tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 6f237f55895..20105623390 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -116,6 +116,10 @@ const ( // SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has SelfHostingPrefix = "self-hosted-" + + // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in + // TODO: This should be changed in the v1.8 dev cycle to a node-BT-specific group instead of the generic Bootstrap Token group that is used now + NodeBootstrapTokenAuthGroup = "system:bootstrappers" ) var ( @@ -143,6 +147,10 @@ var ( // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy MinimumControlPlaneVersion = version.MustParseSemantic("v1.7.0") + + // MinimumCSRAutoApprovalClusterRolesVersion defines whether kubeadm can rely on the built-in CSR approval ClusterRole or not (note, the binding is always created by kubeadm!) + // TODO: Remove this when the v1.9 cycle starts and we bump the minimum supported version to v1.8.0 + MinimumCSRAutoApprovalClusterRolesVersion = version.MustParseSemantic("v1.8.0-alpha.3") ) // GetStaticPodDirectory returns the location on the disk where the Static Pod should be present diff --git a/cmd/kubeadm/app/phases/addons/addons.go b/cmd/kubeadm/app/phases/addons/addons.go index 029e3008acd..9accdb5ce36 100644 --- a/cmd/kubeadm/app/phases/addons/addons.go +++ b/cmd/kubeadm/app/phases/addons/addons.go @@ -30,6 +30,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" ) @@ -100,14 +101,8 @@ func CreateKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientse return fmt.Errorf("unable to decode kube-proxy configmap %v", err) } - if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(kubeproxyConfigMap); err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("unable to create a new kube-proxy configmap: %v", err) - } - - if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(kubeproxyConfigMap); err != nil { - return fmt.Errorf("unable to update the kube-proxy configmap: %v", err) - } + if err := apiclientutil.CreateConfigMapIfNotExists(client, kubeproxyConfigMap); err != nil { + return err } kubeproxyDaemonSet := &extensions.DaemonSet{} diff --git a/cmd/kubeadm/app/phases/apiconfig/clusterroles.go b/cmd/kubeadm/app/phases/apiconfig/clusterroles.go index 4a034fc0cb8..98730cee86e 100644 --- a/cmd/kubeadm/app/phases/apiconfig/clusterroles.go +++ b/cmd/kubeadm/app/phases/apiconfig/clusterroles.go @@ -25,29 +25,18 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1beta1" - bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/util/version" ) const ( // KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole KubeProxyClusterRoleName = "system:node-proxier" - // NodeBootstrapperClusterRoleName sets the name for the TLS Node Bootstrapper ClusterRole - NodeBootstrapperClusterRoleName = "system:node-bootstrapper" - // BootstrapSignerClusterRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-public ns - BootstrapSignerClusterRoleName = "system:bootstrap-signer-clusterinfo" - - clusterRoleKind = "ClusterRole" - roleKind = "Role" - serviceAccountKind = "ServiceAccount" - rbacAPIGroup = "rbac.authorization.k8s.io" - anonymousUser = "system:anonymous" - nodeAutoApproveBootstrap = "kubeadm:node-autoapprove-bootstrap" ) // CreateServiceAccounts creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist. -func CreateServiceAccounts(clientset clientset.Interface) error { +func CreateServiceAccounts(client clientset.Interface) error { + // TODO: Each ServiceAccount should be created per-addon (decentralized) vs here serviceAccounts := []v1.ServiceAccount{ { ObjectMeta: metav1.ObjectMeta{ @@ -64,7 +53,7 @@ func CreateServiceAccounts(clientset clientset.Interface) error { } for _, sa := range serviceAccounts { - if _, err := clientset.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil { + if _, err := client.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil { if !apierrors.IsAlreadyExists(err) { return err } @@ -74,20 +63,11 @@ func CreateServiceAccounts(clientset clientset.Interface) error { } // CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster -func CreateRBACRules(clientset clientset.Interface, k8sVersion *version.Version) error { - if err := createRoles(clientset); err != nil { +func CreateRBACRules(client clientset.Interface, k8sVersion *version.Version) error { + if err := createClusterRoleBindings(client); err != nil { return err } - if err := createRoleBindings(clientset); err != nil { - return err - } - if err := createClusterRoles(clientset); err != nil { - return err - } - if err := createClusterRoleBindings(clientset); err != nil { - return err - } - if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset, k8sVersion); err != nil { + if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(client, k8sVersion); err != nil { return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err) } @@ -95,163 +75,32 @@ func CreateRBACRules(clientset clientset.Interface, k8sVersion *version.Version) return nil } -func createRoles(clientset clientset.Interface) error { - roles := []rbac.Role{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: BootstrapSignerClusterRoleName, - Namespace: metav1.NamespacePublic, - }, - Rules: []rbac.PolicyRule{ - rbachelper.NewRule("get").Groups("").Resources("configmaps").Names("cluster-info").RuleOrDie(), +func createClusterRoleBindings(client clientset.Interface) error { + // TODO: This ClusterRoleBinding should be created by the kube-proxy phase, not here + return apiclientutil.CreateClusterRoleBindingIfNotExists(client, &rbac.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubeadm:node-proxier", + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "ClusterRole", + Name: KubeProxyClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.ServiceAccountKind, + Name: kubeadmconstants.KubeProxyServiceAccountName, + Namespace: metav1.NamespaceSystem, }, }, - } - for _, role := range roles { - if _, err := clientset.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Create(&role); err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("unable to create RBAC role: %v", err) - } - - if _, err := clientset.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Update(&role); err != nil { - return fmt.Errorf("unable to update RBAC role: %v", err) - } - } - } - return nil + }) } -func createRoleBindings(clientset clientset.Interface) error { - roleBindings := []rbac.RoleBinding{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "kubeadm:bootstrap-signer-clusterinfo", - Namespace: metav1.NamespacePublic, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: roleKind, - Name: BootstrapSignerClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: "User", - Name: anonymousUser, - }, - }, - }, - } +func deletePermissiveNodesBindingWhenUsingNodeAuthorization(client clientset.Interface, k8sVersion *version.Version) error { - for _, roleBinding := range roleBindings { - if _, err := clientset.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(&roleBinding); err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("unable to create RBAC rolebinding: %v", err) - } - - if _, err := clientset.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(&roleBinding); err != nil { - return fmt.Errorf("unable to update RBAC rolebinding: %v", err) - } - } - } - return nil -} - -func createClusterRoles(clientset clientset.Interface) error { - clusterRoles := []rbac.ClusterRole{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: nodeAutoApproveBootstrap, - }, - Rules: []rbac.PolicyRule{ - rbachelper.NewRule("create").Groups("certificates.k8s.io").Resources("certificatesigningrequests/nodeclient").RuleOrDie(), - }, - }, - } - - for _, roleBinding := range clusterRoles { - if _, err := clientset.RbacV1beta1().ClusterRoles().Create(&roleBinding); err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("unable to create RBAC clusterrole: %v", err) - } - - if _, err := clientset.RbacV1beta1().ClusterRoles().Update(&roleBinding); err != nil { - return fmt.Errorf("unable to update RBAC clusterrole: %v", err) - } - } - } - return nil -} - -func createClusterRoleBindings(clientset clientset.Interface) error { - clusterRoleBindings := []rbac.ClusterRoleBinding{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "kubeadm:kubelet-bootstrap", - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: clusterRoleKind, - Name: NodeBootstrapperClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: "Group", - Name: bootstrapapi.BootstrapGroup, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: nodeAutoApproveBootstrap, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: clusterRoleKind, - Name: nodeAutoApproveBootstrap, - }, - Subjects: []rbac.Subject{ - { - Kind: "Group", - Name: bootstrapapi.BootstrapGroup, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "kubeadm:node-proxier", - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: clusterRoleKind, - Name: KubeProxyClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: serviceAccountKind, - Name: kubeadmconstants.KubeProxyServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }, - }, - } - - for _, clusterRoleBinding := range clusterRoleBindings { - if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Create(&clusterRoleBinding); err != nil { - if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("unable to create RBAC clusterrolebinding: %v", err) - } - - if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(&clusterRoleBinding); err != nil { - return fmt.Errorf("unable to update RBAC clusterrolebinding: %v", err) - } - } - } - return nil -} - -func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset clientset.Interface, k8sVersion *version.Version) error { - - nodesRoleBinding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{}) + // TODO: When the v1.9 cycle starts (targeting v1.9 at HEAD) and v1.8.0 is the minimum supported version, we can remove this function as the ClusterRoleBinding won't exist + // or already have no such permissive subject + nodesRoleBinding, err := client.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { // Nothing to do; the RoleBinding doesn't exist @@ -271,7 +120,7 @@ func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset clientset. nodesRoleBinding.Subjects = newSubjects - if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil { + if _, err := client.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil { return err } diff --git a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go new file mode 100644 index 00000000000..fc7824094bb --- /dev/null +++ b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go @@ -0,0 +1,105 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterinfo + +import ( + "fmt" + + "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/user" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1beta1" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" +) + +const ( + // BootstrapSignerClusterRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-public ns + BootstrapSignerClusterRoleName = "kubeadm:bootstrap-signer-clusterinfo" +) + +// CreateBootstrapConfigMapIfNotExists creates the kube-public ConfigMap if it doesn't exist already +func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string) error { + + fmt.Printf("[bootstraptoken] Creating the %q ConfigMap in the %q namespace\n", bootstrapapi.ConfigMapClusterInfo, metav1.NamespacePublic) + + adminConfig, err := clientcmd.LoadFromFile(file) + if err != nil { + return fmt.Errorf("failed to load admin kubeconfig [%v]", err) + } + + adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster + // Copy the cluster from admin.conf to the bootstrap kubeconfig, contains the CA cert and the server URL + bootstrapConfig := &clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + "": adminConfig.Clusters[adminCluster], + }, + } + bootstrapBytes, err := clientcmd.Write(*bootstrapConfig) + if err != nil { + return err + } + + // Create or update the ConfigMap in the kube-public namespace + return apiclientutil.CreateConfigMapIfNotExists(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: bootstrapapi.ConfigMapClusterInfo, + Namespace: metav1.NamespacePublic, + }, + Data: map[string]string{ + bootstrapapi.KubeConfigKey: string(bootstrapBytes), + }, + }) +} + +// CreateClusterInfoRBACRules creates the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace to unauthenticated users +func CreateClusterInfoRBACRules(client clientset.Interface) error { + err := apiclientutil.CreateRoleIfNotExists(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: BootstrapSignerClusterRoleName, + Namespace: metav1.NamespacePublic, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(bootstrapapi.ConfigMapClusterInfo).RuleOrDie(), + }, + }) + if err != nil { + return err + } + + return apiclientutil.CreateRoleBindingIfNotExists(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: BootstrapSignerClusterRoleName, + Namespace: metav1.NamespacePublic, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: BootstrapSignerClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.UserKind, + Name: user.Anonymous, + }, + }, + }) +} diff --git a/cmd/kubeadm/app/phases/token/bootstrap_test.go b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo_test.go similarity index 66% rename from cmd/kubeadm/app/phases/token/bootstrap_test.go rename to cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo_test.go index 7a001e2e521..73c7a947b49 100644 --- a/cmd/kubeadm/app/phases/token/bootstrap_test.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo_test.go @@ -14,20 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package token +package clusterinfo import ( - "bytes" "io/ioutil" "os" "testing" - "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" clientsetfake "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/pkg/api" ) @@ -47,59 +44,29 @@ preferences: {} users: - name: kubernetes-admin` -func TestEncodeTokenSecretData(t *testing.T) { - var tests = []struct { - token *kubeadmapi.TokenDiscovery - t time.Duration - }{ - {token: &kubeadmapi.TokenDiscovery{ID: "foo", Secret: "bar"}}, // should use default - {token: &kubeadmapi.TokenDiscovery{ID: "foo", Secret: "bar"}, t: time.Second}, // should use default - } - for _, rt := range tests { - actual := encodeTokenSecretData(rt.token.ID, rt.token.Secret, rt.t, []string{}, "") - if !bytes.Equal(actual["token-id"], []byte(rt.token.ID)) { - t.Errorf( - "failed EncodeTokenSecretData:\n\texpected: %s\n\t actual: %s", - rt.token.ID, - actual["token-id"], - ) - } - if !bytes.Equal(actual["token-secret"], []byte(rt.token.Secret)) { - t.Errorf( - "failed EncodeTokenSecretData:\n\texpected: %s\n\t actual: %s", - rt.token.Secret, - actual["token-secret"], - ) - } - if rt.t > 0 { - if actual["expiration"] == nil { - t.Errorf( - "failed EncodeTokenSecretData, duration was not added to time", - ) - } - } - } -} - func TestCreateBootstrapConfigMapIfNotExists(t *testing.T) { tests := []struct { name string createErr error + updateErr error expectErr bool }{ { "successful case should have no error", nil, + nil, false, }, { - "duplicate creation should have no error", + "if both create and update errors, return error", apierrors.NewAlreadyExists(api.Resource("configmaps"), "test"), - false, + apierrors.NewUnauthorized("go away!"), + true, }, { "unexpected error should be returned", apierrors.NewUnauthorized("go away!"), + nil, true, }, } @@ -119,6 +86,11 @@ func TestCreateBootstrapConfigMapIfNotExists(t *testing.T) { return true, nil, tc.createErr }) } + if tc.updateErr != nil { + client.PrependReactor("update", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, tc.updateErr + }) + } err = CreateBootstrapConfigMapIfNotExists(client, file.Name()) if tc.expectErr && err == nil { diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go b/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go new file mode 100644 index 00000000000..cb53271029b --- /dev/null +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go @@ -0,0 +1,106 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + + rbac "k8s.io/api/rbac/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1beta1" + "k8s.io/kubernetes/pkg/util/version" +) + +const ( + // NodeBootstrapperClusterRoleName defines the name of the auto-bootstrapped ClusterRole for letting someone post a CSR + // TODO: This value should be defined in an other, generic authz package instead of here + NodeBootstrapperClusterRoleName = "system:node-bootstrapper" + // NodeKubeletBootstrap defines the name of the ClusterRoleBinding that lets kubelets post CSRs + NodeKubeletBootstrap = "kubeadm:kubelet-bootstrap" + + // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR + // TODO: This value should be defined in an other, generic authz package instead of here + CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + // NodeAutoApproveBootstrap defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs + NodeAutoApproveBootstrap = "kubeadm:node-autoapprove-bootstrap" +) + +// AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Node Bootstrap Tokens able to post CSRs +func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error { + + fmt.Println("[bootstraptoken] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials") + + return apiclientutil.CreateClusterRoleBindingIfNotExists(client, &rbac.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: NodeKubeletBootstrap, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "ClusterRole", + Name: NodeBootstrapperClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.GroupKind, + Name: constants.NodeBootstrapTokenAuthGroup, + }, + }, + }) +} + +// AutoApproveNodeBootstrapTokens creates RBAC rules in a way that makes Node Bootstrap Tokens' CSR auto-approved by the csrapprover controller +func AutoApproveNodeBootstrapTokens(client clientset.Interface, k8sVersion *version.Version) error { + + fmt.Println("[bootstraptoken] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token") + + // TODO: When the v1.9 cycle starts (targeting v1.9 at HEAD) and v1.8.0 is the minimum supported version, we can remove this function as the ClusterRole will always exist + if k8sVersion.LessThan(constants.MinimumCSRAutoApprovalClusterRolesVersion) { + + err := apiclientutil.CreateClusterRoleIfNotExists(client, &rbac.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: CSRAutoApprovalClusterRoleName, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("create").Groups("certificates.k8s.io").Resources("certificatesigningrequests/nodeclient").RuleOrDie(), + }, + }) + if err != nil { + return err + } + } + + // Always create this kubeadm-specific binding though + return apiclientutil.CreateClusterRoleBindingIfNotExists(client, &rbac.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: NodeAutoApproveBootstrap, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "ClusterRole", + Name: CSRAutoApprovalClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: "Group", + Name: constants.NodeBootstrapTokenAuthGroup, + }, + }, + }) +} diff --git a/cmd/kubeadm/app/phases/token/bootstrap.go b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go similarity index 71% rename from cmd/kubeadm/app/phases/token/bootstrap.go rename to cmd/kubeadm/app/phases/bootstraptoken/node/token.go index 2bb3c0a5d27..e4c29912e4a 100644 --- a/cmd/kubeadm/app/phases/token/bootstrap.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package token +package node import ( "fmt" @@ -24,14 +24,14 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" ) const tokenCreateRetries = 5 +// TODO(mattmoyer): Move CreateNewToken, UpdateOrCreateToken and encodeTokenSecretData out of this package to client-go for a generic abstraction and client for a Bootstrap Token + // CreateNewToken tries to create a token and fails if one with the same ID already exists func CreateNewToken(client clientset.Interface, token string, tokenDuration time.Duration, usages []string, description string) error { return UpdateOrCreateToken(client, token, true, tokenDuration, usages, description) @@ -55,9 +55,8 @@ func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists secret.Data = encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, description) if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Update(secret); err == nil { return nil - } else { - lastErr = err } + lastErr = err continue } @@ -72,9 +71,8 @@ func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists } if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret); err == nil { return nil - } else { - lastErr = err } + lastErr = err continue } @@ -86,45 +84,10 @@ func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists ) } -// CreateBootstrapConfigMapIfNotExists creates the public cluster-info ConfigMap (if it doesn't already exist) -func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string) error { - adminConfig, err := clientcmd.LoadFromFile(file) - if err != nil { - return fmt.Errorf("failed to load admin kubeconfig [%v]", err) - } - - adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster - // Copy the cluster from admin.conf to the bootstrap kubeconfig, contains the CA cert and the server URL - bootstrapConfig := &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": adminConfig.Clusters[adminCluster], - }, - } - bootstrapBytes, err := clientcmd.Write(*bootstrapConfig) - if err != nil { - return err - } - - bootstrapConfigMap := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: bootstrapapi.ConfigMapClusterInfo}, - Data: map[string]string{ - bootstrapapi.KubeConfigKey: string(bootstrapBytes), - }, - } - - if _, err := client.CoreV1().ConfigMaps(metav1.NamespacePublic).Create(&bootstrapConfigMap); err != nil { - if apierrors.IsAlreadyExists(err) { - return nil - } - return err - } - return nil -} - // encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret -func encodeTokenSecretData(tokenId, tokenSecret string, duration time.Duration, usages []string, description string) map[string][]byte { +func encodeTokenSecretData(tokenID, tokenSecret string, duration time.Duration, usages []string, description string) map[string][]byte { data := map[string][]byte{ - bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), } diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go b/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go new file mode 100644 index 00000000000..48a6f80e982 --- /dev/null +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "bytes" + "testing" + "time" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +func TestEncodeTokenSecretData(t *testing.T) { + var tests = []struct { + token *kubeadmapi.TokenDiscovery + t time.Duration + }{ + {token: &kubeadmapi.TokenDiscovery{ID: "foo", Secret: "bar"}}, // should use default + {token: &kubeadmapi.TokenDiscovery{ID: "foo", Secret: "bar"}, t: time.Second}, // should use default + } + for _, rt := range tests { + actual := encodeTokenSecretData(rt.token.ID, rt.token.Secret, rt.t, []string{}, "") + if !bytes.Equal(actual["token-id"], []byte(rt.token.ID)) { + t.Errorf( + "failed EncodeTokenSecretData:\n\texpected: %s\n\t actual: %s", + rt.token.ID, + actual["token-id"], + ) + } + if !bytes.Equal(actual["token-secret"], []byte(rt.token.Secret)) { + t.Errorf( + "failed EncodeTokenSecretData:\n\texpected: %s\n\t actual: %s", + rt.token.Secret, + actual["token-secret"], + ) + } + if rt.t > 0 { + if actual["expiration"] == nil { + t.Errorf( + "failed EncodeTokenSecretData, duration was not added to time", + ) + } + } + } +} diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go new file mode 100644 index 00000000000..4db6f98795b --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -0,0 +1,98 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + clientset "k8s.io/client-go/kubernetes" +) + +// TODO: We should invent a dynamic mechanism for this using the dynamic client instead of hard-coding these functions per-type + +// CreateClusterRoleIfNotExists creates a ClusterRole if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateClusterRoleIfNotExists(client clientset.Interface, clusterRole *rbac.ClusterRole) error { + if _, err := client.RbacV1beta1().ClusterRoles().Create(clusterRole); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create RBAC clusterrole: %v", err) + } + + if _, err := client.RbacV1beta1().ClusterRoles().Update(clusterRole); err != nil { + return fmt.Errorf("unable to update RBAC clusterrole: %v", err) + } + } + return nil +} + +// CreateClusterRoleBindingIfNotExists creates a ClusterRoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateClusterRoleBindingIfNotExists(client clientset.Interface, clusterRoleBinding *rbac.ClusterRoleBinding) error { + if _, err := client.RbacV1beta1().ClusterRoleBindings().Create(clusterRoleBinding); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create RBAC clusterrolebinding: %v", err) + } + + if _, err := client.RbacV1beta1().ClusterRoleBindings().Update(clusterRoleBinding); err != nil { + return fmt.Errorf("unable to update RBAC clusterrolebinding: %v", err) + } + } + return nil +} + +// CreateRoleIfNotExists creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateRoleIfNotExists(client clientset.Interface, role *rbac.Role) error { + if _, err := client.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Create(role); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create RBAC role: %v", err) + } + + if _, err := client.RbacV1beta1().Roles(role.ObjectMeta.Namespace).Update(role); err != nil { + return fmt.Errorf("unable to update RBAC role: %v", err) + } + } + return nil +} + +// CreateRoleBindingIfNotExists creates a RoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateRoleBindingIfNotExists(client clientset.Interface, roleBinding *rbac.RoleBinding) error { + if _, err := client.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(roleBinding); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create RBAC rolebinding: %v", err) + } + + if _, err := client.RbacV1beta1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(roleBinding); err != nil { + return fmt.Errorf("unable to update RBAC rolebinding: %v", err) + } + } + return nil +} + +// CreateConfigMapIfNotExists creates a ConfigMap if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateConfigMapIfNotExists(client clientset.Interface, cm *v1.ConfigMap) error { + if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(cm); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create configmap: %v", err) + } + + if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(cm); err != nil { + return fmt.Errorf("unable to update configmap: %v", err) + } + } + return nil +}