From 6914a8b937f50b11dd16a3faf0c362c7446c7adc Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Thu, 16 Nov 2017 20:20:18 +0800 Subject: [PATCH] Add phase kubelet --- cmd/kubeadm/app/cmd/init.go | 82 +----------- cmd/kubeadm/app/cmd/join.go | 10 +- cmd/kubeadm/app/cmd/util/cmdutil.go | 48 ------- cmd/kubeadm/app/constants/constants.go | 3 + cmd/kubeadm/app/phases/kubelet/kubelet.go | 150 ++++++++++++++++++++++ 5 files changed, 161 insertions(+), 132 deletions(-) create mode 100644 cmd/kubeadm/app/phases/kubelet/kubelet.go diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 79b270f9915..815a7ce81c3 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -30,15 +30,11 @@ import ( "github.com/spf13/cobra" flag "github.com/spf13/pflag" - "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/images" @@ -51,6 +47,7 @@ import ( controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" @@ -62,17 +59,9 @@ import ( kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" "k8s.io/kubernetes/pkg/api/legacyscheme" - rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" - kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" - kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" utilsexec "k8s.io/utils/exec" ) -const ( - // KubeletBaseConfigMapRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-system ns - KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap" -) - var ( initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(` Your Kubernetes master has initialized successfully! @@ -349,34 +338,6 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error creating client: %v", err) } - // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { - _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() - if err != nil { - return err - } - kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(i.cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1alpha1.SchemeGroupVersion, *kubeletCodecs) - if err != nil { - return err - } - - if err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes), - }, - }); err != nil { - return err - } - - if err := CreateKubeletBaseConfigMapRBACRules(client, i.cfg.NodeName); err != nil { - return fmt.Errorf("error creating kubelet base configmap RBAC rules: %v", err) - } - } - // waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled waiter := getWaiter(i.dryRun, client) @@ -393,10 +354,11 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("couldn't initialize a Kubernetes cluster") } + // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { - err = cmdutil.UpdateNodeWithConfigMap(client, i.cfg.NodeName) - if err != nil { - return nil + // Create base kubelet configuration for dynamic kubelet configuration feature. + if err := kubeletphase.CreateBaseKubeletConfiguration(i.cfg, client); err != nil { + return fmt.Errorf("error uploading configuration: %v", err) } } @@ -581,37 +543,3 @@ func waitForAPIAndKubelet(waiter apiclient.Waiter) error { // This call is blocking until one of the goroutines sends to errorChan return <-errorChan } - -// CreateKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the kubelet base ConfigMap in the kube-system namespace to unauthenticated users -func CreateKubeletBaseConfigMapRBACRules(client clientset.Interface, nodeName string) error { - err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: KubeletBaseConfigMapRoleName, - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(), - }, - }) - if err != nil { - return err - } - - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: KubeletBaseConfigMapRoleName, - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: KubeletBaseConfigMapRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: "Group", - Name: kubeadmconstants.NodesGroup, - }, - }, - }) -} diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index f01fd0eab61..4656b8cda85 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -32,10 +32,10 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" "k8s.io/kubernetes/cmd/kubeadm/app/features" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" @@ -239,12 +239,8 @@ func (j *Join) Run(out io.Writer) error { // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { - client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) - if err != nil { - return err - } - - if err = cmdutil.UpdateNodeWithConfigMap(client, j.cfg.NodeName); err != nil { + // Update the node with remote base kubelet configuration + if err := kubeletphase.UpdateNodeWithBaseKubeletConfiguration(j.cfg); err != nil { return err } } diff --git a/cmd/kubeadm/app/cmd/util/cmdutil.go b/cmd/kubeadm/app/cmd/util/cmdutil.go index 90cfc257a6c..87dcdff67f0 100644 --- a/cmd/kubeadm/app/cmd/util/cmdutil.go +++ b/cmd/kubeadm/app/cmd/util/cmdutil.go @@ -17,18 +17,9 @@ limitations under the License. package util import ( - "encoding/json" "fmt" "github.com/spf13/cobra" - - "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - clientset "k8s.io/client-go/kubernetes" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) // SubCmdRunE returns a function that handles a case where a subcommand must be specified @@ -66,42 +57,3 @@ func ValidateExactArgNumber(args []string, supportedArgs []string) error { } return nil } - -// UpdateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap -func UpdateNodeWithConfigMap(client clientset.Interface, nodeName string) error { - node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - - oldData, err := json.Marshal(node) - if err != nil { - return err - } - - kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{}) - if err != nil { - return err - } - - node.Spec.ConfigSource.ConfigMapRef.UID = kubeletCfg.UID - - newData, err := json.Marshal(node) - if err != nil { - return err - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return err - } - - if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil { - if apierrs.IsConflict(err) { - fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") - } - return err - } - - return nil -} diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index aa711cf1735..35f6c9d23f2 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -115,6 +115,9 @@ const ( // system:nodes group subject is removed if present. NodesClusterRoleBinding = "system:node" + // KubeletBaseConfigMapRoleName defines the base kubelet configuration ConfigMap. + KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap" + // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation APICallRetryInterval = 500 * time.Millisecond // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the master when doing discovery diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go new file mode 100644 index 00000000000..fb05785e3c0 --- /dev/null +++ b/cmd/kubeadm/app/phases/kubelet/kubelet.go @@ -0,0 +1,150 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "encoding/json" + "fmt" + + "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" + kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" +) + +// CreateBaseKubeletConfiguration creates base kubelet configuration for dynamic kubelet configuration feature. +func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { + _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() + if err != nil { + return err + } + kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1alpha1.SchemeGroupVersion, *kubeletCodecs) + if err != nil { + return err + } + + if err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes), + }, + }); err != nil { + return err + } + + if err := createKubeletBaseConfigMapRBACRules(client); err != nil { + return fmt.Errorf("error creating base kubelet configmap RBAC rules: %v", err) + } + + return updateNodeWithConfigMap(client, cfg.NodeName) +} + +// UpdateNodeWithBaseKubeletConfiguration updates node with remote base kubelet configuration +func UpdateNodeWithBaseKubeletConfiguration(cfg *kubeadmapi.NodeConfiguration) error { + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) + if err != nil { + return err + } + + return updateNodeWithConfigMap(client, cfg.NodeName) +} + +// createKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users +func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error { + if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeletBaseConfigMapRoleName, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(), + }, + }); err != nil { + return err + } + + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeletBaseConfigMapRoleName, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: kubeadmconstants.KubeletBaseConfigMapRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: "Group", + Name: kubeadmconstants.NodesGroup, + }, + }, + }) +} + +// updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap +func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error { + node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + oldData, err := json.Marshal(node) + if err != nil { + return err + } + + kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{}) + if err != nil { + return err + } + + node.Spec.ConfigSource.ConfigMapRef.UID = kubeletCfg.UID + + newData, err := json.Marshal(node) + if err != nil { + return err + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) + if err != nil { + return err + } + + if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil { + if apierrs.IsConflict(err) { + fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") + } + return err + } + + return nil +}