From 6034ef48a9868f8240e8add51ae1701622b525cd Mon Sep 17 00:00:00 2001 From: xiangpengzhao Date: Wed, 22 Nov 2017 15:52:23 +0800 Subject: [PATCH] Fix deadlock of writing file on node. --- cmd/kubeadm/app/cmd/init.go | 8 ++++ cmd/kubeadm/app/phases/kubelet/kubelet.go | 51 ++++++++++++----------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 3bdfe3c6628..62b9292a536 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -338,6 +338,14 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error printing files on dryrun: %v", err) } + // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + // Write base kubelet configuration for dynamic kubelet configuration feature. + if err := kubeletphase.WriteInitKubeletConfigToDiskOnMaster(i.cfg); err != nil { + return fmt.Errorf("error writing base kubelet configuration to disk: %v", err) + } + } + // Create a kubernetes client and wait for the API server to be healthy (if not dryrunning) client, err := createClient(i.cfg, i.dryRun) if err != nil { diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go index 5096bf8982a..79db338d1c2 100644 --- a/cmd/kubeadm/app/phases/kubelet/kubelet.go +++ b/cmd/kubeadm/app/phases/kubelet/kubelet.go @@ -33,7 +33,6 @@ import ( clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" @@ -56,10 +55,6 @@ func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client return err } - if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil { - return fmt.Errorf("failed to write initial remote configuration of kubelet to disk for node %s: %v", cfg.NodeName, err) - } - if err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, @@ -81,7 +76,7 @@ func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client // ConsumeBaseKubeletConfiguration consumes base kubelet configuration for dynamic kubelet configuration feature. func ConsumeBaseKubeletConfiguration(nodeName string) error { - client, err := getTLSBootstrappedClient() + client, err := getLocalNodeTLSBootstrappedClient() if err != nil { return err } @@ -164,7 +159,7 @@ func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error { return err } - if err := apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: kubeadmconstants.KubeletBaseConfigMapRoleName, Namespace: metav1.NamespaceSystem, @@ -176,24 +171,9 @@ func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error { }, Subjects: []rbac.Subject{ { - Kind: "Group", + Kind: rbac.GroupKind, Name: kubeadmconstants.NodesGroup, }, - }, - }); err != nil { - return err - } - - return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: tokenphase.NodeKubeletBootstrap, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: tokenphase.NodeBootstrapperClusterRoleName, - }, - Subjects: []rbac.Subject{ { Kind: rbac.GroupKind, Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, @@ -202,9 +182,9 @@ func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error { }) } -// getTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap +// getLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap // and then creates a client from config file /etc/kubernetes/kubelet.conf -func getTLSBootstrappedClient() (clientset.Interface, error) { +func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) { fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...") kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) @@ -221,6 +201,27 @@ func getTLSBootstrappedClient() (clientset.Interface, error) { return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) } +// WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master. +func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error { + fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s", cfg.NodeName) + + _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() + if err != nil { + return err + } + + kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1alpha1.SchemeGroupVersion, *kubeletCodecs) + if err != nil { + return err + } + + if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil { + return fmt.Errorf("failed to write base configuration of kubelet to disk on master node %s: %v", cfg.NodeName, err) + } + + return nil +} + func writeInitKubeletConfigToDisk(kubeletConfig []byte) error { baseCongfigFile := filepath.Join(kubeadmconstants.KubeletBaseConfigurationDir, kubeadmconstants.KubeletBaseConfigurationFile) if err := ioutil.WriteFile(baseCongfigFile, kubeletConfig, 0644); err != nil {