From e7a8ac1099852f2512de2fe0915253ef12f3289e Mon Sep 17 00:00:00 2001 From: Yago Nobre Date: Tue, 5 Feb 2019 05:04:45 -0200 Subject: [PATCH] kubeadm: graduate kubelet start join phase --- cmd/kubeadm/app/cmd/BUILD | 4 - cmd/kubeadm/app/cmd/join.go | 126 +----------- cmd/kubeadm/app/cmd/phases/init/kubelet.go | 4 +- cmd/kubeadm/app/cmd/phases/join/BUILD | 8 + .../app/cmd/phases/join/controlplane.go | 2 - cmd/kubeadm/app/cmd/phases/join/kubelet.go | 192 ++++++++++++++++++ 6 files changed, 204 insertions(+), 132 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/phases/join/kubelet.go diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 97cdbf1c563..8d692732261 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -42,9 +42,7 @@ go_library( "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", - "//cmd/kubeadm/app/phases/patchnode:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", @@ -59,11 +57,9 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", "//vendor/github.com/lithammer/dedent:go_default_library", diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 92bc369c64d..42c464d4147 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,10 +28,8 @@ import ( "github.com/spf13/cobra" flag "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" "k8s.io/klog" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" @@ -44,16 +42,11 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - utilsexec "k8s.io/utils/exec" ) var ( @@ -127,19 +120,6 @@ var ( Often times the same token is used for both parts. In this case, the --token flag can be used instead of specifying each token individually. `) - - kubeadmJoinFailMsg = dedent.Dedent(` - Unfortunately, an error has occurred: - %v - - This error is likely caused by: - - The kubelet is not running - - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled) - - If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands: - - 'systemctl status kubelet' - - 'journalctl -xeu kubelet' - `) ) // joinOptions defines all the options exposed via flags by kubeadm join. @@ -201,6 +181,7 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { joinRunner.AppendPhase(phases.NewPreflightPhase()) joinRunner.AppendPhase(phases.NewControlPlanePreparePhase()) joinRunner.AppendPhase(phases.NewCheckEtcdPhase()) + joinRunner.AppendPhase(phases.NewKubeletStartPhase()) // sets the data builder function, that will be used by the runner // both when running the entire workflow or single phases @@ -438,25 +419,11 @@ func (j *joinData) Run() error { // TODO: individual phases should call these: // - phases that need initCfg should call joinData.InitCfg(). // - phases that need tlsBootstrapCfg should call joinData.TLSBootstrapCfg(). - tlsBootstrapCfg, err := j.TLSBootstrapCfg() - if err != nil { - return err - } initCfg, err := j.InitCfg() if err != nil { return err } - // Executes the kubelet TLS bootstrap process, that completes with the node - // joining the cluster with a dedicates set of credentials as required by - // the node authorizer. - // if the node is hosting a new control plane instance, since it uses static pods for the control plane, - // as soon as the kubelet starts it will take charge of creating control plane - // components on the node. - if err := j.BootstrapKubelet(tlsBootstrapCfg, initCfg); err != nil { - return err - } - // if the node is hosting a new control plane instance if j.cfg.ControlPlane != nil { // Completes the control plane setup @@ -485,81 +452,6 @@ func (j *joinData) Run() error { return nil } -// BootstrapKubelet executes the kubelet TLS bootstrap process. -// This process is executed by the kubelet and completes with the node joining the cluster -// with a dedicates set of credentials as required by the node authorizer -func (j *joinData) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config, initConfiguration *kubeadmapi.InitConfiguration) error { - bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() - - // Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk - klog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile) - if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, tlsBootstrapCfg); err != nil { - return errors.Wrap(err, "couldn't save bootstrap-kubelet.conf to disk") - } - - // Write the ca certificate to disk so kubelet can use it for authentication - cluster := tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster - if _, err := os.Stat(j.cfg.CACertPath); os.IsNotExist(err) { - if err := certutil.WriteCert(j.cfg.CACertPath, tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil { - return errors.Wrap(err, "couldn't save the CA certificate to disk") - } - } - - kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) - if err != nil { - return err - } - - bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile) - if err != nil { - return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) - } - - // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet - // Try to stop the kubelet service so no race conditions occur when configuring it - klog.V(1).Infof("Stopping the kubelet") - kubeletphase.TryStopKubelet() - - // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start - if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil { - return err - } - - // Write env file with flags for the kubelet to use. We only want to - // register the joining node with the specified taints if the node - // is not a master. The markmaster phase will register the taints otherwise. - registerTaintsUsingFlags := j.cfg.ControlPlane == nil - if err := kubeletphase.WriteKubeletDynamicEnvFile(&initConfiguration.ClusterConfiguration, &initConfiguration.NodeRegistration, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil { - return err - } - - // Try to start the kubelet service in case it's inactive - klog.V(1).Infof("Starting the kubelet") - kubeletphase.TryStartKubelet() - - // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf - // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process - // times out, display a somewhat user-friendly message. - waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout) - if err := waiter.WaitForKubeletAndFunc(waitForTLSBootstrappedClient); err != nil { - fmt.Printf(kubeadmJoinFailMsg, err) - return err - } - - // When we know the /etc/kubernetes/kubelet.conf file is available, get the client - client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) - if err != nil { - return err - } - - klog.V(1).Infof("[join] preserving the crisocket information for the node") - if err := patchnodephase.AnnotateCRISocket(client, j.cfg.NodeRegistration.Name, j.cfg.NodeRegistration.CRISocket); err != nil { - return errors.Wrap(err, "error uploading crisocket") - } - - return nil -} - // PostInstallControlPlane marks the new node as master and update the cluster status with information about current node func (j *joinData) PostInstallControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName) @@ -599,20 +491,6 @@ func (j *joinData) PostInstallControlPlane(initConfiguration *kubeadmapi.InitCon return nil } -// waitForTLSBootstrappedClient waits for the /etc/kubernetes/kubelet.conf file to be available -func waitForTLSBootstrappedClient() error { - fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...") - - // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, kubeadmconstants.TLSBootstrapTimeout, func() (bool, error) { - // Check that we can create a client set out of the kubelet kubeconfig. This ensures not - // only that the kubeconfig file exists, but that other files required by it also exist (like - // client certificate and key) - _, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) - return (err == nil), nil - }) -} - // fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { // Retrieves the kubeadm configuration diff --git a/cmd/kubeadm/app/cmd/phases/init/kubelet.go b/cmd/kubeadm/app/cmd/phases/init/kubelet.go index 74ec307dc47..ed24084b639 100644 --- a/cmd/kubeadm/app/cmd/phases/init/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/init/kubelet.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ import ( var ( kubeletStartPhaseExample = normalizer.Examples(` # Writes a dynamic environment file with kubelet flags from a InitConfiguration file. - kubeadm init phase kubelet-start --config masterconfig.yaml + kubeadm init phase kubelet-start --config config.yaml `) ) diff --git a/cmd/kubeadm/app/cmd/phases/join/BUILD b/cmd/kubeadm/app/cmd/phases/join/BUILD index 82dcff2919d..7fa558ea707 100644 --- a/cmd/kubeadm/app/cmd/phases/join/BUILD +++ b/cmd/kubeadm/app/cmd/phases/join/BUILD @@ -5,6 +5,7 @@ go_library( srcs = [ "checketcd.go", "controlplane.go", + "kubelet.go", "preflight.go", ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join", @@ -19,10 +20,17 @@ go_library( "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", + "//cmd/kubeadm/app/phases/kubelet:go_default_library", + "//cmd/kubeadm/app/phases/patchnode:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/util/normalizer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//vendor/github.com/lithammer/dedent:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplane.go b/cmd/kubeadm/app/cmd/phases/join/controlplane.go index b98b337584b..d4b97cf5ed0 100644 --- a/cmd/kubeadm/app/cmd/phases/join/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/join/controlplane.go @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" - clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" @@ -34,7 +33,6 @@ import ( type controlPlanePrepareData interface { Cfg() *kubeadmapi.JoinConfiguration - ClientSetFromFile(string) (*clientset.Clientset, error) InitCfg() (*kubeadmapi.InitConfiguration, error) } diff --git a/cmd/kubeadm/app/cmd/phases/join/kubelet.go b/cmd/kubeadm/app/cmd/phases/join/kubelet.go new file mode 100644 index 00000000000..4d7ea52210b --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/join/kubelet.go @@ -0,0 +1,192 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + "os" + + "github.com/lithammer/dedent" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + utilsexec "k8s.io/utils/exec" +) + +var ( + kubeadmJoinFailMsg = dedent.Dedent(` + Unfortunately, an error has occurred: + %v + + This error is likely caused by: + - The kubelet is not running + - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled) + + If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands: + - 'systemctl status kubelet' + - 'journalctl -xeu kubelet' + `) +) + +type kubeletStartData interface { + Cfg() *kubeadmapi.JoinConfiguration + ClientSetFromFile(path string) (*clientset.Clientset, error) + InitCfg() (*kubeadmapi.InitConfiguration, error) + TLSBootstrapCfg() (*clientcmdapi.Config, error) +} + +// NewKubeletStartPhase creates a kubeadm workflow phase that start kubelet on a node. +func NewKubeletStartPhase() workflow.Phase { + return workflow.Phase{ + Name: "kubelet-start", + Short: "Writes kubelet settings, certificates and (re)starts the kubelet", + Long: "Writes a file with KubeletConfiguration and an environment file with node specific kubelet settings, and then (re)starts kubelet.", + Run: runKubeletStartJoinPhase, + InheritFlags: []string{ + options.CfgPath, + options.NodeCRISocket, + options.NodeName, + options.TLSBootstrapToken, + options.TokenDiscovery, + options.TokenDiscoveryCAHash, + options.TokenDiscoverySkipCAHash, + options.TokenStr, + }, + } +} + +func getKubeletStartJoinData(c workflow.RunData) (kubeletStartData, *kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { + data, ok := c.(kubeletStartData) + if !ok { + return nil, nil, nil, nil, errors.New("kubelet-start phase invoked with an invalid data struct") + } + cfg := data.Cfg() + initCfg, err := data.InitCfg() + if err != nil { + return nil, nil, nil, nil, err + } + tlsBootstrapCfg, err := data.TLSBootstrapCfg() + if err != nil { + return nil, nil, nil, nil, err + } + return data, cfg, initCfg, tlsBootstrapCfg, nil +} + +// runKubeletStartJoinPhase executes the kubelet TLS bootstrap process. +// This process is executed by the kubelet and completes with the node joining the cluster +// with a dedicates set of credentials as required by the node authorizer +func runKubeletStartJoinPhase(c workflow.RunData) error { + data, cfg, initCfg, tlsBootstrapCfg, err := getKubeletStartJoinData(c) + if err != nil { + return err + } + bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() + + // Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk + klog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile) + if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, tlsBootstrapCfg); err != nil { + return errors.Wrap(err, "couldn't save bootstrap-kubelet.conf to disk") + } + + // Write the ca certificate to disk so kubelet can use it for authentication + cluster := tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster + if _, err := os.Stat(cfg.CACertPath); os.IsNotExist(err) { + if err := certutil.WriteCert(cfg.CACertPath, tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil { + return errors.Wrap(err, "couldn't save the CA certificate to disk") + } + } + + kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) + if err != nil { + return err + } + + bootstrapClient, err := data.ClientSetFromFile(bootstrapKubeConfigFile) + if err != nil { + return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) + } + + // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet + // Try to stop the kubelet service so no race conditions occur when configuring it + klog.V(1).Infof("[kubelet-start] Stopping the kubelet") + kubeletphase.TryStopKubelet() + + // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start + if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + + // Write env file with flags for the kubelet to use. We only want to + // register the joining node with the specified taints if the node + // is not a master. The markmaster phase will register the taints otherwise. + registerTaintsUsingFlags := cfg.ControlPlane == nil + if err := kubeletphase.WriteKubeletDynamicEnvFile(&initCfg.ClusterConfiguration, &initCfg.NodeRegistration, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + + // Try to start the kubelet service in case it's inactive + klog.V(1).Infof("[kubelet-start] Starting the kubelet") + kubeletphase.TryStartKubelet() + + // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf + // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process + // times out, display a somewhat user-friendly message. + waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout) + if err := waiter.WaitForKubeletAndFunc(waitForTLSBootstrappedClient); err != nil { + fmt.Printf(kubeadmJoinFailMsg, err) + return err + } + + // When we know the /etc/kubernetes/kubelet.conf file is available, get the client + client, err := data.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) + if err != nil { + return err + } + + klog.V(1).Infof("[kubelet-start] preserving the crisocket information for the node") + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return errors.Wrap(err, "error uploading crisocket") + } + + return nil +} + +// waitForTLSBootstrappedClient waits for the /etc/kubernetes/kubelet.conf file to be available +func waitForTLSBootstrappedClient() error { + fmt.Println("[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...") + + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. + return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, kubeadmconstants.TLSBootstrapTimeout, func() (bool, error) { + // Check that we can create a client set out of the kubelet kubeconfig. This ensures not + // only that the kubeconfig file exists, but that other files required by it also exist (like + // client certificate and key) + _, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) + return (err == nil), nil + }) +}