diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index a7f0e206fdd..b1006cb434e 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -38,7 +38,6 @@ go_library( "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 9ce935b2b76..28959bcaa57 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -29,6 +29,7 @@ import ( flag "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" "k8s.io/klog" @@ -42,10 +43,7 @@ import ( cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" @@ -161,6 +159,7 @@ type joinData struct { cfg *kubeadmapi.JoinConfiguration initCfg *kubeadmapi.InitConfiguration tlsBootstrapCfg *clientcmdapi.Config + clientSets map[string]*clientset.Clientset ignorePreflightErrors sets.String outputWriter io.Writer } @@ -200,6 +199,8 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { addJoinOtherFlags(cmd.Flags(), &joinOptions.cfgPath, &joinOptions.ignorePreflightErrors, &joinOptions.controlPlane, &joinOptions.token) joinRunner.AppendPhase(phases.NewPreflightJoinPhase()) + joinRunner.AppendPhase(phases.NewControlPlanePreparePhase()) + joinRunner.AppendPhase(phases.NewCheckEtcdPhase()) // sets the data builder function, that will be used by the runner // both when running the entire workflow or single phases @@ -373,6 +374,7 @@ func newJoinData(cmd *cobra.Command, args []string, options *joinOptions, out io return &joinData{ cfg: cfg, + clientSets: map[string]*clientset.Clientset{}, ignorePreflightErrors: ignorePreflightErrorsSet, outputWriter: out, }, nil @@ -408,6 +410,18 @@ func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { return initCfg, err } +func (j *joinData) ClientSetFromFile(path string) (*clientset.Clientset, error) { + if client, ok := j.clientSets[path]; ok { + return client, nil + } + client, err := kubeconfigutil.ClientSetFromFile(path) + if err != nil { + return nil, errors.Wrap(err, "[join] couldn't create Kubernetes client") + } + j.clientSets[path] = client + return client, nil +} + // IgnorePreflightErrors returns the list of preflight errors to ignore. func (j *joinData) IgnorePreflightErrors() sets.String { return j.ignorePreflightErrors @@ -432,13 +446,6 @@ func (j *joinData) Run() error { if err != nil { return err } - if j.cfg.ControlPlane != nil { - // Prepares the node for hosting a new control plane instance by writing necessary - // kubeconfig files, and static pod manifests - if err := j.PrepareForHostingControlPlane(initCfg); err != nil { - return err - } - } // Executes the kubelet TLS bootstrap process, that completes with the node // joining the cluster with a dedicates set of credentials as required by @@ -478,47 +485,6 @@ func (j *joinData) Run() error { return nil } -// PrepareForHostingControlPlane makes all preparation activities require for a node hosting a new control plane instance -func (j *joinData) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { - - // Generate missing certificates (if any) - if err := certsphase.CreatePKIAssets(initConfiguration); err != nil { - return err - } - - // Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself - // NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in - // following steps of the join --experimental-control plane workflow - if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, initConfiguration); err != nil { - return errors.Wrap(err, "error generating kubeconfig files") - } - - // Creates static pod manifests file for the control plane components to be deployed on this node - // Static pods will be created and managed by the kubelet as soon as it starts - if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil { - return errors.Wrap(err, "error creating static pod manifest files for the control plane components") - } - - // in case of local etcd - if initConfiguration.Etcd.External == nil { - // Checks that the etcd cluster is healthy - // NB. this check cannot be implemented before because it requires the admin.conf and all the certificates - // for connecting to etcd already in place - kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName) - - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - if err != nil { - return errors.Wrap(err, "couldn't create Kubernetes client") - } - - if err := etcdphase.CheckLocalEtcdClusterStatus(client, &initConfiguration.ClusterConfiguration); err != nil { - return err - } - } - - return nil -} - // BootstrapKubelet executes the kubelet TLS bootstrap process. // This process is executed by the kubelet and completes with the node joining the cluster // with a dedicates set of credentials as required by the node authorizer diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index 6313df275e8..4d5af23e79d 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -6,6 +6,7 @@ go_library( "addons.go", "bootstraptoken.go", "certs.go", + "checketcd.go", "controlplane.go", "etcd.go", "kubeconfig.go", diff --git a/cmd/kubeadm/app/cmd/phases/checketcd.go b/cmd/kubeadm/app/cmd/phases/checketcd.go new file mode 100644 index 00000000000..b64e831195a --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/checketcd.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "github.com/pkg/errors" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" +) + +type checkEtcdData interface { + Cfg() *kubeadmapi.JoinConfiguration + ClientSetFromFile(string) (*clientset.Clientset, error) + InitCfg() (*kubeadmapi.InitConfiguration, error) +} + +// NewCheckEtcdPhase is a hidden phase that runs after the control-plane-prepare and +// before the bootstrap-kubelet phase that ensures etcd is healthy +func NewCheckEtcdPhase() workflow.Phase { + return workflow.Phase{ + Name: "check-etcd", + Run: runCheckEtcdPhase, + Hidden: true, + } +} + +func runCheckEtcdPhase(c workflow.RunData) error { + data, ok := c.(checkEtcdData) + if !ok { + return errors.New("check-etcd phase invoked with an invalid data struct") + } + + // Skip if this is not a control plane + if data.Cfg().ControlPlane == nil { + return nil + } + + cfg, err := data.InitCfg() + if err != nil { + return err + } + + if cfg.Etcd.External != nil { + fmt.Println("[check-etcd] Skipping etcd check in external mode") + return nil + } + + fmt.Println("[check-etcd] Checking that the etcd cluster is healthy") + + // Checks that the etcd cluster is healthy + // NB. this check cannot be implemented before because it requires the admin.conf and all the certificates + // for connecting to etcd already in place + client, err := data.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) + if err != nil { + return err + } + + return etcdphase.CheckLocalEtcdClusterStatus(client, &cfg.ClusterConfiguration) +} diff --git a/cmd/kubeadm/app/cmd/phases/controlplane.go b/cmd/kubeadm/app/cmd/phases/controlplane.go index 27b1ac6c40b..8b472b29212 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane.go @@ -21,18 +21,21 @@ import ( "github.com/pkg/errors" + clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" ) var ( controlPlaneExample = normalizer.Examples(` - # Generates all static Pod manifest files for control plane components, + # Generates all static Pod manifest files for control plane components, # functionally equivalent to what is generated by kubeadm init. kubeadm init phase control-plane all @@ -65,6 +68,12 @@ type controlPlaneData interface { ManifestDir() string } +type controlPlanePrepareData interface { + Cfg() *kubeadmapi.JoinConfiguration + ClientSetFromFile(string) (*clientset.Clientset, error) + InitCfg() (*kubeadmapi.InitConfiguration, error) +} + func getPhaseDescription(component string) string { return fmt.Sprintf("Generates the %s static Pod manifest", component) } @@ -83,20 +92,34 @@ func NewControlPlanePhase() workflow.Phase { Example: controlPlaneExample, RunAllSiblings: true, }, - newControlPlaneSubPhase(kubeadmconstants.KubeAPIServer), - newControlPlaneSubPhase(kubeadmconstants.KubeControllerManager), - newControlPlaneSubPhase(kubeadmconstants.KubeScheduler), + newControlPlaneSubphase(kubeadmconstants.KubeAPIServer), + newControlPlaneSubphase(kubeadmconstants.KubeControllerManager), + newControlPlaneSubphase(kubeadmconstants.KubeScheduler), }, Run: runControlPlanePhase, } return phase } -func newControlPlaneSubPhase(component string) workflow.Phase { +// NewControlPlanePreparePhase creates a kubeadm workflow phase that implements the preparation of the node to serve a control plane +func NewControlPlanePreparePhase() workflow.Phase { + return workflow.Phase{ + Name: "control-plane-prepare", + Short: "Prepares the machine for serving a control plane.", + Long: cmdutil.MacroCommandLongDescription, + Phases: []workflow.Phase{ + newControlPlanePrepareCertsSubphase(), + newControlPlanePrepareKubeconfigSubphase(), + newControlPlanePrepareManifestsSubphases(), + }, + } +} + +func newControlPlaneSubphase(component string) workflow.Phase { phase := workflow.Phase{ Name: controlPlanePhaseProperties[component].name, Short: controlPlanePhaseProperties[component].short, - Run: runControlPlaneSubPhase(component), + Run: runControlPlaneSubphase(component), InheritFlags: getControlPlanePhaseFlags(component), } return phase @@ -132,6 +155,65 @@ func getControlPlanePhaseFlags(name string) []string { return flags } +func getControlPlanePreparePhaseFlags() []string { + return []string{ + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.CfgPath, + options.ControlPlane, + options.NodeName, + options.TokenDiscovery, + options.TokenDiscoveryCAHash, + options.TokenDiscoverySkipCAHash, + } +} + +func newControlPlanePrepareCertsSubphase() workflow.Phase { + return workflow.Phase{ + Name: "certs", + Short: "Generates the certificates for the new control plane components", + Run: runControlPlanePrepareCertsPhaseLocal, + InheritFlags: getControlPlanePreparePhaseFlags(), + } +} + +func newControlPlanePrepareKubeconfigSubphase() workflow.Phase { + return workflow.Phase{ + Name: "kubeconfig", + Short: "Generates the kubeconfig for the new control plane components", + Run: runControlPlanePrepareKubeconfigPhaseLocal, + InheritFlags: getControlPlanePreparePhaseFlags(), + } +} + +func newControlPlanePrepareManifestsSubphases() workflow.Phase { + return workflow.Phase{ + Name: "manifests", + Short: "Generates the manifests for the new control plane components", + Phases: []workflow.Phase{ + { + Name: "all", + Short: "Generates all static Pod manifest files", + InheritFlags: getControlPlanePreparePhaseFlags(), + RunAllSiblings: true, + }, + newControlPlanePrepareSubphase(kubeadmconstants.KubeAPIServer), + newControlPlanePrepareSubphase(kubeadmconstants.KubeControllerManager), + newControlPlanePrepareSubphase(kubeadmconstants.KubeScheduler), + }, + InheritFlags: getControlPlanePreparePhaseFlags(), + } +} + +func newControlPlanePrepareSubphase(component string) workflow.Phase { + return workflow.Phase{ + Name: controlPlanePhaseProperties[component].name, + Short: controlPlanePhaseProperties[component].short, + Run: runControlPlanePrepareJoinSubphase(component), + InheritFlags: getControlPlanePreparePhaseFlags(), + } +} + func runControlPlanePhase(c workflow.RunData) error { data, ok := c.(controlPlaneData) if !ok { @@ -142,7 +224,7 @@ func runControlPlanePhase(c workflow.RunData) error { return nil } -func runControlPlaneSubPhase(component string) func(c workflow.RunData) error { +func runControlPlaneSubphase(component string) func(c workflow.RunData) error { return func(c workflow.RunData) error { data, ok := c.(controlPlaneData) if !ok { @@ -154,3 +236,75 @@ func runControlPlaneSubPhase(component string) func(c workflow.RunData) error { return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) } } + +func runControlPlanePrepareCertsPhaseLocal(c workflow.RunData) error { + data, ok := c.(controlPlanePrepareData) + if !ok { + return errors.New("control-plane-prepare phase invoked with an invalid data struct") + } + + // Skip if this is not a control plane + if data.Cfg().ControlPlane == nil { + return nil + } + + cfg, err := data.InitCfg() + if err != nil { + return err + } + + // Generate missing certificates (if any) + return certsphase.CreatePKIAssets(cfg) +} + +func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error { + data, ok := c.(controlPlanePrepareData) + if !ok { + return errors.New("control-plane-prepare phase invoked with an invalid data struct") + } + + // Skip if this is not a control plane + if data.Cfg().ControlPlane == nil { + return nil + } + + cfg, err := data.InitCfg() + if err != nil { + return err + } + + fmt.Println("[control-plane-prepare] Generating kubeconfig files") + + // Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself + // NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in + // following steps of the join --experimental-control plane workflow + if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, cfg); err != nil { + return errors.Wrap(err, "error generating kubeconfig files") + } + + return nil +} + +func runControlPlanePrepareJoinSubphase(component string) func(c workflow.RunData) error { + return func(c workflow.RunData) error { + data, ok := c.(controlPlanePrepareData) + if !ok { + return errors.New("control-plane-prepare phase invoked with an invalid data struct") + } + + // Skip if this is not a control plane + if data.Cfg().ControlPlane == nil { + return nil + } + + cfg, err := data.InitCfg() + if err != nil { + return err + } + + // Creates static pod manifests file for the control plane components to be deployed on this node + // Static pods will be created and managed by the kubelet as soon as it starts + fmt.Printf("[control-plane-prepare] Creating static Pod manifest for %q\n", component) + return controlplane.CreateStaticPodFiles(kubeadmconstants.GetStaticPodDirectory(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) + } +}