From 038a94804e371b2252bef7e595c8d4a72df10f81 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 23 Feb 2024 14:42:07 +0100 Subject: [PATCH] kubeadm: implement ControlPlaneKubeletLocalMode --- cmd/kubeadm/app/cmd/join.go | 2 + .../app/cmd/phases/join/controlplanejoin.go | 30 +++++++ cmd/kubeadm/app/cmd/phases/join/data.go | 17 ++++ cmd/kubeadm/app/cmd/phases/join/kubelet.go | 90 +++++++++++++++++-- cmd/kubeadm/app/features/features.go | 3 + 5 files changed, 135 insertions(+), 7 deletions(-) diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index c685e6bcceb..ae38da06b7f 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -219,6 +219,8 @@ func newCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { joinRunner.AppendPhase(phases.NewControlPlanePreparePhase()) joinRunner.AppendPhase(phases.NewCheckEtcdPhase()) joinRunner.AppendPhase(phases.NewKubeletStartPhase()) + joinRunner.AppendPhase(phases.NewEtcdJoinPhase()) + joinRunner.AppendPhase(phases.NewKubeletWaitBootstrapPhase()) joinRunner.AppendPhase(phases.NewControlPlaneJoinPhase()) joinRunner.AppendPhase(phases.NewWaitControlPlanePhase()) diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go index 7105c928f7f..8fc793cca59 100644 --- a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go +++ b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/features" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" @@ -35,6 +36,11 @@ var controlPlaneJoinExample = cmdutil.Examples(` kubeadm join phase control-plane-join all `) +var etcdJoinExample = cmdutil.Examples(` + # Joins etcd for a control plane instance + kubeadm join phase control-plane-join-etcd all +`) + func getControlPlaneJoinPhaseFlags(name string) []string { flags := []string{ options.CfgPath, @@ -73,6 +79,25 @@ func NewControlPlaneJoinPhase() workflow.Phase { } } +// NewEtcdJoinPhase creates a kubeadm workflow phase that implements joining etcd +func NewEtcdJoinPhase() workflow.Phase { + return workflow.Phase{ + Name: "etcd-join", + Short: fmt.Sprintf("[EXPERIMENTAL] Join etcd for control plane nodes (only used when feature gate %s is enabled)", features.ControlPlaneKubeletLocalMode), + Run: runEtcdPhase, + Example: etcdJoinExample, + InheritFlags: getControlPlaneJoinPhaseFlags("etcd"), + ArgsValidator: cobra.NoArgs, + // TODO: unhide this phase once ControlPlaneKubeletLocalMode goes GA: + // https://github.com/kubernetes/enhancements/issues/4471 + Hidden: true, + // Only run this phase as if `ControlPlaneKubeletLocalMode` is activated. + RunIf: func(c workflow.RunData) (bool, error) { + return checkFeatureState(c, features.ControlPlaneKubeletLocalMode, true) + }, + } +} + func newEtcdLocalSubphase() workflow.Phase { return workflow.Phase{ Name: "etcd", @@ -80,6 +105,11 @@ func newEtcdLocalSubphase() workflow.Phase { Run: runEtcdPhase, InheritFlags: getControlPlaneJoinPhaseFlags("etcd"), ArgsValidator: cobra.NoArgs, + // Only run this phase as subphase of control-plane-join phase if + // `ControlPlaneKubeletLocalMode` is deactivated. + RunIf: func(c workflow.RunData) (bool, error) { + return checkFeatureState(c, features.ControlPlaneKubeletLocalMode, false) + }, } } diff --git a/cmd/kubeadm/app/cmd/phases/join/data.go b/cmd/kubeadm/app/cmd/phases/join/data.go index bbfb8730cbe..c8aa87b3f7c 100644 --- a/cmd/kubeadm/app/cmd/phases/join/data.go +++ b/cmd/kubeadm/app/cmd/phases/join/data.go @@ -20,11 +20,14 @@ package phases import ( "io" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/features" ) // JoinData is the interface to use for join phases. @@ -44,3 +47,17 @@ type JoinData interface { ManifestDir() string CertificateWriteDir() string } + +func checkFeatureState(c workflow.RunData, featureGate string, state bool) (bool, error) { + data, ok := c.(JoinData) + if !ok { + return false, errors.New("control-plane-join phase invoked with an invalid data struct") + } + + cfg, err := data.InitCfg() + if err != nil { + return false, err + } + + return state == features.Enabled(cfg.FeatureGates, featureGate), nil +} diff --git a/cmd/kubeadm/app/cmd/phases/join/kubelet.go b/cmd/kubeadm/app/cmd/phases/join/kubelet.go index a24386df8f6..7c4d27c519b 100644 --- a/cmd/kubeadm/app/cmd/phases/join/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/join/kubelet.go @@ -40,8 +40,10 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" @@ -85,6 +87,27 @@ func NewKubeletStartPhase() workflow.Phase { } } +// NewKubeletWaitBootstrapPhase creates a kubeadm workflow phase that start kubelet on a node. +func NewKubeletWaitBootstrapPhase() workflow.Phase { + return workflow.Phase{ + Name: "kubelet-wait-bootstrap", + Short: "[EXPERIMENTAL] Wait for the kubelet to bootstrap itself (only used when feature gate ControlPlaneKubeletLocalMode is enabled)", + Run: runKubeletWaitBootstrapPhase, + InheritFlags: []string{ + options.CfgPath, + options.NodeCRISocket, + options.DryRun, + }, + // TODO: unhide this phase once ControlPlaneKubeletLocalMode goes GA: + // https://github.com/kubernetes/enhancements/issues/4471 + Hidden: true, + // Only run this phase as if `ControlPlaneKubeletLocalMode` is activated. + RunIf: func(c workflow.RunData) (bool, error) { + return checkFeatureState(c, features.ControlPlaneKubeletLocalMode, true) + }, + } +} + func getKubeletStartJoinData(c workflow.RunData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { data, ok := c.(JoinData) if !ok { @@ -117,8 +140,36 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { } bootstrapKubeConfigFile := filepath.Join(data.KubeConfigDir(), kubeadmconstants.KubeletBootstrapKubeConfigFileName) - // Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk - defer os.Remove(bootstrapKubeConfigFile) + // Do not delete the bootstrapKubeConfigFile at the end of this function when + // using ControlPlaneKubeletLocalMode. The KubeletWaitBootstrapPhase will delete + // it when the feature is enabled. + if !features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + // Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk + defer func() { + _ = os.Remove(bootstrapKubeConfigFile) + }() + } + + // Create the bootstrap client before we possibly overwrite the server address + // for ControlPlaneKubeletLocalMode. + bootstrapClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) + if err != nil { + return errors.Errorf("could not create client from bootstrap kubeconfig") + } + + if features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + // Set the server url to LocalAPIEndpoint if the feature gate is enabled so the config + // which gets passed to the kubelet forces it to talk to the local kube-apiserver. + if cfg.ControlPlane != nil { + for c, conf := range tlsBootstrapCfg.Clusters { + conf.Server, err = kubeadmutil.GetLocalAPIEndpoint(&cfg.ControlPlane.LocalAPIEndpoint) + if err != nil { + return errors.Wrapf(err, "could not get LocalAPIEndpoint when %s is enabled", features.ControlPlaneKubeletLocalMode) + } + tlsBootstrapCfg.Clusters[c] = conf + } + } + } // Write the bootstrap kubelet config file or the TLS-Bootstrapped kubelet config file down to disk klog.V(1).Infof("[kubelet-start] writing bootstrap kubelet config file at %s", bootstrapKubeConfigFile) @@ -142,11 +193,6 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { } } - bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile) - if err != nil { - return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) - } - // Obtain the name of this Node. nodeName, _, err := kubeletphase.GetNodeNameAndHostname(&cfg.NodeRegistration) if err != nil { @@ -205,6 +251,36 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { fmt.Println("[kubelet-start] Starting the kubelet") kubeletphase.TryStartKubelet() + // Run the same code as KubeletWaitBootstrapPhase would do if the ControlPlaneKubeletLocalMode feature gate is disabled. + if !features.Enabled(initCfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + if err := runKubeletWaitBootstrapPhase(c); err != nil { + return err + } + } + + return nil +} + +// runKubeletWaitBootstrapPhase waits for the kubelet to finish its TLS bootstrap process. +// This process is executed by the kubelet and completes with the node joining the cluster +// with a dedicates set of credentials as required by the node authorizer. +func runKubeletWaitBootstrapPhase(c workflow.RunData) (returnErr error) { + data, ok := c.(JoinData) + if !ok { + return errors.New("kubelet-start phase invoked with an invalid data struct") + } + cfg := data.Cfg() + initCfg, err := data.InitCfg() + if err != nil { + return err + } + + bootstrapKubeConfigFile := filepath.Join(data.KubeConfigDir(), kubeadmconstants.KubeletBootstrapKubeConfigFileName) + // Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk + defer func() { + _ = os.Remove(bootstrapKubeConfigFile) + }() + // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process // times out, display a somewhat user-friendly message. diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 525ef3fc7ba..62593fe156b 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -38,6 +38,8 @@ const ( EtcdLearnerMode = "EtcdLearnerMode" // WaitForAllControlPlaneComponents is expected to be alpha in v1.30 WaitForAllControlPlaneComponents = "WaitForAllControlPlaneComponents" + // ControlPlaneKubeletLocalMode is expected to be in alpha in v1.31, beta in v1.32 + ControlPlaneKubeletLocalMode = "ControlPlaneKubeletLocalMode" ) // InitFeatureGates are the default feature gates for the init command @@ -53,6 +55,7 @@ var InitFeatureGates = FeatureList{ }, EtcdLearnerMode: {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}}, WaitForAllControlPlaneComponents: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + ControlPlaneKubeletLocalMode: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, } // Feature represents a feature being gated