diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 2cad0226185..4391cfe6447 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -32,6 +32,21 @@ import ( type MasterConfiguration struct { metav1.TypeMeta + // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. + // After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap + // that is used by `kubeadm upgrade` for instance. + + // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. + // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature + BootstrapTokens []BootstrapToken + + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions + + // Cluster-wide configuration + // TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes + // one cluster. Eventually we want this kind of spec to align well with the Cluster API spec. + // API holds configuration for the k8s apiserver. API API // KubeProxy holds configuration for the k8s service proxy. @@ -45,13 +60,6 @@ type MasterConfiguration struct { // KubernetesVersion is the target version of the control plane. KubernetesVersion string - // NodeRegistration holds fields that relate to registering the new master node to the cluster - NodeRegistration NodeRegistrationOptions - - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - // This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature - BootstrapTokens []BootstrapToken - // APIServerExtraArgs is a set of extra flags to pass to the API Server or override // default ones in form of =. // TODO: This is temporary and ideally we would like to switch all components to @@ -141,10 +149,10 @@ type NodeRegistrationOptions struct { // empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration. Taints []v1.Taint - // ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap // Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - ExtraArgs map[string]string + KubeletExtraArgs map[string]string } // Networking contains elements describing cluster's networking configuration. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go index 9d755d5f8ed..b68184150ca 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go @@ -30,6 +30,21 @@ import ( type MasterConfiguration struct { metav1.TypeMeta `json:",inline"` + // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. + // After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap + // that is used by `kubeadm upgrade` for instance. These fields must be omitempty. + + // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. + // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature + BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"` + + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` + + // Cluster-wide configuration + // TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes + // one cluster. Eventually we want this kind of spec to align well with the Cluster API spec. + // API holds configuration for the k8s apiserver. API API `json:"api"` // KubeProxy holds configuration for the k8s service proxy. @@ -41,16 +56,9 @@ type MasterConfiguration struct { // Networking holds configuration for the networking topology of the cluster. Networking Networking `json:"networking"` - // NodeRegistration holds fields that relate to registering the new master node to the cluster - NodeRegistration NodeRegistrationOptions `json:"nodeRegistration"` - // KubernetesVersion is the target version of the control plane. KubernetesVersion string `json:"kubernetesVersion"` - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - // This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature - BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"` - // APIServerExtraArgs is a set of extra flags to pass to the API Server or override // default ones in form of =. // TODO: This is temporary and ideally we would like to switch all components to @@ -133,10 +141,10 @@ type NodeRegistrationOptions struct { // empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration. Taints []v1.Taint `json:"taints,omitempty"` - // ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap // Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - ExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` + KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` } // Networking contains elements describing cluster's networking configuration diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 637d2a6fc0c..e3c3aafd9eb 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -277,24 +277,35 @@ type Init struct { // Run executes master node provisioning, including certificates, needed static pod manifests, etc. func (i *Init) Run(out io.Writer) error { - // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, - // as we handle that ourselves in the markmaster phase - // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? - if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, false); err != nil { - return err - } - - // Try to start the kubelet service in case it's inactive - glog.V(1).Infof("Starting kubelet") - preflight.TryStartKubelet(i.ignorePreflightErrors) - // Get directories to write files to; can be faked if we're dry-running glog.V(1).Infof("[init] Getting certificates directory from configuration") realCertsDir := i.cfg.CertificatesDir - certsDirToWriteTo, kubeConfigDir, manifestDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir) + certsDirToWriteTo, kubeConfigDir, manifestDir, kubeletDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir) if err != nil { return fmt.Errorf("error getting directories to use: %v", err) } + + // First off, configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet + // Try to stop the kubelet service so no race conditions occur when configuring it + glog.V(1).Infof("Stopping the kubelet") + preflight.TryStopKubelet(i.ignorePreflightErrors) + + // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, + // as we handle that ourselves in the markmaster phase + // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? + if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, i.cfg.FeatureGates, false, kubeletDir); err != nil { + return fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err) + } + + // Write the kubelet configuration file to disk. + if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletDir); err != nil { + return fmt.Errorf("error writing kubelet configuration to disk: %v", err) + } + + // Try to start the kubelet service in case it's inactive + glog.V(1).Infof("Starting the kubelet") + preflight.TryStartKubelet(i.ignorePreflightErrors) + // certsDirToWriteTo is gonna equal cfg.CertificatesDir in the normal case, but gonna be a temp directory if dryrunning i.cfg.CertificatesDir = certsDirToWriteTo @@ -359,16 +370,6 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error printing files on dryrun: %v", err) } - kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) - if err != nil { - return err - } - - // Write the kubelet configuration to disk. - if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig); err != nil { - return fmt.Errorf("error writing kubelet configuration to disk: %v", err) - } - // Create a kubernetes client and wait for the API server to be healthy (if not dryrunning) glog.V(1).Infof("creating Kubernetes client") client, err := createClient(i.cfg, i.dryRun) @@ -426,9 +427,13 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error uploading crisocket: %v", err) } - // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - // This feature is disabled by default, as it is alpha still + // This feature is disabled by default if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) + if err != nil { + return err + } + // Enable dynamic kubelet configuration for the node. if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeRegistration.Name, kubeletVersion); err != nil { return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err) @@ -544,17 +549,17 @@ func createClient(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.I // getDirectoriesToUse returns the (in order) certificates, kubeconfig and Static Pod manifest directories, followed by a possible error // This behaves differently when dry-running vs the normal flow -func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, error) { +func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, string, error) { if dryRun { dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun") if err != nil { - return "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err) + return "", "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err) } // Use the same temp dir for all - return dryRunDir, dryRunDir, dryRunDir, nil + return dryRunDir, dryRunDir, dryRunDir, dryRunDir, nil } - return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), nil + return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubeletRunDirectory, nil } // printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup @@ -569,11 +574,19 @@ func printFilesIfDryRunning(dryRun bool, manifestDir string) error { // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests files := []dryrunutil.FileToPrint{} + // Print static pod manifests for _, component := range kubeadmconstants.MasterComponents { realPath := kubeadmconstants.GetStaticPodFilepath(component, manifestDir) outputPath := kubeadmconstants.GetStaticPodFilepath(component, kubeadmconstants.GetStaticPodDirectory()) files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) } + // Print kubelet config manifests + kubeletConfigFiles := []string{kubeadmconstants.KubeletConfigurationFileName, kubeadmconstants.KubeletEnvFileName} + for _, filename := range kubeletConfigFiles { + realPath := filepath.Join(manifestDir, filename) + outputPath := filepath.Join(kubeadmconstants.KubeletRunDirectory, filename) + files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) + } return dryrunutil.PrintDryRunFiles(files, os.Stdout) } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 00fdd5e9942..4c1f32c180a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -215,7 +215,8 @@ func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight *bo // Join defines struct used by kubeadm join command type Join struct { - cfg *kubeadmapi.NodeConfiguration + cfg *kubeadmapi.NodeConfiguration + ignorePreflightErrors sets.String } // NewJoin instantiates Join struct with given arguments @@ -239,39 +240,31 @@ func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha2.NodeC return nil, err } - // Try to start the kubelet service in case it's inactive - glog.V(1).Infoln("[preflight] starting kubelet service if it's inactive") - preflight.TryStartKubelet(ignorePreflightErrors) - - return &Join{cfg: internalcfg}, nil + return &Join{cfg: internalcfg, ignorePreflightErrors: ignorePreflightErrors}, nil } // Run executes worker node provisioning and tries to join an existing cluster. func (j *Join) Run(out io.Writer) error { - // Write env file with flags for the kubelet to use. Also register taints - if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, true); err != nil { - return err - } - + // Perform the Discovery, which turns a Bootstrap Token and optionally (and preferably) a CA cert hash into a KubeConfig + // file that may be used for the TLS Bootstrapping process the kubelet performs using the Certificates API. glog.V(1).Infoln("[join] retrieving KubeConfig objects") cfg, err := discovery.For(j.cfg) if err != nil { return err } - kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName) + bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() // Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk - glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", kubeconfigFile) - if err := kubeconfigutil.WriteToDisk(kubeconfigFile, cfg); err != nil { + glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile) + if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, cfg); err != nil { return fmt.Errorf("couldn't save bootstrap-kubelet.conf to disk: %v", err) } // Write the ca certificate to disk so kubelet can use it for authentication cluster := cfg.Contexts[cfg.CurrentContext].Cluster - err = certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData) - if err != nil { + if err := certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData); err != nil { return fmt.Errorf("couldn't save the CA certificate to disk: %v", err) } @@ -280,12 +273,31 @@ func (j *Join) Run(out io.Writer) error { return err } - // Write the configuration for the kubelet down to disk so the kubelet can start - if err := kubeletphase.DownloadConfig(kubeconfigFile, kubeletVersion); err != nil { + bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile) + if err != nil { + return fmt.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) + } + + // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet + // Try to stop the kubelet service so no race conditions occur when configuring it + glog.V(1).Infof("Stopping the kubelet") + preflight.TryStopKubelet(j.ignorePreflightErrors) + + // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start + if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil { return err } - // Now the kubelet will perform the TLS Bootstrap, transforming bootstrap-kubeconfig.conf to kubeconfig.conf in /etc/kubernetes + // Write env file with flags for the kubelet to use. Also register taints + if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, true, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + + // Try to start the kubelet service in case it's inactive + glog.V(1).Infof("Starting the kubelet") + preflight.TryStartKubelet(j.ignorePreflightErrors) + + // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf KubeConfig file. If this process // times out, display a somewhat user-friendly message. waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout) @@ -295,8 +307,7 @@ func (j *Join) Run(out io.Writer) error { } // When we know the /etc/kubernetes/kubelet.conf file is available, get the client - kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) - client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) if err != nil { return err } @@ -306,9 +317,7 @@ func (j *Join) Run(out io.Writer) error { return fmt.Errorf("error uploading crisocket: %v", err) } - // NOTE: the "--dynamic-config-dir" flag should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf for this to work - // This feature is disabled by default, as it is alpha still - glog.V(1).Infoln("[join] enabling dynamic kubelet configuration") + // This feature is disabled by default in kubeadm if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeRegistration.Name, kubeletVersion); err != nil { return fmt.Errorf("error consuming base kubelet configuration: %v", err) diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go new file mode 100644 index 00000000000..7aa8d82f98c --- /dev/null +++ b/cmd/kubeadm/app/cmd/options/generic.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import "github.com/spf13/pflag" + +// AddKubeConfigFlag adds the --kubeconfig flag to the given flagset +func AddKubeConfigFlag(fs *pflag.FlagSet, kubeConfigFile *string) { + fs.StringVar(kubeConfigFile, "kubeconfig", *kubeConfigFile, "The KubeConfig file to use when talking to the cluster") +} + +// AddConfigFlag adds the --config flag to the given flagset +func AddConfigFlag(fs *pflag.FlagSet, cfgPath *string) { + fs.StringVar(cfgPath, "config", *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") +} diff --git a/cmd/kubeadm/app/cmd/phases/kubelet.go b/cmd/kubeadm/app/cmd/phases/kubelet.go index bd979d60492..f18171370e0 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet.go @@ -18,45 +18,81 @@ package phases import ( "fmt" - "os" + "io/ioutil" "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" "k8s.io/kubernetes/pkg/util/version" + utilsexec "k8s.io/utils/exec" +) + +const ( + // TODO: Figure out how to get these constants from the API machinery + masterConfig = "MasterConfiguration" + nodeConfig = "NodeConfiguration" ) var ( - kubeletWriteConfigToDiskLongDesc = normalizer.LongDesc(` - Writes kubelet configuration to disk, either based on the kubelet-config-1.X ConfigMap in the cluster, or from the - configuration passed to the command via "--config". + kubeletWriteEnvFileLongDesc = normalizer.LongDesc(` + Writes an environment file with flags that should be passed to the kubelet executing on the master or node. + This --config flag can either consume a MasterConfiguration object or a NodeConfiguration one, as this + function is used for both "kubeadm init" and "kubeadm join". ` + cmdutil.AlphaDisclaimer) - kubeletWriteConfigToDiskExample = normalizer.Examples(` - # Writes kubelet configuration for a node to disk. The information is fetched from the cluster ConfigMap - kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --kubeconfig /etc/kubernetes/kubelet.conf + kubeletWriteEnvFileExample = normalizer.Examples(` + # Writes a dynamic environment file with kubelet flags from a MasterConfiguration file. + kubeadm alpha phase kubelet write-env-file --config masterconfig.yaml - # Writes kubelet configuration down to disk, based on the configuration flag passed to --config - kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --config kubeadm.yaml + # Writes a dynamic environment file with kubelet flags from a NodeConfiguration file. + kubeadm alpha phase kubelet write-env-file --config nodeConfig.yaml `) - kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(` + kubeletConfigUploadLongDesc = normalizer.LongDesc(` Uploads kubelet configuration extracted from the kubeadm MasterConfiguration object to a ConfigMap - of the form kubelet-config-1.X in the cluster, where X is the minor version of the current Kubernetes version + of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version. ` + cmdutil.AlphaDisclaimer) - kubeletUploadDynamicConfigExample = normalizer.Examples(` + kubeletConfigUploadExample = normalizer.Examples(` # Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster. - kubeadm alpha phase kubelet upload-config --config kubeadm.yaml + kubeadm alpha phase kubelet config upload --config kubeadm.yaml `) - kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(` + kubeletConfigDownloadLongDesc = normalizer.LongDesc(` + Downloads the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, + where X is the minor version of the kubelet. Either kubeadm autodetects the kubelet version by exec-ing + "kubelet --version" or respects the --kubelet-version parameter. + ` + cmdutil.AlphaDisclaimer) + + kubeletConfigDownloadExample = normalizer.Examples(` + # Downloads the kubelet configuration from the ConfigMap in the cluster. Autodetects the kubelet version. + kubeadm alpha phase kubelet config download + + # Downloads the kubelet configuration from the ConfigMap in the cluster. Uses a specific desired kubelet version. + kubeadm alpha phase kubelet config download --kubelet-version v1.11.0 + `) + + kubeletConfigWriteToDiskLongDesc = normalizer.LongDesc(` + Writes kubelet configuration to disk, based on the kubeadm configuration passed via "--config". + ` + cmdutil.AlphaDisclaimer) + + kubeletConfigWriteToDiskExample = normalizer.Examples(` + # Extracts the kubelet configuration from a kubeadm configuration file + kubeadm alpha phase kubelet config write-to-disk --config kubeadm.yaml + `) + + kubeletConfigEnableDynamicLongDesc = normalizer.LongDesc(` Enables or updates dynamic kubelet configuration for a Node, against the kubelet-config-1.X ConfigMap in the cluster, where X is the minor version of the desired kubelet version. @@ -65,7 +101,7 @@ var ( ` + cmdutil.AlphaDisclaimer) - kubeletEnableDynamicConfigExample = normalizer.Examples(` + kubeletConfigEnableDynamicExample = normalizer.Examples(` # Enables dynamic kubelet configuration for a Node. kubeadm alpha phase kubelet enable-dynamic-config --node-name node-1 --kubelet-version v1.11.0 @@ -74,32 +110,110 @@ var ( `) ) -// NewCmdKubelet returns main command for Kubelet phase +// NewCmdKubelet returns command for `kubeadm phase kubelet` func NewCmdKubelet() *cobra.Command { - var kubeConfigFile string cmd := &cobra.Command{ Use: "kubelet", + Short: "Commands related to handling the kubelet.", + Long: cmdutil.MacroCommandLongDescription, + } + + cmd.AddCommand(NewCmdKubeletConfig()) + cmd.AddCommand(NewCmdKubeletWriteEnvFile()) + return cmd +} + +// NewCmdKubeletWriteEnvFile calls cobra.Command for writing the dynamic kubelet env file based on a MasterConfiguration or NodeConfiguration object +func NewCmdKubeletWriteEnvFile() *cobra.Command { + var cfgPath string + + cmd := &cobra.Command{ + Use: "write-env-file", + Short: "Writes an environment file with runtime flags for the kubelet.", + Long: kubeletWriteEnvFileLongDesc, + Example: kubeletWriteEnvFileExample, + Run: func(cmd *cobra.Command, args []string) { + err := RunKubeletWriteEnvFile(cfgPath) + kubeadmutil.CheckErr(err) + }, + } + + options.AddConfigFlag(cmd.Flags(), &cfgPath) + return cmd +} + +// RunKubeletWriteEnvFile is the function that is run when "kubeadm phase kubelet write-env-file" is executed +func RunKubeletWriteEnvFile(cfgPath string) error { + b, err := ioutil.ReadFile(cfgPath) + if err != nil { + return err + } + + gvk, err := kubeadmutil.GroupVersionKindFromBytes(b, kubeadmscheme.Codecs) + if err != nil { + return err + } + + var nodeRegistrationObj *kubeadmapi.NodeRegistrationOptions + var featureGates map[string]bool + var registerWithTaints bool + switch gvk.Kind { + case masterConfig: + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) + if err != nil { + return err + } + nodeRegistrationObj = &internalcfg.NodeRegistration + featureGates = internalcfg.FeatureGates + registerWithTaints = false + case nodeConfig: + internalcfg, err := configutil.NodeConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.NodeConfiguration{}) + if err != nil { + return err + } + nodeRegistrationObj = &internalcfg.NodeRegistration + featureGates = internalcfg.FeatureGates + registerWithTaints = true + default: + if err != nil { + return fmt.Errorf("Didn't recognize type with GroupVersionKind: %v", gvk) + } + } + if nodeRegistrationObj == nil { + return fmt.Errorf("couldn't load nodeRegistration field from config file") + } + + if err := kubeletphase.WriteKubeletDynamicEnvFile(nodeRegistrationObj, featureGates, registerWithTaints, constants.KubeletRunDirectory); err != nil { + return fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err) + } + return nil +} + +// NewCmdKubeletConfig returns command for `kubeadm phase kubelet config` +func NewCmdKubeletConfig() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", Short: "Handles kubelet configuration.", Long: cmdutil.MacroCommandLongDescription, } - cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster") - - cmd.AddCommand(NewCmdKubeletWriteConfigToDisk(&kubeConfigFile)) - cmd.AddCommand(NewCmdKubeletUploadConfig(&kubeConfigFile)) - cmd.AddCommand(NewCmdKubeletEnableDynamicConfig(&kubeConfigFile)) + cmd.AddCommand(NewCmdKubeletConfigUpload()) + cmd.AddCommand(NewCmdKubeletConfigDownload()) + cmd.AddCommand(NewCmdKubeletConfigWriteToDisk()) + cmd.AddCommand(NewCmdKubeletConfigEnableDynamic()) return cmd } -// NewCmdKubeletUploadConfig calls cobra.Command for uploading dynamic kubelet configuration -func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { +// NewCmdKubeletConfigUpload calls cobra.Command for uploading dynamic kubelet configuration +func NewCmdKubeletConfigUpload() *cobra.Command { var cfgPath string + kubeConfigFile := constants.GetAdminKubeConfigPath() cmd := &cobra.Command{ - Use: "upload-config", - Short: "Uploads kubelet configuration to a ConfigMap", - Long: kubeletUploadDynamicConfigLongDesc, - Example: kubeletUploadDynamicConfigExample, + Use: "upload", + Short: "Uploads kubelet configuration to a ConfigMap based on a kubeadm MasterConfiguration file.", + Long: kubeletConfigUploadLongDesc, + Example: kubeletConfigUploadExample, Run: func(cmd *cobra.Command, args []string) { if len(cfgPath) == 0 { kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required")) @@ -109,7 +223,7 @@ func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) kubeadmutil.CheckErr(err) - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) err = kubeletphase.CreateConfigMap(internalcfg, client) @@ -117,50 +231,83 @@ func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { }, } - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) + options.AddConfigFlag(cmd.Flags(), &cfgPath) return cmd } -// NewCmdKubeletWriteConfigToDisk calls cobra.Command for writing init kubelet configuration -func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command { - var cfgPath, kubeletVersionStr string +// NewCmdKubeletConfigDownload calls cobra.Command for downloading the kubelet configuration from the kubelet-config-1.X ConfigMap in the cluster +func NewCmdKubeletConfigDownload() *cobra.Command { + var kubeletVersionStr string + // TODO: Be smarter about this and be able to load multiple kubeconfig files in different orders of precedence + kubeConfigFile := constants.GetKubeletKubeConfigPath() + cmd := &cobra.Command{ - Use: "write-config-to-disk", - Short: "Writes kubelet configuration to disk, either based on the --config argument or the kubeadm-config ConfigMap.", - Long: kubeletWriteConfigToDiskLongDesc, - Example: kubeletWriteConfigToDiskExample, + Use: "download", + Short: "Downloads the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet.", + Long: kubeletConfigDownloadLongDesc, + Example: kubeletConfigDownloadExample, Run: func(cmd *cobra.Command, args []string) { - if len(kubeletVersionStr) == 0 { - kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required")) - } - - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeletVersion, err := getKubeletVersion(kubeletVersionStr) kubeadmutil.CheckErr(err) - // This call returns the ready-to-use configuration based on the configuration file - internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath) + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) - err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig) + err = kubeletphase.DownloadConfig(client, kubeletVersion, constants.KubeletRunDirectory) kubeadmutil.CheckErr(err) }, } - cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet") - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) + cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet. Defaults to being autodetected from 'kubelet --version'.") return cmd } -// NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node +func getKubeletVersion(kubeletVersionStr string) (*version.Version, error) { + if len(kubeletVersionStr) > 0 { + return version.ParseSemantic(kubeletVersionStr) + } + return preflight.GetKubeletVersion(utilsexec.New()) +} + +// NewCmdKubeletConfigWriteToDisk calls cobra.Command for writing init kubelet configuration +func NewCmdKubeletConfigWriteToDisk() *cobra.Command { + var cfgPath string + cmd := &cobra.Command{ + Use: "write-to-disk", + Short: "Writes kubelet configuration to disk, either based on the --config argument.", + Long: kubeletConfigWriteToDiskLongDesc, + Example: kubeletConfigWriteToDiskExample, + Run: func(cmd *cobra.Command, args []string) { + if len(cfgPath) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required")) + } + + // This call returns the ready-to-use configuration based on the configuration file + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) + kubeadmutil.CheckErr(err) + + err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, constants.KubeletRunDirectory) + kubeadmutil.CheckErr(err) + }, + } + + options.AddConfigFlag(cmd.Flags(), &cfgPath) + return cmd +} + +// NewCmdKubeletConfigEnableDynamic calls cobra.Command for enabling dynamic kubelet configuration on node // This feature is still in alpha and an experimental state -func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { +func NewCmdKubeletConfigEnableDynamic() *cobra.Command { var nodeName, kubeletVersionStr string + kubeConfigFile := constants.GetAdminKubeConfigPath() cmd := &cobra.Command{ - Use: "enable-dynamic-config", + Use: "enable-dynamic", Short: "EXPERIMENTAL: Enables or updates dynamic kubelet configuration for a Node", - Long: kubeletEnableDynamicConfigLongDesc, - Example: kubeletEnableDynamicConfigExample, + Long: kubeletConfigEnableDynamicLongDesc, + Example: kubeletConfigEnableDynamicExample, Run: func(cmd *cobra.Command, args []string) { if len(nodeName) == 0 { kubeadmutil.CheckErr(fmt.Errorf("The --node-name argument is required")) @@ -172,7 +319,7 @@ func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) kubeadmutil.CheckErr(err) - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) err = kubeletphase.EnableDynamicConfigForNode(client, nodeName, kubeletVersion) @@ -180,6 +327,7 @@ func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { }, } + options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) cmd.Flags().StringVar(&nodeName, "node-name", nodeName, "Name of the node that should enable the dynamic kubelet configuration") cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet") return cmd diff --git a/cmd/kubeadm/app/cmd/phases/kubelet_test.go b/cmd/kubeadm/app/cmd/phases/kubelet_test.go index 89159d26105..4abfdd5d7cb 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet_test.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet_test.go @@ -25,11 +25,12 @@ import ( ) func TestKubeletSubCommandsHasFlags(t *testing.T) { - kubeConfigFile := "foo" subCmds := []*cobra.Command{ - NewCmdKubeletUploadConfig(&kubeConfigFile), - NewCmdKubeletWriteConfigToDisk(&kubeConfigFile), - NewCmdKubeletEnableDynamicConfig(&kubeConfigFile), + NewCmdKubeletWriteEnvFile(), + NewCmdKubeletConfigUpload(), + NewCmdKubeletConfigDownload(), + NewCmdKubeletConfigWriteToDisk(), + NewCmdKubeletConfigEnableDynamic(), } commonFlags := []string{} @@ -39,21 +40,35 @@ func TestKubeletSubCommandsHasFlags(t *testing.T) { additionalFlags []string }{ { - command: "upload-config", + command: "write-env-file", additionalFlags: []string{ "config", }, }, { - command: "write-config-to-disk", + command: "upload", additionalFlags: []string{ + "kubeconfig", + "config", + }, + }, + { + command: "download", + additionalFlags: []string{ + "kubeconfig", "kubelet-version", + }, + }, + { + command: "write-to-disk", + additionalFlags: []string{ "config", }, }, { - command: "enable-dynamic-config", + command: "enable-dynamic", additionalFlags: []string{ + "kubeconfig", "node-name", "kubelet-version", }, diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 1f5f2336ca8..23269ec9ddf 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -54,6 +54,12 @@ type upgradeVariables struct { // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion string) (*upgradeVariables, error) { + + // Set the default for the kubeconfig path if the user didn't override with the flags + if flags.kubeConfigPath == "" { + flags.kubeConfigPath = "/etc/kubernetes/admin.conf" + } + client, err := getClient(flags.kubeConfigPath, dryRun) if err != nil { return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.kubeConfigPath, err) @@ -154,7 +160,10 @@ func getClient(file string, dryRun bool) (clientset.Interface, error) { } // Get the fake clientset - fakeclient := apiclient.NewDryRunClient(dryRunGetter, os.Stdout) + dryRunOpts := apiclient.GetDefaultDryRunClientOptions(dryRunGetter, os.Stdout) + // Print GET and LIST requests + dryRunOpts.PrintGETAndLIST = true + fakeclient := apiclient.NewDryRunClientWithOpts(dryRunOpts) // As we know the return of Discovery() of the fake clientset is of type *fakediscovery.FakeDiscovery // we can convert it to that struct. fakeclientDiscovery, ok := fakeclient.Discovery().(*fakediscovery.FakeDiscovery) diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go new file mode 100644 index 00000000000..28415640d73 --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -0,0 +1,162 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" + "k8s.io/kubernetes/pkg/util/normalizer" + "k8s.io/kubernetes/pkg/util/version" +) + +var ( + upgradeNodeConfigLongDesc = normalizer.LongDesc(` + Downloads the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, + where X is the minor version of the kubelet. kubeadm uses the --kubelet-version parameter to determine + what the _desired_ kubelet version is. Give + `) + + upgradeNodeConfigExample = normalizer.Examples(` + # Downloads the kubelet configuration from the ConfigMap in the cluster. Uses a specific desired kubelet version. + kubeadm upgrade node config --kubelet-version v1.11.0 + + # Simulates the downloading of the kubelet configuration from the ConfigMap in the cluster with a specific desired + # version. Does not change any state locally on the node. + kubeadm upgrade node config --kubelet-version v1.11.0 --dry-run + `) +) + +type nodeUpgradeFlags struct { + parent *cmdUpgradeFlags + kubeletVersionStr string + dryRun bool +} + +// NewCmdNode returns the cobra command for `kubeadm upgrade node` +func NewCmdNode(parentFlags *cmdUpgradeFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "node", + Short: "Upgrade commands for a node in the cluster. Currently only supports upgrading the configuration, not the kubelet itself.", + RunE: cmdutil.SubCmdRunE("node"), + } + cmd.AddCommand(NewCmdUpgradeNodeConfig(parentFlags)) + return cmd +} + +// NewCmdUpgradeNodeConfig returns the cobra.Command for downloading the new/upgrading the kubelet configuration from the kubelet-config-1.X +// ConfigMap in the cluster +func NewCmdUpgradeNodeConfig(parentFlags *cmdUpgradeFlags) *cobra.Command { + flags := &nodeUpgradeFlags{ + parent: parentFlags, + kubeletVersionStr: "", + } + + cmd := &cobra.Command{ + Use: "config", + Short: "Downloads the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet.", + Long: upgradeNodeConfigLongDesc, + Example: upgradeNodeConfigExample, + Run: func(cmd *cobra.Command, args []string) { + err := RunUpgradeNodeConfig(flags) + kubeadmutil.CheckErr(err) + }, + } + + // TODO: Unify the registration of common flags + cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output the actions that would be performed.") + cmd.Flags().StringVar(&flags.kubeletVersionStr, "kubelet-version", flags.kubeletVersionStr, "The *desired* version for the kubelet after the upgrade.") + return cmd +} + +// RunUpgradeNodeConfig is executed when `kubeadm upgrade node config` runs. +func RunUpgradeNodeConfig(flags *nodeUpgradeFlags) error { + if len(flags.kubeletVersionStr) == 0 { + return fmt.Errorf("The --kubelet-version argument is required") + } + + // Set up the kubelet directory to use. If dry-running, use a fake directory + kubeletDir, err := getKubeletDir(flags.dryRun) + if err != nil { + return err + } + + // Set the default for the kubeconfig path if the user didn't override with the flags + // TODO: Be smarter about this and be able to load multiple kubeconfig files in different orders of precedence + if flags.parent.kubeConfigPath == "" { + flags.parent.kubeConfigPath = constants.GetKubeletKubeConfigPath() + } + + client, err := getClient(flags.parent.kubeConfigPath, flags.dryRun) + if err != nil { + return fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.parent.kubeConfigPath, err) + } + + // Parse the desired kubelet version + kubeletVersion, err := version.ParseSemantic(flags.kubeletVersionStr) + if err != nil { + return err + } + // TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered + if err := kubeletphase.DownloadConfig(client, kubeletVersion, kubeletDir); err != nil { + return err + } + + // If we're dry-running, print the generated manifests, otherwise do nothing + if err := printFilesIfDryRunning(flags.dryRun, kubeletDir); err != nil { + return fmt.Errorf("error printing files on dryrun: %v", err) + } + + fmt.Println("[upgrade] The configuration for this node was successfully updated!") + fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") + return nil +} + +// getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not. +func getKubeletDir(dryRun bool) (string, error) { + if dryRun { + dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun") + if err != nil { + return "", fmt.Errorf("couldn't create a temporary directory: %v", err) + } + return dryRunDir, nil + } + return constants.KubeletRunDirectory, nil +} + +// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup +func printFilesIfDryRunning(dryRun bool, kubeletDir string) error { + if !dryRun { + return nil + } + + // Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory + fileToPrint := dryrunutil.FileToPrint{ + RealPath: filepath.Join(kubeletDir, constants.KubeletConfigurationFileName), + PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName), + } + return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout) +} diff --git a/cmd/kubeadm/app/cmd/upgrade/upgrade.go b/cmd/kubeadm/app/cmd/upgrade/upgrade.go index a7928aa01bc..ad2c59b0c2d 100644 --- a/cmd/kubeadm/app/cmd/upgrade/upgrade.go +++ b/cmd/kubeadm/app/cmd/upgrade/upgrade.go @@ -74,6 +74,7 @@ func NewCmdUpgrade(out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdApply(flags)) cmd.AddCommand(NewCmdPlan(flags)) cmd.AddCommand(NewCmdDiff(flags)) + cmd.AddCommand(NewCmdNode(flags)) return cmd } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 4b5f8da9731..5ded24e0701 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -205,8 +205,26 @@ const ( // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" - // KubeletConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet - KubeletConfigurationFile = "/var/lib/kubelet/config.yaml" + // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. + // TODO: Make hard-coded "/var/lib/kubelet" strings reference this constant. + KubeletRunDirectory = "/var/lib/kubelet" + + // KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet + // This file should exist under KubeletRunDirectory + KubeletConfigurationFileName = "config.yaml" + + // DynamicKubeletConfigurationDirectoryName specifies the directory which stores the dynamic configuration checkpoints for the kubelet + // This directory should exist under KubeletRunDirectory + DynamicKubeletConfigurationDirectoryName = "dynamic-config" + + // KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain + // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file + // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. + // This file should exist under KubeletRunDirectory + KubeletEnvFileName = "kubeadm-flags.env" + + // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile + KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.2.17" @@ -266,14 +284,6 @@ const ( // TODO: Import this constant from a consts only package, that does not pull any further dependencies. LeaseEndpointReconcilerType = "lease" - // KubeletEnvFile is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain - // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file - // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. - KubeletEnvFile = "/var/lib/kubelet/kubeadm-flags.env" - - // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile - KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" - // KubeDNSVersion is the version of kube-dns to be deployed if it is used KubeDNSVersion = "1.14.10" @@ -350,6 +360,16 @@ func GetAdminKubeConfigPath() string { return filepath.Join(KubernetesDir, AdminKubeConfigFileName) } +// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default +func GetBootstrapKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) +} + +// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default +func GetKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) +} + // AddSelfHostedPrefix adds the self-hosted- prefix to the component name func AddSelfHostedPrefix(componentName string) string { return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName) diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go index b65475ef0ee..7bcec426224 100644 --- a/cmd/kubeadm/app/constants/constants_test.go +++ b/cmd/kubeadm/app/constants/constants_test.go @@ -50,6 +50,32 @@ func TestGetAdminKubeConfigPath(t *testing.T) { } } +func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) { + expected := "/etc/kubernetes/bootstrap-kubelet.conf" + actual := GetBootstrapKubeletKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + +func TestGetKubeletKubeConfigPath(t *testing.T) { + expected := "/etc/kubernetes/kubelet.conf" + actual := GetKubeletKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + func TestGetStaticPodFilepath(t *testing.T) { var tests = []struct { componentName, manifestsDir, expected string diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index e515d649860..723f92a69db 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -19,6 +19,7 @@ package kubelet import ( "fmt" "io/ioutil" + "path/filepath" "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -28,7 +29,6 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" @@ -37,13 +37,13 @@ import ( // WriteConfigToDisk writes the kubelet config object down to a file // Used at "kubeadm init" and "kubeadm upgrade" time -func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error { +func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletDir string) error { kubeletBytes, err := getConfigBytes(kubeletConfig) if err != nil { return err } - return writeConfigBytesToDisk(kubeletBytes) + return writeConfigBytesToDisk(kubeletBytes, kubeletDir) } // CreateConfigMap creates a ConfigMap with the generic kubelet configuration. @@ -120,7 +120,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve // DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk. // Used at "kubeadm join" time -func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error { +func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error { // Download the ConfigMap from the cluster based on what version the kubelet is configMapName := configMapName(kubeletVersion) @@ -128,17 +128,12 @@ func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) e fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n", configMapName, metav1.NamespaceSystem) - client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) - if err != nil { - return fmt.Errorf("couldn't create client from kubeconfig file %q", kubeletKubeConfig) - } - kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) if err != nil { return err } - return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])) + return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir) } // configMapName returns the right ConfigMap name for the right branch of k8s @@ -162,11 +157,12 @@ func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([ } // writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file -func writeConfigBytesToDisk(b []byte) error { - fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", kubeadmconstants.KubeletConfigurationFile) +func writeConfigBytesToDisk(b []byte, kubeletDir string) error { + configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName) + fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", configFile) - if err := ioutil.WriteFile(kubeadmconstants.KubeletConfigurationFile, b, 0644); err != nil { - return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeadmconstants.KubeletConfigurationFile, err) + if err := ioutil.WriteFile(configFile, b, 0644); err != nil { + return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", configFile, err) } return nil } diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 91f876e0c47..51277b81e73 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -27,34 +27,52 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/procfs" utilsexec "k8s.io/utils/exec" ) +type kubeletFlagsOpts struct { + nodeRegOpts *kubeadmapi.NodeRegistrationOptions + featureGates map[string]bool + registerTaintsUsingFlags bool + execer utilsexec.Interface + pidOfFunc func(string) ([]int, error) + defaultHostname string +} + // WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet. // Used at "kubeadm init" and "kubeadm join" time. -func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) error { +func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, featureGates map[string]bool, registerTaintsUsingFlags bool, kubeletDir string) error { - argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(nodeRegOpts, registerTaintsUsingFlags), nodeRegOpts.ExtraArgs) + flagOpts := kubeletFlagsOpts{ + nodeRegOpts: nodeRegOpts, + featureGates: featureGates, + registerTaintsUsingFlags: registerTaintsUsingFlags, + execer: utilsexec.New(), + pidOfFunc: procfs.PidOf, + defaultHostname: nodeutil.GetHostname(""), + } + stringMap := buildKubeletArgMap(flagOpts) + argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeRegOpts.KubeletExtraArgs) envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) - return writeKubeletFlagBytesToDisk([]byte(envFileContent)) + return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir) } // buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags // that should be given to the local kubelet daemon. -func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) map[string]string { +func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string { kubeletFlags := map[string]string{} - if nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { + if opts.nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { // These flags should only be set when running docker kubeletFlags["network-plugin"] = "cni" kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d" kubeletFlags["cni-bin-dir"] = "/opt/cni/bin" - execer := utilsexec.New() - driver, err := kubeadmutil.GetCgroupDriverDocker(execer) + driver, err := kubeadmutil.GetCgroupDriverDocker(opts.execer) if err != nil { glog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err) } else { @@ -62,27 +80,33 @@ func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registe } } else { kubeletFlags["container-runtime"] = "remote" - kubeletFlags["container-runtime-endpoint"] = nodeRegOpts.CRISocket + kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket } - if registerTaintsUsingFlags && nodeRegOpts.Taints != nil && len(nodeRegOpts.Taints) > 0 { + if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 { taintStrs := []string{} - for _, taint := range nodeRegOpts.Taints { + for _, taint := range opts.nodeRegOpts.Taints { taintStrs = append(taintStrs, taint.ToString()) } kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",") } - if pids, _ := procfs.PidOf("systemd-resolved"); len(pids) > 0 { + if pids, _ := opts.pidOfFunc("systemd-resolved"); len(pids) > 0 { // procfs.PidOf only returns an error if the regex is empty or doesn't compile, so we can ignore it kubeletFlags["resolv-conf"] = "/run/systemd/resolve/resolv.conf" } // Make sure the node name we're passed will work with Kubelet - if nodeRegOpts.Name != "" && nodeRegOpts.Name != nodeutil.GetHostname("") { - glog.V(1).Info("setting kubelet hostname-override to %q", nodeRegOpts.Name) - kubeletFlags["hostname-override"] = nodeRegOpts.Name + if opts.nodeRegOpts.Name != "" && opts.nodeRegOpts.Name != opts.defaultHostname { + glog.V(1).Info("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name) + kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name + } + + // If the user enabled Dynamic Kubelet Configuration (which is disabled by default), set the directory + // in the CLI flags so that the feature actually gets enabled + if features.Enabled(opts.featureGates, features.DynamicKubeletConfig) { + kubeletFlags["dynamic-config-dir"] = filepath.Join(constants.KubeletRunDirectory, constants.DynamicKubeletConfigurationDirectoryName) } // TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker @@ -91,15 +115,16 @@ func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registe } // writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file -func writeKubeletFlagBytesToDisk(b []byte) error { - fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", constants.KubeletEnvFile) +func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error { + kubeletEnvFilePath := filepath.Join(kubeletDir, constants.KubeletEnvFileName) + fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", kubeletEnvFilePath) // creates target folder if not already exists - if err := os.MkdirAll(filepath.Dir(constants.KubeletEnvFile), 0700); err != nil { - return fmt.Errorf("failed to create directory %q: %v", filepath.Dir(constants.KubeletEnvFile), err) + if err := os.MkdirAll(kubeletDir, 0700); err != nil { + return fmt.Errorf("failed to create directory %q: %v", kubeletDir, err) } - if err := ioutil.WriteFile(constants.KubeletEnvFile, b, 0644); err != nil { - return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", constants.KubeletEnvFile, err) + if err := ioutil.WriteFile(kubeletEnvFilePath, b, 0644); err != nil { + return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeletEnvFilePath, err) } return nil } diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go index 32646647d43..b50a658df53 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ b/cmd/kubeadm/app/phases/kubelet/flags_test.go @@ -17,45 +17,260 @@ limitations under the License. package kubelet import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "strings" "testing" + "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - nodeutil "k8s.io/kubernetes/pkg/util/node" + "k8s.io/utils/exec" ) +type fakeCmd struct { + b []byte + err error +} + +func (f fakeCmd) Run() error { return f.err } +func (f fakeCmd) CombinedOutput() ([]byte, error) { return f.b, f.err } +func (f fakeCmd) Output() ([]byte, error) { return f.b, f.err } +func (f fakeCmd) SetDir(dir string) {} +func (f fakeCmd) SetStdin(in io.Reader) {} +func (f fakeCmd) SetStdout(out io.Writer) {} +func (f fakeCmd) SetStderr(out io.Writer) {} +func (f fakeCmd) Stop() {} + +type fakeExecer struct { + ioMap map[string]fakeCmd +} + +func (f fakeExecer) Command(cmd string, args ...string) exec.Cmd { + cmds := []string{cmd} + cmds = append(cmds, args...) + return f.ioMap[strings.Join(cmds, " ")] +} +func (f fakeExecer) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { + return f.Command(cmd, args...) +} +func (f fakeExecer) LookPath(file string) (string, error) { return "", errors.New("unknown binary") } + +var ( + systemdCgroupExecer = fakeExecer{ + ioMap: map[string]fakeCmd{ + "docker info": { + b: []byte(`Cgroup Driver: systemd`), + }, + }, + } + + cgroupfsCgroupExecer = fakeExecer{ + ioMap: map[string]fakeCmd{ + "docker info": { + b: []byte(`Cgroup Driver: cgroupfs`), + }, + }, + } + + errCgroupExecer = fakeExecer{ + ioMap: map[string]fakeCmd{ + "docker info": { + err: fmt.Errorf("no such binary: docker"), + }, + }, + } +) + +func binaryRunningPidOfFunc(_ string) ([]int, error) { + return []int{1, 2, 3}, nil +} + +func binaryNotRunningPidOfFunc(_ string) ([]int, error) { + return []int{}, nil +} + func TestBuildKubeletArgMap(t *testing.T) { tests := []struct { - name string - hostname string - expectedHostname string + name string + opts kubeletFlagsOpts + expected map[string]string }{ { - name: "manually set to current hostname", - hostname: nodeutil.GetHostname(""), - expectedHostname: "", + name: "the simplest case", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "foo", + Taints: []v1.Taint{ // This should be ignored as registerTaintsUsingFlags is false + { + Key: "foo", + Value: "bar", + Effect: "baz", + }, + }, + }, + execer: errCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + }, }, { - name: "unset hostname", - hostname: "", - expectedHostname: "", + name: "nodeRegOpts.Name != default hostname", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "override-name", + }, + execer: errCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "default", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + "hostname-override": "override-name", + }, }, { - name: "override hostname", - hostname: "my-node", - expectedHostname: "my-node", + name: "systemd cgroup driver", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "foo", + }, + execer: systemdCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + "cgroup-driver": "systemd", + }, + }, + { + name: "cgroupfs cgroup driver", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "foo", + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + "cgroup-driver": "cgroupfs", + }, + }, + { + name: "external CRI runtime", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + }, + }, + { + name: "register with taints", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + Taints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + Effect: "baz", + }, + { + Key: "key", + Value: "val", + Effect: "eff", + }, + }, + }, + registerTaintsUsingFlags: true, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + "register-with-taints": "foo=bar:baz,key=val:eff", + }, + }, + { + name: "systemd-resolved running", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + "resolv-conf": "/run/systemd/resolve/resolv.conf", + }, + }, + { + name: "dynamic kubelet config enabled", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + }, + featureGates: map[string]bool{ + "DynamicKubeletConfig": true, + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + "dynamic-config-dir": "/var/lib/kubelet/dynamic-config", + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - opts := &kubeadmapi.NodeRegistrationOptions{ - Name: test.hostname, - } - - m := buildKubeletArgMap(opts, false) - if m["hostname-override"] != test.expectedHostname { - t.Errorf("expected hostname %q, got %q", test.expectedHostname, m["hostname-override"]) + actual := buildKubeletArgMap(test.opts) + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf( + "failed buildKubeletArgMap:\n\texpected: %v\n\t actual: %v", + test.expected, + actual, + ) } }) } diff --git a/cmd/kubeadm/app/phases/markmaster/markmaster.go b/cmd/kubeadm/app/phases/markmaster/markmaster.go index bb1ff4210e0..c9d36989fec 100644 --- a/cmd/kubeadm/app/phases/markmaster/markmaster.go +++ b/cmd/kubeadm/app/phases/markmaster/markmaster.go @@ -31,7 +31,11 @@ func MarkMaster(client clientset.Interface, masterName string, taints []v1.Taint fmt.Printf("[markmaster] Marking the node %s as master by adding the label \"%s=''\"\n", masterName, constants.LabelNodeRoleMaster) if taints != nil && len(taints) > 0 { - fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taints) + taintStrs := []string{} + for _, taint := range taints { + taintStrs = append(taintStrs, taint.ToString()) + } + fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taintStrs) } return apiclient.PatchNode(client, masterName, func(n *v1.Node) { diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go index c1c33c2a45d..7445b436167 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -42,6 +42,8 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I // Removes sensitive info from the data that will be stored in the config map externalcfg.BootstrapTokens = nil + // Clear the NodeRegistration object. + externalcfg.NodeRegistration = kubeadmapiv1alpha2.NodeRegistrationOptions{} cfgYaml, err := util.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha2.SchemeGroupVersion, scheme.Codecs) if err != nil { diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 18c0025f638..6b26e73b673 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -1063,15 +1063,41 @@ func TryStartKubelet(ignorePreflightErrors sets.String) { // If we notice that the kubelet service is inactive, try to start it initSystem, err := initsystem.GetInitSystem() if err != nil { - fmt.Println("[preflight] no supported init system detected, won't ensure kubelet is running.") - } else if initSystem.ServiceExists("kubelet") { + fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet is running properly.") + return + } - fmt.Println("[preflight] Activating the kubelet service") - // This runs "systemctl daemon-reload && systemctl restart kubelet" - if err := initSystem.ServiceRestart("kubelet"); err != nil { - glog.Warningf("[preflight] unable to start the kubelet service: [%v]\n", err) - glog.Warningf("[preflight] please ensure kubelet is running manually.") - } + if !initSystem.ServiceExists("kubelet") { + fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet is running properly.") + } + + fmt.Println("[preflight] Activating the kubelet service") + // This runs "systemctl daemon-reload && systemctl restart kubelet" + if err := initSystem.ServiceRestart("kubelet"); err != nil { + fmt.Printf("[preflight] WARNING: unable to start the kubelet service: [%v]\n", err) + fmt.Printf("[preflight] please ensure kubelet is reloaded and running manually.\n") + } +} + +// TryStopKubelet attempts to bring down the kubelet service momentarily +func TryStopKubelet(ignorePreflightErrors sets.String) { + if setHasItemOrAll(ignorePreflightErrors, "StopKubelet") { + return + } + // If we notice that the kubelet service is inactive, try to start it + initSystem, err := initsystem.GetInitSystem() + if err != nil { + fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.") + return + } + + if !initSystem.ServiceExists("kubelet") { + fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.") + } + + // This runs "systemctl daemon-reload && systemctl stop kubelet" + if err := initSystem.ServiceStop("kubelet"); err != nil { + fmt.Printf("[preflight] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err) } } diff --git a/cmd/kubeadm/app/util/apiclient/dryrunclient.go b/cmd/kubeadm/app/util/apiclient/dryrunclient.go index 6c0d796e6fa..4d4a52d706a 100644 --- a/cmd/kubeadm/app/util/apiclient/dryrunclient.go +++ b/cmd/kubeadm/app/util/apiclient/dryrunclient.go @@ -55,6 +55,18 @@ type DryRunClientOptions struct { PrintGETAndLIST bool } +// GetDefaultDryRunClientOptions returns the default DryRunClientOptions values +func GetDefaultDryRunClientOptions(drg DryRunGetter, w io.Writer) DryRunClientOptions { + return DryRunClientOptions{ + Writer: w, + Getter: drg, + PrependReactors: []core.Reactor{}, + AppendReactors: []core.Reactor{}, + MarshalFunc: DefaultMarshalFunc, + PrintGETAndLIST: false, + } +} + // actionWithName is the generic interface for an action that has a name associated with it // This just makes it easier to catch all actions that has a name; instead of hard-coding all request that has it associated type actionWithName interface { @@ -71,14 +83,7 @@ type actionWithObject interface { // NewDryRunClient is a wrapper for NewDryRunClientWithOpts using some default values func NewDryRunClient(drg DryRunGetter, w io.Writer) clientset.Interface { - return NewDryRunClientWithOpts(DryRunClientOptions{ - Writer: w, - Getter: drg, - PrependReactors: []core.Reactor{}, - AppendReactors: []core.Reactor{}, - MarshalFunc: DefaultMarshalFunc, - PrintGETAndLIST: false, - }) + return NewDryRunClientWithOpts(GetDefaultDryRunClientOptions(drg, w)) } // NewDryRunClientWithOpts returns a clientset.Interface that can be used normally for talking to the Kubernetes API. diff --git a/cmd/kubeadm/app/util/apiclient/init_dryrun.go b/cmd/kubeadm/app/util/apiclient/init_dryrun.go index b21ec7810d0..875ff3f6aee 100644 --- a/cmd/kubeadm/app/util/apiclient/init_dryrun.go +++ b/cmd/kubeadm/app/util/apiclient/init_dryrun.go @@ -61,6 +61,7 @@ func (idr *InitDryRunGetter) HandleGetAction(action core.GetAction) (bool, runti idr.handleGetNode, idr.handleSystemNodesClusterRoleBinding, idr.handleGetBootstrapToken, + idr.handleGetKubeDNSConfigMap, } for _, f := range funcs { handled, obj, err := f(action) @@ -157,3 +158,20 @@ func (idr *InitDryRunGetter) handleGetBootstrapToken(action core.GetAction) (boo // We can safely return a NotFound error here as the code will just proceed normally and create the Bootstrap Token return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "secret not found") } + +// handleGetKubeDNSConfigMap handles the case where kubeadm init will try to read the kube-dns ConfigMap in the cluster +// in order to transform information there to core-dns configuration. We can safely return an empty configmap here +func (idr *InitDryRunGetter) handleGetKubeDNSConfigMap(action core.GetAction) (bool, runtime.Object, error) { + if !strings.HasPrefix(action.GetName(), "kube-dns") || action.GetNamespace() != metav1.NamespaceSystem || action.GetResource().Resource != "configmaps" { + // We can't handle this event + return false, nil, nil + } + // We can safely return an empty configmap here, as we don't have any kube-dns specific config to convert to coredns config + return true, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-dns", + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{}, + }, nil +} diff --git a/pkg/util/initsystem/initsystem.go b/pkg/util/initsystem/initsystem.go index aca7a41b860..6638bab5eb2 100644 --- a/pkg/util/initsystem/initsystem.go +++ b/pkg/util/initsystem/initsystem.go @@ -44,15 +44,26 @@ type InitSystem interface { type SystemdInitSystem struct{} +func (sysd SystemdInitSystem) reloadSystemd() error { + if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { + return fmt.Errorf("failed to reload systemd: %v", err) + } + return nil +} + func (sysd SystemdInitSystem) ServiceStart(service string) error { + // Before we try to start any service, make sure that systemd is ready + if err := sysd.reloadSystemd(); err != nil { + return err + } args := []string{"start", service} - err := exec.Command("systemctl", args...).Run() - return err + return exec.Command("systemctl", args...).Run() } func (sysd SystemdInitSystem) ServiceRestart(service string) error { - if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { - return fmt.Errorf("failed to reload systemd: %v", err) + // Before we try to restart any service, make sure that systemd is ready + if err := sysd.reloadSystemd(); err != nil { + return err } args := []string{"restart", service} return exec.Command("systemctl", args...).Run() @@ -60,8 +71,7 @@ func (sysd SystemdInitSystem) ServiceRestart(service string) error { func (sysd SystemdInitSystem) ServiceStop(service string) error { args := []string{"stop", service} - err := exec.Command("systemctl", args...).Run() - return err + return exec.Command("systemctl", args...).Run() } func (sysd SystemdInitSystem) ServiceExists(service string) bool {