From a1693052070e588401ca620b17bb206112543cfe Mon Sep 17 00:00:00 2001 From: Xianglin Gao Date: Thu, 7 May 2020 17:31:01 +0800 Subject: [PATCH] kubeadm: add pull images check in upgrade apply and upgrade node Signed-off-by: Xianglin Gao --- cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD | 4 + .../app/cmd/phases/upgrade/node/data.go | 2 + .../app/cmd/phases/upgrade/node/preflight.go | 73 +++++++++++++++++++ cmd/kubeadm/app/cmd/upgrade/BUILD | 1 + cmd/kubeadm/app/cmd/upgrade/apply.go | 30 +++++++- cmd/kubeadm/app/cmd/upgrade/node.go | 63 ++++++++++------ 6 files changed, 148 insertions(+), 25 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD b/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD index dfd82184355..f5d74dcac4a 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD @@ -6,6 +6,7 @@ go_library( "controlplane.go", "data.go", "kubeletconfig.go", + "preflight.go", ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node", visibility = ["//visibility:public"], @@ -17,11 +18,14 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/upgrade:go_default_library", + "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/dryrun:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go index 6e2c33604ff..27878ad4308 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go @@ -17,6 +17,7 @@ limitations under the License. package node import ( + "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) @@ -31,5 +32,6 @@ type Data interface { Cfg() *kubeadmapi.InitConfiguration IsControlPlaneNode() bool Client() clientset.Interface + IgnorePreflightErrors() sets.String KustomizeDir() string } diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go new file mode 100644 index 00000000000..4bcfc5aa58a --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go @@ -0,0 +1,73 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + + "github.com/pkg/errors" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + utilsexec "k8s.io/utils/exec" +) + +// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join +func NewPreflightPhase() workflow.Phase { + return workflow.Phase{ + Name: "preflight", + Short: "Run upgrade node pre-flight checks", + Long: "Run pre-flight checks for kubeadm upgrade node.", + Run: runPreflight, + InheritFlags: []string{ + options.IgnorePreflightErrors, + }, + } +} + +// runPreflight executes preflight checks logic. +func runPreflight(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("preflight phase invoked with an invalid data struct") + } + fmt.Println("[preflight] Running pre-flight checks") + + // First, check if we're root separately from the other preflight checks and fail fast + if err := preflight.RunRootCheckOnly(data.IgnorePreflightErrors()); err != nil { + return err + } + + // if this is a control-plane node, pull the basic images + if data.IsControlPlaneNode() { + if !data.DryRun() { + fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster") + fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection") + fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'") + if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil { + return err + } + } else { + fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')") + } + } else { + fmt.Println("[preflight] Skipping prepull. Not a control plane node.") + return nil + } + + return nil +} diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD index a8ea89fc999..a54783b6427 100644 --- a/cmd/kubeadm/app/cmd/upgrade/BUILD +++ b/cmd/kubeadm/app/cmd/upgrade/BUILD @@ -42,6 +42,7 @@ go_library( "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index b9ab8bc5fe4..f2fc89b7ccb 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -18,9 +18,11 @@ package upgrade import ( "fmt" + "time" "github.com/pkg/errors" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog" @@ -28,9 +30,15 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + utilsexec "k8s.io/utils/exec" +) + +const ( + defaultImagePullTimeout = 15 * time.Minute ) // applyFlags holds the information about the flags that can be passed to apply @@ -42,6 +50,7 @@ type applyFlags struct { dryRun bool etcdUpgrade bool renewCerts bool + imagePullTimeout time.Duration kustomizeDir string } @@ -53,9 +62,10 @@ func (f *applyFlags) sessionIsInteractive() bool { // NewCmdApply returns the cobra command for `kubeadm upgrade apply` func NewCmdApply(apf *applyPlanFlags) *cobra.Command { flags := &applyFlags{ - applyPlanFlags: apf, - etcdUpgrade: true, - renewCerts: true, + applyPlanFlags: apf, + imagePullTimeout: defaultImagePullTimeout, + etcdUpgrade: true, + renewCerts: true, } cmd := &cobra.Command{ @@ -80,6 +90,9 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command { cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output what actions would be performed.") cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.") cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") + cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.") + // TODO: The flag was deprecated in 1.19; remove the flag following a GA deprecation policy of 12 months or 2 releases (whichever is longer) + cmd.Flags().MarkDeprecated("image-pull-timeout", "This flag is deprecated and will be removed in a future version.") options.AddKustomizePodsFlag(cmd.Flags(), &flags.kustomizeDir) return cmd @@ -136,6 +149,17 @@ func runApply(flags *applyFlags, userVersion string) error { } } + if !flags.dryRun { + fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster") + fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection") + fmt.Println("[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'") + if err := preflight.RunPullImagesCheck(utilsexec.New(), cfg, sets.NewString(cfg.NodeRegistration.IgnorePreflightErrors...)); err != nil { + return err + } + } else { + fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')") + } + waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout) // Now; perform the upgrade procedure diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index db53d184f4c..ee404130ad9 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -23,8 +23,10 @@ import ( "github.com/spf13/cobra" flag "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" @@ -36,12 +38,13 @@ import ( // Please note that this structure includes the public kubeadm config API, but only a subset of the options // supported by this api will be exposed as a flag. type nodeOptions struct { - kubeConfigPath string - kubeletVersion string - etcdUpgrade bool - renewCerts bool - dryRun bool - kustomizeDir string + kubeConfigPath string + kubeletVersion string + etcdUpgrade bool + renewCerts bool + dryRun bool + kustomizeDir string + ignorePreflightErrors []string } // compile-time assert that the local data object satisfies the phases data interface. @@ -50,14 +53,15 @@ var _ phases.Data = &nodeData{} // nodeData defines all the runtime information used when running the kubeadm upgrade node worklow; // this data is shared across all the phases that are included in the workflow. type nodeData struct { - etcdUpgrade bool - renewCerts bool - dryRun bool - kubeletVersion string - cfg *kubeadmapi.InitConfiguration - isControlPlaneNode bool - client clientset.Interface - kustomizeDir string + etcdUpgrade bool + renewCerts bool + dryRun bool + kubeletVersion string + cfg *kubeadmapi.InitConfiguration + isControlPlaneNode bool + client clientset.Interface + kustomizeDir string + ignorePreflightErrors sets.String } // NewCmdNode returns the cobra command for `kubeadm upgrade node` @@ -80,6 +84,7 @@ func NewCmdNode() *cobra.Command { options.AddKustomizePodsFlag(cmd.Flags(), &nodeOptions.kustomizeDir) // initialize the workflow runner with the list of phases + nodeRunner.AppendPhase(phases.NewPreflightPhase()) nodeRunner.AppendPhase(phases.NewControlPlane()) nodeRunner.AppendPhase(phases.NewKubeletConfigPhase()) @@ -113,6 +118,7 @@ func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) { flagSet.MarkDeprecated(options.KubeletVersion, "This flag is deprecated and will be removed in a future version.") flagSet.BoolVar(&nodeOptions.renewCerts, options.CertificateRenewal, nodeOptions.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") flagSet.BoolVar(&nodeOptions.etcdUpgrade, options.EtcdUpgrade, nodeOptions.etcdUpgrade, "Perform the upgrade of etcd.") + flagSet.StringSliceVar(&nodeOptions.ignorePreflightErrors, options.IgnorePreflightErrors, nodeOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.") } // newNodeData returns a new nodeData struct to be used for the execution of the kubeadm upgrade node workflow. @@ -140,15 +146,23 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) + if err != nil { + return nil, err + } + // Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + return &nodeData{ - etcdUpgrade: options.etcdUpgrade, - renewCerts: options.renewCerts, - dryRun: options.dryRun, - kubeletVersion: options.kubeletVersion, - cfg: cfg, - client: client, - isControlPlaneNode: isControlPlaneNode, - kustomizeDir: options.kustomizeDir, + etcdUpgrade: options.etcdUpgrade, + renewCerts: options.renewCerts, + dryRun: options.dryRun, + kubeletVersion: options.kubeletVersion, + cfg: cfg, + client: client, + isControlPlaneNode: isControlPlaneNode, + kustomizeDir: options.kustomizeDir, + ignorePreflightErrors: ignorePreflightErrorsSet, }, nil } @@ -191,3 +205,8 @@ func (d *nodeData) Client() clientset.Interface { func (d *nodeData) KustomizeDir() string { return d.kustomizeDir } + +// IgnorePreflightErrors returns the list of preflight errors to ignore. +func (d *nodeData) IgnorePreflightErrors() sets.String { + return d.ignorePreflightErrors +}