kubeadm: add pull images check in upgrade apply and upgrade node

Signed-off-by: Xianglin Gao <xianglin.gxl@alibaba-inc.com>

cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD

@@ -6,6 +6,7 @@ go_library(
         "controlplane.go",
         "data.go",
         "kubeletconfig.go",
+        "preflight.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node",
     visibility = ["//visibility:public"],
@@ -17,11 +18,14 @@ go_library(
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/upgrade:go_default_library",
+        "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/dryrun:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )
 

cmd/kubeadm/app/cmd/phases/upgrade/node/data.go

@@ -17,6 +17,7 @@ limitations under the License.
 package node
 
 import (
+	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 )
@@ -31,5 +32,6 @@ type Data interface {
 	Cfg() *kubeadmapi.InitConfiguration
 	IsControlPlaneNode() bool
 	Client() clientset.Interface
+	IgnorePreflightErrors() sets.String
 	KustomizeDir() string
 }

cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
+	utilsexec "k8s.io/utils/exec"
+)
+
+// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join
+func NewPreflightPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "preflight",
+		Short: "Run upgrade node pre-flight checks",
+		Long:  "Run pre-flight checks for kubeadm upgrade node.",
+		Run:   runPreflight,
+		InheritFlags: []string{
+			options.IgnorePreflightErrors,
+		},
+	}
+}
+
+// runPreflight executes preflight checks logic.
+func runPreflight(c workflow.RunData) error {
+	data, ok := c.(Data)
+	if !ok {
+		return errors.New("preflight phase invoked with an invalid data struct")
+	}
+	fmt.Println("[preflight] Running pre-flight checks")
+
+	// First, check if we're root separately from the other preflight checks and fail fast
+	if err := preflight.RunRootCheckOnly(data.IgnorePreflightErrors()); err != nil {
+		return err
+	}
+
+	// if this is a control-plane node, pull the basic images
+	if data.IsControlPlaneNode() {
+		if !data.DryRun() {
+			fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
+			fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
+			fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
+			if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
+				return err
+			}
+		} else {
+			fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
+		}
+	} else {
+		fmt.Println("[preflight] Skipping prepull. Not a control plane node.")
+		return nil
+	}
+
+	return nil
+}

cmd/kubeadm/app/cmd/upgrade/BUILD

@@ -42,6 +42,7 @@ go_library(
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )
 

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -18,9 +18,11 @@ package upgrade
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
@@ -28,9 +30,15 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	utilsexec "k8s.io/utils/exec"
+)
+
+const (
+	defaultImagePullTimeout = 15 * time.Minute
 )
 
 // applyFlags holds the information about the flags that can be passed to apply
@@ -42,6 +50,7 @@ type applyFlags struct {
 	dryRun             bool
 	etcdUpgrade        bool
 	renewCerts         bool
+	imagePullTimeout   time.Duration
 	kustomizeDir       string
 }
 
@@ -53,9 +62,10 @@ func (f *applyFlags) sessionIsInteractive() bool {
 // NewCmdApply returns the cobra command for `kubeadm upgrade apply`
 func NewCmdApply(apf *applyPlanFlags) *cobra.Command {
 	flags := &applyFlags{
-		applyPlanFlags: apf,
-		etcdUpgrade:    true,
-		renewCerts:     true,
+		applyPlanFlags:   apf,
+		imagePullTimeout: defaultImagePullTimeout,
+		etcdUpgrade:      true,
+		renewCerts:       true,
 	}
 
 	cmd := &cobra.Command{
@@ -80,6 +90,9 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command {
 	cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output what actions would be performed.")
 	cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.")
 	cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
+	cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.")
+	// TODO: The flag was deprecated in 1.19; remove the flag following a GA deprecation policy of 12 months or 2 releases (whichever is longer)
+	cmd.Flags().MarkDeprecated("image-pull-timeout", "This flag is deprecated and will be removed in a future version.")
 	options.AddKustomizePodsFlag(cmd.Flags(), &flags.kustomizeDir)
 
 	return cmd
@@ -136,6 +149,17 @@ func runApply(flags *applyFlags, userVersion string) error {
 		}
 	}
 
+	if !flags.dryRun {
+		fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster")
+		fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection")
+		fmt.Println("[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'")
+		if err := preflight.RunPullImagesCheck(utilsexec.New(), cfg, sets.NewString(cfg.NodeRegistration.IgnorePreflightErrors...)); err != nil {
+			return err
+		}
+	} else {
+		fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')")
+	}
+
 	waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout)
 
 	// Now; perform the upgrade procedure

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -23,8 +23,10 @@ import (
 	"github.com/spf13/cobra"
 	flag "github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
@@ -36,12 +38,13 @@ import (
 // Please note that this structure includes the public kubeadm config API, but only a subset of the options
 // supported by this api will be exposed as a flag.
 type nodeOptions struct {
-	kubeConfigPath string
-	kubeletVersion string
-	etcdUpgrade    bool
-	renewCerts     bool
-	dryRun         bool
-	kustomizeDir   string
+	kubeConfigPath        string
+	kubeletVersion        string
+	etcdUpgrade           bool
+	renewCerts            bool
+	dryRun                bool
+	kustomizeDir          string
+	ignorePreflightErrors []string
 }
 
 // compile-time assert that the local data object satisfies the phases data interface.
@@ -50,14 +53,15 @@ var _ phases.Data = &nodeData{}
 // nodeData defines all the runtime information used when running the kubeadm upgrade node worklow;
 // this data is shared across all the phases that are included in the workflow.
 type nodeData struct {
-	etcdUpgrade        bool
-	renewCerts         bool
-	dryRun             bool
-	kubeletVersion     string
-	cfg                *kubeadmapi.InitConfiguration
-	isControlPlaneNode bool
-	client             clientset.Interface
-	kustomizeDir       string
+	etcdUpgrade           bool
+	renewCerts            bool
+	dryRun                bool
+	kubeletVersion        string
+	cfg                   *kubeadmapi.InitConfiguration
+	isControlPlaneNode    bool
+	client                clientset.Interface
+	kustomizeDir          string
+	ignorePreflightErrors sets.String
 }
 
 // NewCmdNode returns the cobra command for `kubeadm upgrade node`
@@ -80,6 +84,7 @@ func NewCmdNode() *cobra.Command {
 	options.AddKustomizePodsFlag(cmd.Flags(), &nodeOptions.kustomizeDir)
 
 	// initialize the workflow runner with the list of phases
+	nodeRunner.AppendPhase(phases.NewPreflightPhase())
 	nodeRunner.AppendPhase(phases.NewControlPlane())
 	nodeRunner.AppendPhase(phases.NewKubeletConfigPhase())
 
@@ -113,6 +118,7 @@ func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) {
 	flagSet.MarkDeprecated(options.KubeletVersion, "This flag is deprecated and will be removed in a future version.")
 	flagSet.BoolVar(&nodeOptions.renewCerts, options.CertificateRenewal, nodeOptions.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
 	flagSet.BoolVar(&nodeOptions.etcdUpgrade, options.EtcdUpgrade, nodeOptions.etcdUpgrade, "Perform the upgrade of etcd.")
+	flagSet.StringSliceVar(&nodeOptions.ignorePreflightErrors, options.IgnorePreflightErrors, nodeOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.")
 }
 
 // newNodeData returns a new nodeData struct to be used for the execution of the kubeadm upgrade node workflow.
@@ -140,15 +146,23 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node
 		return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
 	}
 
+	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
+	if err != nil {
+		return nil, err
+	}
+	// Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration:
+	cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List()
+
 	return &nodeData{
-		etcdUpgrade:        options.etcdUpgrade,
-		renewCerts:         options.renewCerts,
-		dryRun:             options.dryRun,
-		kubeletVersion:     options.kubeletVersion,
-		cfg:                cfg,
-		client:             client,
-		isControlPlaneNode: isControlPlaneNode,
-		kustomizeDir:       options.kustomizeDir,
+		etcdUpgrade:           options.etcdUpgrade,
+		renewCerts:            options.renewCerts,
+		dryRun:                options.dryRun,
+		kubeletVersion:        options.kubeletVersion,
+		cfg:                   cfg,
+		client:                client,
+		isControlPlaneNode:    isControlPlaneNode,
+		kustomizeDir:          options.kustomizeDir,
+		ignorePreflightErrors: ignorePreflightErrorsSet,
 	}, nil
 }
 
@@ -191,3 +205,8 @@ func (d *nodeData) Client() clientset.Interface {
 func (d *nodeData) KustomizeDir() string {
 	return d.kustomizeDir
 }
+
+// IgnorePreflightErrors returns the list of preflight errors to ignore.
+func (d *nodeData) IgnorePreflightErrors() sets.String {
+	return d.ignorePreflightErrors
+}