diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index b47a7cc7eda..042f10796b6 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -151,7 +151,6 @@ func runApply(flags *applyFlags, args []string) error { waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout) // Now; perform the upgrade procedure - klog.V(1).Infoln("[upgrade/apply] performing upgrade") if err := PerformControlPlaneUpgrade(flags, client, waiter, cfg); err != nil { return errors.Wrap(err, "[upgrade/apply] FATAL") } @@ -222,7 +221,8 @@ func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Vers func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error { // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster - fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", internalcfg.KubernetesVersion) + fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n", + internalcfg.KubernetesVersion, upgrade.UpgradeManifestTimeout) if flags.dryRun { return upgrade.DryRunStaticPodUpgrade(flags.patchesDir, internalcfg) diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index 4375a0115d8..524a624c34f 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -321,7 +321,7 @@ func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientse beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeRegistration.Name, constants.Etcd) if err != nil { - return true, errors.Wrap(err, "failed to get etcd pod's hash") + return true, err } // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go index 6eee846d529..ebac2c95d3c 100644 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ b/cmd/kubeadm/app/util/apiclient/wait.go @@ -185,18 +185,19 @@ func (w *KubeWaiter) SetTimeout(timeout time.Duration) { func (w *KubeWaiter) WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error) { componentHash := "" - var err error + var err, lastErr error mirrorPodHashes := map[string]string{} for _, component := range kubeadmconstants.ControlPlaneComponents { err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { componentHash, err = getStaticPodSingleHash(w.client, nodeName, component) if err != nil { + lastErr = err return false, nil } return true, nil }) if err != nil { - return nil, err + return nil, lastErr } mirrorPodHashes[component] = componentHash } @@ -208,27 +209,34 @@ func (w *KubeWaiter) WaitForStaticPodControlPlaneHashes(nodeName string) (map[st func (w *KubeWaiter) WaitForStaticPodSingleHash(nodeName string, component string) (string, error) { componentPodHash := "" - var err error + var err, lastErr error err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { componentPodHash, err = getStaticPodSingleHash(w.client, nodeName, component) if err != nil { + lastErr = err return false, nil } return true, nil }) + if err != nil { + err = lastErr + } return componentPodHash, err } // WaitForStaticPodHashChange blocks until it timeouts or notices that the Mirror Pod (for the Static Pod, respectively) has changed // This implicitly means this function blocks until the kubelet has restarted the Static Pod in question func (w *KubeWaiter) WaitForStaticPodHashChange(nodeName, component, previousHash string) error { - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - + var err, lastErr error + err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { hash, err := getStaticPodSingleHash(w.client, nodeName, component) if err != nil { + lastErr = err return false, nil } + // Set lastErr to nil to be able to later distinguish between getStaticPodSingleHash() and timeout errors + lastErr = nil // We should continue polling until the UID changes if hash == previousHash { return false, nil @@ -236,6 +244,15 @@ func (w *KubeWaiter) WaitForStaticPodHashChange(nodeName, component, previousHas return true, nil }) + + // If lastError is not nil, this must be a getStaticPodSingleHash() error, else if err is not nil there was a poll timeout + if lastErr != nil { + return lastErr + } + if err != nil { + return errors.Wrapf(err, "static Pod hash for component %s on Node %s did not change after %v", component, nodeName, w.timeout) + } + return nil } // getStaticPodSingleHash computes hashes for a single Static Pod resource @@ -244,11 +261,10 @@ func getStaticPodSingleHash(client clientset.Interface, nodeName string, compone staticPodName := fmt.Sprintf("%s-%s", component, nodeName) staticPod, err := client.CoreV1().Pods(metav1.NamespaceSystem).Get(context.TODO(), staticPodName, metav1.GetOptions{}) if err != nil { - return "", err + return "", errors.Wrapf(err, "failed to obtain static Pod hash for component %s on Node %s", component, nodeName) } staticPodHash := staticPod.Annotations["kubernetes.io/config.hash"] - fmt.Printf("Static pod: %s hash: %s\n", staticPodName, staticPodHash) return staticPodHash, nil }