From 7074f28dd2a79eada6833b17abe4f2796753a5cd Mon Sep 17 00:00:00 2001 From: Sandeep Rajan Date: Tue, 29 Oct 2019 11:01:25 -0400 Subject: [PATCH 1/2] retain corefile when migration fails --- cmd/kubeadm/app/phases/addons/dns/dns.go | 7 ++++++- cmd/kubeadm/app/phases/upgrade/preflight.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/kubeadm/app/phases/addons/dns/dns.go b/cmd/kubeadm/app/phases/addons/dns/dns.go index 39e8cf9e401..25cf9ca6a41 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns.go @@ -238,7 +238,12 @@ func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, clien } if IsCoreDNSConfigMapMigrationRequired(corefile) { if err := migrateCoreDNSCorefile(client, coreDNSConfigMap, corefile, currentInstalledCoreDNSVersion); err != nil { - return err + // Errors in Corefile Migration is verified during preflight checks. This part will be executed when a user has chosen + // to ignore preflight check errors. + klog.Warningf("the CoreDNS Configuration was not migrated: %v. The existing CoreDNS Corefile configuration has been retained.", err) + if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { + return err + } } } else { if err := apiclient.CreateOrUpdateConfigMap(client, coreDNSConfigMap); err != nil { diff --git a/cmd/kubeadm/app/phases/upgrade/preflight.go b/cmd/kubeadm/app/phases/upgrade/preflight.go index 0720b06a14f..d0ffe819e3d 100644 --- a/cmd/kubeadm/app/phases/upgrade/preflight.go +++ b/cmd/kubeadm/app/phases/upgrade/preflight.go @@ -108,7 +108,7 @@ func checkMigration(client clientset.Interface) error { _, err = migration.Migrate(currentInstalledCoreDNSversion, kubeadmconstants.CoreDNSVersion, corefile, false) if err != nil { - return err + return errors.Wrap(err, "the CoreDNS configuration will not be migrated, and may be incompatible with the upgraded version of CoreDNS") } return nil } From 16191db3535cb6a892194f7fc8483f0588aa58ed Mon Sep 17 00:00:00 2001 From: Sandeep Rajan Date: Wed, 6 Nov 2019 10:55:54 -0500 Subject: [PATCH 2/2] skip deployment update if migration fails --- cmd/kubeadm/app/phases/addons/dns/dns.go | 16 +++++++++++++--- cmd/kubeadm/app/phases/upgrade/preflight.go | 2 +- cmd/kubeadm/app/util/apiclient/idempotency.go | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cmd/kubeadm/app/phases/addons/dns/dns.go b/cmd/kubeadm/app/phases/addons/dns/dns.go index 25cf9ca6a41..814a993dbbc 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns.go @@ -236,10 +236,13 @@ func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, clien if err != nil { return errors.Wrap(err, "unable to fetch CoreDNS current installed version and ConfigMap.") } + + var hasCoreDNSMigrationFailed bool if IsCoreDNSConfigMapMigrationRequired(corefile) { if err := migrateCoreDNSCorefile(client, coreDNSConfigMap, corefile, currentInstalledCoreDNSVersion); err != nil { // Errors in Corefile Migration is verified during preflight checks. This part will be executed when a user has chosen // to ignore preflight check errors. + hasCoreDNSMigrationFailed = true klog.Warningf("the CoreDNS Configuration was not migrated: %v. The existing CoreDNS Corefile configuration has been retained.", err) if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { return err @@ -286,9 +289,16 @@ func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, clien return errors.Wrapf(err, "%s Deployment", unableToDecodeCoreDNS) } - // Create the Deployment for CoreDNS or update it in case it already exists - if err := apiclient.CreateOrUpdateDeployment(client, coreDNSDeployment); err != nil { - return err + // Create the deployment for CoreDNS or retain it in case the CoreDNS migration has failed during upgrade + if hasCoreDNSMigrationFailed { + if err := apiclient.CreateOrRetainDeployment(client, coreDNSDeployment, kubeadmconstants.CoreDNSDeploymentName); err != nil { + return err + } + } else { + // Create the Deployment for CoreDNS or update it in case it already exists + if err := apiclient.CreateOrUpdateDeployment(client, coreDNSDeployment); err != nil { + return err + } } coreDNSService := &v1.Service{} diff --git a/cmd/kubeadm/app/phases/upgrade/preflight.go b/cmd/kubeadm/app/phases/upgrade/preflight.go index d0ffe819e3d..e7b50c93fcb 100644 --- a/cmd/kubeadm/app/phases/upgrade/preflight.go +++ b/cmd/kubeadm/app/phases/upgrade/preflight.go @@ -108,7 +108,7 @@ func checkMigration(client clientset.Interface) error { _, err = migration.Migrate(currentInstalledCoreDNSversion, kubeadmconstants.CoreDNSVersion, corefile, false) if err != nil { - return errors.Wrap(err, "the CoreDNS configuration will not be migrated, and may be incompatible with the upgraded version of CoreDNS") + return errors.Wrap(err, "CoreDNS will not be upgraded") } return nil } diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index 40900ee52e5..4ae951db3c5 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -147,6 +147,21 @@ func CreateOrUpdateDeployment(client clientset.Interface, deploy *apps.Deploymen return nil } +// CreateOrRetainDeployment creates a Deployment if the target resource doesn't exist. If the resource exists already, this function will retain the resource instead. +func CreateOrRetainDeployment(client clientset.Interface, deploy *apps.Deployment, deployName string) error { + if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Get(deployName, metav1.GetOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + return nil + } + if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Create(deploy); err != nil { + if !apierrors.IsAlreadyExists(err) { + return errors.Wrap(err, "unable to create deployment") + } + } + } + return nil +} + // CreateOrUpdateDaemonSet creates a DaemonSet if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateDaemonSet(client clientset.Interface, ds *apps.DaemonSet) error { if _, err := client.AppsV1().DaemonSets(ds.ObjectMeta.Namespace).Create(ds); err != nil {