add the ability to migrate coredns configmap

This commit is contained in:
Sandeep Rajan 2019-05-01 12:56:41 -04:00
parent f2dd24820a
commit 6821d21260
10 changed files with 423 additions and 12 deletions

View File

@ -101,6 +101,7 @@
"AllowedPrefixes": [
"github.com/beorn7/perks/quantile",
"github.com/blang/semver",
"github.com/coredns/corefile-migration/migration",
"github.com/coreos/etcd/auth/authpb",
"github.com/coreos/etcd/clientv3",
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes",

View File

@ -120,7 +120,7 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin
// Ensure the user is root
klog.V(1).Info("running preflight checks")
if err := runPreflightChecks(ignorePreflightErrorsSet); err != nil {
if err := runPreflightChecks(client, ignorePreflightErrorsSet); err != nil {
return nil, nil, nil, err
}
@ -178,9 +178,17 @@ func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer
}
// runPreflightChecks runs the root preflight check
func runPreflightChecks(ignorePreflightErrors sets.String) error {
func runPreflightChecks(client clientset.Interface, ignorePreflightErrors sets.String) error {
fmt.Println("[preflight] Running pre-flight checks.")
return preflight.RunRootCheckOnly(ignorePreflightErrors)
err := preflight.RunRootCheckOnly(ignorePreflightErrors)
if err != nil {
return err
}
err = upgrade.RunCoreDNSMigrationCheck(client, ignorePreflightErrors)
if err != nil {
return err
}
return nil
}
// getClient gets a real or fake client depending on whether the user is dry-running or not

View File

@ -331,7 +331,7 @@ const (
KubeDNSVersion = "1.14.13"
// CoreDNSVersion is the version of CoreDNS to be deployed if it is used
CoreDNSVersion = "1.3.1"
CoreDNSVersion = "1.5.0"
// ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct
ClusterConfigurationKind = "ClusterConfiguration"

View File

@ -47,6 +47,7 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/github.com/caddyserver/caddy/caddyfile:go_default_library",
"//vendor/github.com/coredns/corefile-migration/migration:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",

View File

@ -23,6 +23,7 @@ import (
"strings"
"github.com/caddyserver/caddy/caddyfile"
"github.com/coredns/corefile-migration/migration"
"github.com/pkg/errors"
apps "k8s.io/api/apps/v1"
@ -228,9 +229,19 @@ func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, clien
return errors.Wrapf(err, "%s ConfigMap", unableToDecodeCoreDNS)
}
// Create the ConfigMap for CoreDNS or retain it in case it already exists
if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil {
return err
// Create the ConfigMap for CoreDNS or update/migrate it in case it already exists
_, corefile, currentInstalledCoreDNSVersion, err := GetCoreDNSInfo(client)
if err != nil {
return errors.Wrapf(err, "unable to fetch CoreDNS current installed version and ConfigMap.")
}
if IsCoreDNSConfigMapMigrationRequired(corefile) {
if err := migrateCoreDNSConfigMap(client, coreDNSConfigMap, corefile, currentInstalledCoreDNSVersion); err != nil {
return err
}
} else {
if err := apiclient.CreateOrUpdateConfigMap(client, coreDNSConfigMap); err != nil {
return err
}
}
coreDNSClusterRoles := &rbac.ClusterRole{}
@ -298,6 +309,70 @@ func createDNSService(dnsService *v1.Service, serviceBytes []byte, client client
return nil
}
// IsCoreDNSConfigMapMigrationRequired checks if a migration of the CoreDNS ConfigMap is required.
func IsCoreDNSConfigMapMigrationRequired(corefile string) bool {
if corefile == "" || migration.Default("", corefile) {
return false
}
return true
}
func migrateCoreDNSConfigMap(client clientset.Interface, cm *v1.ConfigMap, corefile, currentInstalledCoreDNSVersion string) error {
// Since the current Configuration present is the not the default version, try and migrate the Config.
updatedCorefile, err := migration.Migrate(currentInstalledCoreDNSVersion, kubeadmconstants.CoreDNSVersion, corefile, false)
if err != nil {
return errors.Wrap(err, "unable to migrate CoreDNS ConfigMap")
}
if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(&v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.CoreDNSConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
"Corefile": updatedCorefile,
"Corefile-previous": corefile,
},
}); err != nil {
return errors.Wrap(err, "unable to update the CoreDNS ConfigMap")
}
fmt.Println("Migrating CoreDNS Corefile")
changes, err := migration.Deprecated(currentInstalledCoreDNSVersion, kubeadmconstants.CoreDNSVersion, corefile)
if err != nil {
return errors.Wrap(err, "unable to get list of changes to the configuration.")
}
// show the migration changes
klog.V(2).Infof("the CoreDNS configuration has been migrated and applied: %v.", updatedCorefile)
klog.V(2).Infoln("the old migration has been saved in the CoreDNS ConfigMap under the name [Corefile-previous]")
klog.V(2).Infoln("The changes in the new CoreDNS Configuration are as follows:")
for _, change := range changes {
klog.V(2).Infof("%v", change.ToString())
}
return nil
}
// GetCoreDNSInfo gets the current CoreDNS installed and the current Corefile Configuration of CoreDNS.
func GetCoreDNSInfo(client clientset.Interface) (*v1.ConfigMap, string, string, error) {
coreDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.CoreDNSConfigMap, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return nil, "", "", err
}
if apierrors.IsNotFound(err) {
return nil, "", "", nil
}
corefile, ok := coreDNSConfigMap.Data["Corefile"]
if !ok {
return nil, "", "", errors.New("unable to find the CoreDNS Corefile data")
}
_, currentCoreDNSversion, err := DeployedDNSAddon(client)
if err != nil {
return nil, "", "", err
}
return coreDNSConfigMap, corefile, currentCoreDNSversion, nil
}
// translateStubDomainOfKubeDNSToForwardCoreDNS translates StubDomain Data in kube-dns ConfigMap
// in the form of Proxy for the CoreDNS Corefile.
func translateStubDomainOfKubeDNSToForwardCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
@ -351,7 +426,7 @@ func translateStubDomainOfKubeDNSToForwardCoreDNS(dataField string, kubeDNSConfi
// in the form of Proxy for the CoreDNS Corefile.
func translateUpstreamNameServerOfKubeDNSToUpstreamForwardCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
if kubeDNSConfigMap == nil {
return "", nil
return "/etc/resolv.conf", nil
}
if upstreamValues, ok := kubeDNSConfigMap.Data[dataField]; ok {

View File

@ -568,10 +568,11 @@ func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) {
{
name: "CoreDNSDeployment",
manifest: CoreDNSDeployment,
data: struct{ DeploymentName, Image, ControlPlaneTaintKey string }{
data: struct{ DeploymentName, Image, ControlPlaneTaintKey, CoreDNSConfigMapName string }{
DeploymentName: "foo",
Image: "foo",
ControlPlaneTaintKey: "foo",
CoreDNSConfigMapName: "foo",
},
},
}
@ -588,3 +589,150 @@ func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) {
})
}
}
func TestCreateCoreDNSConfigMap(t *testing.T) {
tests := []struct {
name string
initialCorefileData string
expectedCorefileData string
coreDNSVersion string
}{
{
name: "Remove Deprecated options",
initialCorefileData: `.:53 {
errors
health
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
prometheus :9153
forward . /etc/resolv.conf
cache 30
loop
reload
loadbalance
}`,
expectedCorefileData: `.:53 {
errors
health
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
prometheus :9153
forward . /etc/resolv.conf
cache 30
loop
reload
loadbalance
ready
}
`,
coreDNSVersion: "1.3.1",
},
{
name: "Update proxy plugin to forward plugin",
initialCorefileData: `.:53 {
errors
health
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
proxy . /etc/resolv.conf
k8s_external example.com
cache 30
loop
reload
loadbalance
}`,
expectedCorefileData: `.:53 {
errors
health
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf
k8s_external example.com
cache 30
loop
reload
loadbalance
ready
}
`,
coreDNSVersion: "1.3.1",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
client := createClientAndCoreDNSManifest(t, tc.initialCorefileData, tc.coreDNSVersion)
// Get the Corefile and installed CoreDNS version.
cm, corefile, currentInstalledCoreDNSVersion, err := GetCoreDNSInfo(client)
if err != nil {
t.Fatalf("unable to fetch CoreDNS current installed version and ConfigMap.")
}
err = migrateCoreDNSConfigMap(client, cm, corefile, currentInstalledCoreDNSVersion)
if err != nil {
t.Fatalf("error creating the CoreDNS ConfigMap: %v", err)
}
migratedConfigMap, _ := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.CoreDNSConfigMap, metav1.GetOptions{})
if !strings.EqualFold(migratedConfigMap.Data["Corefile"], tc.expectedCorefileData) {
t.Fatalf("expected to get %v, but got %v", tc.expectedCorefileData, migratedConfigMap.Data["Corefile"])
}
})
}
}
func createClientAndCoreDNSManifest(t *testing.T, corefile, coreDNSVersion string) *clientsetfake.Clientset {
client := clientsetfake.NewSimpleClientset()
_, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(&v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.CoreDNSConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
"Corefile": corefile,
},
})
if err != nil {
t.Fatalf("error creating ConfigMap: %v", err)
}
_, err = client.AppsV1().Deployments(metav1.NamespaceSystem).Create(&apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.CoreDNSConfigMap,
Namespace: metav1.NamespaceSystem,
Labels: map[string]string{
"k8s-app": "kube-dns",
},
},
Spec: apps.DeploymentSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Image: "test:" + coreDNSVersion,
},
},
},
},
},
})
if err != nil {
t.Fatalf("error creating deployment: %v", err)
}
return client
}

View File

@ -281,8 +281,8 @@ spec:
failureThreshold: 5
readinessProbe:
httpGet:
path: /health
port: 8080
path: /ready
port: 8181
scheme: HTTP
securityContext:
allowPrivilegeEscalation: false
@ -314,9 +314,9 @@ data:
.:53 {
errors
health
ready
kubernetes {{ .DNSDomain }} in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
ttl 30
}{{ .Federation }}

View File

@ -5,6 +5,7 @@ go_library(
srcs = [
"compute.go",
"health.go",
"migrate.go",
"policy.go",
"postupgrade.go",
"prepull.go",
@ -40,11 +41,14 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/github.com/coredns/corefile-migration/migration:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)

View File

@ -0,0 +1,119 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"os"
"github.com/coredns/corefile-migration/migration"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
)
// CoreDNSCheck validates installed kubelet version
type CoreDNSCheck struct {
name string
client clientset.Interface
f func(clientset.Interface) error
}
// Name is part of the preflight.Checker interface
func (c CoreDNSCheck) Name() string {
return c.name
}
// Check is part of the preflight.Checker interface
func (c CoreDNSCheck) Check() (warnings, errors []error) {
if err := c.f(c.client); err != nil {
return nil, []error{err}
}
return nil, nil
}
// RunCoreDNSMigrationCheck initializes checks related to CoreDNS migration.
func RunCoreDNSMigrationCheck(client clientset.Interface, ignorePreflightErrors sets.String) error {
currentDNSType, _, err := dns.DeployedDNSAddon(client)
if err != nil {
return err
}
if currentDNSType != kubeadmapi.CoreDNS {
return nil
}
migrationChecks := []preflight.Checker{
&CoreDNSCheck{
name: "CoreDNSUnsupportedPlugins",
client: client,
f: checkUnsupportedPlugins,
},
&CoreDNSCheck{
name: "CoreDNSMigration",
client: client,
f: checkMigration,
},
}
return preflight.RunChecks(migrationChecks, os.Stderr, ignorePreflightErrors)
}
// checkUnsupportedPlugins checks if there are any plugins included in the current configuration
// that is unsupported for migration.
func checkUnsupportedPlugins(client clientset.Interface) error {
klog.V(1).Infoln("validating if there are any unsupported CoreDNS plugins in the Corefile")
_, corefile, currentInstalledCoreDNSversion, err := dns.GetCoreDNSInfo(client)
if err != nil {
return err
}
unsupportedCoreDNS, err := migration.Unsupported(currentInstalledCoreDNSversion, kubeadmconstants.CoreDNSVersion, corefile)
if err != nil {
return err
}
if unsupportedCoreDNS != nil {
var UnsupportedPlugins, UnsupportedVersion string
for _, unsup := range unsupportedCoreDNS {
UnsupportedPlugins = unsup.Plugin
UnsupportedVersion = unsup.Version
}
if UnsupportedPlugins != "" || UnsupportedVersion != "" {
return errors.New("there are unsupported plugins in the CoreDNS Corefile")
}
}
return nil
}
// checkMigration validates if migration of the current CoreDNS ConfigMap is possible.
func checkMigration(client clientset.Interface) error {
klog.V(1).Infoln("validating if migration can be done for the current CoreDNS release.")
_, corefile, currentInstalledCoreDNSversion, err := dns.GetCoreDNSInfo(client)
if err != nil {
return err
}
_, err = migration.Migrate(currentInstalledCoreDNSversion, kubeadmconstants.CoreDNSVersion, corefile, false)
if err != nil {
return err
}
return nil
}

View File

@ -17,13 +17,16 @@ limitations under the License.
package upgrade
import (
"fmt"
"os"
"path/filepath"
"github.com/pkg/errors"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
errorsutil "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
@ -94,6 +97,17 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
errs = append(errs, err)
}
coreDNSConfigMap, corefile, _, err := dns.GetCoreDNSInfo(client)
if err != nil {
errs = append(errs, err)
}
isMigrationRequired := dns.IsCoreDNSConfigMapMigrationRequired(corefile)
if isMigrationRequired {
if err := prepareCoreDNSForCorefileMigration(client, coreDNSConfigMap, corefile); err != nil {
errs = append(errs, err)
}
}
// Upgrade kube-dns/CoreDNS and kube-proxy
if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil {
errs = append(errs, err)
@ -109,6 +123,47 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
return errorsutil.NewAggregate(errs)
}
func prepareCoreDNSForCorefileMigration(client clientset.Interface, coreDNSConfigMap *v1.ConfigMap, corefile string) error {
existingCoreDNSConfigMapName := kubeadmconstants.CoreDNSConfigMap + "-previous"
if _, err := client.CoreV1().ConfigMaps(coreDNSConfigMap.ObjectMeta.Namespace).Get(existingCoreDNSConfigMapName, metav1.GetOptions{}); err != nil {
if !apierrors.IsNotFound(err) {
return err
}
if err := client.CoreV1().ConfigMaps(coreDNSConfigMap.ObjectMeta.Namespace).Delete(existingCoreDNSConfigMapName, nil); err != nil {
return errors.Wrap(err, "failed to delete previous CoreDNS ConfigMap")
}
if _, err := client.CoreV1().ConfigMaps(coreDNSConfigMap.ObjectMeta.Namespace).Create(&v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: existingCoreDNSConfigMapName,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
"Corefile": corefile,
},
}); err != nil {
return errors.Wrap(err, "unable to create the migrated CoreDNS ConfigMap")
}
}
if err := patchCoreDNSDeployment(client, existingCoreDNSConfigMapName); err != nil {
return err
}
return nil
}
func patchCoreDNSDeployment(client clientset.Interface, coreDNSConfigMapName string) error {
dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(kubeadmconstants.CoreDNSDeploymentName, metav1.GetOptions{})
if err != nil {
return err
}
patch := fmt.Sprintf(`{"spec":{"template": {"spec":{"volumes":[{"name": "config-volume","configMap":{"name": "%s"}}]}}}}`, coreDNSConfigMapName)
if _, err := client.AppsV1().Deployments(dnsDeployment.ObjectMeta.Namespace).Patch(dnsDeployment.Name, types.StrategicMergePatchType, []byte(patch)); err != nil {
return errors.Wrap(err, "unable to patch the CoreDNS deployment")
}
return nil
}
func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, dryRun bool) error {
return apiclient.TryRunCommand(func() error {
installedDeploymentName := kubeadmconstants.KubeDNSDeploymentName