Merge pull request #85837 from pickledrick/coredns-replicas

kubeadm: persist dns replica count on upgrade

cmd/kubeadm/app/phases/addons/dns/dns.go

@@ -25,9 +25,8 @@ import (
 	"github.com/caddyserver/caddy/caddyfile"
 	"github.com/coredns/corefile-migration/migration"
 	"github.com/pkg/errors"
-
 	apps "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -52,6 +51,8 @@ const (
 	kubeDNSUpstreamNameservers = "upstreamNameservers"
 	kubeDNSFederation          = "federations"
 	unableToDecodeCoreDNS      = "unable to decode CoreDNS"
+	coreDNSReplicas            = 2
+	kubeDNSReplicas            = 1
 )
 
 // DeployedDNSAddon returns the type of DNS addon currently deployed
@@ -80,15 +81,40 @@ func DeployedDNSAddon(client clientset.Interface) (kubeadmapi.DNSAddOnType, stri
 	}
 }
 
+// deployedDNSReplicas returns the replica count for the current DNS deployment
+func deployedDNSReplicas(client clientset.Interface, replicas int32) (*int32, error) {
+	deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem)
+	deployments, err := deploymentsClient.List(metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"})
+	if err != nil {
+		return &replicas, errors.Wrap(err, "couldn't retrieve DNS addon deployments")
+	}
+	switch len(deployments.Items) {
+	case 0:
+		return &replicas, nil
+	case 1:
+		return deployments.Items[0].Spec.Replicas, nil
+	default:
+		return &replicas, errors.Errorf("multiple DNS addon deployments found: %v", deployments.Items)
+	}
+}
+
 // EnsureDNSAddon creates the kube-dns or CoreDNS addon
 func EnsureDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
 	if cfg.DNS.Type == kubeadmapi.CoreDNS {
-		return coreDNSAddon(cfg, client)
+		replicas, err := deployedDNSReplicas(client, coreDNSReplicas)
+		if err != nil {
+			return err
 		}
-	return kubeDNSAddon(cfg, client)
+		return coreDNSAddon(cfg, client, replicas)
+	}
+	replicas, err := deployedDNSReplicas(client, kubeDNSReplicas)
+	if err != nil {
+		return err
+	}
+	return kubeDNSAddon(cfg, client, replicas)
 }
 
-func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
+func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error {
 	if err := CreateServiceAccount(client); err != nil {
 		return err
 	}
@@ -108,7 +134,10 @@ func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interfa
 	}
 
 	dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment,
-		struct{ DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string }{
+		struct {
+			DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string
+			Replicas                                                                                                             *int32
+		}{
 			DeploymentName:       kubeadmconstants.KubeDNSDeploymentName,
 			KubeDNSImage:         images.GetDNSImage(cfg, kubeadmconstants.KubeDNSKubeDNSImageName),
 			DNSMasqImage:         images.GetDNSImage(cfg, kubeadmconstants.KubeDNSDnsMasqNannyImageName),
@@ -117,6 +146,7 @@ func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interfa
 			DNSProbeAddr:         dnsProbeAddr,
 			DNSDomain:            cfg.Networking.DNSDomain,
 			ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleMaster,
+			Replicas:             replicas,
 		})
 	if err != nil {
 		return errors.Wrap(err, "error when parsing kube-dns deployment template")
@@ -162,12 +192,16 @@ func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.I
 	return createDNSService(kubednsService, serviceBytes, client)
 }
 
-func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
+func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error {
 	// Get the YAML manifest
-	coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct{ DeploymentName, Image, ControlPlaneTaintKey string }{
+	coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct {
+		DeploymentName, Image, ControlPlaneTaintKey string
+		Replicas                                    *int32
+	}{
 		DeploymentName:       kubeadmconstants.CoreDNSDeploymentName,
 		Image:                images.GetDNSImage(cfg, kubeadmconstants.CoreDNSImageName),
 		ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleMaster,
+		Replicas:             replicas,
 	})
 	if err != nil {
 		return errors.Wrap(err, "error when parsing CoreDNS deployment template")

cmd/kubeadm/app/phases/addons/dns/dns_test.go

@@ -21,7 +21,7 @@ import (
 	"testing"
 
 	apps "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -91,6 +91,7 @@ func TestCreateServiceAccount(t *testing.T) {
 }
 
 func TestCompileManifests(t *testing.T) {
+	replicas := int32(coreDNSReplicas)
 	var tests = []struct {
 		name     string
 		manifest string
@@ -99,7 +100,10 @@ func TestCompileManifests(t *testing.T) {
 		{
 			name:     "KubeDNSDeployment manifest",
 			manifest: KubeDNSDeployment,
-			data: struct{ DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string }{
+			data: struct {
+				DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string
+				Replicas                                                                                                             *int32
+			}{
 				DeploymentName:       "foo",
 				KubeDNSImage:         "foo",
 				DNSMasqImage:         "foo",
@@ -108,6 +112,7 @@ func TestCompileManifests(t *testing.T) {
 				DNSProbeAddr:         "foo",
 				DNSDomain:            "foo",
 				ControlPlaneTaintKey: "foo",
+				Replicas:             &replicas,
 			},
 		},
 		{
@@ -120,10 +125,14 @@ func TestCompileManifests(t *testing.T) {
 		{
 			name:     "CoreDNSDeployment manifest",
 			manifest: CoreDNSDeployment,
-			data: struct{ DeploymentName, Image, ControlPlaneTaintKey string }{
+			data: struct {
+				DeploymentName, Image, ControlPlaneTaintKey string
+				Replicas                                    *int32
+			}{
 				DeploymentName:       "foo",
 				Image:                "foo",
 				ControlPlaneTaintKey: "foo",
+				Replicas:             &replicas,
 			},
 		},
 		{
@@ -561,6 +570,7 @@ func TestTranslateFederationKubeDNSToCoreDNS(t *testing.T) {
 }
 
 func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) {
+	replicas := int32(coreDNSReplicas)
 	testCases := []struct {
 		name     string
 		manifest string
@@ -569,7 +579,10 @@ func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) {
 		{
 			name:     "KubeDNSDeployment",
 			manifest: KubeDNSDeployment,
-			data: struct{ DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string }{
+			data: struct {
+				DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string
+				Replicas                                                                                                             *int32
+			}{
 				DeploymentName:       "foo",
 				KubeDNSImage:         "foo",
 				DNSMasqImage:         "foo",
@@ -578,16 +591,21 @@ func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) {
 				DNSProbeAddr:         "foo",
 				DNSDomain:            "foo",
 				ControlPlaneTaintKey: "foo",
+				Replicas:             &replicas,
 			},
 		},
 		{
 			name:     "CoreDNSDeployment",
 			manifest: CoreDNSDeployment,
-			data: struct{ DeploymentName, Image, ControlPlaneTaintKey, CoreDNSConfigMapName string }{
+			data: struct {
+				DeploymentName, Image, ControlPlaneTaintKey, CoreDNSConfigMapName string
+				Replicas                                                          *int32
+			}{
 				DeploymentName:       "foo",
 				Image:                "foo",
 				ControlPlaneTaintKey: "foo",
 				CoreDNSConfigMapName: "foo",
+				Replicas:             &replicas,
 			},
 		},
 	}

cmd/kubeadm/app/phases/addons/dns/manifests.go

@@ -27,10 +27,7 @@ metadata:
   labels:
     k8s-app: kube-dns
 spec:
-  # replicas: not specified here:
+  replicas: {{ .Replicas }}
-  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
-  # 2. Default is 1.
-  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
   strategy:
     rollingUpdate:
       maxSurge: 10%
@@ -223,7 +220,7 @@ metadata:
   labels:
     k8s-app: kube-dns
 spec:
-  replicas: 2
+  replicas: {{ .Replicas }}
   strategy:
     type: RollingUpdate
     rollingUpdate: