Add CoreDNS as addon

addons/coredns.go

@@ -0,0 +1,8 @@
+package addons
+
+import "github.com/rancher/rke/templates"
+
+func GetCoreDNSManifest(CoreDNSConfig interface{}) (string, error) {
+
+	return templates.CompileTemplateFromMap(templates.CoreDNSTemplate, CoreDNSConfig)
+}

addons/kubedns.go

@@ -2,19 +2,7 @@ package addons
 
 import "github.com/rancher/rke/templates"
 
-const (
+func GetKubeDNSManifest(KubeDNSConfig interface{}) (string, error) {
-	KubeDNSImage           = "KubeDNSImage"
-	DNSMasqImage           = "DNSMasqImage"
-	KubeDNSSidecarImage    = "KubednsSidecarImage"
-	KubeDNSAutoScalerImage = "KubeDNSAutoScalerImage"
-	KubeDNSServer          = "ClusterDNSServer"
-	KubeDNSClusterDomain   = "ClusterDomain"
-	MetricsServerImage     = "MetricsServerImage"
-	RBAC                   = "RBAC"
-	MetricsServerOptions   = "MetricsServerOptions"
-)
 
-func GetKubeDNSManifest(kubeDNSConfig map[string]string) (string, error) {
+	return templates.CompileTemplateFromMap(templates.KubeDNSTemplate, KubeDNSConfig)
-
-	return templates.CompileTemplateFromMap(templates.KubeDNSTemplate, kubeDNSConfig)
 }

cluster/addons.go

@@ -20,7 +20,6 @@ import (
 )
 
 const (
-	KubeDNSAddonResourceName      = "rke-kubedns-addon"
 	UserAddonResourceName         = "rke-user-addon"
 	IngressAddonResourceName      = "rke-ingress-controller"
 	UserAddonsIncludeResourceName = "rke-user-includes-addons"
@@ -31,8 +30,13 @@ const (
 	NginxIngressAddonAppName       = "ingress-nginx"
 	KubeDNSAddonAppName            = "kube-dns"
 	KubeDNSAutoscalerAppName       = "kube-dns-autoscaler"
+	CoreDNSAutoscalerAppName       = "coredns-autoscaler"
+
+	CoreDNSProvider = "coredns"
 )
 
+var DNSProviders = []string{"kubedns", "coredns"}
+
 type ingressOptions struct {
 	RBACConfig     string
 	Options        map[string]string
@@ -50,6 +54,30 @@ type MetricsServerOptions struct {
 	Version            string
 }
 
+type CoreDNSOptions struct {
+	RBACConfig             string
+	CoreDNSImage           string
+	CoreDNSAutoScalerImage string
+	ClusterDomain          string
+	ClusterDNSServer       string
+	ReverseCIDRs           []string
+	UpstreamNameservers    []string
+	NodeSelector           map[string]string
+}
+
+type KubeDNSOptions struct {
+	RBACConfig             string
+	KubeDNSImage           string
+	DNSMasqImage           string
+	KubeDNSAutoScalerImage string
+	KubeDNSSidecarImage    string
+	ClusterDomain          string
+	ClusterDNSServer       string
+	ReverseCIDRs           []string
+	UpstreamNameservers    []string
+	NodeSelector           map[string]string
+}
+
 type addonError struct {
 	err        string
 	isCritical bool
@@ -59,12 +87,18 @@ func (e *addonError) Error() string {
 	return e.err
 }
 
+func getAddonResourceName(addon string) string {
+	AddonResourceName := "rke-" + addon + "-addon"
+	return AddonResourceName
+}
+
 func (c *Cluster) deployK8sAddOns(ctx context.Context) error {
-	if err := c.deployKubeDNS(ctx); err != nil {
+	if err := c.deployDNS(ctx); err != nil {
 		if err, ok := err.(*addonError); ok && err.isCritical {
 			return err
 		}
-		log.Warnf(ctx, "Failed to deploy addon execute job [%s]: %v", KubeDNSAddonResourceName, err)
+		log.Warnf(ctx, "Failed to deploy DNS addon execute job for provider %s: %v", c.DNS.Provider, err)
+
 	}
 	if err := c.deployMetricServer(ctx); err != nil {
 		if err, ok := err.(*addonError); ok && err.isCritical {
@@ -183,24 +217,48 @@ func getAddonFromURL(yamlURL string) ([]byte, error) {
 }
 
 func (c *Cluster) deployKubeDNS(ctx context.Context) error {
-	log.Infof(ctx, "[addons] Setting up KubeDNS")
+	log.Infof(ctx, "[addons] Setting up %s", c.DNS.Provider)
-	kubeDNSConfig := map[string]string{
+	KubeDNSConfig := KubeDNSOptions{
-		addons.KubeDNSServer:          c.ClusterDNSServer,
+		KubeDNSImage:           c.SystemImages.KubeDNS,
-		addons.KubeDNSClusterDomain:   c.ClusterDomain,
+		KubeDNSSidecarImage:    c.SystemImages.KubeDNSSidecar,
-		addons.KubeDNSImage:           c.SystemImages.KubeDNS,
+		KubeDNSAutoScalerImage: c.SystemImages.KubeDNSAutoscaler,
-		addons.DNSMasqImage:           c.SystemImages.DNSmasq,
+		DNSMasqImage:           c.SystemImages.DNSmasq,
-		addons.KubeDNSSidecarImage:    c.SystemImages.KubeDNSSidecar,
+		RBACConfig:             c.Authorization.Mode,
-		addons.KubeDNSAutoScalerImage: c.SystemImages.KubeDNSAutoscaler,
+		ClusterDomain:          c.ClusterDomain,
-		addons.RBAC:                   c.Authorization.Mode,
+		ClusterDNSServer:       c.ClusterDNSServer,
+		UpstreamNameservers:    c.DNS.UpstreamNameservers,
+		ReverseCIDRs:           c.DNS.ReverseCIDRs,
 	}
-	kubeDNSYaml, err := addons.GetKubeDNSManifest(kubeDNSConfig)
+	kubeDNSYaml, err := addons.GetKubeDNSManifest(KubeDNSConfig)
 	if err != nil {
 		return err
 	}
-	if err := c.doAddonDeploy(ctx, kubeDNSYaml, KubeDNSAddonResourceName, false); err != nil {
+	if err := c.doAddonDeploy(ctx, kubeDNSYaml, getAddonResourceName(c.DNS.Provider), false); err != nil {
 		return err
 	}
-	log.Infof(ctx, "[addons] KubeDNS deployed successfully")
+	log.Infof(ctx, "[addons] %s deployed successfully", c.DNS.Provider)
+	return nil
+}
+
+func (c *Cluster) deployCoreDNS(ctx context.Context) error {
+	log.Infof(ctx, "[addons] Setting up %s", c.DNS.Provider)
+	CoreDNSConfig := CoreDNSOptions{
+		CoreDNSImage:           c.SystemImages.CoreDNS,
+		CoreDNSAutoScalerImage: c.SystemImages.CoreDNSAutoscaler,
+		RBACConfig:             c.Authorization.Mode,
+		ClusterDomain:          c.ClusterDomain,
+		ClusterDNSServer:       c.ClusterDNSServer,
+		UpstreamNameservers:    c.DNS.UpstreamNameservers,
+		ReverseCIDRs:           c.DNS.ReverseCIDRs,
+	}
+	coreDNSYaml, err := addons.GetCoreDNSManifest(CoreDNSConfig)
+	if err != nil {
+		return err
+	}
+	if err := c.doAddonDeploy(ctx, coreDNSYaml, getAddonResourceName(c.DNS.Provider), false); err != nil {
+		return err
+	}
+	log.Infof(ctx, "[addons] CoreDNS deployed successfully..")
 	return nil
 }
 
@@ -392,3 +450,55 @@ func (c *Cluster) deployIngress(ctx context.Context) error {
 	log.Infof(ctx, "[ingress] ingress controller %s deployed successfully", c.Ingress.Provider)
 	return nil
 }
+func (c *Cluster) removeDNSProvider(ctx context.Context, dnsprovider string) error {
+	AddonJobExists, err := addons.AddonJobExists(getAddonResourceName(dnsprovider)+"-deploy-job", c.LocalKubeConfigPath, c.K8sWrapTransport)
+	if err != nil {
+		return err
+	}
+	if AddonJobExists {
+		log.Infof(ctx, "[dns] removing DNS provider %s", dnsprovider)
+		if err := c.doAddonDelete(ctx, getAddonResourceName(dnsprovider), false); err != nil {
+			return err
+		}
+
+		log.Infof(ctx, "[dns] DNS provider %s removed successfully", dnsprovider)
+		return nil
+	}
+	return nil
+}
+
+func (c *Cluster) deployDNS(ctx context.Context) error {
+	for _, dnsprovider := range DNSProviders {
+		if strings.EqualFold(dnsprovider, c.DNS.Provider) {
+			continue
+		}
+		if err := c.removeDNSProvider(ctx, dnsprovider); err != nil {
+			return err
+		}
+	}
+	switch DNSProvider := c.DNS.Provider; DNSProvider {
+	case DefaultDNSProvider:
+		if err := c.deployKubeDNS(ctx); err != nil {
+			if err, ok := err.(*addonError); ok && err.isCritical {
+				return err
+			}
+			log.Warnf(ctx, "Failed to deploy addon execute job [%s]: %v", getAddonResourceName(c.DNS.Provider), err)
+		}
+		log.Infof(ctx, "[dns] DNS provider %s deployed successfully", c.DNS.Provider)
+		return nil
+	case CoreDNSProvider:
+		if err := c.deployCoreDNS(ctx); err != nil {
+			if err, ok := err.(*addonError); ok && err.isCritical {
+				return err
+			}
+			log.Warnf(ctx, "Failed to deploy addon execute job [%s]: %v", getAddonResourceName(c.DNS.Provider), err)
+		}
+		log.Infof(ctx, "[dns] DNS provider %s deployed successfully", c.DNS.Provider)
+		return nil
+	case "none":
+		return nil
+	default:
+		log.Warnf(ctx, "[dns] No valid DNS provider configured: %s", c.DNS.Provider)
+		return nil
+	}
+}

cluster/cluster.go

@@ -491,6 +491,7 @@ func RestartClusterPods(ctx context.Context, kubeCluster *Cluster) error {
 		fmt.Sprintf("%s=%s", KubeAppLabel, DefaultMonitoringProvider),
 		fmt.Sprintf("%s=%s", KubeAppLabel, KubeDNSAddonAppName),
 		fmt.Sprintf("%s=%s", KubeAppLabel, KubeDNSAutoscalerAppName),
+		fmt.Sprintf("%s=%s", KubeAppLabel, CoreDNSAutoscalerAppName),
 	}
 	var errgrp errgroup.Group
 	labelQueue := util.GetObjectQueue(labelsList)

cluster/defaults.go

@@ -45,6 +45,7 @@ const (
 	DefaultMonitoringProvider            = "metrics-server"
 	DefaultEtcdBackupConfigIntervalHours = 12
 	DefaultEtcdBackupConfigRetention     = 6
+	DefaultDNSProvider                   = "kubedns"
 
 	DefaultEtcdHeartbeatIntervalName  = "heartbeat-interval"
 	DefaultEtcdHeartbeatIntervalValue = "500"
@@ -137,7 +138,6 @@ func (c *Cluster) setClusterDefaults(ctx context.Context) error {
 	if len(c.Monitoring.Provider) == 0 {
 		c.Monitoring.Provider = DefaultMonitoringProvider
 	}
-
 	//set docker private registry URL
 	for _, pr := range c.PrivateRegistries {
 		if pr.URL == "" {
@@ -145,10 +145,16 @@ func (c *Cluster) setClusterDefaults(ctx context.Context) error {
 		}
 		c.PrivateRegistriesMap[pr.URL] = pr
 	}
+
 	err := c.setClusterImageDefaults()
 	if err != nil {
 		return err
 	}
+
+	if len(c.DNS.Provider) == 0 {
+		c.DNS.Provider = DefaultDNSProvider
+	}
+
 	c.setClusterServicesDefaults()
 	c.setClusterNetworkDefaults()
 	c.setClusterAuthnDefaults()
@@ -231,6 +237,8 @@ func (c *Cluster) setClusterImageDefaults() error {
 		&c.SystemImages.KubeDNSSidecar:            d(imageDefaults.KubeDNSSidecar, privRegURL),
 		&c.SystemImages.DNSmasq:                   d(imageDefaults.DNSmasq, privRegURL),
 		&c.SystemImages.KubeDNSAutoscaler:         d(imageDefaults.KubeDNSAutoscaler, privRegURL),
+		&c.SystemImages.CoreDNS:                   d(imageDefaults.CoreDNS, privRegURL),
+		&c.SystemImages.CoreDNSAutoscaler:         d(imageDefaults.CoreDNSAutoscaler, privRegURL),
 		&c.SystemImages.KubernetesServicesSidecar: d(imageDefaults.KubernetesServicesSidecar, privRegURL),
 		&c.SystemImages.Etcd:                      d(imageDefaults.Etcd, privRegURL),
 		&c.SystemImages.Kubernetes:                d(imageDefaults.Kubernetes, privRegURL),

templates/coredns.go

@@ -0,0 +1,258 @@
+package templates
+
+const CoreDNSTemplate = `
+---
+{{- if eq .RBACConfig "rbac"}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+{{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        kubernetes {{.ClusterDomain}} {{ if .ReverseCIDRs }}{{ .ReverseCIDRs }}{{ else }}{{ "in-addr.arpa ip6.arpa" }}{{ end }} {
+          pods insecure
+	  {{- if .UpstreamNameservers }}
+          upstream {{range $i, $v := .UpstreamNameservers}}{{if $i}} {{end}}{{.}}{{end}}
+	  {{- else }}
+          upstream
+          {{- end }}
+          fallthrough in-addr.arpa ip6.arpa
+        }
+        prometheus :9153
+	{{- if .UpstreamNameservers }}
+        proxy . {{range $i, $v := .UpstreamNameservers}}{{if $i}} {{end}}{{.}}{{end}}
+	{{- else }}
+        proxy . "/etc/resolv.conf"
+	{{- end }}
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+    spec:
+{{- if eq .RBACConfig "rbac"}}
+      serviceAccountName: coredns
+{{- end }}
+      nodeSelector:
+      {{ range $k, $v := .NodeSelector }}
+        {{ $k }}: {{ $v }}
+      {{ end }}
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      containers:
+      - name: coredns
+        image: {{.CoreDNSImage}}
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+          readOnly: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: {{.ClusterDNSServer}}
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: coredns-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-app: coredns-autoscaler
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: coredns-autoscaler
+    spec:
+      serviceAccountName: coredns-autoscaler
+      containers:
+      - name: autoscaler
+        image: {{.CoreDNSAutoScalerImage}}
+        resources:
+            requests:
+                cpu: "20m"
+                memory: "10Mi"
+        command:
+          - /cluster-proportional-autoscaler
+          - --namespace=kube-system
+          - --configmap=coredns-autoscaler
+          - --target=Deployment/coredns
+          # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate.
+          # If using small nodes, "nodesPerReplica" should dominate.
+          - --default-params={"linear":{"coresPerReplica":128,"nodesPerReplica":4,"min":1}}
+          - --logtostderr=true
+          - --v=2
+{{- if eq .RBACConfig "rbac"}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns-autoscaler
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:coredns-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list"]
+  - apiGroups: [""]
+    resources: ["replicationcontrollers/scale"]
+    verbs: ["get", "update"]
+  - apiGroups: ["extensions"]
+    resources: ["deployments/scale", "replicasets/scale"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:coredns-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: coredns-autoscaler
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: system:coredns-autoscaler
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}`

templates/kubedns.go

@@ -42,7 +42,7 @@ metadata:
   labels:
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-{{- if eq .RBAC "rbac"}}
+{{- if eq .RBACConfig "rbac"}}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -205,8 +205,14 @@ spec:
         - --cache-size=1000
         - --log-facility=-
         - --server=/{{.ClusterDomain}}/127.0.0.1#10053
+	{{- if .ReverseCIDRs }}
+	{{- range .ReverseCIDRs }}
+        - --server=/{{.}}/127.0.0.1#10053
+	{{- end }}
+	{{- else }}
         - --server=/in-addr.arpa/127.0.0.1#10053
         - --server=/ip6.arpa/127.0.0.1#10053
+	{{- end }}
         ports:
         - containerPort: 53
           name: dns
@@ -223,7 +229,7 @@ spec:
         - name: kube-dns-config
           mountPath: /etc/k8s/dns/dnsmasq-nanny
       - name: sidecar
-        image: {{.KubednsSidecarImage}}
+        image: {{.KubeDNSSidecarImage}}
         livenessProbe:
           httpGet:
             path: /metrics
@@ -269,4 +275,15 @@ spec:
     protocol: UDP
   - name: dns-tcp
     port: 53
-    protocol: TCP`
+    protocol: TCP
+{{- if .UpstreamNameservers }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-dns
+  namespace: kube-system
+data:
+  upstreamNameservers: |
+    [{{range $i, $v := .UpstreamNameservers}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
+{{- end }}`