From bbb1d45d9a8d02f881c37ee36cd01f77e2d531e6 Mon Sep 17 00:00:00 2001 From: Krzysztof Siedlecki Date: Mon, 16 Apr 2018 19:01:03 +0200 Subject: [PATCH] Adding kube dns to kubemark --- cluster/kubemark/gce/config-default.sh | 4 + .../kubemark/resources/kube_dns_template.yaml | 188 ++++++++++++++++++ .../kube-dns-binding.yaml | 15 ++ test/kubemark/start-kubemark.sh | 32 ++- 4 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 test/kubemark/resources/kube_dns_template.yaml create mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index 896d8bd89cd..d8195a23ddf 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -117,6 +117,10 @@ if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then fi fi +#Optional: Enable kube dns. +ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-true}" +KUBE_DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}" + # Optional: set feature gates FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" diff --git a/test/kubemark/resources/kube_dns_template.yaml b/test/kubemark/resources/kube_dns_template.yaml new file mode 100644 index 00000000000..480eff46627 --- /dev/null +++ b/test/kubemark/resources/kube_dns_template.yaml @@ -0,0 +1,188 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kubemark + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-dns + namespace: kubemark + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-dns + namespace: kubemark + labels: + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns + namespace: kubemark + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + volumes: + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: secret-volume + secret: + secretName: kubeconfig + containers: + - name: kubedns + image: k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.9 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain={{dns_domain}}. + - --dns-port=10053 + - --config-dir=/kube-dns-config + - --kubecfg-file=/etc/secret-volume/dns.kubeconfig + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + volumeMounts: + - name: kube-dns-config + mountPath: /kube-dns-config + - name: secret-volume + mountPath: /etc/secret-volume + - name: dnsmasq + image: k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.9 + livenessProbe: + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k + - --cache-size=1000 + - --no-negcache + - --log-facility=- + - --server=/{{dns_domain}}/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + # see: https://github.com/kubernetes/kubernetes/issues/29055 for details + resources: + requests: + cpu: 150m + memory: 20Mi + volumeMounts: + - name: kube-dns-config + mountPath: /etc/k8s/dns/dnsmasq-nanny + - name: sidecar + image: k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.9 + livenessProbe: + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{dns_domain}},5,SRV + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{dns_domain}},5,SRV + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + memory: 20Mi + cpu: 10m + dnsPolicy: Default # Don't use cluster DNS. + serviceAccountName: kube-dns diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml new file mode 100644 index 00000000000..3f9b0d3dff3 --- /dev/null +++ b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml @@ -0,0 +1,15 @@ +# This is the role binding for the kubemark kube-dns. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-dns-view-binding + labels: + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-dns +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-dns diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 20f4703652e..e0d6763d440 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -99,6 +99,7 @@ function generate-pki-config { NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) CLUSTER_AUTOSCALER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBE_DNS_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) echo "Generated PKI authentication data for kubemark." } @@ -126,6 +127,7 @@ function write-pki-config-to-master { sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ sudo bash -c \"echo \"${CLUSTER_AUTOSCALER_TOKEN},system:cluster-autoscaler,uid:cluster-autoscaler\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBE_DNS_TOKEN},system:kube-dns,uid:kube-dns\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ sudo bash -c \"echo ${KUBE_PASSWORD},admin,admin > /home/kubernetes/k8s_auth_data/basic_auth.csv\"" execute-cmd-on-master-with-retries "${PKI_SETUP_CMD}" 3 echo "Wrote PKI certs, keys, tokens and admin password to master." @@ -348,6 +350,25 @@ contexts: name: kubemark-context current-context: kubemark-context") + # Create kubeconfig for Kube DNS. + KUBE_DNS_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 +kind: Config +users: +- name: kube-dns + user: + token: ${KUBE_DNS_TOKEN} +clusters: +- name: kubemark + cluster: + insecure-skip-tls-verify: true + server: https://${MASTER_IP} +contexts: +- context: + cluster: kubemark + user: kube-dns + name: kubemark-context +current-context: kubemark-context") + # Create kubemark namespace. "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" @@ -362,7 +383,8 @@ current-context: kubemark-context") --from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \ --from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \ --from-literal=cluster_autoscaler.kubeconfig="${CLUSTER_AUTOSCALER_KUBECONFIG_CONTENTS}" \ - --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" + --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" \ + --from-literal=dns.kubeconfig="${KUBE_DNS_KUBECONFIG_CONTENTS}" # Create addon pods. # Heapster. @@ -380,7 +402,7 @@ current-context: kubemark-context") sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" # Cluster Autoscaler. - if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then + if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then echo "Setting up Cluster Autoscaler" KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}" KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}" @@ -394,6 +416,12 @@ current-context: kubemark-context") sed -i'' -e "s/{{kubemark_autoscaler_max_nodes}}/${KUBEMARK_AUTOSCALER_MAX_NODES}/g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" fi + # Kube DNS. + if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then + echo "Setting up kube-dns" + sed "s/{{dns_domain}}/${KUBE_DNS_DOMAIN}/g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml" + fi + "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" # Create the replication controller for hollow-nodes.