From d961190e6f117e36bff8798f83c3aaa55a69aca3 Mon Sep 17 00:00:00 2001 From: Zihong Zheng Date: Mon, 7 Nov 2016 10:44:42 -0800 Subject: [PATCH 1/2] Deployed DNS horizontal autoscaler as an addon DNS horizontal autoscaling feature is turned on by default on gce. The corresponding env var is piped into almost all other cloud providers. --- .../dns-horizontal-autoscaler.yaml | 50 +++++++++++++++++++ cluster/addons/dns/skydns-rc.yaml.base | 2 + cluster/addons/dns/skydns-rc.yaml.in | 2 + cluster/addons/dns/skydns-rc.yaml.sed | 2 + cluster/aws/config-default.sh | 3 ++ cluster/aws/config-test.sh | 3 ++ cluster/common.sh | 1 + cluster/gce/config-default.sh | 3 ++ cluster/gce/config-test.sh | 3 ++ cluster/gce/configure-vm.sh | 1 + cluster/gce/coreos/configure-node.sh | 3 ++ cluster/gce/gci/configure-helper.sh | 4 ++ cluster/gce/trusty/configure-helper.sh | 4 ++ cluster/gke/config-default.sh | 3 ++ cluster/libvirt-coreos/config-default.sh | 3 ++ cluster/mesos/docker/config-default.sh | 3 ++ .../fragments/configure-salt.yaml | 1 + cluster/photon-controller/config-default.sh | 3 ++ cluster/rackspace/config-default.sh | 3 ++ cluster/saltbase/salt/kube-addons/init.sls | 11 ++++ cluster/ubuntu/config-default.sh | 3 ++ cluster/vagrant/config-default.sh | 3 ++ cluster/vsphere/config-default.sh | 3 ++ 23 files changed, 117 insertions(+) create mode 100644 cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml diff --git a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml new file mode 100644 index 00000000000..70cc55336ab --- /dev/null +++ b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml @@ -0,0 +1,50 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns-autoscaler + namespace: kube-system + labels: + k8s-app: kube-dns-autoscaler + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: kube-dns-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: autoscaler + image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.0.0 + resources: + requests: + cpu: "20m" + memory: "10Mi" + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=kube-dns-autoscaler + - --mode=linear + # Should keep target in sync with cluster/addons/dns/skydns-rc.yaml.base + - --target=ReplicationController/kube-dns-v20 + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"min":1}} + - --logtostderr=true + - --v=2 diff --git a/cluster/addons/dns/skydns-rc.yaml.base b/cluster/addons/dns/skydns-rc.yaml.base index 22cdda791ce..28cf7d7de1f 100644 --- a/cluster/addons/dns/skydns-rc.yaml.base +++ b/cluster/addons/dns/skydns-rc.yaml.base @@ -13,6 +13,8 @@ # limitations under the License. # TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. # __MACHINE_GENERATED_WARNING__ diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in index ee098c4c2e7..616ade17315 100644 --- a/cluster/addons/dns/skydns-rc.yaml.in +++ b/cluster/addons/dns/skydns-rc.yaml.in @@ -13,6 +13,8 @@ # limitations under the License. # TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. # Warning: This is a file generated from the base underscore template file: skydns-rc.yaml.base diff --git a/cluster/addons/dns/skydns-rc.yaml.sed b/cluster/addons/dns/skydns-rc.yaml.sed index 831282cab3c..be59bad988e 100644 --- a/cluster/addons/dns/skydns-rc.yaml.sed +++ b/cluster/addons/dns/skydns-rc.yaml.sed @@ -13,6 +13,8 @@ # limitations under the License. # TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. # Warning: This is a file generated from the base underscore template file: skydns-rc.yaml.base diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index 6aa1a23b97f..58591bae60c 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -122,6 +122,9 @@ DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh index 4ac6492d540..1de0ce1ad73 100755 --- a/cluster/aws/config-test.sh +++ b/cluster/aws/config-test.sh @@ -108,6 +108,9 @@ DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/common.sh b/cluster/common.sh index c4be510641b..1060050b7c6 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -608,6 +608,7 @@ CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE:-}) DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-}) DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) +ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote ${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}) KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 47ae2ebb299..e60273a38d4 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -115,6 +115,9 @@ DNS_SERVER_IP="${KUBE_DNS_SERVER_IP:-10.0.0.10}" DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-true}" + # Optional: Install cluster docker registry. ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-false}" CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_PD:-${INSTANCE_PREFIX}-kube-system-kube-registry}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 31fa409fde1..4b12698fe4d 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -140,6 +140,9 @@ DNS_SERVER_IP="10.0.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-true}" + # Optional: Install cluster docker registry. ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-false}" CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_DISK:-${INSTANCE_PREFIX}-kube-system-kube-registry}" diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 44a56f2ab78..7cace74803c 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -442,6 +442,7 @@ enable_cluster_registry: '$(echo "$ENABLE_CLUSTER_REGISTRY" | sed -e "s/'/''/g") dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' +enable_dns_horizontal_autoscaler: '$(echo "$ENABLE_DNS_HORIZONTAL_AUTOSCALER" | sed -e "s/'/''/g")' admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")' storage_backend: '$(echo "$STORAGE_BACKEND" | sed -e "s/'/''/g")' network_provider: '$(echo "$NETWORK_PROVIDER" | sed -e "s/'/''/g")' diff --git a/cluster/gce/coreos/configure-node.sh b/cluster/gce/coreos/configure-node.sh index bebc240512b..18988b065fc 100755 --- a/cluster/gce/coreos/configure-node.sh +++ b/cluster/gce/coreos/configure-node.sh @@ -135,6 +135,9 @@ function configure-master-addons() { if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns ${addon_dir}/dns + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER}" == "true" ]]; then + evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns-horizontal-autoscaler ${addon_dir}/dns-horizontal-autoscaler + fi fi if [[ "${ENABLE_CLUSTER_UI}" == "true" ]]; then diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index e06b5e6183a..6e0a917e083 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1025,6 +1025,10 @@ function start-kube-addons { sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}" sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then + setup-addon-manifests "addons" "dns-horizontal-autoscaler" + fi + if [[ "${FEDERATION:-}" == "true" ]]; then local federations_domain_map="${FEDERATIONS_DOMAIN_MAP:-}" if [[ -z "${federations_domain_map}" && -n "${FEDERATION_NAME:-}" && -n "${DNS_ZONE_NAME:-}" ]]; then diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 83db29243d8..444497bddfe 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -869,6 +869,10 @@ start_kube_addons() { sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}" sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then + setup_addon_manifests "addons" "dns-horizontal-autoscaler" + fi + if [[ "${FEDERATION:-}" == "true" ]]; then FEDERATIONS_DOMAIN_MAP="${FEDERATIONS_DOMAIN_MAP:-}" if [[ -z "${FEDERATIONS_DOMAIN_MAP}" && -n "${FEDERATION_NAME:-}" && -n "${DNS_ZONE_NAME:-}" ]]; then diff --git a/cluster/gke/config-default.sh b/cluster/gke/config-default.sh index 46697fa67e4..d2efb33b9af 100644 --- a/cluster/gke/config-default.sh +++ b/cluster/gke/config-default.sh @@ -41,4 +41,7 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}" # standalone - Heapster only. Metrics available via Heapster REST API. ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-standalone}" +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false} diff --git a/cluster/libvirt-coreos/config-default.sh b/cluster/libvirt-coreos/config-default.sh index 5d34d2d50b4..05866ff7a83 100644 --- a/cluster/libvirt-coreos/config-default.sh +++ b/cluster/libvirt-coreos/config-default.sh @@ -59,6 +59,9 @@ DNS_SERVER_IP="${SERVICE_CLUSTER_IP_RANGE%.*}.254" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + #Generate dns files sed -f "${KUBE_ROOT}/cluster/addons/dns/transforms2sed.sed" < "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.base" | sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forShellEval.sed" > "${KUBE_ROOT}/cluster/libvirt-coreos/skydns-rc.yaml" sed -f "${KUBE_ROOT}/cluster/addons/dns/transforms2sed.sed" < "${KUBE_ROOT}/cluster/addons/dns/skydns-svc.yaml.base" | sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forShellEval.sed" > "${KUBE_ROOT}/cluster/libvirt-coreos/skydns-svc.yaml" diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh index 54a3cf19675..41fcd6f6ed4 100755 --- a/cluster/mesos/docker/config-default.sh +++ b/cluster/mesos/docker/config-default.sh @@ -34,6 +34,9 @@ DNS_SERVER_IP="10.10.10.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Deploy cluster web interface. ENABLE_CLUSTER_UI=true diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml index 1a04f6df49b..f8ca3fb6f83 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml @@ -51,6 +51,7 @@ write_files: dns_replicas: "1" dns_server: 10.246.0.10 dns_domain: cluster.local + enable_dns_horizontal_autoscaler: "false" federations_domain_map: '' instance_prefix: kubernetes admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,DefaultStorageClass,ResourceQuota diff --git a/cluster/photon-controller/config-default.sh b/cluster/photon-controller/config-default.sh index 23ae87f7949..68f6fc07576 100755 --- a/cluster/photon-controller/config-default.sh +++ b/cluster/photon-controller/config-default.sh @@ -79,6 +79,9 @@ DNS_SERVER_IP="10.244.240.240" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI=true diff --git a/cluster/rackspace/config-default.sh b/cluster/rackspace/config-default.sh index e471791797e..6afb4e37ff2 100755 --- a/cluster/rackspace/config-default.sh +++ b/cluster/rackspace/config-default.sh @@ -58,5 +58,8 @@ DNS_SERVER_IP="10.0.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls index ae8f1dfd36a..502260d91f9 100644 --- a/cluster/saltbase/salt/kube-addons/init.sls +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -90,6 +90,17 @@ addon-dir-create: - makedirs: True {% endif %} +{% if pillar.get('enable_dns_horizontal_autoscaler', '').lower() == 'true' + and pillar.get('enable_cluster_dns', '').lower() == 'true' %} +/etc/kubernetes/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml: + file.managed: + - source: salt://kube-addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml + - user: root + - group: root + - file_mode: 644 + - makedirs: True +{% endif %} + {% if pillar.get('enable_cluster_registry', '').lower() == 'true' %} /etc/kubernetes/addons/registry/registry-svc.yaml: file.managed: diff --git a/cluster/ubuntu/config-default.sh b/cluster/ubuntu/config-default.sh index 14271908bbb..6f7d7003bf1 100755 --- a/cluster/ubuntu/config-default.sh +++ b/cluster/ubuntu/config-default.sh @@ -118,6 +118,9 @@ DNS_SERVER_IP=${DNS_SERVER_IP:-"192.168.3.10"} DNS_DOMAIN=${DNS_DOMAIN:-"cluster.local"} DNS_REPLICAS=${DNS_REPLICAS:-1} +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh index 0d087eb8437..bc4946b5507 100755 --- a/cluster/vagrant/config-default.sh +++ b/cluster/vagrant/config-default.sh @@ -88,6 +88,9 @@ DNS_SERVER_IP="10.247.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh index 7aa672d209b..96f859cd28c 100755 --- a/cluster/vsphere/config-default.sh +++ b/cluster/vsphere/config-default.sh @@ -55,6 +55,9 @@ DNS_SERVER_IP="10.244.240.240" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" From 452e6d8c1185fcd00c1dad52c4c05ebd26618c55 Mon Sep 17 00:00:00 2001 From: Zihong Zheng Date: Mon, 7 Nov 2016 11:28:43 -0800 Subject: [PATCH 2/2] Adds e2e tests for DNS horizontal autoscaling feature The e2e tests cover cases like cluster size changed, parameters changed, ConfigMap got deleted, autoscaler pod got deleted, etc. They are separated into a fast part(could be run parallelly) and a slow part(put in [serial]). The fast part of the e2e tests cost around 50 seconds to run. --- test/e2e/BUILD | 1 + test/e2e/dns_autoscaling.go | 244 ++++++++++++++++++++++++++++++++++++ test/test_owners.csv | 2 + 3 files changed, 247 insertions(+) create mode 100644 test/e2e/dns_autoscaling.go diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 248a7def67a..a539b502699 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -29,6 +29,7 @@ go_library( "deployment.go", "disruption.go", "dns.go", + "dns_autoscaling.go", "e2e.go", "empty.go", "empty_dir_wrapper.go", diff --git a/test/e2e/dns_autoscaling.go b/test/e2e/dns_autoscaling.go new file mode 100644 index 00000000000..de1ab33845c --- /dev/null +++ b/test/e2e/dns_autoscaling.go @@ -0,0 +1,244 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "math" + "reflect" + "strings" + "time" + + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + DNSdefaultTimeout = 5 * time.Minute + DNSscaleUpTimeout = 5 * time.Minute + DNSscaleDownTimeout = 10 * time.Minute + DNSNamespace = "kube-system" + ClusterAddonLabelKey = "k8s-app" + KubeDNSLabelName = "kube-dns" + DNSAutoscalerLabelName = "kube-dns-autoscaler" +) + +var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { + f := framework.NewDefaultFramework("dns-autoscaling") + var c clientset.Interface + var nodeCount int + var previousParams map[string]string + DNSParams_1 := map[string]string{"linear": "{\"nodesPerReplica\": 1}"} + DNSParams_2 := map[string]string{"linear": "{\"nodesPerReplica\": 2}"} + DNSParams_3 := map[string]string{"linear": "{\"nodesPerReplica\": 3}"} + + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce") + c = f.ClientSet + + nodes := framework.GetReadySchedulableNodesOrDie(c) + nodeCount = len(nodes.Items) + Expect(nodeCount).NotTo(BeZero()) + + pcm, err := fetchDNSScalingConfigMap(c) + ExpectNoError(err) + previousParams = pcm.Data + + By("Replace the dns autoscaling parameters with testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) + }) + + AfterEach(func() { + By("Restoring intial dns autoscaling parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))) + }) + + // This test is separated because it is slow and need to run serially + It("[Serial] [Slow] kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func() { + originalSizes := make(map[string]int) + sum := 0 + for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { + size, err := GroupSize(mig) + ExpectNoError(err) + By(fmt.Sprintf("Initial size of %s: %d", mig, size)) + originalSizes[mig] = size + sum += size + } + Expect(nodeCount).Should(Equal(sum)) + + By("Manually increase cluster size") + increasedSize := 0 + increasedSizes := make(map[string]int) + for key, val := range originalSizes { + increasedSizes[key] = val + 1 + increasedSize += increasedSizes[key] + } + setMigSizes(increasedSizes) + ExpectNoError(WaitForClusterSizeFunc(c, + func(size int) bool { return size == increasedSize }, DNSscaleUpTimeout)) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, increasedSize, DNSdefaultTimeout)) + + By("Restoring cluster size") + setMigSizes(originalSizes) + framework.ExpectNoError(framework.WaitForClusterSize(c, nodeCount, DNSscaleDownTimeout)) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) + }) + + It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { + + By("--- Scenario: should scale kube-dns based on changed parameters ---") + By("Replace the dns autoscaling parameters with the second testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) + + By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") + By("Delete the ConfigMap for autoscaler") + err := deleteDNSScalingConfigMap(c) + ExpectNoError(err) + + By("Wait for the ConfigMap got re-created") + configMap, err := waitForDNSConfigMapCreated(c, DNSdefaultTimeout) + ExpectNoError(err) + + By("Check the new created ConfigMap got the same data as we have") + Expect(reflect.DeepEqual(previousParams, configMap.Data)).To(Equal(true)) + + By("Replace the dns autoscaling parameters with the second testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) + + By("--- Scenario: should recover after autoscaler pod got deleted ---") + By("Delete the autoscaler pod for kube-dns") + ExpectNoError(deleteDNSAutoscalerPod(c)) + + By("Replace the dns autoscaling parameters with the third testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/3.0)), DNSdefaultTimeout)) + }) +}) + +func fetchDNSScalingConfigMap(c clientset.Interface) (*api.ConfigMap, error) { + cm, err := c.Core().ConfigMaps(DNSNamespace).Get(DNSAutoscalerLabelName) + if err != nil { + return nil, err + } + return cm, nil +} + +func deleteDNSScalingConfigMap(c clientset.Interface) error { + if err := c.Core().ConfigMaps(DNSNamespace).Delete(DNSAutoscalerLabelName, nil); err != nil { + return err + } + framework.Logf("DNS autoscaling ConfigMap deleted.") + return nil +} + +func packDNSScalingConfigMap(params map[string]string) *api.ConfigMap { + configMap := api.ConfigMap{} + configMap.ObjectMeta.Name = DNSAutoscalerLabelName + configMap.ObjectMeta.Namespace = DNSNamespace + configMap.Data = params + return &configMap +} + +func updateDNSScalingConfigMap(c clientset.Interface, configMap *api.ConfigMap) error { + _, err := c.Core().ConfigMaps(DNSNamespace).Update(configMap) + if err != nil { + return err + } + framework.Logf("DNS autoscaling ConfigMap updated.") + return nil +} + +func getDNSReplicas(c clientset.Interface) (int, error) { + label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: KubeDNSLabelName})) + listOpts := api.ListOptions{LabelSelector: label} + rcs, err := c.Core().ReplicationControllers(DNSNamespace).List(listOpts) + if err != nil { + return 0, err + } + Expect(len(rcs.Items)).Should(Equal(1)) + + rc := rcs.Items[0] + return int(rc.Spec.Replicas), nil +} + +func deleteDNSAutoscalerPod(c clientset.Interface) error { + label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSAutoscalerLabelName})) + listOpts := api.ListOptions{LabelSelector: label} + pods, err := c.Core().Pods(DNSNamespace).List(listOpts) + if err != nil { + return err + } + Expect(len(pods.Items)).Should(Equal(1)) + + podName := pods.Items[0].Name + if err := c.Core().Pods(DNSNamespace).Delete(podName, nil); err != nil { + return err + } + framework.Logf("DNS autoscaling pod %v deleted.", podName) + return nil +} + +func waitForDNSReplicasSatisfied(c clientset.Interface, expected int, timeout time.Duration) (err error) { + var current int + framework.Logf("Waiting up to %v for kube-dns reach %v replicas", timeout, expected) + condition := func() (bool, error) { + current, err = getDNSReplicas(c) + if err != nil { + return false, err + } + if current != expected { + return false, nil + } + return true, nil + } + + if err = wait.Poll(time.Second, timeout, condition); err != nil { + return fmt.Errorf("err waiting for DNS replicas to satisfy %v, got %v: %v", expected, current, err) + } + return nil +} + +func waitForDNSConfigMapCreated(c clientset.Interface, timeout time.Duration) (configMap *api.ConfigMap, err error) { + framework.Logf("Waiting up to %v for DNS autoscaling ConfigMap got re-created", timeout) + condition := func() (bool, error) { + configMap, err = fetchDNSScalingConfigMap(c) + if err != nil { + return false, nil + } + return true, nil + } + + if err = wait.Poll(time.Second, timeout, condition); err != nil { + return nil, fmt.Errorf("err waiting for DNS autoscaling ConfigMap got re-created: %v", err) + } + return configMap, nil +} diff --git a/test/test_owners.csv b/test/test_owners.csv index 03af5a37cbb..6ceb8848566 100644 --- a/test/test_owners.csv +++ b/test/test_owners.csv @@ -38,6 +38,8 @@ Container Lifecycle Hook when create a pod with lifecycle hook when it is http h Container Runtime Conformance Test container runtime conformance blackbox test when running a container with a new image *,Random-Liu,0 Container Runtime Conformance Test container runtime conformance blackbox test when starting a container that exits it should run with the expected status,luxas,1 Container Runtime Conformance Test container runtime conformance blackbox test when starting a container that exits should report termination message if TerminationMessagePath is set,timothysc,1 +DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios,MrHohn,0 +DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods when cluster size changed,MrHohn,0 DNS should provide DNS for ExternalName services,rmmh,1 DNS should provide DNS for pods for Hostname and Subdomain Annotation,mtaufen,1 DNS should provide DNS for services,roberthbailey,1