diff --git a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml new file mode 100644 index 00000000000..70cc55336ab --- /dev/null +++ b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml @@ -0,0 +1,50 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns-autoscaler + namespace: kube-system + labels: + k8s-app: kube-dns-autoscaler + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: kube-dns-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: autoscaler + image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.0.0 + resources: + requests: + cpu: "20m" + memory: "10Mi" + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=kube-dns-autoscaler + - --mode=linear + # Should keep target in sync with cluster/addons/dns/skydns-rc.yaml.base + - --target=ReplicationController/kube-dns-v20 + # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. + # If using small nodes, "nodesPerReplica" should dominate. + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"min":1}} + - --logtostderr=true + - --v=2 diff --git a/cluster/addons/dns/skydns-rc.yaml.base b/cluster/addons/dns/skydns-rc.yaml.base index 22cdda791ce..28cf7d7de1f 100644 --- a/cluster/addons/dns/skydns-rc.yaml.base +++ b/cluster/addons/dns/skydns-rc.yaml.base @@ -13,6 +13,8 @@ # limitations under the License. # TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. # __MACHINE_GENERATED_WARNING__ diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in index ee098c4c2e7..616ade17315 100644 --- a/cluster/addons/dns/skydns-rc.yaml.in +++ b/cluster/addons/dns/skydns-rc.yaml.in @@ -13,6 +13,8 @@ # limitations under the License. # TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. # Warning: This is a file generated from the base underscore template file: skydns-rc.yaml.base diff --git a/cluster/addons/dns/skydns-rc.yaml.sed b/cluster/addons/dns/skydns-rc.yaml.sed index 831282cab3c..be59bad988e 100644 --- a/cluster/addons/dns/skydns-rc.yaml.sed +++ b/cluster/addons/dns/skydns-rc.yaml.sed @@ -13,6 +13,8 @@ # limitations under the License. # TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* +# Should keep target in cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +# in sync with this file. # Warning: This is a file generated from the base underscore template file: skydns-rc.yaml.base diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index 6aa1a23b97f..58591bae60c 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -122,6 +122,9 @@ DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh index 4ac6492d540..1de0ce1ad73 100755 --- a/cluster/aws/config-test.sh +++ b/cluster/aws/config-test.sh @@ -108,6 +108,9 @@ DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/common.sh b/cluster/common.sh index c916e5a1cb5..6d67772dbdd 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -608,6 +608,7 @@ CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE:-}) DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-}) DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) +ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote ${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}) KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index f29a877c2bf..bd5abb86888 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -115,6 +115,9 @@ DNS_SERVER_IP="${KUBE_DNS_SERVER_IP:-10.0.0.10}" DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-true}" + # Optional: Install cluster docker registry. ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-false}" CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_PD:-${INSTANCE_PREFIX}-kube-system-kube-registry}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index fff59511236..8b87b3fe9fa 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -142,6 +142,9 @@ DNS_SERVER_IP="10.0.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-true}" + # Optional: Install cluster docker registry. ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-false}" CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_DISK:-${INSTANCE_PREFIX}-kube-system-kube-registry}" diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 609fd582d89..168dd5b0351 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -442,6 +442,7 @@ enable_cluster_registry: '$(echo "$ENABLE_CLUSTER_REGISTRY" | sed -e "s/'/''/g") dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' +enable_dns_horizontal_autoscaler: '$(echo "$ENABLE_DNS_HORIZONTAL_AUTOSCALER" | sed -e "s/'/''/g")' admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")' network_provider: '$(echo "$NETWORK_PROVIDER" | sed -e "s/'/''/g")' prepull_e2e_images: '$(echo "$PREPULL_E2E_IMAGES" | sed -e "s/'/''/g")' diff --git a/cluster/gce/coreos/configure-node.sh b/cluster/gce/coreos/configure-node.sh index bebc240512b..18988b065fc 100755 --- a/cluster/gce/coreos/configure-node.sh +++ b/cluster/gce/coreos/configure-node.sh @@ -135,6 +135,9 @@ function configure-master-addons() { if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns ${addon_dir}/dns + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER}" == "true" ]]; then + evaluate-manifests-dir ${MANIFESTS_DIR}/addons/dns-horizontal-autoscaler ${addon_dir}/dns-horizontal-autoscaler + fi fi if [[ "${ENABLE_CLUSTER_UI}" == "true" ]]; then diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b02f7846d95..b0e060a12d3 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1030,6 +1030,10 @@ function start-kube-addons { sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}" sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then + setup-addon-manifests "addons" "dns-horizontal-autoscaler" + fi + if [[ "${FEDERATION:-}" == "true" ]]; then local federations_domain_map="${FEDERATIONS_DOMAIN_MAP:-}" if [[ -z "${federations_domain_map}" && -n "${FEDERATION_NAME:-}" && -n "${DNS_ZONE_NAME:-}" ]]; then diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 33be3919f5c..481bae943b9 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -879,6 +879,10 @@ start_kube_addons() { sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${dns_rc_file}" sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${dns_svc_file}" + if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then + setup_addon_manifests "addons" "dns-horizontal-autoscaler" + fi + if [[ "${FEDERATION:-}" == "true" ]]; then FEDERATIONS_DOMAIN_MAP="${FEDERATIONS_DOMAIN_MAP:-}" if [[ -z "${FEDERATIONS_DOMAIN_MAP}" && -n "${FEDERATION_NAME:-}" && -n "${DNS_ZONE_NAME:-}" ]]; then diff --git a/cluster/gke/config-default.sh b/cluster/gke/config-default.sh index 46697fa67e4..d2efb33b9af 100644 --- a/cluster/gke/config-default.sh +++ b/cluster/gke/config-default.sh @@ -41,4 +41,7 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}" # standalone - Heapster only. Metrics available via Heapster REST API. ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-standalone}" +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false} diff --git a/cluster/libvirt-coreos/config-default.sh b/cluster/libvirt-coreos/config-default.sh index 5d34d2d50b4..05866ff7a83 100644 --- a/cluster/libvirt-coreos/config-default.sh +++ b/cluster/libvirt-coreos/config-default.sh @@ -59,6 +59,9 @@ DNS_SERVER_IP="${SERVICE_CLUSTER_IP_RANGE%.*}.254" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + #Generate dns files sed -f "${KUBE_ROOT}/cluster/addons/dns/transforms2sed.sed" < "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.base" | sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forShellEval.sed" > "${KUBE_ROOT}/cluster/libvirt-coreos/skydns-rc.yaml" sed -f "${KUBE_ROOT}/cluster/addons/dns/transforms2sed.sed" < "${KUBE_ROOT}/cluster/addons/dns/skydns-svc.yaml.base" | sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forShellEval.sed" > "${KUBE_ROOT}/cluster/libvirt-coreos/skydns-svc.yaml" diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh index 54a3cf19675..41fcd6f6ed4 100755 --- a/cluster/mesos/docker/config-default.sh +++ b/cluster/mesos/docker/config-default.sh @@ -34,6 +34,9 @@ DNS_SERVER_IP="10.10.10.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Deploy cluster web interface. ENABLE_CLUSTER_UI=true diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml index 1a04f6df49b..f8ca3fb6f83 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml @@ -51,6 +51,7 @@ write_files: dns_replicas: "1" dns_server: 10.246.0.10 dns_domain: cluster.local + enable_dns_horizontal_autoscaler: "false" federations_domain_map: '' instance_prefix: kubernetes admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,DefaultStorageClass,ResourceQuota diff --git a/cluster/photon-controller/config-default.sh b/cluster/photon-controller/config-default.sh index 23ae87f7949..68f6fc07576 100755 --- a/cluster/photon-controller/config-default.sh +++ b/cluster/photon-controller/config-default.sh @@ -79,6 +79,9 @@ DNS_SERVER_IP="10.244.240.240" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI=true diff --git a/cluster/rackspace/config-default.sh b/cluster/rackspace/config-default.sh index e471791797e..6afb4e37ff2 100755 --- a/cluster/rackspace/config-default.sh +++ b/cluster/rackspace/config-default.sh @@ -58,5 +58,8 @@ DNS_SERVER_IP="10.0.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls index ae8f1dfd36a..502260d91f9 100644 --- a/cluster/saltbase/salt/kube-addons/init.sls +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -90,6 +90,17 @@ addon-dir-create: - makedirs: True {% endif %} +{% if pillar.get('enable_dns_horizontal_autoscaler', '').lower() == 'true' + and pillar.get('enable_cluster_dns', '').lower() == 'true' %} +/etc/kubernetes/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml: + file.managed: + - source: salt://kube-addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml + - user: root + - group: root + - file_mode: 644 + - makedirs: True +{% endif %} + {% if pillar.get('enable_cluster_registry', '').lower() == 'true' %} /etc/kubernetes/addons/registry/registry-svc.yaml: file.managed: diff --git a/cluster/ubuntu/config-default.sh b/cluster/ubuntu/config-default.sh index 14271908bbb..6f7d7003bf1 100755 --- a/cluster/ubuntu/config-default.sh +++ b/cluster/ubuntu/config-default.sh @@ -118,6 +118,9 @@ DNS_SERVER_IP=${DNS_SERVER_IP:-"192.168.3.10"} DNS_DOMAIN=${DNS_DOMAIN:-"cluster.local"} DNS_REPLICAS=${DNS_REPLICAS:-1} +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh index 0d087eb8437..bc4946b5507 100755 --- a/cluster/vagrant/config-default.sh +++ b/cluster/vagrant/config-default.sh @@ -88,6 +88,9 @@ DNS_SERVER_IP="10.247.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh index 7aa672d209b..96f859cd28c 100755 --- a/cluster/vsphere/config-default.sh +++ b/cluster/vsphere/config-default.sh @@ -55,6 +55,9 @@ DNS_SERVER_IP="10.244.240.240" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Enable DNS horizontal autoscaler +ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 81d09f09bf2..a68066e8bb1 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -29,6 +29,7 @@ go_library( "density.go", "deployment.go", "dns.go", + "dns_autoscaling.go", "e2e.go", "empty.go", "empty_dir_wrapper.go", diff --git a/test/e2e/dns_autoscaling.go b/test/e2e/dns_autoscaling.go new file mode 100644 index 00000000000..de1ab33845c --- /dev/null +++ b/test/e2e/dns_autoscaling.go @@ -0,0 +1,244 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "math" + "reflect" + "strings" + "time" + + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + DNSdefaultTimeout = 5 * time.Minute + DNSscaleUpTimeout = 5 * time.Minute + DNSscaleDownTimeout = 10 * time.Minute + DNSNamespace = "kube-system" + ClusterAddonLabelKey = "k8s-app" + KubeDNSLabelName = "kube-dns" + DNSAutoscalerLabelName = "kube-dns-autoscaler" +) + +var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { + f := framework.NewDefaultFramework("dns-autoscaling") + var c clientset.Interface + var nodeCount int + var previousParams map[string]string + DNSParams_1 := map[string]string{"linear": "{\"nodesPerReplica\": 1}"} + DNSParams_2 := map[string]string{"linear": "{\"nodesPerReplica\": 2}"} + DNSParams_3 := map[string]string{"linear": "{\"nodesPerReplica\": 3}"} + + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce") + c = f.ClientSet + + nodes := framework.GetReadySchedulableNodesOrDie(c) + nodeCount = len(nodes.Items) + Expect(nodeCount).NotTo(BeZero()) + + pcm, err := fetchDNSScalingConfigMap(c) + ExpectNoError(err) + previousParams = pcm.Data + + By("Replace the dns autoscaling parameters with testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) + }) + + AfterEach(func() { + By("Restoring intial dns autoscaling parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))) + }) + + // This test is separated because it is slow and need to run serially + It("[Serial] [Slow] kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func() { + originalSizes := make(map[string]int) + sum := 0 + for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { + size, err := GroupSize(mig) + ExpectNoError(err) + By(fmt.Sprintf("Initial size of %s: %d", mig, size)) + originalSizes[mig] = size + sum += size + } + Expect(nodeCount).Should(Equal(sum)) + + By("Manually increase cluster size") + increasedSize := 0 + increasedSizes := make(map[string]int) + for key, val := range originalSizes { + increasedSizes[key] = val + 1 + increasedSize += increasedSizes[key] + } + setMigSizes(increasedSizes) + ExpectNoError(WaitForClusterSizeFunc(c, + func(size int) bool { return size == increasedSize }, DNSscaleUpTimeout)) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, increasedSize, DNSdefaultTimeout)) + + By("Restoring cluster size") + setMigSizes(originalSizes) + framework.ExpectNoError(framework.WaitForClusterSize(c, nodeCount, DNSscaleDownTimeout)) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) + }) + + It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { + + By("--- Scenario: should scale kube-dns based on changed parameters ---") + By("Replace the dns autoscaling parameters with the second testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) + + By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") + By("Delete the ConfigMap for autoscaler") + err := deleteDNSScalingConfigMap(c) + ExpectNoError(err) + + By("Wait for the ConfigMap got re-created") + configMap, err := waitForDNSConfigMapCreated(c, DNSdefaultTimeout) + ExpectNoError(err) + + By("Check the new created ConfigMap got the same data as we have") + Expect(reflect.DeepEqual(previousParams, configMap.Data)).To(Equal(true)) + + By("Replace the dns autoscaling parameters with the second testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) + + By("--- Scenario: should recover after autoscaler pod got deleted ---") + By("Delete the autoscaler pod for kube-dns") + ExpectNoError(deleteDNSAutoscalerPod(c)) + + By("Replace the dns autoscaling parameters with the third testing parameters") + ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3))) + By("Wait for kube-dns scaled to expected number") + ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/3.0)), DNSdefaultTimeout)) + }) +}) + +func fetchDNSScalingConfigMap(c clientset.Interface) (*api.ConfigMap, error) { + cm, err := c.Core().ConfigMaps(DNSNamespace).Get(DNSAutoscalerLabelName) + if err != nil { + return nil, err + } + return cm, nil +} + +func deleteDNSScalingConfigMap(c clientset.Interface) error { + if err := c.Core().ConfigMaps(DNSNamespace).Delete(DNSAutoscalerLabelName, nil); err != nil { + return err + } + framework.Logf("DNS autoscaling ConfigMap deleted.") + return nil +} + +func packDNSScalingConfigMap(params map[string]string) *api.ConfigMap { + configMap := api.ConfigMap{} + configMap.ObjectMeta.Name = DNSAutoscalerLabelName + configMap.ObjectMeta.Namespace = DNSNamespace + configMap.Data = params + return &configMap +} + +func updateDNSScalingConfigMap(c clientset.Interface, configMap *api.ConfigMap) error { + _, err := c.Core().ConfigMaps(DNSNamespace).Update(configMap) + if err != nil { + return err + } + framework.Logf("DNS autoscaling ConfigMap updated.") + return nil +} + +func getDNSReplicas(c clientset.Interface) (int, error) { + label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: KubeDNSLabelName})) + listOpts := api.ListOptions{LabelSelector: label} + rcs, err := c.Core().ReplicationControllers(DNSNamespace).List(listOpts) + if err != nil { + return 0, err + } + Expect(len(rcs.Items)).Should(Equal(1)) + + rc := rcs.Items[0] + return int(rc.Spec.Replicas), nil +} + +func deleteDNSAutoscalerPod(c clientset.Interface) error { + label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSAutoscalerLabelName})) + listOpts := api.ListOptions{LabelSelector: label} + pods, err := c.Core().Pods(DNSNamespace).List(listOpts) + if err != nil { + return err + } + Expect(len(pods.Items)).Should(Equal(1)) + + podName := pods.Items[0].Name + if err := c.Core().Pods(DNSNamespace).Delete(podName, nil); err != nil { + return err + } + framework.Logf("DNS autoscaling pod %v deleted.", podName) + return nil +} + +func waitForDNSReplicasSatisfied(c clientset.Interface, expected int, timeout time.Duration) (err error) { + var current int + framework.Logf("Waiting up to %v for kube-dns reach %v replicas", timeout, expected) + condition := func() (bool, error) { + current, err = getDNSReplicas(c) + if err != nil { + return false, err + } + if current != expected { + return false, nil + } + return true, nil + } + + if err = wait.Poll(time.Second, timeout, condition); err != nil { + return fmt.Errorf("err waiting for DNS replicas to satisfy %v, got %v: %v", expected, current, err) + } + return nil +} + +func waitForDNSConfigMapCreated(c clientset.Interface, timeout time.Duration) (configMap *api.ConfigMap, err error) { + framework.Logf("Waiting up to %v for DNS autoscaling ConfigMap got re-created", timeout) + condition := func() (bool, error) { + configMap, err = fetchDNSScalingConfigMap(c) + if err != nil { + return false, nil + } + return true, nil + } + + if err = wait.Poll(time.Second, timeout, condition); err != nil { + return nil, fmt.Errorf("err waiting for DNS autoscaling ConfigMap got re-created: %v", err) + } + return configMap, nil +} diff --git a/test/test_owners.csv b/test/test_owners.csv index 1fe13cb29be..860ca4f5ea3 100644 --- a/test/test_owners.csv +++ b/test/test_owners.csv @@ -38,6 +38,8 @@ Container Lifecycle Hook when create a pod with lifecycle hook when it is http h Container Runtime Conformance Test container runtime conformance blackbox test when running a container with a new image *,Random-Liu,0 Container Runtime Conformance Test container runtime conformance blackbox test when starting a container that exits it should run with the expected status,luxas,1 Container Runtime Conformance Test container runtime conformance blackbox test when starting a container that exits should report termination message if TerminationMessagePath is set,timothysc,1 +DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios,MrHohn,0 +DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods when cluster size changed,MrHohn,0 DNS should provide DNS for ExternalName services,rmmh,1 DNS should provide DNS for pods for Hostname and Subdomain Annotation,mtaufen,1 DNS should provide DNS for services,roberthbailey,1