From b41862b6703bcf9036a61dc8453a5253534545c3 Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Thu, 3 Sep 2015 10:55:26 +0200 Subject: [PATCH] Turning on pod autoscaler on GCE. Implemented optional turning on of pod autoscaler in kube-up script for GCE. --- cluster/gce/config-default.sh | 10 +++++++++- cluster/gce/config-test.sh | 11 ++++++++++- cluster/gce/configure-vm.sh | 6 ++++++ cluster/gce/coreos/helper.sh | 2 ++ cluster/gce/debian/helper.sh | 2 ++ cluster/gce/util.sh | 13 +++++++++++++ .../kube-controller-manager.manifest | 6 +++++- hack/verify-flags/exceptions.txt | 3 +-- 8 files changed, 48 insertions(+), 5 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c0a18753c08..6aae9232ebb 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -44,6 +44,8 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" +RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" +ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 @@ -87,7 +89,6 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. -# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -95,6 +96,13 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi +# Optional: Enable feature for autoscaling number of pods +# Experimental feature, not ready for production use. +ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" +if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then + ENABLE_EXPERIMENTAL_API=true +fi + # Admission Controllers to invoke prior to persisting objects in cluster ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index a45477a3e7d..ef6ffa6f44a 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -45,6 +45,9 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" +RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" +ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" + # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET @@ -92,7 +95,6 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. -# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -100,6 +102,13 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi +# Optional: Enable feature for autoscaling number of pods +# Experimental feature, not ready for production use. +ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" +if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then + ENABLE_EXPERIMENTAL_API=true +fi + ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota # Optional: if set to true kube-up will automatically check for existing resources and clean them up. diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index c199a6643c5..c5ef423800a 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -277,6 +277,7 @@ dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")' +enable_horizontal_pod_autoscaler: '$(echo "$ENABLE_HORIZONTAL_POD_AUTOSCALER" | sed -e "s/'/''/g")' EOF if [ -n "${APISERVER_TEST_ARGS:-}" ]; then @@ -568,6 +569,11 @@ EOF # CIDR range. cat <>/etc/salt/minion.d/grains.conf cbr-cidr: ${MASTER_IP_RANGE} +EOF + fi + if [[ ! -z "${RUNTIME_CONFIG:-}" ]]; then + cat <>/etc/salt/minion.d/grains.conf + runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")' EOF fi } diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 5ef057acf4a..846bb2591b0 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -54,6 +54,8 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) +ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) +RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME}) KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME}) RKT_VERSION: $(yaml-quote ${RKT_VERSION}) diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 6b929c2d9ee..694f29ec85e 100755 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -51,6 +51,8 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) +ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) +RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-}) KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index c884c9c2195..604c855f236 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -53,6 +53,18 @@ function join_csv { # Verify prereqs function verify-prereqs { + if [[ "${ENABLE_EXPERIMENTAL_API}" == "true" ]]; then + if [[ -z "${RUNTIME_CONFIG}" ]]; then + RUNTIME_CONFIG="experimental/v1=true" + else + # TODO: add checking if RUNTIME_CONFIG contains "experimental/v1=false" and appending "experimental/v1=true" if not. + if echo "${RUNTIME_CONFIG}" | grep -q -v "experimental/v1=true"; then + echo "Experimental API should be turned on, but is not turned on in RUNTIME_CONFIG!" + exit 1 + fi + fi + fi + local cmd for cmd in gcloud gsutil; do if ! which "${cmd}" >/dev/null; then @@ -465,6 +477,7 @@ function write-master-env { if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then KUBELET_APISERVER="${MASTER_NAME}" fi + build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" } diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index 7c0e214213a..a4f82888a01 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -1,6 +1,7 @@ {% set cluster_name = "" -%} {% set cluster_cidr = "" -%} {% set allocate_node_cidrs = "" -%} +{% set enable_horizontal_pod_autoscaler = "" -%} {% if pillar['instance_prefix'] is defined -%} {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%} @@ -11,6 +12,9 @@ {% if pillar['allocate_node_cidrs'] is defined -%} {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%} {% endif -%} +{% if pillar['enable_horizontal_pod_autoscaler'] is defined -%} + {% set enable_horizontal_pod_autoscaler = "--enable-horizontal-pod-autoscaler=" + pillar['enable_horizontal_pod_autoscaler'] -%} +{% endif -%} {% set cloud_provider = "" -%} {% set cloud_config = "" -%} @@ -34,7 +38,7 @@ {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} -{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration {% if pillar['controller_manager_test_args'] is defined -%} diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index a8816eff7e5..c541cc21188 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -1,4 +1,3 @@ -cluster/addons/cluster-monitoring/README.md:Heapster enables monitoring of Kubernetes Clusters using [cAdvisor](https://github.com/google/cadvisor). The kubelet will communicate with an instance of cAdvisor running on localhost and proxy container stats to Heapster. Kubelet will attempt to connect to cAdvisor on port 4194 by default but this port can be configured with kubelet's `--cadvisor-port` run flag. Detailed information about heapster can be found [here](https://github.com/GoogleCloudPlatform/heapster). cluster/addons/registry/images/Dockerfile:ADD run_proxy.sh /usr/bin/run_proxy cluster/addons/registry/images/Dockerfile:CMD ["/usr/bin/run_proxy"] cluster/aws/templates/salt-minion.sh:# We set the hostname_override to the full EC2 private dns name @@ -39,7 +38,7 @@ cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control obje cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + " " + basic_auth_file + " " + min_request_timeout -%} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = params + " " + cluster_name + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + " " + token_auth_file + " " + bind_address + " " + pillar['log_level'] + " " + advertise_address + " " + proxy_ssh_options -%} -cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":7080" -%}