From 63531e56c5ecb16b39f93543b67e73d8eb18b27a Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 7 Feb 2017 12:18:15 +0100 Subject: [PATCH 1/4] Default TARGET_STORAGE to etcd3 in etcd manifest --- cluster/gce/container-linux/configure-helper.sh | 6 +++++- cluster/gce/gci/configure-helper.sh | 6 +++++- cluster/gce/trusty/configure-helper.sh | 6 +++++- cluster/saltbase/salt/etcd/etcd.manifest | 3 +-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index f0c3925e891..a0f2ffc3002 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -655,7 +655,11 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" - sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" + else + sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" + fi if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" else diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index af7d732372b..f9206d69b12 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -721,7 +721,11 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" - sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" + else + sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" + fi if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" else diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index fc6aee6244d..53e7e38b874 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -476,7 +476,11 @@ prepare_etcd_manifest() { sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${etcd_temp_file}" sed -i -e "s@{{ *hostname *}}@$host_name@g" "${etcd_temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}" - sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" + else + sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" + fi if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" else diff --git a/cluster/saltbase/salt/etcd/etcd.manifest b/cluster/saltbase/salt/etcd/etcd.manifest index 4d896202fa7..5f1ea9a6d60 100644 --- a/cluster/saltbase/salt/etcd/etcd.manifest +++ b/cluster/saltbase/salt/etcd/etcd.manifest @@ -20,7 +20,6 @@ {% endfor -%} {% set etcd_cluster = vars.etcd_cluster -%} {% set cluster_state = vars.cluster_state -%} -{% set storage_backend = pillar.get('storage_backend', 'etcd3') -%} {% set quota_bytes = '' -%} {% if pillar.get('storage_backend', 'etcd3') == 'etcd3' -%} {% set quota_bytes = '--quota-backend-bytes=4294967296' -%} @@ -52,7 +51,7 @@ ], "env": [ { "name": "TARGET_STORAGE", - "value": "{{ storage_backend }}" + "value": "{{ pillar.get('storage_backend', 'etcd3') }}" }, { "name": "TARGET_VERSION", "value": "{{ pillar.get('etcd_version', '3.0.14') }}" From 3255898fba560ee173c5a98a0a03cca2d0767354 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 7 Feb 2017 12:18:59 +0100 Subject: [PATCH 2/4] Fail etcd migration script if env var are unset --- cluster/images/etcd/migrate-if-needed.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster/images/etcd/migrate-if-needed.sh b/cluster/images/etcd/migrate-if-needed.sh index bb837e56318..db0cffd366a 100755 --- a/cluster/images/etcd/migrate-if-needed.sh +++ b/cluster/images/etcd/migrate-if-needed.sh @@ -40,16 +40,16 @@ set -o errexit set -o nounset if [ -z "${TARGET_STORAGE:-}" ]; then - echo "TARGET_STORAGE variable unset - skipping migration" - exit 0 + echo "TARGET_STORAGE variable unset - unexpected failure" + exit 1 fi if [ -z "${TARGET_VERSION:-}" ]; then - echo "TARGET_VERSION variable unset - skipping migration" - exit 0 + echo "TARGET_VERSION variable unset - unexpected failure" + exit 1 fi if [ -z "${DATA_DIRECTORY:-}" ]; then - echo "DATA_DIRECTORY variable unset - skipping migration" - exit 0 + echo "DATA_DIRECTORY variable unset - unexpected failure" + exit 1 fi if [ "${TARGET_STORAGE}" != "etcd2" -a "${TARGET_STORAGE}" != "etcd3" ]; then From 38281880494cb4cf467b2639cd7f773f98e91a58 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 7 Feb 2017 12:59:33 +0100 Subject: [PATCH 3/4] Set quota bytes by default --- cluster/gce/container-linux/configure-helper.sh | 6 +++++- cluster/gce/gci/configure-helper.sh | 6 +++++- cluster/gce/trusty/configure-helper.sh | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index a0f2ffc3002..b61f1adbfc4 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -655,12 +655,16 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" + # Get default storage backend from manifest file. + local -r default_storage_backend=$(cat "${temp_file}" | \ + grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ + sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") if [[ -n "${STORAGE_BACKEND:-}" ]]; then sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" else sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" fi - if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then + if [[ "${STORAGE_BACKEND:-${default_storage_backend}}" == "etcd3" ]]; then sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" else sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f9206d69b12..bb6a5b2e05a 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -721,12 +721,16 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" + # Get default storage backend from manifest file. + local -r default_storage_backend=$(cat "${temp_file}" | \ + grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ + sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") if [[ -n "${STORAGE_BACKEND:-}" ]]; then sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" else sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" fi - if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then + if [[ "${STORAGE_BACKEND:-${default_storage_backend}}" == "etcd3" ]]; then sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" else sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}" diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 53e7e38b874..2b0bbf912f0 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -476,12 +476,16 @@ prepare_etcd_manifest() { sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${etcd_temp_file}" sed -i -e "s@{{ *hostname *}}@$host_name@g" "${etcd_temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}" + # Get default storage backend from manifest file. + local -r default_storage_backend=$(cat "${temp_file}" | \ + grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ + sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") if [[ -n "${STORAGE_BACKEND:-}" ]]; then sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" else sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" fi - if [[ "${STORAGE_BACKEND:-}" == "etcd3" ]]; then + if [[ "${STORAGE_BACKEND:-${default_storage_backend}}" == "etcd3" ]]; then sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}" else sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}" From a2a861b8f9840aa950fe8c812d573c8bebc1b5a6 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 7 Feb 2017 14:06:52 +0100 Subject: [PATCH 4/4] Fix flag verification --- hack/verify-flags/exceptions.txt | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 600b2ed24d2..a4c870cc98f 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -16,17 +16,26 @@ cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/gce/container-linux/configure-helper.sh: authorization_mode+=",ABAC" cluster/gce/container-linux/configure-helper.sh: authorization_mode+=",Webhook" +cluster/gce/container-linux/configure-helper.sh: grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ +cluster/gce/container-linux/configure-helper.sh: sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") +cluster/gce/container-linux/configure-helper.sh: sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" +cluster/gce/container-linux/configure-helper.sh: sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" cluster/gce/container-linux/configure-helper.sh: local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" cluster/gce/container-linux/configure-helper.sh: local authorization_mode="RBAC" -cluster/gce/container-linux/configure-helper.sh: sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" cluster/gce/container-linux/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/gci/configure-helper.sh: authorization_mode+=",ABAC" cluster/gce/gci/configure-helper.sh: authorization_mode+=",Webhook" +cluster/gce/gci/configure-helper.sh: grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ +cluster/gce/gci/configure-helper.sh: sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") +cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" +cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" cluster/gce/gci/configure-helper.sh: local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" cluster/gce/gci/configure-helper.sh: local authorization_mode="RBAC" -cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" -cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" +cluster/gce/trusty/configure-helper.sh: grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ +cluster/gce/trusty/configure-helper.sh: sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g") +cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}" +cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}" cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py: context['pillar'] = {'num_nodes': get_node_count()} @@ -50,9 +59,8 @@ cluster/photon-controller/util.sh: node_name=${1} cluster/photon-controller/util.sh: ssh_key=$(ssh-add -L | head -1) cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \ cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest:{% set params = pillar['autoscaler_mig_config'] + " " + cloud_config -%} -cluster/saltbase/salt/etcd/etcd.manifest: "value": "{{ storage_backend }}" +cluster/saltbase/salt/etcd/etcd.manifest: "value": "{{ pillar.get('storage_backend', 'etcd3') }}" cluster/saltbase/salt/etcd/etcd.manifest:{% if pillar.get('storage_backend', 'etcd3') == 'etcd3' -%} -cluster/saltbase/salt/etcd/etcd.manifest:{% set storage_backend = pillar.get('storage_backend', 'etcd3') -%} cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% if pillar.get('enable_hostpath_provisioner', '').lower() == 'true' -%}