From 827ee794d6ff52fc36152803b3e88ecf71e6cd92 Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Wed, 20 Jul 2016 16:01:02 +0200 Subject: [PATCH] Modified etcd manifest to support clustering. Modified etcd manifest to support clustering. --- cluster/common.sh | 1 + cluster/gce/configure-vm.sh | 3 ++- cluster/gce/gci/configure-helper.sh | 15 +++++++++++++++ cluster/gce/trusty/configure-helper.sh | 15 +++++++++++++++ cluster/saltbase/salt/etcd/etcd.manifest | 19 ++++++++++++++++++- 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 2a0812829d7..492bdc63a3d 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -644,6 +644,7 @@ ENABLE_MANIFEST_URL: $(yaml-quote ${ENABLE_MANIFEST_URL:-false}) MANIFEST_URL: $(yaml-quote ${MANIFEST_URL:-}) MANIFEST_URL_HEADER: $(yaml-quote ${MANIFEST_URL_HEADER:-}) NUM_NODES: $(yaml-quote ${NUM_NODES}) +INITIAL_ETCD_CLUSTER: $(yaml-quote ${MASTER_NAME}) EOF if [ -n "${APISERVER_TEST_ARGS:-}" ]; then cat >>$file <>/srv/salt-overlay/pillar/cluster-params.sls diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 1e92b7eaae8..5306d236716 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -507,12 +507,27 @@ function start-kube-proxy { # $4: value for variable 'cpulimit' # $5: pod name, which should be either etcd or etcd-events function prepare-etcd-manifest { + local host_name=$(hostname) + local etcd_cluster="" + local cluster_state="new" + for host in $(echo "${INITIAL_ETCD_CLUSTER}" | tr "," "\n"); do + etcd_host="etcd-${host}=http://${host}:$3" + if [[ -n "${etcd_cluster}" ]]; then + etcd_cluster+="," + cluster_state="existing" + fi + etcd_cluster+="${etcd_host}" + done local -r temp_file="/tmp/$5" cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}" + remove-salt-config-comments "${temp_file}" sed -i -e "s@{{ *suffix *}}@$1@g" "${temp_file}" sed -i -e "s@{{ *port *}}@$2@g" "${temp_file}" sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}" sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}" + sed -i -e "s@{{ *host_name *}}@$host_name@g" "${temp_file}" + sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" + sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}" # Replace the volume host path. sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}" mv "${temp_file}" /etc/kubernetes/manifests diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index b5ec9ee958d..2df724088d3 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -409,12 +409,27 @@ create_master_kubelet_auth() { # $4: value for variable 'cpulimit' # $5: pod name, which should be either etcd or etcd-events prepare_etcd_manifest() { + local host_name=$(hostname) + local etcd_cluster="" + local cluster_state="new" + for host in $(echo "${INITIAL_ETCD_CLUSTER}" | tr "," "\n"); do + etcd_host="etcd-${host}=http://${host}:$3" + if [[ -n "${etcd_cluster}" ]]; then + etcd_cluster+="," + cluster_state="existing" + fi + etcd_cluster+="${etcd_host}" + done etcd_temp_file="/tmp/$5" cp /home/kubernetes/kube-manifests/kubernetes/gci-trusty/etcd.manifest "${etcd_temp_file}" + remove_salt_config_comments "${etcd_temp_file}" sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}" sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}" sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}" + sed -i -e "s@{{ *host_name *}}@$host_name@g" "${etcd_temp_file}" + sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}" + sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${etcd_temp_file}" # Replace the volume host path sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}" mv "${etcd_temp_file}" /etc/kubernetes/manifests diff --git a/cluster/saltbase/salt/etcd/etcd.manifest b/cluster/saltbase/salt/etcd/etcd.manifest index bbf22ce4cbc..880a88bc7fd 100644 --- a/cluster/saltbase/salt/etcd/etcd.manifest +++ b/cluster/saltbase/salt/etcd/etcd.manifest @@ -1,3 +1,20 @@ +{% set cluster_state = 'new' -%} +{% set host_name = pillar.get('master_name', '') -%} +{% set etcd_cluster_array = pillar.get('initial_etcd_cluster', '').split(',') -%} +{% set etcd_cluster = '' -%} +{# We use vars dictionary to pass variables set inside the for loop, because jinja defines new variables inside the for loop that hide variables from the outside. #} +{% set vars = {'etcd_cluster': '', 'cluster_state': cluster_state} -%} +{% for host in etcd_cluster_array -%} + {% if etcd_cluster != '' -%} + {% set cluster_state = 'existing' -%} + {% set etcd_cluster = etcd_cluster ~ ',' -%} + {% endif -%} + {% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=http://' ~ host ~ ':' ~ server_port -%} + {% do vars.update({'etcd_cluster': etcd_cluster, 'cluster_state': cluster_state}) -%} +{% endfor -%} +{% set etcd_cluster = vars.etcd_cluster -%} +{% set cluster_state = vars.cluster_state -%} + { "apiVersion": "v1", "kind": "Pod", @@ -19,7 +36,7 @@ "command": [ "/bin/sh", "-c", - "/usr/local/bin/etcd --listen-peer-urls http://127.0.0.1:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} --data-dir /var/etcd/data{{ suffix }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + "/usr/local/bin/etcd --name etcd-{{ host_name }} --listen-peer-urls http://{{ host_name }}:{{ server_port }} --initial-advertise-peer-urls http://{{ host_name }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster {{ etcd_cluster }} --initial-cluster-state {{ cluster_state }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" ], "livenessProbe": { "httpGet": {