From 5740ceb7f6c64b83a522da94ac99ebc0b5dee670 Mon Sep 17 00:00:00 2001 From: Alain Roy Date: Mon, 13 Jun 2016 16:14:18 -0700 Subject: [PATCH] Fixes and improvements to Photon Controller backend for kube-up - Improve reliability of network address detection by using MAC address. VMware has a MAC OUI that reliably distinguishes the VM's NICs from the other NICs (like the CBR). This doesn't rely on the unreliable reporting of the portgroup. - Persist route changes. We configure routes on the master and nodes, but previously we didn't persist them so they didn't last across reboots. This persists them in /etc/network/interfaces - Fix regression that didn't configure auth for kube-apiserver with Photon Controller. - Reliably run apt-get update: Not doing this can cause apt to fail. - Remove unused nginx config in salt --- .../templates/create-dynamic-salt-files.sh | 3 -- .../templates/salt-master.sh | 1 + cluster/photon-controller/util.sh | 41 ++++++++++++++++--- cluster/saltbase/salt/docker/init.sls | 6 +-- cluster/saltbase/salt/kube-apiserver/init.sls | 2 +- .../kube-apiserver/kube-apiserver.manifest | 2 +- hack/verify-flags/exceptions.txt | 3 +- 7 files changed, 43 insertions(+), 15 deletions(-) diff --git a/cluster/photon-controller/templates/create-dynamic-salt-files.sh b/cluster/photon-controller/templates/create-dynamic-salt-files.sh index af69f713881..424c75bd4ff 100755 --- a/cluster/photon-controller/templates/create-dynamic-salt-files.sh +++ b/cluster/photon-controller/templates/create-dynamic-salt-files.sh @@ -125,6 +125,3 @@ cluster_cidr: "$NODE_IP_RANGES" allocate_node_cidrs: "${ALLOCATE_NODE_CIDRS:-true}" admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota EOF - -mkdir -p /srv/salt-overlay/salt/nginx -echo ${MASTER_HTPASSWD} > /srv/salt-overlay/salt/nginx/htpasswd diff --git a/cluster/photon-controller/templates/salt-master.sh b/cluster/photon-controller/templates/salt-master.sh index d83992a4abb..567ddf50765 100755 --- a/cluster/photon-controller/templates/salt-master.sh +++ b/cluster/photon-controller/templates/salt-master.sh @@ -28,6 +28,7 @@ grains: cbr-cidr: $MASTER_IP_RANGE cloud: photon-controller master_extra_sans: $MASTER_EXTRA_SANS + api_servers: $MASTER_NAME EOF # Auto accept all keys from minions that try to join diff --git a/cluster/photon-controller/util.sh b/cluster/photon-controller/util.sh index 55bf9e9733c..a1e6db3d26a 100755 --- a/cluster/photon-controller/util.sh +++ b/cluster/photon-controller/util.sh @@ -71,9 +71,9 @@ function detect-master { fi if [[ -z "${KUBE_MASTER_IP-}" ]]; then - # Make sure to ignore lines where it's not attached to a portgroup + # Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29) # Make sure to ignore lines that have a network interface but no address - KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}') + KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}') fi if [[ -z "${KUBE_MASTER_IP-}" ]]; then kube::log::error "Could not find Kubernetes master node IP. Make sure you've launched a cluster with 'kube-up.sh'" >&2 @@ -114,9 +114,9 @@ function detect-nodes { fi KUBE_NODE_IDS+=("${node_id}") - # Make sure to ignore lines where it's not attached to a portgroup + # Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29) # Make sure to ignore lines that have a network interface but no address - node_ip=$(${PHOTON} vm networks "${node_id}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}') + node_ip=$(${PHOTON} vm networks "${node_id}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}') KUBE_NODE_IP_ADDRESSES+=("${node_ip}") if [[ -z ${silent} ]]; then @@ -323,7 +323,11 @@ function pc-delete-vm { local rc=0 kube::log::status "Deleting VM ${vm_name}" + # In some cases, head exits before photon, so the pipline exits with + # SIGPIPE. We disable the pipefile option to hide that failure. + set +o pipefail ${PHOTON} vm show "${vm_id}" | head -1 | grep STARTED > /dev/null 2>&1 || rc=$? + set +o pipefail if [[ ${rc} -eq 0 ]]; then ${PHOTON} vm stop "${vm_id}" > /dev/null 2>&1 || rc=$? if [[ ${rc} -ne 0 ]]; then @@ -536,6 +540,28 @@ function gen-salt { ) > "${KUBE_TEMP}/${node_name}-salt.sh" } +# +# Generate a script to add a route to a host (master or node) +# The script will do two things: +# 1. Add the route immediately with the route command +# 2. Persist the route by saving it in /etc/network/interfaces +# This was done with a script because it was easier to get the quoting right +# and make it clear. +# +function gen-add-route { + route=${1} + gateway=${2} + ( + echo '#!/bin/bash' + echo '' + echo '# Immediately add route' + echo "sudo route add -net ${route} gw ${gateway}" + echo '' + echo '# Persist route so it lasts over restarts' + echo 'sed -in "s|^iface eth0.*|&\n post-up route add -net' "${route} gw ${gateway}|"'" /etc/network/interfaces' + ) > "${KUBE_TEMP}/add-route.sh" +} + # # Create the Kubernetes master VM # Sets global variables: @@ -721,10 +747,13 @@ function setup-pod-routes { local j for (( i=0; i<${#NODE_NAMES[@]}; i++)); do kube::log::status "Configuring pod routes on ${NODE_NAMES[${i}]}..." - run-ssh-cmd "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}" + gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${i}]}" "${KUBE_NODE_IP_ADDRESSES[${i}]}" + run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/add-route.sh" + for (( j=0; j<${#NODE_NAMES[@]}; j++)); do if [[ "${i}" != "${j}" ]]; then - run-ssh-cmd "${KUBE_NODE_IP_ADDRESSES[${i}]}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${j}]} gw ${KUBE_NODE_IP_ADDRESSES[${j}]}" + gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${j}]}" "${KUBE_NODE_IP_ADDRESSES[${j}]}" + run-script-remotely "${KUBE_NODE_IP_ADDRESSES[${i}]}" "${KUBE_TEMP}/add-route.sh" fi done done diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls index a27779a4efc..fa1174fbd86 100644 --- a/cluster/saltbase/salt/docker/init.sls +++ b/cluster/saltbase/salt/docker/init.sls @@ -94,13 +94,13 @@ fix-service-docker: - require: - pkg: docker-engine -'apt-key': +apt-key: cmd.run: - name: 'apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D' - unless: 'apt-key finger | grep "5811 8E89"' -'apt-update': - cmd.wait: +apt-update: + cmd.run: - name: '/usr/bin/apt-get update -y' - require: - cmd : 'apt-key' diff --git a/cluster/saltbase/salt/kube-apiserver/init.sls b/cluster/saltbase/salt/kube-apiserver/init.sls index 8d3245013dd..80661c6a4f6 100644 --- a/cluster/saltbase/salt/kube-apiserver/init.sls +++ b/cluster/saltbase/salt/kube-apiserver/init.sls @@ -1,4 +1,4 @@ -{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack'] %} +{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} # TODO: generate and distribute tokens on other cloud providers. /srv/kubernetes/known_tokens.csv: file.managed: diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index a8d024061af..cac649a4987 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -76,7 +76,7 @@ {% set basic_auth_file = "" -%} {% set authz_mode = "" -%} {% set abac_policy_file = "" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack'] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%} {% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%} {% set authz_mode = " --authorization-mode=ABAC" -%} diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 91435dfaf79..834f8df336d 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -29,8 +29,9 @@ cluster/log-dump.sh: for node_name in "${NODE_NAMES[@]}"; do cluster/log-dump.sh: local -r node_name="${1}" cluster/log-dump.sh:readonly report_dir="${1:-_artifacts}" cluster/mesos/docker/km/build.sh: km_path=$(find-binary km darwin/amd64) +cluster/photon-controller/templates/salt-master.sh: api_servers: $MASTER_NAME cluster/photon-controller/templates/salt-minion.sh: hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}') -cluster/photon-controller/util.sh: node_ip=$(${PHOTON} vm networks "${node_id}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}') +cluster/photon-controller/util.sh: node_ip=$(${PHOTON} vm networks "${node_id}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}') cluster/photon-controller/util.sh: local cert_dir="/srv/kubernetes" cluster/photon-controller/util.sh: node_name=${1} cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \