Merge pull request #24881 from vmware/vsphere-ui-fix

Fixed vSphere kube-up implementation to allow Kubernetes dashboard (UI) to work

cluster/vsphere/config-default.sh

@@ -57,5 +57,11 @@ DNS_REPLICAS=1
 # Optional: Install Kubernetes UI
 ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
 
+# We need to configure subject alternate names (SANs) for the master's certificate
+# we generate.  While users will connect via the external IP, pods (like the UI)
+# will connect via the cluster IP, from the SERVICE_CLUSTER_IP_RANGE.
+# In addition to the extra SANS here, we'll also add one for for the service IP.
+MASTER_EXTRA_SANS="DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN}"
+
 # Optional: if set to true, kube-up will configure the cluster to run e2e tests.
 E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}

cluster/vsphere/templates/create-dynamic-salt-files.sh

@@ -112,7 +112,7 @@ node_instance_prefix: $NODE_INSTANCE_PREFIX
 service_cluster_ip_range: $SERVICE_CLUSTER_IP_RANGE
 enable_cluster_monitoring: "${ENABLE_CLUSTER_MONITORING:-none}"
 enable_cluster_logging: "${ENABLE_CLUSTER_LOGGING:false}"
-enable_cluster_ui: "${ENABLE_CLUSTER_UI:false}"
+enable_cluster_ui: "${ENABLE_CLUSTER_UI:true}"
 enable_node_logging: "${ENABLE_NODE_LOGGING:false}"
 logging_destination: $LOGGING_DESTINATION
 elasticsearch_replicas: $ELASTICSEARCH_LOGGING_REPLICAS
@@ -123,6 +123,7 @@ dns_domain: $DNS_DOMAIN
 e2e_storage_test_environment: "${E2E_STORAGE_TEST_ENVIRONMENT:-false}"
 cluster_cidr: "$NODE_IP_RANGES"
 allocate_node_cidrs: "${ALLOCATE_NODE_CIDRS:-true}"
+admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
 EOF
 
 mkdir -p /srv/salt-overlay/salt/nginx

cluster/vsphere/templates/salt-master.sh

@@ -27,6 +27,7 @@ grains:
     - kubernetes-master
   cbr-cidr: $MASTER_IP_RANGE
   cloud: vsphere
+  master_extra_sans: $MASTER_EXTRA_SANS
 EOF
 
 # Auto accept all keys from minions that try to join

cluster/vsphere/util.sh

@@ -326,10 +326,12 @@ function setup-pod-routes {
   done
 
 
-  # make the pods visible to each other.
+  # Make the pods visible to each other and to the master.
+  # The master needs have routes to the pods for the UI to work.
   local j
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
      printf "setting up routes for ${NODE_NAMES[$i]}"
+     kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
      for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
         if [[ $i != $j ]]; then
            kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}"
@@ -355,6 +357,18 @@ function kube-up {
   local htpasswd
   htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
 
+  # This calculation of the service IP should work, but if you choose an
+  # alternate subnet, there's a small chance you'd need to modify the
+  # service_ip, below.  We'll choose an IP like 10.244.240.1 by taking
+  # the first three octets of the SERVICE_CLUSTER_IP_RANGE and tacking
+  # on a .1
+  local octets
+  local service_ip
+  octets=($(echo "${SERVICE_CLUSTER_IP_RANGE}" | sed -e 's|/.*||' -e 's/\./ /g'))
+  ((octets[3]+=1))
+  service_ip=$(echo "${octets[*]}" | sed 's/ /./g')
+  MASTER_EXTRA_SANS="IP:${service_ip},DNS:${MASTER_NAME},${MASTER_EXTRA_SANS}"
+
   echo "Starting master VM (this can take a minute)..."
 
   (
@@ -371,6 +385,7 @@ function kube-up {
     echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
     echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
     echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
+    echo "readonly ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI:-false}'"
     echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
     echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
     echo "readonly KUBE_USER='${KUBE_USER:-}'"
@@ -379,6 +394,7 @@ function kube-up {
     echo "readonly SALT_TAR='${SALT_TAR##*/}'"
     echo "readonly MASTER_HTPASSWD='${htpasswd}'"
     echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
+    echo "readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/install-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-master.sh"