Merge pull request #24172 from andyzheng0831/fix

Automatic merge from submit-queue

Trusty: Fixes for running GKE master

This PR includes two fixes for running GKE master on our image: 
(1) The kubelet command line assembly had a missing part for cbr0. We did not catch it because the code path is not covered by OSS k8s tests;

(2) Remove the "" from the variables in the cert files. It causes a parsing issue in GKE. Again, this code path is not covered by k8s tests.

This PR also refactors the code for assembling kubelet flag. I move all logic into a single function assemble_kubelet_flags in configure-helper.sh for better readability and also simplify node.yaml and master.yaml. 

@roberthbailey @dchen1107 please review it, and mark it as cherrypick-candidate. This PR is verified by @maisem. Together with his CL for GKE, we can run GKE cluster with master on our image and nodes on ContainerVM.

cc/ @maisem @fabioy @wonderfly FYI

cluster/gce/trusty/configure-helper.sh

@@ -62,12 +62,12 @@ kind: Config
 users:
 - name: kubelet
   user:
-    client-certificate-data: "${KUBELET_CERT}"
+    client-certificate-data: ${KUBELET_CERT}
-    client-key-data: "${KUBELET_KEY}"
+    client-key-data: ${KUBELET_KEY}
 clusters:
 - name: local
   cluster:
-    certificate-authority-data: "${KUBELET_CA_CERT}"
+    certificate-authority-data: ${KUBELET_CA_CERT}
 contexts:
 - context:
     cluster: local
@@ -85,11 +85,11 @@ kind: Config
 users:
 - name: kube-proxy
   user:
-    token: "${KUBE_PROXY_TOKEN}"
+    token: ${KUBE_PROXY_TOKEN}
 clusters:
 - name: local
   cluster:
-    certificate-authority-data: "${CA_CERT}"
+    certificate-authority-data: ${CA_CERT}
 contexts:
 - context:
     cluster: local
@@ -137,7 +137,7 @@ install_additional_packages() {
 # Assembles kubelet command line flags.
 # It should be called by master and nodes before running kubelet process. The caller
 # needs to source the config file /etc/kube-env. This function sets the following
-# variable that will be used in kubelet command line.
+# variable that will be used as the kubelet command line flags
 #   KUBELET_CMD_FLAGS
 assemble_kubelet_flags() {
   KUBELET_CMD_FLAGS="--v=2"
@@ -150,30 +150,37 @@ assemble_kubelet_flags() {
   if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
     KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}"
   fi
-  if [ "${KUBERNETES_MASTER:-}" = "true" ] && \
+  if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
-     [ ! -z "${KUBELET_APISERVER:-}" ] && \
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-debugging-handlers=false --hairpin-mode=none"
+    if [ ! -z "${KUBELET_APISERVER:-}" ] && \
        [ ! -z "${KUBELET_CERT:-}" ] && \
        [ ! -z "${KUBELET_KEY:-}" ]; then
-    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}"
+      KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER} --register-schedulable=false --reconcile-cidr=false --pod-cidr=10.123.45.0/30"
-    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false"
+    else
-    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30"
+      KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=${MASTER_IP_RANGE}"
     fi
-  if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
+  else # For nodes
-    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-debugging-handlers=true --api-servers=https://${KUBERNETES_MASTER_NAME}"
-  fi
+    if [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
-  if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
-    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none"
-  elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
        [ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \
        [ "${HAIRPIN_MODE:-}" = "none" ]; then
       KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}"
     fi
+  fi
+  if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
+    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
+  fi
   if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then
     KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
   fi
   if [ -n "${NODE_LABELS:-}" ]; then
     KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}"
   fi
+  if [ "${ALLOCATE_NODE_CIDRS:-}" = "true" ]; then
+     KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --configure-cbr0=${ALLOCATE_NODE_CIDRS}"
+  fi
+  # Add the unconditional flags
+  KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --cloud-provider=gce --allow-privileged=true --cgroup-root=/ --system-cgroups=/system --kubelet-cgroups=/kubelet --babysit-daemons=true --config=/etc/kubernetes/manifests --cluster-dns=${DNS_SERVER_IP} --cluster-domain=${DNS_DOMAIN}"
 }
 
 restart_docker_daemon() {
@@ -314,10 +321,15 @@ create_master_auth() {
   if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
     cat <<EOF >/etc/gce.conf
 [global]
-token-url = "${TOKEN_URL}"
+token-url = ${TOKEN_URL}
-token-body = "${TOKEN_BODY}"
+token-body = ${TOKEN_BODY}
-project-id = "${PROJECT_ID}"
+project-id = ${PROJECT_ID}
-network-name = "${NODE_NETWORK}"
+network-name = ${NODE_NETWORK}
+EOF
+  fi
+  if [ -n "${MULTIZONE:-}" ]; then
+    cat <<EOF >>/etc/gce.conf
+multizone = ${MULTIZONE}
 EOF
   fi
 }

cluster/gce/trusty/master.yaml

@@ -127,20 +127,7 @@ script
 	# Assemble command line flags based on env variables, which will put the string
 	# of flags in variable KUBELET_CMD_FLAGS
 	assemble_kubelet_flags
-	
+	/usr/bin/kubelet ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
-	/usr/bin/kubelet \
-		--enable-debugging-handlers=false \
-		--cloud-provider=gce \
-		--config=/etc/kubernetes/manifests \
-		--allow-privileged=true \
-		--cluster-dns=${DNS_SERVER_IP} \
-		--cluster-domain=${DNS_DOMAIN} \
-		--configure-cbr0=${ALLOCATE_NODE_CIDRS} \
-		--cgroup-root=/ \
-		--system-cgroups=/system \
-		--kubelet-cgroups=/kubelet \
-		--babysit-daemons=true \
-		${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
 end script
 
 # Wait for 10s to start kubelet again.

cluster/gce/trusty/node.yaml

@@ -125,21 +125,7 @@ script
 	# Assemble command line flags based on env variables, which will put the string
 	# of flags in variable KUBELET_CMD_FLAGS.
 	assemble_kubelet_flags
-
+	/usr/bin/kubelet ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
-	/usr/bin/kubelet \
-		--api-servers=https://${KUBERNETES_MASTER_NAME} \
-		--enable-debugging-handlers=true \
-		--cloud-provider=gce \
-		--config=/etc/kubernetes/manifests \
-		--allow-privileged=true \
-		--cluster-dns=${DNS_SERVER_IP} \
-		--cluster-domain=${DNS_DOMAIN} \
-		--configure-cbr0=true \
-		--cgroup-root=/ \
-		--system-cgroups=/system \
-		--kubelet-cgroups=/kubelet \
-		--babysit-daemons=true \
-		${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
 end script
 
 # Wait for 10s to start kubelet again.