From 9bf6bc1817fd63a5373d7fe83b45cc39329fc131 Mon Sep 17 00:00:00 2001
From: Yifan Gu <yifan.gu@coreos.com>
Date: Fri, 13 Nov 2015 13:03:44 -0800
Subject: [PATCH] cluster/gce:  Move build-kube-env to util.sh

This allows all 3 distros(coreos, debian, trusty) to share the same
build-kube-env.
---
 cluster/gce/coreos/helper.sh | 132 ++---------------------------------
 cluster/gce/coreos/node.yaml |  20 ++++--
 cluster/gce/debian/helper.sh | 119 -------------------------------
 cluster/gce/trusty/helper.sh |   6 +-
 cluster/gce/util.sh          | 131 +++++++++++++++++++++++++++++++++-
 5 files changed, 150 insertions(+), 258 deletions(-)

diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh
index a6eecb7d673..4e86f16e399 100755
--- a/cluster/gce/coreos/helper.sh
+++ b/cluster/gce/coreos/helper.sh
@@ -16,134 +16,10 @@
 
 # A library of helper functions and constant for coreos os distro
 
-# $1: if 'true', we're building a master yaml, else a node
-function build-kube-env {
-  local master=$1
-  local file=$2
-
-  build-runtime-config
-
-  rm -f ${file}
-  # TODO(dawnchen): master node is still running with debian image
-  if [[ "${master}" == "true" ]]; then
-  cat >$file <<EOF
-KUBERNETES_MASTER: "true"
-ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z))
-INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
-NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
-CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16})
-SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
-SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH})
-SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL})
-SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH})
-SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE})
-ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false})
-ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none})
-ENABLE_CLUSTER_LOGGING: $(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})
-ENABLE_CLUSTER_UI: $(yaml-quote ${ENABLE_CLUSTER_UI:-false})
-ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
-LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
-ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
-ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
-ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false})
-CLUSTER_REGISTRY_DISK: $(yaml-quote ${CLUSTER_REGISTRY_DISK})
-CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE})
-DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
-DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
-DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
-KUBE_USER: $(yaml-quote ${KUBE_USER})
-KUBE_PASSWORD: $(yaml-quote ${KUBE_PASSWORD})
-KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN})
-KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-})
-KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
-ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
-MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
-RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG})
-KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME})
-KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME})
-RKT_VERSION: $(yaml-quote ${RKT_VERSION})
-CA_CERT: $(yaml-quote ${CA_CERT_BASE64})
-MASTER_CERT: $(yaml-quote ${MASTER_CERT_BASE64:-})
-MASTER_KEY: $(yaml-quote ${MASTER_KEY_BASE64:-})
-KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-})
-KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-})
-KUBECFG_CERT: $(yaml-quote ${KUBECFG_CERT_BASE64:-})
-KUBECFG_KEY: $(yaml-quote ${KUBECFG_KEY_BASE64:-})
-KUBELET_APISERVER: $(yaml-quote ${KUBELET_APISERVER:-})
-NUM_MINIONS: $(yaml-quote ${NUM_MINIONS})
-EOF
-  else
-    cat >>$file <<EOF
-KUBERNETES_MASTER="false"
-ENV_TIMESTAMP=$(date -u +%Y-%m-%dT%T%z)
-INSTANCE_PREFIX=${INSTANCE_PREFIX}
-NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX}
-SERVER_BINARY_TAR_URL=${SERVER_BINARY_TAR_URL}
-SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE}
-ENABLE_CLUSTER_MONITORING=${ENABLE_CLUSTER_MONITORING:-none}
-ENABLE_CLUSTER_LOGGING=${ENABLE_CLUSTER_LOGGING:-false}
-ENABLE_CLUSTER_UI=${ENABLE_CLUSTER_UI:-false}
-ENABLE_NODE_LOGGING=${ENABLE_NODE_LOGGING:-false}
-LOGGING_DESTINATION=${LOGGING_DESTINATION:-}
-ELASTICSEARCH_LOGGING_REPLICAS=${ELASTICSEARCH_LOGGING_REPLICAS:-}
-ENABLE_CLUSTER_DNS=${ENABLE_CLUSTER_DNS:-false}
-ENABLE_CLUSTER_REGISTRY=${ENABLE_CLUSTER_REGISTRY:-false}
-NUM_MINIONS=${NUM_MINIONS}
-DNS_REPLICAS=${DNS_REPLICAS:-}
-DNS_SERVER_IP=${DNS_SERVER_IP:-}
-DNS_DOMAIN=${DNS_DOMAIN:-}
-KUBELET_TOKEN=${KUBELET_TOKEN:-}
-KUBE_PROXY_TOKEN=${KUBE_PROXY_TOKEN:-}
-ADMISSION_CONTROL=${ADMISSION_CONTROL:-}
-MASTER_IP_RANGE=${MASTER_IP_RANGE}
-KUBERNETES_MASTER_NAME=${MASTER_NAME}
-ZONE=${ZONE}
-EXTRA_DOCKER_OPTS=${EXTRA_DOCKER_OPTS:-}
-PROJECT_ID=${PROJECT}
-KUBERNETES_CONTAINER_RUNTIME=${CONTAINER_RUNTIME}
-RKT_VERSION=${RKT_VERSION}
-KUBERNETES_CONFIGURE_CBR0=${KUBERNETES_CONFIGURE_CBR0:-true}
-CA_CERT=${CA_CERT_BASE64}
-KUBELET_CERT=${KUBELET_CERT_BASE64:-}
-KUBELET_KEY=${KUBELET_KEY_BASE64:-}
-EOF
-  fi
-}
-
-# create-master-instance creates the master instance. If called with
-# an argument, the argument is used as the name to a reserved IP
-# address for the master. (In the case of upgrade/repair, we re-use
-# the same IP.)
-#
-# It requires a whole slew of assumed variables, partially due to to
-# the call to write-master-env. Listing them would be rather
-# futile. Instead, we list the required calls to ensure any additional
-# variables are set:
-#   ensure-temp-dir
-#   detect-project
-#   get-bearer-token
-#
-# TODO(dawnchen): Convert master node to use coreos image too
-function create-master-instance {
-  local address_opt=""
-  [[ -n ${1:-} ]] && address_opt="--address ${1}"
-
-  write-master-env
-  gcloud compute instances create "${MASTER_NAME}" \
-    ${address_opt} \
-    --project "${PROJECT}" \
-    --zone "${ZONE}" \
-    --machine-type "${MASTER_SIZE}" \
-    --image-project="${MASTER_IMAGE_PROJECT}" \
-    --image "${MASTER_IMAGE}" \
-    --tags "${MASTER_TAG}" \
-    --network "${NETWORK}" \
-    --scopes "storage-ro,compute-rw,monitoring,logging-write" \
-    --can-ip-forward \
-    --metadata-from-file \
-      "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh,kube-env=${KUBE_TEMP}/master-kube-env.yaml" \
-    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
-}
+# By sourcing debian's helper.sh, we use the same create-master-instance
+# functions as debian. But we overwrite the create-node-instance-template
+# function to use coreos.
+source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
 
 # TODO(dawnchen): Check $CONTAINER_RUNTIME to decide which
 # cloud_config yaml file should be passed
diff --git a/cluster/gce/coreos/node.yaml b/cluster/gce/coreos/node.yaml
index 2f2d7870a3b..3ca473be617 100644
--- a/cluster/gce/coreos/node.yaml
+++ b/cluster/gce/coreos/node.yaml
@@ -75,10 +75,13 @@ coreos:
         [Service]
         Type=oneshot
         RemainAfterExit=yes
-        ExecStart=/usr/bin/curl --fail --silent --show-error \
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
         -H "X-Google-Metadata-Request: True" \
-        -o /etc/kube-env \
+        -o /etc/kube-env.yaml \
         http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/cp /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
 
     - name: kubernetes-install-rkt.service
       command: start
@@ -145,16 +148,19 @@ coreos:
         ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/run/setup-auth.sh
         ExecStart=/opt/kubernetes/server/bin/kubelet \
-        --api-servers=https://${KUBERNETES_MASTER_NAME}.c.${PROJECT_ID}.internal \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=true \
+        --cloud-provider=gce \
         --config=/etc/kubernetes/manifests \
-        --allow-privileged=False \
+        --allow-privileged=true \
         --v=2 \
-        --cluster-dns=10.0.0.10 \
-        --cluster-domain=cluster.local \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
         --logtostderr=true \
         --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
         --rkt-path=/opt/rkt/rkt/rkt \
-        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0}
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --pod-cidr=${MASTER_IP_RANGE} \
         Restart=always
         RestartSec=10
 
diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh
index ad01714c657..32a90f1cc10 100755
--- a/cluster/gce/debian/helper.sh
+++ b/cluster/gce/debian/helper.sh
@@ -16,125 +16,6 @@
 
 # A library of helper functions and constant for debian os distro
 
-# $1: if 'true', we're building a master yaml, else a node
-function build-kube-env {
-  local master=$1
-  local file=$2
-
-  build-runtime-config
-
-  rm -f ${file}
-  cat >$file <<EOF
-ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z))
-INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
-NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
-CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16})
-SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
-SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH})
-SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL})
-SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH})
-SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE})
-KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME})
-ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false})
-ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none})
-ENABLE_L7_LOADBALANCING: $(yaml-quote ${ENABLE_L7_LOADBALANCING:-none})
-ENABLE_CLUSTER_LOGGING: $(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})
-ENABLE_CLUSTER_UI: $(yaml-quote ${ENABLE_CLUSTER_UI:-false})
-ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
-LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
-ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
-ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
-ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false})
-CLUSTER_REGISTRY_DISK: $(yaml-quote ${CLUSTER_REGISTRY_DISK})
-CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE})
-DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
-DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
-DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
-KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-})
-KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
-ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
-MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
-RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG})
-CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-})
-KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-})
-KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-})
-NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-})
-OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})
-OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-})
-OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})
-E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-})
-EOF
-  if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
-    cat >>$file <<EOF
-KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote ${KUBE_APISERVER_REQUEST_TIMEOUT})
-EOF
-  fi
-  if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
-    cat >>$file <<EOF
-TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
-EOF
-  fi
-  if [ -n "${TEST_CLUSTER:-}" ]; then
-    cat >>$file <<EOF
-TEST_CLUSTER: $(yaml-quote ${TEST_CLUSTER})
-EOF
-  fi
-  if [[ "${master}" == "true" ]]; then
-    # Master-only env vars.
-    cat >>$file <<EOF
-KUBERNETES_MASTER: "true"
-KUBE_USER: $(yaml-quote ${KUBE_USER})
-KUBE_PASSWORD: $(yaml-quote ${KUBE_PASSWORD})
-KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN})
-MASTER_CERT: $(yaml-quote ${MASTER_CERT_BASE64:-})
-MASTER_KEY: $(yaml-quote ${MASTER_KEY_BASE64:-})
-KUBECFG_CERT: $(yaml-quote ${KUBECFG_CERT_BASE64:-})
-KUBECFG_KEY: $(yaml-quote ${KUBECFG_KEY_BASE64:-})
-KUBELET_APISERVER: $(yaml-quote ${KUBELET_APISERVER:-})
-ENABLE_MANIFEST_URL: $(yaml-quote ${ENABLE_MANIFEST_URL:-false})
-MANIFEST_URL: $(yaml-quote ${MANIFEST_URL:-})
-MANIFEST_URL_HEADER: $(yaml-quote ${MANIFEST_URL_HEADER:-})
-NUM_MINIONS: $(yaml-quote ${NUM_MINIONS})
-EOF
-    if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
-      cat >>$file <<EOF
-APISERVER_TEST_ARGS: $(yaml-quote ${APISERVER_TEST_ARGS})
-EOF
-    fi
-    if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
-      cat >>$file <<EOF
-KUBELET_TEST_ARGS: $(yaml-quote ${KUBELET_TEST_ARGS})
-EOF
-    fi
-    if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
-      cat >>$file <<EOF
-CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote ${CONTROLLER_MANAGER_TEST_ARGS})
-EOF
-    fi
-    if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
-      cat >>$file <<EOF
-SCHEDULER_TEST_ARGS: $(yaml-quote ${SCHEDULER_TEST_ARGS})
-EOF
-    fi
-  else
-    # Node-only env vars.
-    cat >>$file <<EOF
-KUBERNETES_MASTER: "false"
-ZONE: $(yaml-quote ${ZONE})
-EXTRA_DOCKER_OPTS: $(yaml-quote ${EXTRA_DOCKER_OPTS:-})
-EOF
-    if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
-      cat >>$file <<EOF
-KUBELET_TEST_ARGS: $(yaml-quote ${KUBELET_TEST_ARGS})
-EOF
-    fi
-    if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
-      cat >>$file <<EOF
-KUBEPROXY_TEST_ARGS: $(yaml-quote ${KUBEPROXY_TEST_ARGS})
-EOF
-    fi
-  fi
-}
 
 # create-master-instance creates the master instance. If called with
 # an argument, the argument is used as the name to a reserved IP
diff --git a/cluster/gce/trusty/helper.sh b/cluster/gce/trusty/helper.sh
index 58a6fe07322..d5c9892e21e 100755
--- a/cluster/gce/trusty/helper.sh
+++ b/cluster/gce/trusty/helper.sh
@@ -22,9 +22,9 @@
 # replaced upstart with systemd as the init system. Consequently, the
 # configuration cannot work on these images.
 
-# By sourcing debian's helper.sh, we use the same build-kube-env and
-# create-master-instance functions as debian. But we overwrite the
-# create-node-instance-template function to use Ubuntu.
+# By sourcing debian's helper.sh, we use the same  create-master-instance
+# functions as debian. But we overwrite the create-node-instance-template
+# function to use Ubuntu.
 source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
 
 # $1: template name (required)
diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
index d1e80d4783e..655221c9d16 100755
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -351,7 +351,7 @@ function get-template-name-from-version {
 # $2: The scopes flag.
 # $3: The minion start script metadata from file.
 # $4: The kube-env metadata.
-# $5(optional): Additional metadata for Ubuntu trusty nodes.
+# $5 and others: Additional user defined metadata.
 function create-node-template {
   detect-project
   local template_name="$1"
@@ -1254,3 +1254,132 @@ function build-runtime-config() {
       fi
   fi
 }
+
+# $1: if 'true', we're building a master yaml, else a node
+function build-kube-env {
+  local master=$1
+  local file=$2
+
+  build-runtime-config
+
+  rm -f ${file}
+  cat >$file <<EOF
+ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z))
+INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
+NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
+CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16})
+SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
+SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH})
+SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL})
+SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH})
+SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE})
+KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME})
+ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false})
+ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none})
+ENABLE_L7_LOADBALANCING: $(yaml-quote ${ENABLE_L7_LOADBALANCING:-none})
+ENABLE_CLUSTER_LOGGING: $(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})
+ENABLE_CLUSTER_UI: $(yaml-quote ${ENABLE_CLUSTER_UI:-false})
+ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
+LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
+ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
+ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
+ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false})
+CLUSTER_REGISTRY_DISK: $(yaml-quote ${CLUSTER_REGISTRY_DISK})
+CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE})
+DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
+DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
+DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
+KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-})
+KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
+ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
+MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
+RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG})
+CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-})
+KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-})
+KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-})
+NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-})
+OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})
+OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-})
+OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})
+E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-})
+EOF
+  if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
+    cat >>$file <<EOF
+KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote ${KUBE_APISERVER_REQUEST_TIMEOUT})
+EOF
+  fi
+  if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
+    cat >>$file <<EOF
+TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
+EOF
+  fi
+  if [ -n "${TEST_CLUSTER:-}" ]; then
+    cat >>$file <<EOF
+TEST_CLUSTER: $(yaml-quote ${TEST_CLUSTER})
+EOF
+  fi
+  if [[ "${master}" == "true" ]]; then
+    # Master-only env vars.
+    cat >>$file <<EOF
+KUBERNETES_MASTER: $(yaml-quote "true")
+KUBE_USER: $(yaml-quote ${KUBE_USER})
+KUBE_PASSWORD: $(yaml-quote ${KUBE_PASSWORD})
+KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN})
+MASTER_CERT: $(yaml-quote ${MASTER_CERT_BASE64:-})
+MASTER_KEY: $(yaml-quote ${MASTER_KEY_BASE64:-})
+KUBECFG_CERT: $(yaml-quote ${KUBECFG_CERT_BASE64:-})
+KUBECFG_KEY: $(yaml-quote ${KUBECFG_KEY_BASE64:-})
+KUBELET_APISERVER: $(yaml-quote ${KUBELET_APISERVER:-})
+ENABLE_MANIFEST_URL: $(yaml-quote ${ENABLE_MANIFEST_URL:-false})
+MANIFEST_URL: $(yaml-quote ${MANIFEST_URL:-})
+MANIFEST_URL_HEADER: $(yaml-quote ${MANIFEST_URL_HEADER:-})
+NUM_MINIONS: $(yaml-quote ${NUM_MINIONS})
+EOF
+    if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+APISERVER_TEST_ARGS: $(yaml-quote ${APISERVER_TEST_ARGS})
+EOF
+    fi
+    if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+KUBELET_TEST_ARGS: $(yaml-quote ${KUBELET_TEST_ARGS})
+EOF
+    fi
+    if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote ${CONTROLLER_MANAGER_TEST_ARGS})
+EOF
+    fi
+    if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+SCHEDULER_TEST_ARGS: $(yaml-quote ${SCHEDULER_TEST_ARGS})
+EOF
+    fi
+  else
+    # Node-only env vars.
+    cat >>$file <<EOF
+KUBERNETES_MASTER: $(yaml-quote "false")
+ZONE: $(yaml-quote ${ZONE})
+EXTRA_DOCKER_OPTS: $(yaml-quote ${EXTRA_DOCKER_OPTS:-})
+EOF
+    if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+KUBELET_TEST_ARGS: $(yaml-quote ${KUBELET_TEST_ARGS})
+EOF
+    fi
+    if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+KUBEPROXY_TEST_ARGS: $(yaml-quote ${KUBEPROXY_TEST_ARGS})
+EOF
+    fi
+  fi
+  if [[ "${OS_DISTRIBUTION}" == "coreos" ]]; then
+    # CoreOS-only env vars. TODO(yifan): Make them available on other distros.
+    cat >>$file <<EOF
+KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME:-docker})
+RKT_VERSION: $(yaml-quote ${RKT_VERSION:-})
+RKT_PATH: $(yaml-quote ${RKT_PATH:-})
+KUBERNETES_CONFIGURE_CBR0: $(yaml-quote ${KUBERNETES_CONFIGURE_CBR0:-true})
+EOF
+  fi
+}