Merge pull request #24172 from andyzheng0831/fix

Automatic merge from submit-queue

Trusty: Fixes for running GKE master

This PR includes two fixes for running GKE master on our image: 
(1) The kubelet command line assembly had a missing part for cbr0. We did not catch it because the code path is not covered by OSS k8s tests;

(2) Remove the "" from the variables in the cert files. It causes a parsing issue in GKE. Again, this code path is not covered by k8s tests.

This PR also refactors the code for assembling kubelet flag. I move all logic into a single function assemble_kubelet_flags in configure-helper.sh for better readability and also simplify node.yaml and master.yaml. 

@roberthbailey @dchen1107 please review it, and mark it as cherrypick-candidate. This PR is verified by @maisem. Together with his CL for GKE, we can run GKE cluster with master on our image and nodes on ContainerVM.

cc/ @maisem @fabioy @wonderfly FYI
This commit is contained in:
k8s-merge-robot 2016-04-13 20:04:15 -07:00
commit a12cec52e5
3 changed files with 38 additions and 53 deletions

View File

@ -62,12 +62,12 @@ kind: Config
users:
- name: kubelet
user:
client-certificate-data: "${KUBELET_CERT}"
client-key-data: "${KUBELET_KEY}"
client-certificate-data: ${KUBELET_CERT}
client-key-data: ${KUBELET_KEY}
clusters:
- name: local
cluster:
certificate-authority-data: "${KUBELET_CA_CERT}"
certificate-authority-data: ${KUBELET_CA_CERT}
contexts:
- context:
cluster: local
@ -85,11 +85,11 @@ kind: Config
users:
- name: kube-proxy
user:
token: "${KUBE_PROXY_TOKEN}"
token: ${KUBE_PROXY_TOKEN}
clusters:
- name: local
cluster:
certificate-authority-data: "${CA_CERT}"
certificate-authority-data: ${CA_CERT}
contexts:
- context:
cluster: local
@ -137,7 +137,7 @@ install_additional_packages() {
# Assembles kubelet command line flags.
# It should be called by master and nodes before running kubelet process. The caller
# needs to source the config file /etc/kube-env. This function sets the following
# variable that will be used in kubelet command line.
# variable that will be used as the kubelet command line flags
# KUBELET_CMD_FLAGS
assemble_kubelet_flags() {
KUBELET_CMD_FLAGS="--v=2"
@ -150,30 +150,37 @@ assemble_kubelet_flags() {
if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}"
fi
if [ "${KUBERNETES_MASTER:-}" = "true" ] && \
[ ! -z "${KUBELET_APISERVER:-}" ] && \
[ ! -z "${KUBELET_CERT:-}" ] && \
[ ! -z "${KUBELET_KEY:-}" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}"
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false"
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30"
if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-debugging-handlers=false --hairpin-mode=none"
if [ ! -z "${KUBELET_APISERVER:-}" ] && \
[ ! -z "${KUBELET_CERT:-}" ] && \
[ ! -z "${KUBELET_KEY:-}" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER} --register-schedulable=false --reconcile-cidr=false --pod-cidr=10.123.45.0/30"
else
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=${MASTER_IP_RANGE}"
fi
else # For nodes
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-debugging-handlers=true --api-servers=https://${KUBERNETES_MASTER_NAME}"
if [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
[ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \
[ "${HAIRPIN_MODE:-}" = "none" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}"
fi
fi
if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
fi
if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none"
elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \
[ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \
[ "${HAIRPIN_MODE:-}" = "none" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}"
fi
if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
fi
if [ -n "${NODE_LABELS:-}" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}"
fi
if [ "${ALLOCATE_NODE_CIDRS:-}" = "true" ]; then
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --configure-cbr0=${ALLOCATE_NODE_CIDRS}"
fi
# Add the unconditional flags
KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --cloud-provider=gce --allow-privileged=true --cgroup-root=/ --system-cgroups=/system --kubelet-cgroups=/kubelet --babysit-daemons=true --config=/etc/kubernetes/manifests --cluster-dns=${DNS_SERVER_IP} --cluster-domain=${DNS_DOMAIN}"
}
restart_docker_daemon() {
@ -314,10 +321,15 @@ create_master_auth() {
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
cat <<EOF >/etc/gce.conf
[global]
token-url = "${TOKEN_URL}"
token-body = "${TOKEN_BODY}"
project-id = "${PROJECT_ID}"
network-name = "${NODE_NETWORK}"
token-url = ${TOKEN_URL}
token-body = ${TOKEN_BODY}
project-id = ${PROJECT_ID}
network-name = ${NODE_NETWORK}
EOF
fi
if [ -n "${MULTIZONE:-}" ]; then
cat <<EOF >>/etc/gce.conf
multizone = ${MULTIZONE}
EOF
fi
}

View File

@ -127,20 +127,7 @@ script
# Assemble command line flags based on env variables, which will put the string
# of flags in variable KUBELET_CMD_FLAGS
assemble_kubelet_flags
/usr/bin/kubelet \
--enable-debugging-handlers=false \
--cloud-provider=gce \
--config=/etc/kubernetes/manifests \
--allow-privileged=true \
--cluster-dns=${DNS_SERVER_IP} \
--cluster-domain=${DNS_DOMAIN} \
--configure-cbr0=${ALLOCATE_NODE_CIDRS} \
--cgroup-root=/ \
--system-cgroups=/system \
--kubelet-cgroups=/kubelet \
--babysit-daemons=true \
${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
/usr/bin/kubelet ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
end script
# Wait for 10s to start kubelet again.

View File

@ -125,21 +125,7 @@ script
# Assemble command line flags based on env variables, which will put the string
# of flags in variable KUBELET_CMD_FLAGS.
assemble_kubelet_flags
/usr/bin/kubelet \
--api-servers=https://${KUBERNETES_MASTER_NAME} \
--enable-debugging-handlers=true \
--cloud-provider=gce \
--config=/etc/kubernetes/manifests \
--allow-privileged=true \
--cluster-dns=${DNS_SERVER_IP} \
--cluster-domain=${DNS_DOMAIN} \
--configure-cbr0=true \
--cgroup-root=/ \
--system-cgroups=/system \
--kubelet-cgroups=/kubelet \
--babysit-daemons=true \
${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
/usr/bin/kubelet ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1
end script
# Wait for 10s to start kubelet again.