diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f5c1cde1582..065614413ec 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -249,10 +249,17 @@ EOF } function assemble-docker-flags { - local docker_opts="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" + echo "Assemble docker command line flags" + local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" if [[ "${TEST_CLUSTER:-}" == "true" ]]; then docker_opts+=" --debug" fi + local use_net_plugin="true" + if [[ "${NETWORK_PROVIDER:-}" != "kubenet" && "${NETWORK_PROVIDER:-}" != "cni" ]]; then + use_net_plugin="false" + docker_opts+=" --bridge=cbr0" + fi + # Decide whether to enable a docker registry mirror. This is taken from # the "kube-env" metadata value. if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then @@ -261,6 +268,12 @@ function assemble-docker-flags { fi echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker + # If using a network plugin, we need to explicitly restart docker daemon, because + # kubelet will not do it. + if [[ "${use_net_plugin}" == "true" ]]; then + echo "Docker command line is updated. Restart docker to pick it up" + systemctl restart docker + fi } # A helper function for loading a docker image. It keeps trying up to 5 times. @@ -321,14 +334,15 @@ function start-kubelet { if [[ -n "${KUBELET_PORT:-}" ]]; then flags+=" --port=${KUBELET_PORT}" fi + local reconcile_cidr="true" if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then flags+=" --enable-debugging-handlers=false" flags+=" --hairpin-mode=none" if [[ ! -z "${KUBELET_APISERVER:-}" && ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then flags+=" --api-servers=https://${KUBELET_APISERVER}" flags+=" --register-schedulable=false" - flags+=" --reconcile-cidr=false" flags+=" --pod-cidr=10.123.45.0/30" + reconcile_cidr="false" else flags+=" --pod-cidr=${MASTER_IP_RANGE}" fi @@ -341,6 +355,15 @@ function start-kubelet { flags+=" --hairpin-mode=${HAIRPIN_MODE}" fi fi + # Network plugin + if [[ -n "${NETWORK_PROVIDER:-}" ]]; then + flags+=" --network-plugin-dir=/home/kubernetes/bin" + flags+=" --network-plugin=${NETWORK_PROVIDER}" + fi + flags+=" --reconcile-cidr=${reconcile_cidr}" + if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then + flag+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}" + fi if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then flags+=" --manifest-url=${MANIFEST_URL}" flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}" @@ -591,7 +614,9 @@ function start-kube-controller-manager { if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi - if [[ "${ALLOCATE_NODE_CIDRS:-}" == "true" ]]; then + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then + params+=" --allocate-node-cidrs=true" + elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" fi if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then @@ -798,7 +823,6 @@ function start-lb-controller { fi } - function reset-motd { # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl) local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")" diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 211517c273c..fa94ee8f610 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -146,6 +146,17 @@ function install-kube-binary-config { else rm -f "${kube_bin}/kubelet" fi + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \ + [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then + #TODO(andyzheng0831): We should make the cni version number as a k8s env variable. + local -r cni_tar="cni-26b61728ac940c3faf827927782326e921be17b0.tar.gz" + download-or-bust "" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}" + tar xzf "${KUBE_HOME}/${cni_tar}" -C "${kube_bin}" --overwrite + mv "${kube_bin}/bin"/* "${kube_bin}" + rmdir "${kube_bin}/bin" + rm -f "${KUBE_HOME}/${cni_tar}" + fi + cp "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}" # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/. diff --git a/cluster/gce/gci/health-monitor.sh b/cluster/gce/gci/health-monitor.sh index a23fdee4588..3318a89752b 100644 --- a/cluster/gce/gci/health-monitor.sh +++ b/cluster/gce/gci/health-monitor.sh @@ -38,8 +38,9 @@ function docker_monitoring { } function kubelet_monitoring { - echo "waiting a minute for startup" - sleep 60 + echo "Wait for 2 minutes for kubelet to be fuctional" + # TODO(andyzheng0831): replace it with a more reliable method if possible. + sleep 120 local -r max_seconds=10 while [ 1 ]; do if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 91435dfaf79..f16ff229f90 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -14,7 +14,9 @@ cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed +cluster/gce/gci/configure-helper.sh: reconcile_cidr="false" cluster/gce/gci/configure-helper.sh: local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" +cluster/gce/gci/configure-helper.sh: local reconcile_cidr="true" cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \