Merge pull request #27027 from andyzheng0831/gci-network

Automatic merge from submit-queue

GCI: add support for network plugin

I had run e2e against a cluster with both master and nodes on GCI a couple of times. The PR auto tests will cover the hybrid cluster with just master on GCI.

cc/ @roberthbailey @fabioy @kubernetes/goog-image

cluster/gce/gci/configure-helper.sh

@@ -249,10 +249,17 @@ EOF
 }
 
 function assemble-docker-flags {
-  local docker_opts="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false"
+  echo "Assemble docker command line flags"
+  local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
   if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
     docker_opts+=" --debug"
   fi
+  local use_net_plugin="true"
+  if [[ "${NETWORK_PROVIDER:-}" != "kubenet" && "${NETWORK_PROVIDER:-}" != "cni" ]]; then
+    use_net_plugin="false"
+    docker_opts+=" --bridge=cbr0"
+  fi
+
   # Decide whether to enable a docker registry mirror. This is taken from
   # the "kube-env" metadata value.
   if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then
@@ -261,6 +268,12 @@ function assemble-docker-flags {
   fi
 
   echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
+  # If using a network plugin, we need to explicitly restart docker daemon, because
+  # kubelet will not do it. 
+  if [[ "${use_net_plugin}" == "true" ]]; then
+    echo "Docker command line is updated. Restart docker to pick it up"
+    systemctl restart docker
+  fi
 }
 
 # A helper function for loading a docker image. It keeps trying up to 5 times.
@@ -321,14 +334,15 @@ function start-kubelet {
   if [[ -n "${KUBELET_PORT:-}" ]]; then
     flags+=" --port=${KUBELET_PORT}"
   fi
+  local reconcile_cidr="true"
   if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
     flags+=" --enable-debugging-handlers=false"
     flags+=" --hairpin-mode=none"
     if [[ ! -z "${KUBELET_APISERVER:-}" && ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then
       flags+=" --api-servers=https://${KUBELET_APISERVER}"
       flags+=" --register-schedulable=false"
-      flags+=" --reconcile-cidr=false"
       flags+=" --pod-cidr=10.123.45.0/30"
+      reconcile_cidr="false"
     else
       flags+=" --pod-cidr=${MASTER_IP_RANGE}"
     fi
@@ -341,6 +355,15 @@ function start-kubelet {
       flags+=" --hairpin-mode=${HAIRPIN_MODE}"
     fi
   fi
+  # Network plugin
+  if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
+    flags+=" --network-plugin-dir=/home/kubernetes/bin"
+    flags+=" --network-plugin=${NETWORK_PROVIDER}"
+  fi
+  flags+=" --reconcile-cidr=${reconcile_cidr}"
+  if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then
+    flag+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
+  fi
   if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
     flags+=" --manifest-url=${MANIFEST_URL}"
     flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}"
@@ -591,7 +614,9 @@ function start-kube-controller-manager {
   if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
     params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
   fi
-  if [[ "${ALLOCATE_NODE_CIDRS:-}" == "true" ]]; then
+  if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then
+    params+=" --allocate-node-cidrs=true"
+  elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then
     params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
   fi
   if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
@@ -798,7 +823,6 @@ function start-lb-controller {
   fi
 }
 
-
 function reset-motd {
   # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
   local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"

cluster/gce/gci/configure.sh

@@ -146,6 +146,17 @@ function install-kube-binary-config {
   else
     rm -f "${kube_bin}/kubelet"
   fi
+  if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \
+     [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
+    #TODO(andyzheng0831): We should make the cni version number as a k8s env variable.
+    local -r cni_tar="cni-26b61728ac940c3faf827927782326e921be17b0.tar.gz"
+    download-or-bust "" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}"
+    tar xzf "${KUBE_HOME}/${cni_tar}" -C "${kube_bin}" --overwrite
+    mv "${kube_bin}/bin"/* "${kube_bin}"
+    rmdir "${kube_bin}/bin"
+    rm -f "${KUBE_HOME}/${cni_tar}"
+  fi
+
   cp "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}"
 
   # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/.

cluster/gce/gci/health-monitor.sh

@@ -38,8 +38,9 @@ function docker_monitoring {
 }
 
 function kubelet_monitoring {
-  echo "waiting a minute for startup"
-  sleep 60
+  echo "Wait for 2 minutes for kubelet to be fuctional"
+  # TODO(andyzheng0831): replace it with a more reliable method if possible.
+  sleep 120
   local -r max_seconds=10
   while [ 1 ]; do
     if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then

hack/verify-flags/exceptions.txt

@@ -14,7 +14,9 @@ cluster/gce/configure-vm.sh:  cloud_config: ${CLOUD_CONFIG}
 cluster/gce/configure-vm.sh:  env-to-grains "runtime_config"
 cluster/gce/configure-vm.sh:  kubelet_api_servers: '${KUBELET_APISERVER}'
 cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed
+cluster/gce/gci/configure-helper.sh:      reconcile_cidr="false"
 cluster/gce/gci/configure-helper.sh:  local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
+cluster/gce/gci/configure-helper.sh:  local reconcile_cidr="true"
 cluster/gce/gci/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/trusty/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/util.sh:    local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \