Merge pull request #27371 from andyzheng0831/network-plugin

Automatic merge from submit-queue

Revert "Revert "GCI: add support for network plugin""

PR #27027 added the network plugin support in GCI config, but later a bug in the network plugin broke e2e tests (see issue #27118). The bug was fixed by #27141 and we have been repeatedly run the serial e2e tests more than 10 times to verify the fix. Now it should be safe to put the GCI network plugin support back.

We will first merge in the master branch and monitor the Jenkins serial tests for a while and then cherry-pick it into release-1.3 branch.

cluster/gce/gci/configure-helper.sh

@@ -268,10 +268,17 @@ EOF
 }
 
 function assemble-docker-flags {
-  local docker_opts="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false"
+  echo "Assemble docker command line flags"
+  local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
   if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
     docker_opts+=" --debug"
   fi
+  local use_net_plugin="true"
+  if [[ "${NETWORK_PROVIDER:-}" != "kubenet" && "${NETWORK_PROVIDER:-}" != "cni" ]]; then
+    use_net_plugin="false"
+    docker_opts+=" --bridge=cbr0"
+  fi
+
   # Decide whether to enable a docker registry mirror. This is taken from
   # the "kube-env" metadata value.
   if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then
@@ -280,6 +287,12 @@ function assemble-docker-flags {
   fi
 
   echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
+  # If using a network plugin, we need to explicitly restart docker daemon, because
+  # kubelet will not do it. 
+  if [[ "${use_net_plugin}" == "true" ]]; then
+    echo "Docker command line is updated. Restart docker to pick it up"
+    systemctl restart docker
+  fi
 }
 
 # A helper function for loading a docker image. It keeps trying up to 5 times.
@@ -340,14 +353,15 @@ function start-kubelet {
   if [[ -n "${KUBELET_PORT:-}" ]]; then
     flags+=" --port=${KUBELET_PORT}"
   fi
+  local reconcile_cidr="true"
   if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
     flags+=" --enable-debugging-handlers=false"
     flags+=" --hairpin-mode=none"
     if [[ ! -z "${KUBELET_APISERVER:-}" && ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then
       flags+=" --api-servers=https://${KUBELET_APISERVER}"
       flags+=" --register-schedulable=false"
-      flags+=" --reconcile-cidr=false"
       flags+=" --pod-cidr=10.123.45.0/30"
+      reconcile_cidr="false"
     else
       flags+=" --pod-cidr=${MASTER_IP_RANGE}"
     fi
@@ -360,6 +374,15 @@ function start-kubelet {
       flags+=" --hairpin-mode=${HAIRPIN_MODE}"
     fi
   fi
+  # Network plugin
+  if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
+    flags+=" --network-plugin-dir=/home/kubernetes/bin"
+    flags+=" --network-plugin=${NETWORK_PROVIDER}"
+  fi
+  flags+=" --reconcile-cidr=${reconcile_cidr}"
+  if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then
+    flag+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
+  fi
   if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
     flags+=" --manifest-url=${MANIFEST_URL}"
     flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}"
@@ -611,7 +634,9 @@ function start-kube-controller-manager {
   if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
     params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
   fi
-  if [[ "${ALLOCATE_NODE_CIDRS:-}" == "true" ]]; then
+  if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then
+    params+=" --allocate-node-cidrs=true"
+  elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then
     params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
   fi
   if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
@@ -818,7 +843,6 @@ function start-lb-controller {
   fi
 }
 
-
 function reset-motd {
   # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
   local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"

cluster/gce/gci/configure.sh

@@ -146,6 +146,17 @@ function install-kube-binary-config {
   else
     rm -f "${kube_bin}/kubelet"
   fi
+  if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \
+     [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
+    #TODO(andyzheng0831): We should make the cni version number as a k8s env variable.
+    local -r cni_tar="cni-26b61728ac940c3faf827927782326e921be17b0.tar.gz"
+    download-or-bust "" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}"
+    tar xzf "${KUBE_HOME}/${cni_tar}" -C "${kube_bin}" --overwrite
+    mv "${kube_bin}/bin"/* "${kube_bin}"
+    rmdir "${kube_bin}/bin"
+    rm -f "${KUBE_HOME}/${cni_tar}"
+  fi
+
   cp "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}"
 
   # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/.

cluster/gce/gci/health-monitor.sh

@@ -38,8 +38,9 @@ function docker_monitoring {
 }
 
 function kubelet_monitoring {
-  echo "waiting a minute for startup"
-  sleep 60
+  echo "Wait for 2 minutes for kubelet to be fuctional"
+  # TODO(andyzheng0831): replace it with a more reliable method if possible.
+  sleep 120
   local -r max_seconds=10
   while [ 1 ]; do
     if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then

hack/verify-flags/exceptions.txt

@@ -14,7 +14,9 @@ cluster/gce/configure-vm.sh:  cloud_config: ${CLOUD_CONFIG}
 cluster/gce/configure-vm.sh:  env-to-grains "runtime_config"
 cluster/gce/configure-vm.sh:  kubelet_api_servers: '${KUBELET_APISERVER}'
 cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed
+cluster/gce/gci/configure-helper.sh:      reconcile_cidr="false"
 cluster/gce/gci/configure-helper.sh:  local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
+cluster/gce/gci/configure-helper.sh:  local reconcile_cidr="true"
 cluster/gce/gci/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/trusty/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/util.sh:    local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \