Merge pull request #54964 from Random-Liu/add-containerd-e2e

Automatic merge from submit-queue (batch tested with PRs 54488, 54838, 54964). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add support to for alternative container runtime in `kube-up.sh`

For https://github.com/kubernetes/features/issues/286.

This PR added 4 new environment variables in `kube-up.sh` to support alternative container runtime:
1) `KUBE_MASTER_EXTRA_METADATA` and `KUBE_NODE_EXTRA_METADATA`. Add extra metadata on master and node instance. With this we could specify different cloud-init for a different container runtime, and also add extra metadata for the new cloud-init, e.g. [master.yaml](7d73966214/test/e2e/master.yaml)
2) `KUBE_CONTAINER_RUNTIME_ENDPOINT`. Specify different sock for different container runtime. It's only used when it's not empty.
3) `KUBE_LOAD_IMAGE_COMMAND`. Specify different load image command for different container runtime.

An example for cri-containerd:
```
export KUBE_MASTER_EXTRA_METADATA="user-data=${GOPATH}/src/github.com/kubernetes-incubator/cri-containerd/test/e2e/master.yaml,cri-containerd-configure-sh=${GOPATH}/src/github.com/kubernetes-incubator/cri-containerd/test/configure.sh"
export KUBE_NODE_EXTRA_METADATA="user-data=${GOPATH}/src/github.com/kubernetes-incubator/cri-containerd/test/e2e/node.yaml,cri-containerd-configure-sh=${GOPATH}/src/github.com/kubernetes-incubator/cri-containerd/test/configure.sh"
export KUBE_CONTAINER_RUNTIME="remote"
export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock"
export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/cri-containerd load"
export NETWORK_POLICY_PROVIDER="calico"
```

Signed-off-by: Lantao Liu <lantaol@google.com>

```release-note
none
```
/cc @yujuhong @dchen1107 @feiskyer @mikebrow @abhi @mrunalp @runcom 
/cc @kubernetes/sig-node-pr-reviews

cluster/common.sh

@@ -666,6 +666,9 @@ PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote ${PROMETHEUS_TO_SD_ENDPOINT:-})
 PROMETHEUS_TO_SD_PREFIX: $(yaml-quote ${PROMETHEUS_TO_SD_PREFIX:-})
 ENABLE_PROMETHEUS_TO_SD: $(yaml-quote ${ENABLE_PROMETHEUS_TO_SD:-false})
 ENABLE_POD_PRIORITY: $(yaml-quote ${ENABLE_POD_PRIORITY:-})
+CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME:-})
+CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote ${CONTAINER_RUNTIME_ENDPOINT:-})
+LOAD_IMAGE_COMMAND: $(yaml-quote ${LOAD_IMAGE_COMMAND:-})
 EOF
   if [ -n "${KUBELET_PORT:-}" ]; then
     cat >>$file <<EOF

cluster/gce/config-default.sh

@@ -81,8 +81,14 @@ NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}
 NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 NODE_SERVICE_ACCOUNT=${KUBE_GCE_NODE_SERVICE_ACCOUNT:-default}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
+CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-}
+LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-docker load -i}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}
 RKT_STAGE1_IMAGE=${KUBE_RKT_STAGE1_IMAGE:-coreos.com/rkt/stage1-coreos}
+# MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas.
+MASTER_EXTRA_METADATA=${KUBE_MASTER_EXTRA_METADATA:-${KUBE_EXTRA_METADATA:-}}
+# MASTER_EXTRA_METADATA is the extra instance metadata on node instance separated by commas.
+NODE_EXTRA_METADATA=${KUBE_NODE_EXTRA_METADATA:-${KUBE_EXTRA_METADATA:-}}
 
 NETWORK=${KUBE_GCE_NETWORK:-default}
 if [[ "${CREATE_CUSTOM_NETWORK}" == true ]]; then

cluster/gce/config-test.sh

@@ -80,9 +80,15 @@ NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}
 NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 NODE_SERVICE_ACCOUNT=${KUBE_GCE_NODE_SERVICE_ACCOUNT:-default}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
+CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-}
+LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-docker load -i}
 GCI_DOCKER_VERSION=${KUBE_GCI_DOCKER_VERSION:-}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}
 RKT_STAGE1_IMAGE=${KUBE_RKT_STAGE1_IMAGE:-coreos.com/rkt/stage1-coreos}
+# MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas.
+MASTER_EXTRA_METADATA=${KUBE_MASTER_EXTRA_METADATA:-${KUBE_EXTRA_METADATA:-}}
+# MASTER_EXTRA_METADATA is the extra instance metadata on node instance separated by commas.
+NODE_EXTRA_METADATA=${KUBE_NODE_EXTRA_METADATA:-${KUBE_EXTRA_METADATA:-}}
 
 NETWORK=${KUBE_GCE_NETWORK:-e2e-test-${USER}}
 if [[ "${CREATE_CUSTOM_NETWORK}" == true ]]; then

cluster/gce/container-linux/configure-helper.sh

@@ -1497,7 +1497,7 @@ else
   fi
 fi
 
-if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
+if [[ "${KUBERNETES_CONTAINER_RUNTIME:-}" == "rkt" ]]; then
   systemctl stop docker
   systemctl disable docker
   setup-rkt

cluster/gce/gci/configure-helper.sh

@@ -1005,6 +1005,13 @@ function start-kubelet {
   if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
     flags+=" --rotate-certificates=true"
   fi
+  if [[ -n "${CONTAINER_RUNTIME:-}" ]]; then
+    flags+=" --container-runtime=${CONTAINER_RUNTIME}"
+  fi
+  if [[ -n "${CONTAINER_RUNTIME_ENDPOINT:-}" ]]; then
+    flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
+  fi
+
 
   local -r kubelet_env_file="/etc/default/kubelet"
   echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
@@ -1035,6 +1042,7 @@ function start-node-problem-detector {
   echo "Start node problem detector"
   local -r npd_bin="${KUBE_HOME}/bin/node-problem-detector"
   local -r km_config="${KUBE_HOME}/node-problem-detector/config/kernel-monitor.json"
+  # TODO(random-liu): Handle this for alternative container runtime.
   local -r dm_config="${KUBE_HOME}/node-problem-detector/config/docker-monitor.json"
   echo "Using node problem detector binary at ${npd_bin}"
   local flags="${NPD_TEST_LOG_LEVEL:-"--v=2"} ${NPD_TEST_ARGS:-}"
@@ -2034,7 +2042,9 @@ fi
 
 override-kubectl
 # Run the containerized mounter once to pre-cache the container image.
-assemble-docker-flags
+if [[ "${CONTAINER_RUNTIME:-}" == "docker" ]]; then
+  assemble-docker-flags
+fi
 start-kubelet
 
 if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then

cluster/gce/gci/configure.sh

@@ -241,7 +241,7 @@ function try-load-docker-image {
   set +e
   local -r max_attempts=5
   local -i attempt_num=1
-  until timeout 30 docker load -i "${img}"; do
+  until timeout 30 ${LOAD_IMAGE_COMMAND} "${img}"; do
     if [[ "${attempt_num}" == "${max_attempts}" ]]; then
       echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!"
       exit 1

cluster/gce/gci/master-helper.sh

@@ -100,6 +100,7 @@ function create-master-instance-internal() {
   metadata="${metadata},gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt"
   metadata="${metadata},gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
   metadata="${metadata},kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml"
+  metadata="${metadata},${MASTER_EXTRA_METADATA}"
 
   local disk="name=${master_name}-pd"
   disk="${disk},device-name=master-pd"

cluster/gce/gci/node-helper.sh

@@ -25,7 +25,8 @@ function get-node-instance-metadata {
   metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt,"
   metadata+="gci-update-strategy=${KUBE_TEMP}/gci-update.txt,"
   metadata+="gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,"
-  metadata+="gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
+  metadata+="gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,"
+  metadata+="${NODE_EXTRA_METADATA}"
   echo "${metadata}"
 }