Merge pull request #28516 from Random-Liu/disable-cni-for-local-node-e2e

Automatic merge from submit-queue

Node E2E: Disable kubenet for local node e2e test.

After https://github.com/kubernetes/kubernetes/pull/28196, we must manually setup cni and nsenter in local node to run `make test_e2e_node`, which may not be necessary for local development.

I've tried to move cni downloading logic into `BeforeSuite`, however it is still hard to figure out who should install nsenter, manually installed by every developer? in the `setup_host.sh` script? in `BeforeSuite`?

This PR:
* Added a flag to disable kubenet and disabled kubenet in local test.
* Cleaned up the CNI installation logic a bit.

/cc @yujuhong @freehan 
[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/.github/PULL_REQUEST_TEMPLATE.md?pixel)]()

hack/e2e-node-test.sh

@@ -129,6 +129,6 @@ else
   # Test using the host the script was run on
   # Provided for backwards compatibility
   "${ginkgo}" --focus=$focus --skip=$skip "${KUBE_ROOT}/test/e2e_node/" --report-dir=${report} \
-    -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true
+    -- --alsologtostderr --v 2 --node-name $(hostname) --disable-kubenet=true --build-services=true --start-services=true --stop-services=true
   exit $?
 fi

hack/verify-flags/known-flags.txt

@@ -100,6 +100,7 @@ deployment-label-key
 deserialization-cache-size
 dest-file
 disable-filter
+disable-kubenet
 dns-port
 dns-provider
 dns-provider-config

test/e2e_node/e2e_remote.go

@@ -42,10 +42,11 @@ var sshOptionsMap map[string]string
 
 const (
 	archiveName  = "e2e_node_test.tar.gz"
-	CNI_RELEASE = "c864f0e1ea73719b8f4582402b0847064f9883b0"
+	CNIRelease   = "c864f0e1ea73719b8f4582402b0847064f9883b0"
+	CNIDirectory = "cni"
 )
 
-var CNI_URL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-%s.tar.gz", CNI_RELEASE)
+var CNIURL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-%s.tar.gz", CNIRelease)
 
 var hostnameIpOverrides = struct {
 	sync.RWMutex
@@ -157,14 +158,6 @@ func RunRemote(archive string, host string, cleanup bool, junitFileNumber int, s
 		}
 	}
 
-	// Install the cni plugin. Note that /opt/cni does not get cleaned up after
-	// the test completes.
-	if _, err := RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c",
-		getSshCommand(" ; ", "sudo mkdir -p /opt/cni", fmt.Sprintf("sudo wget -O - %s | sudo tar -xz -C /opt/cni", CNI_URL))); err != nil {
-		// Exit failure with the error
-		return "", false, err
-	}
-
 	// Create the temp staging directory
 	glog.Infof("Staging test binaries on %s", host)
 	tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
@@ -182,6 +175,15 @@ func RunRemote(archive string, host string, cleanup bool, junitFileNumber int, s
 		}()
 	}
 
+	// Install the cni plugin.
+	cniPath := filepath.Join(tmp, CNIDirectory)
+	if _, err := RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c",
+		getSshCommand(" ; ", fmt.Sprintf("sudo mkdir -p %s", cniPath),
+			fmt.Sprintf("sudo wget -O - %s | sudo tar -xz -C %s", CNIURL, cniPath))); err != nil {
+		// Exit failure with the error
+		return "", false, err
+	}
+
 	// Copy the archive to the staging directory
 	_, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), tmp))
 	if err != nil {

test/e2e_node/e2e_service.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -233,9 +234,17 @@ func (es *e2eService) startKubeletServer() (*killCmd, error) {
 		"--config", es.kubeletStaticPodDir,
 		"--file-check-frequency", "10s", // Check file frequently so tests won't wait too long
 		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
-		"--network-plugin=kubenet",
 		"--pod-cidr=10.180.0.0/24", // Assign a fixed CIDR to the node because there is no node controller.
 	)
+	if !*disableKubenet {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		cmdArgs = append(cmdArgs,
+			"--network-plugin=kubenet",
+			"--network-plugin-dir", filepath.Join(cwd, CNIDirectory, "bin")) // Enable kubenet
+	}
 	cmd := exec.Command("sudo", cmdArgs...)
 	hcc := newHealthCheckCommand(
 		"http://127.0.0.1:10255/healthz",

test/e2e_node/util.go

@@ -26,6 +26,7 @@ import (
 var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
 var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080", "Host and port of the api server")
 
+var disableKubenet = flag.Bool("disable-kubenet", false, "If true, start kubelet without kubenet")
 var buildServices = flag.Bool("build-services", true, "If true, build local executables")
 var startServices = flag.Bool("start-services", true, "If true, start local node services")
 var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")