Merge pull request #49443 from yguo0905/gke-tests

Automatic merge from submit-queue (batch tested with PRs 45813, 49594, 49443, 49167, 47539)

Add node e2e tests for GKE environment

Ref: https://github.com/kubernetes/kubernetes/issues/46891

This PR adds node e2e tests for validating images used on GKE.

- We pass the `SYSTEM_SPEC_NAME` to the node e2e test process via the flag `--system-spec-name` so that we can skip the environment specific tests using `RunIfSystemSpecNameIs()`.
- Also added `SkipIfContainerRuntimeIs()` as the opposite of `RunIfContainerRuntimeIs()`.

**Release note**:
```
None
```

test/e2e/framework/test_context.go

@@ -135,6 +135,10 @@ type NodeTestContextType struct {
 	KubeletConfig componentconfig.KubeletConfiguration
 	// ImageDescription is the description of the image on which the test is running.
 	ImageDescription string
+	// SystemSpecName is the name of the system spec (e.g., gke) that's used in
+	// the node e2e test. If empty, the default one (system.DefaultSpec) is
+	// used. The system specs are in test/e2e_node/system/specs/.
+	SystemSpecName string
 }
 
 type CloudConfig struct {
@@ -256,6 +260,7 @@ func RegisterNodeFlags() {
 	flag.BoolVar(&TestContext.NodeConformance, "conformance", false, "If true, the test suite will not start kubelet, and fetch system log (kernel, docker, kubelet log etc.) to the report directory.")
 	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
 	flag.StringVar(&TestContext.ImageDescription, "image-description", "", "The description of the image which the test will be running on.")
+	flag.StringVar(&TestContext.SystemSpecName, "system-spec-name", "", "The name of the system spec (e.g., gke) that's used in the node e2e test. The system specs are in test/e2e_node/system/specs/. This is used by the test framework to determine which tests to run for validating the system requirements.")
 }
 
 // ViperizeFlags sets up all flag and config processing. Future configuration info should be added to viper, not to flags.

test/e2e/framework/util.go

@@ -356,6 +356,24 @@ func SkipIfContainerRuntimeIs(runtimes ...string) {
 	}
 }
 
+func RunIfContainerRuntimeIs(runtimes ...string) {
+	for _, runtime := range runtimes {
+		if runtime == TestContext.ContainerRuntime {
+			return
+		}
+	}
+	Skipf("Skipped because container runtime %q is not in %s", TestContext.ContainerRuntime, runtimes)
+}
+
+func RunIfSystemSpecNameIs(names ...string) {
+	for _, name := range names {
+		if name == TestContext.SystemSpecName {
+			return
+		}
+	}
+	Skipf("Skipped because system spec name %q is not in %v", TestContext.SystemSpecName, names)
+}
+
 func ProviderIs(providers ...string) bool {
 	for _, provider := range providers {
 		if strings.ToLower(provider) == strings.ToLower(TestContext.Provider) {

test/e2e_node/BUILD

@@ -75,6 +75,7 @@ go_test(
         "dynamic_kubelet_configuration_test.go",
         "e2e_node_suite_test.go",
         "garbage_collector_test.go",
+        "gke_environment_test.go",
         "image_id_test.go",
         "inode_eviction_test.go",
         "kubelet_test.go",

test/e2e_node/conformance/build/Dockerfile

@@ -43,5 +43,7 @@ ENTRYPOINT ginkgo --focus="$FOCUS" \
 	--prepull-images=false \
 	--report-dir="$REPORT_PATH" \
 	# This is a placeholder that will be substituted in the Makefile.
+	--system-spec-name=SYSTEM_SPEC_NAME \
+	# This is a placeholder that will be substituted in the Makefile.
 	--system-spec-file=SYSTEM_SPEC_FILE_PATH \
 	$TEST_ARGS

test/e2e_node/conformance/build/Makefile

@@ -64,6 +64,7 @@ endif
 	cd ${TEMP_DIR} && sed -i.back \
 	    "s|BASEIMAGE|${BASEIMAGE}|g;\
 	     s|COPY_SYSTEM_SPEC_FILE|${COPY_SYSTEM_SPEC_FILE}|g;\
+	     s|SYSTEM_SPEC_NAME|${SYSTEM_SPEC_NAME}|g;\
 	     s|SYSTEM_SPEC_FILE_PATH|${SYSTEM_SPEC_FILE_PATH}|g" Dockerfile
 
 	# Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer

test/e2e_node/gke_environment_test.go

@@ -0,0 +1,412 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+// checkProcess checks whether there's a process whose command line contains
+// the specified pattern and whose parent process id is ppid using the
+// pre-built information in cmdToProcessMap.
+func checkProcess(pattern string, ppid int, cmdToProcessMap map[string][]process) error {
+	for cmd, processes := range cmdToProcessMap {
+		if !strings.Contains(cmd, pattern) {
+			continue
+		}
+		for _, p := range processes {
+			if p.ppid == ppid {
+				return nil
+			}
+		}
+	}
+	return fmt.Errorf("failed to find the process whose cmdline contains %q with ppid = %d", pattern, ppid)
+}
+
+// checkIPTables checks whether the functionality required by kube-proxy works
+// in iptables.
+func checkIPTables() (err error) {
+	cmds := [][]string{
+		{"iptables", "-N", "KUBE-PORTALS-HOST", "-t", "nat"},
+		{"iptables", "-I", "OUTPUT", "-t", "nat", "-m", "comment", "--comment", "ClusterIPs", "-j", "KUBE-PORTALS-HOST"},
+		{"iptables", "-A", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-1:", "-p", "tcp", "-m", "tcp", "--dport", "443", "-d", "10.0.0.1/32", "-j", "DNAT", "--to-destination", "10.240.0.1:11111"},
+		{"iptables", "-C", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-1:", "-p", "tcp", "-m", "tcp", "--dport", "443", "-d", "10.0.0.1/32", "-j", "DNAT", "--to-destination", "10.240.0.1:11111"},
+		{"iptables", "-A", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-2:", "-p", "tcp", "-m", "tcp", "--dport", "80", "-d", "10.0.0.1/32", "-j", "REDIRECT", "--to-ports", "22222"},
+		{"iptables", "-C", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-2:", "-p", "tcp", "-m", "tcp", "--dport", "80", "-d", "10.0.0.1/32", "-j", "REDIRECT", "--to-ports", "22222"},
+	}
+	cleanupCmds := [][]string{
+		{"iptables", "-F", "KUBE-PORTALS-HOST", "-t", "nat"},
+		{"iptables", "-D", "OUTPUT", "-t", "nat", "-m", "comment", "--comment", "ClusterIPs", "-j", "KUBE-PORTALS-HOST"},
+		{"iptables", "-X", "KUBE-PORTALS-HOST", "-t", "nat"},
+	}
+	defer func() {
+		for _, cmd := range cleanupCmds {
+			if _, cleanupErr := runCommand(cmd...); cleanupErr != nil && err == nil {
+				err = cleanupErr
+				return
+			}
+		}
+	}()
+	for _, cmd := range cmds {
+		if _, err := runCommand(cmd...); err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// checkPublicGCR checks the access to the public Google Container Registry by
+// pulling the busybox image.
+func checkPublicGCR() error {
+	const image = "gcr.io/google-containers/busybox"
+	output, err := runCommand("docker", "images", "-q", image)
+	if len(output) != 0 {
+		if _, err := runCommand("docker", "rmi", "-f", image); err != nil {
+			return err
+		}
+	}
+	output, err = runCommand("docker", "pull", image)
+	if len(output) == 0 {
+		return fmt.Errorf("failed to pull %s", image)
+	}
+	if _, err = runCommand("docker", "rmi", "-f", image); err != nil {
+		return err
+	}
+	return nil
+}
+
+// checkDockerConfig runs docker's check-config.sh script and ensures that all
+// expected kernel configs are enabled.
+func checkDockerConfig() error {
+	var (
+		re   = regexp.MustCompile("\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]")
+		bins = []string{
+			"/usr/share/docker.io/contrib/check-config.sh",
+			"/usr/share/docker/contrib/check-config.sh",
+		}
+		whitelist = map[string]bool{
+			"CONFIG_MEMCG_SWAP_ENABLED": true,
+			"CONFIG_RT_GROUP_SCHED":     true,
+			"CONFIG_EXT3_FS":            true,
+			"CONFIG_EXT3_FS_XATTR":      true,
+			"CONFIG_EXT3_FS_POSIX_ACL":  true,
+			"CONFIG_EXT3_FS_SECURITY":   true,
+			"/dev/zfs":                  true,
+			"zfs command":               true,
+			"zpool command":             true,
+		}
+		missing = map[string]bool{}
+	)
+	for _, bin := range bins {
+		if _, err := os.Stat(bin); os.IsNotExist(err) {
+			continue
+		}
+		output, err := runCommand(bin)
+		if err != nil {
+			return err
+		}
+		for _, line := range strings.Split(output, "\n") {
+			if !strings.Contains(line, "missing") {
+				continue
+			}
+			line = re.ReplaceAllString(line, "")
+			fields := strings.Split(line, ":")
+			if len(fields) != 2 {
+				continue
+			}
+			key := strings.TrimFunc(fields[0], func(c rune) bool {
+				return c == ' ' || c == '-'
+			})
+			if _, found := whitelist[key]; !found {
+				missing[key] = true
+			}
+		}
+		if len(missing) != 0 {
+			return fmt.Errorf("missing docker config: %v", missing)
+		}
+		break
+	}
+	return nil
+}
+
+// checkDockerNetworkClient checks client networking by pinging an external IP
+// address from a container.
+func checkDockerNetworkClient() error {
+	const imageName = "gcr.io/google-containers/busybox"
+	output, err := runCommand("docker", "run", "--rm", imageName, "sh", "-c", "ping -w 5 -q google.com")
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(output, `0% packet loss`) {
+		return fmt.Errorf("failed to ping from container: %s", output)
+	}
+	return nil
+}
+
+// checkDockerNetworkServer checks server networking by running an echo server
+// within a container and accessing it from outside.
+func checkDockerNetworkServer() error {
+	const (
+		imageName     = "gcr.io/google-containers/nginx:1.7.9"
+		hostAddr      = "127.0.0.1"
+		hostPort      = "8088"
+		containerPort = "80"
+		containerID   = "nginx"
+		message       = "Welcome to nginx!"
+	)
+	var (
+		portMapping = fmt.Sprintf("%s:%s", hostPort, containerPort)
+		host        = fmt.Sprintf("http://%s:%s", hostAddr, hostPort)
+	)
+	runCommand("docker", "rm", "-f", containerID)
+	if _, err := runCommand("docker", "run", "-d", "--name", containerID, "-p", portMapping, imageName); err != nil {
+		return err
+	}
+	output, err := runCommand("curl", host)
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(output, message) {
+		return fmt.Errorf("failed to connect to container")
+	}
+	// Clean up
+	if _, err = runCommand("docker", "rm", "-f", containerID); err != nil {
+		return err
+	}
+	if _, err = runCommand("docker", "rmi", imageName); err != nil {
+		return err
+	}
+	return nil
+}
+
+// checkDockerAppArmor checks whether AppArmor is enabled and has the
+// "docker-default" profile.
+func checkDockerAppArmor() error {
+	buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled")
+	if err != nil {
+		return err
+	}
+	if string(buf) != "Y\n" {
+		return fmt.Errorf("apparmor module is not loaded")
+	}
+
+	// Checks that the "docker-default" profile is loaded and enforced.
+	buf, err = ioutil.ReadFile("/sys/kernel/security/apparmor/profiles")
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(string(buf), "docker-default (enforce)") {
+		return fmt.Errorf("'docker-default' profile is not loaded and enforced")
+	}
+
+	// Checks that the `apparmor_parser` binary is present.
+	_, err = exec.LookPath("apparmor_parser")
+	if err != nil {
+		return fmt.Errorf("'apparmor_parser' is not in directories named by the PATH env")
+	}
+	return nil
+}
+
+// checkDockerSeccomp checks whether the Docker supports seccomp.
+func checkDockerSeccomp() error {
+	const (
+		seccompProfileFileName = "/tmp/no_mkdir.json"
+		seccompProfile         = `{
+        "defaultAction": "SCMP_ACT_ALLOW",
+        "syscalls": [
+            {
+                "name": "mkdir",
+                "action": "SCMP_ACT_ERRNO"
+            }
+        ]}`
+		image = "gcr.io/google-appengine/debian8:2017-06-07-171918"
+	)
+	if err := ioutil.WriteFile(seccompProfileFileName, []byte(seccompProfile), 0644); err != nil {
+		return err
+	}
+	// Starts a container with no seccomp profile and ensures that unshare
+	// succeeds.
+	_, err := runCommand("docker", "run", "--rm", "-i", "--security-opt", "seccomp=unconfined", image, "unshare", "-r", "whoami")
+	if err != nil {
+		return err
+	}
+	// Starts a container with the default seccomp profile and ensures that
+	// unshare (a blacklisted system call in the default profile) fails.
+	cmd := []string{"docker", "run", "--rm", "-i", image, "unshare", "-r", "whoami"}
+	_, err = runCommand(cmd...)
+	if err == nil {
+		return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
+	}
+	// Starts a container with a custom seccomp profile that blacklists mkdir
+	// and ensures that unshare succeeds.
+	_, err = runCommand("docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "unshare", "-r", "whoami")
+	if err != nil {
+		return err
+	}
+	// Starts a container with a custom seccomp profile that blacklists mkdir
+	// and ensures that mkdir fails.
+	cmd = []string{"docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "mkdir", "-p", "/tmp/foo"}
+	_, err = runCommand(cmd...)
+	if err == nil {
+		return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
+	}
+	return nil
+}
+
+// checkDockerStorageDriver checks whether the current storage driver used by
+// Docker is overlay.
+func checkDockerStorageDriver() error {
+	output, err := runCommand("docker", "info")
+	if err != nil {
+		return err
+	}
+	for _, line := range strings.Split(string(output), "\n") {
+		if !strings.Contains(line, "Storage Driver:") {
+			continue
+		}
+		if !strings.Contains(line, "overlay") {
+			return fmt.Errorf("storage driver is not 'overlay': %s", line)
+		}
+		return nil
+	}
+	return fmt.Errorf("failed to find storage driver")
+}
+
+var _ = framework.KubeDescribe("GKE system requirements [Conformance] [Feature:GKEEnv]", func() {
+	BeforeEach(func() {
+		framework.RunIfSystemSpecNameIs("gke")
+	})
+
+	It("The required processes should be running", func() {
+		cmdToProcessMap, err := getCmdToProcessMap()
+		framework.ExpectNoError(err)
+		for _, p := range []struct {
+			cmd  string
+			ppid int
+		}{
+			{"google_accounts_daemon", 1},
+			{"google_clock_skew_daemon", 1},
+			{"google_ip_forwarding_daemon", 1},
+		} {
+			framework.ExpectNoError(checkProcess(p.cmd, p.ppid, cmdToProcessMap))
+		}
+	})
+	It("The iptable rules should work (required by kube-proxy)", func() {
+		framework.ExpectNoError(checkIPTables())
+	})
+	It("The GCR is accessible", func() {
+		framework.ExpectNoError(checkPublicGCR())
+	})
+	It("The docker configuration validation should pass", func() {
+		framework.RunIfContainerRuntimeIs("docker")
+		framework.ExpectNoError(checkDockerConfig())
+	})
+	It("The docker container network should work", func() {
+		framework.RunIfContainerRuntimeIs("docker")
+		framework.ExpectNoError(checkDockerNetworkServer())
+		framework.ExpectNoError(checkDockerNetworkClient())
+	})
+	It("The docker daemon should support AppArmor and seccomp", func() {
+		framework.RunIfContainerRuntimeIs("docker")
+		framework.ExpectNoError(checkDockerAppArmor())
+		framework.ExpectNoError(checkDockerSeccomp())
+	})
+	It("The docker storage driver should work", func() {
+		framework.Skipf("GKE does not currently require overlay")
+		framework.ExpectNoError(checkDockerStorageDriver())
+	})
+})
+
+// runCommand runs the cmd and returns the combined stdout and stderr, or an
+// error if the command failed.
+func runCommand(cmd ...string) (string, error) {
+	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
+	}
+	return string(output), nil
+}
+
+// getPPID returns the PPID for the pid.
+func getPPID(pid int) (int, error) {
+	statusFile := "/proc/" + strconv.Itoa(pid) + "/status"
+	content, err := ioutil.ReadFile(statusFile)
+	if err != nil {
+		return 0, err
+	}
+	for _, line := range strings.Split(string(content), "\n") {
+		if !strings.HasPrefix(line, "PPid:") {
+			continue
+		}
+		s := strings.TrimSpace(strings.TrimPrefix(line, "PPid:"))
+		ppid, err := strconv.Atoi(s)
+		if err != nil {
+			return 0, err
+		}
+		return ppid, nil
+	}
+	return 0, fmt.Errorf("no PPid in %s", statusFile)
+}
+
+// process contains a process ID and its parent's process ID.
+type process struct {
+	pid  int
+	ppid int
+}
+
+// getCmdToProcessMap returns a mapping from the process command line to its
+// process ids.
+func getCmdToProcessMap() (map[string][]process, error) {
+	root, err := os.Open("/proc")
+	if err != nil {
+		return nil, err
+	}
+	defer root.Close()
+	dirs, err := root.Readdirnames(0)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[string][]process)
+	for _, dir := range dirs {
+		pid, err := strconv.Atoi(dir)
+		if err != nil {
+			continue
+		}
+		ppid, err := getPPID(pid)
+		if err != nil {
+			continue
+		}
+		content, err := ioutil.ReadFile("/proc/" + dir + "/cmdline")
+		if err != nil || len(content) == 0 {
+			continue
+		}
+		cmd := string(bytes.Replace(content, []byte("\x00"), []byte(" "), -1))
+		result[cmd] = append(result[cmd], process{pid, ppid})
+	}
+	return result, nil
+}

test/e2e_node/remote/node_e2e.go

@@ -211,8 +211,8 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr
 	glog.V(2).Infof("Starting tests on %q", host)
 	cmd := getSSHCommand(" && ",
 		fmt.Sprintf("cd %s", workspace),
-		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-file=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s",
+		fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s",
-			timeout.Seconds(), ginkgoArgs, systemSpecFile, host, results, junitFilePrefix, imageDesc, testArgs),
+			timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, host, results, junitFilePrefix, imageDesc, testArgs),
 	)
 	return SSH(host, "sh", "-c", cmd)
 }

test/e2e_node/runner/local/run_local.go

@@ -66,7 +66,7 @@ func main() {
 		glog.Fatalf("Failed to get k8s root directory: %v", err)
 	}
 	systemSpecFile := filepath.Join(rootDir, systemSpecPath, *systemSpecName+".yaml")
-	runCommand(ginkgo, *ginkgoFlags, test, "--", fmt.Sprintf("--system-spec-file=%s", systemSpecFile), *testFlags)
+	runCommand(ginkgo, *ginkgoFlags, test, "--", fmt.Sprintf("--system-spec-name=%s --system-spec-file=%s", *systemSpecName, systemSpecFile), *testFlags)
 	return
 }