Merge pull request #32427 from Random-Liu/system-verification

Automatic merge from submit-queue

Node Conformance Test: Add system verification

For #30122 and #29081.

This PR introduces system verification test in node e2e and conformance test. It will run before the real test. Once the system verification fails, the test will just fail. The output of the system verification is like this:

```
I0909 23:33:20.622122    2717 validators.go:45] Validating os...
OS: Linux
I0909 23:33:20.623274    2717 validators.go:45] Validating kernel...
I0909 23:33:20.624037    2717 kernel_validator.go:79] Validating kernel version
KERNEL_VERSION: 3.16.0-4-amd64
I0909 23:33:20.624146    2717 kernel_validator.go:93] Validating kernel config
CONFIG_NAMESPACES: enabled
CONFIG_NET_NS: enabled
CONFIG_PID_NS: enabled
CONFIG_IPC_NS: enabled
CONFIG_UTS_NS: enabled
CONFIG_CGROUPS: enabled
CONFIG_CGROUP_CPUACCT: enabled
CONFIG_CGROUP_DEVICE: enabled
CONFIG_CGROUP_FREEZER: enabled
CONFIG_CGROUP_SCHED: enabled
CONFIG_CPUSETS: enabled
CONFIG_MEMCG: enabled
I0909 23:33:20.679328    2717 validators.go:45] Validating cgroups...
CGROUPS_CPU: enabled
CGROUPS_CPUACCT: enabled
CGROUPS_CPUSET: enabled
CGROUPS_DEVICES: enabled
CGROUPS_FREEZER: enabled
CGROUPS_MEMORY: enabled
I0909 23:33:20.679454    2717 validators.go:45] Validating docker...
DOCKER_GRAPH_DRIVER: aufs
```

It verifies the system following a predefined `SysSpec`:

``` go
// DefaultSysSpec is the default SysSpec.
 var DefaultSysSpec = SysSpec{
    OS:            "Linux",
    KernelVersion: []string{`3\.[1-9][0-9].*`, `4\..*`}, // Requires 3.10+ or 4+
    // TODO(random-liu): Add more config
    KernelConfig: KernelConfig{
        Required: []string{
            "NAMESPACES", "NET_NS", "PID_NS", "IPC_NS", "UTS_NS",
            "CGROUPS", "CGROUP_CPUACCT", "CGROUP_DEVICE", "CGROUP_FREEZER",
            "CGROUP_SCHED", "CPUSETS", "MEMCG",
        },
        Forbidden: []string{},
    },
    Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"},
    RuntimeSpec: RuntimeSpec{
        DockerSpec: &DockerSpec{
            Version: []string{`1\.(9|\d{2,})\..*`}, // Requires 1.9+
            GraphDriver: []string{"aufs", "overlay", "devicemapper"},
        },
    },
 }
```

Currently, it only supports:
- Kernel validation: version validation and kernel configuration validation
- Cgroup validation: validating whether required cgroups subsystems are enabled.
- Runtime Validation: currently, only validates docker graph driver.

The validating framework is ready. The specific validation items could be added over time.

@dchen1107 
/cc @kubernetes/sig-node

hack/make-rules/test.sh

@@ -53,6 +53,7 @@ kube::test::find_dirs() {
     find -L . \
         -path './_output' -prune \
         -o -path './vendor/k8s.io/client-go/*' \
+        -o -path './test/e2e_node/system/*' \
       -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | LC_ALL=C sort -u
   )
 }

hack/verify-flags/known-flags.txt

@@ -554,6 +554,7 @@ system-cgroups
 system-container
 system-pods-startup-timeout
 system-reserved
+system-validate-mode
 target-port
 target-ram-mb
 tcp-services

test/e2e_node/BUILD

@@ -98,9 +98,11 @@ go_test(
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e_node/services:go_default_library",
+        "//test/e2e_node/system:go_default_library",
         "//test/utils:go_default_library",
         "//vendor:github.com/davecgh/go-spew/spew",
         "//vendor:github.com/golang/glog",
+        "//vendor:github.com/kardianos/osext",
         "//vendor:github.com/onsi/ginkgo",
         "//vendor:github.com/onsi/ginkgo/config",
         "//vendor:github.com/onsi/ginkgo/reporters",

test/e2e_node/e2e_node_suite_test.go

@@ -36,8 +36,10 @@ import (
 	commontest "k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e_node/services"
+	"k8s.io/kubernetes/test/e2e_node/system"
 
 	"github.com/golang/glog"
+	"github.com/kardianos/osext"
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	more_reporters "github.com/onsi/ginkgo/reporters"
@@ -47,7 +49,9 @@ import (
 
 var e2es *services.E2EServices
 
+// TODO(random-liu): Change the following modes to sub-command.
 var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
+var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.")
 
 func init() {
 	framework.RegisterCommonFlags()
@@ -73,6 +77,13 @@ func TestE2eNode(t *testing.T) {
 		services.RunE2EServices()
 		return
 	}
+	if *systemValidateMode {
+		// If system-validate-mode is specified, only run system validation in current process.
+		if err := system.Validate(); err != nil {
+			glog.Exitf("system validation failed: %v", err)
+		}
+		return
+	}
 	// If run-services-mode is not specified, run test.
 	rand.Seed(time.Now().UTC().UnixNano())
 	RegisterFailHandler(Fail)
@@ -100,6 +111,10 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		Expect(err).NotTo(HaveOccurred(), "should be able to get node name")
 		framework.TestContext.NodeName = hostname
 	}
+
+	// Run system validation test.
+	Expect(validateSystem()).To(Succeed(), "system validation")
+
 	// Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue
 	// This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling.
 	if framework.TestContext.PrepullImages {
@@ -151,6 +166,22 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
 	glog.Infof("Tests Finished")
 })
 
+// validateSystem runs system validation in a separate process and returns error if validation fails.
+func validateSystem() error {
+	testBin, err := osext.Executable()
+	if err != nil {
+		return fmt.Errorf("can't get current binary: %v", err)
+	}
+	// TODO(random-liu): Remove sudo in containerize PR.
+	output, err := exec.Command("sudo", testBin, "--system-validate-mode").CombinedOutput()
+	// The output of system validation should have been formatted, directly print here.
+	fmt.Print(string(output))
+	if err != nil {
+		return fmt.Errorf("system validation failed")
+	}
+	return nil
+}
+
 func maskLocksmithdOnCoreos() {
 	data, err := ioutil.ReadFile("/etc/os-release")
 	if err != nil {

test/e2e_node/system/BUILD

@@ -0,0 +1,48 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cgroup_validator.go",
+        "docker_validator.go",
+        "kernel_validator.go",
+        "os_validator.go",
+        "types.go",
+        "util.go",
+        "validators.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/util/errors:go_default_library",
+        "//vendor:github.com/docker/engine-api/client",
+        "//vendor:github.com/docker/engine-api/types",
+        "//vendor:github.com/golang/glog",
+        "//vendor:golang.org/x/net/context",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cgroup_validator_test.go",
+        "docker_validator_test.go",
+        "kernel_validator_test.go",
+        "os_validator_test.go",
+    ],
+    library = "go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//vendor:github.com/docker/engine-api/types",
+        "//vendor:github.com/stretchr/testify/assert",
+    ],
+)

test/e2e_node/system/cgroup_validator.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+var _ Validator = &CgroupsValidator{}
+
+type CgroupsValidator struct{}
+
+func (c *CgroupsValidator) Name() string {
+	return "cgroups"
+}
+
+const (
+	cgroupsConfigPrefix = "CGROUPS_"
+)
+
+func (c *CgroupsValidator) Validate(spec SysSpec) error {
+	subsystems, err := c.getCgroupSubsystems()
+	if err != nil {
+		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
+	}
+	return c.validateCgroupSubsystems(spec.Cgroups, subsystems)
+}
+
+func (c *CgroupsValidator) validateCgroupSubsystems(cgroupSpec, subsystems []string) error {
+	missing := []string{}
+	for _, cgroup := range cgroupSpec {
+		found := false
+		for _, subsystem := range subsystems {
+			if cgroup == subsystem {
+				found = true
+				break
+			}
+		}
+		item := cgroupsConfigPrefix + strings.ToUpper(cgroup)
+		if found {
+			report(item, "enabled", good)
+		} else {
+			report(item, "missing", bad)
+			missing = append(missing, cgroup)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("missing cgroups: %s", strings.Join(missing, " "))
+	}
+	return nil
+
+}
+
+func (c *CgroupsValidator) getCgroupSubsystems() ([]string, error) {
+	f, err := os.Open("/proc/cgroups")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	subsystems := []string{}
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+		text := s.Text()
+		if text[0] != '#' {
+			parts := strings.Fields(text)
+			if len(parts) >= 4 && parts[3] != "0" {
+				subsystems = append(subsystems, parts[0])
+			}
+		}
+	}
+	return subsystems, nil
+}

test/e2e_node/system/cgroup_validator_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateCgroupSubsystem(t *testing.T) {
+	v := &CgroupsValidator{}
+	cgroupSpec := []string{"system1", "system2"}
+	for desc, test := range map[string]struct {
+		cgroupSpec []string
+		subsystems []string
+		err        bool
+	}{
+		"missing cgroup subsystem should report error": {
+			subsystems: []string{"system1"},
+			err:        true,
+		},
+		"extra cgroup subsystems should not report error": {
+			subsystems: []string{"system1", "system2", "system3"},
+			err:        false,
+		},
+		"subsystems the same with spec should not report error": {
+			subsystems: []string{"system1", "system2"},
+			err:        false,
+		},
+	} {
+		err := v.validateCgroupSubsystems(cgroupSpec, test.subsystems)
+		if !test.err {
+			assert.Nil(t, err, "%q: Expect error not to occur with cgroup", desc)
+		} else {
+			assert.NotNil(t, err, "%q: Expect error to occur with docker info", desc)
+		}
+
+	}
+}

test/e2e_node/system/docker_validator.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/docker/engine-api/client"
+	"github.com/docker/engine-api/types"
+	"golang.org/x/net/context"
+)
+
+var _ Validator = &DockerValidator{}
+
+// DockerValidator validates docker configuration.
+type DockerValidator struct{}
+
+func (d *DockerValidator) Name() string {
+	return "docker"
+}
+
+const (
+	dockerEndpoint     = "unix:///var/run/docker.sock"
+	dockerConfigPrefix = "DOCKER_"
+)
+
+// TODO(random-liu): Add more validating items.
+func (d *DockerValidator) Validate(spec SysSpec) error {
+	if spec.RuntimeSpec.DockerSpec == nil {
+		// If DockerSpec is not specified, assume current runtime is not
+		// docker, skip the docker configuration validation.
+		return nil
+	}
+	c, err := client.NewClient(dockerEndpoint, "", nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create docker client: %v", err)
+	}
+	info, err := c.Info(context.Background())
+	if err != nil {
+		return fmt.Errorf("failed to get docker info: %v", err)
+	}
+	return d.validateDockerInfo(spec.RuntimeSpec.DockerSpec, info)
+}
+
+func (d *DockerValidator) validateDockerInfo(spec *DockerSpec, info types.Info) error {
+	// Validate docker version.
+	matched := false
+	for _, v := range spec.Version {
+		r := regexp.MustCompile(v)
+		if r.MatchString(info.ServerVersion) {
+			report(dockerConfigPrefix+"VERSION", info.ServerVersion, good)
+			matched = true
+		}
+	}
+	if !matched {
+		report(dockerConfigPrefix+"VERSION", info.ServerVersion, bad)
+		return fmt.Errorf("unsupported docker version: %s", info.ServerVersion)
+	}
+	// Validate graph driver.
+	item := dockerConfigPrefix + "GRAPH_DRIVER"
+	for _, d := range spec.GraphDriver {
+		if info.Driver == d {
+			report(item, info.Driver, good)
+			return nil
+		}
+	}
+	report(item, info.Driver, bad)
+	return fmt.Errorf("unsupported graph driver: %s", info.Driver)
+}

test/e2e_node/system/docker_validator_test.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"testing"
+
+	"github.com/docker/engine-api/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateDockerInfo(t *testing.T) {
+	v := &DockerValidator{}
+	spec := &DockerSpec{
+		Version:     []string{`1\.(9|\d{2,})\..*`},
+		GraphDriver: []string{"driver_1", "driver_2"},
+	}
+	for _, test := range []struct {
+		info types.Info
+		err  bool
+	}{
+		{
+			info: types.Info{Driver: "driver_1", ServerVersion: "1.10.1"},
+			err:  false,
+		},
+		{
+			info: types.Info{Driver: "bad_driver", ServerVersion: "1.9.1"},
+			err:  true,
+		},
+		{
+			info: types.Info{Driver: "driver_2", ServerVersion: "1.8.1"},
+			err:  true,
+		},
+	} {
+		err := v.validateDockerInfo(spec, test.info)
+		if !test.err {
+			assert.Nil(t, err, "Expect error not to occur with docker info %+v", test.info)
+		} else {
+			assert.NotNil(t, err, "Expect error to occur with docker info %+v", test.info)
+		}
+
+	}
+}

test/e2e_node/system/kernel_validator.go

@@ -0,0 +1,255 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/util/errors"
+)
+
+var _ Validator = &KernelValidator{}
+
+// KernelValidator validates kernel. Currently only validate kernel version
+// and kernel configuration.
+type KernelValidator struct {
+	kernelRelease string
+}
+
+func (k *KernelValidator) Name() string {
+	return "kernel"
+}
+
+// kConfigOption is the possible kernel config option.
+type kConfigOption string
+
+const (
+	builtIn  kConfigOption = "y"
+	asModule kConfigOption = "m"
+	leftOut  kConfigOption = "n"
+
+	// validKConfigRegex is the regex matching kernel configuration line.
+	validKConfigRegex = "^CONFIG_[A-Z0-9_]+=[myn]"
+	// kConfigPrefix is the prefix of kernel configuration.
+	kConfigPrefix = "CONFIG_"
+)
+
+func (k *KernelValidator) Validate(spec SysSpec) error {
+	out, err := exec.Command("uname", "-r").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to get kernel release: %v", err)
+	}
+	k.kernelRelease = strings.TrimSpace(string(out))
+	var errs []error
+	errs = append(errs, k.validateKernelVersion(spec.KernelSpec))
+	errs = append(errs, k.validateKernelConfig(spec.KernelSpec))
+	return errors.NewAggregate(errs)
+}
+
+// validateKernelVersion validates the kernel version.
+func (k *KernelValidator) validateKernelVersion(kSpec KernelSpec) error {
+	glog.Infof("Validating kernel version")
+	versionRegexps := kSpec.Versions
+	for _, versionRegexp := range versionRegexps {
+		r := regexp.MustCompile(versionRegexp)
+		if r.MatchString(k.kernelRelease) {
+			report("KERNEL_VERSION", k.kernelRelease, good)
+			return nil
+		}
+	}
+	report("KERNEL_VERSION", k.kernelRelease, bad)
+	return fmt.Errorf("unsupported kernel release: %s", k.kernelRelease)
+}
+
+// validateKernelConfig validates the kernel configurations.
+func (k *KernelValidator) validateKernelConfig(kSpec KernelSpec) error {
+	glog.Infof("Validating kernel config")
+	allConfig, err := k.getKernelConfig()
+	if err != nil {
+		return fmt.Errorf("failed to parse kernel config: %v", err)
+	}
+	return k.validateCachedKernelConfig(allConfig, kSpec)
+}
+
+// validateCachedKernelConfig validates the kernel confgiurations cached in internal data type.
+func (k *KernelValidator) validateCachedKernelConfig(allConfig map[string]kConfigOption, kSpec KernelSpec) error {
+	badConfigs := []string{}
+	// reportAndRecord is a helper function to record bad config when
+	// report.
+	reportAndRecord := func(name, msg, desc string, result resultType) {
+		if result == bad {
+			badConfigs = append(badConfigs, name)
+		}
+		// report description when the config is bad or warn.
+		if result != good && desc != "" {
+			msg = msg + " - " + desc
+		}
+		report(name, msg, result)
+	}
+	const (
+		required = iota
+		optional
+		forbidden
+	)
+	validateOpt := func(config KernelConfig, expect int) {
+		var found, missing resultType
+		switch expect {
+		case required:
+			found, missing = good, bad
+		case optional:
+			found, missing = good, warn
+		case forbidden:
+			found, missing = bad, good
+		}
+		var name string
+		var opt kConfigOption
+		var ok bool
+		for _, name = range append([]string{config.Name}, config.Aliases...) {
+			name = kConfigPrefix + name
+			if opt, ok = allConfig[name]; ok {
+				break
+			}
+		}
+		if !ok {
+			reportAndRecord(name, "not set", config.Description, missing)
+			return
+		}
+		switch opt {
+		case builtIn:
+			reportAndRecord(name, "enabled", config.Description, found)
+		case asModule:
+			reportAndRecord(name, "enabled (as module)", config.Description, found)
+		case leftOut:
+			reportAndRecord(name, "disabled", config.Description, missing)
+		default:
+			reportAndRecord(name, fmt.Sprintf("unknown option: %s", opt), config.Description, missing)
+		}
+	}
+	for _, config := range kSpec.Required {
+		validateOpt(config, required)
+	}
+	for _, config := range kSpec.Optional {
+		validateOpt(config, optional)
+	}
+	for _, config := range kSpec.Forbidden {
+		validateOpt(config, forbidden)
+	}
+	if len(badConfigs) > 0 {
+		return fmt.Errorf("unexpected kernel config: %s", strings.Join(badConfigs, " "))
+	}
+	return nil
+}
+
+// getKernelConfigReader search kernel config file in a predefined list. Once the kernel config
+// file is found it will read the configurations into a byte buffer and return. If the kernel
+// config file is not found, it will try to load kernel config module and retry again.
+func (k *KernelValidator) getKernelConfigReader() (io.Reader, error) {
+	possibePaths := []string{
+		"/proc/config.gz",
+		"/boot/config-" + k.kernelRelease,
+		"/usr/src/linux-" + k.kernelRelease + "/.config",
+		"/usr/src/linux/.config",
+	}
+	configsModule := "configs"
+	modprobeCmd := "modprobe"
+	// loadModule indicates whether we've tried to load kernel config module ourselves.
+	loadModule := false
+	for {
+		for _, path := range possibePaths {
+			_, err := os.Stat(path)
+			if err != nil {
+				continue
+			}
+			// Buffer the whole file, so that we can close the file and unload
+			// kernel config module in this function.
+			b, err := ioutil.ReadFile(path)
+			if err != nil {
+				return nil, err
+			}
+			var r io.Reader
+			r = bytes.NewReader(b)
+			// This is a gzip file (config.gz), unzip it.
+			if filepath.Ext(path) == ".gz" {
+				r, err = gzip.NewReader(r)
+				if err != nil {
+					return nil, err
+				}
+			}
+			return r, nil
+		}
+		// If we've tried to load kernel config module, break and return error.
+		if loadModule {
+			break
+		}
+		// If the kernel config file is not found, try to load the kernel
+		// config module and check again.
+		// TODO(random-liu): Remove "sudo" in containerize test PR #31093
+		output, err := exec.Command("sudo", modprobeCmd, configsModule).CombinedOutput()
+		if err != nil {
+			return nil, fmt.Errorf("unable to load kernel module %q: output - %q, err - %v",
+				configsModule, output, err)
+		}
+		// Unload the kernel config module to make sure the validation have no side effect.
+		defer exec.Command("sudo", modprobeCmd, "-r", configsModule).Run()
+		loadModule = true
+	}
+	return nil, fmt.Errorf("no config path in %v is available", possibePaths)
+}
+
+// getKernelConfig gets kernel config from kernel config file and convert kernel config to internal type.
+func (k *KernelValidator) getKernelConfig() (map[string]kConfigOption, error) {
+	r, err := k.getKernelConfigReader()
+	if err != nil {
+		return nil, err
+	}
+	return k.parseKernelConfig(r)
+}
+
+// parseKernelConfig converts kernel config to internal type.
+func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOption, error) {
+	config := map[string]kConfigOption{}
+	regex := regexp.MustCompile(validKConfigRegex)
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+		line := strings.TrimSpace(s.Text())
+		if !regex.MatchString(line) {
+			continue
+		}
+		fields := strings.Split(line, "=")
+		if len(fields) != 2 {
+			glog.Errorf("Unexpected fields number in config %q", line)
+			continue
+		}
+		config[fields[0]] = kConfigOption(fields[1])
+	}
+	return config, nil
+
+}

test/e2e_node/system/kernel_validator_test.go

@@ -0,0 +1,191 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateKernelVersion(t *testing.T) {
+	v := &KernelValidator{}
+	// Currently, testRegex is align with DefaultSysSpec.KernelVersion, but in the future
+	// they may be different.
+	// This is fine, because the test mainly tests the kernel version validation logic,
+	// not the DefaultSysSpec. The DefaultSysSpec should be tested with node e2e.
+	testRegex := []string{`3\.[1-9][0-9].*`, `4\..*`}
+	for _, test := range []struct {
+		version string
+		err     bool
+	}{
+		// first version regex matches
+		{
+			version: "3.19.9-99-test",
+			err:     false,
+		},
+		// one of version regexes matches
+		{
+			version: "4.4.14+",
+			err:     false,
+		},
+		// no version regex matches
+		{
+			version: "2.0.0",
+			err:     true,
+		},
+		{
+			version: "5.0.0",
+			err:     true,
+		},
+		{
+			version: "3.9.0",
+			err:     true,
+		},
+	} {
+		v.kernelRelease = test.version
+		err := v.validateKernelVersion(KernelSpec{Versions: testRegex})
+		if !test.err {
+			assert.Nil(t, err, "Expect error not to occur with kernel version %q", test.version)
+		} else {
+			assert.NotNil(t, err, "Expect error to occur with kenrel version %q", test.version)
+		}
+	}
+}
+
+func TestValidateCachedKernelConfig(t *testing.T) {
+	v := &KernelValidator{}
+	testKernelSpec := KernelSpec{
+		Required: []KernelConfig{{Name: "REQUIRED_1"}, {Name: "REQUIRED_2", Aliases: []string{"ALIASE_REQUIRED_2"}}},
+		Optional: []KernelConfig{{Name: "OPTIONAL_1"}, {Name: "OPTIONAL_2"}},
+		Forbidden: []KernelConfig{
+			{Name: "FORBIDDEN_1", Description: "TEST FORBIDDEN"},
+			{Name: "FORBIDDEN_2", Aliases: []string{"ALIASE_FORBIDDEN_2"}},
+		},
+	}
+	for c, test := range []struct {
+		desc   string
+		config map[string]kConfigOption
+		err    bool
+	}{
+		{
+			desc: "meet all required configurations should not report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1": builtIn,
+				"REQUIRED_2": asModule,
+			},
+			err: false,
+		},
+		{
+			desc: "one required configuration disabled should report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1": leftOut,
+				"REQUIRED_2": builtIn,
+			},
+			err: true,
+		},
+		{
+			desc: "one required configuration missing should report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1": builtIn,
+			},
+			err: true,
+		},
+		{
+			desc: "alias of required configuration should not report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1":        builtIn,
+				"ALIASE_REQUIRED_2": asModule,
+			},
+			err: false,
+		},
+		{
+			desc: "optional configuration set or not should not report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1": builtIn,
+				"REQUIRED_2": asModule,
+				"OPTIONAL_1": builtIn,
+			},
+			err: false,
+		},
+		{
+			desc: "forbidden configuration disabled should not report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1":  builtIn,
+				"REQUIRED_2":  asModule,
+				"FORBIDDEN_1": leftOut,
+			},
+			err: false,
+		},
+		{
+			desc: "forbidden configuration built-in should report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1":  builtIn,
+				"REQUIRED_2":  asModule,
+				"FORBIDDEN_1": builtIn,
+			},
+			err: true,
+		},
+		{
+			desc: "forbidden configuration built as module should report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1":  builtIn,
+				"REQUIRED_2":  asModule,
+				"FORBIDDEN_1": asModule,
+			},
+			err: true,
+		},
+		{
+			desc: "alias of forbidden configuration should report error.",
+			config: map[string]kConfigOption{
+				"REQUIRED_1":         builtIn,
+				"REQUIRED_2":         asModule,
+				"ALIASE_FORBIDDEN_2": asModule,
+			},
+			err: true,
+		},
+	} {
+		t.Logf("TestCase #%d %s", c, test.desc)
+		// Add kernel config prefix.
+		for k, v := range test.config {
+			delete(test.config, k)
+			test.config[kConfigPrefix+k] = v
+		}
+		err := v.validateCachedKernelConfig(test.config, testKernelSpec)
+		if !test.err {
+			assert.Nil(t, err, "Expect error not to occur with kernel config %q", test.config)
+		} else {
+			assert.NotNil(t, err, "Expect error to occur with kenrel config %q", test.config)
+		}
+	}
+}
+
+func TestValidateParseKernelConfig(t *testing.T) {
+	config := `CONFIG_1=y
+CONFIG_2=m
+CONFIG_3=n`
+	expected := map[string]kConfigOption{
+		"CONFIG_1": builtIn,
+		"CONFIG_2": asModule,
+		"CONFIG_3": leftOut,
+	}
+	v := &KernelValidator{}
+	got, err := v.parseKernelConfig(bytes.NewReader([]byte(config)))
+	assert.Nil(t, err, "Expect error not to occur when parse kernel configuration %q", config)
+	assert.Equal(t, expected, got)
+}

test/e2e_node/system/os_validator.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+var _ Validator = &OSValidator{}
+
+type OSValidator struct{}
+
+func (c *OSValidator) Name() string {
+	return "os"
+}
+
+func (c *OSValidator) Validate(spec SysSpec) error {
+	out, err := exec.Command("uname").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to get os name: %v", err)
+	}
+	return c.validateOS(strings.TrimSpace(string(out)), spec.OS)
+}
+
+func (c *OSValidator) validateOS(os, specOS string) error {
+	if os != specOS {
+		report("OS", os, bad)
+		return fmt.Errorf("unsupported operating system: %s", os)
+	}
+	report("OS", os, good)
+	return nil
+}

test/e2e_node/system/os_validator_test.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateOS(t *testing.T) {
+	v := &OSValidator{}
+	specOS := "Linux"
+	for _, test := range []struct {
+		os  string
+		err bool
+	}{
+		{
+			os:  "Linux",
+			err: false,
+		},
+		{
+			os:  "Windows",
+			err: true,
+		},
+		{
+			os:  "Darwin",
+			err: true,
+		},
+	} {
+		err := v.validateOS(test.os, specOS)
+		if !test.err {
+			assert.Nil(t, err, "Expect error not to occur with os %q", test.os)
+		} else {
+			assert.NotNil(t, err, "Expect error to occur with os %q", test.os)
+		}
+	}
+}

test/e2e_node/system/types.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+// KernelConfig defines one kernel configration item.
+type KernelConfig struct {
+	// Name is the general name of the kernel configuration. It is used to
+	// match kernel configuration.
+	Name string
+	// Aliases are aliases of the kernel configuration. Some configuration
+	// has different names in different kernel version. Names of different
+	// versions will be treated as aliases.
+	Aliases []string
+	// Description is the description of the kernel configuration, for example:
+	//  * What is it used for?
+	//  * Why is it needed?
+	//  * Who needs it?
+	Description string
+}
+
+// KernelSpec defines the specification for the kernel. Currently, it contains
+// specification for:
+//   * Kernel Version
+//   * Kernel Configuration
+type KernelSpec struct {
+	// Versions define supported kernel version. It is a group of regexps.
+	Versions []string
+	// Required contains all kernel configurations required to be enabled
+	// (built in or as module).
+	Required []KernelConfig
+	// Optional contains all kernel configurations are required for optional
+	// features.
+	Optional []KernelConfig
+	// Forbidden contains all kernel configurations which areforbidden (disabled
+	// or not set)
+	Forbidden []KernelConfig
+}
+
+// DockerSpec defines the requirement configuration for docker. Currently, it only
+// contains spec for graph driver.
+type DockerSpec struct {
+	// Version is a group of regex matching supported docker versions.
+	Version []string
+	// GraphDriver is the graph drivers supported by kubelet.
+	GraphDriver []string
+}
+
+// RuntimeSpec is the abstract layer for different runtimes. Different runtimes
+// should put their spec inside the RuntimeSpec.
+type RuntimeSpec struct {
+	*DockerSpec
+}
+
+// SysSpec defines the requirement of supported system. Currently, it only contains
+// spec for OS, Kernel and Cgroups.
+type SysSpec struct {
+	// OS is the operating system of the SysSpec.
+	OS string
+	// KernelConfig defines the spec for kernel.
+	KernelSpec KernelSpec
+	// Cgroups is the required cgroups.
+	Cgroups []string
+	// RuntimeSpec defines the spec for runtime.
+	RuntimeSpec RuntimeSpec
+}
+
+// DefaultSysSpec is the default SysSpec.
+var DefaultSysSpec = SysSpec{
+	OS: "Linux",
+	KernelSpec: KernelSpec{
+		Versions: []string{`3\.[1-9][0-9].*`, `4\..*`}, // Requires 3.10+ or 4+
+		// TODO(random-liu): Add more config
+		// TODO(random-liu): Add description for each kernel configuration:
+		Required: []KernelConfig{
+			{Name: "NAMESPACES"},
+			{Name: "NET_NS"},
+			{Name: "PID_NS"},
+			{Name: "IPC_NS"},
+			{Name: "UTS_NS"},
+			{Name: "CGROUPS"},
+			{Name: "CGROUP_CPUACCT"},
+			{Name: "CGROUP_DEVICE"},
+			{Name: "CGROUP_FREEZER"},
+			{Name: "CGROUP_SCHED"},
+			{Name: "CPUSETS"},
+			{Name: "MEMCG"},
+			{Name: "INET"},
+			{Name: "EXT4_FS"},
+			{Name: "PROC_FS"},
+			{Name: "NETFILTER_XT_TARGET_REDIRECT", Aliases: []string{"IP_NF_TARGET_REDIRECT"}},
+			{Name: "NETFILTER_XT_MATCH_COMMENT"},
+		},
+		Optional: []KernelConfig{
+			{Name: "OVERLAY_FS", Aliases: []string{"OVERLAYFS_FS"}, Description: "Required for overlayfs."},
+			{Name: "AUFS_FS", Description: "Required for aufs."},
+			{Name: "BLK_DEV_DM", Description: "Required for devicemapper."},
+		},
+		Forbidden: []KernelConfig{},
+	},
+	Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"},
+	RuntimeSpec: RuntimeSpec{
+		DockerSpec: &DockerSpec{
+			Version: []string{`1\.(9|\d{2,})\..*`}, // Requires 1.9+
+			// TODO(random-liu): Validate overlay2.
+			GraphDriver: []string{"aufs", "overlay", "devicemapper"},
+		},
+	},
+}

test/e2e_node/system/util.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"fmt"
+)
+
+// resultType is type of the validation result. Different validation results
+// corresponds to different colors.
+type resultType int
+
+const (
+	good resultType = iota
+	bad
+	warn
+)
+
+// color is the color of the message.
+type color int
+
+const (
+	red    color = 31
+	green        = 32
+	yellow       = 33
+	white        = 37
+)
+
+func wrap(s string, c color) string {
+	return fmt.Sprintf("\033[0;%dm%s\033[0m", c, s)
+}
+
+// report reports "item: r". item is white, and the color of r depends on the
+// result type.
+func report(item, r string, t resultType) {
+	var c color
+	switch t {
+	case good:
+		c = green
+	case bad:
+		c = red
+	case warn:
+		c = yellow
+	default:
+		c = white
+	}
+	fmt.Printf("%s: %s\n", wrap(item, white), wrap(r, c))
+}

test/e2e_node/system/validators.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package system
+
+import (
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/util/errors"
+)
+
+// Validator is the interface for all validators.
+type Validator interface {
+	// Name is the name of the validator.
+	Name() string
+	// Validate is the validate function.
+	Validate(SysSpec) error
+}
+
+// validators are all the validators.
+var validators = []Validator{
+	&OSValidator{},
+	&KernelValidator{},
+	&CgroupsValidator{},
+	&DockerValidator{},
+}
+
+// Validate uses all validators to validate the system.
+func Validate() error {
+	var errs []error
+	spec := DefaultSysSpec
+	for _, v := range validators {
+		glog.Infof("Validating %s...", v.Name())
+		errs = append(errs, v.Validate(spec))
+	}
+	return errors.NewAggregate(errs)
+}