github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 23:15:14 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #32427 from Random-Liu/system-verification

Browse Source 
Automatic merge from submit-queue

Node Conformance Test: Add system verification

For #30122 and #29081.

This PR introduces system verification test in node e2e and conformance test. It will run before the real test. Once the system verification fails, the test will just fail. The output of the system verification is like this:

```
I0909 23:33:20.622122    2717 validators.go:45] Validating os...
OS: Linux
I0909 23:33:20.623274    2717 validators.go:45] Validating kernel...
I0909 23:33:20.624037    2717 kernel_validator.go:79] Validating kernel version
KERNEL_VERSION: 3.16.0-4-amd64
I0909 23:33:20.624146    2717 kernel_validator.go:93] Validating kernel config
CONFIG_NAMESPACES: enabled
CONFIG_NET_NS: enabled
CONFIG_PID_NS: enabled
CONFIG_IPC_NS: enabled
CONFIG_UTS_NS: enabled
CONFIG_CGROUPS: enabled
CONFIG_CGROUP_CPUACCT: enabled
CONFIG_CGROUP_DEVICE: enabled
CONFIG_CGROUP_FREEZER: enabled
CONFIG_CGROUP_SCHED: enabled
CONFIG_CPUSETS: enabled
CONFIG_MEMCG: enabled
I0909 23:33:20.679328    2717 validators.go:45] Validating cgroups...
CGROUPS_CPU: enabled
CGROUPS_CPUACCT: enabled
CGROUPS_CPUSET: enabled
CGROUPS_DEVICES: enabled
CGROUPS_FREEZER: enabled
CGROUPS_MEMORY: enabled
I0909 23:33:20.679454    2717 validators.go:45] Validating docker...
DOCKER_GRAPH_DRIVER: aufs
```

It verifies the system following a predefined `SysSpec`:

``` go
// DefaultSysSpec is the default SysSpec.
 var DefaultSysSpec = SysSpec{
    OS:            "Linux",
    KernelVersion: []string{`3\.[1-9][0-9].*`, `4\..*`}, // Requires 3.10+ or 4+
    // TODO(random-liu): Add more config
    KernelConfig: KernelConfig{
        Required: []string{
            "NAMESPACES", "NET_NS", "PID_NS", "IPC_NS", "UTS_NS",
            "CGROUPS", "CGROUP_CPUACCT", "CGROUP_DEVICE", "CGROUP_FREEZER",
            "CGROUP_SCHED", "CPUSETS", "MEMCG",
        },
        Forbidden: []string{},
    },
    Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"},
    RuntimeSpec: RuntimeSpec{
        DockerSpec: &DockerSpec{
            Version: []string{`1\.(9|\d{2,})\..*`}, // Requires 1.9+
            GraphDriver: []string{"aufs", "overlay", "devicemapper"},
        },
    },
 }
```

Currently, it only supports:
- Kernel validation: version validation and kernel configuration validation
- Cgroup validation: validating whether required cgroups subsystems are enabled.
- Runtime Validation: currently, only validates docker graph driver.

The validating framework is ready. The specific validation items could be added over time.

@dchen1107 
/cc @kubernetes/sig-node
This commit is contained in:
Kubernetes Submit Queue 2016-11-06 17:12:39 -08:00 committed by GitHub
commit 9534c4f563
16 changed files with 1150 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hack/make-rules/test.sh
View File

@ -53,6 +53,7 @@ kube::test::find_dirs() {
find -L . \
-path './_output' -prune \
-o -path './vendor/k8s.io/client-go/*' \
-o -path './test/e2e_node/system/*' \
-name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | LC_ALL=C sort -u
)
}

1
hack/verify-flags/known-flags.txt
View File

@ -554,6 +554,7 @@ system-cgroups
system-container
system-pods-startup-timeout
system-reserved
system-validate-mode
target-port
target-ram-mb
tcp-services

2
test/e2e_node/BUILD
View File

@ -98,9 +98,11 @@ go_test(
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e_node/services:go_default_library",
"//test/e2e_node/system:go_default_library",
"//test/utils:go_default_library",
"//vendor:github.com/davecgh/go-spew/spew",
"//vendor:github.com/golang/glog",
"//vendor:github.com/kardianos/osext",
"//vendor:github.com/onsi/ginkgo",
"//vendor:github.com/onsi/ginkgo/config",
"//vendor:github.com/onsi/ginkgo/reporters",

31
test/e2e_node/e2e_node_suite_test.go
View File

@ -36,8 +36,10 @@ import (
commontest "k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e_node/services"
"k8s.io/kubernetes/test/e2e_node/system"
"github.com/golang/glog"
"github.com/kardianos/osext"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/config"
more_reporters "github.com/onsi/ginkgo/reporters"
@ -47,7 +49,9 @@ import (
var e2es *services.E2EServices
// TODO(random-liu): Change the following modes to sub-command.
var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.")
func init() {
framework.RegisterCommonFlags()
@ -73,6 +77,13 @@ func TestE2eNode(t *testing.T) {
services.RunE2EServices()
return
}
if *systemValidateMode {
// If system-validate-mode is specified, only run system validation in current process.
if err := system.Validate(); err != nil {
glog.Exitf("system validation failed: %v", err)
}
return
}
// If run-services-mode is not specified, run test.
rand.Seed(time.Now().UTC().UnixNano())
RegisterFailHandler(Fail)
@ -100,6 +111,10 @@ var _ = SynchronizedBeforeSuite(func() []byte {
Expect(err).NotTo(HaveOccurred(), "should be able to get node name")
framework.TestContext.NodeName = hostname
}
// Run system validation test.
Expect(validateSystem()).To(Succeed(), "system validation")
// Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue
// This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling.
if framework.TestContext.PrepullImages {
@ -151,6 +166,22 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
glog.Infof("Tests Finished")
})
// validateSystem runs system validation in a separate process and returns error if validation fails.
func validateSystem() error {
testBin, err := osext.Executable()
if err != nil {
return fmt.Errorf("can't get current binary: %v", err)
}
// TODO(random-liu): Remove sudo in containerize PR.
output, err := exec.Command("sudo", testBin, "--system-validate-mode").CombinedOutput()
// The output of system validation should have been formatted, directly print here.
fmt.Print(string(output))
if err != nil {
return fmt.Errorf("system validation failed")
}
return nil
}
func maskLocksmithdOnCoreos() {
data, err := ioutil.ReadFile("/etc/os-release")
if err != nil {

48
test/e2e_node/system/BUILD Normal file
View File

@ -0,0 +1,48 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
"go_test",
"cgo_library",
)
go_library(
name = "go_default_library",
srcs = [
"cgroup_validator.go",
"docker_validator.go",
"kernel_validator.go",
"os_validator.go",
"types.go",
"util.go",
"validators.go",
],
tags = ["automanaged"],
deps = [
"//pkg/util/errors:go_default_library",
"//vendor:github.com/docker/engine-api/client",
"//vendor:github.com/docker/engine-api/types",
"//vendor:github.com/golang/glog",
"//vendor:golang.org/x/net/context",
],
)
go_test(
name = "go_default_test",
srcs = [
"cgroup_validator_test.go",
"docker_validator_test.go",
"kernel_validator_test.go",
"os_validator_test.go",
],
library = "go_default_library",
tags = ["automanaged"],
deps = [
"//vendor:github.com/docker/engine-api/types",
"//vendor:github.com/stretchr/testify/assert",
],
)

93
test/e2e_node/system/cgroup_validator.go Normal file
View File

@ -0,0 +1,93 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"bufio"
"fmt"
"os"
"strings"
)
var _ Validator = &CgroupsValidator{}
type CgroupsValidator struct{}
func (c *CgroupsValidator) Name() string {
return "cgroups"
}
const (
cgroupsConfigPrefix = "CGROUPS_"
)
func (c *CgroupsValidator) Validate(spec SysSpec) error {
subsystems, err := c.getCgroupSubsystems()
if err != nil {
return fmt.Errorf("failed to get cgroup subsystems: %v", err)
}
return c.validateCgroupSubsystems(spec.Cgroups, subsystems)
}
func (c *CgroupsValidator) validateCgroupSubsystems(cgroupSpec, subsystems []string) error {
missing := []string{}
for _, cgroup := range cgroupSpec {
found := false
for _, subsystem := range subsystems {
if cgroup == subsystem {
found = true
break
}
}
item := cgroupsConfigPrefix + strings.ToUpper(cgroup)
if found {
report(item, "enabled", good)
} else {
report(item, "missing", bad)
missing = append(missing, cgroup)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing cgroups: %s", strings.Join(missing, " "))
}
return nil
}
func (c *CgroupsValidator) getCgroupSubsystems() ([]string, error) {
f, err := os.Open("/proc/cgroups")
if err != nil {
return nil, err
}
defer f.Close()
subsystems := []string{}
s := bufio.NewScanner(f)
for s.Scan() {
if err := s.Err(); err != nil {
return nil, err
}
text := s.Text()
if text[0] != '#' {
parts := strings.Fields(text)
if len(parts) >= 4 && parts[3] != "0" {
subsystems = append(subsystems, parts[0])
}
}
}
return subsystems, nil
}

54
test/e2e_node/system/cgroup_validator_test.go Normal file
View File

@ -0,0 +1,54 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateCgroupSubsystem(t *testing.T) {
v := &CgroupsValidator{}
cgroupSpec := []string{"system1", "system2"}
for desc, test := range map[string]struct {
cgroupSpec []string
subsystems []string
err bool
}{
"missing cgroup subsystem should report error": {
subsystems: []string{"system1"},
err: true,
},
"extra cgroup subsystems should not report error": {
subsystems: []string{"system1", "system2", "system3"},
err: false,
},
"subsystems the same with spec should not report error": {
subsystems: []string{"system1", "system2"},
err: false,
},
} {
err := v.validateCgroupSubsystems(cgroupSpec, test.subsystems)
if !test.err {
assert.Nil(t, err, "%q: Expect error not to occur with cgroup", desc)
} else {
assert.NotNil(t, err, "%q: Expect error to occur with docker info", desc)
}
}
}

84
test/e2e_node/system/docker_validator.go Normal file
View File

@ -0,0 +1,84 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"fmt"
"regexp"
"github.com/docker/engine-api/client"
"github.com/docker/engine-api/types"
"golang.org/x/net/context"
)
var _ Validator = &DockerValidator{}
// DockerValidator validates docker configuration.
type DockerValidator struct{}
func (d *DockerValidator) Name() string {
return "docker"
}
const (
dockerEndpoint = "unix:///var/run/docker.sock"
dockerConfigPrefix = "DOCKER_"
)
// TODO(random-liu): Add more validating items.
func (d *DockerValidator) Validate(spec SysSpec) error {
if spec.RuntimeSpec.DockerSpec == nil {
// If DockerSpec is not specified, assume current runtime is not
// docker, skip the docker configuration validation.
return nil
}
c, err := client.NewClient(dockerEndpoint, "", nil, nil)
if err != nil {
return fmt.Errorf("failed to create docker client: %v", err)
}
info, err := c.Info(context.Background())
if err != nil {
return fmt.Errorf("failed to get docker info: %v", err)
}
return d.validateDockerInfo(spec.RuntimeSpec.DockerSpec, info)
}
func (d *DockerValidator) validateDockerInfo(spec *DockerSpec, info types.Info) error {
// Validate docker version.
matched := false
for _, v := range spec.Version {
r := regexp.MustCompile(v)
if r.MatchString(info.ServerVersion) {
report(dockerConfigPrefix+"VERSION", info.ServerVersion, good)
matched = true
}
}
if !matched {
report(dockerConfigPrefix+"VERSION", info.ServerVersion, bad)
return fmt.Errorf("unsupported docker version: %s", info.ServerVersion)
}
// Validate graph driver.
item := dockerConfigPrefix + "GRAPH_DRIVER"
for _, d := range spec.GraphDriver {
if info.Driver == d {
report(item, info.Driver, good)
return nil
}
}
report(item, info.Driver, bad)
return fmt.Errorf("unsupported graph driver: %s", info.Driver)
}

57
test/e2e_node/system/docker_validator_test.go Normal file
View File

@ -0,0 +1,57 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"testing"
"github.com/docker/engine-api/types"
"github.com/stretchr/testify/assert"
)
func TestValidateDockerInfo(t *testing.T) {
v := &DockerValidator{}
spec := &DockerSpec{
Version: []string{`1\.(9|\d{2,})\..*`},
GraphDriver: []string{"driver_1", "driver_2"},
}
for _, test := range []struct {
info types.Info
err bool
}{
{
info: types.Info{Driver: "driver_1", ServerVersion: "1.10.1"},
err: false,
},
{
info: types.Info{Driver: "bad_driver", ServerVersion: "1.9.1"},
err: true,
},
{
info: types.Info{Driver: "driver_2", ServerVersion: "1.8.1"},
err: true,
},
} {
err := v.validateDockerInfo(spec, test.info)
if !test.err {
assert.Nil(t, err, "Expect error not to occur with docker info %+v", test.info)
} else {
assert.NotNil(t, err, "Expect error to occur with docker info %+v", test.info)
}
}
}

255
test/e2e_node/system/kernel_validator.go Normal file
View File

@ -0,0 +1,255 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/errors"
)
var _ Validator = &KernelValidator{}
// KernelValidator validates kernel. Currently only validate kernel version
// and kernel configuration.
type KernelValidator struct {
kernelRelease string
}
func (k *KernelValidator) Name() string {
return "kernel"
}
// kConfigOption is the possible kernel config option.
type kConfigOption string
const (
builtIn kConfigOption = "y"
asModule kConfigOption = "m"
leftOut kConfigOption = "n"
// validKConfigRegex is the regex matching kernel configuration line.
validKConfigRegex = "^CONFIG_[A-Z0-9_]+=[myn]"
// kConfigPrefix is the prefix of kernel configuration.
kConfigPrefix = "CONFIG_"
)
func (k *KernelValidator) Validate(spec SysSpec) error {
out, err := exec.Command("uname", "-r").CombinedOutput()
if err != nil {
return fmt.Errorf("failed to get kernel release: %v", err)
}
k.kernelRelease = strings.TrimSpace(string(out))
var errs []error
errs = append(errs, k.validateKernelVersion(spec.KernelSpec))
errs = append(errs, k.validateKernelConfig(spec.KernelSpec))
return errors.NewAggregate(errs)
}
// validateKernelVersion validates the kernel version.
func (k *KernelValidator) validateKernelVersion(kSpec KernelSpec) error {
glog.Infof("Validating kernel version")
versionRegexps := kSpec.Versions
for _, versionRegexp := range versionRegexps {
r := regexp.MustCompile(versionRegexp)
if r.MatchString(k.kernelRelease) {
report("KERNEL_VERSION", k.kernelRelease, good)
return nil
}
}
report("KERNEL_VERSION", k.kernelRelease, bad)
return fmt.Errorf("unsupported kernel release: %s", k.kernelRelease)
}
// validateKernelConfig validates the kernel configurations.
func (k *KernelValidator) validateKernelConfig(kSpec KernelSpec) error {
glog.Infof("Validating kernel config")
allConfig, err := k.getKernelConfig()
if err != nil {
return fmt.Errorf("failed to parse kernel config: %v", err)
}
return k.validateCachedKernelConfig(allConfig, kSpec)
}
// validateCachedKernelConfig validates the kernel confgiurations cached in internal data type.
func (k *KernelValidator) validateCachedKernelConfig(allConfig map[string]kConfigOption, kSpec KernelSpec) error {
badConfigs := []string{}
// reportAndRecord is a helper function to record bad config when
// report.
reportAndRecord := func(name, msg, desc string, result resultType) {
if result == bad {
badConfigs = append(badConfigs, name)
}
// report description when the config is bad or warn.
if result != good && desc != "" {
msg = msg + " - " + desc
}
report(name, msg, result)
}
const (
required = iota
optional
forbidden
)
validateOpt := func(config KernelConfig, expect int) {
var found, missing resultType
switch expect {
case required:
found, missing = good, bad
case optional:
found, missing = good, warn
case forbidden:
found, missing = bad, good
}
var name string
var opt kConfigOption
var ok bool
for _, name = range append([]string{config.Name}, config.Aliases...) {
name = kConfigPrefix + name
if opt, ok = allConfig[name]; ok {
break
}
}
if !ok {
reportAndRecord(name, "not set", config.Description, missing)
return
}
switch opt {
case builtIn:
reportAndRecord(name, "enabled", config.Description, found)
case asModule:
reportAndRecord(name, "enabled (as module)", config.Description, found)
case leftOut:
reportAndRecord(name, "disabled", config.Description, missing)
default:
reportAndRecord(name, fmt.Sprintf("unknown option: %s", opt), config.Description, missing)
}
}
for _, config := range kSpec.Required {
validateOpt(config, required)
}
for _, config := range kSpec.Optional {
validateOpt(config, optional)
}
for _, config := range kSpec.Forbidden {
validateOpt(config, forbidden)
}
if len(badConfigs) > 0 {
return fmt.Errorf("unexpected kernel config: %s", strings.Join(badConfigs, " "))
}
return nil
}
// getKernelConfigReader search kernel config file in a predefined list. Once the kernel config
// file is found it will read the configurations into a byte buffer and return. If the kernel
// config file is not found, it will try to load kernel config module and retry again.
func (k *KernelValidator) getKernelConfigReader() (io.Reader, error) {
possibePaths := []string{
"/proc/config.gz",
"/boot/config-" + k.kernelRelease,
"/usr/src/linux-" + k.kernelRelease + "/.config",
"/usr/src/linux/.config",
}
configsModule := "configs"
modprobeCmd := "modprobe"
// loadModule indicates whether we've tried to load kernel config module ourselves.
loadModule := false
for {
for _, path := range possibePaths {
_, err := os.Stat(path)
if err != nil {
continue
}
// Buffer the whole file, so that we can close the file and unload
// kernel config module in this function.
b, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
var r io.Reader
r = bytes.NewReader(b)
// This is a gzip file (config.gz), unzip it.
if filepath.Ext(path) == ".gz" {
r, err = gzip.NewReader(r)
if err != nil {
return nil, err
}
}
return r, nil
}
// If we've tried to load kernel config module, break and return error.
if loadModule {
break
}
// If the kernel config file is not found, try to load the kernel
// config module and check again.
// TODO(random-liu): Remove "sudo" in containerize test PR #31093
output, err := exec.Command("sudo", modprobeCmd, configsModule).CombinedOutput()
if err != nil {
return nil, fmt.Errorf("unable to load kernel module %q: output - %q, err - %v",
configsModule, output, err)
}
// Unload the kernel config module to make sure the validation have no side effect.
defer exec.Command("sudo", modprobeCmd, "-r", configsModule).Run()
loadModule = true
}
return nil, fmt.Errorf("no config path in %v is available", possibePaths)
}
// getKernelConfig gets kernel config from kernel config file and convert kernel config to internal type.
func (k *KernelValidator) getKernelConfig() (map[string]kConfigOption, error) {
r, err := k.getKernelConfigReader()
if err != nil {
return nil, err
}
return k.parseKernelConfig(r)
}
// parseKernelConfig converts kernel config to internal type.
func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOption, error) {
config := map[string]kConfigOption{}
regex := regexp.MustCompile(validKConfigRegex)
s := bufio.NewScanner(r)
for s.Scan() {
if err := s.Err(); err != nil {
return nil, err
}
line := strings.TrimSpace(s.Text())
if !regex.MatchString(line) {
continue
}
fields := strings.Split(line, "=")
if len(fields) != 2 {
glog.Errorf("Unexpected fields number in config %q", line)
continue
}
config[fields[0]] = kConfigOption(fields[1])
}
return config, nil
}

191
test/e2e_node/system/kernel_validator_test.go Normal file
View File

@ -0,0 +1,191 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateKernelVersion(t *testing.T) {
v := &KernelValidator{}
// Currently, testRegex is align with DefaultSysSpec.KernelVersion, but in the future
// they may be different.
// This is fine, because the test mainly tests the kernel version validation logic,
// not the DefaultSysSpec. The DefaultSysSpec should be tested with node e2e.
testRegex := []string{`3\.[1-9][0-9].*`, `4\..*`}
for _, test := range []struct {
version string
err bool
}{
// first version regex matches
{
version: "3.19.9-99-test",
err: false,
},
// one of version regexes matches
{
version: "4.4.14+",
err: false,
},
// no version regex matches
{
version: "2.0.0",
err: true,
},
{
version: "5.0.0",
err: true,
},
{
version: "3.9.0",
err: true,
},
} {
v.kernelRelease = test.version
err := v.validateKernelVersion(KernelSpec{Versions: testRegex})
if !test.err {
assert.Nil(t, err, "Expect error not to occur with kernel version %q", test.version)
} else {
assert.NotNil(t, err, "Expect error to occur with kenrel version %q", test.version)
}
}
}
func TestValidateCachedKernelConfig(t *testing.T) {
v := &KernelValidator{}
testKernelSpec := KernelSpec{
Required: []KernelConfig{{Name: "REQUIRED_1"}, {Name: "REQUIRED_2", Aliases: []string{"ALIASE_REQUIRED_2"}}},
Optional: []KernelConfig{{Name: "OPTIONAL_1"}, {Name: "OPTIONAL_2"}},
Forbidden: []KernelConfig{
{Name: "FORBIDDEN_1", Description: "TEST FORBIDDEN"},
{Name: "FORBIDDEN_2", Aliases: []string{"ALIASE_FORBIDDEN_2"}},
},
}
for c, test := range []struct {
desc string
config map[string]kConfigOption
err bool
}{
{
desc: "meet all required configurations should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
},
err: false,
},
{
desc: "one required configuration disabled should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": leftOut,
"REQUIRED_2": builtIn,
},
err: true,
},
{
desc: "one required configuration missing should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
},
err: true,
},
{
desc: "alias of required configuration should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"ALIASE_REQUIRED_2": asModule,
},
err: false,
},
{
desc: "optional configuration set or not should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"OPTIONAL_1": builtIn,
},
err: false,
},
{
desc: "forbidden configuration disabled should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"FORBIDDEN_1": leftOut,
},
err: false,
},
{
desc: "forbidden configuration built-in should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"FORBIDDEN_1": builtIn,
},
err: true,
},
{
desc: "forbidden configuration built as module should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"FORBIDDEN_1": asModule,
},
err: true,
},
{
desc: "alias of forbidden configuration should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"ALIASE_FORBIDDEN_2": asModule,
},
err: true,
},
} {
t.Logf("TestCase #%d %s", c, test.desc)
// Add kernel config prefix.
for k, v := range test.config {
delete(test.config, k)
test.config[kConfigPrefix+k] = v
}
err := v.validateCachedKernelConfig(test.config, testKernelSpec)
if !test.err {
assert.Nil(t, err, "Expect error not to occur with kernel config %q", test.config)
} else {
assert.NotNil(t, err, "Expect error to occur with kenrel config %q", test.config)
}
}
}
func TestValidateParseKernelConfig(t *testing.T) {
config := `CONFIG_1=y
CONFIG_2=m
CONFIG_3=n`
expected := map[string]kConfigOption{
"CONFIG_1": builtIn,
"CONFIG_2": asModule,
"CONFIG_3": leftOut,
}
v := &KernelValidator{}
got, err := v.parseKernelConfig(bytes.NewReader([]byte(config)))
assert.Nil(t, err, "Expect error not to occur when parse kernel configuration %q", config)
assert.Equal(t, expected, got)
}

48
test/e2e_node/system/os_validator.go Normal file
View File

@ -0,0 +1,48 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"fmt"
"os/exec"
"strings"
)
var _ Validator = &OSValidator{}
type OSValidator struct{}
func (c *OSValidator) Name() string {
return "os"
}
func (c *OSValidator) Validate(spec SysSpec) error {
out, err := exec.Command("uname").CombinedOutput()
if err != nil {
return fmt.Errorf("failed to get os name: %v", err)
}
return c.validateOS(strings.TrimSpace(string(out)), spec.OS)
}
func (c *OSValidator) validateOS(os, specOS string) error {
if os != specOS {
report("OS", os, bad)
return fmt.Errorf("unsupported operating system: %s", os)
}
report("OS", os, good)
return nil
}

52
test/e2e_node/system/os_validator_test.go Normal file
View File

@ -0,0 +1,52 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateOS(t *testing.T) {
v := &OSValidator{}
specOS := "Linux"
for _, test := range []struct {
os string
err bool
}{
{
os: "Linux",
err: false,
},
{
os: "Windows",
err: true,
},
{
os: "Darwin",
err: true,
},
} {
err := v.validateOS(test.os, specOS)
if !test.err {
assert.Nil(t, err, "Expect error not to occur with os %q", test.os)
} else {
assert.NotNil(t, err, "Expect error to occur with os %q", test.os)
}
}
}

122
test/e2e_node/system/types.go Normal file
View File

@ -0,0 +1,122 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
// KernelConfig defines one kernel configration item.
type KernelConfig struct {
// Name is the general name of the kernel configuration. It is used to
// match kernel configuration.
Name string
// Aliases are aliases of the kernel configuration. Some configuration
// has different names in different kernel version. Names of different
// versions will be treated as aliases.
Aliases []string
// Description is the description of the kernel configuration, for example:
// * What is it used for?
// * Why is it needed?
// * Who needs it?
Description string
}
// KernelSpec defines the specification for the kernel. Currently, it contains
// specification for:
// * Kernel Version
// * Kernel Configuration
type KernelSpec struct {
// Versions define supported kernel version. It is a group of regexps.
Versions []string
// Required contains all kernel configurations required to be enabled
// (built in or as module).
Required []KernelConfig
// Optional contains all kernel configurations are required for optional
// features.
Optional []KernelConfig
// Forbidden contains all kernel configurations which areforbidden (disabled
// or not set)
Forbidden []KernelConfig
}
// DockerSpec defines the requirement configuration for docker. Currently, it only
// contains spec for graph driver.
type DockerSpec struct {
// Version is a group of regex matching supported docker versions.
Version []string
// GraphDriver is the graph drivers supported by kubelet.
GraphDriver []string
}
// RuntimeSpec is the abstract layer for different runtimes. Different runtimes
// should put their spec inside the RuntimeSpec.
type RuntimeSpec struct {
*DockerSpec
}
// SysSpec defines the requirement of supported system. Currently, it only contains
// spec for OS, Kernel and Cgroups.
type SysSpec struct {
// OS is the operating system of the SysSpec.
OS string
// KernelConfig defines the spec for kernel.
KernelSpec KernelSpec
// Cgroups is the required cgroups.
Cgroups []string
// RuntimeSpec defines the spec for runtime.
RuntimeSpec RuntimeSpec
}
// DefaultSysSpec is the default SysSpec.
var DefaultSysSpec = SysSpec{
OS: "Linux",
KernelSpec: KernelSpec{
Versions: []string{`3\.[1-9][0-9].*`, `4\..*`}, // Requires 3.10+ or 4+
// TODO(random-liu): Add more config
// TODO(random-liu): Add description for each kernel configuration:
Required: []KernelConfig{
{Name: "NAMESPACES"},
{Name: "NET_NS"},
{Name: "PID_NS"},
{Name: "IPC_NS"},
{Name: "UTS_NS"},
{Name: "CGROUPS"},
{Name: "CGROUP_CPUACCT"},
{Name: "CGROUP_DEVICE"},
{Name: "CGROUP_FREEZER"},
{Name: "CGROUP_SCHED"},
{Name: "CPUSETS"},
{Name: "MEMCG"},
{Name: "INET"},
{Name: "EXT4_FS"},
{Name: "PROC_FS"},
{Name: "NETFILTER_XT_TARGET_REDIRECT", Aliases: []string{"IP_NF_TARGET_REDIRECT"}},
{Name: "NETFILTER_XT_MATCH_COMMENT"},
},
Optional: []KernelConfig{
{Name: "OVERLAY_FS", Aliases: []string{"OVERLAYFS_FS"}, Description: "Required for overlayfs."},
{Name: "AUFS_FS", Description: "Required for aufs."},
{Name: "BLK_DEV_DM", Description: "Required for devicemapper."},
},
Forbidden: []KernelConfig{},
},
Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"},
RuntimeSpec: RuntimeSpec{
DockerSpec: &DockerSpec{
Version: []string{`1\.(9|\d{2,})\..*`}, // Requires 1.9+
// TODO(random-liu): Validate overlay2.
GraphDriver: []string{"aufs", "overlay", "devicemapper"},
},
},
}

62
test/e2e_node/system/util.go Normal file
View File

@ -0,0 +1,62 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"fmt"
)
// resultType is type of the validation result. Different validation results
// corresponds to different colors.
type resultType int
const (
good resultType = iota
bad
warn
)
// color is the color of the message.
type color int
const (
red color = 31
green = 32
yellow = 33
white = 37
)
func wrap(s string, c color) string {
return fmt.Sprintf("\033[0;%dm%s\033[0m", c, s)
}
// report reports "item: r". item is white, and the color of r depends on the
// result type.
func report(item, r string, t resultType) {
var c color
switch t {
case good:
c = green
case bad:
c = red
case warn:
c = yellow
default:
c = white
}
fmt.Printf("%s: %s\n", wrap(item, white), wrap(r, c))
}

49
test/e2e_node/system/validators.go Normal file
View File

@ -0,0 +1,49 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/errors"
)
// Validator is the interface for all validators.
type Validator interface {
// Name is the name of the validator.
Name() string
// Validate is the validate function.
Validate(SysSpec) error
}
// validators are all the validators.
var validators = []Validator{
&OSValidator{},
&KernelValidator{},
&CgroupsValidator{},
&DockerValidator{},
}
// Validate uses all validators to validate the system.
func Validate() error {
var errs []error
spec := DefaultSysSpec
for _, v := range validators {
glog.Infof("Validating %s...", v.Name())
errs = append(errs, v.Validate(spec))
}
return errors.NewAggregate(errs)
}