From a5fdf3850cf17b2152dda5ebc52f4f37eb26329f Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Fri, 9 Sep 2016 17:20:45 -0700 Subject: [PATCH 1/3] Add system verification. --- hack/verify-flags/known-flags.txt | 1 + test/e2e_node/e2e_node_suite_test.go | 31 +++ test/e2e_node/system/cgroup_validator.go | 93 +++++++++ test/e2e_node/system/docker_validator.go | 84 ++++++++ test/e2e_node/system/kernel_validator.go | 255 +++++++++++++++++++++++ test/e2e_node/system/os_validator.go | 48 +++++ test/e2e_node/system/types.go | 122 +++++++++++ test/e2e_node/system/util.go | 62 ++++++ test/e2e_node/system/validators.go | 49 +++++ 9 files changed, 745 insertions(+) create mode 100644 test/e2e_node/system/cgroup_validator.go create mode 100644 test/e2e_node/system/docker_validator.go create mode 100644 test/e2e_node/system/kernel_validator.go create mode 100644 test/e2e_node/system/os_validator.go create mode 100644 test/e2e_node/system/types.go create mode 100644 test/e2e_node/system/util.go create mode 100644 test/e2e_node/system/validators.go diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index b2cd367909a..cbee20a07a4 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -549,6 +549,7 @@ system-cgroups system-container system-pods-startup-timeout system-reserved +system-validate-mode target-port target-ram-mb tcp-services diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 73a539893a9..821089e635e 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -36,8 +36,10 @@ import ( commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/services" + "k8s.io/kubernetes/test/e2e_node/system" "github.com/golang/glog" + "github.com/kardianos/osext" . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" more_reporters "github.com/onsi/ginkgo/reporters" @@ -47,7 +49,9 @@ import ( var e2es *services.E2EServices +// TODO(random-liu): Change the following modes to sub-command. var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.") +var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.") func init() { framework.RegisterCommonFlags() @@ -73,6 +77,13 @@ func TestE2eNode(t *testing.T) { services.RunE2EServices() return } + if *systemValidateMode { + // If system-validate-mode is specified, only run system validation in current process. + if err := system.Validate(); err != nil { + glog.Exitf("system validation failed: %v", err) + } + return + } // If run-services-mode is not specified, run test. rand.Seed(time.Now().UTC().UnixNano()) RegisterFailHandler(Fail) @@ -100,6 +111,10 @@ var _ = SynchronizedBeforeSuite(func() []byte { Expect(err).NotTo(HaveOccurred(), "should be able to get node name") framework.TestContext.NodeName = hostname } + + // Run system validation test. + Expect(validateSystem()).To(Succeed(), "system validation") + // Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue // This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling. if framework.TestContext.PrepullImages { @@ -151,6 +166,22 @@ var _ = SynchronizedAfterSuite(func() {}, func() { glog.Infof("Tests Finished") }) +// validateSystem runs system validation in a separate process and returns error if validation fails. +func validateSystem() error { + testBin, err := osext.Executable() + if err != nil { + return fmt.Errorf("can't get current binary: %v", err) + } + // TODO(random-liu): Remove sudo in containerize PR. + output, err := exec.Command("sudo", testBin, "--system-validate-mode").CombinedOutput() + // The output of system validation should have been formatted, directly print here. + fmt.Print(string(output)) + if err != nil { + return fmt.Errorf("system validation failed") + } + return nil +} + func maskLocksmithdOnCoreos() { data, err := ioutil.ReadFile("/etc/os-release") if err != nil { diff --git a/test/e2e_node/system/cgroup_validator.go b/test/e2e_node/system/cgroup_validator.go new file mode 100644 index 00000000000..6ec1d76fcfa --- /dev/null +++ b/test/e2e_node/system/cgroup_validator.go @@ -0,0 +1,93 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +var _ Validator = &CgroupsValidator{} + +type CgroupsValidator struct{} + +func (c *CgroupsValidator) Name() string { + return "cgroups" +} + +const ( + cgroupsConfigPrefix = "CGROUPS_" +) + +func (c *CgroupsValidator) Validate(spec SysSpec) error { + subsystems, err := c.getCgroupSubsystems() + if err != nil { + return fmt.Errorf("failed to get cgroup subsystems: %v", err) + } + return c.validateCgroupSubsystems(spec.Cgroups, subsystems) +} + +func (c *CgroupsValidator) validateCgroupSubsystems(cgroupSpec, subsystems []string) error { + missing := []string{} + for _, cgroup := range cgroupSpec { + found := false + for _, subsystem := range subsystems { + if cgroup == subsystem { + found = true + break + } + } + item := cgroupsConfigPrefix + strings.ToUpper(cgroup) + if found { + report(item, "enabled", good) + } else { + report(item, "missing", bad) + missing = append(missing, cgroup) + } + } + if len(missing) > 0 { + return fmt.Errorf("missing cgroups: %s", strings.Join(missing, " ")) + } + return nil + +} + +func (c *CgroupsValidator) getCgroupSubsystems() ([]string, error) { + f, err := os.Open("/proc/cgroups") + if err != nil { + return nil, err + } + defer f.Close() + + subsystems := []string{} + s := bufio.NewScanner(f) + for s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + text := s.Text() + if text[0] != '#' { + parts := strings.Fields(text) + if len(parts) >= 4 && parts[3] != "0" { + subsystems = append(subsystems, parts[0]) + } + } + } + return subsystems, nil +} diff --git a/test/e2e_node/system/docker_validator.go b/test/e2e_node/system/docker_validator.go new file mode 100644 index 00000000000..0e207df8cb4 --- /dev/null +++ b/test/e2e_node/system/docker_validator.go @@ -0,0 +1,84 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "fmt" + "regexp" + + "github.com/docker/engine-api/client" + "github.com/docker/engine-api/types" + "golang.org/x/net/context" +) + +var _ Validator = &DockerValidator{} + +// DockerValidator validates docker configuration. +type DockerValidator struct{} + +func (d *DockerValidator) Name() string { + return "docker" +} + +const ( + dockerEndpoint = "unix:///var/run/docker.sock" + dockerConfigPrefix = "DOCKER_" +) + +// TODO(random-liu): Add more validating items. +func (d *DockerValidator) Validate(spec SysSpec) error { + if spec.RuntimeSpec.DockerSpec == nil { + // If DockerSpec is not specified, assume current runtime is not + // docker, skip the docker configuration validation. + return nil + } + c, err := client.NewClient(dockerEndpoint, "", nil, nil) + if err != nil { + return fmt.Errorf("failed to create docker client: %v", err) + } + info, err := c.Info(context.Background()) + if err != nil { + return fmt.Errorf("failed to get docker info: %v", err) + } + return d.validateDockerInfo(spec.RuntimeSpec.DockerSpec, info) +} + +func (d *DockerValidator) validateDockerInfo(spec *DockerSpec, info types.Info) error { + // Validate docker version. + matched := false + for _, v := range spec.Version { + r := regexp.MustCompile(v) + if r.MatchString(info.ServerVersion) { + report(dockerConfigPrefix+"VERSION", info.ServerVersion, good) + matched = true + } + } + if !matched { + report(dockerConfigPrefix+"VERSION", info.ServerVersion, bad) + return fmt.Errorf("unsupported docker version: %s", info.ServerVersion) + } + // Validate graph driver. + item := dockerConfigPrefix + "GRAPH_DRIVER" + for _, d := range spec.GraphDriver { + if info.Driver == d { + report(item, info.Driver, good) + return nil + } + } + report(item, info.Driver, bad) + return fmt.Errorf("unsupported graph driver: %s", info.Driver) +} diff --git a/test/e2e_node/system/kernel_validator.go b/test/e2e_node/system/kernel_validator.go new file mode 100644 index 00000000000..f71d7072e32 --- /dev/null +++ b/test/e2e_node/system/kernel_validator.go @@ -0,0 +1,255 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "bufio" + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/errors" +) + +var _ Validator = &KernelValidator{} + +// KernelValidator validates kernel. Currently only validate kernel version +// and kernel configuration. +type KernelValidator struct { + kernelRelease string +} + +func (k *KernelValidator) Name() string { + return "kernel" +} + +// kConfigOption is the possible kernel config option. +type kConfigOption string + +const ( + builtIn kConfigOption = "y" + asModule kConfigOption = "m" + leftOut kConfigOption = "n" + + // validKConfigRegex is the regex matching kernel configuration line. + validKConfigRegex = "^CONFIG_[A-Z0-9_]+=[myn]" + // kConfigPrefix is the prefix of kernel configuration. + kConfigPrefix = "CONFIG_" +) + +func (k *KernelValidator) Validate(spec SysSpec) error { + out, err := exec.Command("uname", "-r").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to get kernel release: %v", err) + } + k.kernelRelease = strings.TrimSpace(string(out)) + var errs []error + errs = append(errs, k.validateKernelVersion(spec.KernelSpec)) + errs = append(errs, k.validateKernelConfig(spec.KernelSpec)) + return errors.NewAggregate(errs) +} + +// validateKernelVersion validates the kernel version. +func (k *KernelValidator) validateKernelVersion(kSpec KernelSpec) error { + glog.Infof("Validating kernel version") + versionRegexps := kSpec.Versions + for _, versionRegexp := range versionRegexps { + r := regexp.MustCompile(versionRegexp) + if r.MatchString(k.kernelRelease) { + report("KERNEL_VERSION", k.kernelRelease, good) + return nil + } + } + report("KERNEL_VERSION", k.kernelRelease, bad) + return fmt.Errorf("unsupported kernel release: %s", k.kernelRelease) +} + +// validateKernelConfig validates the kernel configurations. +func (k *KernelValidator) validateKernelConfig(kSpec KernelSpec) error { + glog.Infof("Validating kernel config") + allConfig, err := k.getKernelConfig() + if err != nil { + return fmt.Errorf("failed to parse kernel config: %v", err) + } + return k.validateCachedKernelConfig(allConfig, kSpec) +} + +// validateCachedKernelConfig validates the kernel confgiurations cached in internal data type. +func (k *KernelValidator) validateCachedKernelConfig(allConfig map[string]kConfigOption, kSpec KernelSpec) error { + badConfigs := []string{} + // reportAndRecord is a helper function to record bad config when + // report. + reportAndRecord := func(name, msg, desc string, result resultType) { + if result == bad { + badConfigs = append(badConfigs, name) + } + // report description when the config is bad or warn. + if result != good && desc != "" { + msg = msg + " - " + desc + } + report(name, msg, result) + } + const ( + required = iota + optional + forbidden + ) + validateOpt := func(config KernelConfig, expect int) { + var found, missing resultType + switch expect { + case required: + found, missing = good, bad + case optional: + found, missing = good, warn + case forbidden: + found, missing = bad, good + } + var name string + var opt kConfigOption + var ok bool + for _, name = range append([]string{config.Name}, config.Aliases...) { + name = kConfigPrefix + name + if opt, ok = allConfig[name]; ok { + break + } + } + if !ok { + reportAndRecord(name, "not set", config.Description, missing) + return + } + switch opt { + case builtIn: + reportAndRecord(name, "enabled", config.Description, found) + case asModule: + reportAndRecord(name, "enabled (as module)", config.Description, found) + case leftOut: + reportAndRecord(name, "disabled", config.Description, missing) + default: + reportAndRecord(name, fmt.Sprintf("unknown option: %s", opt), config.Description, missing) + } + } + for _, config := range kSpec.Required { + validateOpt(config, required) + } + for _, config := range kSpec.Optional { + validateOpt(config, optional) + } + for _, config := range kSpec.Forbidden { + validateOpt(config, forbidden) + } + if len(badConfigs) > 0 { + return fmt.Errorf("unexpected kernel config: %s", strings.Join(badConfigs, " ")) + } + return nil +} + +// getKernelConfigReader search kernel config file in a predefined list. Once the kernel config +// file is found it will read the configurations into a byte buffer and return. If the kernel +// config file is not found, it will try to load kernel config module and retry again. +func (k *KernelValidator) getKernelConfigReader() (io.Reader, error) { + possibePaths := []string{ + "/proc/config.gz", + "/boot/config-" + k.kernelRelease, + "/usr/src/linux-" + k.kernelRelease + "/.config", + "/usr/src/linux/.config", + } + configsModule := "configs" + modprobeCmd := "modprobe" + // loadModule indicates whether we've tried to load kernel config module ourselves. + loadModule := false + for { + for _, path := range possibePaths { + _, err := os.Stat(path) + if err != nil { + continue + } + // Buffer the whole file, so that we can close the file and unload + // kernel config module in this function. + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var r io.Reader + r = bytes.NewReader(b) + // This is a gzip file (config.gz), unzip it. + if filepath.Ext(path) == ".gz" { + r, err = gzip.NewReader(r) + if err != nil { + return nil, err + } + } + return r, nil + } + // If we've tried to load kernel config module, break and return error. + if loadModule { + break + } + // If the kernel config file is not found, try to load the kernel + // config module and check again. + // TODO(random-liu): Remove "sudo" in containerize test PR #31093 + output, err := exec.Command("sudo", modprobeCmd, configsModule).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("unable to load kernel module %q: output - %q, err - %v", + configsModule, output, err) + } + // Unload the kernel config module to make sure the validation have no side effect. + defer exec.Command("sudo", modprobeCmd, "-r", configsModule).Run() + loadModule = true + } + return nil, fmt.Errorf("no config path in %v is available", possibePaths) +} + +// getKernelConfig gets kernel config from kernel config file and convert kernel config to internal type. +func (k *KernelValidator) getKernelConfig() (map[string]kConfigOption, error) { + r, err := k.getKernelConfigReader() + if err != nil { + return nil, err + } + return k.parseKernelConfig(r) +} + +// parseKernelConfig converts kernel config to internal type. +func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOption, error) { + config := map[string]kConfigOption{} + regex := regexp.MustCompile(validKConfigRegex) + s := bufio.NewScanner(r) + for s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + line := strings.TrimSpace(s.Text()) + if !regex.MatchString(line) { + continue + } + fields := strings.Split(line, "=") + if len(fields) != 2 { + glog.Errorf("Unexpected fields number in config %q", line) + continue + } + config[fields[0]] = kConfigOption(fields[1]) + } + return config, nil + +} diff --git a/test/e2e_node/system/os_validator.go b/test/e2e_node/system/os_validator.go new file mode 100644 index 00000000000..fc228a3418c --- /dev/null +++ b/test/e2e_node/system/os_validator.go @@ -0,0 +1,48 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "fmt" + "os/exec" + "strings" +) + +var _ Validator = &OSValidator{} + +type OSValidator struct{} + +func (c *OSValidator) Name() string { + return "os" +} + +func (c *OSValidator) Validate(spec SysSpec) error { + out, err := exec.Command("uname").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to get os name: %v", err) + } + return c.validateOS(strings.TrimSpace(string(out)), spec.OS) +} + +func (c *OSValidator) validateOS(os, specOS string) error { + if os != specOS { + report("OS", os, bad) + return fmt.Errorf("unsupported operating system: %s", os) + } + report("OS", os, good) + return nil +} diff --git a/test/e2e_node/system/types.go b/test/e2e_node/system/types.go new file mode 100644 index 00000000000..79e0d779852 --- /dev/null +++ b/test/e2e_node/system/types.go @@ -0,0 +1,122 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +// KernelConfig defines one kernel configration item. +type KernelConfig struct { + // Name is the general name of the kernel configuration. It is used to + // match kernel configuration. + Name string + // Aliases are aliases of the kernel configuration. Some configuration + // has different names in different kernel version. Names of different + // versions will be treated as aliases. + Aliases []string + // Description is the description of the kernel configuration, for example: + // * What is it used for? + // * Why is it needed? + // * Who needs it? + Description string +} + +// KernelSpec defines the specification for the kernel. Currently, it contains +// specification for: +// * Kernel Version +// * Kernel Configuration +type KernelSpec struct { + // Versions define supported kernel version. It is a group of regexps. + Versions []string + // Required contains all kernel configurations required to be enabled + // (built in or as module). + Required []KernelConfig + // Optional contains all kernel configurations are required for optional + // features. + Optional []KernelConfig + // Forbidden contains all kernel configurations which areforbidden (disabled + // or not set) + Forbidden []KernelConfig +} + +// DockerSpec defines the requirement configuration for docker. Currently, it only +// contains spec for graph driver. +type DockerSpec struct { + // Version is a group of regex matching supported docker versions. + Version []string + // GraphDriver is the graph drivers supported by kubelet. + GraphDriver []string +} + +// RuntimeSpec is the abstract layer for different runtimes. Different runtimes +// should put their spec inside the RuntimeSpec. +type RuntimeSpec struct { + *DockerSpec +} + +// SysSpec defines the requirement of supported system. Currently, it only contains +// spec for OS, Kernel and Cgroups. +type SysSpec struct { + // OS is the operating system of the SysSpec. + OS string + // KernelConfig defines the spec for kernel. + KernelSpec KernelSpec + // Cgroups is the required cgroups. + Cgroups []string + // RuntimeSpec defines the spec for runtime. + RuntimeSpec RuntimeSpec +} + +// DefaultSysSpec is the default SysSpec. +var DefaultSysSpec = SysSpec{ + OS: "Linux", + KernelSpec: KernelSpec{ + Versions: []string{`3\.[1-9][0-9].*`, `4\..*`}, // Requires 3.10+ or 4+ + // TODO(random-liu): Add more config + // TODO(random-liu): Add description for each kernel configuration: + Required: []KernelConfig{ + {Name: "NAMESPACES"}, + {Name: "NET_NS"}, + {Name: "PID_NS"}, + {Name: "IPC_NS"}, + {Name: "UTS_NS"}, + {Name: "CGROUPS"}, + {Name: "CGROUP_CPUACCT"}, + {Name: "CGROUP_DEVICE"}, + {Name: "CGROUP_FREEZER"}, + {Name: "CGROUP_SCHED"}, + {Name: "CPUSETS"}, + {Name: "MEMCG"}, + {Name: "INET"}, + {Name: "EXT4_FS"}, + {Name: "PROC_FS"}, + {Name: "NETFILTER_XT_TARGET_REDIRECT", Aliases: []string{"IP_NF_TARGET_REDIRECT"}}, + {Name: "NETFILTER_XT_MATCH_COMMENT"}, + }, + Optional: []KernelConfig{ + {Name: "OVERLAY_FS", Aliases: []string{"OVERLAYFS_FS"}, Description: "Required for overlayfs."}, + {Name: "AUFS_FS", Description: "Required for aufs."}, + {Name: "BLK_DEV_DM", Description: "Required for devicemapper."}, + }, + Forbidden: []KernelConfig{}, + }, + Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"}, + RuntimeSpec: RuntimeSpec{ + DockerSpec: &DockerSpec{ + Version: []string{`1\.(9|\d{2,})\..*`}, // Requires 1.9+ + // TODO(random-liu): Validate overlay2. + GraphDriver: []string{"aufs", "overlay", "devicemapper"}, + }, + }, +} diff --git a/test/e2e_node/system/util.go b/test/e2e_node/system/util.go new file mode 100644 index 00000000000..612891fcac2 --- /dev/null +++ b/test/e2e_node/system/util.go @@ -0,0 +1,62 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "fmt" +) + +// resultType is type of the validation result. Different validation results +// corresponds to different colors. +type resultType int + +const ( + good resultType = iota + bad + warn +) + +// color is the color of the message. +type color int + +const ( + red color = 31 + green = 32 + yellow = 33 + white = 37 +) + +func wrap(s string, c color) string { + return fmt.Sprintf("\033[0;%dm%s\033[0m", c, s) +} + +// report reports "item: r". item is white, and the color of r depends on the +// result type. +func report(item, r string, t resultType) { + var c color + switch t { + case good: + c = green + case bad: + c = red + case warn: + c = yellow + default: + c = white + } + fmt.Printf("%s: %s\n", wrap(item, white), wrap(r, c)) +} diff --git a/test/e2e_node/system/validators.go b/test/e2e_node/system/validators.go new file mode 100644 index 00000000000..c799f8ff34d --- /dev/null +++ b/test/e2e_node/system/validators.go @@ -0,0 +1,49 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/errors" +) + +// Validator is the interface for all validators. +type Validator interface { + // Name is the name of the validator. + Name() string + // Validate is the validate function. + Validate(SysSpec) error +} + +// validators are all the validators. +var validators = []Validator{ + &OSValidator{}, + &KernelValidator{}, + &CgroupsValidator{}, + &DockerValidator{}, +} + +// Validate uses all validators to validate the system. +func Validate() error { + var errs []error + spec := DefaultSysSpec + for _, v := range validators { + glog.Infof("Validating %s...", v.Name()) + errs = append(errs, v.Validate(spec)) + } + return errors.NewAggregate(errs) +} From b76b2f218b058b6af4d23cffad3d4fc708a64835 Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Thu, 3 Nov 2016 20:36:52 -0700 Subject: [PATCH 2/3] Add unit test for system verification --- hack/make-rules/test.sh | 1 + test/e2e_node/system/cgroup_validator_test.go | 54 +++++ test/e2e_node/system/docker_validator_test.go | 57 ++++++ test/e2e_node/system/kernel_validator_test.go | 191 ++++++++++++++++++ test/e2e_node/system/os_validator_test.go | 52 +++++ 5 files changed, 355 insertions(+) create mode 100644 test/e2e_node/system/cgroup_validator_test.go create mode 100644 test/e2e_node/system/docker_validator_test.go create mode 100644 test/e2e_node/system/kernel_validator_test.go create mode 100644 test/e2e_node/system/os_validator_test.go diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index db6a3998496..5ff912f1bfb 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -52,6 +52,7 @@ kube::test::find_dirs() { find -L . \ -path './_output' -prune \ -o -path './vendor/k8s.io/client-go/*' \ + -o -path './test/e2e_node/system/*' \ -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | LC_ALL=C sort -u ) } diff --git a/test/e2e_node/system/cgroup_validator_test.go b/test/e2e_node/system/cgroup_validator_test.go new file mode 100644 index 00000000000..d306154b96d --- /dev/null +++ b/test/e2e_node/system/cgroup_validator_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateCgroupSubsystem(t *testing.T) { + v := &CgroupsValidator{} + cgroupSpec := []string{"system1", "system2"} + for desc, test := range map[string]struct { + cgroupSpec []string + subsystems []string + err bool + }{ + "missing cgroup subsystem should report error": { + subsystems: []string{"system1"}, + err: true, + }, + "extra cgroup subsystems should not report error": { + subsystems: []string{"system1", "system2", "system3"}, + err: false, + }, + "subsystems the same with spec should not report error": { + subsystems: []string{"system1", "system2"}, + err: false, + }, + } { + err := v.validateCgroupSubsystems(cgroupSpec, test.subsystems) + if !test.err { + assert.Nil(t, err, "%q: Expect error not to occur with cgroup", desc) + } else { + assert.NotNil(t, err, "%q: Expect error to occur with docker info", desc) + } + + } +} diff --git a/test/e2e_node/system/docker_validator_test.go b/test/e2e_node/system/docker_validator_test.go new file mode 100644 index 00000000000..7e3f43171b0 --- /dev/null +++ b/test/e2e_node/system/docker_validator_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "testing" + + "github.com/docker/engine-api/types" + "github.com/stretchr/testify/assert" +) + +func TestValidateDockerInfo(t *testing.T) { + v := &DockerValidator{} + spec := &DockerSpec{ + Version: []string{`1\.(9|\d{2,})\..*`}, + GraphDriver: []string{"driver_1", "driver_2"}, + } + for _, test := range []struct { + info types.Info + err bool + }{ + { + info: types.Info{Driver: "driver_1", ServerVersion: "1.10.1"}, + err: false, + }, + { + info: types.Info{Driver: "bad_driver", ServerVersion: "1.9.1"}, + err: true, + }, + { + info: types.Info{Driver: "driver_2", ServerVersion: "1.8.1"}, + err: true, + }, + } { + err := v.validateDockerInfo(spec, test.info) + if !test.err { + assert.Nil(t, err, "Expect error not to occur with docker info %+v", test.info) + } else { + assert.NotNil(t, err, "Expect error to occur with docker info %+v", test.info) + } + + } +} diff --git a/test/e2e_node/system/kernel_validator_test.go b/test/e2e_node/system/kernel_validator_test.go new file mode 100644 index 00000000000..be1420a4a20 --- /dev/null +++ b/test/e2e_node/system/kernel_validator_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateKernelVersion(t *testing.T) { + v := &KernelValidator{} + // Currently, testRegex is align with DefaultSysSpec.KernelVersion, but in the future + // they may be different. + // This is fine, because the test mainly tests the kernel version validation logic, + // not the DefaultSysSpec. The DefaultSysSpec should be tested with node e2e. + testRegex := []string{`3\.[1-9][0-9].*`, `4\..*`} + for _, test := range []struct { + version string + err bool + }{ + // first version regex matches + { + version: "3.19.9-99-test", + err: false, + }, + // one of version regexes matches + { + version: "4.4.14+", + err: false, + }, + // no version regex matches + { + version: "2.0.0", + err: true, + }, + { + version: "5.0.0", + err: true, + }, + { + version: "3.9.0", + err: true, + }, + } { + v.kernelRelease = test.version + err := v.validateKernelVersion(KernelSpec{Versions: testRegex}) + if !test.err { + assert.Nil(t, err, "Expect error not to occur with kernel version %q", test.version) + } else { + assert.NotNil(t, err, "Expect error to occur with kenrel version %q", test.version) + } + } +} + +func TestValidateCachedKernelConfig(t *testing.T) { + v := &KernelValidator{} + testKernelSpec := KernelSpec{ + Required: []KernelConfig{{Name: "REQUIRED_1"}, {Name: "REQUIRED_2", Aliases: []string{"ALIASE_REQUIRED_2"}}}, + Optional: []KernelConfig{{Name: "OPTIONAL_1"}, {Name: "OPTIONAL_2"}}, + Forbidden: []KernelConfig{ + {Name: "FORBIDDEN_1", Description: "TEST FORBIDDEN"}, + {Name: "FORBIDDEN_2", Aliases: []string{"ALIASE_FORBIDDEN_2"}}, + }, + } + for c, test := range []struct { + desc string + config map[string]kConfigOption + err bool + }{ + { + desc: "meet all required configurations should not report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "REQUIRED_2": asModule, + }, + err: false, + }, + { + desc: "one required configuration disabled should report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": leftOut, + "REQUIRED_2": builtIn, + }, + err: true, + }, + { + desc: "one required configuration missing should report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + }, + err: true, + }, + { + desc: "alias of required configuration should not report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "ALIASE_REQUIRED_2": asModule, + }, + err: false, + }, + { + desc: "optional configuration set or not should not report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "REQUIRED_2": asModule, + "OPTIONAL_1": builtIn, + }, + err: false, + }, + { + desc: "forbidden configuration disabled should not report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "REQUIRED_2": asModule, + "FORBIDDEN_1": leftOut, + }, + err: false, + }, + { + desc: "forbidden configuration built-in should report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "REQUIRED_2": asModule, + "FORBIDDEN_1": builtIn, + }, + err: true, + }, + { + desc: "forbidden configuration built as module should report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "REQUIRED_2": asModule, + "FORBIDDEN_1": asModule, + }, + err: true, + }, + { + desc: "alias of forbidden configuration should report error.", + config: map[string]kConfigOption{ + "REQUIRED_1": builtIn, + "REQUIRED_2": asModule, + "ALIASE_FORBIDDEN_2": asModule, + }, + err: true, + }, + } { + t.Logf("TestCase #%d %s", c, test.desc) + // Add kernel config prefix. + for k, v := range test.config { + delete(test.config, k) + test.config[kConfigPrefix+k] = v + } + err := v.validateCachedKernelConfig(test.config, testKernelSpec) + if !test.err { + assert.Nil(t, err, "Expect error not to occur with kernel config %q", test.config) + } else { + assert.NotNil(t, err, "Expect error to occur with kenrel config %q", test.config) + } + } +} + +func TestValidateParseKernelConfig(t *testing.T) { + config := `CONFIG_1=y +CONFIG_2=m +CONFIG_3=n` + expected := map[string]kConfigOption{ + "CONFIG_1": builtIn, + "CONFIG_2": asModule, + "CONFIG_3": leftOut, + } + v := &KernelValidator{} + got, err := v.parseKernelConfig(bytes.NewReader([]byte(config))) + assert.Nil(t, err, "Expect error not to occur when parse kernel configuration %q", config) + assert.Equal(t, expected, got) +} diff --git a/test/e2e_node/system/os_validator_test.go b/test/e2e_node/system/os_validator_test.go new file mode 100644 index 00000000000..f9e70a752c1 --- /dev/null +++ b/test/e2e_node/system/os_validator_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateOS(t *testing.T) { + v := &OSValidator{} + specOS := "Linux" + for _, test := range []struct { + os string + err bool + }{ + { + os: "Linux", + err: false, + }, + { + os: "Windows", + err: true, + }, + { + os: "Darwin", + err: true, + }, + } { + err := v.validateOS(test.os, specOS) + if !test.err { + assert.Nil(t, err, "Expect error not to occur with os %q", test.os) + } else { + assert.NotNil(t, err, "Expect error to occur with os %q", test.os) + } + } +} From f9b50f09497fd95a1ca2ffa7440e7399e847eefe Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Thu, 3 Nov 2016 18:27:03 -0700 Subject: [PATCH 3/3] Update bazel. --- test/e2e_node/BUILD | 2 ++ test/e2e_node/system/BUILD | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 test/e2e_node/system/BUILD diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index c9493a2f01b..a68ecd6f90c 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -98,9 +98,11 @@ go_test( "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e_node/services:go_default_library", + "//test/e2e_node/system:go_default_library", "//test/utils:go_default_library", "//vendor:github.com/davecgh/go-spew/spew", "//vendor:github.com/golang/glog", + "//vendor:github.com/kardianos/osext", "//vendor:github.com/onsi/ginkgo", "//vendor:github.com/onsi/ginkgo/config", "//vendor:github.com/onsi/ginkgo/reporters", diff --git a/test/e2e_node/system/BUILD b/test/e2e_node/system/BUILD new file mode 100644 index 00000000000..646b3d4970a --- /dev/null +++ b/test/e2e_node/system/BUILD @@ -0,0 +1,48 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", + "cgo_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "cgroup_validator.go", + "docker_validator.go", + "kernel_validator.go", + "os_validator.go", + "types.go", + "util.go", + "validators.go", + ], + tags = ["automanaged"], + deps = [ + "//pkg/util/errors:go_default_library", + "//vendor:github.com/docker/engine-api/client", + "//vendor:github.com/docker/engine-api/types", + "//vendor:github.com/golang/glog", + "//vendor:golang.org/x/net/context", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "cgroup_validator_test.go", + "docker_validator_test.go", + "kernel_validator_test.go", + "os_validator_test.go", + ], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//vendor:github.com/docker/engine-api/types", + "//vendor:github.com/stretchr/testify/assert", + ], +)