diff --git a/test/e2e_node/jenkins/conformance/conformance-jenkins.sh b/test/e2e_node/jenkins/conformance/conformance-jenkins.sh new file mode 100755 index 00000000000..751768eb06c --- /dev/null +++ b/test/e2e_node/jenkins/conformance/conformance-jenkins.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script executed by jenkins to run node conformance test against gce +# Usage: test/e2e_node/jenkins/conformance-node-jenkins.sh + +set -e +set -x + +: "${1:?Usage test/e2e_node/jenkins/conformance-node-jenkins.sh }" + +. $1 + +make generated_files + +WORKSPACE=${WORKSPACE:-"/tmp/"} +ARTIFACTS=${WORKSPACE}/_artifacts +TIMEOUT=${TIMEOUT:-"45m"} + +mkdir -p ${ARTIFACTS} + +go run test/e2e_node/runner/remote/run_remote.go conformance \ + --logtostderr --vmodule=*=4 --ssh-env="gce" \ + --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \ + --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \ + --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \ + --results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \ + --test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \ + --instance-metadata="$GCE_INSTANCE_METADATA" diff --git a/test/e2e_node/jenkins/conformance/jenkins-conformance.properties b/test/e2e_node/jenkins/conformance/jenkins-conformance.properties new file mode 100644 index 00000000000..f465d8081a9 --- /dev/null +++ b/test/e2e_node/jenkins/conformance/jenkins-conformance.properties @@ -0,0 +1,6 @@ +GCE_HOSTS= +GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml +GCE_ZONE=us-central1-f +GCE_PROJECT=k8s-jkns-ci-node-e2e +CLEANUP=true +KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go new file mode 100644 index 00000000000..e914942c3e2 --- /dev/null +++ b/test/e2e_node/remote/node_conformance.go @@ -0,0 +1,298 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/golang/glog" + + "k8s.io/kubernetes/test/e2e_node/builder" +) + +// ConformanceRemote contains the specific functions in the node conformance test suite. +type ConformanceRemote struct{} + +func InitConformanceRemote() TestSuite { + return &ConformanceRemote{} +} + +// getConformanceDirectory gets node conformance test build directory. +func getConformanceDirectory() (string, error) { + k8sRoot, err := builder.GetK8sRootDir() + if err != nil { + return "", err + } + return filepath.Join(k8sRoot, "test", "e2e_node", "conformance", "build"), nil +} + +// commandToString is a helper function which formats command to string. +func commandToString(c *exec.Cmd) string { + return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ") +} + +// Image path constants. +const ( + conformanceRegistry = "gcr.io/google_containers" + conformanceArch = runtime.GOARCH + conformanceTarfile = "node_conformance.tar" + conformanceTestBinary = "e2e_node.test" + conformanceImageLoadTimeout = time.Duration(30) * time.Second +) + +// timestamp is used as an unique id of current test. +var timestamp = getTimestamp() + +// getConformanceImageRepo returns conformance image full repo name. +func getConformanceImageRepo() string { + return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp) +} + +// buildConformanceTest builds node conformance test image tarball into binDir. +func buildConformanceTest(binDir string) error { + // Get node conformance directory. + conformancePath, err := getConformanceDirectory() + if err != nil { + return fmt.Errorf("failed to get node conformance directory: %v", err) + } + // Build docker image. + cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir, + "REGISTRY="+conformanceRegistry, + "ARCH="+conformanceArch, + "VERSION="+timestamp) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q", + commandToString(cmd), err, output) + } + // Save docker image into tar file. + cmd = exec.Command("docker", "save", getConformanceImageRepo(), "-o", filepath.Join(binDir, conformanceTarfile)) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q", + commandToString(cmd), err, output) + } + return nil +} + +// SetupTestPackage sets up the test package with binaries k8s required for node conformance test +func (c *ConformanceRemote) SetupTestPackage(tardir string) error { + // Build the executables + if err := builder.BuildGo(); err != nil { + return fmt.Errorf("failed to build the depedencies: %v", err) + } + + // Make sure we can find the newly built binaries + buildOutputDir, err := builder.GetK8sBuildOutputDir() + if err != nil { + return fmt.Errorf("failed to locate kubernetes build output directory %v", err) + } + + // Build node conformance tarball. + if err := buildConformanceTest(buildOutputDir); err != nil { + return fmt.Errorf("failed to build node conformance test %v", err) + } + + // Copy files + requiredFiles := []string{"kubelet", conformanceTestBinary, conformanceTarfile} + for _, file := range requiredFiles { + source := filepath.Join(buildOutputDir, file) + if _, err := os.Stat(source); err != nil { + return fmt.Errorf("failed to locate test file %s: %v", file, err) + } + output, err := exec.Command("cp", source, filepath.Join(tardir, file)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy %q: error - %v output - %q", file, err, output) + } + } + + return nil +} + +// loadConformanceImage loads node conformance image from tar file. +func loadConformanceImage(host, workspace string) error { + tarfile := filepath.Join(workspace, conformanceTarfile) + if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(), + "docker", "load", "-i", tarfile); err != nil { + return fmt.Errorf("failed to load node conformance image from tar file %q: error - %v output - %q", + tarfile, err, output) + } + return nil +} + +// kubeletLauncherLog is the log of kubelet launcher. +const kubeletLauncherLog = "kubelet-launcher.log" + +// kubeletPodManifestPath is a fixed known pod manifest path. We can not use the random pod +// manifest directory generated in e2e_node.test because we need to mount the directory into +// the conformance test container, it's easier if it's a known directory. +// TODO(random-liu): Get rid of this once we switch to cluster e2e node bootstrap script. +var kubeletPodManifestPath = "conformance-pod-manifest-" + timestamp + +// getPodManifestPath returns pod manifest full path. +func getPodManifestPath(workspace string) string { + return filepath.Join(workspace, kubeletPodManifestPath) +} + +// isSystemd returns whether the node is a systemd node. +func isSystemd(host string) (bool, error) { + // Returns "systemd" if /run/systemd/system is found, empty string otherwise. + output, err := SSH(host, "test", "-e", "/run/systemd/system", "&&", "echo", "systemd", "||", "true") + if err != nil { + return false, fmt.Errorf("failed to check systemd: error - %v output - %q", err, output) + } + return strings.TrimSpace(output) != "", nil +} + +// launchKubelet launches kubelet by running e2e_node.test binary in run-kubelet-mode. +// This is a temporary solution, we should change node e2e to use the same node bootstrap +// with cluster e2e and launch kubelet outside of the test for both regular node e2e and +// node conformance test. +// TODO(random-liu): Switch to use standard node bootstrap script. +func launchKubelet(host, workspace, results, testArgs string) error { + podManifestPath := getPodManifestPath(workspace) + if output, err := SSH(host, "mkdir", podManifestPath); err != nil { + return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q", + podManifestPath, err, output) + } + startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --logtostderr --node-name=%s"+ + " --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1", + conformanceTestBinary, host, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog)) + var cmd []string + systemd, err := isSystemd(host) + if err != nil { + return fmt.Errorf("failed to check systemd: %v", err) + } + if systemd { + cmd = []string{ + "systemd-run", "sh", "-c", getSSHCommand(" && ", + // Switch to workspace. + fmt.Sprintf("cd %s", workspace), + // Launch kubelet by running e2e_node.test in run-kubelet-mode. + startKubeletCmd, + ), + } + } else { + cmd = []string{ + "sh", "-c", getSSHCommand(" && ", + // Switch to workspace. + fmt.Sprintf("cd %s", workspace), + // Launch kubelet by running e2e_node.test in run-kubelet-mode with nohup. + fmt.Sprintf("(nohup %s &)", startKubeletCmd), + ), + } + } + glog.Infof("Launch kubelet with command: %v", cmd) + output, err := SSH(host, cmd...) + if err != nil { + return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q", + cmd, err, output) + } + glog.Info("Successfully launch kubelet") + return nil +} + +// kubeletStopGracePeriod is the grace period to wait before forcibly killing kubelet. +const kubeletStopGracePeriod = 10 * time.Second + +// stopKubelet stops kubelet launcher and kubelet gracefully. +func stopKubelet(host, workspace string) error { + glog.Info("Gracefully stop kubelet launcher") + if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil { + return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q", + err, output) + } + glog.Info("Wait for kubelet launcher to stop") + stopped := false + for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) { + // Check whehther the process is still running. + output, err := SSH(host, "pidof", conformanceTestBinary, "||", "true") + if err != nil { + return fmt.Errorf("failed to check kubelet stopping: error - %v output -%q", + err, output) + } + // Kubelet is stopped + if strings.TrimSpace(output) == "" { + stopped = true + break + } + } + if !stopped { + glog.Info("Forcibly stop kubelet") + if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil { + return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q", + err, output) + } + } + glog.Info("Successfully stop kubelet") + // Clean up the pod manifest path + podManifestPath := getPodManifestPath(workspace) + if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil { + return fmt.Errorf("failed to cleanup pod manifest directory %q: error - %v, output - %q", + podManifestPath, err, output) + } + return nil +} + +// RunTest runs test on the node. +func (c *ConformanceRemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) { + // Install the cni plugin. + if err := installCNI(host, workspace); err != nil { + return "", err + } + + // Configure iptables firewall rules. + if err := configureFirewall(host); err != nil { + return "", err + } + + // Kill any running node processes. + cleanupNodeProcesses(host) + + // Load node conformance image. + if err := loadConformanceImage(host, workspace); err != nil { + return "", err + } + + // Launch kubelet. + if err := launchKubelet(host, workspace, results, testArgs); err != nil { + return "", err + } + // Stop kubelet. + defer func() { + if err := stopKubelet(host, workspace); err != nil { + // Only log an error if failed to stop kubelet because it is not critical. + glog.Errorf("failed to stop kubelet: %v", err) + } + }() + + // Run the tests + glog.Infof("Starting tests on %q", host) + podManifestPath := getPodManifestPath(workspace) + cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result %s'", + timeout.Seconds(), podManifestPath, podManifestPath, results, getConformanceImageRepo()) + testOutput, err := SSH(host, "sh", "-c", cmd) + if err != nil { + return testOutput, err + } + + return testOutput, nil +} diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index a7d3fd84a85..1a3c2b19554 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -98,8 +98,15 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, junit return "", false, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output) } + // Create the test result directory. + resultDir := filepath.Join(workspace, "results") + if output, err := SSHNoSudo(host, "mkdir", resultDir); err != nil { + // Exit failure with the error + return "", false, fmt.Errorf("failed to create test result directory %q on host %q: %v output: %q", resultDir, host, err, output) + } + glog.Infof("Running test on %q", host) - output, err := suite.RunTest(host, workspace, filepath.Join(workspace, "results"), junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds) + output, err := suite.RunTest(host, workspace, resultDir, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds) aggErrs := []error{} // Do not log the output here, let the caller deal with the test output. diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index d5d3c523fa0..c2bd9f229e8 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -134,6 +134,8 @@ func parseFlags() { // Parse subcommand. subcommand := os.Args[1] switch subcommand { + case "conformance": + suite = remote.InitConformanceRemote() // TODO: Add subcommand for node soaking, node conformance, cri validation. default: // Use node e2e suite by default if no subcommand is specified.