mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-04 02:37:36 +00:00
re-organize remote e2e test to be pluggable
Signed-off-by: Davanum Srinivas <davanum@gmail.com>
This commit is contained in:
Davanum Srinivas
@@ -22,232 +22,23 @@ package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e_node/remote"
|
||||
"k8s.io/kubernetes/test/e2e_node/system"
|
||||
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
"k8s.io/kubernetes/test/e2e_node/remote"
|
||||
)
|
||||
|
||||
var mode = flag.String("mode", "gce", "Mode to operate in. One of gce|ssh. Defaults to gce")
|
||||
var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
|
||||
var testSuite = flag.String("test-suite", "default", "Test suite the runner initializes with. Currently support default|cadvisor|conformance")
|
||||
var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
|
||||
var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run")
|
||||
var imageConfigDir = flag.String("image-config-dir", "", "(optional) path to image config files")
|
||||
var images = flag.String("images", "", "images to test")
|
||||
var hosts = flag.String("hosts", "", "hosts to test")
|
||||
var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances")
|
||||
var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created")
|
||||
var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
|
||||
var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
|
||||
var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
|
||||
var systemSpecName = flag.String("system-spec-name", "", fmt.Sprintf("The name of the system spec used for validating the image in the node conformance test. The specs are at %s. If unspecified, the default built-in spec (system.DefaultSpec) will be used.", system.SystemSpecPath))
|
||||
var extraEnvs = flag.String("extra-envs", "", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2")
|
||||
var runtimeConfig = flag.String("runtime-config", "", "The runtime configuration for the API server on the node e2e tests.. Format: a list of key=value pairs, e.g., env1=val1,env2=val2")
|
||||
var kubeletConfigFile = flag.String("kubelet-config-file", "", "The KubeletConfiguration file that should be applied to the kubelet")
|
||||
var (
|
||||
arc Archive
|
||||
)
|
||||
|
||||
// Archive contains path info in the archive.
|
||||
type Archive struct {
|
||||
sync.Once
|
||||
path string
|
||||
err error
|
||||
}
|
||||
|
||||
func main() {
|
||||
klog.InitFlags(nil)
|
||||
flag.Parse()
|
||||
|
||||
var suite remote.TestSuite
|
||||
switch *testSuite {
|
||||
case "conformance":
|
||||
suite = remote.InitConformanceRemote()
|
||||
case "cadvisor":
|
||||
suite = remote.InitCAdvisorE2ERemote()
|
||||
// TODO: Add subcommand for node soaking, node conformance, cri validation.
|
||||
case "default":
|
||||
// Use node e2e suite by default if no subcommand is specified.
|
||||
suite = remote.InitNodeE2ERemote()
|
||||
default:
|
||||
klog.Fatalf("--test-suite must be one of default, cadvisor, or conformance")
|
||||
}
|
||||
|
||||
// Listen for SIGINT and ignore the first one. In case SIGINT is sent to this
|
||||
// process and all its children, we ignore it here, while our children ssh connections
|
||||
// are stopped. This allows us to gather artifacts and print out test state before
|
||||
// being killed.
|
||||
c := make(chan os.Signal, 2)
|
||||
signal.Notify(c, os.Interrupt)
|
||||
go func() {
|
||||
<-c
|
||||
fmt.Printf("Received SIGINT. Will exit on next SIGINT.\n")
|
||||
<-c
|
||||
fmt.Printf("Received another SIGINT. Will exit.\n")
|
||||
os.Exit(1)
|
||||
}()
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
if *buildOnly {
|
||||
// Build the archive and exit
|
||||
remote.CreateTestArchive(suite, *systemSpecName, *kubeletConfigFile)
|
||||
return
|
||||
}
|
||||
|
||||
// Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh
|
||||
allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags)
|
||||
fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags)
|
||||
|
||||
var runner remote.Runner
|
||||
cfg := remote.Config{
|
||||
InstanceNamePrefix: *instanceNamePrefix,
|
||||
ImageConfigFile: *imageConfigFile,
|
||||
ImageConfigDir: *imageConfigDir,
|
||||
Images: splitCommaList(*images),
|
||||
Hosts: parseHostsList(*hosts),
|
||||
GinkgoFlags: allGinkgoFlags,
|
||||
DeleteInstances: *deleteInstances,
|
||||
Cleanup: *cleanup,
|
||||
TestArgs: *testArgs,
|
||||
ExtraEnvs: *extraEnvs,
|
||||
RuntimeConfig: *runtimeConfig,
|
||||
SystemSpecName: *systemSpecName,
|
||||
}
|
||||
|
||||
var sshRunner remote.Runner
|
||||
switch *mode {
|
||||
case "gce":
|
||||
runner = remote.NewGCERunner(cfg)
|
||||
sshRunner = remote.NewSSHRunner(cfg)
|
||||
case "ssh":
|
||||
runner = remote.NewSSHRunner(cfg)
|
||||
}
|
||||
|
||||
if err := runner.Validate(); err != nil {
|
||||
klog.Fatalf("validating remote config, %s", err)
|
||||
}
|
||||
|
||||
// Setup coloring
|
||||
stat, _ := os.Stdout.Stat()
|
||||
useColor := (stat.Mode() & os.ModeCharDevice) != 0
|
||||
blue := ""
|
||||
noColour := ""
|
||||
if useColor {
|
||||
blue = "\033[0;34m"
|
||||
noColour = "\033[0m"
|
||||
}
|
||||
|
||||
results := make(chan *remote.TestResult)
|
||||
|
||||
path, err := arc.getArchive(suite)
|
||||
suite, err := remote.GetTestSuite(*testSuite)
|
||||
if err != nil {
|
||||
log.Fatalf("unable to create test archive: %s", err)
|
||||
klog.Fatalf("error looking up testsuite [%v] - registered test suites [%v]",
|
||||
err,
|
||||
remote.GetTestSuiteKeys())
|
||||
}
|
||||
defer arc.deleteArchive()
|
||||
|
||||
running := runner.StartTests(suite, path, results)
|
||||
// You can potentially run SSH based tests while running image based test as well. The GCE provider does this, see
|
||||
// test-e2e-node.sh.
|
||||
if sshRunner != nil && len(cfg.Hosts) > 0 {
|
||||
running += sshRunner.StartTests(suite, path, results)
|
||||
}
|
||||
|
||||
// Wait for all tests to complete and emit the results
|
||||
errCount := 0
|
||||
exitOk := true
|
||||
for i := 0; i < running; i++ {
|
||||
tr := <-results
|
||||
host := tr.Host
|
||||
fmt.Println() // Print an empty line
|
||||
fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour)
|
||||
fmt.Printf("%s> START TEST >%s\n", blue, noColour)
|
||||
fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour)
|
||||
fmt.Printf("Start Test Suite on Host %s\n", host)
|
||||
fmt.Printf("%s\n", tr.Output)
|
||||
if tr.Err != nil {
|
||||
errCount++
|
||||
fmt.Printf("Failure Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n%v\n", host, tr.Err)
|
||||
} else {
|
||||
fmt.Printf("Success Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n", host)
|
||||
}
|
||||
exitOk = exitOk && tr.ExitOK
|
||||
fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
|
||||
fmt.Printf("%s< FINISH TEST <%s\n", blue, noColour)
|
||||
fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour)
|
||||
fmt.Println() // Print an empty line
|
||||
}
|
||||
// Set the exit code if there were failures
|
||||
if !exitOk {
|
||||
fmt.Printf("Failure: %d errors encountered.\n", errCount)
|
||||
callGubernator(*gubernator)
|
||||
arc.deleteArchive()
|
||||
os.Exit(1)
|
||||
}
|
||||
callGubernator(*gubernator)
|
||||
}
|
||||
|
||||
func splitCommaList(s string) []string {
|
||||
if len(s) == 0 {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(s, ",")
|
||||
}
|
||||
|
||||
func callGubernator(gubernator bool) {
|
||||
if gubernator {
|
||||
fmt.Println("Running gubernator.sh")
|
||||
output, err := exec.Command("./test/e2e_node/gubernator.sh", "y").Output()
|
||||
|
||||
if err != nil {
|
||||
fmt.Println("gubernator.sh Failed")
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s", output)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (a *Archive) getArchive(suite remote.TestSuite) (string, error) {
|
||||
a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName, *kubeletConfigFile) })
|
||||
return a.path, a.err
|
||||
}
|
||||
|
||||
func (a *Archive) deleteArchive() {
|
||||
path, err := a.getArchive(nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
os.Remove(path)
|
||||
}
|
||||
|
||||
// parseHostsList splits a host list of the form a=1.2.3.4,b=5.6.7.8 into the list of hosts [a,b] while registering the
|
||||
// given addresses
|
||||
func parseHostsList(hostList string) []string {
|
||||
if len(hostList) == 0 {
|
||||
return nil
|
||||
}
|
||||
hosts := strings.Split(hostList, ",")
|
||||
var hostsOnly []string
|
||||
for _, host := range hosts {
|
||||
segs := strings.Split(host, "=")
|
||||
if len(segs) == 2 {
|
||||
remote.AddHostnameIP(segs[0], segs[1])
|
||||
} else if len(segs) > 2 {
|
||||
klog.Fatalf("invalid format of host %q", hostList)
|
||||
}
|
||||
hostsOnly = append(hostsOnly, segs[0])
|
||||
}
|
||||
return hostsOnly
|
||||
remote.RunRemoteTestSuite(suite)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user