From 82c58f26588f0898577466f7fa13cdf0892c99fe Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 16 May 2017 09:50:14 -0700 Subject: [PATCH] Generate individual test results for upgrade tests --- test/e2e/BUILD | 1 + test/e2e/cluster_upgrade.go | 215 ++++++++++++++++++++++++------------ 2 files changed, 145 insertions(+), 71 deletions(-) diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 9e41db0f293..608cd11b404 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -172,6 +172,7 @@ go_library( "//test/e2e_federation:go_default_library", "//test/images/net/nat:go_default_library", "//test/utils:go_default_library", + "//test/utils/junit:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/github.com/elazarl/goproxy:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 4168c8dae38..8b6ea79721e 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -17,11 +17,20 @@ limitations under the License. package e2e import ( + "encoding/xml" + "fmt" + "os" + "path/filepath" + "sync" + "time" + "k8s.io/client-go/discovery" "k8s.io/kubernetes/pkg/util/version" "k8s.io/kubernetes/test/e2e/chaosmonkey" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/ginkgowrapper" "k8s.io/kubernetes/test/e2e/upgrades" + "k8s.io/kubernetes/test/utils/junit" . "github.com/onsi/ginkgo" ) @@ -45,32 +54,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { // Create the frameworks here because we can only create them // in a "Describe". - testFrameworks := map[string]*framework.Framework{} - for _, t := range upgradeTests { - testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name()) - } - + testFrameworks := createUpgradeFrameworks() framework.KubeDescribe("master upgrade", func() { It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() { upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) - cm := chaosmonkey.New(func() { + testSuite := &junit.TestSuite{Name: "Master upgrade"} + masterUpgradeTest := &junit.TestCase{Name: "master-upgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest) + + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, masterUpgradeTest) target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(framework.MasterUpgrade(target)) framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) - }) - for _, t := range upgradeTests { - cma := chaosMonkeyAdapter{ - test: t, - framework: testFrameworks[t.Name()], - upgradeType: upgrades.MasterUpgrade, - upgCtx: *upgCtx, - } - cm.Register(cma.Test) } - - cm.Do() + runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.MasterUpgrade, upgradeFunc) }) }) @@ -79,21 +80,17 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) - cm := chaosmonkey.New(func() { + testSuite := &junit.TestSuite{Name: "Node upgrade"} + nodeUpgradeTest := &junit.TestCase{Name: "node-upgrade", Classname: "upgrade_tests"} + + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, nodeUpgradeTest) target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage)) framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) - }) - for _, t := range upgradeTests { - cma := chaosMonkeyAdapter{ - test: t, - framework: testFrameworks[t.Name()], - upgradeType: upgrades.NodeUpgrade, - upgCtx: *upgCtx, - } - cm.Register(cma.Test) } - cm.Do() + runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.NodeUpgrade, upgradeFunc) }) }) @@ -102,23 +99,19 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) - cm := chaosmonkey.New(func() { + testSuite := &junit.TestSuite{Name: "Cluster upgrade"} + clusterUpgradeTest := &junit.TestCase{Name: "cluster-upgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, clusterUpgradeTest) + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, clusterUpgradeTest) target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(framework.MasterUpgrade(target)) framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage)) framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) - }) - for _, t := range upgradeTests { - cma := chaosMonkeyAdapter{ - test: t, - framework: testFrameworks[t.Name()], - upgradeType: upgrades.ClusterUpgrade, - upgCtx: *upgCtx, - } - cm.Register(cma.Test) } - cm.Do() + runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -138,24 +131,21 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() { upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) - cm := chaosmonkey.New(func() { + testSuite := &junit.TestSuite{Name: "Cluster downgrade"} + clusterDowngradeTest := &junit.TestCase{Name: "cluster-downgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, clusterDowngradeTest) + + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, clusterDowngradeTest) // Yes this really is a downgrade. And nodes must downgrade first. target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage)) framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) framework.ExpectNoError(framework.MasterUpgrade(target)) framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) - }) - for _, t := range upgradeTests { - cma := chaosMonkeyAdapter{ - test: t, - framework: testFrameworks[t.Name()], - upgradeType: upgrades.ClusterUpgrade, - upgCtx: *upgCtx, - } - cm.Register(cma.Test) } - cm.Do() + runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -165,55 +155,138 @@ var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() { // Create the frameworks here because we can only create them // in a "Describe". - testFrameworks := map[string]*framework.Framework{} - for _, t := range upgradeTests { - testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name()) - } - + testFrameworks := createUpgradeFrameworks() framework.KubeDescribe("etcd upgrade", func() { It("should maintain a functioning cluster", func() { upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "") framework.ExpectNoError(err) - cm := chaosmonkey.New(func() { - framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion)) - // TODO(mml): verify the etcd version - }) - for _, t := range upgradeTests { - cma := chaosMonkeyAdapter{ - test: t, - framework: testFrameworks[t.Name()], - upgradeType: upgrades.EtcdUpgrade, - upgCtx: *upgCtx, - } - cm.Register(cma.Test) - } + testSuite := &junit.TestSuite{Name: "Etcd upgrade"} + etcdTest := &junit.TestCase{Name: "etcd-upgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, etcdTest) - cm.Do() + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, etcdTest) + framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion)) + } + runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.EtcdUpgrade, upgradeFunc) }) }) }) type chaosMonkeyAdapter struct { test upgrades.Test + testReport *junit.TestCase framework *framework.Framework upgradeType upgrades.UpgradeType upgCtx upgrades.UpgradeContext } func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) { + start := time.Now() + var once sync.Once + ready := func() { + once.Do(func() { + sem.Ready() + }) + } + defer finalizeUpgradeTest(start, cma.testReport) + defer ready() if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) { By("skipping test " + cma.test.Name()) - sem.Ready() + cma.testReport.Skipped = "skipping test " + cma.test.Name() return } defer cma.test.Teardown(cma.framework) cma.test.Setup(cma.framework) - sem.Ready() + ready() cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType) } +func finalizeUpgradeTest(start time.Time, tc *junit.TestCase) { + tc.Time = time.Since(start).Seconds() + r := recover() + if r == nil { + return + } + + switch r := r.(type) { + case ginkgowrapper.FailurePanic: + tc.Failures = []*junit.Failure{ + { + Message: r.Message, + Type: "Failure", + Value: fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace), + }, + } + case ginkgowrapper.SkipPanic: + tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message) + default: + tc.Errors = []*junit.Error{ + { + Message: fmt.Sprintf("%v", r), + Type: "Panic", + Value: fmt.Sprintf("%v", r), + }, + } + } +} + +func createUpgradeFrameworks() map[string]*framework.Framework { + testFrameworks := map[string]*framework.Framework{} + for _, t := range upgradeTests { + testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name()) + } + return testFrameworks +} + +func runUpgradeSuite( + f *framework.Framework, + testFrameworks map[string]*framework.Framework, + testSuite *junit.TestSuite, + upgCtx *upgrades.UpgradeContext, + upgradeType upgrades.UpgradeType, + upgradeFunc func(), +) { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + + cm := chaosmonkey.New(upgradeFunc) + for _, t := range upgradeTests { + testCase := &junit.TestCase{ + Name: t.Name(), + Classname: "upgrade_tests", + } + testSuite.TestCases = append(testSuite.TestCases, testCase) + cma := chaosMonkeyAdapter{ + test: t, + testReport: testCase, + framework: testFrameworks[t.Name()], + upgradeType: upgradeType, + upgCtx: *upgCtx, + } + cm.Register(cma.Test) + } + + start := time.Now() + defer func() { + testSuite.Update() + testSuite.Time = time.Since(start).Seconds() + if framework.TestContext.ReportDir != "" { + fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%supgrades.xml", framework.TestContext.ReportPrefix)) + f, err := os.Create(fname) + if err != nil { + return + } + defer f.Close() + xml.NewEncoder(f).Encode(testSuite) + } + }() + cm.Do() +} + func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) { current, err := c.ServerVersion() if err != nil {