mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-05 18:24:07 +00:00
Generate individual test results for upgrade tests
This commit is contained in:
Kris
parent
6da0d384cf
commit
82c58f2658
@ -172,6 +172,7 @@ go_library(
|
||||
"//test/e2e_federation:go_default_library",
|
||||
"//test/images/net/nat:go_default_library",
|
||||
"//test/utils:go_default_library",
|
||||
"//test/utils/junit:go_default_library",
|
||||
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
|
||||
"//vendor/github.com/elazarl/goproxy:go_default_library",
|
||||
"//vendor/github.com/ghodss/yaml:go_default_library",
|
||||
|
||||
@ -17,11 +17,20 @@ limitations under the License.
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"k8s.io/client-go/discovery"
|
||||
"k8s.io/kubernetes/pkg/util/version"
|
||||
"k8s.io/kubernetes/test/e2e/chaosmonkey"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
|
||||
"k8s.io/kubernetes/test/e2e/upgrades"
|
||||
"k8s.io/kubernetes/test/utils/junit"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
)
|
||||
@ -45,32 +54,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
|
||||
|
||||
// Create the frameworks here because we can only create them
|
||||
// in a "Describe".
|
||||
testFrameworks := map[string]*framework.Framework{}
|
||||
for _, t := range upgradeTests {
|
||||
testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
|
||||
}
|
||||
|
||||
testFrameworks := createUpgradeFrameworks()
|
||||
framework.KubeDescribe("master upgrade", func() {
|
||||
It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
cm := chaosmonkey.New(func() {
|
||||
testSuite := &junit.TestSuite{Name: "Master upgrade"}
|
||||
masterUpgradeTest := &junit.TestCase{Name: "master-upgrade", Classname: "upgrade_tests"}
|
||||
testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest)
|
||||
|
||||
upgradeFunc := func() {
|
||||
start := time.Now()
|
||||
defer finalizeUpgradeTest(start, masterUpgradeTest)
|
||||
target := upgCtx.Versions[1].Version.String()
|
||||
framework.ExpectNoError(framework.MasterUpgrade(target))
|
||||
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
|
||||
})
|
||||
for _, t := range upgradeTests {
|
||||
cma := chaosMonkeyAdapter{
|
||||
test: t,
|
||||
framework: testFrameworks[t.Name()],
|
||||
upgradeType: upgrades.MasterUpgrade,
|
||||
upgCtx: *upgCtx,
|
||||
}
|
||||
cm.Register(cma.Test)
|
||||
}
|
||||
|
||||
cm.Do()
|
||||
runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.MasterUpgrade, upgradeFunc)
|
||||
})
|
||||
})
|
||||
|
||||
@ -79,21 +80,17 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
cm := chaosmonkey.New(func() {
|
||||
testSuite := &junit.TestSuite{Name: "Node upgrade"}
|
||||
nodeUpgradeTest := &junit.TestCase{Name: "node-upgrade", Classname: "upgrade_tests"}
|
||||
|
||||
upgradeFunc := func() {
|
||||
start := time.Now()
|
||||
defer finalizeUpgradeTest(start, nodeUpgradeTest)
|
||||
target := upgCtx.Versions[1].Version.String()
|
||||
framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
|
||||
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
|
||||
})
|
||||
for _, t := range upgradeTests {
|
||||
cma := chaosMonkeyAdapter{
|
||||
test: t,
|
||||
framework: testFrameworks[t.Name()],
|
||||
upgradeType: upgrades.NodeUpgrade,
|
||||
upgCtx: *upgCtx,
|
||||
}
|
||||
cm.Register(cma.Test)
|
||||
}
|
||||
cm.Do()
|
||||
runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.NodeUpgrade, upgradeFunc)
|
||||
})
|
||||
})
|
||||
|
||||
@ -102,23 +99,19 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
cm := chaosmonkey.New(func() {
|
||||
testSuite := &junit.TestSuite{Name: "Cluster upgrade"}
|
||||
clusterUpgradeTest := &junit.TestCase{Name: "cluster-upgrade", Classname: "upgrade_tests"}
|
||||
testSuite.TestCases = append(testSuite.TestCases, clusterUpgradeTest)
|
||||
upgradeFunc := func() {
|
||||
start := time.Now()
|
||||
defer finalizeUpgradeTest(start, clusterUpgradeTest)
|
||||
target := upgCtx.Versions[1].Version.String()
|
||||
framework.ExpectNoError(framework.MasterUpgrade(target))
|
||||
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
|
||||
framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
|
||||
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
|
||||
})
|
||||
for _, t := range upgradeTests {
|
||||
cma := chaosMonkeyAdapter{
|
||||
test: t,
|
||||
framework: testFrameworks[t.Name()],
|
||||
upgradeType: upgrades.ClusterUpgrade,
|
||||
upgCtx: *upgCtx,
|
||||
}
|
||||
cm.Register(cma.Test)
|
||||
}
|
||||
cm.Do()
|
||||
runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc)
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -138,24 +131,21 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
cm := chaosmonkey.New(func() {
|
||||
testSuite := &junit.TestSuite{Name: "Cluster downgrade"}
|
||||
clusterDowngradeTest := &junit.TestCase{Name: "cluster-downgrade", Classname: "upgrade_tests"}
|
||||
testSuite.TestCases = append(testSuite.TestCases, clusterDowngradeTest)
|
||||
|
||||
upgradeFunc := func() {
|
||||
start := time.Now()
|
||||
defer finalizeUpgradeTest(start, clusterDowngradeTest)
|
||||
// Yes this really is a downgrade. And nodes must downgrade first.
|
||||
target := upgCtx.Versions[1].Version.String()
|
||||
framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
|
||||
framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
|
||||
framework.ExpectNoError(framework.MasterUpgrade(target))
|
||||
framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
|
||||
})
|
||||
for _, t := range upgradeTests {
|
||||
cma := chaosMonkeyAdapter{
|
||||
test: t,
|
||||
framework: testFrameworks[t.Name()],
|
||||
upgradeType: upgrades.ClusterUpgrade,
|
||||
upgCtx: *upgCtx,
|
||||
}
|
||||
cm.Register(cma.Test)
|
||||
}
|
||||
cm.Do()
|
||||
runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc)
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -165,55 +155,138 @@ var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
|
||||
|
||||
// Create the frameworks here because we can only create them
|
||||
// in a "Describe".
|
||||
testFrameworks := map[string]*framework.Framework{}
|
||||
for _, t := range upgradeTests {
|
||||
testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
|
||||
}
|
||||
|
||||
testFrameworks := createUpgradeFrameworks()
|
||||
framework.KubeDescribe("etcd upgrade", func() {
|
||||
It("should maintain a functioning cluster", func() {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
cm := chaosmonkey.New(func() {
|
||||
framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion))
|
||||
// TODO(mml): verify the etcd version
|
||||
})
|
||||
for _, t := range upgradeTests {
|
||||
cma := chaosMonkeyAdapter{
|
||||
test: t,
|
||||
framework: testFrameworks[t.Name()],
|
||||
upgradeType: upgrades.EtcdUpgrade,
|
||||
upgCtx: *upgCtx,
|
||||
}
|
||||
cm.Register(cma.Test)
|
||||
}
|
||||
testSuite := &junit.TestSuite{Name: "Etcd upgrade"}
|
||||
etcdTest := &junit.TestCase{Name: "etcd-upgrade", Classname: "upgrade_tests"}
|
||||
testSuite.TestCases = append(testSuite.TestCases, etcdTest)
|
||||
|
||||
cm.Do()
|
||||
upgradeFunc := func() {
|
||||
start := time.Now()
|
||||
defer finalizeUpgradeTest(start, etcdTest)
|
||||
framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion))
|
||||
}
|
||||
runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.EtcdUpgrade, upgradeFunc)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
type chaosMonkeyAdapter struct {
|
||||
test upgrades.Test
|
||||
testReport *junit.TestCase
|
||||
framework *framework.Framework
|
||||
upgradeType upgrades.UpgradeType
|
||||
upgCtx upgrades.UpgradeContext
|
||||
}
|
||||
|
||||
func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) {
|
||||
start := time.Now()
|
||||
var once sync.Once
|
||||
ready := func() {
|
||||
once.Do(func() {
|
||||
sem.Ready()
|
||||
})
|
||||
}
|
||||
defer finalizeUpgradeTest(start, cma.testReport)
|
||||
defer ready()
|
||||
if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) {
|
||||
By("skipping test " + cma.test.Name())
|
||||
sem.Ready()
|
||||
cma.testReport.Skipped = "skipping test " + cma.test.Name()
|
||||
return
|
||||
}
|
||||
|
||||
defer cma.test.Teardown(cma.framework)
|
||||
cma.test.Setup(cma.framework)
|
||||
sem.Ready()
|
||||
ready()
|
||||
cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType)
|
||||
}
|
||||
|
||||
func finalizeUpgradeTest(start time.Time, tc *junit.TestCase) {
|
||||
tc.Time = time.Since(start).Seconds()
|
||||
r := recover()
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch r := r.(type) {
|
||||
case ginkgowrapper.FailurePanic:
|
||||
tc.Failures = []*junit.Failure{
|
||||
{
|
||||
Message: r.Message,
|
||||
Type: "Failure",
|
||||
Value: fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace),
|
||||
},
|
||||
}
|
||||
case ginkgowrapper.SkipPanic:
|
||||
tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message)
|
||||
default:
|
||||
tc.Errors = []*junit.Error{
|
||||
{
|
||||
Message: fmt.Sprintf("%v", r),
|
||||
Type: "Panic",
|
||||
Value: fmt.Sprintf("%v", r),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createUpgradeFrameworks() map[string]*framework.Framework {
|
||||
testFrameworks := map[string]*framework.Framework{}
|
||||
for _, t := range upgradeTests {
|
||||
testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
|
||||
}
|
||||
return testFrameworks
|
||||
}
|
||||
|
||||
func runUpgradeSuite(
|
||||
f *framework.Framework,
|
||||
testFrameworks map[string]*framework.Framework,
|
||||
testSuite *junit.TestSuite,
|
||||
upgCtx *upgrades.UpgradeContext,
|
||||
upgradeType upgrades.UpgradeType,
|
||||
upgradeFunc func(),
|
||||
) {
|
||||
upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
cm := chaosmonkey.New(upgradeFunc)
|
||||
for _, t := range upgradeTests {
|
||||
testCase := &junit.TestCase{
|
||||
Name: t.Name(),
|
||||
Classname: "upgrade_tests",
|
||||
}
|
||||
testSuite.TestCases = append(testSuite.TestCases, testCase)
|
||||
cma := chaosMonkeyAdapter{
|
||||
test: t,
|
||||
testReport: testCase,
|
||||
framework: testFrameworks[t.Name()],
|
||||
upgradeType: upgradeType,
|
||||
upgCtx: *upgCtx,
|
||||
}
|
||||
cm.Register(cma.Test)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
testSuite.Update()
|
||||
testSuite.Time = time.Since(start).Seconds()
|
||||
if framework.TestContext.ReportDir != "" {
|
||||
fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%supgrades.xml", framework.TestContext.ReportPrefix))
|
||||
f, err := os.Create(fname)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
xml.NewEncoder(f).Encode(testSuite)
|
||||
}
|
||||
}()
|
||||
cm.Do()
|
||||
}
|
||||
|
||||
func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) {
|
||||
current, err := c.ServerVersion()
|
||||
if err != nil {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user