diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 64bafd78f8c..74c55158eaf 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -196,6 +196,7 @@ go_library( "//vendor:k8s.io/apimachinery/pkg/watch", "//vendor:k8s.io/apiserver/pkg/authentication/serviceaccount", "//vendor:k8s.io/apiserver/pkg/registry/generic/registry", + "//vendor:k8s.io/client-go/discovery", "//vendor:k8s.io/client-go/kubernetes", "//vendor:k8s.io/client-go/pkg/api/v1", "//vendor:k8s.io/client-go/pkg/apis/extensions/v1beta1", diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 136ba5ddfba..6f9d49a6388 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -17,6 +17,8 @@ limitations under the License. package e2e import ( + "k8s.io/client-go/discovery" + "k8s.io/kubernetes/pkg/util/version" "k8s.io/kubernetes/test/e2e/chaosmonkey" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/upgrades" @@ -50,18 +52,22 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { framework.KubeDescribe("master upgrade", func() { It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + cm := chaosmonkey.New(func() { - v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) - framework.ExpectNoError(err) - framework.ExpectNoError(framework.MasterUpgrade(v)) - framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v)) + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(framework.MasterUpgrade(target)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) }) for _, t := range upgradeTests { - cm.RegisterInterface(&chaosMonkeyAdapter{ + cma := chaosMonkeyAdapter{ test: t, framework: testFrameworks[t.Name()], upgradeType: upgrades.MasterUpgrade, - }) + upgCtx: *upgCtx, + } + cm.Register(cma.Test) } cm.Do() @@ -70,18 +76,22 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { framework.KubeDescribe("node upgrade", func() { It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + cm := chaosmonkey.New(func() { - v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) - framework.ExpectNoError(err) - framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) - framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v)) + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) }) for _, t := range upgradeTests { - cm.RegisterInterface(&chaosMonkeyAdapter{ + cma := chaosMonkeyAdapter{ test: t, framework: testFrameworks[t.Name()], upgradeType: upgrades.NodeUpgrade, - }) + upgCtx: *upgCtx, + } + cm.Register(cma.Test) } cm.Do() }) @@ -89,20 +99,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { framework.KubeDescribe("cluster upgrade", func() { It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + cm := chaosmonkey.New(func() { - v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) - framework.ExpectNoError(err) - framework.ExpectNoError(framework.MasterUpgrade(v)) - framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v)) - framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) - framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v)) + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(framework.MasterUpgrade(target)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) + framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) }) for _, t := range upgradeTests { - cm.RegisterInterface(&chaosMonkeyAdapter{ + cma := chaosMonkeyAdapter{ test: t, framework: testFrameworks[t.Name()], upgradeType: upgrades.ClusterUpgrade, - }) + upgCtx: *upgCtx, + } + cm.Register(cma.Test) } cm.Do() }) @@ -121,21 +135,25 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() { framework.KubeDescribe("cluster downgrade", func() { It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + cm := chaosmonkey.New(func() { // Yes this really is a downgrade. And nodes must downgrade first. - v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) - framework.ExpectNoError(err) - framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) - framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v)) - framework.ExpectNoError(framework.MasterUpgrade(v)) - framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v)) + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) + framework.ExpectNoError(framework.MasterUpgrade(target)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) }) for _, t := range upgradeTests { - cm.RegisterInterface(&chaosMonkeyAdapter{ + cma := chaosMonkeyAdapter{ test: t, framework: testFrameworks[t.Name()], upgradeType: upgrades.ClusterUpgrade, - }) + upgCtx: *upgCtx, + } + cm.Register(cma.Test) } cm.Do() }) @@ -143,6 +161,8 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() { }) var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() { + f := framework.NewDefaultFramework("etc-upgrade") + // Create the frameworks here because we can only create them // in a "Describe". testFrameworks := map[string]*framework.Framework{} @@ -152,16 +172,21 @@ var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() { framework.KubeDescribe("etcd upgrade", func() { It("should maintain a functioning cluster", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "") + framework.ExpectNoError(err) + cm := chaosmonkey.New(func() { framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion)) // TODO(mml): verify the etcd version }) for _, t := range upgradeTests { - cm.RegisterInterface(&chaosMonkeyAdapter{ + cma := chaosMonkeyAdapter{ test: t, framework: testFrameworks[t.Name()], upgradeType: upgrades.EtcdUpgrade, - }) + upgCtx: *upgCtx, + } + cm.Register(cma.Test) } cm.Do() @@ -173,16 +198,60 @@ type chaosMonkeyAdapter struct { test upgrades.Test framework *framework.Framework upgradeType upgrades.UpgradeType + upgCtx upgrades.UpgradeContext } -func (cma *chaosMonkeyAdapter) Setup() { +func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) { + if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) { + By("skipping test " + cma.test.Name()) + sem.Ready() + return + } + cma.test.Setup(cma.framework) + defer cma.test.Teardown(cma.framework) + sem.Ready() + cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType) } -func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) { - cma.test.Test(cma.framework, stopCh, cma.upgradeType) -} +func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) { + current, err := c.ServerVersion() + if err != nil { + return nil, err + } -func (cma *chaosMonkeyAdapter) Teardown() { - cma.test.Teardown(cma.framework) + curVer, err := version.ParseSemantic(current.String()) + if err != nil { + return nil, err + } + + upgCtx := &upgrades.UpgradeContext{ + Versions: []upgrades.VersionContext{ + { + Version: *curVer, + NodeImage: framework.TestContext.NodeOSDistro, + }, + }, + } + + if len(upgradeTarget) == 0 { + return upgCtx, nil + } + + next, err := framework.RealVersion(upgradeTarget) + if err != nil { + return nil, err + } + + nextVer, err := version.ParseSemantic(next) + if err != nil { + return nil, err + } + + upgCtx.Versions = append(upgCtx.Versions, upgrades.VersionContext{ + Version: *nextVer, + NodeImage: framework.TestContext.UpgradeImage, + }) + + return upgCtx, nil } diff --git a/test/e2e/common/apparmor.go b/test/e2e/common/apparmor.go index 914f0daf8cf..68906cf70b8 100644 --- a/test/e2e/common/apparmor.go +++ b/test/e2e/common/apparmor.go @@ -32,8 +32,11 @@ const ( appArmorDeniedPath = "/expect_permission_denied" ) +// AppArmorDistros are distros with AppArmor support +var AppArmorDistros = []string{"gci", "ubuntu"} + func SkipIfAppArmorNotSupported() { - framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu") + framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...) } func LoadAppArmorProfiles(f *framework.Framework) { diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index a7fa15e8375..89bec5160a8 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -33,6 +33,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/controller/deployment/util:go_default_library", "//pkg/kubelet/sysctl:go_default_library", + "//pkg/util/version:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//vendor:github.com/onsi/ginkgo", diff --git a/test/e2e/upgrades/apparmor.go b/test/e2e/upgrades/apparmor.go index 44cf56aba71..fe8fdc689eb 100644 --- a/test/e2e/upgrades/apparmor.go +++ b/test/e2e/upgrades/apparmor.go @@ -34,9 +34,22 @@ type AppArmorUpgradeTest struct { func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" } +func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool { + var supportedImages map[string]bool + for _, d := range common.AppArmorDistros { + supportedImages[d] = true + } + + for _, vCtx := range upgCtx.Versions { + if !supportedImages[vCtx.NodeImage] { + return true + } + } + return false +} + // Setup creates a secret and then verifies that a pod can consume it. func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) { - common.SkipIfAppArmorNotSupported() By("Loading AppArmor profiles to nodes") common.LoadAppArmorProfiles(f) diff --git a/test/e2e/upgrades/statefulset.go b/test/e2e/upgrades/statefulset.go index eea3a203d91..7fad63d5759 100644 --- a/test/e2e/upgrades/statefulset.go +++ b/test/e2e/upgrades/statefulset.go @@ -22,6 +22,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1" + "k8s.io/kubernetes/pkg/util/version" "k8s.io/kubernetes/test/e2e/framework" ) @@ -35,6 +36,17 @@ type StatefulSetUpgradeTest struct { func (StatefulSetUpgradeTest) Name() string { return "statefulset-upgrade" } +func (StatefulSetUpgradeTest) Skip(upgCtx UpgradeContext) bool { + minVersion := version.MustParseSemantic("1.5.0") + + for _, vCtx := range upgCtx.Versions { + if vCtx.Version.LessThan(minVersion) { + return true + } + } + return false +} + // Setup creates a StatefulSet and a HeadlessService. It verifies the basic SatefulSet properties func (t *StatefulSetUpgradeTest) Setup(f *framework.Framework) { ssName := "ss" diff --git a/test/e2e/upgrades/upgrade.go b/test/e2e/upgrades/upgrade.go index 2f1f88f6d4e..8efc26373e8 100644 --- a/test/e2e/upgrades/upgrade.go +++ b/test/e2e/upgrades/upgrade.go @@ -18,7 +18,10 @@ limitations under the License. // features before, during, and after different types of upgrades. package upgrades -import "k8s.io/kubernetes/test/e2e/framework" +import ( + "k8s.io/kubernetes/pkg/util/version" + "k8s.io/kubernetes/test/e2e/framework" +) // UpgradeType represents different types of upgrades. type UpgradeType int @@ -57,3 +60,24 @@ type Test interface { // aren't already cleaned up by the framework. Teardown(f *framework.Framework) } + +// Skippable is an interface that an upgrade test can implement to be +// able to indicate that it should be skipped. +type Skippable interface { + // Skip should return true if test should be skipped. upgCtx + // provides information about the upgrade that is going to + // occur. + Skip(upgCtx UpgradeContext) bool +} + +// UpgradeContext contains information about all the stages of the +// upgrade that is going to occur. +type UpgradeContext struct { + Versions []VersionContext +} + +// VersionContext represents a stage of the upgrade. +type VersionContext struct { + Version version.Version + NodeImage string +}