From 5744e44cadd73eb0f1604a6fdfb217a3cb7c5125 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Thu, 11 Mar 2021 14:23:37 +0100 Subject: [PATCH] Move GPU upgrade tests to node directory --- test/e2e/cloud/gcp/cluster_upgrade.go | 80 ++++--------------- .../cloud/gcp/{ => common}/upgrade_context.go | 2 +- .../gcp/{ => common}/upgrade_mechanics.go | 4 +- test/e2e/cloud/gcp/imports.go | 22 +++++ test/e2e/cloud/gcp/node/framework.go | 24 ++++++ test/e2e/cloud/gcp/node/gpu.go | 75 +++++++++++++++++ 6 files changed, 140 insertions(+), 67 deletions(-) rename test/e2e/cloud/gcp/{ => common}/upgrade_context.go (99%) rename test/e2e/cloud/gcp/{ => common}/upgrade_mechanics.go (99%) create mode 100644 test/e2e/cloud/gcp/imports.go create mode 100644 test/e2e/cloud/gcp/node/framework.go create mode 100644 test/e2e/cloud/gcp/node/gpu.go diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index ac5c24e70ff..b78bf8ada9c 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -19,6 +19,7 @@ package gcp import ( "fmt" + "k8s.io/kubernetes/test/e2e/cloud/gcp/common" "k8s.io/kubernetes/test/e2e/framework" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/upgrades" @@ -33,8 +34,6 @@ import ( "github.com/onsi/ginkgo" ) -const etcdImage = "3.4.9-1" - var upgradeTests = []upgrades.Test{ &network.ServiceUpgradeTest{}, &node.SecretUpgradeTest{}, @@ -50,10 +49,6 @@ var upgradeTests = []upgrades.Test{ &storage.VolumeModeDowngradeTest{}, } -var gpuUpgradeTests = []upgrades.Test{ - &node.NvidiaGPUUpgradeTest{}, -} - var statefulsetUpgradeTests = []upgrades.Test{ &apps.MySQLUpgradeTest{}, &apps.EtcdUpgradeTest{}, @@ -86,7 +81,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { // in a "Describe". ginkgo.Describe("master upgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Master upgrade"} @@ -96,21 +91,21 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { } testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest, nil) - upgradeFunc := ControlPlaneUpgradeFunc(f, upgCtx, masterUpgradeTest, nil) + upgradeFunc := common.ControlPlaneUpgradeFunc(f, upgCtx, masterUpgradeTest, nil) upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) ginkgo.Describe("cluster upgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Cluster upgrade"} clusterUpgradeTest := &junit.TestCase{Name: "[sig-cloud-provider-gcp] cluster-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, clusterUpgradeTest) - upgradeFunc := ClusterUpgradeFunc(f, upgCtx, clusterUpgradeTest, nil, nil) + upgradeFunc := common.ClusterUpgradeFunc(f, upgCtx, clusterUpgradeTest, nil, nil) upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -122,78 +117,33 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { ginkgo.Describe("cluster downgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Cluster downgrade"} clusterDowngradeTest := &junit.TestCase{Name: "[sig-cloud-provider-gcp] cluster-downgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, clusterDowngradeTest) - upgradeFunc := ClusterDowngradeFunc(f, upgCtx, clusterDowngradeTest, nil, nil) + upgradeFunc := common.ClusterDowngradeFunc(f, upgCtx, clusterDowngradeTest, nil, nil) upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) -// TODO(#98326): Split the test by SIGs, move to appropriate directories and use SIGDescribe. -var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { - f := framework.NewDefaultFramework("gpu-upgrade") - - ginkgo.Describe("master upgrade", func() { - ginkgo.It("should NOT disrupt gpu pod [Feature:GPUMasterUpgrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) - framework.ExpectNoError(err) - - testSuite := &junit.TestSuite{Name: "GPU master upgrade"} - gpuUpgradeTest := &junit.TestCase{Name: "[sig-node] gpu-master-upgrade", Classname: "upgrade_tests"} - testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) - - upgradeFunc := ControlPlaneUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil) - upgrades.RunUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) - }) - }) - ginkgo.Describe("cluster upgrade", func() { - ginkgo.It("should be able to run gpu pod after upgrade [Feature:GPUClusterUpgrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) - framework.ExpectNoError(err) - - testSuite := &junit.TestSuite{Name: "GPU cluster upgrade"} - gpuUpgradeTest := &junit.TestCase{Name: "[sig-node] gpu-cluster-upgrade", Classname: "upgrade_tests"} - testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) - - upgradeFunc := ClusterUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil, nil) - upgrades.RunUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) - }) - }) - ginkgo.Describe("cluster downgrade", func() { - ginkgo.It("should be able to run gpu pod after downgrade [Feature:GPUClusterDowngrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) - framework.ExpectNoError(err) - - testSuite := &junit.TestSuite{Name: "GPU cluster downgrade"} - gpuDowngradeTest := &junit.TestCase{Name: "[sig-node] gpu-cluster-downgrade", Classname: "upgrade_tests"} - testSuite.TestCases = append(testSuite.TestCases, gpuDowngradeTest) - - upgradeFunc := ClusterDowngradeFunc(f, upgCtx, gpuDowngradeTest, nil, nil) - upgrades.RunUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) - }) - }) -}) - // TODO(#98326): Move the test to sig-aps dir and use SIGDescribe. var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func() { f := framework.NewDefaultFramework("stateful-upgrade") ginkgo.Describe("stateful upgrade", func() { ginkgo.It("should maintain a functioning cluster", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Stateful upgrade"} statefulUpgradeTest := &junit.TestCase{Name: "[sig-apps] stateful-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, statefulUpgradeTest) - upgradeFunc := ClusterUpgradeFunc(f, upgCtx, statefulUpgradeTest, nil, nil) + upgradeFunc := common.ClusterUpgradeFunc(f, upgCtx, statefulUpgradeTest, nil, nil) upgrades.RunUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -209,7 +159,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati ginkgo.Describe("Upgrade kube-proxy from static pods to a DaemonSet", func() { ginkgo.It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetUpgrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "kube-proxy upgrade"} @@ -220,14 +170,14 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati testSuite.TestCases = append(testSuite.TestCases, kubeProxyUpgradeTest) extraEnvs := kubeProxyDaemonSetExtraEnvs(true) - upgradeFunc := ClusterUpgradeFunc(f, upgCtx, kubeProxyUpgradeTest, extraEnvs, extraEnvs) + upgradeFunc := common.ClusterUpgradeFunc(f, upgCtx, kubeProxyUpgradeTest, extraEnvs, extraEnvs) upgrades.RunUpgradeSuite(upgCtx, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) ginkgo.Describe("Downgrade kube-proxy from a DaemonSet to static pods", func() { ginkgo.It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetDowngrade]", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "kube-proxy downgrade"} @@ -238,7 +188,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati testSuite.TestCases = append(testSuite.TestCases, kubeProxyDowngradeTest) extraEnvs := kubeProxyDaemonSetExtraEnvs(false) - upgradeFunc := ClusterDowngradeFunc(f, upgCtx, kubeProxyDowngradeTest, extraEnvs, extraEnvs) + upgradeFunc := common.ClusterDowngradeFunc(f, upgCtx, kubeProxyDowngradeTest, extraEnvs, extraEnvs) upgrades.RunUpgradeSuite(upgCtx, kubeProxyDowngradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -250,7 +200,7 @@ var _ = ginkgo.Describe("[sig-auth] ServiceAccount admission controller migratio ginkgo.Describe("master upgrade", func() { ginkgo.It("should maintain a functioning cluster", func() { - upgCtx, err := GetUpgradeContext(f.ClientSet.Discovery()) + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "ServiceAccount admission controller migration"} @@ -261,7 +211,7 @@ var _ = ginkgo.Describe("[sig-auth] ServiceAccount admission controller migratio testSuite.TestCases = append(testSuite.TestCases, serviceaccountAdmissionControllerMigrationTest) extraEnvs := []string{"KUBE_FEATURE_GATES=BoundServiceAccountTokenVolume=true"} - upgradeFunc := ControlPlaneUpgradeFunc(f, upgCtx, serviceaccountAdmissionControllerMigrationTest, extraEnvs) + upgradeFunc := common.ControlPlaneUpgradeFunc(f, upgCtx, serviceaccountAdmissionControllerMigrationTest, extraEnvs) upgrades.RunUpgradeSuite(upgCtx, serviceaccountAdmissionControllerMigrationTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) diff --git a/test/e2e/cloud/gcp/upgrade_context.go b/test/e2e/cloud/gcp/common/upgrade_context.go similarity index 99% rename from test/e2e/cloud/gcp/upgrade_context.go rename to test/e2e/cloud/gcp/common/upgrade_context.go index ea86dc0d292..19d09c5c347 100644 --- a/test/e2e/cloud/gcp/upgrade_context.go +++ b/test/e2e/cloud/gcp/common/upgrade_context.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gcp +package common import ( "fmt" diff --git a/test/e2e/cloud/gcp/upgrade_mechanics.go b/test/e2e/cloud/gcp/common/upgrade_mechanics.go similarity index 99% rename from test/e2e/cloud/gcp/upgrade_mechanics.go rename to test/e2e/cloud/gcp/common/upgrade_mechanics.go index 2c1407d31e1..10b2d65f4f7 100644 --- a/test/e2e/cloud/gcp/upgrade_mechanics.go +++ b/test/e2e/cloud/gcp/common/upgrade_mechanics.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gcp +package common import ( "fmt" @@ -72,6 +72,8 @@ func ClusterDowngradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContex } } +const etcdImage = "3.4.9-1" + // controlPlaneUpgrade upgrades control plane node on GCE/GKE. func controlPlaneUpgrade(f *framework.Framework, v string, extraEnvs []string) error { switch framework.TestContext.Provider { diff --git a/test/e2e/cloud/gcp/imports.go b/test/e2e/cloud/gcp/imports.go new file mode 100644 index 00000000000..fb35d820dc0 --- /dev/null +++ b/test/e2e/cloud/gcp/imports.go @@ -0,0 +1,22 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcp + +import ( + // ensure these packages are scanned by ginkgo for e2e tests + _ "k8s.io/kubernetes/test/e2e/cloud/gcp/node" +) diff --git a/test/e2e/cloud/gcp/node/framework.go b/test/e2e/cloud/gcp/node/framework.go new file mode 100644 index 00000000000..f32393e98e1 --- /dev/null +++ b/test/e2e/cloud/gcp/node/framework.go @@ -0,0 +1,24 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import "github.com/onsi/ginkgo" + +// SIGDescribe annotates the test with the SIG label. +func SIGDescribe(text string, body func()) bool { + return ginkgo.Describe("[sig-node] "+text, body) +} diff --git a/test/e2e/cloud/gcp/node/gpu.go b/test/e2e/cloud/gcp/node/gpu.go new file mode 100644 index 00000000000..7afb88c7c30 --- /dev/null +++ b/test/e2e/cloud/gcp/node/gpu.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "k8s.io/kubernetes/test/e2e/cloud/gcp/common" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/upgrades" + "k8s.io/kubernetes/test/e2e/upgrades/node" + "k8s.io/kubernetes/test/utils/junit" + + "github.com/onsi/ginkgo" +) + +var upgradeTests = []upgrades.Test{ + &node.NvidiaGPUUpgradeTest{}, +} + +var _ = SIGDescribe("gpu Upgrade [Feature:GPUUpgrade]", func() { + f := framework.NewDefaultFramework("gpu-upgrade") + + ginkgo.Describe("master upgrade", func() { + ginkgo.It("should NOT disrupt gpu pod [Feature:GPUMasterUpgrade]", func() { + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) + framework.ExpectNoError(err) + + testSuite := &junit.TestSuite{Name: "GPU master upgrade"} + gpuUpgradeTest := &junit.TestCase{Name: "[sig-node] gpu-master-upgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) + + upgradeFunc := common.ControlPlaneUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + }) + }) + ginkgo.Describe("cluster upgrade", func() { + ginkgo.It("should be able to run gpu pod after upgrade [Feature:GPUClusterUpgrade]", func() { + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) + framework.ExpectNoError(err) + + testSuite := &junit.TestSuite{Name: "GPU cluster upgrade"} + gpuUpgradeTest := &junit.TestCase{Name: "[sig-node] gpu-cluster-upgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) + + upgradeFunc := common.ClusterUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil, nil) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + }) + }) + ginkgo.Describe("cluster downgrade", func() { + ginkgo.It("should be able to run gpu pod after downgrade [Feature:GPUClusterDowngrade]", func() { + upgCtx, err := common.GetUpgradeContext(f.ClientSet.Discovery()) + framework.ExpectNoError(err) + + testSuite := &junit.TestSuite{Name: "GPU cluster downgrade"} + gpuDowngradeTest := &junit.TestCase{Name: "[sig-node] gpu-cluster-downgrade", Classname: "upgrade_tests"} + testSuite.TestCases = append(testSuite.TestCases, gpuDowngradeTest) + + upgradeFunc := common.ClusterDowngradeFunc(f, upgCtx, gpuDowngradeTest, nil, nil) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + }) + }) +})