diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 7b3142f6bc3..28aa7819b72 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -14,7 +14,6 @@ go_library( "framework.go", "get-kubemark-resource-usage.go", "google_compute.go", - "gpu_util.go", "jobs_util.go", "kubelet_stats.go", "log_size_monitoring.go", @@ -156,6 +155,7 @@ filegroup( ":package-srcs", "//test/e2e/framework/config:all-srcs", "//test/e2e/framework/ginkgowrapper:all-srcs", + "//test/e2e/framework/gpu:all-srcs", "//test/e2e/framework/ingress:all-srcs", "//test/e2e/framework/metrics:all-srcs", "//test/e2e/framework/podlogs:all-srcs", diff --git a/test/e2e/framework/gpu/BUILD b/test/e2e/framework/gpu/BUILD new file mode 100644 index 00000000000..340b4bc2ed0 --- /dev/null +++ b/test/e2e/framework/gpu/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["gpu_util.go"], + importpath = "k8s.io/kubernetes/test/e2e/framework/gpu", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//test/e2e/framework:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/framework/gpu_util.go b/test/e2e/framework/gpu/gpu_util.go similarity index 92% rename from test/e2e/framework/gpu_util.go rename to test/e2e/framework/gpu/gpu_util.go index 8a5aac81d0c..415d4b1a93d 100644 --- a/test/e2e/framework/gpu_util.go +++ b/test/e2e/framework/gpu/gpu_util.go @@ -14,13 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework +package gpu import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" + "k8s.io/kubernetes/test/e2e/framework" ) const ( @@ -48,8 +49,8 @@ func NumberOfNVIDIAGPUs(node *v1.Node) int64 { // NVIDIADevicePlugin returns the official Google Device Plugin pod for NVIDIA GPU in GKE func NVIDIADevicePlugin() *v1.Pod { - ds, err := DsFromManifest(GPUDevicePluginDSYAML) - ExpectNoError(err) + ds, err := framework.DsFromManifest(GPUDevicePluginDSYAML) + framework.ExpectNoError(err) p := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "device-plugin-nvidia-gpu-" + string(uuid.NewUUID()), @@ -64,7 +65,7 @@ func NVIDIADevicePlugin() *v1.Pod { // GetGPUDevicePluginImage returns the image of GPU device plugin. func GetGPUDevicePluginImage() string { - ds, err := DsFromManifest(GPUDevicePluginDSYAML) + ds, err := framework.DsFromManifest(GPUDevicePluginDSYAML) if err != nil { klog.Errorf("Failed to parse the device plugin image: %v", err) return "" diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 0b635c1455c..0add13cd5cb 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -37,6 +37,7 @@ go_library( "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/config:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/instrumentation/common:go_default_library", "//test/e2e/scheduling:go_default_library", diff --git a/test/e2e/instrumentation/monitoring/accelerator.go b/test/e2e/instrumentation/monitoring/accelerator.go index 9654d3ef638..502fb2d5603 100644 --- a/test/e2e/instrumentation/monitoring/accelerator.go +++ b/test/e2e/instrumentation/monitoring/accelerator.go @@ -24,11 +24,12 @@ import ( "github.com/onsi/ginkgo" "golang.org/x/oauth2/google" gcm "google.golang.org/api/monitoring/v3" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" "k8s.io/kubernetes/test/e2e/scheduling" "k8s.io/kubernetes/test/utils/image" @@ -88,7 +89,7 @@ func testStackdriverAcceleratorMonitoring(f *framework.Framework) { Args: []string{"nvidia-smi && sleep infinity"}, Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - framework.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), + gpu.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), }, }, }, diff --git a/test/e2e/scheduling/BUILD b/test/e2e/scheduling/BUILD index 8f934fb2ff1..452733eb8e9 100644 --- a/test/e2e/scheduling/BUILD +++ b/test/e2e/scheduling/BUILD @@ -44,6 +44,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/providers/gce:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/e2e/scheduling/nvidia-gpus.go b/test/e2e/scheduling/nvidia-gpus.go index 8dc0a3873dd..37d50f76723 100644 --- a/test/e2e/scheduling/nvidia-gpus.go +++ b/test/e2e/scheduling/nvidia-gpus.go @@ -20,12 +20,13 @@ import ( "os" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" imageutils "k8s.io/kubernetes/test/utils/image" . "github.com/onsi/ginkgo" @@ -114,7 +115,7 @@ func SetupNVIDIAGPUNode(f *framework.Framework, setupResourceGatherer bool) *fra } else { dsYamlUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/daemonset.yaml" } - gpuResourceName = framework.NVIDIAGPUResourceName + gpuResourceName = gpu.NVIDIAGPUResourceName framework.Logf("Using %v", dsYamlUrl) // Creates the DaemonSet that installs Nvidia Drivers. diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index 4bd5c3f5d7e..a2465882edf 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -37,6 +37,7 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/scheduling:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/e2e/upgrades/nvidia-gpu.go b/test/e2e/upgrades/nvidia-gpu.go index 3b52989bad8..459de01bcf6 100644 --- a/test/e2e/upgrades/nvidia-gpu.go +++ b/test/e2e/upgrades/nvidia-gpu.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" "k8s.io/kubernetes/test/e2e/scheduling" imageutils "k8s.io/kubernetes/test/utils/image" @@ -78,7 +79,7 @@ func (t *NvidiaGPUUpgradeTest) startJob(f *framework.Framework) { Command: []string{"/bin/sh", "-c", "./vectorAdd && sleep 60"}, Resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - framework.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), + gpu.NVIDIAGPUResourceName: *resource.NewQuantity(1, resource.DecimalSI), }, }, }, diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index c8fe18d9763..d4c423a0bb4 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -47,6 +47,7 @@ go_library( "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/blang/semver:go_default_library", diff --git a/test/e2e_node/gpu_device_plugin.go b/test/e2e_node/gpu_device_plugin.go index 5ea0b089b22..30b9ee2c45a 100644 --- a/test/e2e_node/gpu_device_plugin.go +++ b/test/e2e_node/gpu_device_plugin.go @@ -21,10 +21,11 @@ import ( "strconv" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" "k8s.io/kubernetes/test/e2e/framework/metrics" . "github.com/onsi/ginkgo" @@ -46,15 +47,15 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi } By("Creating the Google Device Plugin pod for NVIDIA GPU in GKE") - devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(framework.NVIDIADevicePlugin()) + devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(gpu.NVIDIADevicePlugin()) framework.ExpectNoError(err) By("Waiting for GPUs to become available on the local node") Eventually(func() bool { - return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 + return gpu.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 }, 5*time.Minute, framework.Poll).Should(BeTrue()) - if framework.NumberOfNVIDIAGPUs(getLocalNode(f)) < 2 { + if gpu.NumberOfNVIDIAGPUs(getLocalNode(f)) < 2 { Skip("Not enough GPUs to execute this test (at least two needed)") } }) @@ -75,7 +76,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi It("checks that when Kubelet restarts exclusive GPU assignation to pods is kept.", func() { By("Creating one GPU pod on a node with at least two GPUs") podRECMD := "devs=$(ls /dev/ | egrep '^nvidia[0-9]+$') && echo gpu devices: $devs" - p1 := f.PodClient().CreateSync(makeBusyboxPod(framework.NVIDIAGPUResourceName, podRECMD)) + p1 := f.PodClient().CreateSync(makeBusyboxPod(gpu.NVIDIAGPUResourceName, podRECMD)) deviceIDRE := "gpu devices: (nvidia[0-9]+)" devId1 := parseLog(f, p1.Name, p1.Name, deviceIDRE) @@ -94,9 +95,9 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi restartKubelet() framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout) Eventually(func() bool { - return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 + return gpu.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0 }, 5*time.Minute, framework.Poll).Should(BeTrue()) - p2 := f.PodClient().CreateSync(makeBusyboxPod(framework.NVIDIAGPUResourceName, podRECMD)) + p2 := f.PodClient().CreateSync(makeBusyboxPod(gpu.NVIDIAGPUResourceName, podRECMD)) By("Checking that pods got a different GPU") devId2 := parseLog(f, p2.Name, p2.Name, deviceIDRE) @@ -109,7 +110,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi Eventually(func() bool { node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) - return framework.NumberOfNVIDIAGPUs(node) <= 0 + return gpu.NumberOfNVIDIAGPUs(node) <= 0 }, 10*time.Minute, framework.Poll).Should(BeTrue()) By("Checking that scheduled pods can continue to run even after we delete device plugin.") ensurePodContainerRestart(f, p1.Name, p1.Name) diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index f6c17261792..34c548029b1 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -30,6 +30,7 @@ import ( runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/gpu" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -52,7 +53,7 @@ var NodeImageWhiteList = sets.NewString( imageutils.GetE2EImage(imageutils.Netexec), imageutils.GetE2EImage(imageutils.Nonewprivs), imageutils.GetPauseImageName(), - framework.GetGPUDevicePluginImage(), + gpu.GetGPUDevicePluginImage(), "gcr.io/kubernetes-e2e-test-images/node-perf/npb-is:1.0", "gcr.io/kubernetes-e2e-test-images/node-perf/npb-ep:1.0", "gcr.io/kubernetes-e2e-test-images/node-perf/tf-wide-deep-amd64:1.0",