|
|
|
|
@@ -80,10 +80,6 @@ func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
|
// TODO(vikasc): Instead of hard-coding number of devices, provide number of devices in the sample-device-plugin using configmap
|
|
|
|
|
// and then use the same here
|
|
|
|
|
expectedSampleDevsAmount int64 = 2
|
|
|
|
|
|
|
|
|
|
// This is the sleep interval specified in the command executed in the pod to ensure container is running "forever" in the test timescale
|
|
|
|
|
sleepIntervalForever string = "24h"
|
|
|
|
|
|
|
|
|
|
@@ -181,16 +177,16 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
ginkgo.By("Waiting for devices to become available on the local node")
|
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready && CountSampleDeviceCapacity(node) > 0
|
|
|
|
|
return ready && e2enode.CountSampleDeviceCapacity(node) > 0
|
|
|
|
|
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrueBecause("expected devices to be available on local node"))
|
|
|
|
|
framework.Logf("Successfully created device plugin pod")
|
|
|
|
|
|
|
|
|
|
ginkgo.By(fmt.Sprintf("Waiting for the resource exported by the sample device plugin to become available on the local node (instances: %d)", expectedSampleDevsAmount))
|
|
|
|
|
ginkgo.By(fmt.Sprintf("Waiting for the resource exported by the sample device plugin to become available on the local node (instances: %d)", e2enode.SampleDevsAmount))
|
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready &&
|
|
|
|
|
CountSampleDeviceCapacity(node) == expectedSampleDevsAmount &&
|
|
|
|
|
CountSampleDeviceAllocatable(node) == expectedSampleDevsAmount
|
|
|
|
|
e2enode.CountSampleDeviceCapacity(node) == e2enode.SampleDevsAmount &&
|
|
|
|
|
e2enode.CountSampleDeviceAllocatable(node) == e2enode.SampleDevsAmount
|
|
|
|
|
}, 30*time.Second, framework.Poll).Should(gomega.BeTrueBecause("expected resource to be available on local node"))
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
@@ -215,7 +211,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
ginkgo.By("Waiting for devices to become unavailable on the local node")
|
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready && CountSampleDeviceCapacity(node) <= 0
|
|
|
|
|
return ready && e2enode.CountSampleDeviceCapacity(node) <= 0
|
|
|
|
|
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrueBecause("expected devices to be unavailable on local node"))
|
|
|
|
|
|
|
|
|
|
ginkgo.By("devices now unavailable on the local node")
|
|
|
|
|
@@ -223,7 +219,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
|
|
|
|
|
ginkgo.It("Can schedule a pod that requires a device", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalWithRestart)
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID1, err := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
|
|
|
|
|
framework.ExpectNoError(err, "getting logs for pod %q", pod1.Name)
|
|
|
|
|
@@ -275,8 +271,8 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
gomega.Expect(v1alphaResourcesForOurPod.Containers[0].Devices).To(gomega.HaveLen(1))
|
|
|
|
|
gomega.Expect(v1ResourcesForOurPod.Containers[0].Devices).To(gomega.HaveLen(1))
|
|
|
|
|
|
|
|
|
|
gomega.Expect(v1alphaResourcesForOurPod.Containers[0].Devices[0].ResourceName).To(gomega.Equal(SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(v1ResourcesForOurPod.Containers[0].Devices[0].ResourceName).To(gomega.Equal(SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(v1alphaResourcesForOurPod.Containers[0].Devices[0].ResourceName).To(gomega.Equal(e2enode.SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(v1ResourcesForOurPod.Containers[0].Devices[0].ResourceName).To(gomega.Equal(e2enode.SampleDeviceResourceName))
|
|
|
|
|
|
|
|
|
|
gomega.Expect(v1alphaResourcesForOurPod.Containers[0].Devices[0].DeviceIds).To(gomega.HaveLen(1))
|
|
|
|
|
gomega.Expect(v1ResourcesForOurPod.Containers[0].Devices[0].DeviceIds).To(gomega.HaveLen(1))
|
|
|
|
|
@@ -285,7 +281,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
f.It("can make a CDI device accessible in a container", func(ctx context.Context) {
|
|
|
|
|
// check if CDI_DEVICE env variable is set
|
|
|
|
|
// and only one correspondent device node /tmp/<CDI_DEVICE> is available inside a container
|
|
|
|
|
podObj := makeBusyboxPod(SampleDeviceResourceName, "[ $(ls /tmp/CDI-Dev-[1,2] | wc -l) -eq 1 -a -b /tmp/$CDI_DEVICE ]")
|
|
|
|
|
podObj := makeBusyboxPod(e2enode.SampleDeviceResourceName, "[ $(ls /tmp/CDI-Dev-[1,2] | wc -l) -eq 1 -a -b /tmp/$CDI_DEVICE ]")
|
|
|
|
|
podObj.Spec.RestartPolicy = v1.RestartPolicyNever
|
|
|
|
|
pod := e2epod.NewPodClient(f).Create(ctx, podObj)
|
|
|
|
|
framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
|
|
|
|
|
@@ -296,7 +292,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
// restarts. For the sake of brevity we however check just the fist restart.
|
|
|
|
|
ginkgo.It("Keeps device plugin assignments across pod restarts (no kubelet restart, no device plugin restart)", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalWithRestart)
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID1, err := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
|
|
|
|
|
framework.ExpectNoError(err, "getting logs for pod %q", pod1.Name)
|
|
|
|
|
@@ -321,11 +317,11 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
if err != nil {
|
|
|
|
|
framework.ExpectNoError(err, "getting pod resources assignment after pod restart")
|
|
|
|
|
}
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after pod restart")
|
|
|
|
|
|
|
|
|
|
ginkgo.By("Creating another pod")
|
|
|
|
|
pod2 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod2 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
err = e2epod.WaitTimeoutForPodRunningInNamespace(ctx, f.ClientSet, pod2.Name, f.Namespace.Name, 1*time.Minute)
|
|
|
|
|
framework.ExpectNoError(err)
|
|
|
|
|
|
|
|
|
|
@@ -340,9 +336,9 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
if err != nil {
|
|
|
|
|
framework.ExpectNoError(err, "getting pod resources assignment after pod restart")
|
|
|
|
|
}
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after extra container restart - pod1")
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod2.Namespace, pod2.Name, pod2.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID2})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod2.Namespace, pod2.Name, pod2.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID2})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after extra container restart - pod2")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
@@ -352,7 +348,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
// and both the device plugin and the actual consumer (container) are stable.
|
|
|
|
|
ginkgo.It("Keeps device plugin assignments across kubelet restarts (no pod restart, no device plugin restart)", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalForever)
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID1, err := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
|
|
|
|
|
framework.ExpectNoError(err, "getting logs for pod %q", pod1.Name)
|
|
|
|
|
@@ -372,8 +368,8 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
gomega.Eventually(ctx, func() bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready &&
|
|
|
|
|
CountSampleDeviceCapacity(node) == expectedSampleDevsAmount &&
|
|
|
|
|
CountSampleDeviceAllocatable(node) == expectedSampleDevsAmount
|
|
|
|
|
e2enode.CountSampleDeviceCapacity(node) == e2enode.SampleDevsAmount &&
|
|
|
|
|
e2enode.CountSampleDeviceAllocatable(node) == e2enode.SampleDevsAmount
|
|
|
|
|
}, 30*time.Second, framework.Poll).Should(gomega.BeTrueBecause("expected resource to be available after restart"))
|
|
|
|
|
|
|
|
|
|
ginkgo.By("Checking the same instance of the pod is still running")
|
|
|
|
|
@@ -397,7 +393,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
return err
|
|
|
|
|
}, 30*time.Second, framework.Poll).ShouldNot(gomega.HaveOccurred(), "cannot fetch the compute resource assignment after kubelet restart")
|
|
|
|
|
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod2.Namespace, pod2.Name, pod2.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod2.Namespace, pod2.Name, pod2.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after pod restart")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
@@ -406,7 +402,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
// performs the device allocation, and both the device plugin is stable.
|
|
|
|
|
ginkgo.It("Keeps device plugin assignments across pod and kubelet restarts (no device plugin restart)", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalWithRestart)
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID1, err := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
|
|
|
|
|
framework.ExpectNoError(err, "getting logs for pod %q", pod1.Name)
|
|
|
|
|
@@ -462,13 +458,13 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
return err
|
|
|
|
|
}, 30*time.Second, framework.Poll).ShouldNot(gomega.HaveOccurred(), "cannot fetch the compute resource assignment after kubelet restart")
|
|
|
|
|
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after pod restart")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
ginkgo.It("will not attempt to admit the succeeded pod after the kubelet restart and device plugin removed", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalToCompletion)
|
|
|
|
|
podSpec := makeBusyboxPod(SampleDeviceResourceName, podRECMD)
|
|
|
|
|
podSpec := makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD)
|
|
|
|
|
podSpec.Spec.RestartPolicy = v1.RestartPolicyNever
|
|
|
|
|
// Making sure the pod will not be garbage collected and will stay thru the kubelet restart after
|
|
|
|
|
// it reached the terminated state. Using finalizers makes the test more reliable.
|
|
|
|
|
@@ -494,7 +490,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
e2epod.NewPodClient(f).DeleteSync(ctx, devicePluginPod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete)
|
|
|
|
|
waitForContainerRemoval(ctx, devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace)
|
|
|
|
|
|
|
|
|
|
gomega.Eventually(getNodeResourceValues, devicePluginGracefulTimeout, f.Timeouts.Poll).WithContext(ctx).WithArguments(SampleDeviceResourceName).Should(gomega.Equal(ResourceValue{Allocatable: 0, Capacity: int(expectedSampleDevsAmount)}))
|
|
|
|
|
gomega.Eventually(getNodeResourceValues, devicePluginGracefulTimeout, f.Timeouts.Poll).WithContext(ctx).WithArguments(e2enode.SampleDeviceResourceName).Should(gomega.Equal(ResourceValue{Allocatable: 0, Capacity: int(e2enode.SampleDevsAmount)}))
|
|
|
|
|
|
|
|
|
|
ginkgo.By("Restarting Kubelet")
|
|
|
|
|
restartKubelet(ctx, true)
|
|
|
|
|
@@ -521,7 +517,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
// test should pass.
|
|
|
|
|
ginkgo.It("Keeps device plugin assignments after the device plugin has restarted (no kubelet restart, pod restart)", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalForever)
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID1, err := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
|
|
|
|
|
framework.ExpectNoError(err, "getting logs for pod %q", pod1.Name)
|
|
|
|
|
@@ -550,8 +546,8 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
gomega.Eventually(ctx, func() bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready &&
|
|
|
|
|
CountSampleDeviceCapacity(node) == expectedSampleDevsAmount &&
|
|
|
|
|
CountSampleDeviceAllocatable(node) == expectedSampleDevsAmount
|
|
|
|
|
e2enode.CountSampleDeviceCapacity(node) == e2enode.SampleDevsAmount &&
|
|
|
|
|
e2enode.CountSampleDeviceAllocatable(node) == e2enode.SampleDevsAmount
|
|
|
|
|
}, 30*time.Second, framework.Poll).Should(gomega.BeTrueBecause("expected resource to be available after re-registration"))
|
|
|
|
|
|
|
|
|
|
// crosscheck that after device plugin restart the device assignment is preserved and
|
|
|
|
|
@@ -565,7 +561,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
return err
|
|
|
|
|
}, 30*time.Second, framework.Poll).ShouldNot(gomega.HaveOccurred(), "cannot fetch the compute resource assignment after kubelet restart")
|
|
|
|
|
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after pod restart")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
@@ -574,7 +570,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
// orchestrate the device allocation: the actual consumer (container) is stable.
|
|
|
|
|
ginkgo.It("Keeps device plugin assignments after kubelet restart and device plugin restart (no pod restart)", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalForever) // the pod has to run "forever" in the timescale of this test
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID1, err := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
|
|
|
|
|
framework.ExpectNoError(err, "getting logs for pod %q", pod1.Name)
|
|
|
|
|
@@ -607,7 +603,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
return err
|
|
|
|
|
}, 30*time.Second, framework.Poll).ShouldNot(gomega.HaveOccurred(), "cannot fetch the compute resource assignment after kubelet restart")
|
|
|
|
|
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
err, _ = checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{devID1})
|
|
|
|
|
framework.ExpectNoError(err, "inconsistent device assignment after pod restart")
|
|
|
|
|
|
|
|
|
|
ginkgo.By("Re-Register resources by deleting the plugin pod")
|
|
|
|
|
@@ -625,8 +621,8 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
gomega.Eventually(ctx, func() bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready &&
|
|
|
|
|
CountSampleDeviceCapacity(node) == expectedSampleDevsAmount &&
|
|
|
|
|
CountSampleDeviceAllocatable(node) == expectedSampleDevsAmount
|
|
|
|
|
e2enode.CountSampleDeviceCapacity(node) == e2enode.SampleDevsAmount &&
|
|
|
|
|
e2enode.CountSampleDeviceAllocatable(node) == e2enode.SampleDevsAmount
|
|
|
|
|
}, 30*time.Second, framework.Poll).Should(gomega.BeTrueBecause("expected resource to be available after restart"))
|
|
|
|
|
|
|
|
|
|
ginkgo.By("Checking the same instance of the pod is still running after the device plugin restart")
|
|
|
|
|
@@ -639,7 +635,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
|
|
|
|
|
ginkgo.It("[OrphanedPods] Ensures pods consuming devices deleted while kubelet is down are cleaned up correctly", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalWithRestart)
|
|
|
|
|
pod := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
|
|
|
|
|
pod := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD))
|
|
|
|
|
|
|
|
|
|
deviceIDRE := "stub devices: (Dev-[0-9]+)"
|
|
|
|
|
devID, err := parseLog(ctx, f, pod.Name, pod.Name, deviceIDRE)
|
|
|
|
|
@@ -687,7 +683,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
v1PodResources, err = getV1NodeDevices(ctx)
|
|
|
|
|
return err
|
|
|
|
|
}, 30*time.Second, framework.Poll).ShouldNot(gomega.HaveOccurred(), "cannot fetch the compute resource assignment after kubelet restart")
|
|
|
|
|
err, allocated := checkPodResourcesAssignment(v1PodResources, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, SampleDeviceResourceName, []string{})
|
|
|
|
|
err, allocated := checkPodResourcesAssignment(v1PodResources, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{})
|
|
|
|
|
if err == nil || allocated {
|
|
|
|
|
framework.Fail(fmt.Sprintf("stale device assignment after pod deletion while kubelet was down allocated=%v error=%v", allocated, err))
|
|
|
|
|
}
|
|
|
|
|
@@ -696,7 +692,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
f.It("Can schedule a pod with a restartable init container", framework.WithNodeConformance(), func(ctx context.Context) {
|
|
|
|
|
podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s"
|
|
|
|
|
sleepOneSecond := "1s"
|
|
|
|
|
rl := v1.ResourceList{v1.ResourceName(SampleDeviceResourceName): *resource.NewQuantity(1, resource.DecimalSI)}
|
|
|
|
|
rl := v1.ResourceList{v1.ResourceName(e2enode.SampleDeviceResourceName): *resource.NewQuantity(1, resource.DecimalSI)}
|
|
|
|
|
pod := &v1.Pod{
|
|
|
|
|
ObjectMeta: metav1.ObjectMeta{Name: "device-plugin-test-" + string(uuid.NewUUID())},
|
|
|
|
|
Spec: v1.PodSpec{
|
|
|
|
|
@@ -778,11 +774,11 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
for _, container := range resourcesForOurPod.Containers {
|
|
|
|
|
if container.Name == pod1.Spec.InitContainers[1].Name {
|
|
|
|
|
gomega.Expect(container.Devices).To(gomega.HaveLen(1))
|
|
|
|
|
gomega.Expect(container.Devices[0].ResourceName).To(gomega.Equal(SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(container.Devices[0].ResourceName).To(gomega.Equal(e2enode.SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(container.Devices[0].DeviceIds).To(gomega.HaveLen(1))
|
|
|
|
|
} else if container.Name == pod1.Spec.Containers[0].Name {
|
|
|
|
|
gomega.Expect(container.Devices).To(gomega.HaveLen(1))
|
|
|
|
|
gomega.Expect(container.Devices[0].ResourceName).To(gomega.Equal(SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(container.Devices[0].ResourceName).To(gomega.Equal(e2enode.SampleDeviceResourceName))
|
|
|
|
|
gomega.Expect(container.Devices[0].DeviceIds).To(gomega.HaveLen(1))
|
|
|
|
|
} else {
|
|
|
|
|
framework.Failf("unexpected container name: %s", container.Name)
|
|
|
|
|
@@ -854,15 +850,15 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ginkgo.By("Scheduling a sample device plugin pod")
|
|
|
|
|
data, err := e2etestfiles.Read(SampleDevicePluginControlRegistrationDSYAML)
|
|
|
|
|
data, err := e2etestfiles.Read(e2enode.SampleDevicePluginControlRegistrationDSYAML)
|
|
|
|
|
if err != nil {
|
|
|
|
|
framework.Fail(fmt.Sprintf("error reading test data %q: %v", SampleDevicePluginControlRegistrationDSYAML, err))
|
|
|
|
|
framework.Fail(fmt.Sprintf("error reading test data %q: %v", e2enode.SampleDevicePluginControlRegistrationDSYAML, err))
|
|
|
|
|
}
|
|
|
|
|
ds := readDaemonSetV1OrDie(data)
|
|
|
|
|
|
|
|
|
|
dp := &v1.Pod{
|
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
|
Name: SampleDevicePluginName,
|
|
|
|
|
Name: e2enode.SampleDevicePluginName,
|
|
|
|
|
},
|
|
|
|
|
Spec: ds.Spec.Template.Spec,
|
|
|
|
|
}
|
|
|
|
|
@@ -892,16 +888,16 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
ginkgo.By("Waiting for devices to become available on the local node")
|
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready && CountSampleDeviceCapacity(node) > 0
|
|
|
|
|
return ready && e2enode.CountSampleDeviceCapacity(node) > 0
|
|
|
|
|
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrueBecause("expected devices to be available on the local node"))
|
|
|
|
|
framework.Logf("Successfully created device plugin pod")
|
|
|
|
|
|
|
|
|
|
ginkgo.By(fmt.Sprintf("Waiting for the resource exported by the sample device plugin to become available on the local node (instances: %d)", expectedSampleDevsAmount))
|
|
|
|
|
ginkgo.By(fmt.Sprintf("Waiting for the resource exported by the sample device plugin to become available on the local node (instances: %d)", e2enode.SampleDevsAmount))
|
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready &&
|
|
|
|
|
CountSampleDeviceCapacity(node) == expectedSampleDevsAmount &&
|
|
|
|
|
CountSampleDeviceAllocatable(node) == expectedSampleDevsAmount
|
|
|
|
|
e2enode.CountSampleDeviceCapacity(node) == e2enode.SampleDevsAmount &&
|
|
|
|
|
e2enode.CountSampleDeviceAllocatable(node) == e2enode.SampleDevsAmount
|
|
|
|
|
}, 2*time.Minute, framework.Poll).Should(gomega.BeTrueBecause("expected resource exported by the sample device plugin to be available on local node"))
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
@@ -930,7 +926,7 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
ginkgo.By("Waiting for devices to become unavailable on the local node")
|
|
|
|
|
gomega.Eventually(ctx, func(ctx context.Context) bool {
|
|
|
|
|
node, ready := getLocalTestNode(ctx, f)
|
|
|
|
|
return ready && CountSampleDeviceCapacity(node) <= 0
|
|
|
|
|
return ready && e2enode.CountSampleDeviceCapacity(node) <= 0
|
|
|
|
|
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrueBecause("expected devices to be unavailable on local node"))
|
|
|
|
|
|
|
|
|
|
ginkgo.By("devices now unavailable on the local node")
|
|
|
|
|
@@ -941,7 +937,7 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
// exposing those devices as resource has restarted. The expected behavior is that the application pod fails at admission time.
|
|
|
|
|
framework.It("Does not keep device plugin assignments across node reboots if fails admission (no pod restart, no device plugin re-registration)", func(ctx context.Context) {
|
|
|
|
|
podRECMD := fmt.Sprintf("devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep %s", sleepIntervalForever)
|
|
|
|
|
testPod := makeBusyboxPod(SampleDeviceResourceName, podRECMD)
|
|
|
|
|
testPod := makeBusyboxPod(e2enode.SampleDeviceResourceName, podRECMD)
|
|
|
|
|
// Set a short termination grace period to speed up the test.
|
|
|
|
|
// After kubelet restart, this pod will fail admission and need to be terminated.
|
|
|
|
|
// The default 30s grace period causes the test to timeout waiting for the pod
|
|
|
|
|
@@ -1005,7 +1001,7 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
|
|
|
|
|
framework.Logf("Failed to get node devices: %v, will retry", err)
|
|
|
|
|
return true // Return true to continue polling on error
|
|
|
|
|
}
|
|
|
|
|
_, found := checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, SampleDeviceResourceName, []string{})
|
|
|
|
|
_, found := checkPodResourcesAssignment(v1PodResources, pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name, e2enode.SampleDeviceResourceName, []string{})
|
|
|
|
|
return found
|
|
|
|
|
}, 30*time.Second, framework.Poll).Should(gomega.BeFalseBecause("%s/%s/%s failed admission, so it must not appear in podresources list", pod1.Namespace, pod1.Name, pod1.Spec.Containers[0].Name))
|
|
|
|
|
})
|
|
|
|
|
@@ -1116,7 +1112,7 @@ func matchContainerDevices(ident string, contDevs []*kubeletpodresourcesv1.Conta
|
|
|
|
|
|
|
|
|
|
// getSampleDevicePluginPod returns the Sample Device Plugin pod to be used e2e tests.
|
|
|
|
|
func getSampleDevicePluginPod(pluginSockDir string) *v1.Pod {
|
|
|
|
|
data, err := e2etestfiles.Read(SampleDevicePluginDSYAML)
|
|
|
|
|
data, err := e2etestfiles.Read(e2enode.SampleDevicePluginDSYAML)
|
|
|
|
|
if err != nil {
|
|
|
|
|
framework.Fail(err.Error())
|
|
|
|
|
}
|
|
|
|
|
@@ -1124,12 +1120,12 @@ func getSampleDevicePluginPod(pluginSockDir string) *v1.Pod {
|
|
|
|
|
ds := readDaemonSetV1OrDie(data)
|
|
|
|
|
dp := &v1.Pod{
|
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
|
Name: SampleDevicePluginName,
|
|
|
|
|
Name: e2enode.SampleDevicePluginName,
|
|
|
|
|
},
|
|
|
|
|
Spec: ds.Spec.Template.Spec,
|
|
|
|
|
}
|
|
|
|
|
for i := range dp.Spec.Containers[0].Env {
|
|
|
|
|
if dp.Spec.Containers[0].Env[i].Name == SampleDeviceEnvVarNamePluginSockDir {
|
|
|
|
|
if dp.Spec.Containers[0].Env[i].Name == e2enode.SampleDeviceEnvVarNamePluginSockDir {
|
|
|
|
|
dp.Spec.Containers[0].Env[i].Value = pluginSockDir
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|