mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-18 04:54:54 +00:00
Merge pull request #134047 from pohly/dra-e2e-node-metric-flake
DRA E2E node: fix test cleanup
This commit is contained in:
Kubernetes Prow Robot
GitHub
@@ -106,7 +106,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
|
||||
f.Context("Resource Kubelet Plugin", f.WithSerial(), func() {
|
||||
ginkgo.It("must register after Kubelet restart", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
oldCalls := kubeletPlugin.GetGRPCCalls()
|
||||
getNewCalls := func() []testdriver.GRPCCall {
|
||||
@@ -122,11 +122,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must register after plugin restart", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
ginkgo.By("restart Kubelet Plugin")
|
||||
kubeletPlugin.Stop()
|
||||
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
ginkgo.By("wait for Kubelet plugin re-registration")
|
||||
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
@@ -135,7 +135,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
// Test that the kubelet plugin manager retries plugin registration
|
||||
// when the GetInfo call fails, and succeeds once the call passes.
|
||||
ginkgo.It("must recover and register after registration failure", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
ginkgo.By("set GetInfo failure mode")
|
||||
kubeletPlugin.SetGetInfoError(fmt.Errorf("simulated GetInfo failure"))
|
||||
@@ -156,7 +156,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must process pod created when kubelet is not running", func(ctx context.Context) {
|
||||
newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
// Stop Kubelet
|
||||
ginkgo.By("stop kubelet")
|
||||
@@ -175,7 +175,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must keep pod in pending state if NodePrepareResources times out", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unblockNodePrepareResources := kubeletPlugin.BlockNodePrepareResources()
|
||||
defer unblockNodePrepareResources()
|
||||
@@ -195,7 +195,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must run pod if NodePrepareResources fails and then succeeds", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unsetNodePrepareResourcesFailureMode := kubeletPlugin.SetNodePrepareResourcesFailureMode()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
|
||||
@@ -220,7 +220,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must run pod if NodeUnprepareResources fails and then succeeds", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
|
||||
@@ -242,7 +242,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must retry NodePrepareResources after Kubelet restart", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unsetNodePrepareResourcesFailureMode := kubeletPlugin.SetNodePrepareResourcesFailureMode()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
|
||||
@@ -273,7 +273,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must retry NodeUnprepareResources after Kubelet restart", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
|
||||
@@ -300,7 +300,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must call NodeUnprepareResources for deleted pod", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
|
||||
@@ -324,7 +324,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must call NodeUnprepareResources for deleted pod after Kubelet restart", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
|
||||
@@ -355,7 +355,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
})
|
||||
|
||||
ginkgo.It("must not call NodePrepareResources for deleted pod after Kubelet restart", func(ctx context.Context) {
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
unblockNodePrepareResources := kubeletPlugin.BlockNodePrepareResources()
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
|
||||
@@ -394,7 +394,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
|
||||
ginkgo.By("start DRA plugin service")
|
||||
draService := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...)
|
||||
draService := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...)
|
||||
|
||||
pod := createTestObjects(ctx, f.ClientSet, nodeName, f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName})
|
||||
|
||||
@@ -426,7 +426,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
|
||||
ginkgo.By("start DRA plugin service")
|
||||
draService := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...)
|
||||
draService := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...)
|
||||
|
||||
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drasleeppod" /* enables sleeping */, false /* pod is deleted below */, []string{driverName})
|
||||
|
||||
@@ -444,12 +444,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
gomega.Eventually(ctx, listResources(f.ClientSet)).Should(gomega.BeEmpty(), "ResourceSlices without plugin")
|
||||
|
||||
ginkgo.By("restarting plugin")
|
||||
draService = newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...)
|
||||
draService = newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...)
|
||||
|
||||
ginkgo.By("stopping pod")
|
||||
ginkgo.By("stopping pod and waiting for it to get removed, indicating that NodeUnprepareResources must have been called")
|
||||
err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
|
||||
framework.ExpectNoError(err)
|
||||
framework.ExpectNoError(err, "delete pod")
|
||||
gomega.Eventually(draService.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdrivergomega.NodeUnprepareResourcesSucceeded)
|
||||
err = e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete)
|
||||
framework.ExpectNoError(err, "pod + claim shutdown")
|
||||
}
|
||||
ginkgo.DescribeTable("must be functional after service reconnect",
|
||||
functionalAfterServiceReconnect,
|
||||
@@ -464,7 +466,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
|
||||
failOnClosedListener := func(
|
||||
ctx context.Context,
|
||||
service func(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin,
|
||||
service func(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin,
|
||||
listenerOptionFun func(listen func(ctx context.Context, path string) (net.Listener, error)) kubeletplugin.Option,
|
||||
) {
|
||||
ginkgo.By("create a custom listener")
|
||||
@@ -483,6 +485,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
service(
|
||||
ctx,
|
||||
f.ClientSet,
|
||||
f.Namespace.Name,
|
||||
getNodeName(ctx, f),
|
||||
driverName,
|
||||
"",
|
||||
@@ -500,7 +503,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
// The wrappedNewRegistrar function is used to create a new registrar
|
||||
// with the same signature as the newDRAService function, so that it can be
|
||||
// used in the DescribeTable.
|
||||
wrappedNewRegistrar := func(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin {
|
||||
wrappedNewRegistrar := func(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin {
|
||||
return newRegistrar(ctx, clientSet, nodeName, driverName, opts...)
|
||||
}
|
||||
ginkgo.DescribeTable("must report gRPC serving error",
|
||||
@@ -513,8 +516,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() {
|
||||
// start creates plugins which will get stopped when the context gets canceled.
|
||||
start := func(ctx context.Context) (*testdriver.ExamplePlugin, *testdriver.ExamplePlugin) {
|
||||
kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin1Name)
|
||||
kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin2Name)
|
||||
kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), kubeletPlugin1Name)
|
||||
kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), kubeletPlugin2Name)
|
||||
|
||||
ginkgo.By("wait for Kubelet plugin registration")
|
||||
gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
@@ -551,13 +554,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
framework.ExpectNoError(err)
|
||||
gomega.Expect(kubeletPlugin1.GetGRPCCalls()).Should(testdrivergomega.NodePrepareResourcesSucceeded, "Plugin 1 should have prepared resources.")
|
||||
gomega.Expect(kubeletPlugin2.GetGRPCCalls()).Should(testdrivergomega.NodePrepareResourcesSucceeded, "Plugin 2 should have prepared resources.")
|
||||
driverName := func(element any) string {
|
||||
el := element.(*testutil.Sample)
|
||||
return string(el.Metric[testutil.LabelName("driver_name")])
|
||||
}
|
||||
|
||||
gomega.Expect(getKubeletMetrics(ctx)).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
|
||||
"dra_resource_claims_in_use": gstruct.MatchAllElements(driverName, gstruct.Elements{
|
||||
"dra_resource_claims_in_use": gstruct.MatchAllElements(driverNameLabelKey, gstruct.Elements{
|
||||
"<any>": timelessSample(1),
|
||||
kubeletPlugin1Name: timelessSample(1),
|
||||
kubeletPlugin2Name: timelessSample(1),
|
||||
@@ -572,10 +570,10 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
gomega.Expect(kubeletPlugin1.GetGRPCCalls()).Should(testdrivergomega.NodeUnprepareResourcesSucceeded, "Plugin 2 should have unprepared resources.")
|
||||
gomega.Expect(kubeletPlugin2.GetGRPCCalls()).Should(testdrivergomega.NodeUnprepareResourcesSucceeded, "Plugin 2 should have unprepared resources.")
|
||||
gomega.Expect(getKubeletMetrics(ctx)).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
|
||||
"dra_resource_claims_in_use": gstruct.MatchAllElements(driverName, gstruct.Elements{
|
||||
"dra_resource_claims_in_use": gstruct.MatchAllElements(driverNameLabelKey, gstruct.Elements{
|
||||
"<any>": timelessSample(0),
|
||||
}),
|
||||
}), "metrics while pod is running")
|
||||
}), "metrics after deleting pod")
|
||||
})
|
||||
|
||||
ginkgo.It("must run pod if NodePrepareResources fails for one plugin and then succeeds", func(ctx context.Context) {
|
||||
@@ -714,7 +712,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName))
|
||||
|
||||
ginkgo.By("start plugin and wait for ResourceSlice")
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
gomega.Eventually(ctx, listResources(f.ClientSet)).Should(matchNode, "ResourceSlice from kubelet plugin")
|
||||
gomega.Consistently(ctx, listResources(f.ClientSet)).WithTimeout(5*time.Second).Should(matchNode, "ResourceSlice from kubelet plugin")
|
||||
|
||||
@@ -734,7 +732,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
|
||||
ginkgo.By("start DRA plugin service")
|
||||
kubeletPlugin := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...)
|
||||
kubeletPlugin := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...)
|
||||
|
||||
ginkgo.By("wait for ResourceSlice to be created by plugin")
|
||||
matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName))
|
||||
@@ -786,7 +784,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
|
||||
ginkgo.By("start DRA plugin service")
|
||||
kubeletPlugin := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...)
|
||||
kubeletPlugin := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...)
|
||||
|
||||
ginkgo.By("wait for ResourceSlice to be created by plugin")
|
||||
matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName))
|
||||
@@ -804,7 +802,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
ginkgo.By("restarting plugin")
|
||||
newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...)
|
||||
newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...)
|
||||
|
||||
ginkgo.By("ensuring unchanged ResourceSlices")
|
||||
gomega.Consistently(ctx, listResources(f.ClientSet)).WithTimeout(time.Minute).Should(gomega.Equal(slices), "ResourceSlices")
|
||||
@@ -827,7 +825,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
// reported by a DRA plugin are correctly reflected in the Pod's status.
|
||||
ginkgo.It("should reflect device health changes in the Pod's status", func(ctx context.Context) {
|
||||
ginkgo.By("Starting the test driver with channel-based control")
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
className := "health-test-class"
|
||||
claimName := "health-test-claim"
|
||||
@@ -881,7 +879,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
// stops and recovers to "Healthy" upon plugin restart.
|
||||
ginkgo.It("should update health to Unknown when plugin stops and recover upon restart", func(ctx context.Context) {
|
||||
ginkgo.By("Starting the test driver")
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
className := "unknown-test-class"
|
||||
claimName := "unknown-test-claim"
|
||||
@@ -914,7 +912,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
|
||||
ginkgo.By("Restarting the DRA plugin to simulate recovery")
|
||||
// Re-initialize the plugin, which will re-register with the Kubelet.
|
||||
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
|
||||
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName)
|
||||
|
||||
ginkgo.By("Forcing a 'Healthy' status update after restart")
|
||||
kubeletPlugin.HealthControlChan <- testdriver.DeviceHealthUpdate{
|
||||
@@ -947,7 +945,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
ginkgo.It("should not add health status to Pod when feature gate is disabled", func(ctx context.Context) {
|
||||
|
||||
ginkgo.By("Starting a test driver")
|
||||
kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName, withHealthService(false))
|
||||
newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName, withHealthService(false))
|
||||
|
||||
className := "gate-disabled-class"
|
||||
claimName := "gate-disabled-claim"
|
||||
@@ -976,10 +974,6 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami
|
||||
}
|
||||
return fmt.Errorf("could not find container 'testcontainer' in pod status")
|
||||
}).WithContext(ctx).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(gomega.Succeed(), "The allocatedResourcesStatus field should be absent when the feature gate is disabled")
|
||||
// Clean up and wait for the system to settle before the test ends
|
||||
ginkgo.By("Cleaning up pod and waiting for resources to be unprepared")
|
||||
e2epod.DeletePodOrFail(ctx, f.ClientSet, pod.Namespace, pod.Name)
|
||||
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdrivergomega.NodeUnprepareResourcesSucceeded)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -995,7 +989,7 @@ func withHealthService(enabled bool) pluginOption {
|
||||
}
|
||||
|
||||
// Run Kubelet plugin and wait until it's registered
|
||||
func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName string, options ...pluginOption) *testdriver.ExamplePlugin {
|
||||
func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, testNamespace string, nodeName, driverName string, options ...pluginOption) *testdriver.ExamplePlugin {
|
||||
ginkgo.By("start Kubelet plugin")
|
||||
logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "DRA kubelet plugin "+driverName), "node", nodeName)
|
||||
ctx = klog.NewContext(ctx, logger)
|
||||
@@ -1017,7 +1011,7 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN
|
||||
}
|
||||
|
||||
plugin, err := testdriver.StartPlugin(
|
||||
ctx,
|
||||
context.WithoutCancel(ctx), // Stopping is handled in draServiceCleanup.
|
||||
cdiDir,
|
||||
driverName,
|
||||
clientSet,
|
||||
@@ -1041,14 +1035,9 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN
|
||||
)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) { draServiceCleanup(ctx, clientSet, driverName, testNamespace, plugin) })
|
||||
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
// kubelet should do this eventually, but better make sure.
|
||||
// A separate test checks this explicitly.
|
||||
framework.ExpectNoError(clientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName}))
|
||||
})
|
||||
ginkgo.DeferCleanup(plugin.Stop)
|
||||
gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered)
|
||||
|
||||
return plugin
|
||||
}
|
||||
@@ -1094,7 +1083,7 @@ func newRegistrar(ctx context.Context, clientSet kubernetes.Interface, nodeName,
|
||||
//
|
||||
// Returns:
|
||||
// - A pointer to the started ExamplePlugin instance.
|
||||
func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin {
|
||||
func newDRAService(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin {
|
||||
ginkgo.By("start only Kubelet plugin")
|
||||
logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin "+driverName), "node", nodeName)
|
||||
ctx = klog.NewContext(ctx, logger)
|
||||
@@ -1120,7 +1109,7 @@ func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName
|
||||
framework.ExpectNoError(err, "create DRA socket directory")
|
||||
|
||||
plugin, err := testdriver.StartPlugin(
|
||||
ctx,
|
||||
context.WithoutCancel(ctx), // Stopping is handled in draServiceCleanup.
|
||||
cdiDir,
|
||||
driverName,
|
||||
clientSet,
|
||||
@@ -1144,14 +1133,40 @@ func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName
|
||||
)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) {
|
||||
ginkgo.DeferCleanup(func(ctx context.Context) { draServiceCleanup(ctx, clientSet, driverName, testNamespace, plugin) })
|
||||
|
||||
return plugin
|
||||
}
|
||||
|
||||
func draServiceCleanup(ctx context.Context, clientSet kubernetes.Interface, driverName string, testNamespace string, plugin *testdriver.ExamplePlugin) {
|
||||
// Always stop at the end, even in the case of failures.
|
||||
defer func() {
|
||||
ginkgo.By("Stopping the driver and deleting ResourceSlices...")
|
||||
plugin.Stop()
|
||||
|
||||
// kubelet should do this eventually, but better make sure.
|
||||
// A separate test checks this explicitly.
|
||||
framework.ExpectNoError(clientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName}))
|
||||
})
|
||||
ginkgo.DeferCleanup(plugin.Stop)
|
||||
}()
|
||||
|
||||
return plugin
|
||||
// Some test was leaking an in-use ResourceClaim
|
||||
// (https://github.com/kubernetes/kubernetes/issues/133304) because it
|
||||
// didn't delete its pod before the driver shutdown. The namespace
|
||||
// deletion usually takes care of this, but that happens later and
|
||||
// without the driver the pods stay pending. Therefore we delete all of
|
||||
// the pods in the test namespace *before* proceeding with the driver
|
||||
// shutdown.
|
||||
ginkgo.By("Deleting pods and waiting for ResourceClaims to be released...")
|
||||
framework.ExpectNoError(clientSet.CoreV1().Pods(testNamespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}))
|
||||
|
||||
// We allow for a delay because it takes a while till the kubelet fully deals with pod deletion.
|
||||
// Typically there is no dra_resource_claims_in_use entry for the driver (covered by gstruct.IgnoreMissing).
|
||||
// If there is one, it must be zero.
|
||||
gomega.Eventually(ctx, getKubeletMetrics).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{
|
||||
"dra_resource_claims_in_use": gstruct.MatchElements(driverNameLabelKey, gstruct.IgnoreExtras|gstruct.IgnoreMissing, gstruct.Elements{
|
||||
driverName: timelessSample(0),
|
||||
}),
|
||||
}), "The test should have deleted all pods.\nInstead some claims prepared by the driver %s are still in use at the time when the driver gets uninstalled.", driverName)
|
||||
}
|
||||
|
||||
// createTestObjects creates objects required by the test
|
||||
@@ -1518,3 +1533,10 @@ func (l *errorOnCloseListener) Close() error {
|
||||
func (*errorOnCloseListener) Addr() net.Addr {
|
||||
return &net.UnixAddr{Name: "errorOnCloseListener", Net: "unix"}
|
||||
}
|
||||
|
||||
// driverNameLabelKey maps metric samples to their driver_name label value.
|
||||
// Used with gstruct.MatchAllElements.
|
||||
func driverNameLabelKey(element any) string {
|
||||
el := element.(*testutil.Sample)
|
||||
return string(el.Metric[testutil.LabelName("driver_name")])
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user