From 76b4973b424b1f78792acf9f52d80fe3a9baf45f Mon Sep 17 00:00:00 2001 From: Jiawei Wang Date: Fri, 31 Jul 2020 15:47:19 -0700 Subject: [PATCH] Wait for mock CSI Driver bringup to perform e2e test In our current mock CSI driver e2e test, we are not waiting for the CSI driver register successfully to perform test including provision PVC. This can lead to timeout when the csi driver takes longer to register the socket. This change adds the waiting part so that the system will wait for up to 10 minutes for the driver to be ready. This normally won't take this long. However, under a resource constraint environment it can take longer than expected time. https://github.com/kubernetes/kubernetes/issues/93358 --- test/e2e/storage/csi_mock_volume.go | 7 ++++++- test/e2e/storage/drivers/csi.go | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index e8643e192ea..c6ba11a6faf 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -151,11 +151,16 @@ var _ = utils.SIGDescribe("CSI mock volume", func() { if tp.registerDriver { err = waitForCSIDriver(cs, m.config.GetUniqueDriverName()) - framework.ExpectNoError(err, "Failed to get CSIDriver : %v", err) + framework.ExpectNoError(err, "Failed to get CSIDriver %v", m.config.GetUniqueDriverName()) m.testCleanups = append(m.testCleanups, func() { destroyCSIDriver(cs, m.config.GetUniqueDriverName()) }) } + + // Wait for the CSIDriver actually get deployed and CSINode object to be generated. + // This indicates the mock CSI driver pod is up and running healthy. + err = drivers.WaitForCSIDriverRegistrationOnNode(m.config.ClientNodeSelection.Name, m.config.GetUniqueDriverName(), cs) + framework.ExpectNoError(err, "Failed to register CSIDriver %v", m.config.GetUniqueDriverName()) } createPod := func(ephemeral bool) (class *storagev1.StorageClass, claim *v1.PersistentVolumeClaim, pod *v1.Pod) { diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 88b9ef7a888..6783e6bf31d 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -608,17 +608,25 @@ func waitForCSIDriverRegistrationOnAllNodes(driverName string, cs clientset.Inte return err } for _, node := range nodes.Items { - if err := waitForCSIDriverRegistrationOnNode(node.Name, driverName, cs); err != nil { + if err := WaitForCSIDriverRegistrationOnNode(node.Name, driverName, cs); err != nil { return err } } return nil } -func waitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs clientset.Interface) error { - const csiNodeRegisterTimeout = 5 * time.Minute +// WaitForCSIDriverRegistrationOnNode waits for the CSINode object generated by the node-registrar on a certain node +func WaitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs clientset.Interface) error { + framework.Logf("waiting for CSIDriver %v to register on node %v", driverName, nodeName) - waitErr := wait.PollImmediate(10*time.Second, csiNodeRegisterTimeout, func() (bool, error) { + // About 8.6 minutes timeout + backoff := wait.Backoff{ + Duration: 2 * time.Second, + Factor: 1.5, + Steps: 12, + } + + waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) { csiNode, err := cs.StorageV1().CSINodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { return false, err