Merge pull request #64308 from jsafrane/rbd-startup

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Move Ceph server secret creation to common code.

The secret should be created only on one place.

**Release note**:

```release-note
NONE
```

@jeffvance @copejon @rootfs @msau42 PTAL

test/e2e/framework/volume_util.go

@@ -186,7 +186,7 @@ func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTest
 }
 
 // CephRBD-specific wrapper for CreateStorageServer.
-func NewRBDServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
+func NewRBDServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, secret *v1.Secret, ip string) {
 	config = VolumeTestConfig{
 		Namespace:   namespace,
 		Prefix:      "rbd",
@@ -205,7 +205,28 @@ func NewRBDServer(cs clientset.Interface, namespace string) (config VolumeTestCo
 	Logf("sleeping a bit to give ceph server time to initialize")
 	time.Sleep(VolumeServerPodStartupSleep)
 
-	return config, pod, ip
+	// create secrets for the server
+	secret = &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Prefix + "-secret",
+		},
+		Data: map[string][]byte{
+			// from test/images/volumes-tester/rbd/keyring
+			"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
+		},
+		Type: "kubernetes.io/rbd",
+	}
+
+	secret, err := cs.CoreV1().Secrets(config.Namespace).Create(secret)
+	if err != nil {
+		Failf("Failed to create secrets for Ceph RBD: %v", err)
+	}
+
+	return config, pod, secret, ip
 }
 
 // Wrapper for StartVolumeServer(). A storage server config is passed in, and a pod pointer

test/e2e/storage/volume_io.go

@@ -385,33 +385,11 @@ var _ = utils.SIGDescribe("Volume plugin streaming [Slow]", func() {
 	Describe("Ceph-RBD [Feature:Volumes]", func() {
 		var (
 			secret *v1.Secret
-			name   string
 		)
 		testFile := "ceph-rbd_io_test"
 
 		BeforeEach(func() {
-			config, serverPod, serverIP = framework.NewRBDServer(cs, ns)
+			config, serverPod, secret, serverIP = framework.NewRBDServer(cs, ns)
-			name = config.Prefix + "-server"
-
-			// create server secret
-			secret = &v1.Secret{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "Secret",
-					APIVersion: "v1",
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name: name,
-				},
-				Data: map[string][]byte{
-					// from test/images/volumes-tester/rbd/keyring
-					"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
-				},
-				Type: "kubernetes.io/rbd",
-			}
-			var err error
-			secret, err = cs.CoreV1().Secrets(ns).Create(secret)
-			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("BeforeEach: failed to create secret %q for Ceph-RBD: %v", name, err))
-
 			volSource = v1.VolumeSource{
 				RBD: &v1.RBDVolumeSource{
 					CephMonitors: []string{serverIP},
@@ -419,7 +397,7 @@ var _ = utils.SIGDescribe("Volume plugin streaming [Slow]", func() {
 					RBDImage:     "foo",
 					RadosUser:    "admin",
 					SecretRef: &v1.LocalObjectReference{
-						Name: name,
+						Name: secret.Name,
 					},
 					FSType:   "ext2",
 					ReadOnly: false,
@@ -428,13 +406,13 @@ var _ = utils.SIGDescribe("Volume plugin streaming [Slow]", func() {
 		})
 
 		AfterEach(func() {
-			framework.Logf("AfterEach: deleting Ceph-RDB server secret %q...", name)
+			framework.Logf("AfterEach: deleting Ceph-RDB server secret %q...", secret.Name)
-			secErr := cs.CoreV1().Secrets(ns).Delete(name, &metav1.DeleteOptions{})
+			secErr := cs.CoreV1().Secrets(ns).Delete(secret.Name, &metav1.DeleteOptions{})
 			framework.Logf("AfterEach: deleting Ceph-RDB server pod %q...", serverPod.Name)
 			err := framework.DeletePodWithWait(f, cs, serverPod)
 			if secErr != nil || err != nil {
 				if secErr != nil {
-					framework.Logf("AfterEach: Ceph-RDB delete secret failed: %v", err)
+					framework.Logf("AfterEach: Ceph-RDB delete secret failed: %v", secErr)
 				}
 				if err != nil {
 					framework.Logf("AfterEach: Ceph-RDB server pod delete failed: %v", err)

test/e2e/storage/volumes.go

@@ -57,7 +57,6 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 	vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere"
-	imageutils "k8s.io/kubernetes/test/utils/image"
 )
 
 func DeleteCinderVolume(name string) error {
@@ -200,34 +199,9 @@ var _ = utils.SIGDescribe("Volumes", func() {
 
 	Describe("Ceph RBD [Feature:Volumes]", func() {
 		It("should be mountable", func() {
-			config, _, serverIP := framework.NewRBDServer(cs, namespace.Name)
+			config, _, secret, serverIP := framework.NewRBDServer(cs, namespace.Name)
 			defer framework.VolumeTestCleanup(f, config)
-
+			defer cs.CoreV1().Secrets(config.Namespace).Delete(secret.Name, nil)
-			// create secrets for the server
-			secret := v1.Secret{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "Secret",
-					APIVersion: "v1",
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name: config.Prefix + "-secret",
-				},
-				Data: map[string][]byte{
-					// from test/images/volumes-tester/rbd/keyring
-					"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
-				},
-				Type: "kubernetes.io/rbd",
-			}
-
-			secClient := cs.CoreV1().Secrets(config.Namespace)
-
-			defer func() {
-				secClient.Delete(config.Prefix+"-secret", nil)
-			}()
-
-			if _, err := secClient.Create(&secret); err != nil {
-				framework.Failf("Failed to create secrets for Ceph RBD: %v", err)
-			}
 
 			tests := []framework.VolumeTest{
 				{
@@ -238,7 +212,7 @@ var _ = utils.SIGDescribe("Volumes", func() {
 							RBDImage:     "foo",
 							RadosUser:    "admin",
 							SecretRef: &v1.LocalObjectReference{
-								Name: config.Prefix + "-secret",
+								Name: secret.Name,
 							},
 							FSType: "ext2",
 						},
@@ -258,44 +232,9 @@ var _ = utils.SIGDescribe("Volumes", func() {
 	////////////////////////////////////////////////////////////////////////
 	Describe("CephFS [Feature:Volumes]", func() {
 		It("should be mountable", func() {
-			config := framework.VolumeTestConfig{
+			config, _, secret, serverIP := framework.NewRBDServer(cs, namespace.Name)
-				Namespace:   namespace.Name,
-				Prefix:      "cephfs",
-				ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer),
-				ServerPorts: []int{6789},
-			}
-
 			defer framework.VolumeTestCleanup(f, config)
-			_, serverIP := framework.CreateStorageServer(cs, config)
+			defer cs.CoreV1().Secrets(config.Namespace).Delete(secret.Name, nil)
-			By("sleeping a bit to give ceph server time to initialize")
-			time.Sleep(framework.VolumeServerPodStartupSleep)
-
-			// create ceph secret
-			secret := &v1.Secret{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "Secret",
-					APIVersion: "v1",
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name: config.Prefix + "-secret",
-				},
-				// from test/images/volumes-tester/rbd/keyring
-				Data: map[string][]byte{
-					"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
-				},
-				Type: "kubernetes.io/cephfs",
-			}
-
-			defer func() {
-				if err := cs.CoreV1().Secrets(namespace.Name).Delete(secret.Name, nil); err != nil {
-					framework.Failf("unable to delete secret %v: %v", secret.Name, err)
-				}
-			}()
-
-			var err error
-			if secret, err = cs.CoreV1().Secrets(namespace.Name).Create(secret); err != nil {
-				framework.Failf("unable to create test secret %s: %v", secret.Name, err)
-			}
 
 			tests := []framework.VolumeTest{
 				{
@@ -303,7 +242,7 @@ var _ = utils.SIGDescribe("Volumes", func() {
 						CephFS: &v1.CephFSVolumeSource{
 							Monitors:  []string{serverIP + ":6789"},
 							User:      "kube",
-							SecretRef: &v1.LocalObjectReference{Name: config.Prefix + "-secret"},
+							SecretRef: &v1.LocalObjectReference{Name: secret.Name},
 							ReadOnly:  true,
 						},
 					},