Merge pull request #37380 from jsafrane/rbd-errors

Automatic merge from submit-queue (batch tested with PRs 39092, 39126, 37380, 37093, 39237)

Improve error reporting in Ceph RBD provisioner.

- We should report an error when user references a secret that cannot be found
- We should report output of rbd create/delete commands, logging "exit code 1"
  is not enough.


Before:
```
Events:
  FirstSeen     LastSeen        Count   From                            SubobjectPath   Type            Reason                  Message
  ---------     --------        -----   ----                            -------------   --------        ------                  -------
  33m           33m             1       {persistentvolume-controller }                  Warning         ProvisioningFailed      Failed to provision volume with StorageClass "cephrbdprovisioner": rbd: create volume failed, err: exit status 1
```

After:

```
Events:
  FirstSeen     LastSeen        Count   From                            SubobjectPath   Type            Reason                  Message
  ---------     --------        -----   ----                            -------------   --------        ------                  -------
  33m           33m             1       {persistentvolume-controller }                  Warning         ProvisioningFailed      Failed to provision volume with StorageClass "cephrbdprovisioner": failed to create rbd image: exit status 1, command output: rbd: couldn't connect to the cluster
```


@rootfs, PTAL

pkg/volume/rbd/rbd.go

@@ -201,8 +201,7 @@ func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
 
 	secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
 	if err != nil {
-		// log error but don't return yet
+		return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
-		glog.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
 	}
 	return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
 }
@@ -282,8 +281,7 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
 		return nil, fmt.Errorf("missing Ceph admin secret name")
 	}
 	if secret, err = parsePVSecret(adminSecretNamespace, adminSecretName, r.plugin.host.GetKubeClient()); err != nil {
-		// log error but don't return yet
+		return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
-		glog.Errorf("failed to get admin secret from [%q/%q]", adminSecretNamespace, adminSecretName)
 	}
 	r.adminSecret = secret
 	if len(r.Mon) < 1 {
@@ -308,7 +306,7 @@ func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
 	rbd, sizeMB, err := r.manager.CreateImage(r)
 	if err != nil {
 		glog.Errorf("rbd: create volume failed, err: %v", err)
-		return nil, fmt.Errorf("rbd: create volume failed, err: %v", err)
+		return nil, err
 	}
 	glog.Infof("successfully created rbd image %q", image)
 	pv := new(v1.PersistentVolume)

pkg/volume/rbd/rbd_util.go

@@ -317,6 +317,7 @@ func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error {
 }
 
 func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, size int, err error) {
+	var output []byte
 	capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
 	volSizeBytes := capacity.Value()
 	// convert to MB that rbd defaults on
@@ -330,7 +331,6 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource
 	for i := start; i < start+l; i++ {
 		mon := p.Mon[i%l]
 		glog.V(4).Infof("rbd: create %s size %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
-		var output []byte
 		output, err = p.rbdMounter.plugin.execCommand("rbd",
 			[]string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", "1"})
 		if err == nil {
@@ -341,8 +341,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource
 	}
 
 	if err != nil {
-		glog.Errorf("rbd: Error creating rbd image: %v", err)
+		return nil, 0, fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output))
-		return nil, 0, err
 	}
 
 	return &v1.RBDVolumeSource{
@@ -375,7 +374,7 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
 		if err == nil {
 			return nil
 		} else {
-			glog.Errorf("failed to delete rbd image, error %v output %v", err, string(output))
+			glog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output))
 		}
 	}
 	return err