Merge pull request #55313 from petrutlucian94/openstack_hv_pv

Automatic merge from submit-queue (batch tested with PRs 55642, 55897, 55835, 55496, 55313). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

OpenStack: fetch volume path from metadata service

**What this PR does / why we need it**:
Updates the OpenStack cloud provider to use the Nova metadata service as a fallback when retrieving mounted PV disk paths. Note that the Nova instance device metadata will contain the disk address and bus, which allows finding its path. 

This is needed as the *standard* mechanism of retrieving disk paths is not available when running k8s under OpenStack Hyper-V hosts.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #55312

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

pkg/cloudprovider/providers/openstack/metadata.go

@@ -33,11 +33,12 @@ import (
 )
 
 const (
-	// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4
-	// link-local address as documented in "OpenStack Cloud Administrator Guide",
-	// chapter Compute - Networking with nova-network.
+	// metadataUrlTemplate allows building an OpenStack Metadata service URL.
+	// It's a hardcoded IPv4 link-local address as documented in "OpenStack Cloud
+	// Administrator Guide", chapter Compute - Networking with nova-network.
 	// https://docs.openstack.org/admin-guide/compute-networking-nova.html#metadata-service
-	metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json"
+	defaultMetadataVersion = "2012-08-10"
+	metadataUrlTemplate    = "http://169.254.169.254/openstack/%s/meta_data.json"
 
 	// metadataID is used as an identifier on the metadata search order configuration.
 	metadataID = "metadataService"
@@ -45,8 +46,8 @@ const (
 	// Config drive is defined as an iso9660 or vfat (deprecated) drive
 	// with the "config-2" label.
 	// http://docs.openstack.org/user-guide/cli-config-drive.html
-	configDriveLabel = "config-2"
-	configDrivePath  = "openstack/2012-08-10/meta_data.json"
+	configDriveLabel        = "config-2"
+	configDrivePathTemplate = "openstack/%s/meta_data.json"
 
 	// configDriveID is used as an identifier on the metadata search order configuration.
 	configDriveID = "configDrive"
@@ -54,12 +55,23 @@ const (
 
 var ErrBadMetadata = errors.New("invalid OpenStack metadata, got empty uuid")
 
+// There are multiple device types. To keep it simple, we're using a single structure
+// for all device metadata types.
+type DeviceMetadata struct {
+	Type    string `json:"type"`
+	Bus     string `json:"bus,omitempty"`
+	Serial  string `json:"serial,omitempty"`
+	Address string `json:"address,omitempty"`
+	// .. and other fields.
+}
+
 // Assumes the "2012-08-10" meta_data.json format.
 // See http://docs.openstack.org/user-guide/cli_config_drive.html
 type Metadata struct {
-	Uuid             string `json:"uuid"`
-	Name             string `json:"name"`
-	AvailabilityZone string `json:"availability_zone"`
+	Uuid             string           `json:"uuid"`
+	Name             string           `json:"name"`
+	AvailabilityZone string           `json:"availability_zone"`
+	Devices          []DeviceMetadata `json:"devices,omitempty"`
 	// .. and other fields we don't care about.  Expand as necessary.
 }
 
@@ -79,7 +91,15 @@ func parseMetadata(r io.Reader) (*Metadata, error) {
 	return &metadata, nil
 }
 
-func getMetadataFromConfigDrive() (*Metadata, error) {
+func getMetadataUrl(metadataVersion string) string {
+	return fmt.Sprintf(metadataUrlTemplate, metadataVersion)
+}
+
+func getConfigDrivePath(metadataVersion string) string {
+	return fmt.Sprintf(configDrivePathTemplate, metadataVersion)
+}
+
+func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) {
 	// Try to read instance UUID from config drive.
 	dev := "/dev/disk/by-label/" + configDriveLabel
 	if _, err := os.Stat(dev); os.IsNotExist(err) {
@@ -114,6 +134,7 @@ func getMetadataFromConfigDrive() (*Metadata, error) {
 
 	glog.V(4).Infof("Configdrive mounted on %s", mntdir)
 
+	configDrivePath := getConfigDrivePath(metadataVersion)
 	f, err := os.Open(
 		filepath.Join(mntdir, configDrivePath))
 	if err != nil {
@@ -124,8 +145,9 @@ func getMetadataFromConfigDrive() (*Metadata, error) {
 	return parseMetadata(f)
 }
 
-func getMetadataFromMetadataService() (*Metadata, error) {
+func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) {
 	// Try to get JSON from metadata server.
+	metadataUrl := getMetadataUrl(metadataVersion)
 	glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl)
 	resp, err := http.Get(metadataUrl)
 	if err != nil {
@@ -154,9 +176,9 @@ func getMetadata(order string) (*Metadata, error) {
 			id = strings.TrimSpace(id)
 			switch id {
 			case configDriveID:
-				md, err = getMetadataFromConfigDrive()
+				md, err = getMetadataFromConfigDrive(defaultMetadataVersion)
 			case metadataID:
-				md, err = getMetadataFromMetadataService()
+				md, err = getMetadataFromMetadataService(defaultMetadataVersion)
 			default:
 				err = fmt.Errorf("%s is not a valid metadata search order option. Supported options are %s and %s", id, configDriveID, metadataID)
 			}

pkg/cloudprovider/providers/openstack/metadata_test.go

@@ -64,7 +64,16 @@ func TestParseMetadata(t *testing.T) {
     "public_keys": {
         "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n"
     },
-    "uuid": "83679162-1378-4288-a2d4-70e13ec132aa"
+    "uuid": "83679162-1378-4288-a2d4-70e13ec132aa",
+    "devices": [
+        {
+            "bus": "scsi",
+            "serial": "6df1888b-f373-41cf-b960-3786e60a28ef",
+            "tags": ["fake_tag"],
+            "type": "disk",
+            "address": "0:0:0:0"
+        }
+    ]
 }
 `)
 	md, err := parseMetadata(data)
@@ -83,4 +92,20 @@ func TestParseMetadata(t *testing.T) {
 	if md.AvailabilityZone != "nova" {
 		t.Errorf("incorrect az: %s", md.AvailabilityZone)
 	}
+
+	if len(md.Devices) != 1 {
+		t.Errorf("expecting to find 1 device, found %d", len(md.Devices))
+	}
+
+	if md.Devices[0].Bus != "scsi" {
+		t.Errorf("incorrect disk bus: %s", md.Devices[0].Bus)
+	}
+
+	if md.Devices[0].Address != "0:0:0:0" {
+		t.Errorf("incorrect disk address: %s", md.Devices[0].Address)
+	}
+
+	if md.Devices[0].Type != "disk" {
+		t.Errorf("incorrect device type: %s", md.Devices[0].Type)
+	}
 }

pkg/cloudprovider/providers/openstack/openstack_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
 	"testing"
@@ -498,6 +499,8 @@ func TestZones(t *testing.T) {
 	}
 }
 
+var diskPathRegexp = regexp.MustCompile("/dev/disk/(?:by-id|by-path)/")
+
 func TestVolumes(t *testing.T) {
 	cfg, ok := configFromEnv()
 	if !ok {
@@ -534,7 +537,7 @@ func TestVolumes(t *testing.T) {
 	WaitForVolumeStatus(t, os, vol, volumeInUseStatus)
 
 	devicePath := os.GetDevicePath(diskId)
-	if !strings.HasPrefix(devicePath, "/dev/disk/by-id/") {
+	if diskPathRegexp.FindString(devicePath) == "" {
 		t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath)
 	}
 	t.Logf("Volume (%s) found at path: %s\n", vol, devicePath)

pkg/cloudprovider/providers/openstack/openstack_volumes.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -78,6 +79,11 @@ const (
 	VolumeInUseStatus     = "in-use"
 	VolumeDeletedStatus   = "deleted"
 	VolumeErrorStatus     = "error"
+
+	// On some environments, we need to query the metadata service in order
+	// to locate disks. We'll use the Newton version, which includes device
+	// metadata.
+	NewtonMetadataVersion = "2016-06-30"
 )
 
 func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) {
@@ -305,8 +311,9 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str
 }
 
 // GetDevicePath returns the path of an attached block storage volume, specified by its id.
-func (os *OpenStack) GetDevicePath(volumeID string) string {
-	// Build a list of candidate device paths
+func (os *OpenStack) GetDevicePathBySerialId(volumeID string) string {
+	// Build a list of candidate device paths.
+	// Certain Nova drivers will set the disk serial ID, including the Cinder volume id.
 	candidateDeviceNodes := []string{
 		// KVM
 		fmt.Sprintf("virtio-%s", volumeID[:20]),
@@ -327,10 +334,74 @@ func (os *OpenStack) GetDevicePath(volumeID string) string {
 		}
 	}
 
-	glog.Warningf("Failed to find device for the volumeID: %q\n", volumeID)
+	glog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID)
 	return ""
 }
 
+func (os *OpenStack) GetDevicePathFromInstanceMetadata(volumeID string) string {
+	// Nova Hyper-V hosts cannot override disk SCSI IDs. In order to locate
+	// volumes, we're querying the metadata service. Note that the Hyper-V
+	// driver will include device metadata for untagged volumes as well.
+	//
+	// We're avoiding using cached metadata (or the configdrive),
+	// relying on the metadata service.
+	instanceMetadata, err := getMetadataFromMetadataService(
+		NewtonMetadataVersion)
+
+	if err != nil {
+		glog.V(4).Infof(
+			"Could not retrieve instance metadata. Error: %v", err)
+		return ""
+	}
+
+	for _, device := range instanceMetadata.Devices {
+		if device.Type == "disk" && device.Serial == volumeID {
+			glog.V(4).Infof(
+				"Found disk metadata for volumeID %q. Bus: %q, Address: %q",
+				volumeID, device.Bus, device.Address)
+
+			diskPattern := fmt.Sprintf(
+				"/dev/disk/by-path/*-%s-%s",
+				device.Bus, device.Address)
+			diskPaths, err := filepath.Glob(diskPattern)
+			if err != nil {
+				glog.Errorf(
+					"could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v",
+					volumeID, diskPattern, err)
+				return ""
+			}
+
+			if len(diskPaths) == 1 {
+				return diskPaths[0]
+			}
+
+			glog.Errorf(
+				"expecting to find one disk path for volumeID %q, found %d: %v",
+				volumeID, len(diskPaths), diskPaths)
+			return ""
+		}
+	}
+
+	glog.V(4).Infof(
+		"Could not retrieve device metadata for volumeID: %q", volumeID)
+	return ""
+}
+
+// GetDevicePath returns the path of an attached block storage volume, specified by its id.
+func (os *OpenStack) GetDevicePath(volumeID string) string {
+	devicePath := os.GetDevicePathBySerialId(volumeID)
+
+	if devicePath == "" {
+		devicePath = os.GetDevicePathFromInstanceMetadata(volumeID)
+	}
+
+	if devicePath == "" {
+		glog.Warningf("Failed to find device for the volumeID: %q", volumeID)
+	}
+
+	return devicePath
+}
+
 func (os *OpenStack) DeleteVolume(volumeID string) error {
 	used, err := os.diskIsUsed(volumeID)
 	if err != nil {