openstack: fetch volume path from metadata service

The OpenStack cloud provider retrieves mounted Cinder volume paths
by looking in /dev/disk/by-id, expecting the disk serial IDs (e.g.
SCSI ID) to include the volume ID.

The issue is that not all hypervisors are able to expose this.
For example, Hyper-V will just preserve the original Cinder volume
lun SCSI ID (without setting the volume id). For this reason,
disk path lookups will fail.

In order to be able to leverage Hyper-V based OpenStack providers,
as a fallback, we're querying the metadata service, searching for
disk device metadata. Note that starting with Nova Queens, the Hyper-V
driver always provides disk address information through the instance
metadata.

pkg/cloudprovider/providers/openstack/metadata.go

@@ -33,11 +33,12 @@ import (
 )
 
 const (
-	// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4
-	// link-local address as documented in "OpenStack Cloud Administrator Guide",
-	// chapter Compute - Networking with nova-network.
+	// metadataUrlTemplate allows building an OpenStack Metadata service URL.
+	// It's a hardcoded IPv4 link-local address as documented in "OpenStack Cloud
+	// Administrator Guide", chapter Compute - Networking with nova-network.
 	// https://docs.openstack.org/admin-guide/compute-networking-nova.html#metadata-service
-	metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json"
+	defaultMetadataVersion = "2012-08-10"
+	metadataUrlTemplate    = "http://169.254.169.254/openstack/%s/meta_data.json"
 
 	// metadataID is used as an identifier on the metadata search order configuration.
 	metadataID = "metadataService"
@@ -45,8 +46,8 @@ const (
 	// Config drive is defined as an iso9660 or vfat (deprecated) drive
 	// with the "config-2" label.
 	// http://docs.openstack.org/user-guide/cli-config-drive.html
-	configDriveLabel = "config-2"
-	configDrivePath  = "openstack/2012-08-10/meta_data.json"
+	configDriveLabel        = "config-2"
+	configDrivePathTemplate = "openstack/%s/meta_data.json"
 
 	// configDriveID is used as an identifier on the metadata search order configuration.
 	configDriveID = "configDrive"
@@ -54,12 +55,23 @@ const (
 
 var ErrBadMetadata = errors.New("invalid OpenStack metadata, got empty uuid")
 
+// There are multiple device types. To keep it simple, we're using a single structure
+// for all device metadata types.
+type DeviceMetadata struct {
+	Type    string `json:"type"`
+	Bus     string `json:"bus,omitempty"`
+	Serial  string `json:"serial,omitempty"`
+	Address string `json:"address,omitempty"`
+	// .. and other fields.
+}
+
 // Assumes the "2012-08-10" meta_data.json format.
 // See http://docs.openstack.org/user-guide/cli_config_drive.html
 type Metadata struct {
-	Uuid             string `json:"uuid"`
-	Name             string `json:"name"`
-	AvailabilityZone string `json:"availability_zone"`
+	Uuid             string           `json:"uuid"`
+	Name             string           `json:"name"`
+	AvailabilityZone string           `json:"availability_zone"`
+	Devices          []DeviceMetadata `json:"devices,omitempty"`
 	// .. and other fields we don't care about.  Expand as necessary.
 }
 
@@ -79,7 +91,15 @@ func parseMetadata(r io.Reader) (*Metadata, error) {
 	return &metadata, nil
 }
 
-func getMetadataFromConfigDrive() (*Metadata, error) {
+func getMetadataUrl(metadataVersion string) string {
+	return fmt.Sprintf(metadataUrlTemplate, metadataVersion)
+}
+
+func getConfigDrivePath(metadataVersion string) string {
+	return fmt.Sprintf(configDrivePathTemplate, metadataVersion)
+}
+
+func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) {
 	// Try to read instance UUID from config drive.
 	dev := "/dev/disk/by-label/" + configDriveLabel
 	if _, err := os.Stat(dev); os.IsNotExist(err) {
@@ -114,6 +134,7 @@ func getMetadataFromConfigDrive() (*Metadata, error) {
 
 	glog.V(4).Infof("Configdrive mounted on %s", mntdir)
 
+	configDrivePath := getConfigDrivePath(metadataVersion)
 	f, err := os.Open(
 		filepath.Join(mntdir, configDrivePath))
 	if err != nil {
@@ -124,8 +145,9 @@ func getMetadataFromConfigDrive() (*Metadata, error) {
 	return parseMetadata(f)
 }
 
-func getMetadataFromMetadataService() (*Metadata, error) {
+func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) {
 	// Try to get JSON from metadata server.
+	metadataUrl := getMetadataUrl(metadataVersion)
 	glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl)
 	resp, err := http.Get(metadataUrl)
 	if err != nil {
@@ -154,9 +176,9 @@ func getMetadata(order string) (*Metadata, error) {
 			id = strings.TrimSpace(id)
 			switch id {
 			case configDriveID:
-				md, err = getMetadataFromConfigDrive()
+				md, err = getMetadataFromConfigDrive(defaultMetadataVersion)
 			case metadataID:
-				md, err = getMetadataFromMetadataService()
+				md, err = getMetadataFromMetadataService(defaultMetadataVersion)
 			default:
 				err = fmt.Errorf("%s is not a valid metadata search order option. Supported options are %s and %s", id, configDriveID, metadataID)
 			}

pkg/cloudprovider/providers/openstack/metadata_test.go

@@ -64,7 +64,16 @@ func TestParseMetadata(t *testing.T) {
     "public_keys": {
         "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n"
     },
-    "uuid": "83679162-1378-4288-a2d4-70e13ec132aa"
+    "uuid": "83679162-1378-4288-a2d4-70e13ec132aa",
+    "devices": [
+        {
+            "bus": "scsi",
+            "serial": "6df1888b-f373-41cf-b960-3786e60a28ef",
+            "tags": ["fake_tag"],
+            "type": "disk",
+            "address": "0:0:0:0"
+        }
+    ]
 }
 `)
 	md, err := parseMetadata(data)
@@ -83,4 +92,20 @@ func TestParseMetadata(t *testing.T) {
 	if md.AvailabilityZone != "nova" {
 		t.Errorf("incorrect az: %s", md.AvailabilityZone)
 	}
+
+	if len(md.Devices) != 1 {
+		t.Errorf("expecting to find 1 device, found %d", len(md.Devices))
+	}
+
+	if md.Devices[0].Bus != "scsi" {
+		t.Errorf("incorrect disk bus: %s", md.Devices[0].Bus)
+	}
+
+	if md.Devices[0].Address != "0:0:0:0" {
+		t.Errorf("incorrect disk address: %s", md.Devices[0].Address)
+	}
+
+	if md.Devices[0].Type != "disk" {
+		t.Errorf("incorrect device type: %s", md.Devices[0].Type)
+	}
 }

pkg/cloudprovider/providers/openstack/openstack_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"regexp"
 	"sort"
 	"strings"
 	"testing"
@@ -498,6 +499,8 @@ func TestZones(t *testing.T) {
 	}
 }
 
+var diskPathRegexp = regexp.MustCompile("/dev/disk/(?:by-id|by-path)/")
+
 func TestVolumes(t *testing.T) {
 	cfg, ok := configFromEnv()
 	if !ok {
@@ -534,7 +537,7 @@ func TestVolumes(t *testing.T) {
 	WaitForVolumeStatus(t, os, vol, volumeInUseStatus)
 
 	devicePath := os.GetDevicePath(diskId)
-	if !strings.HasPrefix(devicePath, "/dev/disk/by-id/") {
+	if diskPathRegexp.FindString(devicePath) == "" {
 		t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath)
 	}
 	t.Logf("Volume (%s) found at path: %s\n", vol, devicePath)

pkg/cloudprovider/providers/openstack/openstack_volumes.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -78,6 +79,11 @@ const (
 	VolumeInUseStatus     = "in-use"
 	VolumeDeletedStatus   = "deleted"
 	VolumeErrorStatus     = "error"
+
+	// On some environments, we need to query the metadata service in order
+	// to locate disks. We'll use the Newton version, which includes device
+	// metadata.
+	NewtonMetadataVersion = "2016-06-30"
 )
 
 func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) {
@@ -305,8 +311,9 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str
 }
 
 // GetDevicePath returns the path of an attached block storage volume, specified by its id.
-func (os *OpenStack) GetDevicePath(volumeID string) string {
-	// Build a list of candidate device paths
+func (os *OpenStack) GetDevicePathBySerialId(volumeID string) string {
+	// Build a list of candidate device paths.
+	// Certain Nova drivers will set the disk serial ID, including the Cinder volume id.
 	candidateDeviceNodes := []string{
 		// KVM
 		fmt.Sprintf("virtio-%s", volumeID[:20]),
@@ -327,10 +334,74 @@ func (os *OpenStack) GetDevicePath(volumeID string) string {
 		}
 	}
 
-	glog.Warningf("Failed to find device for the volumeID: %q\n", volumeID)
+	glog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID)
 	return ""
 }
 
+func (os *OpenStack) GetDevicePathFromInstanceMetadata(volumeID string) string {
+	// Nova Hyper-V hosts cannot override disk SCSI IDs. In order to locate
+	// volumes, we're querying the metadata service. Note that the Hyper-V
+	// driver will include device metadata for untagged volumes as well.
+	//
+	// We're avoiding using cached metadata (or the configdrive),
+	// relying on the metadata service.
+	instanceMetadata, err := getMetadataFromMetadataService(
+		NewtonMetadataVersion)
+
+	if err != nil {
+		glog.V(4).Infof(
+			"Could not retrieve instance metadata. Error: %v", err)
+		return ""
+	}
+
+	for _, device := range instanceMetadata.Devices {
+		if device.Type == "disk" && device.Serial == volumeID {
+			glog.V(4).Infof(
+				"Found disk metadata for volumeID %q. Bus: %q, Address: %q",
+				volumeID, device.Bus, device.Address)
+
+			diskPattern := fmt.Sprintf(
+				"/dev/disk/by-path/*-%s-%s",
+				device.Bus, device.Address)
+			diskPaths, err := filepath.Glob(diskPattern)
+			if err != nil {
+				glog.Errorf(
+					"could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v",
+					volumeID, diskPattern, err)
+				return ""
+			}
+
+			if len(diskPaths) == 1 {
+				return diskPaths[0]
+			}
+
+			glog.Errorf(
+				"expecting to find one disk path for volumeID %q, found %d: %v",
+				volumeID, len(diskPaths), diskPaths)
+			return ""
+		}
+	}
+
+	glog.V(4).Infof(
+		"Could not retrieve device metadata for volumeID: %q", volumeID)
+	return ""
+}
+
+// GetDevicePath returns the path of an attached block storage volume, specified by its id.
+func (os *OpenStack) GetDevicePath(volumeID string) string {
+	devicePath := os.GetDevicePathBySerialId(volumeID)
+
+	if devicePath == "" {
+		devicePath = os.GetDevicePathFromInstanceMetadata(volumeID)
+	}
+
+	if devicePath == "" {
+		glog.Warningf("Failed to find device for the volumeID: %q", volumeID)
+	}
+
+	return devicePath
+}
+
 func (os *OpenStack) DeleteVolume(volumeID string) error {
 	used, err := os.diskIsUsed(volumeID)
 	if err != nil {