From 559f83a93daa5078603dd30430f2a18fbd7c1f4c Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Mon, 6 Nov 2017 16:14:28 +0200 Subject: [PATCH] openstack: fetch volume path from metadata service The OpenStack cloud provider retrieves mounted Cinder volume paths by looking in /dev/disk/by-id, expecting the disk serial IDs (e.g. SCSI ID) to include the volume ID. The issue is that not all hypervisors are able to expose this. For example, Hyper-V will just preserve the original Cinder volume lun SCSI ID (without setting the volume id). For this reason, disk path lookups will fail. In order to be able to leverage Hyper-V based OpenStack providers, as a fallback, we're querying the metadata service, searching for disk device metadata. Note that starting with Nova Queens, the Hyper-V driver always provides disk address information through the instance metadata. --- .../providers/openstack/metadata.go | 48 ++++++++---- .../providers/openstack/metadata_test.go | 27 ++++++- .../providers/openstack/openstack_test.go | 5 +- .../providers/openstack/openstack_volumes.go | 77 ++++++++++++++++++- 4 files changed, 139 insertions(+), 18 deletions(-) diff --git a/pkg/cloudprovider/providers/openstack/metadata.go b/pkg/cloudprovider/providers/openstack/metadata.go index 959d91b92e5..d3cf22d9238 100644 --- a/pkg/cloudprovider/providers/openstack/metadata.go +++ b/pkg/cloudprovider/providers/openstack/metadata.go @@ -33,11 +33,12 @@ import ( ) const ( - // metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4 - // link-local address as documented in "OpenStack Cloud Administrator Guide", - // chapter Compute - Networking with nova-network. + // metadataUrlTemplate allows building an OpenStack Metadata service URL. + // It's a hardcoded IPv4 link-local address as documented in "OpenStack Cloud + // Administrator Guide", chapter Compute - Networking with nova-network. // https://docs.openstack.org/admin-guide/compute-networking-nova.html#metadata-service - metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json" + defaultMetadataVersion = "2012-08-10" + metadataUrlTemplate = "http://169.254.169.254/openstack/%s/meta_data.json" // metadataID is used as an identifier on the metadata search order configuration. metadataID = "metadataService" @@ -45,8 +46,8 @@ const ( // Config drive is defined as an iso9660 or vfat (deprecated) drive // with the "config-2" label. // http://docs.openstack.org/user-guide/cli-config-drive.html - configDriveLabel = "config-2" - configDrivePath = "openstack/2012-08-10/meta_data.json" + configDriveLabel = "config-2" + configDrivePathTemplate = "openstack/%s/meta_data.json" // configDriveID is used as an identifier on the metadata search order configuration. configDriveID = "configDrive" @@ -54,12 +55,23 @@ const ( var ErrBadMetadata = errors.New("invalid OpenStack metadata, got empty uuid") +// There are multiple device types. To keep it simple, we're using a single structure +// for all device metadata types. +type DeviceMetadata struct { + Type string `json:"type"` + Bus string `json:"bus,omitempty"` + Serial string `json:"serial,omitempty"` + Address string `json:"address,omitempty"` + // .. and other fields. +} + // Assumes the "2012-08-10" meta_data.json format. // See http://docs.openstack.org/user-guide/cli_config_drive.html type Metadata struct { - Uuid string `json:"uuid"` - Name string `json:"name"` - AvailabilityZone string `json:"availability_zone"` + Uuid string `json:"uuid"` + Name string `json:"name"` + AvailabilityZone string `json:"availability_zone"` + Devices []DeviceMetadata `json:"devices,omitempty"` // .. and other fields we don't care about. Expand as necessary. } @@ -79,7 +91,15 @@ func parseMetadata(r io.Reader) (*Metadata, error) { return &metadata, nil } -func getMetadataFromConfigDrive() (*Metadata, error) { +func getMetadataUrl(metadataVersion string) string { + return fmt.Sprintf(metadataUrlTemplate, metadataVersion) +} + +func getConfigDrivePath(metadataVersion string) string { + return fmt.Sprintf(configDrivePathTemplate, metadataVersion) +} + +func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { // Try to read instance UUID from config drive. dev := "/dev/disk/by-label/" + configDriveLabel if _, err := os.Stat(dev); os.IsNotExist(err) { @@ -114,6 +134,7 @@ func getMetadataFromConfigDrive() (*Metadata, error) { glog.V(4).Infof("Configdrive mounted on %s", mntdir) + configDrivePath := getConfigDrivePath(metadataVersion) f, err := os.Open( filepath.Join(mntdir, configDrivePath)) if err != nil { @@ -124,8 +145,9 @@ func getMetadataFromConfigDrive() (*Metadata, error) { return parseMetadata(f) } -func getMetadataFromMetadataService() (*Metadata, error) { +func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) { // Try to get JSON from metadata server. + metadataUrl := getMetadataUrl(metadataVersion) glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl) resp, err := http.Get(metadataUrl) if err != nil { @@ -154,9 +176,9 @@ func getMetadata(order string) (*Metadata, error) { id = strings.TrimSpace(id) switch id { case configDriveID: - md, err = getMetadataFromConfigDrive() + md, err = getMetadataFromConfigDrive(defaultMetadataVersion) case metadataID: - md, err = getMetadataFromMetadataService() + md, err = getMetadataFromMetadataService(defaultMetadataVersion) default: err = fmt.Errorf("%s is not a valid metadata search order option. Supported options are %s and %s", id, configDriveID, metadataID) } diff --git a/pkg/cloudprovider/providers/openstack/metadata_test.go b/pkg/cloudprovider/providers/openstack/metadata_test.go index feeb04b0f96..d224a598bbb 100644 --- a/pkg/cloudprovider/providers/openstack/metadata_test.go +++ b/pkg/cloudprovider/providers/openstack/metadata_test.go @@ -64,7 +64,16 @@ func TestParseMetadata(t *testing.T) { "public_keys": { "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n" }, - "uuid": "83679162-1378-4288-a2d4-70e13ec132aa" + "uuid": "83679162-1378-4288-a2d4-70e13ec132aa", + "devices": [ + { + "bus": "scsi", + "serial": "6df1888b-f373-41cf-b960-3786e60a28ef", + "tags": ["fake_tag"], + "type": "disk", + "address": "0:0:0:0" + } + ] } `) md, err := parseMetadata(data) @@ -83,4 +92,20 @@ func TestParseMetadata(t *testing.T) { if md.AvailabilityZone != "nova" { t.Errorf("incorrect az: %s", md.AvailabilityZone) } + + if len(md.Devices) != 1 { + t.Errorf("expecting to find 1 device, found %d", len(md.Devices)) + } + + if md.Devices[0].Bus != "scsi" { + t.Errorf("incorrect disk bus: %s", md.Devices[0].Bus) + } + + if md.Devices[0].Address != "0:0:0:0" { + t.Errorf("incorrect disk address: %s", md.Devices[0].Address) + } + + if md.Devices[0].Type != "disk" { + t.Errorf("incorrect device type: %s", md.Devices[0].Type) + } } diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/pkg/cloudprovider/providers/openstack/openstack_test.go index 8b08d317238..0875185b63d 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_test.go +++ b/pkg/cloudprovider/providers/openstack/openstack_test.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "reflect" + "regexp" "sort" "strings" "testing" @@ -498,6 +499,8 @@ func TestZones(t *testing.T) { } } +var diskPathRegexp = regexp.MustCompile("/dev/disk/(?:by-id|by-path)/") + func TestVolumes(t *testing.T) { cfg, ok := configFromEnv() if !ok { @@ -534,7 +537,7 @@ func TestVolumes(t *testing.T) { WaitForVolumeStatus(t, os, vol, volumeInUseStatus) devicePath := os.GetDevicePath(diskId) - if !strings.HasPrefix(devicePath, "/dev/disk/by-id/") { + if diskPathRegexp.FindString(devicePath) == "" { t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath) } t.Logf("Volume (%s) found at path: %s\n", vol, devicePath) diff --git a/pkg/cloudprovider/providers/openstack/openstack_volumes.go b/pkg/cloudprovider/providers/openstack/openstack_volumes.go index ebe7970f797..8d308fe547f 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_volumes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_volumes.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "path" + "path/filepath" "strings" "time" @@ -78,6 +79,11 @@ const ( VolumeInUseStatus = "in-use" VolumeDeletedStatus = "deleted" VolumeErrorStatus = "error" + + // On some environments, we need to query the metadata service in order + // to locate disks. We'll use the Newton version, which includes device + // metadata. + NewtonMetadataVersion = "2016-06-30" ) func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) { @@ -305,8 +311,9 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str } // GetDevicePath returns the path of an attached block storage volume, specified by its id. -func (os *OpenStack) GetDevicePath(volumeID string) string { - // Build a list of candidate device paths +func (os *OpenStack) GetDevicePathBySerialId(volumeID string) string { + // Build a list of candidate device paths. + // Certain Nova drivers will set the disk serial ID, including the Cinder volume id. candidateDeviceNodes := []string{ // KVM fmt.Sprintf("virtio-%s", volumeID[:20]), @@ -327,10 +334,74 @@ func (os *OpenStack) GetDevicePath(volumeID string) string { } } - glog.Warningf("Failed to find device for the volumeID: %q\n", volumeID) + glog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID) return "" } +func (os *OpenStack) GetDevicePathFromInstanceMetadata(volumeID string) string { + // Nova Hyper-V hosts cannot override disk SCSI IDs. In order to locate + // volumes, we're querying the metadata service. Note that the Hyper-V + // driver will include device metadata for untagged volumes as well. + // + // We're avoiding using cached metadata (or the configdrive), + // relying on the metadata service. + instanceMetadata, err := getMetadataFromMetadataService( + NewtonMetadataVersion) + + if err != nil { + glog.V(4).Infof( + "Could not retrieve instance metadata. Error: %v", err) + return "" + } + + for _, device := range instanceMetadata.Devices { + if device.Type == "disk" && device.Serial == volumeID { + glog.V(4).Infof( + "Found disk metadata for volumeID %q. Bus: %q, Address: %q", + volumeID, device.Bus, device.Address) + + diskPattern := fmt.Sprintf( + "/dev/disk/by-path/*-%s-%s", + device.Bus, device.Address) + diskPaths, err := filepath.Glob(diskPattern) + if err != nil { + glog.Errorf( + "could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v", + volumeID, diskPattern, err) + return "" + } + + if len(diskPaths) == 1 { + return diskPaths[0] + } + + glog.Errorf( + "expecting to find one disk path for volumeID %q, found %d: %v", + volumeID, len(diskPaths), diskPaths) + return "" + } + } + + glog.V(4).Infof( + "Could not retrieve device metadata for volumeID: %q", volumeID) + return "" +} + +// GetDevicePath returns the path of an attached block storage volume, specified by its id. +func (os *OpenStack) GetDevicePath(volumeID string) string { + devicePath := os.GetDevicePathBySerialId(volumeID) + + if devicePath == "" { + devicePath = os.GetDevicePathFromInstanceMetadata(volumeID) + } + + if devicePath == "" { + glog.Warningf("Failed to find device for the volumeID: %q", volumeID) + } + + return devicePath +} + func (os *OpenStack) DeleteVolume(volumeID string) error { used, err := os.diskIsUsed(volumeID) if err != nil {