diff --git a/pkg/cloudprovider/providers/openstack/metadata.go b/pkg/cloudprovider/providers/openstack/metadata.go index 959d91b92e5..d3cf22d9238 100644 --- a/pkg/cloudprovider/providers/openstack/metadata.go +++ b/pkg/cloudprovider/providers/openstack/metadata.go @@ -33,11 +33,12 @@ import ( ) const ( - // metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4 - // link-local address as documented in "OpenStack Cloud Administrator Guide", - // chapter Compute - Networking with nova-network. + // metadataUrlTemplate allows building an OpenStack Metadata service URL. + // It's a hardcoded IPv4 link-local address as documented in "OpenStack Cloud + // Administrator Guide", chapter Compute - Networking with nova-network. // https://docs.openstack.org/admin-guide/compute-networking-nova.html#metadata-service - metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json" + defaultMetadataVersion = "2012-08-10" + metadataUrlTemplate = "http://169.254.169.254/openstack/%s/meta_data.json" // metadataID is used as an identifier on the metadata search order configuration. metadataID = "metadataService" @@ -45,8 +46,8 @@ const ( // Config drive is defined as an iso9660 or vfat (deprecated) drive // with the "config-2" label. // http://docs.openstack.org/user-guide/cli-config-drive.html - configDriveLabel = "config-2" - configDrivePath = "openstack/2012-08-10/meta_data.json" + configDriveLabel = "config-2" + configDrivePathTemplate = "openstack/%s/meta_data.json" // configDriveID is used as an identifier on the metadata search order configuration. configDriveID = "configDrive" @@ -54,12 +55,23 @@ const ( var ErrBadMetadata = errors.New("invalid OpenStack metadata, got empty uuid") +// There are multiple device types. To keep it simple, we're using a single structure +// for all device metadata types. +type DeviceMetadata struct { + Type string `json:"type"` + Bus string `json:"bus,omitempty"` + Serial string `json:"serial,omitempty"` + Address string `json:"address,omitempty"` + // .. and other fields. +} + // Assumes the "2012-08-10" meta_data.json format. // See http://docs.openstack.org/user-guide/cli_config_drive.html type Metadata struct { - Uuid string `json:"uuid"` - Name string `json:"name"` - AvailabilityZone string `json:"availability_zone"` + Uuid string `json:"uuid"` + Name string `json:"name"` + AvailabilityZone string `json:"availability_zone"` + Devices []DeviceMetadata `json:"devices,omitempty"` // .. and other fields we don't care about. Expand as necessary. } @@ -79,7 +91,15 @@ func parseMetadata(r io.Reader) (*Metadata, error) { return &metadata, nil } -func getMetadataFromConfigDrive() (*Metadata, error) { +func getMetadataUrl(metadataVersion string) string { + return fmt.Sprintf(metadataUrlTemplate, metadataVersion) +} + +func getConfigDrivePath(metadataVersion string) string { + return fmt.Sprintf(configDrivePathTemplate, metadataVersion) +} + +func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) { // Try to read instance UUID from config drive. dev := "/dev/disk/by-label/" + configDriveLabel if _, err := os.Stat(dev); os.IsNotExist(err) { @@ -114,6 +134,7 @@ func getMetadataFromConfigDrive() (*Metadata, error) { glog.V(4).Infof("Configdrive mounted on %s", mntdir) + configDrivePath := getConfigDrivePath(metadataVersion) f, err := os.Open( filepath.Join(mntdir, configDrivePath)) if err != nil { @@ -124,8 +145,9 @@ func getMetadataFromConfigDrive() (*Metadata, error) { return parseMetadata(f) } -func getMetadataFromMetadataService() (*Metadata, error) { +func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) { // Try to get JSON from metadata server. + metadataUrl := getMetadataUrl(metadataVersion) glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl) resp, err := http.Get(metadataUrl) if err != nil { @@ -154,9 +176,9 @@ func getMetadata(order string) (*Metadata, error) { id = strings.TrimSpace(id) switch id { case configDriveID: - md, err = getMetadataFromConfigDrive() + md, err = getMetadataFromConfigDrive(defaultMetadataVersion) case metadataID: - md, err = getMetadataFromMetadataService() + md, err = getMetadataFromMetadataService(defaultMetadataVersion) default: err = fmt.Errorf("%s is not a valid metadata search order option. Supported options are %s and %s", id, configDriveID, metadataID) } diff --git a/pkg/cloudprovider/providers/openstack/metadata_test.go b/pkg/cloudprovider/providers/openstack/metadata_test.go index feeb04b0f96..d224a598bbb 100644 --- a/pkg/cloudprovider/providers/openstack/metadata_test.go +++ b/pkg/cloudprovider/providers/openstack/metadata_test.go @@ -64,7 +64,16 @@ func TestParseMetadata(t *testing.T) { "public_keys": { "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n" }, - "uuid": "83679162-1378-4288-a2d4-70e13ec132aa" + "uuid": "83679162-1378-4288-a2d4-70e13ec132aa", + "devices": [ + { + "bus": "scsi", + "serial": "6df1888b-f373-41cf-b960-3786e60a28ef", + "tags": ["fake_tag"], + "type": "disk", + "address": "0:0:0:0" + } + ] } `) md, err := parseMetadata(data) @@ -83,4 +92,20 @@ func TestParseMetadata(t *testing.T) { if md.AvailabilityZone != "nova" { t.Errorf("incorrect az: %s", md.AvailabilityZone) } + + if len(md.Devices) != 1 { + t.Errorf("expecting to find 1 device, found %d", len(md.Devices)) + } + + if md.Devices[0].Bus != "scsi" { + t.Errorf("incorrect disk bus: %s", md.Devices[0].Bus) + } + + if md.Devices[0].Address != "0:0:0:0" { + t.Errorf("incorrect disk address: %s", md.Devices[0].Address) + } + + if md.Devices[0].Type != "disk" { + t.Errorf("incorrect device type: %s", md.Devices[0].Type) + } } diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/pkg/cloudprovider/providers/openstack/openstack_test.go index 8b08d317238..0875185b63d 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_test.go +++ b/pkg/cloudprovider/providers/openstack/openstack_test.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "reflect" + "regexp" "sort" "strings" "testing" @@ -498,6 +499,8 @@ func TestZones(t *testing.T) { } } +var diskPathRegexp = regexp.MustCompile("/dev/disk/(?:by-id|by-path)/") + func TestVolumes(t *testing.T) { cfg, ok := configFromEnv() if !ok { @@ -534,7 +537,7 @@ func TestVolumes(t *testing.T) { WaitForVolumeStatus(t, os, vol, volumeInUseStatus) devicePath := os.GetDevicePath(diskId) - if !strings.HasPrefix(devicePath, "/dev/disk/by-id/") { + if diskPathRegexp.FindString(devicePath) == "" { t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath) } t.Logf("Volume (%s) found at path: %s\n", vol, devicePath) diff --git a/pkg/cloudprovider/providers/openstack/openstack_volumes.go b/pkg/cloudprovider/providers/openstack/openstack_volumes.go index ebe7970f797..8d308fe547f 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_volumes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_volumes.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "path" + "path/filepath" "strings" "time" @@ -78,6 +79,11 @@ const ( VolumeInUseStatus = "in-use" VolumeDeletedStatus = "deleted" VolumeErrorStatus = "error" + + // On some environments, we need to query the metadata service in order + // to locate disks. We'll use the Newton version, which includes device + // metadata. + NewtonMetadataVersion = "2016-06-30" ) func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) { @@ -305,8 +311,9 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str } // GetDevicePath returns the path of an attached block storage volume, specified by its id. -func (os *OpenStack) GetDevicePath(volumeID string) string { - // Build a list of candidate device paths +func (os *OpenStack) GetDevicePathBySerialId(volumeID string) string { + // Build a list of candidate device paths. + // Certain Nova drivers will set the disk serial ID, including the Cinder volume id. candidateDeviceNodes := []string{ // KVM fmt.Sprintf("virtio-%s", volumeID[:20]), @@ -327,10 +334,74 @@ func (os *OpenStack) GetDevicePath(volumeID string) string { } } - glog.Warningf("Failed to find device for the volumeID: %q\n", volumeID) + glog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID) return "" } +func (os *OpenStack) GetDevicePathFromInstanceMetadata(volumeID string) string { + // Nova Hyper-V hosts cannot override disk SCSI IDs. In order to locate + // volumes, we're querying the metadata service. Note that the Hyper-V + // driver will include device metadata for untagged volumes as well. + // + // We're avoiding using cached metadata (or the configdrive), + // relying on the metadata service. + instanceMetadata, err := getMetadataFromMetadataService( + NewtonMetadataVersion) + + if err != nil { + glog.V(4).Infof( + "Could not retrieve instance metadata. Error: %v", err) + return "" + } + + for _, device := range instanceMetadata.Devices { + if device.Type == "disk" && device.Serial == volumeID { + glog.V(4).Infof( + "Found disk metadata for volumeID %q. Bus: %q, Address: %q", + volumeID, device.Bus, device.Address) + + diskPattern := fmt.Sprintf( + "/dev/disk/by-path/*-%s-%s", + device.Bus, device.Address) + diskPaths, err := filepath.Glob(diskPattern) + if err != nil { + glog.Errorf( + "could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v", + volumeID, diskPattern, err) + return "" + } + + if len(diskPaths) == 1 { + return diskPaths[0] + } + + glog.Errorf( + "expecting to find one disk path for volumeID %q, found %d: %v", + volumeID, len(diskPaths), diskPaths) + return "" + } + } + + glog.V(4).Infof( + "Could not retrieve device metadata for volumeID: %q", volumeID) + return "" +} + +// GetDevicePath returns the path of an attached block storage volume, specified by its id. +func (os *OpenStack) GetDevicePath(volumeID string) string { + devicePath := os.GetDevicePathBySerialId(volumeID) + + if devicePath == "" { + devicePath = os.GetDevicePathFromInstanceMetadata(volumeID) + } + + if devicePath == "" { + glog.Warningf("Failed to find device for the volumeID: %q", volumeID) + } + + return devicePath +} + func (os *OpenStack) DeleteVolume(volumeID string) error { used, err := os.diskIsUsed(volumeID) if err != nil {