Add portals field to iscsi volume source to achieve multipathing.

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

examples/volumes/iscsi/README.md

@@ -18,7 +18,7 @@ If there isn't one in place then it is possible to setup a software version on L
 
 ## Creating the pod with iSCSI persistent storage
 
-Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required.
+Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. If you have more than one target portals for a single IQN, you can mention other portal IPs in *portals* field.
 
 If you want to use an iSCSI offload card or other open-iscsi transports besides tcp, setup an iSCSI interface and provide *iscsiInterface* in the pod YAML. The default name for an iscsi iface (open-iscsi parameter iface.iscsi\_ifacename) is in the format transport\_name.hwaddress when generated by iscsiadm. See [open-iscsi](http://www.open-iscsi.org/docs/README) or [openstack](http://docs.openstack.org/kilo/config-reference/content/iscsi-iface-config.html) for detailed configuration information.
 
@@ -48,19 +48,23 @@ For a non mpio device the output should look like the following
 
 ```console
 # mount |grep kub
-/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered)
+/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
-/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered)
+/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (ro,relatime,data=ordered)
-/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered)
+/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
 /dev/sdc on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
+/dev/sdd on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
+/dev/sdd on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
 ```
 
 And for a node with mpio enabled the expected output would be similar to the following
 
 ```console
 # mount |grep kub
-/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered)
+/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
 /dev/mapper/mpatha on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered)
-/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered)
+/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
+/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
+/dev/mapper/mpathc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
 /dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
 ```
 
@@ -70,7 +74,6 @@ If you ssh to that machine, you can run `docker ps` to see the actual pod.
 ```console
 # docker ps
 CONTAINER ID        IMAGE                                  COMMAND             CREATED             STATUS              PORTS               NAMES
-f855336407f4        kubernetes/pause                       "/pause"            6 minutes ago       Up 6 minutes                            k8s_iscsipd-ro.d130ec3e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_5409a4cb
 3b8a772515d2        kubernetes/pause                       "/pause"            6 minutes ago       Up 6 minutes                            k8s_iscsipd-rw.ed58ec4e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_d25592c5
 ```
 

examples/volumes/iscsi/iscsi.yaml

@@ -5,28 +5,17 @@ metadata:
   name: iscsipd
 spec:
   containers:
-  - name: iscsipd-ro
-    image: kubernetes/pause
-    volumeMounts:
-    - mountPath: "/mnt/iscsipd"
-      name: iscsipd-ro
   - name: iscsipd-rw
     image: kubernetes/pause
     volumeMounts:
     - mountPath: "/mnt/iscsipd"
       name: iscsipd-rw
   volumes:
-  - name: iscsipd-ro
+  - name: iscsipd-rw
     iscsi:
       targetPortal: 10.0.2.15:3260
+      portals: ['10.0.2.16:3260', '10.0.2.17:3260']
       iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz
       lun: 0
       fsType: ext4
       readOnly: true
-  - name: iscsipd-rw
-    iscsi:
-      targetPortal: 10.0.2.15:3260
-      iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz
-      lun: 1
-      fsType: ext4
-      readOnly: false

pkg/api/types.go

@@ -623,6 +623,10 @@ type ISCSIVolumeSource struct {
 	// the ReadOnly setting in VolumeMounts.
 	// +optional
 	ReadOnly bool
+	// Required: list of iSCSI target portal ips for high availability.
+	// the portal is either an IP or ip_addr:port if port is other than default (typically TCP ports 860 and 3260)
+	// +optional
+	Portals []string
 }
 
 // Represents a Fibre Channel volume.

pkg/api/v1/types.go

@@ -988,6 +988,10 @@ type ISCSIVolumeSource struct {
 	// Defaults to false.
 	// +optional
 	ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,6,opt,name=readOnly"`
+	// iSCSI target portal List. The portal is either an IP or ip_addr:port if the port
+	// is other than default (typically TCP ports 860 and 3260).
+	// +optional
+	Portals []string `json:"portals" protobuf:"bytes,7,opt,name=portals"`
 }
 
 // Represents a Fibre Channel volume.

pkg/volume/iscsi/iscsi.go

@@ -104,14 +104,18 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI
 
 	lun := strconv.Itoa(int(iscsi.Lun))
 	portal := portalMounter(iscsi.TargetPortal)
-
+	var bkportal []string
+	bkportal = append(bkportal, portal)
+	for _, tp := range iscsi.Portals {
+		bkportal = append(bkportal, portalMounter(string(tp)))
+	}
 	iface := iscsi.ISCSIInterface
 
 	return &iscsiDiskMounter{
 		iscsiDisk: &iscsiDisk{
 			podUID:  podUID,
 			volName: spec.Name(),
-			portal:  portal,
+			portals: bkportal,
 			iqn:     iscsi.IQN,
 			lun:     lun,
 			iface:   iface,
@@ -162,7 +166,7 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v
 type iscsiDisk struct {
 	volName string
 	podUID  types.UID
-	portal  string
+	portals []string
 	iqn     string
 	lun     string
 	iface   string

pkg/volume/iscsi/iscsi_util.go

@@ -98,13 +98,13 @@ func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun s
 type ISCSIUtil struct{}
 
 func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
-	return makePDNameInternal(iscsi.plugin.host, iscsi.portal, iscsi.iqn, iscsi.lun)
+	return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun)
 }
 
 func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
 	var devicePath string
+	var devicePaths []string
 	var iscsiTransport string
-
 	out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.iface, "-o", "show"})
 	if err != nil {
 		glog.Errorf("iscsi: could not read iface %s error: %s", b.iface, string(out))
@@ -112,34 +112,50 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
 	}
 
 	iscsiTransport = extractTransportname(string(out))
+	bkpPortal := b.portals
+	for _, tp := range bkpPortal {
+		if iscsiTransport == "" {
+			glog.Errorf("iscsi: could not find transport name in iface %s", b.iface)
+			return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface))
+		} else if iscsiTransport == "tcp" {
+			devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-")
+		} else {
+			devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-")
+		}
+		exist := waitForPathToExist(devicePath, 1, iscsiTransport)
+		if exist == false {
+			// discover iscsi target
+			out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discovery", "-t", "sendtargets", "-p", tp, "-I", b.iface})
+			if err != nil {
+				glog.Errorf("iscsi: failed to sendtargets to portal %s error: %s", tp, string(out))
+				continue
+			}
+			// login to iscsi target
+			out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-I", b.iface, "--login"})
+			if err != nil {
+				glog.Errorf("iscsi: failed to attach disk:Error: %s (%v)", string(out), err)
+				continue
+			}
+			exist = waitForPathToExist(devicePath, 10, iscsiTransport)
+			if !exist {
+				glog.Errorf("Could not attach disk: Timeout after 10s")
+			} else {
+				devicePaths = append(devicePaths, devicePath)
+			}
+		} else {
+			glog.V(4).Infof("iscsi: devicepath (%s) exists", devicePath)
+			devicePaths = append(devicePaths, devicePath)
+		}
+	}
 
-	if iscsiTransport == "" {
+	if len(devicePaths) == 0 {
-		glog.Errorf("iscsi: could not find transport name in iface %s", b.iface)
+		glog.Errorf("iscsi: failed to get any path for iscsi disk")
-		return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface))
+		return errors.New("failed to get any path for iscsi disk")
-	} else if iscsiTransport == "tcp" {
-		devicePath = strings.Join([]string{"/dev/disk/by-path/ip", b.portal, "iscsi", b.iqn, "lun", b.lun}, "-")
-	} else {
-		devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", b.portal, "iscsi", b.iqn, "lun", b.lun}, "-")
-	}
-	exist := waitForPathToExist(devicePath, 1, iscsiTransport)
-	if exist == false {
-		// discover iscsi target
-		out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discovery", "-t", "sendtargets", "-p", b.portal, "-I", b.iface})
-		if err != nil {
-			glog.Errorf("iscsi: failed to sendtargets to portal %s error: %s", b.portal, string(out))
-			return err
-		}
-		// login to iscsi target
-		out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", b.portal, "-T", b.iqn, "-I", b.iface, "--login"})
-		if err != nil {
-			glog.Errorf("iscsi: failed to attach disk:Error: %s (%v)", string(out), err)
-			return err
-		}
-		exist = waitForPathToExist(devicePath, 10, iscsiTransport)
-		if !exist {
-			return errors.New("Could not attach disk: Timeout after 10s")
-		}
 	}
+
+	//Make sure we use a valid devicepath to find mpio device.
+	devicePath = devicePaths[0]
+
 	// mount it
 	globalPDPath := b.manager.MakeGlobalPDName(*b.iscsiDisk)
 	notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
@@ -153,9 +169,17 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
 		return err
 	}
 
-	// check if the dev is using mpio and if so mount it via the dm-XX device
+	for _, path := range devicePaths {
-	if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(devicePath); mappedDevicePath != "" {
+		// There shouldnt be any empty device paths. However adding this check
-		devicePath = mappedDevicePath
+		// for safer side to avoid the possibility of an empty entry.
+		if path == "" {
+			continue
+		}
+		// check if the dev is using mpio and if so mount it via the dm-XX device
+		if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
+			devicePath = mappedDevicePath
+			break
+		}
 	}
 	err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil)
 	if err != nil {