From 332e26dc8c09cd4cb9777fee881c49f258522355 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Mon, 6 Feb 2017 17:06:33 +0530 Subject: [PATCH] Add portals field to iscsi volume source to achieve multipathing. Signed-off-by: Humble Chirammal --- examples/volumes/iscsi/README.md | 17 +++--- examples/volumes/iscsi/iscsi.yaml | 15 +----- pkg/api/types.go | 4 ++ pkg/api/v1/types.go | 4 ++ pkg/volume/iscsi/iscsi.go | 10 ++-- pkg/volume/iscsi/iscsi_util.go | 86 ++++++++++++++++++++----------- 6 files changed, 82 insertions(+), 54 deletions(-) diff --git a/examples/volumes/iscsi/README.md b/examples/volumes/iscsi/README.md index b9741269351..268b34fe226 100644 --- a/examples/volumes/iscsi/README.md +++ b/examples/volumes/iscsi/README.md @@ -18,7 +18,7 @@ If there isn't one in place then it is possible to setup a software version on L ## Creating the pod with iSCSI persistent storage -Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. +Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. If you have more than one target portals for a single IQN, you can mention other portal IPs in *portals* field. If you want to use an iSCSI offload card or other open-iscsi transports besides tcp, setup an iSCSI interface and provide *iscsiInterface* in the pod YAML. The default name for an iscsi iface (open-iscsi parameter iface.iscsi\_ifacename) is in the format transport\_name.hwaddress when generated by iscsiadm. See [open-iscsi](http://www.open-iscsi.org/docs/README) or [openstack](http://docs.openstack.org/kilo/config-reference/content/iscsi-iface-config.html) for detailed configuration information. @@ -48,19 +48,23 @@ For a non mpio device the output should look like the following ```console # mount |grep kub -/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered) -/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered) -/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered) +/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (ro,relatime,data=ordered) +/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) /dev/sdc on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +/dev/sdd on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/sdd on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) ``` And for a node with mpio enabled the expected output would be similar to the following ```console # mount |grep kub -/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered) +/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) /dev/mapper/mpatha on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered) -/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered) /dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) ``` @@ -70,7 +74,6 @@ If you ssh to that machine, you can run `docker ps` to see the actual pod. ```console # docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -f855336407f4 kubernetes/pause "/pause" 6 minutes ago Up 6 minutes k8s_iscsipd-ro.d130ec3e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_5409a4cb 3b8a772515d2 kubernetes/pause "/pause" 6 minutes ago Up 6 minutes k8s_iscsipd-rw.ed58ec4e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_d25592c5 ``` diff --git a/examples/volumes/iscsi/iscsi.yaml b/examples/volumes/iscsi/iscsi.yaml index 4c951f6dc11..46736eda8f7 100644 --- a/examples/volumes/iscsi/iscsi.yaml +++ b/examples/volumes/iscsi/iscsi.yaml @@ -5,28 +5,17 @@ metadata: name: iscsipd spec: containers: - - name: iscsipd-ro - image: kubernetes/pause - volumeMounts: - - mountPath: "/mnt/iscsipd" - name: iscsipd-ro - name: iscsipd-rw image: kubernetes/pause volumeMounts: - mountPath: "/mnt/iscsipd" name: iscsipd-rw volumes: - - name: iscsipd-ro + - name: iscsipd-rw iscsi: targetPortal: 10.0.2.15:3260 + portals: ['10.0.2.16:3260', '10.0.2.17:3260'] iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz lun: 0 fsType: ext4 readOnly: true - - name: iscsipd-rw - iscsi: - targetPortal: 10.0.2.15:3260 - iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz - lun: 1 - fsType: ext4 - readOnly: false diff --git a/pkg/api/types.go b/pkg/api/types.go index 457a5f64566..837c5ed5c06 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -623,6 +623,10 @@ type ISCSIVolumeSource struct { // the ReadOnly setting in VolumeMounts. // +optional ReadOnly bool + // Required: list of iSCSI target portal ips for high availability. + // the portal is either an IP or ip_addr:port if port is other than default (typically TCP ports 860 and 3260) + // +optional + Portals []string } // Represents a Fibre Channel volume. diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index e90ef580e4e..e56ade1c805 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -988,6 +988,10 @@ type ISCSIVolumeSource struct { // Defaults to false. // +optional ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,6,opt,name=readOnly"` + // iSCSI target portal List. The portal is either an IP or ip_addr:port if the port + // is other than default (typically TCP ports 860 and 3260). + // +optional + Portals []string `json:"portals" protobuf:"bytes,7,opt,name=portals"` } // Represents a Fibre Channel volume. diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 70bf1a5017c..126d81c923d 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -104,14 +104,18 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI lun := strconv.Itoa(int(iscsi.Lun)) portal := portalMounter(iscsi.TargetPortal) - + var bkportal []string + bkportal = append(bkportal, portal) + for _, tp := range iscsi.Portals { + bkportal = append(bkportal, portalMounter(string(tp))) + } iface := iscsi.ISCSIInterface return &iscsiDiskMounter{ iscsiDisk: &iscsiDisk{ podUID: podUID, volName: spec.Name(), - portal: portal, + portals: bkportal, iqn: iscsi.IQN, lun: lun, iface: iface, @@ -162,7 +166,7 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v type iscsiDisk struct { volName string podUID types.UID - portal string + portals []string iqn string lun string iface string diff --git a/pkg/volume/iscsi/iscsi_util.go b/pkg/volume/iscsi/iscsi_util.go index 7224a179652..0b48fb61b64 100644 --- a/pkg/volume/iscsi/iscsi_util.go +++ b/pkg/volume/iscsi/iscsi_util.go @@ -98,13 +98,13 @@ func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun s type ISCSIUtil struct{} func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string { - return makePDNameInternal(iscsi.plugin.host, iscsi.portal, iscsi.iqn, iscsi.lun) + return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun) } func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { var devicePath string + var devicePaths []string var iscsiTransport string - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.iface, "-o", "show"}) if err != nil { glog.Errorf("iscsi: could not read iface %s error: %s", b.iface, string(out)) @@ -112,34 +112,50 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { } iscsiTransport = extractTransportname(string(out)) + bkpPortal := b.portals + for _, tp := range bkpPortal { + if iscsiTransport == "" { + glog.Errorf("iscsi: could not find transport name in iface %s", b.iface) + return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface)) + } else if iscsiTransport == "tcp" { + devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-") + } else { + devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-") + } + exist := waitForPathToExist(devicePath, 1, iscsiTransport) + if exist == false { + // discover iscsi target + out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discovery", "-t", "sendtargets", "-p", tp, "-I", b.iface}) + if err != nil { + glog.Errorf("iscsi: failed to sendtargets to portal %s error: %s", tp, string(out)) + continue + } + // login to iscsi target + out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-I", b.iface, "--login"}) + if err != nil { + glog.Errorf("iscsi: failed to attach disk:Error: %s (%v)", string(out), err) + continue + } + exist = waitForPathToExist(devicePath, 10, iscsiTransport) + if !exist { + glog.Errorf("Could not attach disk: Timeout after 10s") + } else { + devicePaths = append(devicePaths, devicePath) + } + } else { + glog.V(4).Infof("iscsi: devicepath (%s) exists", devicePath) + devicePaths = append(devicePaths, devicePath) + } + } - if iscsiTransport == "" { - glog.Errorf("iscsi: could not find transport name in iface %s", b.iface) - return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface)) - } else if iscsiTransport == "tcp" { - devicePath = strings.Join([]string{"/dev/disk/by-path/ip", b.portal, "iscsi", b.iqn, "lun", b.lun}, "-") - } else { - devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", b.portal, "iscsi", b.iqn, "lun", b.lun}, "-") - } - exist := waitForPathToExist(devicePath, 1, iscsiTransport) - if exist == false { - // discover iscsi target - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discovery", "-t", "sendtargets", "-p", b.portal, "-I", b.iface}) - if err != nil { - glog.Errorf("iscsi: failed to sendtargets to portal %s error: %s", b.portal, string(out)) - return err - } - // login to iscsi target - out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", b.portal, "-T", b.iqn, "-I", b.iface, "--login"}) - if err != nil { - glog.Errorf("iscsi: failed to attach disk:Error: %s (%v)", string(out), err) - return err - } - exist = waitForPathToExist(devicePath, 10, iscsiTransport) - if !exist { - return errors.New("Could not attach disk: Timeout after 10s") - } + if len(devicePaths) == 0 { + glog.Errorf("iscsi: failed to get any path for iscsi disk") + return errors.New("failed to get any path for iscsi disk") } + + //Make sure we use a valid devicepath to find mpio device. + devicePath = devicePaths[0] + // mount it globalPDPath := b.manager.MakeGlobalPDName(*b.iscsiDisk) notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath) @@ -153,9 +169,17 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { return err } - // check if the dev is using mpio and if so mount it via the dm-XX device - if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(devicePath); mappedDevicePath != "" { - devicePath = mappedDevicePath + for _, path := range devicePaths { + // There shouldnt be any empty device paths. However adding this check + // for safer side to avoid the possibility of an empty entry. + if path == "" { + continue + } + // check if the dev is using mpio and if so mount it via the dm-XX device + if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" { + devicePath = mappedDevicePath + break + } } err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil) if err != nil {