github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 11:50:44 +00:00
Code Issues Projects Releases Wiki Activity

Add portals field to iscsi volume source to achieve multipathing.

Browse Source 
Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
This commit is contained in:
Humble Chirammal 2017-02-06 17:06:33 +05:30
parent 1df1ad9d34
commit 332e26dc8c
6 changed files with 82 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
examples/volumes/iscsi/README.md
View File

@ -18,7 +18,7 @@ If there isn't one in place then it is possible to setup a software version on L
## Creating the pod with iSCSI persistent storage
Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required.
Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. If you have more than one target portals for a single IQN, you can mention other portal IPs in *portals* field.
If you want to use an iSCSI offload card or other open-iscsi transports besides tcp, setup an iSCSI interface and provide *iscsiInterface* in the pod YAML. The default name for an iscsi iface (open-iscsi parameter iface.iscsi\_ifacename) is in the format transport\_name.hwaddress when generated by iscsiadm. See [open-iscsi](http://www.open-iscsi.org/docs/README) or [openstack](http://docs.openstack.org/kilo/config-reference/content/iscsi-iface-config.html) for detailed configuration information.
@ -48,19 +48,23 @@ For a non mpio device the output should look like the following
```console
# mount |grep kub
/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered)
/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered)
/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered)
/dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
/dev/sdb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (ro,relatime,data=ordered)
/dev/sdc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
/dev/sdc on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
/dev/sdd on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
/dev/sdd on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
```
And for a node with mpio enabled the expected output would be similar to the following
```console
# mount |grep kub
/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered)
/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
/dev/mapper/mpatha on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered)
/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered)
/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.16:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
/dev/mapper/mpathc on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.17:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (rw,relatime,data=ordered)
/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered)
```
@ -70,7 +74,6 @@ If you ssh to that machine, you can run `docker ps` to see the actual pod.
```console
# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
f855336407f4 kubernetes/pause "/pause" 6 minutes ago Up 6 minutes k8s_iscsipd-ro.d130ec3e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_5409a4cb
3b8a772515d2 kubernetes/pause "/pause" 6 minutes ago Up 6 minutes k8s_iscsipd-rw.ed58ec4e_iscsipd_default_f527ca5b-6d87-11e5-aa7e-080027ff6387_d25592c5
```

15
examples/volumes/iscsi/iscsi.yaml
View File

@ -5,28 +5,17 @@ metadata:
name: iscsipd
spec:
containers:
- name: iscsipd-ro
image: kubernetes/pause
volumeMounts:
- mountPath: "/mnt/iscsipd"
name: iscsipd-ro
- name: iscsipd-rw
image: kubernetes/pause
volumeMounts:
- mountPath: "/mnt/iscsipd"
name: iscsipd-rw
volumes:
- name: iscsipd-ro
- name: iscsipd-rw
iscsi:
targetPortal: 10.0.2.15:3260
portals: ['10.0.2.16:3260', '10.0.2.17:3260']
iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz
lun: 0
fsType: ext4
readOnly: true
- name: iscsipd-rw
iscsi:
targetPortal: 10.0.2.15:3260
iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz
lun: 1
fsType: ext4
readOnly: false

4
pkg/api/types.go
View File

@ -623,6 +623,10 @@ type ISCSIVolumeSource struct {
// the ReadOnly setting in VolumeMounts.
// +optional
ReadOnly bool
// Required: list of iSCSI target portal ips for high availability.
// the portal is either an IP or ip_addr:port if port is other than default (typically TCP ports 860 and 3260)
// +optional
Portals []string
}
// Represents a Fibre Channel volume.

4
pkg/api/v1/types.go
View File

@ -988,6 +988,10 @@ type ISCSIVolumeSource struct {
// Defaults to false.
// +optional
ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,6,opt,name=readOnly"`
// iSCSI target portal List. The portal is either an IP or ip_addr:port if the port
// is other than default (typically TCP ports 860 and 3260).
// +optional
Portals []string `json:"portals" protobuf:"bytes,7,opt,name=portals"`
}
// Represents a Fibre Channel volume.

10
pkg/volume/iscsi/iscsi.go
View File

@ -104,14 +104,18 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI
lun := strconv.Itoa(int(iscsi.Lun))
portal := portalMounter(iscsi.TargetPortal)
var bkportal []string
bkportal = append(bkportal, portal)
for _, tp := range iscsi.Portals {
bkportal = append(bkportal, portalMounter(string(tp)))
}
iface := iscsi.ISCSIInterface
return &iscsiDiskMounter{
iscsiDisk: &iscsiDisk{
podUID: podUID,
volName: spec.Name(),
portal: portal,
portals: bkportal,
iqn: iscsi.IQN,
lun: lun,
iface: iface,
@ -162,7 +166,7 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v
type iscsiDisk struct {
volName string
podUID types.UID
portal string
portals []string
iqn string
lun string
iface string

86
pkg/volume/iscsi/iscsi_util.go
View File

@ -98,13 +98,13 @@ func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun s
type ISCSIUtil struct{}
func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
return makePDNameInternal(iscsi.plugin.host, iscsi.portal, iscsi.iqn, iscsi.lun)
return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun)
}
func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
var devicePath string
var devicePaths []string
var iscsiTransport string
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.iface, "-o", "show"})
if err != nil {
glog.Errorf("iscsi: could not read iface %s error: %s", b.iface, string(out))
@ -112,34 +112,50 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
}
iscsiTransport = extractTransportname(string(out))
bkpPortal := b.portals
for _, tp := range bkpPortal {
if iscsiTransport == "" {
glog.Errorf("iscsi: could not find transport name in iface %s", b.iface)
return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface))
} else if iscsiTransport == "tcp" {
devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-")
} else {
devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-")
}
exist := waitForPathToExist(devicePath, 1, iscsiTransport)
if exist == false {
// discover iscsi target
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discovery", "-t", "sendtargets", "-p", tp, "-I", b.iface})
if err != nil {
glog.Errorf("iscsi: failed to sendtargets to portal %s error: %s", tp, string(out))
continue
}
// login to iscsi target
out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-I", b.iface, "--login"})
if err != nil {
glog.Errorf("iscsi: failed to attach disk:Error: %s (%v)", string(out), err)
continue
}
exist = waitForPathToExist(devicePath, 10, iscsiTransport)
if !exist {
glog.Errorf("Could not attach disk: Timeout after 10s")
} else {
devicePaths = append(devicePaths, devicePath)
}
} else {
glog.V(4).Infof("iscsi: devicepath (%s) exists", devicePath)
devicePaths = append(devicePaths, devicePath)
}
}
if iscsiTransport == "" {
glog.Errorf("iscsi: could not find transport name in iface %s", b.iface)
return errors.New(fmt.Sprintf("Could not parse iface file for %s", b.iface))
} else if iscsiTransport == "tcp" {
devicePath = strings.Join([]string{"/dev/disk/by-path/ip", b.portal, "iscsi", b.iqn, "lun", b.lun}, "-")
} else {
devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", b.portal, "iscsi", b.iqn, "lun", b.lun}, "-")
}
exist := waitForPathToExist(devicePath, 1, iscsiTransport)
if exist == false {
// discover iscsi target
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discovery", "-t", "sendtargets", "-p", b.portal, "-I", b.iface})
if err != nil {
glog.Errorf("iscsi: failed to sendtargets to portal %s error: %s", b.portal, string(out))
return err
}
// login to iscsi target
out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", b.portal, "-T", b.iqn, "-I", b.iface, "--login"})
if err != nil {
glog.Errorf("iscsi: failed to attach disk:Error: %s (%v)", string(out), err)
return err
}
exist = waitForPathToExist(devicePath, 10, iscsiTransport)
if !exist {
return errors.New("Could not attach disk: Timeout after 10s")
}
if len(devicePaths) == 0 {
glog.Errorf("iscsi: failed to get any path for iscsi disk")
return errors.New("failed to get any path for iscsi disk")
}
//Make sure we use a valid devicepath to find mpio device.
devicePath = devicePaths[0]
// mount it
globalPDPath := b.manager.MakeGlobalPDName(*b.iscsiDisk)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
@ -153,9 +169,17 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
return err
}
// check if the dev is using mpio and if so mount it via the dm-XX device
if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(devicePath); mappedDevicePath != "" {
devicePath = mappedDevicePath
for _, path := range devicePaths {
// There shouldnt be any empty device paths. However adding this check
// for safer side to avoid the possibility of an empty entry.
if path == "" {
continue
}
// check if the dev is using mpio and if so mount it via the dm-XX device
if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
devicePath = mappedDevicePath
break
}
}
err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil)
if err != nil {