From 1811ded39605dc1fbb085581d6c0d17fec089db3 Mon Sep 17 00:00:00 2001 From: tobad357 Date: Tue, 19 Apr 2016 13:10:00 +0800 Subject: [PATCH] This is an update that allows the iscsi volume to check if a iscsi device belongs to a mpio device If it does belong to the device then we make sure we mount the mpio device instead of the raw device. Heuristics Login into /dev/disk/by-path/iqn-example.com.2999 -> /dev/sde Check if sde existsin in /sys/block/[dm-X]/slaves/xx If it does mount /dev/[dm-x] which will look like /dev/mapper/mpiodevicename in mount examples/iscsi has more details --- examples/iscsi/README.md | 62 ++++++---- examples/iscsi/iscsi.json | 53 -------- examples/iscsi/iscsi.yaml | 32 +++++ pkg/volume/fc/fc_util.go | 2 + pkg/volume/iscsi/iscsi.go | 15 ++- pkg/volume/iscsi/iscsi_util.go | 4 + pkg/volume/util/device_util.go | 31 +++++ pkg/volume/util/device_util_linux.go | 61 +++++++++ pkg/volume/util/device_util_linux_test.go | 136 +++++++++++++++++++++ pkg/volume/util/device_util_unsupported.go | 24 ++++ pkg/volume/util/io_util.go | 48 ++++++++ 11 files changed, 383 insertions(+), 85 deletions(-) delete mode 100644 examples/iscsi/iscsi.json create mode 100644 examples/iscsi/iscsi.yaml create mode 100644 pkg/volume/util/device_util.go create mode 100644 pkg/volume/util/device_util_linux.go create mode 100644 pkg/volume/util/device_util_linux_test.go create mode 100644 pkg/volume/util/device_util_unsupported.go create mode 100644 pkg/volume/util/io_util.go diff --git a/examples/iscsi/README.md b/examples/iscsi/README.md index 4aa4f9e4536..349f11f4c31 100644 --- a/examples/iscsi/README.md +++ b/examples/iscsi/README.md @@ -32,48 +32,45 @@ Documentation for other releases can be found at -## Step 1. Setting up iSCSI target and iSCSI initiator +## Introduction -**Setup A.** On Fedora 21 nodes +The Kubernetes iSCSI implementation can connect to iSCSI devices via open-iscsi and multipathd on Linux. +Currently supported features are + * Connecting to one portal + * Mounting a device directly or via multipathd + * Formatting and partitioning any new device connected -If you use Fedora 21 on Kubernetes node, then first install iSCSI initiator on the node: +## Prerequisites - # yum -y install iscsi-initiator-utils +This example expects there to be a working iSCSI target to connect to. +If there isn't one in place then it is possible to setup a software version on Linux by following these guides -then edit */etc/iscsi/iscsid.conf* to match your iSCSI target configuration. + * [Setup a iSCSI target on Fedora](http://www.server-world.info/en/note?os=Fedora_21&p=iscsi) + * [Install the iSCSI initiator on Fedora](http://www.server-world.info/en/note?os=Fedora_21&p=iscsi&f=2) + * [Install multipathd for mpio support if required](http://www.linuxstories.eu/2014/07/how-to-setup-dm-multipath-on-rhel.html) -I mostly followed these [instructions](http://www.server-world.info/en/note?os=Fedora_21&p=iscsi) to setup iSCSI target. and these [instructions](http://www.server-world.info/en/note?os=Fedora_21&p=iscsi&f=2) to setup iSCSI initiator. -**Setup B.** On Unbuntu 12.04 and Debian 7 nodes on Google Compute Engine (GCE) +## Creating the pod with iSCSI persistent storage -GCE does not provide preconfigured Fedora 21 image, so I set up the iSCSI target on a preconfigured Ubuntu 12.04 image, mostly following these [instructions](http://www.server-world.info/en/note?os=Ubuntu_12.04&p=iscsi). My Kubernetes cluster on GCE was running Debian 7 images, so I followed these [instructions](http://www.server-world.info/en/note?os=Debian_7.0&p=iscsi&f=2) to set up the iSCSI initiator. +Once you have configured the iSCSI initiator, you can create a pod based on the example *iscsi.yaml*. In the pod YAML, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. -## Step 2. Creating the pod with iSCSI persistent storage - -Once you have installed iSCSI initiator and new Kubernetes, you can create a pod based on the example *iscsi.json*. In the pod JSON, you need to provide *targetPortal* (the iSCSI target's **IP** address and *port* if not the default port 3260), target's *iqn*, *lun*, and the type of the filesystem that has been created on the lun, and *readOnly* boolean. No initiator information is required. - -If you want to use an iSCSI offload card or other open-iscsi transports besides tcp, setup an iSCSI interface and provide *iscsiInterface* in the pod JSON. The default name for an iscsi iface (open-iscsi parameter iface.iscsi\_ifacename) is in the format transport\_name.hwaddress when generated by iscsiadm. See [open-iscsi](http://www.open-iscsi.org/docs/README) or [openstack](http://docs.openstack.org/kilo/config-reference/content/iscsi-iface-config.html) for detailed configuration information. +If you want to use an iSCSI offload card or other open-iscsi transports besides tcp, setup an iSCSI interface and provide *iscsiInterface* in the pod YAML. The default name for an iscsi iface (open-iscsi parameter iface.iscsi\_ifacename) is in the format transport\_name.hwaddress when generated by iscsiadm. See [open-iscsi](http://www.open-iscsi.org/docs/README) or [openstack](http://docs.openstack.org/kilo/config-reference/content/iscsi-iface-config.html) for detailed configuration information. **Note:** If you have followed the instructions in the links above you may have partitioned the device, the iSCSI volume plugin does not -currently support partitions so format the device as one partition. -Make sure you have the correct device name then run the following as -root to format it: +currently support partitions so format the device as one partition or leave the device raw and Kubernetes will partition and format it one first mount. + + +Once the pod config is created, run it on the Kubernetes master: ```console -mkfs.ext4 /dev/ +kubectl create -f ./your_new_pod.yaml ``` -Once your pod is created, run it on the Kubernetes master: +Here is the example pod created and expected output: ```console -kubectl create -f ./your_new_pod.json -``` - -Here is my command and output: - -```console -# kubectl create -f examples/iscsi/iscsi.json +# kubectl create -f examples/iscsi/iscsi.yaml # kubectl get pods NAME READY STATUS RESTARTS AGE iscsipd 2/2 RUNNING 0 2m @@ -81,6 +78,8 @@ iscsipd 2/2 RUNNING 0 2m On the Kubernetes node, verify the mount output +For a non mpio device the output should look like the following + ```console # mount |grep kub /dev/sdb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered) @@ -89,6 +88,17 @@ On the Kubernetes node, verify the mount output /dev/sdc on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) ``` +And for a node with mpio enabled the expected output would be similar to the following + +```console +# mount |grep kub +/dev/mapper/mpatha on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-0 type ext4 (ro,relatime,data=ordered) +/dev/mapper/mpatha on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro type ext4 (ro,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/plugins/kubernetes.io/iscsi/10.0.2.15:3260-iqn.2001-04.com.example:storage.kube.sys1.xyz-lun-1 type ext4 (rw,relatime,data=ordered) +/dev/mapper/mpathb on /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-rw type ext4 (rw,relatime,data=ordered) +``` + + If you ssh to that machine, you can run `docker ps` to see the actual pod. ```console @@ -100,7 +110,7 @@ f855336407f4 kubernetes/pause "/pause" 6 Run *docker inspect* and verify the container mounted the host directory into the their */mnt/iscsipd* directory. -```console +```console # docker inspect --format '{{ range .Mounts }}{{ if eq .Destination "/mnt/iscsipd" }}{{ .Source }}{{ end }}{{ end }}' f855336407f4 /var/lib/kubelet/pods/f527ca5b-6d87-11e5-aa7e-080027ff6387/volumes/kubernetes.io~iscsi/iscsipd-ro diff --git a/examples/iscsi/iscsi.json b/examples/iscsi/iscsi.json deleted file mode 100644 index 3c597147f15..00000000000 --- a/examples/iscsi/iscsi.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "name": "iscsipd" - }, - "spec": { - "containers": [ - { - "name": "iscsipd-ro", - "image": "kubernetes/pause", - "volumeMounts": [ - { - "mountPath": "/mnt/iscsipd", - "name": "iscsipd-ro" - } - ] - }, - { - "name": "iscsipd-rw", - "image": "kubernetes/pause", - "volumeMounts": [ - { - "mountPath": "/mnt/iscsipd", - "name": "iscsipd-rw" - } - ] - } - ], - "volumes": [ - { - "name": "iscsipd-ro", - "iscsi": { - "targetPortal": "10.0.2.15:3260", - "iqn": "iqn.2001-04.com.example:storage.kube.sys1.xyz", - "lun": 0, - "fsType": "ext4", - "readOnly": true - } - }, - { - "name": "iscsipd-rw", - "iscsi": { - "targetPortal": "10.0.2.15:3260", - "iqn": "iqn.2001-04.com.example:storage.kube.sys1.xyz", - "lun": 1, - "fsType": "ext4", - "readOnly": false - } - } - ] - } -} diff --git a/examples/iscsi/iscsi.yaml b/examples/iscsi/iscsi.yaml new file mode 100644 index 00000000000..4c951f6dc11 --- /dev/null +++ b/examples/iscsi/iscsi.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: iscsipd +spec: + containers: + - name: iscsipd-ro + image: kubernetes/pause + volumeMounts: + - mountPath: "/mnt/iscsipd" + name: iscsipd-ro + - name: iscsipd-rw + image: kubernetes/pause + volumeMounts: + - mountPath: "/mnt/iscsipd" + name: iscsipd-rw + volumes: + - name: iscsipd-ro + iscsi: + targetPortal: 10.0.2.15:3260 + iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz + lun: 0 + fsType: ext4 + readOnly: true + - name: iscsipd-rw + iscsi: + targetPortal: 10.0.2.15:3260 + iqn: iqn.2001-04.com.example:storage.kube.sys1.xyz + lun: 1 + fsType: ext4 + readOnly: false diff --git a/pkg/volume/fc/fc_util.go b/pkg/volume/fc/fc_util.go index 2a36610a936..3da53966874 100644 --- a/pkg/volume/fc/fc_util.go +++ b/pkg/volume/fc/fc_util.go @@ -51,6 +51,8 @@ func (handler *osIOHandler) WriteFile(filename string, data []byte, perm os.File } // given a disk path like /dev/sdx, find the devicemapper parent +// TODO #23192 Convert this code to use the generic code in ../util +// which is used by the iSCSI implementation func findMultipathDeviceMapper(disk string, io ioHandler) string { sys_path := "/sys/block/" if dirs, err := io.ReadDir(sys_path); err == nil { diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 2f3c52bb521..a778bab1b65 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" + ioutil "k8s.io/kubernetes/pkg/volume/util" ) // This is the primary entrypoint for volume plugins. @@ -103,9 +104,10 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI iface: iface, manager: manager, plugin: plugin}, - fsType: iscsi.FSType, - readOnly: readOnly, - mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + fsType: iscsi.FSType, + readOnly: readOnly, + mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()), }, nil } @@ -152,9 +154,10 @@ func (iscsi *iscsiDisk) GetPath() string { type iscsiDiskMounter struct { *iscsiDisk - readOnly bool - fsType string - mounter *mount.SafeFormatAndMount + readOnly bool + fsType string + mounter *mount.SafeFormatAndMount + deviceUtil ioutil.DeviceUtil } var _ volume.Mounter = &iscsiDiskMounter{} diff --git a/pkg/volume/iscsi/iscsi_util.go b/pkg/volume/iscsi/iscsi_util.go index 6cbb86ba3b9..c9c92beb404 100644 --- a/pkg/volume/iscsi/iscsi_util.go +++ b/pkg/volume/iscsi/iscsi_util.go @@ -136,6 +136,10 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { return err } + // check if the dev is using mpio and if so mount it via the dm-XX device + if mappedDevicePath := b.deviceUtil.FindMultipathDeviceForDevice(devicePath); mappedDevicePath != "" { + devicePath = mappedDevicePath + } err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil) if err != nil { glog.Errorf("iscsi: failed to mount iscsi volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err) diff --git a/pkg/volume/util/device_util.go b/pkg/volume/util/device_util.go new file mode 100644 index 00000000000..5f18806158f --- /dev/null +++ b/pkg/volume/util/device_util.go @@ -0,0 +1,31 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +//DeviceUtil is a util for common device methods +type DeviceUtil interface { + FindMultipathDeviceForDevice(disk string) string +} + +type deviceHandler struct { + get_io IoUtil +} + +//NewDeviceHandler Create a new IoHandler implementation +func NewDeviceHandler(io IoUtil) DeviceUtil { + return &deviceHandler{get_io: io} +} diff --git a/pkg/volume/util/device_util_linux.go b/pkg/volume/util/device_util_linux.go new file mode 100644 index 00000000000..8933087cc6c --- /dev/null +++ b/pkg/volume/util/device_util_linux.go @@ -0,0 +1,61 @@ +// +build linux + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "errors" + "strings" +) + +// FindMultipathDeviceForDevice given a device name like /dev/sdx, find the devicemapper parent +func (handler *deviceHandler) FindMultipathDeviceForDevice(device string) string { + io := handler.get_io + disk, err := findDeviceForPath(device, io) + if err != nil { + return "" + } + sysPath := "/sys/block/" + if dirs, err := io.ReadDir(sysPath); err == nil { + for _, f := range dirs { + name := f.Name() + if strings.HasPrefix(name, "dm-") { + if _, err1 := io.Lstat(sysPath + name + "/slaves/" + disk); err1 == nil { + return "/dev/" + name + } + } + } + } + return "" +} + +// findDeviceForPath Find the underlaying disk for a linked path such as /dev/disk/by-path/XXXX or /dev/mapper/XXXX +// will return sdX or hdX etc, if /dev/sdX is passed in then sdX will be returned +func findDeviceForPath(path string, io IoUtil) (string, error) { + devicePath, err := io.EvalSymlinks(path) + if err != nil { + return "", err + } + // if path /dev/hdX split into "", "dev", "hdX" then we will + // return just the last part + parts := strings.Split(devicePath, "/") + if len(parts) == 3 && strings.HasPrefix(parts[1], "dev") { + return parts[2], nil + } + return "", errors.New("Illegal path for device " + devicePath) +} diff --git a/pkg/volume/util/device_util_linux_test.go b/pkg/volume/util/device_util_linux_test.go new file mode 100644 index 00000000000..83bc8f1a71e --- /dev/null +++ b/pkg/volume/util/device_util_linux_test.go @@ -0,0 +1,136 @@ +// +build linux + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "errors" + "os" + "testing" + "time" +) + +type mockOsIOHandler struct{} + +func (handler *mockOsIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) { + switch dirname { + case "/sys/block/dm-2/slaves/": + f := &fakeFileInfo{ + name: "sda", + } + return []os.FileInfo{f}, nil + case "/sys/block/": + f1 := &fakeFileInfo{ + name: "sda", + } + f2 := &fakeFileInfo{ + name: "dm-1", + } + return []os.FileInfo{f1, f2}, nil + } + return nil, nil +} + +func (handler *mockOsIOHandler) Lstat(name string) (os.FileInfo, error) { + links := map[string]string{ + "/sys/block/dm-1/slaves/sda": "sda", + "/dev/sda": "sda", + } + if dev, ok := links[name]; ok { + return &fakeFileInfo{name: dev}, nil + } + return nil, errors.New("Not Implemented for Mock") +} + +func (handler *mockOsIOHandler) EvalSymlinks(path string) (string, error) { + links := map[string]string{ + "/returns/a/dev": "/dev/sde", + "/returns/non/dev": "/sys/block", + "/dev/disk/by-path/127.0.0.1:3260-eui.02004567A425678D-lun-0": "/dev/sda", + "/dev/dm-2": "/dev/dm-2", + "/dev/dm-3": "/dev/dm-3", + "/dev/sde": "/dev/sde", + } + return links[path], nil +} + +func (handler *mockOsIOHandler) WriteFile(filename string, data []byte, perm os.FileMode) error { + return errors.New("Not Implemented for Mock") +} + +type fakeFileInfo struct { + name string +} + +func (fi *fakeFileInfo) Name() string { + return fi.name +} + +func (fi *fakeFileInfo) Size() int64 { + return 0 +} + +func (fi *fakeFileInfo) Mode() os.FileMode { + return 777 +} + +func (fi *fakeFileInfo) ModTime() time.Time { + return time.Now() +} +func (fi *fakeFileInfo) IsDir() bool { + return false +} + +func (fi *fakeFileInfo) Sys() interface{} { + return nil +} + +func TestFindMultipathDeviceForDevice(t *testing.T) { + mockDeviceUtil := NewDeviceHandler(&mockOsIOHandler{}) + dev := mockDeviceUtil.FindMultipathDeviceForDevice("/dev/disk/by-path/127.0.0.1:3260-eui.02004567A425678D-lun-0") + if dev != "/dev/dm-1" { + t.Fatalf("mpio device not found dm-1 expected got [%s]", dev) + } + dev = mockDeviceUtil.FindMultipathDeviceForDevice("/dev/disk/by-path/empty") + if dev != "" { + t.Fatalf("mpio device not found '' expected got [%s]", dev) + } +} + +func TestfindDeviceForPath(t *testing.T) { + io := &mockOsIOHandler{} + + disk, err := findDeviceForPath("/dev/sde", io) + if disk != "sde" { + t.Fatalf("disk [%s] didn't match expected sde", disk) + } + disk, err = findDeviceForPath("/returns/a/dev", io) + if disk != "sde" { + t.Fatalf("disk [%s] didn't match expected sde", disk) + } + _, err = findDeviceForPath("/returns/non/dev", io) + if err == nil { + t.Fatalf("link is to incorrect dev") + } + + _, err = findDeviceForPath("/path/doesnt/exist", &osIOHandler{}) + if err == nil { + t.Fatalf("path shouldn't exist but still doesn't give an error") + } + +} diff --git a/pkg/volume/util/device_util_unsupported.go b/pkg/volume/util/device_util_unsupported.go new file mode 100644 index 00000000000..dbc03874a5a --- /dev/null +++ b/pkg/volume/util/device_util_unsupported.go @@ -0,0 +1,24 @@ +// +build !linux + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +// FindMultipathDeviceForDevice unsupported returns "" +func (handler *deviceHandler) FindMultipathDeviceForDevice(device string) string { + return "" +} diff --git a/pkg/volume/util/io_util.go b/pkg/volume/util/io_util.go new file mode 100644 index 00000000000..95acbe94109 --- /dev/null +++ b/pkg/volume/util/io_util.go @@ -0,0 +1,48 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +//IoUtil is a util for common IO operations +//it also backports certain operations from golang 1.5 +type IoUtil interface { + ReadDir(dirname string) ([]os.FileInfo, error) + Lstat(name string) (os.FileInfo, error) + EvalSymlinks(path string) (string, error) +} + +type osIOHandler struct{} + +//NewIOHandler Create a new IoHandler implementation +func NewIOHandler() IoUtil { + return &osIOHandler{} +} + +func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) { + return ioutil.ReadDir(dirname) +} +func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) { + return os.Lstat(name) +} +func (handler *osIOHandler) EvalSymlinks(path string) (string, error) { + return filepath.EvalSymlinks(path) +}