github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-03 01:06:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #46239 from mtanino/issue/45394

Browse Source 
Automatic merge from submit-queue

Log out from multiple target portals when using iscsi storage plugin

**What this PR does / why we need it**:

When using iscsi storage with multiple target portal (TP) addresses
and multipathing the volume manager logs on to the IQN for all
portal addresses, but when a pod gets destroyed the volume manager
only logs out for the primary TP and sessions for another TPs are
always remained.

This patch adds mount points for all TPs, and then log out from all
TPs when a pod is destroyed. If a TP is referred from another pods,
the connection will be remained as usual.



**Which issue this PR fixes** 
fixes #45394

**Special notes for your reviewer**:

**Release note**:

```
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2017-06-02 23:27:14 -07:00 committed by GitHub
commit fcf183dcaa
3 changed files with 109 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/volume/iscsi/iscsi.go
View File

@ -136,10 +136,10 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI
iscsiDisk: &iscsiDisk{ iscsiDisk: &iscsiDisk{
podUID: podUID, podUID: podUID,
volName: spec.Name(), volName: spec.Name(),
portals: bkportal, Portals: bkportal,
iqn: iscsi.IQN, Iqn: iscsi.IQN,
lun: lun, lun: lun,
iface: iface, Iface: iface,
chap_discovery: iscsi.DiscoveryCHAPAuth, chap_discovery: iscsi.DiscoveryCHAPAuth,
chap_session: iscsi.SessionCHAPAuth, chap_session: iscsi.SessionCHAPAuth,
secret: secret, secret: secret,
@ -191,10 +191,10 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v
type iscsiDisk struct { type iscsiDisk struct {
volName string volName string
podUID types.UID podUID types.UID
portals []string Portals []string
iqn string Iqn string
lun string lun string
iface string Iface string
chap_discovery bool chap_discovery bool
chap_session bool chap_session bool
secret map[string]string secret map[string]string

136
pkg/volume/iscsi/iscsi_util.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package iscsi package iscsi
import ( import (
"encoding/json"
"fmt" "fmt"
"os" "os"
"path" "path"
@ -45,7 +46,7 @@ var (
func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error { func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error {
if b.chap_discovery { if b.chap_discovery {
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP"}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP"})
if err != nil { if err != nil {
return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", string(out)) return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", string(out))
} }
@ -53,7 +54,7 @@ func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error {
for _, k := range chap_st { for _, k := range chap_st {
v := b.secret[k] v := b.secret[k]
if len(v) > 0 { if len(v) > 0 {
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.iface, "-o", "update", "-n", k, "-v", v}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", k, "-v", v})
if err != nil { if err != nil {
return fmt.Errorf("iscsi: failed to update discoverydb key %q with value %q error: %v", k, v, string(out)) return fmt.Errorf("iscsi: failed to update discoverydb key %q with value %q error: %v", k, v, string(out))
} }
@ -65,7 +66,7 @@ func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error {
func updateISCSINode(b iscsiDiskMounter, tp string) error { func updateISCSINode(b iscsiDiskMounter, tp string) error {
if b.chap_session { if b.chap_session {
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-I", b.iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP"}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP"})
if err != nil { if err != nil {
return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", string(out)) return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", string(out))
} }
@ -73,7 +74,7 @@ func updateISCSINode(b iscsiDiskMounter, tp string) error {
for _, k := range chap_sess { for _, k := range chap_sess {
v := b.secret[k] v := b.secret[k]
if len(v) > 0 { if len(v) > 0 {
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-I", b.iface, "-o", "update", "-n", k, "-v", v}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", k, "-v", v})
if err != nil { if err != nil {
return fmt.Errorf("iscsi: failed to update node session key %q with value %q error: %v", k, v, string(out)) return fmt.Errorf("iscsi: failed to update node session key %q with value %q error: %v", k, v, string(out))
} }
@ -150,7 +151,36 @@ func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun s
type ISCSIUtil struct{} type ISCSIUtil struct{}
func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string { func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
return makePDNameInternal(iscsi.plugin.host, iscsi.portals[0], iscsi.iqn, iscsi.lun, iscsi.iface) return makePDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.lun, iscsi.Iface)
}
func (util *ISCSIUtil) persistISCSI(conf iscsiDisk, mnt string) error {
file := path.Join(mnt, "iscsi.json")
fp, err := os.Create(file)
if err != nil {
return fmt.Errorf("iscsi: create %s err %s", file, err)
}
defer fp.Close()
encoder := json.NewEncoder(fp)
if err = encoder.Encode(conf); err != nil {
return fmt.Errorf("iscsi: encode err: %v.", err)
}
return nil
}
func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error {
// NOTE: The iscsi config json is not deleted after logging out from target portals.
file := path.Join(mnt, "iscsi.json")
fp, err := os.Open(file)
if err != nil {
return fmt.Errorf("iscsi: open %s err %s", file, err)
}
defer fp.Close()
decoder := json.NewDecoder(fp)
if err = decoder.Decode(conf); err != nil {
return fmt.Errorf("iscsi: decode err: %v.", err)
}
return nil
} }
func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
@ -159,45 +189,45 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
var iscsiTransport string var iscsiTransport string
var lastErr error var lastErr error
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.iface, "-o", "show"}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.Iface, "-o", "show"})
if err != nil { if err != nil {
glog.Errorf("iscsi: could not read iface %s error: %s", b.iface, string(out)) glog.Errorf("iscsi: could not read iface %s error: %s", b.Iface, string(out))
return err return err
} }
iscsiTransport = extractTransportname(string(out)) iscsiTransport = extractTransportname(string(out))
bkpPortal := b.portals bkpPortal := b.Portals
for _, tp := range bkpPortal { for _, tp := range bkpPortal {
// Rescan sessions to discover newly mapped LUNs. Do not specify the interface when rescanning // Rescan sessions to discover newly mapped LUNs. Do not specify the interface when rescanning
// to avoid establishing additional sessions to the same target. // to avoid establishing additional sessions to the same target.
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-R"}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-R"})
if err != nil { if err != nil {
glog.Errorf("iscsi: failed to rescan session with error: %s (%v)", string(out), err) glog.Errorf("iscsi: failed to rescan session with error: %s (%v)", string(out), err)
} }
if iscsiTransport == "" { if iscsiTransport == "" {
glog.Errorf("iscsi: could not find transport name in iface %s", b.iface) glog.Errorf("iscsi: could not find transport name in iface %s", b.Iface)
return fmt.Errorf("Could not parse iface file for %s", b.iface) return fmt.Errorf("Could not parse iface file for %s", b.Iface)
} else if iscsiTransport == "tcp" { } else if iscsiTransport == "tcp" {
devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-") devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.Iqn, "lun", b.lun}, "-")
} else { } else {
devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.iqn, "lun", b.lun}, "-") devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.Iqn, "lun", b.lun}, "-")
} }
exist := waitForPathToExist(devicePath, 1, iscsiTransport) exist := waitForPathToExist(devicePath, 1, iscsiTransport)
if exist == false { if exist == false {
// build discoverydb and discover iscsi target // build discoverydb and discover iscsi target
b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.iface, "-o", "new"}) b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new"})
// update discoverydb with CHAP secret // update discoverydb with CHAP secret
err = updateISCSIDiscoverydb(b, tp) err = updateISCSIDiscoverydb(b, tp)
if err != nil { if err != nil {
lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err) lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err)
continue continue
} }
out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.iface, "--discover"}) out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "--discover"})
if err != nil { if err != nil {
// delete discoverydb record // delete discoverydb record
b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.iface, "-o", "delete"}) b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "delete"})
lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, string(out), err) lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, string(out), err)
continue continue
} }
@ -208,10 +238,10 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
continue continue
} }
// login to iscsi target // login to iscsi target
out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.iqn, "-I", b.iface, "--login"}) out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "--login"})
if err != nil { if err != nil {
// delete the node record from database // delete the node record from database
b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-I", b.iface, "-T", b.iqn, "-o", "delete"}) b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-I", b.Iface, "-T", b.Iqn, "-o", "delete"})
lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", string(out), err) lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", string(out), err)
continue continue
} }
@ -251,6 +281,9 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error {
return err return err
} }
// Persist iscsi disk config to json file for DetachDisk path
util.persistISCSI(*(b.iscsiDisk), globalPDPath)
for _, path := range devicePaths { for _, path := range devicePaths {
// There shouldnt be any empty device paths. However adding this check // There shouldnt be any empty device paths. However adding this check
// for safer side to avoid the possibility of an empty entry. // for safer side to avoid the possibility of an empty entry.
@ -290,37 +323,41 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
} }
refCount, err := getDevicePrefixRefCount(c.mounter, prefix) refCount, err := getDevicePrefixRefCount(c.mounter, prefix)
if err == nil && refCount == 0 { if err == nil && refCount == 0 {
// This portal/iqn/iface is no longer referenced, log out. var bkpPortal []string
// Extract the portal and iqn from device path. var iqn, iface string
portal, iqn, err := extractPortalAndIqn(device) found := true
if err != nil {
return err // load iscsi disk config from json file
} if err := util.loadISCSI(c.iscsiDisk, mntPath); err == nil {
// Extract the iface from the mountPath and use it to log out. If the iface bkpPortal, iqn, iface = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Iface
// is not found, maintain the previous behavior to facilitate kubelet upgrade.
// Logout may fail as no session may exist for the portal/IQN on the specified interface.
iface, found := extractIface(mntPath)
if found {
glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "-I", iface, "--logout"})
if err != nil {
glog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
}
// Delete the node record
glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn)
out, err = c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "-I", iface, "-o", "delete"})
if err != nil {
glog.Errorf("iscsi: failed to delete node record Error: %s", string(out))
}
} else { } else {
glog.Infof("iscsi: log out target %s iqn %s", portal, iqn) // If the iscsi disk config is not found, fall back to the original behavior.
out, err := c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}) // This portal/iqn/iface is no longer referenced, log out.
// Extract the portal and iqn from device path.
bkpPortal[0], iqn, err = extractPortalAndIqn(device)
if err != nil {
return err
}
// Extract the iface from the mountPath and use it to log out. If the iface
// is not found, maintain the previous behavior to facilitate kubelet upgrade.
// Logout may fail as no session may exist for the portal/IQN on the specified interface.
iface, found = extractIface(mntPath)
}
for _, portal := range removeDuplicate(bkpPortal) {
logout := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}
delete := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"}
if found {
logout = append(logout, []string{"-I", iface}...)
delete = append(delete, []string{"-I", iface}...)
}
glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
out, err := c.plugin.execCommand("iscsiadm", logout)
if err != nil { if err != nil {
glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) glog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
} }
// Delete the node record // Delete the node record
glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn) glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn)
out, err = c.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"}) out, err = c.plugin.execCommand("iscsiadm", delete)
if err != nil { if err != nil {
glog.Errorf("iscsi: failed to delete node record Error: %s", string(out)) glog.Errorf("iscsi: failed to delete node record Error: %s", string(out))
} }
@ -390,3 +427,16 @@ func extractPortalAndIqn(device string) (string, string, error) {
iqn := device[ind2:ind] iqn := device[ind2:ind]
return portal, iqn, nil return portal, iqn, nil
} }
// Remove duplicates or string
func removeDuplicate(s []string) []string {
m := map[string]bool{}
for _, v := range s {
if v != "" && !m[v] {
s[len(m)] = v
m[v] = true
}
}
s = s[:len(m)]
return s
}

10
pkg/volume/iscsi/iscsi_util_test.go
View File

@ -19,6 +19,7 @@ package iscsi
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"testing" "testing"
"k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/mount"
@ -91,6 +92,15 @@ func TestExtractPortalAndIqn(t *testing.T) {
} }
} }
func TestRemoveDuplicate(t *testing.T) {
dupPortals := []string{"127.0.0.1:3260", "127.0.0.1:3260", "127.0.0.100:3260"}
portals := removeDuplicate(dupPortals)
want := []string{"127.0.0.1:3260", "127.0.0.100:3260"}
if reflect.DeepEqual(portals, want) == false {
t.Errorf("removeDuplicate: want: %s, got: %s", want, portals)
}
}
func fakeOsStat(devicePath string) (fi os.FileInfo, err error) { func fakeOsStat(devicePath string) (fi os.FileInfo, err error) {
var cmd os.FileInfo var cmd os.FileInfo
return cmd, nil return cmd, nil