/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package nvmf import ( b64 "encoding/base64" "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" "strings" "github.com/kubernetes-csi/csi-driver-nvmf/pkg/utils" "k8s.io/klog/v2" ) type Connector struct { VolumeID string DeviceUUID string TargetNqn string TargetAddr string TargetPort string Transport string HostNqn string RetryCount int32 CheckInterval int32 } func getNvmfConnector(nvmfInfo *nvmfDiskInfo, hostnqn string) *Connector { return &Connector{ VolumeID: nvmfInfo.VolName, DeviceUUID: nvmfInfo.DeviceUUID, TargetNqn: nvmfInfo.Nqn, TargetAddr: nvmfInfo.Addr, TargetPort: nvmfInfo.Port, Transport: nvmfInfo.Transport, HostNqn: hostnqn, } } // connector provides a struct to hold all of the needed parameters to make nvmf connection func _connect(argStr string) error { file, err := os.OpenFile("/dev/nvme-fabrics", os.O_RDWR, 0666) if err != nil { klog.Errorf("Connect: open NVMf fabrics error: %v", err) return err } defer file.Close() err = utils.WriteStringToFile(file, argStr) if err != nil { klog.Errorf("Connect: write arg to connect file error: %v", err) return err } // todo: read file to verify lines, err := utils.ReadLinesFromFile(file) klog.Infof("Connect: read string %s", lines) return nil } func _disconnect(sysfs_path string) error { file, err := os.OpenFile(sysfs_path, os.O_WRONLY, 0755) if err != nil { return err } err = utils.WriteStringToFile(file, "1") if err != nil { klog.Errorf("Disconnect: write 1 to delete_controller error: %v", err) return err } return nil } func disconnectSubsysWithHostNqn(nqn, hostnqn, ctrl string) error { sysfs_subsysnqn_path := fmt.Sprintf("%s/%s/subsysnqn", SYS_NVMF, ctrl) sysfs_hostnqn_path := fmt.Sprintf("%s/%s/hostnqn", SYS_NVMF, ctrl) sysfs_del_path := fmt.Sprintf("%s/%s/delete_controller", SYS_NVMF, ctrl) file, err := os.Open(sysfs_subsysnqn_path) if err != nil { klog.Errorf("Disconnect: open file %s err: %v", sysfs_subsysnqn_path, err) return &NoControllerError{Nqn: nqn, Hostnqn: hostnqn} } defer file.Close() lines, err := utils.ReadLinesFromFile(file) if err != nil { klog.Errorf("Disconnect: read file %s err: %v", file.Name(), err) return &NoControllerError{Nqn: nqn, Hostnqn: hostnqn} } if lines[0] != nqn { klog.Warningf("Disconnect: not this subsystem, skip") return &NoControllerError{Nqn: nqn, Hostnqn: hostnqn} } file, err = os.Open(sysfs_hostnqn_path) if err != nil { klog.Errorf("Disconnect: open file %s err: %v", sysfs_hostnqn_path, err) return &UnsupportedHostnqnError{Target: sysfs_hostnqn_path} } defer file.Close() lines, err = utils.ReadLinesFromFile(file) if err != nil { klog.Errorf("Disconnect: read file %s err: %v", file.Name(), err) return &NoControllerError{Nqn: nqn, Hostnqn: hostnqn} } if lines[0] != hostnqn { klog.Warningf("Disconnect: not this subsystem, skip") return &NoControllerError{Nqn: nqn, Hostnqn: hostnqn} } err = _disconnect(sysfs_del_path) if err != nil { klog.Errorf("Disconnect: disconnect error: %s", err) return &NoControllerError{Nqn: nqn, Hostnqn: hostnqn} } return nil } func disconnectSubsys(nqn, ctrl string) error { sysfs_subsysnqn_path := fmt.Sprintf("%s/%s/subsysnqn", SYS_NVMF, ctrl) sysfs_del_path := fmt.Sprintf("%s/%s/delete_controller", SYS_NVMF, ctrl) file, err := os.Open(sysfs_subsysnqn_path) if err != nil { klog.Errorf("Disconnect: open file %s err: %v", sysfs_subsysnqn_path, err) return &NoControllerError{Nqn: nqn, Hostnqn: ""} } defer file.Close() lines, err := utils.ReadLinesFromFile(file) if err != nil { klog.Errorf("Disconnect: read file %s err: %v", file.Name(), err) return &NoControllerError{Nqn: nqn, Hostnqn: ""} } if lines[0] != nqn { klog.Warningf("Disconnect: not this subsystem, skip") return &NoControllerError{Nqn: nqn, Hostnqn: ""} } err = _disconnect(sysfs_del_path) if err != nil { klog.Errorf("Disconnect: disconnect error: %s", err) return &NoControllerError{Nqn: nqn, Hostnqn: ""} } return nil } func disconnectByNqn(nqn, hostnqn string) int { ret := 0 if len(nqn) > NVMF_NQN_SIZE { klog.Errorf("Disconnect: nqn %s is too long ", nqn) return -EINVAL } // delete hostnqn file hostnqnPath := filepath.Join(RUN_NVMF, nqn, b64.StdEncoding.EncodeToString([]byte(hostnqn))) os.Remove(hostnqnPath) // delete nqn directory if has no hostnqn files nqnPath := filepath.Join(RUN_NVMF, nqn) hostnqns, err := ioutil.ReadDir(nqnPath) if err != nil { klog.Errorf("Disconnect: readdir %s err: %v", nqnPath, err) return -ENOENT } if len(hostnqns) <= 0 { os.RemoveAll(nqnPath) } devices, err := ioutil.ReadDir(SYS_NVMF) if err != nil { klog.Errorf("Disconnect: readdir %s err: %s", SYS_NVMF, err) return -ENOENT } for _, device := range devices { if err := disconnectSubsysWithHostNqn(nqn, hostnqn, device.Name()); err != nil { if _, ok := err.(*UnsupportedHostnqnError); ok { klog.Infof("Fallback because you have no hostnqn supports!") // disconnect all controllers if has no hostnqn files if len(hostnqns) <= 0 { devices, err := ioutil.ReadDir(SYS_NVMF) if err != nil { klog.Errorf("Disconnect: readdir %s err: %s", SYS_NVMF, err) return -ENOENT } for _, device := range devices { if err := disconnectSubsys(nqn, device.Name()); err == nil { ret++ } } } return ret } } else { ret++ } } return ret } // connect to volume to this node and return devicePath func (c *Connector) Connect() (string, error) { if c.RetryCount == 0 { c.RetryCount = 10 } if c.CheckInterval == 0 { c.CheckInterval = 1 } if c.RetryCount < 0 || c.CheckInterval < 0 { return "", fmt.Errorf("Invalid RetryCount and CheckInterval combinaitons "+ "RetryCount: %d, CheckInterval: %d ", c.RetryCount, c.CheckInterval) } if strings.ToLower(c.Transport) != "tcp" && strings.ToLower(c.Transport) != "rdma" { return "", fmt.Errorf("csi transport only support tcp/rdma ") } baseString := fmt.Sprintf("nqn=%s,transport=%s,traddr=%s,trsvcid=%s,hostnqn=%s", c.TargetNqn, c.Transport, c.TargetAddr, c.TargetPort, c.HostNqn) devicePath := strings.Join([]string{"/dev/disk/by-id/nvme-uuid", c.DeviceUUID}, ".") // connect to nvmf disk err := _connect(baseString) if err != nil { return "", err } klog.Infof("Connect Volume %s success nqn: %s, hostnqn: %s", c.VolumeID, c.TargetNqn, c.HostNqn) retries := int(c.RetryCount / c.CheckInterval) if exists, err := waitForPathToExist(devicePath, retries, int(c.CheckInterval), c.Transport); !exists { klog.Errorf("connect nqn %s error %v, rollback!!!", c.TargetNqn, err) ret := disconnectByNqn(c.TargetNqn, c.HostNqn) if ret < 0 { klog.Errorf("rollback error !!!") } return "", err } // create nqn directory nqnPath := filepath.Join(RUN_NVMF, c.TargetNqn) if err := os.MkdirAll(nqnPath, 0750); err != nil { klog.Errorf("create nqn directory %s error %v, rollback!!!", c.TargetNqn, err) ret := disconnectByNqn(c.TargetNqn, c.HostNqn) if ret < 0 { klog.Errorf("rollback error !!!") } return "", err } // create hostnqn file hostnqnPath := filepath.Join(RUN_NVMF, c.TargetNqn, b64.StdEncoding.EncodeToString([]byte(c.HostNqn))) file, err := os.Create(hostnqnPath) if err != nil { klog.Errorf("create hostnqn file %s:%s error %v, rollback!!!", c.TargetNqn, c.HostNqn, err) ret := disconnectByNqn(c.TargetNqn, c.HostNqn) if ret < 0 { klog.Errorf("rollback error !!!") } return "", err } defer file.Close() klog.Infof("After connect we're returning devicePath: %s", devicePath) return devicePath, nil } // we disconnect only by nqn func (c *Connector) Disconnect() error { ret := disconnectByNqn(c.TargetNqn, c.HostNqn) if ret < 0 { return fmt.Errorf("Disconnect: failed to disconnect by nqn: %s ", c.TargetNqn) } return nil } // PersistConnector persists the provided Connector to the specified file (ie /var/lib/pfile/myConnector.json) func persistConnectorFile(c *Connector, filePath string) error { f, err := os.Create(filePath) if err != nil { return fmt.Errorf("error creating nvmf persistence file %s: %s", filePath, err) } defer f.Close() encoder := json.NewEncoder(f) if err = encoder.Encode(c); err != nil { return fmt.Errorf("error encoding connector: %v", err) } return nil } func removeConnectorFile(targetPath string) { // todo: here maybe be attack for os.Remove can operate any file, fix? if err := os.Remove(targetPath + ".json"); err != nil { klog.Errorf("DetachDisk: Can't remove connector file: %s", targetPath) } if err := os.RemoveAll(targetPath); err != nil { klog.Errorf("DetachDisk: failed to remove mount path Error: %v", err) } } func GetConnectorFromFile(filePath string) (*Connector, error) { f, err := ioutil.ReadFile(filePath) if err != nil { return &Connector{}, err } data := Connector{} err = json.Unmarshal([]byte(f), &data) if err != nil { return &Connector{}, err } return &data, nil }