diff --git a/cmd/nvmfplugin/nvmfplugin.go b/cmd/nvmfplugin/nvmfplugin.go index ea36dce..0b15f1e 100644 --- a/cmd/nvmfplugin/nvmfplugin.go +++ b/cmd/nvmfplugin/nvmfplugin.go @@ -38,7 +38,9 @@ func init() { flag.StringVar(&conf.DriverName, "drivername", nvmf.DefaultDriverName, "CSI Driver") flag.StringVar(&conf.Region, "region", "test_region", "Region") flag.StringVar(&conf.Version, "version", nvmf.DefaultDriverVersion, "Version") + flag.StringVar(&conf.NVMfVolumeMapDir, "nvmfVolumeMapDir", nvmf.DefaultVolumeMapPath, "Persistent volume") + flag.StringVar(&conf.NVMfBackendEndpoint, "nvmfBackendEndpoint", nvmf.DefaultBackendEndpoint, "NVMf Volume backend controller") } func main() { diff --git a/pkg/client/createVolumeResponse.go b/pkg/client/createVolumeResponse.go new file mode 100644 index 0000000..243de2f --- /dev/null +++ b/pkg/client/createVolumeResponse.go @@ -0,0 +1,55 @@ +package client + +type CreateVolumeResponse struct { + ID string `json:"ID,omitempty"` + CapacityBytes uint64 `json:"capacityBytes,omitempty"` + TargetType string `json:"targetType,omitempty"` + TargetConfig *TargetConfig `json:"targetConfig,omitempty"` +} + +type TargetConfig struct { + Hosts []*Host `json:"hosts,omitempty"` + Ports []*Port `json:"ports,omitempty"` + Subsystems []*Subsystem `json:"subsystems,omitempty"` +} + +type Host struct { + NQN string `json:"NQN,omitempty"` +} + +type Port struct { + Addr *Addr `json:"addr,omitempty"` + PortID uint64 `json:"portID,omitempty"` + Subsystems []string `json:"subsystems,omitempty"` +} + +type Addr struct { + AdrFam string `json:"adrFam,omitempty"` + TrAddr string `json:"trAddr,omitempty"` + TrSvcID string `json:"trSvcID,omitempty"` + TrType string `json:"trType,omitempty"` +} + +type Subsystem struct { + AllowedHosts []string `json:"AllowedHosts,omitempty"` + Attr *Attr `json:"attr,omitempty"` + Namespaces []*Namespace `json:"namespaces,omitempty"` + NQN string `json:"NQN,omitempty"` +} + +type Attr struct { + AllowAnyHost string `json:"allowAnyHost,omitempty"` + Serial string `json:"serial,omitempty"` +} + +type Namespace struct { + Device *Device `json:"device,omitempty"` + Enable uint32 `json:"enable,omitempty"` + NSID uint64 `json:"NSID,omitempty"` +} + +type Device struct { + NGUID string `json:"NGUID,omitempty"` + Path string `json:"path,omitempty"` + UUID string `json:"UUID,omitempty"` +} diff --git a/pkg/client/response.go b/pkg/client/response.go index e4d3d29..513a180 100644 --- a/pkg/client/response.go +++ b/pkg/client/response.go @@ -1,6 +1,11 @@ package client -import "net/http" +import ( + "encoding/json" + "net/http" + + "k8s.io/klog/v2" +) type Response struct { statusCode int @@ -8,3 +13,23 @@ type Response struct { body []byte err error } + +func (r *Response) StatusCode() int { + return r.statusCode +} + +func (r *Response) Body() []byte { + return r.body +} + +func (r *Response) Err() error { + return r.err +} + +func (r *Response) Parse(m interface{}) error { + if r.err != nil { + return r.err + } + klog.Infof("Json unmarshal: %v", string(r.body)) + return json.Unmarshal(r.body, m) +} diff --git a/pkg/nvmf/config.go b/pkg/nvmf/config.go new file mode 100644 index 0000000..d283b80 --- /dev/null +++ b/pkg/nvmf/config.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package nvmf + +const ( + NVMF_NQN_SIZE = 223 + SYS_NVMF = "/sys/class/nvmf" +) + +// Here erron +const ( + ENOENT = 1 /* No such file or directory */ + EINVAL = 2 /* Invalid argument */ +) + +const ( + DefaultDriverName = "csi.nvmf.com" + DefaultDriverServicePort = "12230" + DefaultDriverVersion = "v1.0.0" + + DefaultVolumeMapPath = "/var/lib/nvmf/volumes" + DefaultBackendEndpoint = "" +) + +type GlobalConfig struct { + NVMfVolumeMapDir string + NVMfBackendEndpoint string + DriverName string + Region string + NodeID string + Endpoint string // CSI endpoint + Version string + IsControllerServer bool + LogLevel string +} diff --git a/pkg/nvmf/const.go b/pkg/nvmf/const.go index 42a74c8..220cc27 100644 --- a/pkg/nvmf/const.go +++ b/pkg/nvmf/const.go @@ -1,46 +1,9 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ package nvmf const ( - NVMF_NQN_SIZE = 223 - SYS_NVMF = "/sys/class/nvmf" + TargetTrAddr = "targetTrAddr" + TargetTrPort = "targetTrPort" + TargetTrType = "targetTrType" + DeviceUUID = "deviceUUID" + NQN = "nqn" ) - -// Here erron -const ( - ENOENT = 1 /* No such file or directory */ - EINVAL = 2 /* Invalid argument */ -) - -const ( - DefaultDriverName = "csi.nvmf.com" - DefaultDriverServicePort = "12230" - DefaultDriverVersion = "v1.0.0" - - DefaultVolumeMapPath = "/var/lib/nvmf/volumes" -) - -type GlobalConfig struct { - NVMfVolumeMapDir string - DriverName string - Region string - NodeID string - Endpoint string // CSI endpoint - Version string - IsControllerServer bool - LogLevel string -} diff --git a/pkg/nvmf/driver.go b/pkg/nvmf/driver.go index c8726fc..7b15e6d 100644 --- a/pkg/nvmf/driver.go +++ b/pkg/nvmf/driver.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-driver-nvmf/pkg/client" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog" @@ -37,6 +38,8 @@ type driver struct { nodeServer *NodeServer controllerServer *ControllerServer + client *client.Client + cap []*csi.VolumeCapability_AccessMode cscap []*csi.ControllerServiceCapability } @@ -59,6 +62,7 @@ func NewDriver(conf *GlobalConfig) *driver { } func (d *driver) Run(conf *GlobalConfig) { + var err error d.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, @@ -71,6 +75,11 @@ func (d *driver) Run(conf *GlobalConfig) { d.idServer = NewIdentityServer(d) d.nodeServer = NewNodeServer(d) if conf.IsControllerServer { + d.client, err = client.NewClient(conf.NVMfBackendEndpoint) + if err != nil { + klog.Fatal("create http client failed.") + return + } d.controllerServer = NewControllerServer(d) } diff --git a/pkg/nvmf/nvmf.go b/pkg/nvmf/nvmf.go index af44c60..4f27972 100644 --- a/pkg/nvmf/nvmf.go +++ b/pkg/nvmf/nvmf.go @@ -55,11 +55,11 @@ func getNVMfDiskInfo(req *csi.NodePublishVolumeRequest) (*nvmfDiskInfo, error) { volName := req.GetVolumeId() volOpts := req.GetVolumeContext() - targetTrAddr := volOpts["targetTrAddr"] - targetTrPort := volOpts["targetTrPort"] - targetTrType := volOpts["targetTrType"] - deviceUUID := volOpts["deviceUUID"] - nqn := volOpts["nqn"] + targetTrAddr := volOpts[TargetTrAddr] + targetTrPort := volOpts[TargetTrPort] + targetTrType := volOpts[TargetTrType] + deviceUUID := volOpts[DeviceUUID] + nqn := volOpts[NQN] if targetTrAddr == "" || nqn == "" || targetTrPort == "" || targetTrType == "" { return nil, fmt.Errorf("Some Nvme target info is missing, volID: %s ", volName) diff --git a/pkg/nvmf/volume.go b/pkg/nvmf/volume.go new file mode 100644 index 0000000..2ea87a7 --- /dev/null +++ b/pkg/nvmf/volume.go @@ -0,0 +1,82 @@ +package nvmf + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-driver-nvmf/pkg/client" +) + +//todo: add allowHostNqn request para +type CreateVolumeRequest struct { + Name string + SizeByte int64 + // AllowHostNqn string + ReadOnly bool +} + +type DeleteVolumeRequest struct { + VolumeId string +} + +func ParseCreateVolumeParameters(parameters map[string]string) (volReq *CreateVolumeRequest, err error) { + //todo: need more parameters for nvmf + + // readonly + readonly, ok := parameters["readonly"] + if !ok { + volReq.ReadOnly = false + } else { + readonly = strings.ToLower(readonly) + if readonly == "yes" || readonly == "true" || readonly == "1" { + volReq.ReadOnly = true + } else { + volReq.ReadOnly = false + } + } + + return volReq, nil +} + +func GetVolumeContext(volRes *client.CreateVolumeResponse) (map[string]string, error) { + //todo: volume context parse failed? + volContext := make(map[string]string) + + volContext[TargetTrAddr] = volRes.TargetConfig.Ports[0].Addr.TrAddr + volContext[TargetTrPort] = volRes.TargetConfig.Ports[0].Addr.TrSvcID + volContext[TargetTrType] = volRes.TargetConfig.Ports[0].Addr.TrAddr + volContext[DeviceUUID] = volRes.TargetConfig.Subsystems[0].Namespaces[0].Device.UUID + volContext[NQN] = volRes.TargetConfig.Subsystems[0].NQN + + return volContext, nil +} + +func PersistVolumeInfo(v *csi.Volume, filePath string) error { + f, err := os.Create(filePath) + if err != nil { + fmt.Errorf("error creating nvme volume persistence file %s: %s", filePath, err) + } + defer f.Close() + encoder := json.NewEncoder(f) + if err = encoder.Encode(v); err != nil { + return fmt.Errorf("error encoding volume: %v", err) + } + return nil +} + +func GetVolumeInfoFromFile(filePath string) (*csi.Volume, error) { + f, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + data := csi.Volume{} + err = json.Unmarshal([]byte(f), &data) + if err != nil { + return nil, err + } + return &data, nil +}