feature: support request back_end controller to Create/Delete Volume

1. send a http request to Create/Delete Volume
2. a most based code.

Signed-off-by: Meinhard Zhou <zhouenhua@bytedance.com>

cmd/nvmfplugin/nvmfplugin.go

@@ -38,7 +38,9 @@ func init() {
 	flag.StringVar(&conf.DriverName, "drivername", nvmf.DefaultDriverName, "CSI Driver")
 	flag.StringVar(&conf.Region, "region", "test_region", "Region")
 	flag.StringVar(&conf.Version, "version", nvmf.DefaultDriverVersion, "Version")
+
 	flag.StringVar(&conf.NVMfVolumeMapDir, "nvmfVolumeMapDir", nvmf.DefaultVolumeMapPath, "Persistent volume")
+	flag.StringVar(&conf.NVMfBackendEndpoint, "nvmfBackendEndpoint", nvmf.DefaultBackendEndpoint, "NVMf Volume backend controller")
 }
 
 func main() {

pkg/client/createVolumeResponse.go

@@ -0,0 +1,55 @@
+package client
+
+type CreateVolumeResponse struct {
+	ID            string        `json:"ID,omitempty"`
+	CapacityBytes uint64        `json:"capacityBytes,omitempty"`
+	TargetType    string        `json:"targetType,omitempty"`
+	TargetConfig  *TargetConfig `json:"targetConfig,omitempty"`
+}
+
+type TargetConfig struct {
+	Hosts      []*Host      `json:"hosts,omitempty"`
+	Ports      []*Port      `json:"ports,omitempty"`
+	Subsystems []*Subsystem `json:"subsystems,omitempty"`
+}
+
+type Host struct {
+	NQN string `json:"NQN,omitempty"`
+}
+
+type Port struct {
+	Addr       *Addr    `json:"addr,omitempty"`
+	PortID     uint64   `json:"portID,omitempty"`
+	Subsystems []string `json:"subsystems,omitempty"`
+}
+
+type Addr struct {
+	AdrFam  string `json:"adrFam,omitempty"`
+	TrAddr  string `json:"trAddr,omitempty"`
+	TrSvcID string `json:"trSvcID,omitempty"`
+	TrType  string `json:"trType,omitempty"`
+}
+
+type Subsystem struct {
+	AllowedHosts []string     `json:"AllowedHosts,omitempty"`
+	Attr         *Attr        `json:"attr,omitempty"`
+	Namespaces   []*Namespace `json:"namespaces,omitempty"`
+	NQN          string       `json:"NQN,omitempty"`
+}
+
+type Attr struct {
+	AllowAnyHost string `json:"allowAnyHost,omitempty"`
+	Serial       string `json:"serial,omitempty"`
+}
+
+type Namespace struct {
+	Device *Device `json:"device,omitempty"`
+	Enable uint32  `json:"enable,omitempty"`
+	NSID   uint64  `json:"NSID,omitempty"`
+}
+
+type Device struct {
+	NGUID string `json:"NGUID,omitempty"`
+	Path  string `json:"path,omitempty"`
+	UUID  string `json:"UUID,omitempty"`
+}

pkg/client/response.go

@@ -1,6 +1,11 @@
 package client
 
-import "net/http"
+import (
+	"encoding/json"
+	"net/http"
+
+	"k8s.io/klog/v2"
+)
 
 type Response struct {
 	statusCode int
@@ -8,3 +13,23 @@ type Response struct {
 	body       []byte
 	err        error
 }
+
+func (r *Response) StatusCode() int {
+	return r.statusCode
+}
+
+func (r *Response) Body() []byte {
+	return r.body
+}
+
+func (r *Response) Err() error {
+	return r.err
+}
+
+func (r *Response) Parse(m interface{}) error {
+	if r.err != nil {
+		return r.err
+	}
+	klog.Infof("Json unmarshal: %v", string(r.body))
+	return json.Unmarshal(r.body, m)
+}

pkg/nvmf/config.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package nvmf
+
+const (
+	NVMF_NQN_SIZE = 223
+	SYS_NVMF      = "/sys/class/nvmf"
+)
+
+// Here erron
+const (
+	ENOENT = 1 /* No such file or directory */
+	EINVAL = 2 /* Invalid argument */
+)
+
+const (
+	DefaultDriverName        = "csi.nvmf.com"
+	DefaultDriverServicePort = "12230"
+	DefaultDriverVersion     = "v1.0.0"
+
+	DefaultVolumeMapPath   = "/var/lib/nvmf/volumes"
+	DefaultBackendEndpoint = ""
+)
+
+type GlobalConfig struct {
+	NVMfVolumeMapDir    string
+	NVMfBackendEndpoint string
+	DriverName          string
+	Region              string
+	NodeID              string
+	Endpoint            string // CSI endpoint
+	Version             string
+	IsControllerServer  bool
+	LogLevel            string
+}

pkg/nvmf/const.go

@@ -1,46 +1,9 @@
-/*
-Copyright 2021 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
 package nvmf
 
 const (
-	NVMF_NQN_SIZE = 223
+	TargetTrAddr = "targetTrAddr"
-	SYS_NVMF      = "/sys/class/nvmf"
+	TargetTrPort = "targetTrPort"
+	TargetTrType = "targetTrType"
+	DeviceUUID   = "deviceUUID"
+	NQN          = "nqn"
 )
-
-// Here erron
-const (
-	ENOENT = 1 /* No such file or directory */
-	EINVAL = 2 /* Invalid argument */
-)
-
-const (
-	DefaultDriverName        = "csi.nvmf.com"
-	DefaultDriverServicePort = "12230"
-	DefaultDriverVersion     = "v1.0.0"
-
-	DefaultVolumeMapPath = "/var/lib/nvmf/volumes"
-)
-
-type GlobalConfig struct {
-	NVMfVolumeMapDir   string
-	DriverName         string
-	Region             string
-	NodeID             string
-	Endpoint           string // CSI endpoint
-	Version            string
-	IsControllerServer bool
-	LogLevel           string
-}

pkg/nvmf/driver.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/csi-driver-nvmf/pkg/client"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"k8s.io/klog"
@@ -37,6 +38,8 @@ type driver struct {
 	nodeServer       *NodeServer
 	controllerServer *ControllerServer
 
+	client *client.Client
+
 	cap   []*csi.VolumeCapability_AccessMode
 	cscap []*csi.ControllerServiceCapability
 }
@@ -59,6 +62,7 @@ func NewDriver(conf *GlobalConfig) *driver {
 }
 
 func (d *driver) Run(conf *GlobalConfig) {
+	var err error
 
 	d.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
@@ -71,6 +75,11 @@ func (d *driver) Run(conf *GlobalConfig) {
 	d.idServer = NewIdentityServer(d)
 	d.nodeServer = NewNodeServer(d)
 	if conf.IsControllerServer {
+		d.client, err = client.NewClient(conf.NVMfBackendEndpoint)
+		if err != nil {
+			klog.Fatal("create http client failed.")
+			return
+		}
 		d.controllerServer = NewControllerServer(d)
 	}
 

pkg/nvmf/nvmf.go

@@ -55,11 +55,11 @@ func getNVMfDiskInfo(req *csi.NodePublishVolumeRequest) (*nvmfDiskInfo, error) {
 	volName := req.GetVolumeId()
 
 	volOpts := req.GetVolumeContext()
-	targetTrAddr := volOpts["targetTrAddr"]
+	targetTrAddr := volOpts[TargetTrAddr]
-	targetTrPort := volOpts["targetTrPort"]
+	targetTrPort := volOpts[TargetTrPort]
-	targetTrType := volOpts["targetTrType"]
+	targetTrType := volOpts[TargetTrType]
-	deviceUUID := volOpts["deviceUUID"]
+	deviceUUID := volOpts[DeviceUUID]
-	nqn := volOpts["nqn"]
+	nqn := volOpts[NQN]
 
 	if targetTrAddr == "" || nqn == "" || targetTrPort == "" || targetTrType == "" {
 		return nil, fmt.Errorf("Some Nvme target info is missing, volID: %s ", volName)

pkg/nvmf/volume.go

@@ -0,0 +1,82 @@
+package nvmf
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/csi-driver-nvmf/pkg/client"
+)
+
+//todo: add allowHostNqn request para
+type CreateVolumeRequest struct {
+	Name     string
+	SizeByte int64
+	// AllowHostNqn string
+	ReadOnly bool
+}
+
+type DeleteVolumeRequest struct {
+	VolumeId string
+}
+
+func ParseCreateVolumeParameters(parameters map[string]string) (volReq *CreateVolumeRequest, err error) {
+	//todo: need more parameters for nvmf
+
+	// readonly
+	readonly, ok := parameters["readonly"]
+	if !ok {
+		volReq.ReadOnly = false
+	} else {
+		readonly = strings.ToLower(readonly)
+		if readonly == "yes" || readonly == "true" || readonly == "1" {
+			volReq.ReadOnly = true
+		} else {
+			volReq.ReadOnly = false
+		}
+	}
+
+	return volReq, nil
+}
+
+func GetVolumeContext(volRes *client.CreateVolumeResponse) (map[string]string, error) {
+	//todo: volume context parse failed?
+	volContext := make(map[string]string)
+
+	volContext[TargetTrAddr] = volRes.TargetConfig.Ports[0].Addr.TrAddr
+	volContext[TargetTrPort] = volRes.TargetConfig.Ports[0].Addr.TrSvcID
+	volContext[TargetTrType] = volRes.TargetConfig.Ports[0].Addr.TrAddr
+	volContext[DeviceUUID] = volRes.TargetConfig.Subsystems[0].Namespaces[0].Device.UUID
+	volContext[NQN] = volRes.TargetConfig.Subsystems[0].NQN
+
+	return volContext, nil
+}
+
+func PersistVolumeInfo(v *csi.Volume, filePath string) error {
+	f, err := os.Create(filePath)
+	if err != nil {
+		fmt.Errorf("error creating nvme volume persistence file %s: %s", filePath, err)
+	}
+	defer f.Close()
+	encoder := json.NewEncoder(f)
+	if err = encoder.Encode(v); err != nil {
+		return fmt.Errorf("error encoding volume: %v", err)
+	}
+	return nil
+}
+
+func GetVolumeInfoFromFile(filePath string) (*csi.Volume, error) {
+	f, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	data := csi.Volume{}
+	err = json.Unmarshal([]byte(f), &data)
+	if err != nil {
+		return nil, err
+	}
+	return &data, nil
+}