diff --git a/src/tools/csi-kata-directvolume/.gitignore b/src/tools/csi-kata-directvolume/.gitignore index e660fd93d3..08f7deafae 100644 --- a/src/tools/csi-kata-directvolume/.gitignore +++ b/src/tools/csi-kata-directvolume/.gitignore @@ -1 +1,2 @@ bin/ +deploy/kata-directvolume/kata-directvol-rbac.yaml diff --git a/src/tools/csi-kata-directvolume/README.md b/src/tools/csi-kata-directvolume/README.md index 7c9eccb040..206f950c02 100644 --- a/src/tools/csi-kata-directvolume/README.md +++ b/src/tools/csi-kata-directvolume/README.md @@ -30,50 +30,9 @@ cd tools/csi-kata-directvolume/ && make ## Building the Container Image -If you want to build the container image yourself, you can do so with the following command from a specified path. -Here, we just use `buildah/podman` as an example: +If you want to build the container image yourself, you can do so with the following command: ```shell -$ tree -L 2 buildah-directv/ -buildah-directv/ -├── bin -│   └── directvolplugin -└── Dockerfile - -$ buildah bud -t kata-directvolume:v1.0.19 -STEP 1/7: FROM alpine -STEP 2/7: LABEL maintainers="Kata Containers Authors" -STEP 3/7: LABEL description="Kata DirectVolume Driver" -STEP 4/7: ARG binary=./bin/directvolplugin -STEP 5/7: RUN apk add util-linux coreutils e2fsprogs xfsprogs xfsprogs-extra btrfs-progs && apk update && apk upgrade -fetch https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/APKINDEX.tar.gz -fetch https://dl-cdn.alpinelinux.org/alpine/v3.19/community/x86_64/APKINDEX.tar.gz -(1/66) Installing libblkid (2.39.3-r0) -... -(66/66) Installing xfsprogs-extra (6.5.0-r0) -Executing busybox-1.36.1-r15.trigger -OK: 64 MiB in 81 packages -fetch https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64/APKINDEX.tar.gz -fetch https://dl-cdn.alpinelinux.org/alpine/v3.19/community/x86_64/APKINDEX.tar.gz -v3.19.0-19-ga0ddaee500e [https://dl-cdn.alpinelinux.org/alpine/v3.19/main] -v3.19.0-18-gec62a609516 [https://dl-cdn.alpinelinux.org/alpine/v3.19/community] -OK: 22983 distinct packages available -OK: 64 MiB in 81 packages -STEP 6/7: COPY ${binary} /kata-directvol-plugin -STEP 7/7: ENTRYPOINT ["/kata-directvol-plugin"] -COMMIT kata-directvolume:v1.0.19 -Getting image source signatures -Copying blob 5af4f8f59b76 skipped: already exists -Copying blob a55645705de3 done -Copying config 244001cc51 done -Writing manifest to image destination -Storing signatures ---> 244001cc51d -Successfully tagged localhost/kata-directvolume:v1.0.19 -244001cc51d77302c4ed5e1a0ec347d12d85dec4576ea1313f700f66e2a7d36d -$ podman save localhost/kata-directvolume:v1.0.19 -o kata-directvolume-v1.0.19.tar -$ ctr -n k8s.io image import kata-directvolume-v1.0.19.tar -unpacking localhost/kata-directvolume:v1.0.19 (sha256:1bdc33ff7f9cee92e74cbf77a9d79d00dce6dbb9ba19b9811f683e1a087f8fbf)...done -$ crictl images |grep 1.0.19 -localhost/kata-directvolume v1.0.19 244001cc51d77 83.8MB +$ cd src/tools/csi-kata-directvolume +$ docker build -t localhost/kata-directvolume:v1.0.18 . ``` diff --git a/src/tools/csi-kata-directvolume/deploy/deploy.sh b/src/tools/csi-kata-directvolume/deploy/deploy.sh old mode 100644 new mode 100755 diff --git a/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md b/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md index c57eda4480..ccfea48a57 100644 --- a/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md +++ b/src/tools/csi-kata-directvolume/docs/deploy-csi-kata-directvol.md @@ -17,87 +17,58 @@ The easiest way to deploy the `Direct Volume CSI driver` is to run the `deploy.s the cluster as shown below for Kubernetes 1.28.2. ```shell -sudo deploy/deploy.sh -``` - -You'll get an output similar to the following, indicating the application of `RBAC rules` and the successful deployment of `csi-provisioner`, `node-driver-registrar`, `kata directvolume csi driver`(`csi-kata-directvol-plugin`), liveness-probe. Please note that the following output is specific to Kubernetes 1.28.2. - -```shell +$ ./deploy/deploy.sh Creating Namespace kata-directvolume ... -kubectl apply -f /tmp/tmp.kN43BWUGQ5/kata-directvol-ns.yaml +kubectl apply -f /tmp/tmp.lAAPNQ1aI2/kata-directvol-ns.yaml namespace/kata-directvolume created Namespace kata-directvolume created Done ! Applying RBAC rules ... -curl https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/v3.6.0/deploy/kubernetes/rbac.yaml --output /tmp/tmp.kN43BWUGQ5/rbac.yaml --silent --location -kubectl apply -f ./kata-directvolume/kata-directvol-rbac.yaml +curl https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/v3.6.0/deploy/kubernetes/rbac.yaml --output /tmp/tmp.lAAPNQ1aI2/rbac.yaml --silent --location +kubectl apply -f ./deploy/kata-directvolume/kata-directvol-rbac.yaml serviceaccount/csi-provisioner created clusterrole.rbac.authorization.k8s.io/external-provisioner-runner created clusterrolebinding.rbac.authorization.k8s.io/csi-provisioner-role created role.rbac.authorization.k8s.io/external-provisioner-cfg created rolebinding.rbac.authorization.k8s.io/csi-provisioner-role-cfg created - -$ ./directvol-deploy.sh +Applying RBAC rules Done! deploying kata directvolume components - ./kata-directvolume/csi-directvol-driverinfo.yaml + ./deploy/kata-directvolume/csi-directvol-driverinfo.yaml csidriver.storage.k8s.io/directvolume.csi.katacontainers.io created - ./kata-directvolume/csi-directvol-plugin.yaml + ./deploy/kata-directvolume/csi-directvol-plugin.yaml kata-directvolume plugin using image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 kata-directvolume plugin using image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 -kata-directvolume plugin using image: localhost/kata-directvolume:v1.0.52 +kata-directvolume plugin using image: localhost/kata-directvolume:v1.0.19 kata-directvolume plugin using image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 daemonset.apps/csi-kata-directvol-plugin created - ./kata-directvolume/kata-directvol-ns.yaml -namespace/kata-directvolume unchanged - ./kata-directvolume/kata-directvol-rbac.yaml -serviceaccount/csi-provisioner unchanged -clusterrole.rbac.authorization.k8s.io/external-provisioner-runner configured -clusterrolebinding.rbac.authorization.k8s.io/csi-provisioner-role unchanged -role.rbac.authorization.k8s.io/external-provisioner-cfg unchanged -rolebinding.rbac.authorization.k8s.io/csi-provisioner-role-cfg unchanged -NAMESPACE NAME READY STATUS RESTARTS AGE -default pod/kata-driectvol-01 1/1 Running 0 3h57m -kata-directvolume pod/csi-kata-directvol-plugin-92smp 4/4 Running 0 4s -kube-flannel pod/kube-flannel-ds-vq796 1/1 Running 1 (67d ago) 67d -kube-system pod/coredns-66f779496c-9bmp2 1/1 Running 3 (67d ago) 67d -kube-system pod/coredns-66f779496c-qlq6d 1/1 Running 1 (67d ago) 67d -kube-system pod/etcd-tnt001 1/1 Running 19 (67d ago) 67d -kube-system pod/kube-apiserver-tnt001 1/1 Running 5 (67d ago) 67d -kube-system pod/kube-controller-manager-tnt001 1/1 Running 8 (67d ago) 67d -kube-system pod/kube-proxy-p9t6t 1/1 Running 6 (67d ago) 67d -kube-system pod/kube-scheduler-tnt001 1/1 Running 8 (67d ago) 67d +NAMESPACE NAME READY STATUS RESTARTS AGE +kata-directvolume pod/csi-kata-directvol-plugin-9vvhc 4/4 Running 0 3s +[...TRUNCATED...] -NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE -kata-directvolume daemonset.apps/csi-kata-directvol-plugin 1 1 1 1 1 4s -kube-flannel daemonset.apps/kube-flannel-ds 1 1 1 1 1 67d -kube-system daemonset.apps/kube-proxy 1 1 1 1 1 kubernetes.io/os=linux 67d +NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +kata-directvolume daemonset.apps/csi-kata-directvol-plugin 1 1 1 1 1 3s +[...TRUNCATED...] ``` ## How to Run a Kata Pod and Validate it -First, ensure all expected pods are running properly, including `csi-provisioner`, `node-driver-registrar`, `kata-directvolume` `csi driver(csi-kata-directvol-plugin)`, liveness-probe: +First, ensure all expected containers are running properly: ```shell -$ kubectl get po -A -NAMESPACE NAME READY STATUS RESTARTS AGE -default csi-kata-directvol-plugin-dlphw 4/4 Running 0 68m -kube-flannel kube-flannel-ds-vq796 1/1 Running 1 (52d ago) 52d -kube-system coredns-66f779496c-9bmp2 1/1 Running 3 (52d ago) 52d -kube-system coredns-66f779496c-qlq6d 1/1 Running 1 (52d ago) 52d -kube-system etcd-node001 1/1 Running 19 (52d ago) 52d -kube-system kube-apiserver-node001 1/1 Running 5 (52d ago) 52d -kube-system kube-controller-manager-node001 1/1 Running 8 (52d ago) 52d -kube-system kube-proxy-p9t6t 1/1 Running 6 (52d ago) 52d -kube-system kube-scheduler-node001 1/1 Running 8 (52d ago) 52d +$ kubectl get po -n kata-directvolume +NAME READY STATUS RESTARTS AGE +csi-kata-directvol-plugin-9vvhc 4/4 Running 0 6m14s ``` -From the root directory, deploy the application pods including a storage class, a `PVC`, and a pod which uses direct block device based volume. The details can be seen in `/examples/pod-with-directvol/*.yaml`: +Deploy the application pods including a storage class, a `PVC`, and a +pod which uses direct block device based volume: ```shell -kubectl apply -f ${BASE_DIR}/csi-storageclass.yaml -kubectl apply -f ${BASE_DIR}/csi-pvc.yaml -kubectl apply -f ${BASE_DIR}/csi-app.yaml +$ cd src/tools/csi-kata-directvolume/examples/pod-with-directvol +$ kubectl apply -f csi-storageclass.yaml +$ kubectl apply -f csi-pvc.yaml +$ kubectl apply -f csi-app.yaml ``` Let's validate the components are deployed: diff --git a/src/tools/csi-kata-directvolume/pkg/directvolume/controllerserver.go b/src/tools/csi-kata-directvolume/pkg/directvolume/controllerserver.go index 7ee6be4d71..f99a1b1a50 100644 --- a/src/tools/csi-kata-directvolume/pkg/directvolume/controllerserver.go +++ b/src/tools/csi-kata-directvolume/pkg/directvolume/controllerserver.go @@ -49,6 +49,7 @@ func (dv *directVolume) CreateVolume(ctx context.Context, req *csi.CreateVolumeR volumeCtx := make(map[string]string) volumeCtx[utils.IsDirectVolume] = "False" + volumeCtx[utils.KataContainersDirectLoop] = "False" for key, value := range req.GetParameters() { switch strings.ToLower(key) { @@ -56,12 +57,18 @@ func (dv *directVolume) CreateVolume(ctx context.Context, req *csi.CreateVolumeR if value == utils.DirectVolumeTypeName { volumeCtx[utils.IsDirectVolume] = "True" } + volumeCtx[utils.KataContainersDirectVolumeType] = value case utils.KataContainersDirectFsType: volumeCtx[utils.KataContainersDirectFsType] = value + case utils.KataContainersDirectLoop: + volumeCtx[utils.KataContainersDirectLoop] = value default: - continue + klog.Warningf("unknown parameter: %s", key) } } + if isLoopDevice(volumeCtx) { + volumeCtx[utils.IsDirectVolume] = "True" + } contentSrc := req.GetVolumeContentSource() diff --git a/src/tools/csi-kata-directvolume/pkg/directvolume/nodeserver.go b/src/tools/csi-kata-directvolume/pkg/directvolume/nodeserver.go index 9847c38722..2e0c84aee1 100644 --- a/src/tools/csi-kata-directvolume/pkg/directvolume/nodeserver.go +++ b/src/tools/csi-kata-directvolume/pkg/directvolume/nodeserver.go @@ -10,8 +10,10 @@ package directvolume import ( "fmt" "os" + "os/exec" "path/filepath" "strconv" + "strings" "kata-containers/csi-kata-directvolume/pkg/utils" @@ -68,8 +70,12 @@ func (dv *directVolume) NodePublishVolume(ctx context.Context, req *csi.NodePubl attrib := req.GetVolumeContext() devicePath := dv.config.VolumeDevices[volumeID] - klog.Infof("target %v\nfstype %v\ndevice %v\nreadonly %v\nvolumeID %v\n", - targetPath, fsType, devicePath, readOnly, volumeID) + klog.Infoln("target", targetPath) + klog.Infoln("volType", volType) + klog.Infoln("fstype", fsType) + klog.Infoln("device", devicePath) + klog.Infoln("readonly", readOnly) + klog.Infoln("volumeID", volumeID) options := []string{"bind"} if readOnly { @@ -93,13 +99,20 @@ func (dv *directVolume) NodePublishVolume(ctx context.Context, req *csi.NodePubl return nil, status.Error(codes.Aborted, errMsg) } + var guestOptions []string + if isLoopDevice(attrib) { + guestOptions = []string{} + } else { + guestOptions = options + } + // kata-containers DirectVolume add mountInfo := utils.MountInfo{ VolumeType: volType, Device: devicePath, FsType: fsType, Metadata: attrib, - Options: options, + Options: guestOptions, } if err := utils.AddDirectVolume(targetPath, mountInfo); err != nil { klog.Errorf("add direct volume with source %s and mountInfo %v failed", targetPath, mountInfo) @@ -196,8 +209,27 @@ func (dv *directVolume) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUn return &csi.NodeUnpublishVolumeResponse{}, nil } +func parseBool(s string) bool { + if b, err := strconv.ParseBool(s); err != nil { + return false + } else { + return b + } +} + func isDirectVolume(VolumeCtx map[string]string) bool { - return VolumeCtx[utils.IsDirectVolume] == "True" + return parseBool(VolumeCtx[utils.IsDirectVolume]) +} + +func isLoopDevice(VolumeCtx map[string]string) bool { + return parseBool(VolumeCtx[utils.KataContainersDirectLoop]) +} + +// getDeviceSymlinkPath returns the path of the symlink that is used to +// point to the loop device from inside the specified stagingTargetPath +// directory. +func getDeviceSymlinkPath(stagingTargetPath string) string { + return filepath.Join(stagingTargetPath, "device") } func (dv *directVolume) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { @@ -224,14 +256,14 @@ func (dv *directVolume) NodeStageVolume(ctx context.Context, req *csi.NodeStageV defer dv.mutex.Unlock() capacityInBytes := req.VolumeContext[utils.CapabilityInBytes] - devicePath, err := utils.CreateDirectBlockDevice(volumeID, capacityInBytes, dv.config.StoragePath) + imagePath, err := utils.CreateDirectBlockDevice(volumeID, capacityInBytes, dv.config.StoragePath) if err != nil { errMsg := status.Errorf(codes.Internal, "setup storage for volume '%s' failed", volumeID) return &csi.NodeStageVolumeResponse{}, errMsg } // /full_path_on_host/VolumeId/ - deviceUpperPath := filepath.Dir(*devicePath) + imageUpperPath := filepath.Dir(*imagePath) if canMnt, err := utils.CanDoBindmount(dv.config.safeMounter, stagingTargetPath); err != nil { return nil, err } else if !canMnt { @@ -240,8 +272,8 @@ func (dv *directVolume) NodeStageVolume(ctx context.Context, req *csi.NodeStageV } options := []string{"bind"} - if err := dv.config.safeMounter.DoBindmount(deviceUpperPath, stagingTargetPath, "", options); err != nil { - klog.Errorf("safe mounter: %v do bind mount %v failed, with error: %v", deviceUpperPath, stagingTargetPath, err.Error()) + if err := dv.config.safeMounter.DoBindmount(imageUpperPath, stagingTargetPath, "", options); err != nil { + klog.Errorf("safe mounter: %v do bind mount %v failed, with error: %v", imageUpperPath, stagingTargetPath, err.Error()) return nil, err } @@ -251,11 +283,33 @@ func (dv *directVolume) NodeStageVolume(ctx context.Context, req *csi.NodeStageV fsType = utils.DefaultFsType } - if err := dv.config.safeMounter.SafeFormatWithFstype(*devicePath, fsType, options); err != nil { + if err := dv.config.safeMounter.SafeFormatWithFstype(*imagePath, fsType, options); err != nil { return nil, err } - dv.config.VolumeDevices[volumeID] = *devicePath + if isLoopDevice(req.VolumeContext) { + deviceLink := getDeviceSymlinkPath(stagingTargetPath) + + losetupOut, err := exec.Command("losetup", "-f", "--show", *imagePath).Output() + if err != nil { + var stderr []byte + if exitErr, isExitError := err.(*exec.ExitError); isExitError { + stderr = exitErr.Stderr + } + errMsg := status.Errorf(codes.Internal, "failed to set up loop device from %s: %v: %s", *imagePath, err, stderr) + return &csi.NodeStageVolumeResponse{}, errMsg + } + + devicePath := strings.TrimSuffix(string(losetupOut), "\n") + + if err := os.Symlink(devicePath, deviceLink); err != nil { + return nil, status.Errorf(codes.Internal, "failed to create symlink at %s: %v", deviceLink, err) + } + + dv.config.VolumeDevices[volumeID] = devicePath + } else { + dv.config.VolumeDevices[volumeID] = *imagePath + } klog.Infof("directvolume: volume %s has been staged.", stagingTargetPath) @@ -305,6 +359,24 @@ func (dv *directVolume) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnst dv.mutex.Lock() defer dv.mutex.Unlock() + deviceLink := getDeviceSymlinkPath(stagingTargetPath) + + if _, err := os.Stat(deviceLink); err != nil { + if !os.IsNotExist(err) { + return nil, status.Errorf(codes.Internal, "failed to stat file %s: %v", deviceLink, err) + } + // Else this volume didn't use a loop device, so do nothing. + } else { + // We have to resolve the symlink first because losetup won't follow it. + canonicalDevice, err := filepath.EvalSymlinks(deviceLink) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to resolve device symlink %s: %v", deviceLink, err) + } + if err := exec.Command("losetup", "-d", canonicalDevice).Run(); err != nil { + return nil, status.Errorf(codes.Internal, "failed to detach loop device %s: %v", deviceLink, err) + } + } + // Unmount only if the target path is really a mount point. if isMnt, err := dv.config.safeMounter.IsMountPoint(stagingTargetPath); err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("check staging target path: %v", err)) diff --git a/src/tools/csi-kata-directvolume/pkg/utils/utils.go b/src/tools/csi-kata-directvolume/pkg/utils/utils.go index cdd80147a4..1e1f1db4d8 100644 --- a/src/tools/csi-kata-directvolume/pkg/utils/utils.go +++ b/src/tools/csi-kata-directvolume/pkg/utils/utils.go @@ -26,6 +26,7 @@ import ( const ( KataContainersDirectVolumeType = "katacontainers.direct.volume/volumetype" KataContainersDirectFsType = "katacontainers.direct.volume/fstype" + KataContainersDirectLoop = "katacontainers.direct.volume/loop" DirectVolumeTypeName = "directvol" IsDirectVolume = "is_directvolume" )