mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 12:43:23 +00:00
Merge pull request #55866 from zhangxiaoyu-zidif/make-k8s-support-ceph-fuse
Automatic merge from submit-queue (batch tested with PRs 55954, 56037, 55866, 55984, 54994). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. make k8s support cephfs fuse mount **What this PR does / why we need it**: make k8s support cephfs fuse. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes https://github.com/kubernetes/kubernetes/issues/46636, https://github.com/kubernetes/kubernetes/issues/44486 **Special notes for your reviewer**: **Release note**: ```release-note K8s supports cephfs fuse mount. ```
This commit is contained in:
commit
69b9007667
@ -19,6 +19,9 @@ package cephfs
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
@ -231,15 +234,32 @@ func (cephfsVolume *cephfsMounter) SetUpAt(dir string, fsGroup *int64) error {
|
|||||||
}
|
}
|
||||||
os.MkdirAll(dir, 0750)
|
os.MkdirAll(dir, 0750)
|
||||||
|
|
||||||
err = cephfsVolume.execMount(dir)
|
// check whether it belongs to fuse, if not, default to use kernel mount.
|
||||||
if err == nil {
|
if cephfsVolume.checkFuseMount() {
|
||||||
return nil
|
glog.V(4).Infof("CephFS fuse mount.")
|
||||||
|
err = cephfsVolume.execFuseMount(dir)
|
||||||
|
// cleanup no matter if fuse mount fail.
|
||||||
|
keyringPath := cephfsVolume.GetKeyringPath()
|
||||||
|
_, StatErr := os.Stat(keyringPath)
|
||||||
|
if !os.IsNotExist(StatErr) {
|
||||||
|
os.RemoveAll(keyringPath)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
// cephfs fuse mount succeeded.
|
||||||
|
return nil
|
||||||
|
} else {
|
||||||
|
// if cephfs fuse mount failed, fallback to kernel mount.
|
||||||
|
glog.V(4).Infof("CephFS fuse mount failed: %v, fallback to kernel mount.", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
glog.V(4).Infof("CephFS kernel mount.")
|
||||||
// cleanup upon failure
|
err = cephfsVolume.execMount(dir)
|
||||||
util.UnmountPath(dir, cephfsVolume.mounter)
|
if err != nil {
|
||||||
// return error
|
// cleanup upon failure.
|
||||||
return err
|
util.UnmountPath(dir, cephfsVolume.mounter)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type cephfsUnmounter struct {
|
type cephfsUnmounter struct {
|
||||||
@ -264,6 +284,14 @@ func (cephfsVolume *cephfs) GetPath() string {
|
|||||||
return cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cephfsVolume.volName)
|
return cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cephfsVolume.volName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetKeyringPath creates cephfuse keyring path
|
||||||
|
func (cephfsVolume *cephfs) GetKeyringPath() string {
|
||||||
|
name := cephfsPluginName
|
||||||
|
volumeDir := cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cephfsVolume.volName)
|
||||||
|
volumeKeyringDir := volumeDir + "~keyring"
|
||||||
|
return volumeKeyringDir
|
||||||
|
}
|
||||||
|
|
||||||
func (cephfsVolume *cephfs) execMount(mountpoint string) error {
|
func (cephfsVolume *cephfs) execMount(mountpoint string) error {
|
||||||
// cephfs mount option
|
// cephfs mount option
|
||||||
ceph_opt := ""
|
ceph_opt := ""
|
||||||
@ -299,6 +327,92 @@ func (cephfsVolume *cephfs) execMount(mountpoint string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cephfsMounter *cephfsMounter) checkFuseMount() bool {
|
||||||
|
execute := cephfsMounter.plugin.host.GetExec(cephfsMounter.plugin.GetPluginName())
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "linux":
|
||||||
|
retBytes, err := execute.Run("/bin/ls", "/sbin/mount.fuse.ceph")
|
||||||
|
if err == nil && string(retBytes) == "/sbin/mount.fuse.ceph\n" {
|
||||||
|
glog.V(4).Infof("/sbin/mount.fuse.ceph exists, it should be fuse mount")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cephfsVolume *cephfs) execFuseMount(mountpoint string) error {
|
||||||
|
// cephfs keyring file
|
||||||
|
keyring_file := ""
|
||||||
|
// override secretfile if secret is provided
|
||||||
|
if cephfsVolume.secret != "" {
|
||||||
|
// TODO: cephfs fuse currently doesn't support secret option,
|
||||||
|
// remove keyring file create once secret option is supported.
|
||||||
|
glog.V(4).Infof("cephfs mount begin using fuse.")
|
||||||
|
|
||||||
|
keyringPath := cephfsVolume.GetKeyringPath()
|
||||||
|
os.MkdirAll(keyringPath, 0750)
|
||||||
|
|
||||||
|
payload := make(map[string]util.FileProjection, 1)
|
||||||
|
var fileProjection util.FileProjection
|
||||||
|
|
||||||
|
keyring := fmt.Sprintf("[client.%s]\n", cephfsVolume.id) + "key = " + cephfsVolume.secret + "\n"
|
||||||
|
|
||||||
|
fileProjection.Data = []byte(keyring)
|
||||||
|
fileProjection.Mode = int32(0644)
|
||||||
|
fileName := cephfsVolume.id + ".keyring"
|
||||||
|
|
||||||
|
payload[fileName] = fileProjection
|
||||||
|
|
||||||
|
writerContext := fmt.Sprintf("cephfuse:%v.keyring", cephfsVolume.id)
|
||||||
|
writer, err := util.NewAtomicWriter(keyringPath, writerContext)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to create atomic writer: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = writer.Write(payload)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to write payload to dir: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
keyring_file = path.Join(keyringPath, fileName)
|
||||||
|
|
||||||
|
} else {
|
||||||
|
keyring_file = cephfsVolume.secret_file
|
||||||
|
}
|
||||||
|
|
||||||
|
// build src like mon1:6789,mon2:6789,mon3:6789:/
|
||||||
|
hosts := cephfsVolume.mon
|
||||||
|
l := len(hosts)
|
||||||
|
// pass all monitors and let ceph randomize and fail over
|
||||||
|
i := 0
|
||||||
|
src := ""
|
||||||
|
for i = 0; i < l-1; i++ {
|
||||||
|
src += hosts[i] + ","
|
||||||
|
}
|
||||||
|
src += hosts[i]
|
||||||
|
|
||||||
|
mountArgs := []string{}
|
||||||
|
mountArgs = append(mountArgs, "-k")
|
||||||
|
mountArgs = append(mountArgs, keyring_file)
|
||||||
|
mountArgs = append(mountArgs, "-m")
|
||||||
|
mountArgs = append(mountArgs, src)
|
||||||
|
mountArgs = append(mountArgs, mountpoint)
|
||||||
|
mountArgs = append(mountArgs, "-r")
|
||||||
|
mountArgs = append(mountArgs, cephfsVolume.path)
|
||||||
|
|
||||||
|
glog.V(4).Infof("Mounting cmd ceph-fuse with arguments (%s)", mountArgs)
|
||||||
|
command := exec.Command("ceph-fuse", mountArgs...)
|
||||||
|
output, err := command.CombinedOutput()
|
||||||
|
if err != nil || !(strings.Contains(string(output), "starting fuse")) {
|
||||||
|
return fmt.Errorf("Ceph-fuse failed: %v\narguments: %s\nOutput: %s\n", err, mountArgs, string(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func getVolumeSource(spec *volume.Spec) ([]string, string, string, string, bool, error) {
|
func getVolumeSource(spec *volume.Spec) ([]string, string, string, string, bool, error) {
|
||||||
if spec.Volume != nil && spec.Volume.CephFS != nil {
|
if spec.Volume != nil && spec.Volume.CephFS != nil {
|
||||||
mon := spec.Volume.CephFS.Monitors
|
mon := spec.Volume.CephFS.Monitors
|
||||||
|
Loading…
Reference in New Issue
Block a user