From b821a5df4c1adb154613b80f981ba16684ed8b80 Mon Sep 17 00:00:00 2001 From: Harshal Patil Date: Fri, 1 Jun 2018 15:17:40 +0530 Subject: [PATCH] virtcontainers: Add support for ephemeral volumes Ephemeral volumes should not be passed at 9pfs mounts. They should be created inside the VM. This patch disables ephemeral volumes from getting mounted as 9pfs from the host and instead a corresponding tmpfs is created inside the VM. Fixes : #61 Signed-off-by: Harshal Patil --- cli/create.go | 17 ++++++++++++++++- cli/create_test.go | 18 ++++++++++++++++++ cli/utils.go | 25 ++++++++++++++++++++++++- cli/utils_test.go | 14 ++++++++++++++ virtcontainers/kata_agent.go | 27 +++++++++++++++++++++++++++ virtcontainers/kata_agent_test.go | 19 +++++++++++++++++++ 6 files changed, 118 insertions(+), 2 deletions(-) diff --git a/cli/create.go b/cli/create.go index d7e5abec8..33ae22b81 100644 --- a/cli/create.go +++ b/cli/create.go @@ -103,7 +103,6 @@ func create(containerID, bundlePath, console, pidFilePath string, detach bool, disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) var process vc.Process - switch containerType { case vc.PodSandbox: process, err = createSandbox(ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput) @@ -246,9 +245,25 @@ func createSandbox(ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, return containers[0].Process(), nil } +// setEphemeralStorageType sets the mount type to 'ephemeral' +// if the mount source path is provisioned by k8s for ephemeral storage. +// For the given pod ephemeral volume is created only once +// backed by tmpfs inside the VM. For successive containers +// of the same pod the already existing volume is reused. +func setEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec { + for idx, mnt := range ociSpec.Mounts { + if IsEphemeralStorage(mnt.Source) { + ociSpec.Mounts[idx].Type = "ephemeral" + } + } + return ociSpec +} + func createContainer(ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) { + ociSpec = setEphemeralStorageType(ociSpec) + contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput) if err != nil { return vc.Process{}, err diff --git a/cli/create_test.go b/cli/create_test.go index 914373546..fd4dd892a 100644 --- a/cli/create_test.go +++ b/cli/create_test.go @@ -1010,6 +1010,24 @@ func TestCreateCreateContainerFail(t *testing.T) { } } +func TestSetEphemeralStorageType(t *testing.T) { + assert := assert.New(t) + + ociSpec := oci.CompatOCISpec{} + var ociMounts []specs.Mount + mount := specs.Mount{ + Source: "/var/lib/kubelet/pods/366c3a77-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume", + } + + ociMounts = append(ociMounts, mount) + ociSpec.Mounts = ociMounts + ociSpec = setEphemeralStorageType(ociSpec) + + mountType := ociSpec.Mounts[0].Type + assert.Equal(mountType, "ephemeral", + "Unexpected mount type, got %s expected ephemeral", mountType) +} + func TestCreateCreateContainer(t *testing.T) { assert := assert.New(t) diff --git a/cli/utils.go b/cli/utils.go index 61a8ee1ed..3993f3e08 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -15,7 +15,10 @@ import ( "strings" ) -const unknown = "<>" +const ( + unknown = "<>" + k8sEmptyDir = "kubernetes.io~empty-dir" +) // variables to allow tests to modify the values var ( @@ -43,6 +46,26 @@ func getFileContents(file string) (string, error) { return string(bytes), nil } +// IsEphemeralStorage returns true if the given path +// to the storage belongs to kubernetes ephemeral storage +// +// This method depends on a specific path used by k8s +// to detect if it's of type ephemeral. As of now, +// this is a very k8s specific solution that works +// but in future there should be a better way for this +// method to determine if the path is for ephemeral +// volume type +func IsEphemeralStorage(path string) bool { + splitSourceSlice := strings.Split(path, "/") + if len(splitSourceSlice) > 1 { + storageType := splitSourceSlice[len(splitSourceSlice)-2] + if storageType == k8sEmptyDir { + return true + } + } + return false +} + func getKernelVersion() (string, error) { contents, err := getFileContents(procVersion) if err != nil { diff --git a/cli/utils_test.go b/cli/utils_test.go index 69a131c9f..3f8529b1d 100644 --- a/cli/utils_test.go +++ b/cli/utils_test.go @@ -38,6 +38,20 @@ func TestFileExists(t *testing.T) { fmt.Sprintf("File %q should exist", file)) } +func TestIsEphemeralStorage(t *testing.T) { + sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume" + isEphe := IsEphemeralStorage(sampleEphePath) + if !isEphe { + t.Fatalf("Unable to correctly determine volume type") + } + + sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume" + isEphe = IsEphemeralStorage(sampleEphePath) + if isEphe { + t.Fatalf("Unable to correctly determine volume type") + } +} + func TestGetFileContents(t *testing.T) { type testData struct { contents string diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index b49efae9d..17063d3cb 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -51,6 +51,7 @@ var ( sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L", "nodev"} shmDir = "shm" kataEphemeralDevType = "ephemeral" + ephemeralPath = filepath.Join(kataGuestSandboxDir, kataEphemeralDevType) ) // KataAgentConfig is a structure storing information needed @@ -781,6 +782,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, return nil, err } + epheStorages := k.handleEphemeralStorage(ociSpec.Mounts) + ctrStorages = append(ctrStorages, epheStorages...) + // We replace all OCI mount sources that match our container mount // with the right source path (The guest one). if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil { @@ -846,6 +850,29 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, k.state.URL, c.config.Cmd, createNSList, enterNSList) } +// handleEphemeralStorage handles ephemeral storages by +// creating a Storage from corresponding source of the mount point +func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage { + var epheStorages []*grpc.Storage + for idx, mnt := range mounts { + if mnt.Type == kataEphemeralDevType { + // Set the mount source path to a path that resides inside the VM + mounts[idx].Source = filepath.Join(ephemeralPath, filepath.Base(mnt.Source)) + + // Create a storage struct so that kata agent is able to create + // tmpfs backed volume inside the VM + epheStorage := &grpc.Storage{ + Driver: kataEphemeralDevType, + Source: "tmpfs", + Fstype: "tmpfs", + MountPoint: mounts[idx].Source, + } + epheStorages = append(epheStorages, epheStorage) + } + } + return epheStorages +} + // handleBlockVolumes handles volumes that are block devices files // by passing the block devices as Storage to the agent. func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index 92846e891..33a8a51fc 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -369,6 +369,25 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { } +func TestHandleEphemeralStorage(t *testing.T) { + k := kataAgent{} + var ociMounts []specs.Mount + mountSource := "/tmp/mountPoint" + + mount := specs.Mount{ + Type: kataEphemeralDevType, + Source: mountSource, + } + + ociMounts = append(ociMounts, mount) + epheStorages := k.handleEphemeralStorage(ociMounts) + + epheMountPoint := epheStorages[0].GetMountPoint() + expected := filepath.Join(ephemeralPath, filepath.Base(mountSource)) + assert.Equal(t, epheMountPoint, expected, + "Ephemeral mount point didn't match: got %s, expecting %s", epheMountPoint, expected) +} + func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { k := kataAgent{}