Merge pull request #452 from Sarsate/extvol-hostdir

Initial framework for external volumes.

pkg/api/types.go

@@ -62,8 +62,31 @@ type Volume struct {
 	// Required: This must be a DNS_LABEL.  Each volume in a pod must have
 	// a unique name.
 	Name string `yaml:"name" json:"name"`
+	// Source represents the location and type of a volume to mount.
+	// This is optional for now. If not specified, the Volume is implied to be an EmptyDir.
+	// This implied behavior is deprecated and will be removed in a future version.
+        Source *VolumeSource `yaml:"source" json:"source"`
 }
 
+type VolumeSource struct {
+	// Only one of the following sources may be specified
+	// HostDirectory represents a pre-existing directory on the host machine that is directly
+	// exposed to the container. This is generally used for system agents or other privileged
+	// things that are allowed to see the host machine. Most containers will NOT need this.
+	// TODO(jonesdl) We need to restrict who can use host directory mounts and
+	// who can/can not mount host directories as read/write.
+	HostDirectory *HostDirectory `yaml:"hostDir" json:"hostDir"`
+	// EmptyDirectory represents a temporary directory that shares a pod's lifetime.
+	EmptyDirectory *EmptyDirectory `yaml:"emptyDir" json:"emptyDir"`
+}
+
+// Bare host directory volume.
+type HostDirectory struct {
+	Path string `yaml:"path" json:"path"`
+}
+
+type EmptyDirectory struct {}
+
 // Port represents a network port in a single container
 type Port struct {
 	// Optional: If specified, this must be a DNS_LABEL.  Each named port
@@ -92,6 +115,7 @@ type VolumeMount struct {
 	MountPath string `yaml:"mountPath,omitempty" json:"mountPath,omitempty"`
 	Path      string `yaml:"path,omitempty" json:"path,omitempty"`
 	// One of: "LOCAL" (local volume) or "HOST" (external mount from the host). Default: LOCAL.
+	// DEPRECATED: MountType will be removed in a future version of the API.
 	MountType string `yaml:"mountType,omitempty" json:"mountType,omitempty"`
 }
 

pkg/api/validation.go

@@ -76,17 +76,50 @@ func validateVolumes(volumes []Volume) (util.StringSet, errorList) {
 	allNames := util.StringSet{}
 	for i := range volumes {
 		vol := &volumes[i] // so we can set default values
+		errs := errorList{}
+		// TODO(thockin) enforce that a source is set once we deprecate the implied form.
+		if vol.Source != nil {
+			errs = validateSource(vol.Source)
+		}
 		if !util.IsDNSLabel(vol.Name) {
-			allErrs.Append(makeInvalidError("Volume.Name", vol.Name))
+			errs.Append(makeInvalidError("Volume.Name", vol.Name))
 		} else if allNames.Has(vol.Name) {
-			allErrs.Append(makeDuplicateError("Volume.Name", vol.Name))
-		} else {
+			errs.Append(makeDuplicateError("Volume.Name", vol.Name))
+		}
+		if len(errs) == 0 {
 			allNames.Insert(vol.Name)
+		} else {
+			allErrs.Append(errs...)
 		}
 	}
 	return allNames, allErrs
 }
 
+func validateSource(source *VolumeSource) errorList {
+	numVolumes := 0
+	allErrs := errorList{}
+	if source.HostDirectory != nil {
+		numVolumes++
+		allErrs.Append(validateHostDir(source.HostDirectory)...)
+	}
+	if source.EmptyDirectory != nil {
+		numVolumes++
+		//EmptyDirs have nothing to validate
+	}
+	if numVolumes != 1 {
+		allErrs.Append(makeInvalidError("Volume.Source", source))
+	}
+	return allErrs
+}
+
+func validateHostDir(hostDir *HostDirectory) errorList {
+	allErrs := errorList{}
+	if hostDir.Path == "" {
+		allErrs.Append(makeNotFoundError("HostDir.Path", hostDir.Path))
+	}
+	return allErrs
+}
+
 var supportedPortProtocols = util.NewStringSet("TCP", "UDP")
 
 func validatePorts(ports []Port) errorList {
@@ -163,6 +196,9 @@ func validateVolumeMounts(mounts []VolumeMount, volumes util.StringSet) errorLis
 				mnt.Path = ""
 			}
 		}
+		if len(mnt.MountType) != 0 {
+			glog.Warning("DEPRECATED: VolumeMount.MountType will be removed. The Volume struct will handle types")
+		}
 	}
 	return allErrs
 }

pkg/api/validation_test.go

@@ -26,14 +26,15 @@ import (
 func TestValidateVolumes(t *testing.T) {
 	successCase := []Volume{
 		{Name: "abc"},
-		{Name: "123"},
-		{Name: "abc-123"},
+		{Name: "123", Source: &VolumeSource{HostDirectory: &HostDirectory{"/mnt/path2"}}},
+		{Name: "abc-123", Source: &VolumeSource{HostDirectory: &HostDirectory{"/mnt/path3"}}},
+		{Name: "empty", Source: &VolumeSource{EmptyDirectory: &EmptyDirectory{}}},
 	}
 	names, errs := validateVolumes(successCase)
 	if len(errs) != 0 {
 		t.Errorf("expected success: %v", errs)
 	}
-	if len(names) != 3 || !names.Has("abc") || !names.Has("123") || !names.Has("abc-123") {
+	if len(names) != 4 || !names.Has("abc") || !names.Has("123") || !names.Has("abc-123") || !names.Has("empty") {
 		t.Errorf("wrong names result: %v", names)
 	}
 
@@ -206,7 +207,8 @@ func TestValidateManifest(t *testing.T) {
 		{
 			Version: "v1beta1",
 			ID:      "abc",
-			Volumes: []Volume{{Name: "vol1"}, {Name: "vol2"}},
+			Volumes: []Volume{{Name: "vol1", Source: &VolumeSource{HostDirectory: &HostDirectory{"/mnt/vol1"}}},
+			                  {Name: "vol2", Source: &VolumeSource{HostDirectory: &HostDirectory{"/mnt/vol2"}}}},
 			Containers: []Container{
 				{
 					Name:       "abc",

pkg/kubelet/kubelet.go

@@ -36,6 +36,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/health"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/volume"
 	"github.com/coreos/go-etcd/etcd"
 	"github.com/fsouza/go-dockerclient"
 	"github.com/golang/glog"
@@ -62,6 +63,8 @@ func New() *Kubelet {
 	return &Kubelet{}
 }
 
+type volumeMap map[string]volume.Interface
+
 // Kubelet is the main kubelet implementation.
 type Kubelet struct {
 	Hostname           string
@@ -178,15 +181,20 @@ func makeEnvironmentVariables(container *api.Container) []string {
 	return result
 }
 
-func makeVolumesAndBinds(manifestID string, container *api.Container) (map[string]struct{}, []string) {
+func makeVolumesAndBinds(manifestID string, container *api.Container, podVolumes volumeMap) (map[string]struct{}, []string) {
 	volumes := map[string]struct{}{}
 	binds := []string{}
 	for _, volume := range container.VolumeMounts {
 		var basePath string
-		if volume.MountType == "HOST" {
+		if vol, ok := podVolumes[volume.Name]; ok {
 			// Host volumes are not Docker volumes and are directly mounted from the host.
+			basePath = fmt.Sprintf("%s:%s", vol.GetPath(), volume.MountPath)
+		} else if volume.MountType == "HOST" {
+			// DEPRECATED: VolumeMount.MountType will be handled by the Volume struct.
 			basePath = fmt.Sprintf("%s:%s", volume.MountPath, volume.MountPath)
 		} else {
+			// TODO(jonesdl) This clause should be deleted and an error should be thrown. The default
+			// behavior is now supported by the EmptyDirectory type.
 			volumes[volume.MountPath] = struct{}{}
 			basePath = fmt.Sprintf("/exports/%s/%s:%s", manifestID, volume.Name, volume.MountPath)
 		}
@@ -237,10 +245,28 @@ func milliCPUToShares(milliCPU int) int {
 	return shares
 }
 
+func (kl *Kubelet) mountExternalVolumes(manifest *api.ContainerManifest) (volumeMap, error) {
+	podVolumes := make(volumeMap)
+	for _, vol := range manifest.Volumes {
+		extVolume, err := volume.CreateVolume(&vol, manifest.ID)
+		if err != nil {
+			return nil, err
+		}
+		// TODO(jonesdl) When the default volume behavior is no longer supported, this case
+		// should never occur and an error should be thrown instead.
+		if extVolume == nil {
+			continue
+		}
+		podVolumes[vol.Name] = extVolume
+		extVolume.SetUp()
+	}
+	return podVolumes, nil
+}
+
 // Run a single container from a manifest. Returns the docker container ID
-func (kl *Kubelet) runContainer(manifest *api.ContainerManifest, container *api.Container, netMode string) (id DockerID, err error) {
+func (kl *Kubelet) runContainer(manifest *api.ContainerManifest, container *api.Container, podVolumes volumeMap, netMode string) (id DockerID, err error) {
 	envVariables := makeEnvironmentVariables(container)
-	volumes, binds := makeVolumesAndBinds(manifest.ID, container)
+	volumes, binds := makeVolumesAndBinds(manifest.ID, container, podVolumes)
 	exposedPorts, portBindings := makePortsAndBindings(container)
 
 	opts := docker.CreateContainerOptions{
@@ -533,7 +559,7 @@ func (kl *Kubelet) createNetworkContainer(manifest *api.ContainerManifest) (Dock
 		Ports:   ports,
 	}
 	kl.DockerPuller.Pull("busybox")
-	return kl.runContainer(manifest, container, "")
+	return kl.runContainer(manifest, container, nil, "")
 }
 
 func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, dockerContainers DockerContainers, keepChannel chan<- DockerID) error {
@@ -550,7 +576,10 @@ func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, dockerContainer
 		netID = dockerNetworkID
 	}
 	keepChannel <- netID
-
+	podVolumes, err := kl.mountExternalVolumes(manifest)
+	if err != nil {
+		glog.Errorf("Unable to mount volumes for manifest %s: (%v)", manifest.ID, err)
+	}
 	for _, container := range manifest.Containers {
 		if dockerContainer, found := dockerContainers.FindPodContainer(manifest.ID, container.Name); found {
 			containerID := DockerID(dockerContainer.ID)
@@ -580,7 +609,7 @@ func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, dockerContainer
 			glog.Errorf("Failed to create container: %v skipping manifest %s container %s.", err, manifest.ID, container.Name)
 			continue
 		}
-		containerID, err := kl.runContainer(manifest, &container, "container:"+string(netID))
+		containerID, err := kl.runContainer(manifest, &container, podVolumes, "container:"+string(netID))
 		if err != nil {
 			// TODO(bburns) : Perhaps blacklist a container after N failures?
 			glog.Errorf("Error running manifest %s container %s: %v", manifest.ID, container.Name, err)

pkg/kubelet/kubelet_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/health"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/volume"
 	"github.com/coreos/go-etcd/etcd"
 	"github.com/fsouza/go-dockerclient"
 	"github.com/google/cadvisor/info"
@@ -528,6 +529,31 @@ func TestMakeEnvVariables(t *testing.T) {
 	}
 }
 
+func TestMountExternalVolumes(t *testing.T) {
+	kubelet, _, _ := makeTestKubelet(t)
+	manifest := api.ContainerManifest{
+		Volumes: []api.Volume{
+			{
+				Name: "host-dir",
+				Source: &api.VolumeSource{
+					HostDirectory: &api.HostDirectory{"/dir/path"},
+				},
+			},
+		},
+	}
+	podVolumes, _ := kubelet.mountExternalVolumes(&manifest)
+	expectedPodVolumes := make(volumeMap)
+	expectedPodVolumes["host-dir"] = &volume.HostDirectory{"/dir/path"}
+	if len(expectedPodVolumes) != len(podVolumes) {
+		t.Errorf("Unexpected volumes. Expected %#v got %#v.  Manifest was: %#v", expectedPodVolumes, podVolumes, manifest)
+	}
+	for name, expectedVolume := range expectedPodVolumes {
+		if _, ok := podVolumes[name]; !ok {
+			t.Errorf("Pod volumes map is missing key: %s. %#v", expectedVolume, podVolumes)
+		}
+	}
+}
+
 func TestMakeVolumesAndBinds(t *testing.T) {
 	container := api.Container{
 		VolumeMounts: []api.VolumeMount{
@@ -548,12 +574,22 @@ func TestMakeVolumesAndBinds(t *testing.T) {
 				ReadOnly:  false,
 				MountType: "HOST",
 			},
+			{
+				MountPath: "/mnt/path4",
+				Name:      "disk4",
+				ReadOnly:  false,
+			},
 		},
 	}
-	volumes, binds := makeVolumesAndBinds("pod", &container)
+
+	podVolumes := make(volumeMap)
+	podVolumes["disk4"] = &volume.HostDirectory{"/mnt/host"}
+
+	volumes, binds := makeVolumesAndBinds("pod", &container, podVolumes)
 
 	expectedVolumes := []string{"/mnt/path", "/mnt/path2"}
-	expectedBinds := []string{"/exports/pod/disk:/mnt/path", "/exports/pod/disk2:/mnt/path2:ro", "/mnt/path3:/mnt/path3"}
+	expectedBinds := []string{"/exports/pod/disk:/mnt/path", "/exports/pod/disk2:/mnt/path2:ro", "/mnt/path3:/mnt/path3",
+				  "/mnt/host:/mnt/path4"}
 	if len(volumes) != len(expectedVolumes) {
 		t.Errorf("Unexpected volumes. Expected %#v got %#v.  Container was: %#v", expectedVolumes, volumes, container)
 	}

pkg/volume/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package volume includes internal representations of external volume types
+// as well as utility methods required to mount/unmount volumes to kubelets.
+package volume

pkg/volume/volume.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/golang/glog"
+)
+
+
+// All volume types are expected to implement this interface
+type Interface interface {
+	// Prepares and mounts/unpacks the volume to a directory path.
+	// This procedure must be idempotent.
+	SetUp()
+	// Returns the directory path the volume is mounted to.
+	GetPath() string
+	// Unmounts the volume and removes traces of the SetUp procedure.
+	// This procedure must be idempotent.
+	TearDown()
+}
+
+// Host Directory volumes represent a bare host directory mount.
+// The directory in Path will be directly exposed to the container.
+type HostDirectory struct {
+	Path string
+}
+
+// Host directory mounts require no setup or cleanup, but still
+// need to fulfill the interface definitions.
+func (hostVol *HostDirectory) SetUp() {}
+
+func (hostVol *HostDirectory) TearDown() {}
+
+func (hostVol *HostDirectory) GetPath() string {
+	return hostVol.Path
+}
+
+// EmptyDirectory volumes are temporary directories exposed to the pod.
+// These do not persist beyond the lifetime of a pod.
+type EmptyDirectory struct {
+	Name string
+	PodID string
+}
+
+// SetUp creates the new directory.
+func (emptyDir *EmptyDirectory) SetUp() {
+	if _, err := os.Stat(emptyDir.GetPath()); os.IsNotExist(err) {
+		os.MkdirAll(emptyDir.GetPath(), 0750)
+	} else {
+		glog.Warningf("Directory already exists: (%v)", emptyDir.GetPath())
+	}
+}
+
+// TODO(jonesdl) when we can properly invoke TearDown(), we should delete
+// the directory created by SetUp.
+func (emptyDir *EmptyDirectory) TearDown() {}
+
+func (emptyDir *EmptyDirectory) GetPath() string {
+	// TODO(jonesdl) We will want to add a flag to designate a root
+	// directory for kubelet to write to. For now this will just be /exports
+	return fmt.Sprintf("/exports/%v/%v", emptyDir.PodID, emptyDir.Name)
+}
+
+// Interprets API volume as a HostDirectory
+func createHostDirectory(volume *api.Volume) *HostDirectory {
+	return &HostDirectory{volume.Source.HostDirectory.Path}
+}
+
+// Interprets API volume as an EmptyDirectory
+func createEmptyDirectory(volume *api.Volume, podID string) *EmptyDirectory {
+	return &EmptyDirectory{volume.Name, podID}
+}
+
+// CreateVolume returns an Interface capable of mounting a volume described by an
+// *api.Volume and whether or not it is mounted, or an error.
+func CreateVolume(volume *api.Volume, podID string) (Interface, error) {
+	source := volume.Source
+	// TODO(jonesdl) We will want to throw an error here when we no longer
+	// support the default behavior.
+	if source == nil {
+		return nil, nil
+	}
+	var vol Interface
+	// TODO(jonesdl) We should probably not check every pointer and directly
+	// resolve these types instead.
+	if source.HostDirectory != nil {
+		vol = createHostDirectory(volume)
+	} else if source.EmptyDirectory != nil {
+		vol = createEmptyDirectory(volume, podID)
+	} else {
+		return nil, errors.New("Unsupported volume type.")
+	}
+	return vol, nil
+}

pkg/volume/volume_test.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2014 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"testing"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+)
+
+func TestCreateVolumes(t *testing.T) {
+	volumes := []api.Volume{
+		{
+			Name: "host-dir",
+			Source: &api.VolumeSource{
+				HostDirectory: &api.HostDirectory{"/dir/path"},
+			},
+		},
+		{
+			Name: "empty-dir",
+			Source: &api.VolumeSource{
+				EmptyDirectory: &api.EmptyDirectory{},
+			},
+		},
+	}
+	fakePodID := "my-id"
+	expectedPaths := []string{"/dir/path", "/exports/my-id/empty-dir"}
+	for i, volume := range volumes {
+		extVolume, _ := CreateVolume(&volume, fakePodID)
+		expectedPath := expectedPaths[i]
+		path := extVolume.GetPath()
+		if expectedPath != path {
+			t.Errorf("Unexpected bind path. Expected %v, got %v", expectedPath, path)
+		}
+	}
+}