From f84ff740f06af0bf1312498b95861ad41bde7a71 Mon Sep 17 00:00:00 2001 From: Danny Jones Date: Mon, 14 Jul 2014 18:39:30 -0700 Subject: [PATCH] Adds initial volumes package; Supports host-dirs Adds the framework for external volume mounts. Currently supports bare host directory mounts. Modifies the API to support host directory mounts from Volumes instead of VolumeMounts. --- pkg/api/types.go | 10 ++++++ pkg/api/validation.go | 17 ++++++++++ pkg/api/validation_test.go | 9 ++--- pkg/kubelet/kubelet.go | 37 +++++++++++++++++---- pkg/kubelet/kubelet_test.go | 40 +++++++++++++++++++++-- pkg/volume/doc.go | 19 +++++++++++ pkg/volume/volume.go | 65 +++++++++++++++++++++++++++++++++++++ pkg/volume/volume_test.go | 40 +++++++++++++++++++++++ 8 files changed, 224 insertions(+), 13 deletions(-) create mode 100644 pkg/volume/doc.go create mode 100644 pkg/volume/volume.go create mode 100644 pkg/volume/volume_test.go diff --git a/pkg/api/types.go b/pkg/api/types.go index bdaec3756ae..3ad179e3b0d 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -62,6 +62,15 @@ type Volume struct { // Required: This must be a DNS_LABEL. Each volume in a pod must have // a unique name. Name string `yaml:"name" json:"name"` + // When multiple volume types are supported, only one of them may be specified. + HostDirectory *HostDirectory `yaml:"hostDir" json:"hostDir"` + // DEPRECATED: If no volume type is specified, HostDirectory will be assumed. + // The path of the directory will be specified in the VolumeMount struct in this case. +} + +// Bare host directory volume. +type HostDirectory struct { + Path string `yaml:"path" json:"path"` } // Port represents a network port in a single container @@ -92,6 +101,7 @@ type VolumeMount struct { MountPath string `yaml:"mountPath,omitempty" json:"mountPath,omitempty"` Path string `yaml:"path,omitempty" json:"path,omitempty"` // One of: "LOCAL" (local volume) or "HOST" (external mount from the host). Default: LOCAL. + // DEPRECATED: MountType will be removed in a future version of the API. MountType string `yaml:"mountType,omitempty" json:"mountType,omitempty"` } diff --git a/pkg/api/validation.go b/pkg/api/validation.go index a29dec05c0d..92dd44e40b4 100644 --- a/pkg/api/validation.go +++ b/pkg/api/validation.go @@ -76,6 +76,10 @@ func validateVolumes(volumes []Volume) (util.StringSet, errorList) { allNames := util.StringSet{} for i := range volumes { vol := &volumes[i] // so we can set default values + if vol.HostDirectory != nil { + errs := validateHostDir(vol.HostDirectory) + allErrs.Append(errs...) + } if !util.IsDNSLabel(vol.Name) { allErrs.Append(makeInvalidError("Volume.Name", vol.Name)) } else if allNames.Has(vol.Name) { @@ -87,6 +91,14 @@ func validateVolumes(volumes []Volume) (util.StringSet, errorList) { return allNames, allErrs } +func validateHostDir(hostDir *HostDirectory) errorList { + allErrs := errorList{} + if hostDir.Path == "" { + allErrs.Append(makeNotFoundError("Volume.HostDir.Path", hostDir.Path)) + } + return allErrs +} + var supportedPortProtocols = util.NewStringSet("TCP", "UDP") func validatePorts(ports []Port) errorList { @@ -163,6 +175,11 @@ func validateVolumeMounts(mounts []VolumeMount, volumes util.StringSet) errorLis mnt.Path = "" } } + if len(mnt.MountType) != 0 { + glog.Warning("DEPRECATED: VolumeMount.MountType will be removed. The Volume struct will handle types") + } + + } return allErrs } diff --git a/pkg/api/validation_test.go b/pkg/api/validation_test.go index 84ae5ff20c5..eabd0cba37a 100644 --- a/pkg/api/validation_test.go +++ b/pkg/api/validation_test.go @@ -25,9 +25,9 @@ import ( func TestValidateVolumes(t *testing.T) { successCase := []Volume{ - {Name: "abc"}, - {Name: "123"}, - {Name: "abc-123"}, + {Name: "abc", HostDirectory: &HostDirectory{"/mnt/path1"}}, + {Name: "123", HostDirectory: &HostDirectory{"/mnt/path2"}}, + {Name: "abc-123", HostDirectory: &HostDirectory{"/mnt/path3"}}, } names, errs := validateVolumes(successCase) if len(errs) != 0 { @@ -206,7 +206,8 @@ func TestValidateManifest(t *testing.T) { { Version: "v1beta1", ID: "abc", - Volumes: []Volume{{Name: "vol1"}, {Name: "vol2"}}, + Volumes: []Volume{{Name: "vol1", HostDirectory: &HostDirectory{"/mnt/vol1"}}, + {Name: "vol2", HostDirectory: &HostDirectory{"/mnt/vol2"}}}, Containers: []Container{ { Name: "abc", diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6909411121e..65e3644d2bf 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -36,6 +36,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/health" "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/GoogleCloudPlatform/kubernetes/pkg/volume" "github.com/coreos/go-etcd/etcd" "github.com/fsouza/go-dockerclient" "github.com/golang/glog" @@ -62,6 +63,8 @@ func New() *Kubelet { return &Kubelet{} } +type volumeMap map[string]volume.Interface + // Kubelet is the main kubelet implementation. type Kubelet struct { Hostname string @@ -74,6 +77,7 @@ type Kubelet struct { HTTPCheckFrequency time.Duration pullLock sync.Mutex HealthChecker health.HealthChecker + extVolumes map[string]volumes.Interface } type manifestUpdate struct { @@ -178,13 +182,16 @@ func makeEnvironmentVariables(container *api.Container) []string { return result } -func makeVolumesAndBinds(manifestID string, container *api.Container) (map[string]struct{}, []string) { +func makeVolumesAndBinds(manifestID string, container *api.Container, podVolumes volumeMap) (map[string]struct{}, []string) { volumes := map[string]struct{}{} binds := []string{} for _, volume := range container.VolumeMounts { var basePath string - if volume.MountType == "HOST" { + if hostVol, ok := podVolumes[volume.Name]; ok { // Host volumes are not Docker volumes and are directly mounted from the host. + basePath = fmt.Sprintf("%s:%s", hostVol.GetPath(), volume.MountPath) + } else if volume.MountType == "HOST" { + // DEPRECATED: VolumeMount.MountType will be moved to the Volume struct. basePath = fmt.Sprintf("%s:%s", volume.MountPath, volume.MountPath) } else { volumes[volume.MountPath] = struct{}{} @@ -237,10 +244,26 @@ func milliCPUToShares(milliCPU int) int { return shares } +func (kl *Kubelet) mountExternalVolumes(manifest *api.ContainerManifest) (volumeMap, error) { + podVolumes := make(volumeMap) + for _, vol := range manifest.Volumes { + extVolume, err := volume.CreateVolume(&vol, manifest.ID) + if err != nil { + return nil, err + } + podVolumes[vol.Name] = extVolume + // Only prepare the volume if it is not already mounted. + if _, err := os.Stat(extVolume.GetPath()); os.IsNotExist(err) { + extVolume.SetUp() + } + } + return podVolumes, nil +} + // Run a single container from a manifest. Returns the docker container ID -func (kl *Kubelet) runContainer(manifest *api.ContainerManifest, container *api.Container, netMode string) (id DockerID, err error) { +func (kl *Kubelet) runContainer(manifest *api.ContainerManifest, container *api.Container, podVolumes volumeMap, netMode string) (id DockerID, err error) { envVariables := makeEnvironmentVariables(container) - volumes, binds := makeVolumesAndBinds(manifest.ID, container) + volumes, binds := makeVolumesAndBinds(manifest.ID, container, podVolumes) exposedPorts, portBindings := makePortsAndBindings(container) opts := docker.CreateContainerOptions{ @@ -540,7 +563,7 @@ func (kl *Kubelet) createNetworkContainer(manifest *api.ContainerManifest) (Dock Ports: ports, } kl.DockerPuller.Pull("busybox") - return kl.runContainer(manifest, container, "") + return kl.runContainer(manifest, container, nil, "") } func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, dockerContainers DockerContainers, keepChannel chan<- DockerID) error { @@ -557,7 +580,7 @@ func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, dockerContainer netID = dockerNetworkID } keepChannel <- netID - + podVolumes, err := kl.mountExternalVolumes(manifest) for _, container := range manifest.Containers { if dockerContainer, found := dockerContainers.FindPodContainer(manifest.ID, container.Name); found { containerID := DockerID(dockerContainer.ID) @@ -587,7 +610,7 @@ func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, dockerContainer glog.Errorf("Failed to create container: %v skipping manifest %s container %s.", err, manifest.ID, container.Name) continue } - containerID, err := kl.runContainer(manifest, &container, "container:"+string(netID)) + containerID, err := kl.runContainer(manifest, &container, podVolumes, "container:"+string(netID)) if err != nil { // TODO(bburns) : Perhaps blacklist a container after N failures? glog.Errorf("Error running manifest %s container %s: %v", manifest.ID, container.Name, err) diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 1777e6fec4c..26dae7911db 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -29,6 +29,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/health" "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/GoogleCloudPlatform/kubernetes/pkg/volume" "github.com/coreos/go-etcd/etcd" "github.com/fsouza/go-dockerclient" "github.com/google/cadvisor/info" @@ -528,6 +529,31 @@ func TestMakeEnvVariables(t *testing.T) { } } +func TestMountExternalVolumes(t *testing.T) { + kubelet, _, _ := makeTestKubelet(t) + manifest := api.ContainerManifest{ + Volumes: []api.Volume{ + { + Name: "host-dir", + HostDirectory: &api.HostDirectory{"/dir/path"}, + }, + }, + } + podVolumes, _ := kubelet.mountExternalVolumes(&manifest) + expectedPodVolumes := make(map[string]volumes.Interface) + expectedPodVolumes["host-dir"] = &volumes.HostDirectoryVolume{"/dir/path"} + if len(expectedPodVolumes) != len(podVolumes) { + t.Errorf("Unexpected volumes. Expected %#v got %#v. Manifest was: %#v", expectedPodVolumes, podVolumes, manifest) + } + for name, expectedVolume := range expectedPodVolumes { + if _, ok := podVolumes[name]; !ok { + t.Errorf("Pod volumes map is missing key: %s. %#v", expectedVolume, podVolumes) + } + } +} + + + func TestMakeVolumesAndBinds(t *testing.T) { container := api.Container{ VolumeMounts: []api.VolumeMount{ @@ -548,12 +574,22 @@ func TestMakeVolumesAndBinds(t *testing.T) { ReadOnly: false, MountType: "HOST", }, + { + MountPath: "/mnt/path4", + Name: "disk4", + ReadOnly: false, + }, }, } - volumes, binds := makeVolumesAndBinds("pod", &container) + + podVolumes := make(volumeMap) + podVolumes["disk4"] = &volume.HostDirectory{"/mnt/host"} + + volumes, binds := makeVolumesAndBinds("pod", &container, podVolumes) expectedVolumes := []string{"/mnt/path", "/mnt/path2"} - expectedBinds := []string{"/exports/pod/disk:/mnt/path", "/exports/pod/disk2:/mnt/path2:ro", "/mnt/path3:/mnt/path3"} + expectedBinds := []string{"/exports/pod/disk:/mnt/path", "/exports/pod/disk2:/mnt/path2:ro", "/mnt/path3:/mnt/path3", + "/mnt/host:/mnt/path4"} if len(volumes) != len(expectedVolumes) { t.Errorf("Unexpected volumes. Expected %#v got %#v. Container was: %#v", expectedVolumes, volumes, container) } diff --git a/pkg/volume/doc.go b/pkg/volume/doc.go new file mode 100644 index 00000000000..e0f8d95904c --- /dev/null +++ b/pkg/volume/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package volumes includes internal representations of external volume types +// as well as utility methods required to mount/unmount volumes to kubelets. +package volume diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go new file mode 100644 index 00000000000..878ce37ed7a --- /dev/null +++ b/pkg/volume/volume.go @@ -0,0 +1,65 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumes + +import ( + "errors" + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" +) + + +// All volume types are expected to implement this interface +type Interface interface { + // Prepares and mounts/unpacks the volume to a directory path. + SetUp() + // Returns the directory path the volume is mounted to. + GetPath() string + // Unmounts the volume and removes traces of the SetUp procedure. + TearDown() +} + +// Host Directory Volumes represent a bare host directory mount. +type HostDirectoryVolume struct { + Path string +} + +// Simple host directory mounts require no setup or cleanup, but still +// need to fulfill the interface definitions. +func (hostVol *HostDirectoryVolume) SetUp() {} + +func (hostVol *HostDirectoryVolume) TearDown() {} + +func (hostVol *HostDirectoryVolume) GetPath() string { + return hostVol.Path +} + +// Interprets API volume as a HostDirectory +func createHostDirectoryVolume(volume *api.Volume) *HostDirectoryVolume { + return &HostDirectoryVolume{volume.HostDirectory.Path} +} + +// Interprets parameters passed in the API as an internal structure +// with utility procedures for mounting. +func CreateVolume(volume *api.Volume) (Interface, error) { + // TODO(jonesdl) We should probably not check every + // pointer and directly resolve these types instead. + if volume.HostDirectory != nil { + return createHostDirectoryVolume(volume), nil + } else { + return nil, errors.New("Unsupported volume type.") + } +} diff --git a/pkg/volume/volume_test.go b/pkg/volume/volume_test.go new file mode 100644 index 00000000000..280beba160e --- /dev/null +++ b/pkg/volume/volume_test.go @@ -0,0 +1,40 @@ +opyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumes + +import ( + "testing" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" +) + +func TestCreateVolumes(t *testing.T) { + volumes := []api.Volume{ + { + Name: "host-dir", + HostDirectory: &api.HostDirectory{"/dir/path"}, + }, + } + expectedPaths := []string{"/dir/path"} + for i, volume := range volumes { + extVolume, _ := CreateVolume(&volume) + expectedPath := expectedPaths[i] + path := extVolume.GetPath() + if expectedPath != path { + t.Errorf("Unexpected bind path. Expected %v, got %v", expectedPath, path) + } + } +}