Merge pull request #25852 from vishh/network-volumes

Add metrics support for a GCE PD, EC2 EBS & Azure File volumes

pkg/volume/aws_ebs/aws_ebs.go

@@ -30,7 +30,7 @@ import (
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/mount"
-	utilstrings "k8s.io/kubernetes/pkg/util/strings"
+	kstrings "k8s.io/kubernetes/pkg/util/strings"
 	"k8s.io/kubernetes/pkg/volume"
 )
 
@@ -52,6 +52,10 @@ const (
 	awsElasticBlockStorePluginName = "kubernetes.io/aws-ebs"
 )
 
+func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
+	return host.GetPodVolumeDir(uid, kstrings.EscapeQualifiedNameForDisk(awsElasticBlockStorePluginName), volName)
+}
+
 func (plugin *awsElasticBlockStorePlugin) Init(host volume.VolumeHost) error {
 	plugin.host = host
 	return nil
@@ -99,13 +103,14 @@ func (plugin *awsElasticBlockStorePlugin) newMounterInternal(spec *volume.Spec,
 
 	return &awsElasticBlockStoreMounter{
 		awsElasticBlockStore: &awsElasticBlockStore{
-			podUID:    podUID,
+			podUID:          podUID,
-			volName:   spec.Name(),
+			volName:         spec.Name(),
-			volumeID:  volumeID,
+			volumeID:        volumeID,
-			partition: partition,
+			partition:       partition,
-			manager:   manager,
+			manager:         manager,
-			mounter:   mounter,
+			mounter:         mounter,
-			plugin:    plugin,
+			plugin:          plugin,
+			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
 		},
 		fsType:      fsType,
 		readOnly:    readOnly,
@@ -119,11 +124,12 @@ func (plugin *awsElasticBlockStorePlugin) NewUnmounter(volName string, podUID ty
 
 func (plugin *awsElasticBlockStorePlugin) newUnmounterInternal(volName string, podUID types.UID, manager ebsManager, mounter mount.Interface) (volume.Unmounter, error) {
 	return &awsElasticBlockStoreUnmounter{&awsElasticBlockStore{
-		podUID:  podUID,
+		podUID:          podUID,
-		volName: volName,
+		volName:         volName,
-		manager: manager,
+		manager:         manager,
-		mounter: mounter,
+		mounter:         mounter,
-		plugin:  plugin,
+		plugin:          plugin,
+		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
 	}}, nil
 }
 
@@ -187,7 +193,7 @@ type awsElasticBlockStore struct {
 	// Mounter interface that provides system calls to mount the global path to the pod local path.
 	mounter mount.Interface
 	plugin  *awsElasticBlockStorePlugin
-	volume.MetricsNil
+	volume.MetricsProvider
 }
 
 func detachDiskLogError(ebs *awsElasticBlockStore) {
@@ -313,8 +319,7 @@ func getVolumeIDFromGlobalMount(host volume.VolumeHost, globalPath string) (stri
 }
 
 func (ebs *awsElasticBlockStore) GetPath() string {
-	name := awsElasticBlockStorePluginName
+	return getPath(ebs.podUID, ebs.volName, ebs.plugin.host)
-	return ebs.plugin.host.GetPodVolumeDir(ebs.podUID, utilstrings.EscapeQualifiedNameForDisk(name), ebs.volName)
 }
 
 type awsElasticBlockStoreUnmounter struct {
@@ -392,8 +397,7 @@ type awsElasticBlockStoreDeleter struct {
 var _ volume.Deleter = &awsElasticBlockStoreDeleter{}
 
 func (d *awsElasticBlockStoreDeleter) GetPath() string {
-	name := awsElasticBlockStorePluginName
+	return getPath(d.podUID, d.volName, d.plugin.host)
-	return d.plugin.host.GetPodVolumeDir(d.podUID, utilstrings.EscapeQualifiedNameForDisk(name), d.volName)
 }
 
 func (d *awsElasticBlockStoreDeleter) Delete() error {

pkg/volume/azure_file/azure_file.go

@@ -23,7 +23,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/mount"
-	"k8s.io/kubernetes/pkg/util/strings"
+	kstrings "k8s.io/kubernetes/pkg/util/strings"
 	"k8s.io/kubernetes/pkg/volume"
 
 	"github.com/golang/glog"
@@ -45,6 +45,10 @@ const (
 	azureFilePluginName = "kubernetes.io/azure-file"
 )
 
+func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
+	return host.GetPodVolumeDir(uid, kstrings.EscapeQualifiedNameForDisk(azureFilePluginName), volName)
+}
+
 func (plugin *azureFilePlugin) Init(host volume.VolumeHost) error {
 	plugin.host = host
 	return nil
@@ -84,10 +88,11 @@ func (plugin *azureFilePlugin) newMounterInternal(spec *volume.Spec, pod *api.Po
 	}
 	return &azureFileMounter{
 		azureFile: &azureFile{
-			volName: spec.Name(),
+			volName:         spec.Name(),
-			mounter: mounter,
+			mounter:         mounter,
-			pod:     pod,
+			pod:             pod,
-			plugin:  plugin,
+			plugin:          plugin,
+			MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host)),
 		},
 		util:       util,
 		secretName: source.SecretName,
@@ -102,10 +107,11 @@ func (plugin *azureFilePlugin) NewUnmounter(volName string, podUID types.UID) (v
 
 func (plugin *azureFilePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
 	return &azureFileUnmounter{&azureFile{
-		volName: volName,
+		volName:         volName,
-		mounter: mounter,
+		mounter:         mounter,
-		pod:     &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
+		pod:             &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
-		plugin:  plugin,
+		plugin:          plugin,
+		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
 	}}, nil
 }
 
@@ -115,12 +121,11 @@ type azureFile struct {
 	pod     *api.Pod
 	mounter mount.Interface
 	plugin  *azureFilePlugin
-	volume.MetricsNil
+	volume.MetricsProvider
 }
 
 func (azureFileVolume *azureFile) GetPath() string {
-	name := azureFilePluginName
+	return getPath(azureFileVolume.pod.UID, azureFileVolume.volName, azureFileVolume.plugin.host)
-	return azureFileVolume.plugin.host.GetPodVolumeDir(azureFileVolume.pod.UID, strings.EscapeQualifiedNameForDisk(name), azureFileVolume.volName)
 }
 
 type azureFileMounter struct {

pkg/volume/empty_dir/empty_dir.go

@@ -54,6 +54,10 @@ const (
 	emptyDirPluginName = "kubernetes.io/empty-dir"
 )
 
+func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
+	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(emptyDirPluginName), volName)
+}
+
 func (plugin *emptyDirPlugin) Init(host volume.VolumeHost) error {
 	plugin.host = host
 
@@ -88,7 +92,7 @@ func (plugin *emptyDirPlugin) newMounterInternal(spec *volume.Spec, pod *api.Pod
 		mountDetector:   mountDetector,
 		plugin:          plugin,
 		rootContext:     opts.RootContext,
-		MetricsProvider: volume.NewMetricsDu(GetPath(pod.UID, spec.Name(), plugin.host)),
+		MetricsProvider: volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host)),
 	}, nil
 }
 
@@ -105,7 +109,7 @@ func (plugin *emptyDirPlugin) newUnmounterInternal(volName string, podUID types.
 		mounter:         mounter,
 		mountDetector:   mountDetector,
 		plugin:          plugin,
-		MetricsProvider: volume.NewMetricsDu(GetPath(podUID, volName, plugin.host)),
+		MetricsProvider: volume.NewMetricsDu(getPath(podUID, volName, plugin.host)),
 	}
 	return ed, nil
 }
@@ -271,12 +275,7 @@ func (ed *emptyDir) setupDir(dir string) error {
 }
 
 func (ed *emptyDir) GetPath() string {
-	return GetPath(ed.pod.UID, ed.volName, ed.plugin.host)
+	return getPath(ed.pod.UID, ed.volName, ed.plugin.host)
-}
-
-func GetPath(uid types.UID, volName string, host volume.VolumeHost) string {
-	name := emptyDirPluginName
-	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(name), volName)
 }
 
 // TearDown simply discards everything in the directory.

pkg/volume/gce_pd/gce_pd.go

@@ -49,6 +49,10 @@ const (
 	gcePersistentDiskPluginName = "kubernetes.io/gce-pd"
 )
 
+func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
+	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(gcePersistentDiskPluginName), volName)
+}
+
 func (plugin *gcePersistentDiskPlugin) Init(host volume.VolumeHost) error {
 	plugin.host = host
 	return nil
@@ -103,13 +107,14 @@ func (plugin *gcePersistentDiskPlugin) newMounterInternal(spec *volume.Spec, pod
 
 	return &gcePersistentDiskMounter{
 		gcePersistentDisk: &gcePersistentDisk{
-			podUID:    podUID,
+			podUID:          podUID,
-			volName:   spec.Name(),
+			volName:         spec.Name(),
-			pdName:    pdName,
+			pdName:          pdName,
-			partition: partition,
+			partition:       partition,
-			mounter:   mounter,
+			mounter:         mounter,
-			manager:   manager,
+			manager:         manager,
-			plugin:    plugin,
+			plugin:          plugin,
+			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
 		},
 		readOnly: readOnly}, nil
 }
@@ -121,11 +126,12 @@ func (plugin *gcePersistentDiskPlugin) NewUnmounter(volName string, podUID types
 
 func (plugin *gcePersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) {
 	return &gcePersistentDiskUnmounter{&gcePersistentDisk{
-		podUID:  podUID,
+		podUID:          podUID,
-		volName: volName,
+		volName:         volName,
-		manager: manager,
+		manager:         manager,
-		mounter: mounter,
+		mounter:         mounter,
-		plugin:  plugin,
+		plugin:          plugin,
+		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
 	}}, nil
 }
 
@@ -185,7 +191,7 @@ type gcePersistentDisk struct {
 	// Mounter interface that provides system calls to mount the global path to the pod local path.
 	mounter mount.Interface
 	plugin  *gcePersistentDiskPlugin
-	volume.MetricsNil
+	volume.MetricsProvider
 }
 
 type gcePersistentDiskMounter struct {
@@ -270,9 +276,8 @@ func makeGlobalPDName(host volume.VolumeHost, devName string) string {
 	return path.Join(host.GetPluginDir(gcePersistentDiskPluginName), "mounts", devName)
 }
 
-func (pd *gcePersistentDisk) GetPath() string {
+func (b *gcePersistentDiskMounter) GetPath() string {
-	name := gcePersistentDiskPluginName
+	return getPath(b.podUID, b.volName, b.plugin.host)
-	return pd.plugin.host.GetPodVolumeDir(pd.podUID, strings.EscapeQualifiedNameForDisk(name), pd.volName)
 }
 
 type gcePersistentDiskUnmounter struct {
@@ -281,7 +286,12 @@ type gcePersistentDiskUnmounter struct {
 
 var _ volume.Unmounter = &gcePersistentDiskUnmounter{}
 
-// TearDown unmounts the bind mount
+func (c *gcePersistentDiskUnmounter) GetPath() string {
+	return getPath(c.podUID, c.volName, c.plugin.host)
+}
+
+// Unmounts the bind mount, and detaches the disk only if the PD
+// resource was the last reference to that disk on the kubelet.
 func (c *gcePersistentDiskUnmounter) TearDown() error {
 	return c.TearDownAt(c.GetPath())
 }
@@ -316,8 +326,7 @@ type gcePersistentDiskDeleter struct {
 var _ volume.Deleter = &gcePersistentDiskDeleter{}
 
 func (d *gcePersistentDiskDeleter) GetPath() string {
-	name := gcePersistentDiskPluginName
+	return getPath(d.podUID, d.volName, d.plugin.host)
-	return d.plugin.host.GetPodVolumeDir(d.podUID, strings.EscapeQualifiedNameForDisk(name), d.volName)
 }
 
 func (d *gcePersistentDiskDeleter) Delete() error {

pkg/volume/metrics_du.go

@@ -72,7 +72,7 @@ func (md *metricsDu) runDu(metrics *Metrics) error {
 
 // getFsInfo writes metrics.Capacity and metrics.Available from the filesystem info
 func (md *metricsDu) getFsInfo(metrics *Metrics) error {
-	available, capacity, err := util.FsInfo(md.path)
+	available, capacity, _, err := util.FsInfo(md.path)
 	if err != nil {
 		return fmt.Errorf("Failed to get FsInfo due to error %v", err)
 	}

pkg/volume/metrics_statfs.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volume
+
+import (
+	"errors"
+	"fmt"
+
+	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/volume/util"
+)
+
+var _ MetricsProvider = &metricsStatFS{}
+
+// metricsStatFS represents a MetricsProvider that calculates the used and available
+// Volume space by stat'ing and gathering filesystem info for the Volume path.
+type metricsStatFS struct {
+	// the directory path the volume is mounted to.
+	path string
+}
+
+// NewMetricsStatfs creates a new metricsStatFS with the Volume path.
+func NewMetricsStatFS(path string) MetricsProvider {
+	return &metricsStatFS{path}
+}
+
+// See MetricsProvider.GetMetrics
+// GetMetrics calculates the volume usage and device free space by executing "du"
+// and gathering filesystem info for the Volume path.
+func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
+	metrics := &Metrics{}
+	if md.path == "" {
+		return metrics, errors.New("no path defined for disk usage metrics.")
+	}
+
+	err := md.getFsInfo(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
+	return metrics, nil
+}
+
+// getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info
+func (md *metricsStatFS) getFsInfo(metrics *Metrics) error {
+	available, capacity, usage, err := util.FsInfo(md.path)
+	if err != nil {
+		return fmt.Errorf("Failed to get FsInfo due to error %v", err)
+	}
+	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
+	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
+	metrics.Used = resource.NewQuantity(usage, resource.BinarySI)
+	return nil
+}

pkg/volume/secret/secret.go

@@ -49,6 +49,10 @@ var wrappedVolumeSpec = volume.Spec{
 	Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}},
 }
 
+func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
+	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(secretPluginName), volName)
+}
+
 func (plugin *secretPlugin) Init(host volume.VolumeHost) error {
 	plugin.host = host
 	return nil
@@ -70,7 +74,7 @@ func (plugin *secretPlugin) NewMounter(spec *volume.Spec, pod *api.Pod, opts vol
 			plugin,
 			plugin.host.GetMounter(),
 			plugin.host.GetWriter(),
-			volume.NewCachedMetrics(volume.NewMetricsDu(getPathFromHost(plugin.host, pod.UID, spec.Name()))),
+			volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
 		},
 		source: *spec.Volume.Secret,
 		pod:    *pod,
@@ -86,7 +90,7 @@ func (plugin *secretPlugin) NewUnmounter(volName string, podUID types.UID) (volu
 			plugin,
 			plugin.host.GetMounter(),
 			plugin.host.GetWriter(),
-			volume.NewCachedMetrics(volume.NewMetricsDu(getPathFromHost(plugin.host, podUID, volName))),
+			volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
 		},
 	}, nil
 }
@@ -103,11 +107,7 @@ type secretVolume struct {
 var _ volume.Volume = &secretVolume{}
 
 func (sv *secretVolume) GetPath() string {
-	return getPathFromHost(sv.plugin.host, sv.podUID, sv.volName)
+	return getPath(sv.podUID, sv.volName, sv.plugin.host)
-}
-
-func getPathFromHost(host volume.VolumeHost, podUID types.UID, volName string) string {
-	return host.GetPodVolumeDir(podUID, strings.EscapeQualifiedNameForDisk(secretPluginName), volName)
 }
 
 // secretVolumeMounter handles retrieving secrets from the API server

pkg/volume/util/fs.go

@@ -1,4 +1,4 @@
-// +build linux
+// +build linux darwin
 
 /*
 Copyright 2014 The Kubernetes Authors All rights reserved.
@@ -27,22 +27,25 @@ import (
 	"k8s.io/kubernetes/pkg/api/resource"
 )
 
-// FSInfo linux returns (available bytes, byte capacity, error) for the filesystem that
+// FSInfo linux returns (available bytes, byte capacity, byte usage, error) for the filesystem that
 // path resides upon.
-func FsInfo(path string) (int64, int64, error) {
+func FsInfo(path string) (int64, int64, int64, error) {
 	statfs := &syscall.Statfs_t{}
 	err := syscall.Statfs(path, statfs)
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, 0, err
 	}
-
+	// TODO(vishh): Include inodes space
 	// Available is blocks available * fragment size
-	available := int64(statfs.Bavail) * int64(statfs.Frsize)
+	available := int64(statfs.Bavail) * int64(statfs.Bsize)
 
 	// Capacity is total block count * fragment size
-	capacity := int64(statfs.Blocks) * int64(statfs.Frsize)
+	capacity := int64(statfs.Blocks) * int64(statfs.Bsize)
 
-	return available, capacity, nil
+	// Usage is block being used * fragment size (aka block size).
+	usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize)
+
+	return available, capacity, usage, nil
 }
 
 func Du(path string) (*resource.Quantity, error) {

pkg/volume/util/fs_darwin.go

@@ -1,47 +0,0 @@
-// +build darwin
-
-/*
-Copyright 2016 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"errors"
-	"fmt"
-	"os/exec"
-	"strings"
-
-	"k8s.io/kubernetes/pkg/api/resource"
-)
-
-// FSInfo linux returns (available bytes, byte capacity, error) for the filesystem that
-// path resides upon.
-func FsInfo(path string) (int64, int64, error) {
-	return 0, 0, errors.New("FsInfo not supported for this build.")
-}
-
-func Du(path string) (*resource.Quantity, error) {
-	out, err := exec.Command("nice", "-n", "19", "du", "-s", path).CombinedOutput()
-	if err != nil {
-		return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -s) on path %s with error %v", path, err)
-	}
-	used, err := resource.ParseQuantity(strings.Fields(string(out))[0])
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err)
-	}
-	used.Format = resource.BinarySI
-	return &used, nil
-}

pkg/volume/util/fs_unsupported.go

@@ -26,8 +26,8 @@ import (
 )
 
 // FSInfo unsupported returns 0 values for available and capacity and an error.
-func FsInfo(path string) (int64, int64, error) {
+func FsInfo(path string) (int64, int64, int64, error) {
-	return 0, 0, errors.New("FsInfo not supported for this build.")
+	return 0, 0, 0, errors.New("FsInfo not supported for this build.")
 }
 
 func Du(path string) (*resource.Quantity, error) {