From 98392532335407f948ade593cb8cd777673600ec Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 25 May 2015 08:13:30 -0400 Subject: [PATCH 1/2] Add logging to volume tear-down to help understand mount behaviour --- pkg/util/mount/mount.go | 12 +++++++++--- pkg/volume/aws_ebs/aws_ebs.go | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 9733f5a02eb..73311749b48 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -18,6 +18,8 @@ limitations under the License. // an alternate platform, we will need to abstract further. package mount +import "github.com/golang/glog" + type Interface interface { // Mount mounts source to target as fstype with given options. Mount(source string, target string, fstype string, options []string) error @@ -66,9 +68,13 @@ func GetMountRefs(mounter Interface, mountPath string) ([]string, error) { // Find all references to the device. var refs []string - for i := range mps { - if mps[i].Device == deviceName && mps[i].Path != mountPath { - refs = append(refs, mps[i].Path) + if deviceName == "" { + glog.Warningf("could not determine device for path: %s", mountPath) + } else { + for i := range mps { + if mps[i].Device == deviceName && mps[i].Path != mountPath { + refs = append(refs, mps[i].Path) + } } } return refs, nil diff --git a/pkg/volume/aws_ebs/aws_ebs.go b/pkg/volume/aws_ebs/aws_ebs.go index f276db68e31..b10ed14a422 100644 --- a/pkg/volume/aws_ebs/aws_ebs.go +++ b/pkg/volume/aws_ebs/aws_ebs.go @@ -289,6 +289,9 @@ func (pd *awsElasticBlockStore) TearDownAt(dir string) error { glog.V(2).Info("Error getting mountrefs for ", dir, ": ", err) return err } + if len(refs) == 0 { + glog.Warning("Did not find pod-mount for ", dir, " during tear-down") + } // Unmount the bind-mount inside this pod if err := pd.mounter.Unmount(dir); err != nil { glog.V(2).Info("Error unmounting dir ", dir, ": ", err) @@ -307,6 +310,8 @@ func (pd *awsElasticBlockStore) TearDownAt(dir string) error { glog.V(2).Info("Error detaching disk ", pd.volumeID, ": ", err) return err } + } else { + glog.V(2).Infof("Found multiple refs; won't detach EBS volume: %v", refs) } mountpoint, mntErr := pd.mounter.IsMountPoint(dir) if mntErr != nil { From c4a2631593bf83a274e228bd008bafe2d5b9a8b7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 25 May 2015 08:15:26 -0400 Subject: [PATCH 2/2] Mount logic breaks if /var/lib/kubelet is a symlink Pass the correct kubelet root-dir on AWS --- cluster/aws/templates/format-disks.sh | 1 + cluster/aws/templates/salt-master.sh | 6 ++++++ cluster/aws/templates/salt-minion.sh | 6 ++++++ cluster/saltbase/salt/kubelet/default | 7 ++++++- pkg/util/mount/mount.go | 2 +- 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/cluster/aws/templates/format-disks.sh b/cluster/aws/templates/format-disks.sh index be2cd5d99c2..32590ad8840 100644 --- a/cluster/aws/templates/format-disks.sh +++ b/cluster/aws/templates/format-disks.sh @@ -75,5 +75,6 @@ else fi mkdir -p /mnt/kubelet ln -s /mnt/kubelet /var/lib/kubelet + KUBELET_ROOT="/mnt/kubelet" fi diff --git a/cluster/aws/templates/salt-master.sh b/cluster/aws/templates/salt-master.sh index 6aa5f3415a7..eaa15f5eb08 100755 --- a/cluster/aws/templates/salt-master.sh +++ b/cluster/aws/templates/salt-master.sh @@ -38,6 +38,12 @@ if [[ -n "${DOCKER_ROOT}" ]]; then EOF fi +if [[ -n "${KUBELET_ROOT}" ]]; then + cat <>/etc/salt/minion.d/grains.conf + kubelet_root: '$(echo "$KUBELET_ROOT" | sed -e "s/'/''/g")' +EOF +fi + # Auto accept all keys from minions that try to join mkdir -p /etc/salt/master.d cat </etc/salt/master.d/auto-accept.conf diff --git a/cluster/aws/templates/salt-minion.sh b/cluster/aws/templates/salt-minion.sh index 763114da4bd..0462ab9ad72 100755 --- a/cluster/aws/templates/salt-minion.sh +++ b/cluster/aws/templates/salt-minion.sh @@ -55,6 +55,12 @@ if [[ -n "${DOCKER_ROOT}" ]]; then EOF fi +if [[ -n "${KUBELET_ROOT}" ]]; then + cat <>/etc/salt/minion.d/grains.conf + kubelet_root: '$(echo "$KUBELET_ROOT" | sed -e "s/'/''/g")' +EOF +fi + install-salt service salt-minion start diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 02c39c49821..53f64765f35 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -53,6 +53,11 @@ {% set docker_root = " --docker_root=" + grains.docker_root -%} {% endif -%} +{% set kubelet_root = "" -%} +{% if grains.kubelet_root is defined -%} + {% set kubelet_root = " --root_dir=" + grains.kubelet_root -%} +{% endif -%} + {% set configure_cbr0 = "" -%} {% if pillar['allocate_node_cidrs'] is defined -%} {% set configure_cbr0 = "--configure-cbr0=" + pillar['allocate_node_cidrs'] -%} @@ -66,4 +71,4 @@ {% set cgroup_root = "--cgroup_root=/" -%} {% endif -%} -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{cloud_provider}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{hostname_override}} {{cloud_provider}} {{config}} --allow_privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}}" diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 73311749b48..e1d839b231a 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -69,7 +69,7 @@ func GetMountRefs(mounter Interface, mountPath string) ([]string, error) { // Find all references to the device. var refs []string if deviceName == "" { - glog.Warningf("could not determine device for path: %s", mountPath) + glog.Warningf("could not determine device for path: %q", mountPath) } else { for i := range mps { if mps[i].Device == deviceName && mps[i].Path != mountPath {