From 90644d773bd5a3b69c317a7e02c150637e9e52c6 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Mon, 13 Jun 2016 12:02:07 -0700 Subject: [PATCH 1/2] Add petset directory validation in e2e --- test/e2e/petset.go | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/test/e2e/petset.go b/test/e2e/petset.go index 931075417f2..fcd59f26160 100644 --- a/test/e2e/petset.go +++ b/test/e2e/petset.go @@ -109,17 +109,23 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { By("Saturating pet set " + ps.Name) pst.saturate(ps) + By("Verifying petset mounted data directory is usable") + ExpectNoError(pst.verifyDirectoryIsUsable("/data")) + cmd := "echo $(hostname) > /data/hostname; sync;" By("Running " + cmd + " in all pets") - pst.execInPets(ps, cmd) + ExpectNoError(pst.execInPets(ps, cmd)) By("Restarting pet set " + ps.Name) pst.restart(ps) pst.saturate(ps) + By("Verifying petset mounted data directory is usable") + ExpectNoError(pst.verifyDirectoryIsUsable("/data")) + cmd = "if [ \"$(cat /data/hostname)\" = \"$(hostname)\" ]; then exit 0; else exit 1; fi" By("Running " + cmd + " in all pets") - pst.execInPets(ps, cmd) + ExpectNoError(pst.execInPets(ps, cmd)) }) It("should handle healthy pet restarts during scale [Feature:PetSet]", func() { @@ -414,13 +420,31 @@ func (p *petSetTester) createPetSet(manifestPath, ns string) *apps.PetSet { return ps } -func (p *petSetTester) execInPets(ps *apps.PetSet, cmd string) { +func (p *petSetTester) verifyDirectoryIsUsable(dirPath string) error { + for _, cmd := range []string{ + // Print inode, size etc + fmt.Sprintf("ls -idlh %v", dirPath), + // Print subdirs + fmt.Sprintf("find %v", dirPath), + // Try writing + fmt.Sprintf("touch %v", filepath.Join(dirPath, fmt.Sprintf("%v", time.Now().UnixNano()))), + } { + if err := p.execInPets(ps, cmd); err != nil { + return fmt.Errorf("failed to execute %v, error: %v", cmd, err) + } + } +} + +func (p *petSetTester) execInPets(ps *apps.PetSet, cmd string) error { podList := p.getPodList(ps) for _, pet := range podList.Items { stdout, err := framework.RunHostCmd(pet.Namespace, pet.Name, cmd) - ExpectNoError(err) framework.Logf("stdout of %v on %v: %v", cmd, pet.Name, stdout) + if err != nil { + return err + } } + return nil } func (p *petSetTester) saturate(ps *apps.PetSet) { From 4e2f97a80e0394df7c3abdeae388f85ace18acb5 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Mon, 13 Jun 2016 13:55:49 -0700 Subject: [PATCH 2/2] Add some logging around ro flag in GCE volume plugin --- pkg/volume/gce_pd/attacher.go | 1 + pkg/volume/gce_pd/gce_pd.go | 5 +++-- test/e2e/petset.go | 13 +++++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/volume/gce_pd/attacher.go b/pkg/volume/gce_pd/attacher.go index e208f2aa690..76d57eaa6a4 100644 --- a/pkg/volume/gce_pd/attacher.go +++ b/pkg/volume/gce_pd/attacher.go @@ -166,6 +166,7 @@ func (attacher *gcePersistentDiskAttacher) MountDevice(spec *volume.Spec, device os.Remove(deviceMountPath) return err } + glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) } return nil } diff --git a/pkg/volume/gce_pd/gce_pd.go b/pkg/volume/gce_pd/gce_pd.go index 942df71a7e3..cbc89aeed69 100644 --- a/pkg/volume/gce_pd/gce_pd.go +++ b/pkg/volume/gce_pd/gce_pd.go @@ -86,11 +86,12 @@ func getVolumeSource(spec *volume.Spec) (*api.GCEPersistentDiskVolumeSource, boo if spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil { volumeSource = spec.Volume.GCEPersistentDisk readOnly = volumeSource.ReadOnly + glog.V(4).Infof("volume source %v spec %v, readonly flag retrieved from source: %v", volumeSource.PDName, spec.Name(), readOnly) } else { volumeSource = spec.PersistentVolume.Spec.GCEPersistentDisk readOnly = spec.ReadOnly + glog.V(4).Infof("volume source %v spec %v, readonly flag retrieved from spec: %v", volumeSource.PDName, spec.Name(), readOnly) } - return volumeSource, readOnly } @@ -219,7 +220,7 @@ func (b *gcePersistentDiskMounter) SetUp(fsGroup *int64) error { func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error { // TODO: handle failed mounts here. notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("PersistentDisk set up: %s %v %v", dir, !notMnt, err) + glog.V(4).Infof("PersistentDisk set up: %s %v %v, pd name %v readOnly %v", dir, !notMnt, err, b.pdName, b.readOnly) if err != nil && !os.IsNotExist(err) { return err } diff --git a/test/e2e/petset.go b/test/e2e/petset.go index fcd59f26160..60c8803ec5e 100644 --- a/test/e2e/petset.go +++ b/test/e2e/petset.go @@ -110,7 +110,7 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { pst.saturate(ps) By("Verifying petset mounted data directory is usable") - ExpectNoError(pst.verifyDirectoryIsUsable("/data")) + ExpectNoError(pst.checkMount(ps, "/data")) cmd := "echo $(hostname) > /data/hostname; sync;" By("Running " + cmd + " in all pets") @@ -121,7 +121,7 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() { pst.saturate(ps) By("Verifying petset mounted data directory is usable") - ExpectNoError(pst.verifyDirectoryIsUsable("/data")) + ExpectNoError(pst.checkMount(ps, "/data")) cmd = "if [ \"$(cat /data/hostname)\" = \"$(hostname)\" ]; then exit 0; else exit 1; fi" By("Running " + cmd + " in all pets") @@ -420,19 +420,20 @@ func (p *petSetTester) createPetSet(manifestPath, ns string) *apps.PetSet { return ps } -func (p *petSetTester) verifyDirectoryIsUsable(dirPath string) error { +func (p *petSetTester) checkMount(ps *apps.PetSet, mountPath string) error { for _, cmd := range []string{ // Print inode, size etc - fmt.Sprintf("ls -idlh %v", dirPath), + fmt.Sprintf("ls -idlh %v", mountPath), // Print subdirs - fmt.Sprintf("find %v", dirPath), + fmt.Sprintf("find %v", mountPath), // Try writing - fmt.Sprintf("touch %v", filepath.Join(dirPath, fmt.Sprintf("%v", time.Now().UnixNano()))), + fmt.Sprintf("touch %v", filepath.Join(mountPath, fmt.Sprintf("%v", time.Now().UnixNano()))), } { if err := p.execInPets(ps, cmd); err != nil { return fmt.Errorf("failed to execute %v, error: %v", cmd, err) } } + return nil } func (p *petSetTester) execInPets(ps *apps.PetSet, cmd string) error {