diff --git a/hack/.golint_failures b/hack/.golint_failures index 6cd1db4fbb1..1f2f6c9061f 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -796,6 +796,8 @@ test/e2e/scalability test/e2e/scheduling test/e2e/servicecatalog test/e2e/storage +test/e2e/storage/utils +test/e2e/storage/vsphere test/e2e/ui test/e2e/upgrades test/e2e/upgrades/apps diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index 13969e42c24..f7c76328535 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -10,66 +10,42 @@ go_library( srcs = [ "empty_dir_wrapper.go", "flexvolume.go", - "framework.go", "pd.go", "persistent_volumes.go", "persistent_volumes-disruptive.go", "persistent_volumes-gce.go", "persistent_volumes-local.go", - "persistent_volumes-vsphere.go", - "pv_reclaimpolicy.go", - "pvc_label_selector.go", "pvc_protection.go", "volume_expand.go", "volume_io.go", "volume_metrics.go", "volume_provisioning.go", "volumes.go", - "vsphere_scale.go", - "vsphere_statefulsets.go", - "vsphere_stress.go", - "vsphere_utils.go", - "vsphere_volume_cluster_ds.go", - "vsphere_volume_datastore.go", - "vsphere_volume_diskformat.go", - "vsphere_volume_disksize.go", - "vsphere_volume_fstype.go", - "vsphere_volume_master_restart.go", - "vsphere_volume_node_poweroff.go", - "vsphere_volume_ops_storm.go", - "vsphere_volume_perf.go", - "vsphere_volume_placement.go", - "vsphere_volume_vsan_policy.go", ], importpath = "k8s.io/kubernetes/test/e2e/storage", deps = [ "//pkg/api/testapi:go_default_library", "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/storage/v1/util:go_default_library", - "//pkg/cloudprovider/providers/vsphere:go_default_library", - "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/util/slice:go_default_library", "//pkg/volume/util:go_default_library", - "//pkg/volume/util/volumehelper:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/generated:go_default_library", + "//test/e2e/storage/utils:go_default_library", + "//test/e2e/storage/vsphere:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", - "//vendor/github.com/vmware/govmomi/find:go_default_library", - "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", - "//vendor/golang.org/x/net/context:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/k8s.io/api/apps/v1beta1:go_default_library", "//vendor/k8s.io/api/batch/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/api/policy/v1beta1:go_default_library", "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", @@ -101,6 +77,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//test/e2e/storage/utils:all-srcs", + "//test/e2e/storage/vsphere:all-srcs", + ], tags = ["automanaged"], ) diff --git a/test/e2e/storage/empty_dir_wrapper.go b/test/e2e/storage/empty_dir_wrapper.go index 653c2a0f970..4f1a45898ed 100644 --- a/test/e2e/storage/empty_dir_wrapper.go +++ b/test/e2e/storage/empty_dir_wrapper.go @@ -30,6 +30,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -51,7 +53,7 @@ const ( wrappedVolumeRaceRCNamePrefix = "wrapped-volume-race-" ) -var _ = SIGDescribe("EmptyDir wrapper volumes", func() { +var _ = utils.SIGDescribe("EmptyDir wrapper volumes", func() { f := framework.NewDefaultFramework("emptydir-wrapper") It("should not conflict", func() { diff --git a/test/e2e/storage/flexvolume.go b/test/e2e/storage/flexvolume.go index 862c692c286..99aafb95c06 100644 --- a/test/e2e/storage/flexvolume.go +++ b/test/e2e/storage/flexvolume.go @@ -28,6 +28,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/generated" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -136,7 +137,7 @@ func sshAndLog(cmd, host string) { } } -var _ = SIGDescribe("Flexvolumes [Disruptive] [Feature:FlexVolume]", func() { +var _ = utils.SIGDescribe("Flexvolumes [Disruptive] [Feature:FlexVolume]", func() { f := framework.NewDefaultFramework("flexvolume") // note that namespace deletion is handled by delete-namespace flag diff --git a/test/e2e/storage/pd.go b/test/e2e/storage/pd.go index ae7041d89e1..6ffecb194ea 100644 --- a/test/e2e/storage/pd.go +++ b/test/e2e/storage/pd.go @@ -40,6 +40,7 @@ import ( v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -52,7 +53,7 @@ const ( minNodes = 2 ) -var _ = SIGDescribe("Pod Disks", func() { +var _ = utils.SIGDescribe("Pod Disks", func() { var ( ns string cs clientset.Interface diff --git a/test/e2e/storage/persistent_volumes-disruptive.go b/test/e2e/storage/persistent_volumes-disruptive.go index b43d17e2e19..feb15c6a5a9 100644 --- a/test/e2e/storage/persistent_volumes-disruptive.go +++ b/test/e2e/storage/persistent_volumes-disruptive.go @@ -18,18 +18,17 @@ package storage import ( "fmt" - "strings" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) @@ -37,17 +36,12 @@ type disruptiveTest struct { testItStmt string runTest testBody } -type kubeletOpt string const ( - MinNodes = 2 - NodeStateTimeout = 1 * time.Minute - kStart kubeletOpt = "start" - kStop kubeletOpt = "stop" - kRestart kubeletOpt = "restart" + MinNodes = 2 ) -var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() { +var _ = utils.SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() { f := framework.NewDefaultFramework("disruptive-pv") var ( @@ -223,11 +217,11 @@ var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() { disruptiveTestTable := []disruptiveTest{ { testItStmt: "Should test that a file written to the mount before kubelet restart is readable after restart.", - runTest: testKubeletRestartsAndRestoresMount, + runTest: utils.TestKubeletRestartsAndRestoresMount, }, { testItStmt: "Should test that a volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns.", - runTest: testVolumeUnmountsFromDeletedPod, + runTest: utils.TestVolumeUnmountsFromDeletedPod, }, } @@ -243,61 +237,6 @@ var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() { }) }) -// testKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts -func testKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) { - By("Writing to the volume.") - file := "/mnt/_SUCCESS" - out, err := podExec(clientPod, fmt.Sprintf("touch %s", file)) - framework.Logf(out) - Expect(err).NotTo(HaveOccurred()) - - By("Restarting kubelet") - kubeletCommand(kRestart, c, clientPod) - - By("Testing that written file is accessible.") - out, err = podExec(clientPod, fmt.Sprintf("cat %s", file)) - framework.Logf(out) - Expect(err).NotTo(HaveOccurred()) - framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, file) -} - -// testVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down. -func testVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) { - nodeIP, err := framework.GetHostExternalAddress(c, clientPod) - Expect(err).NotTo(HaveOccurred()) - nodeIP = nodeIP + ":22" - - By("Expecting the volume mount to be found.") - result, err := framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider) - framework.LogSSHResult(result) - Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.") - Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code)) - - By("Stopping the kubelet.") - kubeletCommand(kStop, c, clientPod) - defer func() { - if err != nil { - kubeletCommand(kStart, c, clientPod) - } - }() - By(fmt.Sprintf("Deleting Pod %q", clientPod.Name)) - err = c.CoreV1().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - By("Starting the kubelet and waiting for pod to delete.") - kubeletCommand(kStart, c, clientPod) - err = f.WaitForPodTerminated(clientPod.Name, "") - if !apierrs.IsNotFound(err) && err != nil { - Expect(err).NotTo(HaveOccurred(), "Expected pod to terminate.") - } - - By("Expecting the volume mount not to be found.") - result, err = framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider) - framework.LogSSHResult(result) - Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.") - Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).") - framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName) -} - // initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed // by the test. func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig framework.PersistentVolumeConfig, pvcConfig framework.PersistentVolumeClaimConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) { @@ -339,101 +278,3 @@ func tearDownTestCase(c clientset.Interface, f *framework.Framework, ns string, framework.DeletePersistentVolumeClaim(c, pvc.Name, ns) framework.DeletePersistentVolume(c, pv.Name) } - -// kubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod and waits -// for the desired statues.. -// - First issues the command via `systemctl` -// - If `systemctl` returns stderr "command not found, issues the command via `service` -// - If `service` also returns stderr "command not found", the test is aborted. -// Allowed kubeletOps are `kStart`, `kStop`, and `kRestart` -func kubeletCommand(kOp kubeletOpt, c clientset.Interface, pod *v1.Pod) { - command := "" - sudoPresent := false - systemctlPresent := false - kubeletPid := "" - - nodeIP, err := framework.GetHostExternalAddress(c, pod) - Expect(err).NotTo(HaveOccurred()) - nodeIP = nodeIP + ":22" - - framework.Logf("Checking if sudo command is present") - sshResult, err := framework.SSH("sudo --version", nodeIP, framework.TestContext.Provider) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName)) - if !strings.Contains(sshResult.Stderr, "command not found") { - sudoPresent = true - } - - framework.Logf("Checking if systemctl command is present") - sshResult, err = framework.SSH("systemctl --version", nodeIP, framework.TestContext.Provider) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName)) - if !strings.Contains(sshResult.Stderr, "command not found") { - command = fmt.Sprintf("systemctl %s kubelet", string(kOp)) - systemctlPresent = true - } else { - command = fmt.Sprintf("service kubelet %s", string(kOp)) - } - if sudoPresent { - command = fmt.Sprintf("sudo %s", command) - } - - if kOp == kRestart { - kubeletPid = getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent) - } - - framework.Logf("Attempting `%s`", command) - sshResult, err = framework.SSH(command, nodeIP, framework.TestContext.Provider) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName)) - framework.LogSSHResult(sshResult) - Expect(sshResult.Code).To(BeZero(), "Failed to [%s] kubelet:\n%#v", string(kOp), sshResult) - - if kOp == kStop { - if ok := framework.WaitForNodeToBeNotReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok { - framework.Failf("Node %s failed to enter NotReady state", pod.Spec.NodeName) - } - } - if kOp == kRestart { - // Wait for a minute to check if kubelet Pid is getting changed - isPidChanged := false - for start := time.Now(); time.Since(start) < 1*time.Minute; time.Sleep(2 * time.Second) { - kubeletPidAfterRestart := getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent) - if kubeletPid != kubeletPidAfterRestart { - isPidChanged = true - break - } - } - Expect(isPidChanged).To(BeTrue(), "Kubelet PID remained unchanged after restarting Kubelet") - framework.Logf("Noticed that kubelet PID is changed. Waiting for 30 Seconds for Kubelet to come back") - time.Sleep(30 * time.Second) - } - if kOp == kStart || kOp == kRestart { - // For kubelet start and restart operations, Wait until Node becomes Ready - if ok := framework.WaitForNodeToBeReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok { - framework.Failf("Node %s failed to enter Ready state", pod.Spec.NodeName) - } - } -} - -// return the Main PID of the Kubelet Process -func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) string { - command := "" - if systemctlPresent { - command = "systemctl status kubelet | grep 'Main PID'" - } else { - command = "service kubelet status | grep 'Main PID'" - } - if sudoPresent { - command = fmt.Sprintf("sudo %s", command) - } - framework.Logf("Attempting `%s`", command) - sshResult, err := framework.SSH(command, nodeIP, framework.TestContext.Provider) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", nodeIP)) - framework.LogSSHResult(sshResult) - Expect(sshResult.Code).To(BeZero(), "Failed to get kubelet PID") - Expect(sshResult.Stdout).NotTo(BeEmpty(), "Kubelet Main PID should not be Empty") - return sshResult.Stdout -} - -// podExec wraps RunKubectl to execute a bash cmd in target pod -func podExec(pod *v1.Pod, bashExec string) (string, error) { - return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec) -} diff --git a/test/e2e/storage/persistent_volumes-gce.go b/test/e2e/storage/persistent_volumes-gce.go index 4609fd56074..08a2c4afa4a 100644 --- a/test/e2e/storage/persistent_volumes-gce.go +++ b/test/e2e/storage/persistent_volumes-gce.go @@ -26,6 +26,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) // verifyGCEDiskAttached performs a sanity check to verify the PD attached to the node @@ -51,7 +52,7 @@ func initializeGCETestSpec(c clientset.Interface, ns string, pvConfig framework. } // Testing configurations of single a PV/PVC pair attached to a GCE PD -var _ = SIGDescribe("PersistentVolumes GCEPD", func() { +var _ = utils.SIGDescribe("PersistentVolumes GCEPD", func() { var ( c clientset.Interface diskName string diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index fb873a9f771..4c09ed26099 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -44,6 +44,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" ) @@ -138,7 +139,7 @@ var ( Level: "s0:c0,c1"} ) -var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [Serial]", func() { +var _ = utils.SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [Serial]", func() { f := framework.NewDefaultFramework("persistent-local-volumes-test") var ( @@ -835,7 +836,7 @@ func createFileDoesntExistCmd(testFileDir string, testFile string) string { // Execute a read or write command in a pod. // Fail on error func podRWCmdExec(pod *v1.Pod, cmd string) string { - out, err := podExec(pod, cmd) + out, err := utils.PodExec(pod, cmd) Expect(err).NotTo(HaveOccurred()) return out } diff --git a/test/e2e/storage/persistent_volumes.go b/test/e2e/storage/persistent_volumes.go index 416bb34c6cf..53d28b39bcf 100644 --- a/test/e2e/storage/persistent_volumes.go +++ b/test/e2e/storage/persistent_volumes.go @@ -29,6 +29,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) // Validate PV/PVC, create and verify writer pod, delete the PVC, and validate the PV's @@ -85,7 +86,7 @@ func completeMultiTest(f *framework.Framework, c clientset.Interface, ns string, return nil } -var _ = SIGDescribe("PersistentVolumes", func() { +var _ = utils.SIGDescribe("PersistentVolumes", func() { // global vars for the Context()s and It()'s below f := framework.NewDefaultFramework("pv") diff --git a/test/e2e/storage/pvc_protection.go b/test/e2e/storage/pvc_protection.go index b43ee5d4d91..5ead6272635 100644 --- a/test/e2e/storage/pvc_protection.go +++ b/test/e2e/storage/pvc_protection.go @@ -30,9 +30,10 @@ import ( "k8s.io/kubernetes/pkg/util/slice" volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) -var _ = SIGDescribe("PVC Protection [Feature:PVCProtection]", func() { +var _ = utils.SIGDescribe("PVC Protection [Feature:PVCProtection]", func() { var ( client clientset.Interface nameSpace string diff --git a/test/e2e/storage/utils/BUILD b/test/e2e/storage/utils/BUILD new file mode 100644 index 00000000000..7f7debdce11 --- /dev/null +++ b/test/e2e/storage/utils/BUILD @@ -0,0 +1,37 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "framework.go", + "utils.go", + ], + importpath = "k8s.io/kubernetes/test/e2e/storage/utils", + deps = [ + "//test/e2e/framework:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/e2e/storage/framework.go b/test/e2e/storage/utils/framework.go similarity index 97% rename from test/e2e/storage/framework.go rename to test/e2e/storage/utils/framework.go index 443c9926bcc..b8ea82b9bd2 100644 --- a/test/e2e/storage/framework.go +++ b/test/e2e/storage/utils/framework.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package utils import "github.com/onsi/ginkgo" diff --git a/test/e2e/storage/utils/utils.go b/test/e2e/storage/utils/utils.go new file mode 100644 index 00000000000..5b27dea6d97 --- /dev/null +++ b/test/e2e/storage/utils/utils.go @@ -0,0 +1,240 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" +) + +type KubeletOpt string + +const ( + NodeStateTimeout = 1 * time.Minute + KStart KubeletOpt = "start" + KStop KubeletOpt = "stop" + KRestart KubeletOpt = "restart" +) + +// PodExec wraps RunKubectl to execute a bash cmd in target pod +func PodExec(pod *v1.Pod, bashExec string) (string, error) { + return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec) +} + +// KubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod and waits +// for the desired statues.. +// - First issues the command via `systemctl` +// - If `systemctl` returns stderr "command not found, issues the command via `service` +// - If `service` also returns stderr "command not found", the test is aborted. +// Allowed kubeletOps are `KStart`, `KStop`, and `KRestart` +func KubeletCommand(kOp KubeletOpt, c clientset.Interface, pod *v1.Pod) { + command := "" + sudoPresent := false + systemctlPresent := false + kubeletPid := "" + + nodeIP, err := framework.GetHostExternalAddress(c, pod) + Expect(err).NotTo(HaveOccurred()) + nodeIP = nodeIP + ":22" + + framework.Logf("Checking if sudo command is present") + sshResult, err := framework.SSH("sudo --version", nodeIP, framework.TestContext.Provider) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName)) + if !strings.Contains(sshResult.Stderr, "command not found") { + sudoPresent = true + } + + framework.Logf("Checking if systemctl command is present") + sshResult, err = framework.SSH("systemctl --version", nodeIP, framework.TestContext.Provider) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName)) + if !strings.Contains(sshResult.Stderr, "command not found") { + command = fmt.Sprintf("systemctl %s kubelet", string(kOp)) + systemctlPresent = true + } else { + command = fmt.Sprintf("service kubelet %s", string(kOp)) + } + if sudoPresent { + command = fmt.Sprintf("sudo %s", command) + } + + if kOp == KRestart { + kubeletPid = getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent) + } + + framework.Logf("Attempting `%s`", command) + sshResult, err = framework.SSH(command, nodeIP, framework.TestContext.Provider) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName)) + framework.LogSSHResult(sshResult) + Expect(sshResult.Code).To(BeZero(), "Failed to [%s] kubelet:\n%#v", string(kOp), sshResult) + + if kOp == KStop { + if ok := framework.WaitForNodeToBeNotReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok { + framework.Failf("Node %s failed to enter NotReady state", pod.Spec.NodeName) + } + } + if kOp == KRestart { + // Wait for a minute to check if kubelet Pid is getting changed + isPidChanged := false + for start := time.Now(); time.Since(start) < 1*time.Minute; time.Sleep(2 * time.Second) { + kubeletPidAfterRestart := getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent) + if kubeletPid != kubeletPidAfterRestart { + isPidChanged = true + break + } + } + Expect(isPidChanged).To(BeTrue(), "Kubelet PID remained unchanged after restarting Kubelet") + framework.Logf("Noticed that kubelet PID is changed. Waiting for 30 Seconds for Kubelet to come back") + time.Sleep(30 * time.Second) + } + if kOp == KStart || kOp == KRestart { + // For kubelet start and restart operations, Wait until Node becomes Ready + if ok := framework.WaitForNodeToBeReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok { + framework.Failf("Node %s failed to enter Ready state", pod.Spec.NodeName) + } + } +} + +// getKubeletMainPid return the Main PID of the Kubelet Process +func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) string { + command := "" + if systemctlPresent { + command = "systemctl status kubelet | grep 'Main PID'" + } else { + command = "service kubelet status | grep 'Main PID'" + } + if sudoPresent { + command = fmt.Sprintf("sudo %s", command) + } + framework.Logf("Attempting `%s`", command) + sshResult, err := framework.SSH(command, nodeIP, framework.TestContext.Provider) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", nodeIP)) + framework.LogSSHResult(sshResult) + Expect(sshResult.Code).To(BeZero(), "Failed to get kubelet PID") + Expect(sshResult.Stdout).NotTo(BeEmpty(), "Kubelet Main PID should not be Empty") + return sshResult.Stdout +} + +// TestKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts +func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) { + By("Writing to the volume.") + file := "/mnt/_SUCCESS" + out, err := PodExec(clientPod, fmt.Sprintf("touch %s", file)) + framework.Logf(out) + Expect(err).NotTo(HaveOccurred()) + + By("Restarting kubelet") + KubeletCommand(KRestart, c, clientPod) + + By("Testing that written file is accessible.") + out, err = PodExec(clientPod, fmt.Sprintf("cat %s", file)) + framework.Logf(out) + Expect(err).NotTo(HaveOccurred()) + framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, file) +} + +// TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down. +func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) { + nodeIP, err := framework.GetHostExternalAddress(c, clientPod) + Expect(err).NotTo(HaveOccurred()) + nodeIP = nodeIP + ":22" + + By("Expecting the volume mount to be found.") + result, err := framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider) + framework.LogSSHResult(result) + Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.") + Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code)) + + By("Stopping the kubelet.") + KubeletCommand(KStop, c, clientPod) + defer func() { + if err != nil { + KubeletCommand(KStart, c, clientPod) + } + }() + By(fmt.Sprintf("Deleting Pod %q", clientPod.Name)) + err = c.CoreV1().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Starting the kubelet and waiting for pod to delete.") + KubeletCommand(KStart, c, clientPod) + err = f.WaitForPodTerminated(clientPod.Name, "") + if !apierrs.IsNotFound(err) && err != nil { + Expect(err).NotTo(HaveOccurred(), "Expected pod to terminate.") + } + + By("Expecting the volume mount not to be found.") + result, err = framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider) + framework.LogSSHResult(result) + Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.") + Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).") + framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName) +} + +// RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory. +func RunInPodWithVolume(c clientset.Interface, ns, claimName, command string) { + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: "busybox", + Command: []string{"/bin/sh"}, + Args: []string{"-c", command}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: claimName, + ReadOnly: false, + }, + }, + }, + }, + }, + } + pod, err := c.CoreV1().Pods(ns).Create(pod) + framework.ExpectNoError(err, "Failed to create pod: %v", err) + defer func() { + framework.DeletePodOrFail(c, ns, pod.Name) + }() + framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace)) +} diff --git a/test/e2e/storage/volume_expand.go b/test/e2e/storage/volume_expand.go index f8ef16d2f14..6382a7cbc51 100644 --- a/test/e2e/storage/volume_expand.go +++ b/test/e2e/storage/volume_expand.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -38,7 +39,7 @@ const ( totalResizeWaitPeriod = 20 * time.Minute ) -var _ = SIGDescribe("Volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() { +var _ = utils.SIGDescribe("Volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() { var ( c clientset.Interface ns string diff --git a/test/e2e/storage/volume_io.go b/test/e2e/storage/volume_io.go index 0be71d44d77..e9205e4dfc3 100644 --- a/test/e2e/storage/volume_io.go +++ b/test/e2e/storage/volume_io.go @@ -40,6 +40,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -126,7 +127,7 @@ func writeToFile(pod *v1.Pod, fpath, dd_input string, fsize int64) error { By(fmt.Sprintf("writing %d bytes to test file %s", fsize, fpath)) loopCnt := fsize / minFileSize writeCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do dd if=%s bs=%d >>%s 2>/dev/null; let i+=1; done", loopCnt, dd_input, minFileSize, fpath) - _, err := podExec(pod, writeCmd) + _, err := utils.PodExec(pod, writeCmd) return err } @@ -134,7 +135,7 @@ func writeToFile(pod *v1.Pod, fpath, dd_input string, fsize int64) error { // Verify that the test file is the expected size and contains the expected content. func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) error { By("verifying file size") - rtnstr, err := podExec(pod, fmt.Sprintf("stat -c %%s %s", fpath)) + rtnstr, err := utils.PodExec(pod, fmt.Sprintf("stat -c %%s %s", fpath)) if err != nil || rtnstr == "" { return fmt.Errorf("unable to get file size via `stat %s`: %v", fpath, err) } @@ -147,7 +148,7 @@ func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) er } By("verifying file hash") - rtnstr, err = podExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath)) + rtnstr, err = utils.PodExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath)) if err != nil { return fmt.Errorf("unable to test file hash via `md5sum %s`: %v", fpath, err) } @@ -168,7 +169,7 @@ func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) er // Delete `fpath` to save some disk space on host. Delete errors are logged but ignored. func deleteFile(pod *v1.Pod, fpath string) { By(fmt.Sprintf("deleting test file %s...", fpath)) - _, err := podExec(pod, fmt.Sprintf("rm -f %s", fpath)) + _, err := utils.PodExec(pod, fmt.Sprintf("rm -f %s", fpath)) if err != nil { // keep going, the test dir will be deleted when the volume is unmounted framework.Logf("unable to delete test file %s: %v\nerror ignored, continuing test", fpath, err) @@ -237,7 +238,7 @@ func testVolumeIO(f *framework.Framework, cs clientset.Interface, config framewo // These tests need privileged containers which are disabled by default. // TODO: support all of the plugins tested in storage/volumes.go -var _ = SIGDescribe("Volume plugin streaming [Slow]", func() { +var _ = utils.SIGDescribe("Volume plugin streaming [Slow]", func() { f := framework.NewDefaultFramework("volume-io") var ( config framework.VolumeTestConfig diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index 85e04ab1c32..303df4855f8 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -30,11 +30,12 @@ import ( kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/metrics" + "k8s.io/kubernetes/test/e2e/storage/utils" ) // This test needs to run in serial because other tests could interfere // with metrics being tested here. -var _ = SIGDescribe("[Serial] Volume metrics", func() { +var _ = utils.SIGDescribe("[Serial] Volume metrics", func() { var ( c clientset.Interface ns string diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index 38d1b257930..ded7fea3c00 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -45,6 +45,7 @@ import ( storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) type storageClassTest struct { @@ -229,7 +230,7 @@ func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error { return nil } -var _ = SIGDescribe("Dynamic Provisioning", func() { +var _ = utils.SIGDescribe("Dynamic Provisioning", func() { f := framework.NewDefaultFramework("volume-provisioning") // filled in BeforeEach diff --git a/test/e2e/storage/volumes.go b/test/e2e/storage/volumes.go index da4ad0bc67c..f5e82ec3470 100644 --- a/test/e2e/storage/volumes.go +++ b/test/e2e/storage/volumes.go @@ -55,6 +55,8 @@ import ( clientset "k8s.io/client-go/kubernetes" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" + vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere" ) func DeleteCinderVolume(name string) error { @@ -79,7 +81,7 @@ func DeleteCinderVolume(name string) error { } // These tests need privileged containers, which are disabled by default. -var _ = SIGDescribe("Volumes", func() { +var _ = utils.SIGDescribe("Volumes", func() { f := framework.NewDefaultFramework("volume") // note that namespace deletion is handled by delete-namespace flag @@ -510,10 +512,10 @@ var _ = SIGDescribe("Volumes", func() { if err != nil { return } - vsp, err := getVSphere(c) + vsp, err := vspheretest.GetVSphere(c) Expect(err).NotTo(HaveOccurred()) - volumePath, err = createVSphereVolume(vsp, nil) + volumePath, err = vspheretest.CreateVSphereVolume(vsp, nil) Expect(err).NotTo(HaveOccurred()) defer func() { diff --git a/test/e2e/storage/vsphere/BUILD b/test/e2e/storage/vsphere/BUILD new file mode 100644 index 00000000000..d582d820046 --- /dev/null +++ b/test/e2e/storage/vsphere/BUILD @@ -0,0 +1,67 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "persistent_volumes-vsphere.go", + "pv_reclaimpolicy.go", + "pvc_label_selector.go", + "vsphere_scale.go", + "vsphere_statefulsets.go", + "vsphere_stress.go", + "vsphere_utils.go", + "vsphere_volume_cluster_ds.go", + "vsphere_volume_datastore.go", + "vsphere_volume_diskformat.go", + "vsphere_volume_disksize.go", + "vsphere_volume_fstype.go", + "vsphere_volume_master_restart.go", + "vsphere_volume_node_poweroff.go", + "vsphere_volume_ops_storm.go", + "vsphere_volume_perf.go", + "vsphere_volume_placement.go", + "vsphere_volume_vsan_policy.go", + ], + importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere", + deps = [ + "//pkg/cloudprovider/providers/vsphere:go_default_library", + "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", + "//pkg/volume/util/volumehelper:go_default_library", + "//test/e2e/framework:go_default_library", + "//test/e2e/storage/utils:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/vmware/govmomi/find:go_default_library", + "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", + "//vendor/k8s.io/api/storage/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/e2e/storage/persistent_volumes-vsphere.go b/test/e2e/storage/vsphere/persistent_volumes-vsphere.go similarity index 96% rename from test/e2e/storage/persistent_volumes-vsphere.go rename to test/e2e/storage/vsphere/persistent_volumes-vsphere.go index 1d49361f068..46556e446a4 100644 --- a/test/e2e/storage/persistent_volumes-vsphere.go +++ b/test/e2e/storage/vsphere/persistent_volumes-vsphere.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "time" @@ -28,10 +28,11 @@ import ( clientset "k8s.io/client-go/kubernetes" vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) // Testing configurations of single a PV/PVC pair attached to a vSphere Disk -var _ = SIGDescribe("PersistentVolumes:vsphere", func() { +var _ = utils.SIGDescribe("PersistentVolumes:vsphere", func() { var ( c clientset.Interface ns string @@ -182,7 +183,7 @@ var _ = SIGDescribe("PersistentVolumes:vsphere", func() { 3. Verify that written file is accessible after kubelet restart */ It("should test that a file written to the vspehre volume mount before kubelet restart can be read after restart [Disruptive]", func() { - testKubeletRestartsAndRestoresMount(c, f, clientPod, pvc, pv) + utils.TestKubeletRestartsAndRestoresMount(c, f, clientPod, pvc, pv) }) /* @@ -197,7 +198,7 @@ var _ = SIGDescribe("PersistentVolumes:vsphere", func() { 5. Verify that volume mount not to be found. */ It("should test that a vspehre volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns [Disruptive]", func() { - testVolumeUnmountsFromDeletedPod(c, f, clientPod, pvc, pv) + utils.TestVolumeUnmountsFromDeletedPod(c, f, clientPod, pvc, pv) }) /* diff --git a/test/e2e/storage/pv_reclaimpolicy.go b/test/e2e/storage/vsphere/pv_reclaimpolicy.go similarity index 97% rename from test/e2e/storage/pv_reclaimpolicy.go rename to test/e2e/storage/vsphere/pv_reclaimpolicy.go index 8713ce7d7eb..79211c292b8 100644 --- a/test/e2e/storage/pv_reclaimpolicy.go +++ b/test/e2e/storage/vsphere/pv_reclaimpolicy.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "strconv" @@ -29,9 +29,10 @@ import ( clientset "k8s.io/client-go/kubernetes" vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) -var _ = SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() { +var _ = utils.SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() { f := framework.NewDefaultFramework("persistentvolumereclaim") var ( c clientset.Interface @@ -47,7 +48,7 @@ var _ = SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() { framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) }) - SIGDescribe("persistentvolumereclaim:vsphere", func() { + utils.SIGDescribe("persistentvolumereclaim:vsphere", func() { BeforeEach(func() { framework.SkipUnlessProviderIs("vsphere") pv = nil diff --git a/test/e2e/storage/pvc_label_selector.go b/test/e2e/storage/vsphere/pvc_label_selector.go similarity index 96% rename from test/e2e/storage/pvc_label_selector.go rename to test/e2e/storage/vsphere/pvc_label_selector.go index d389386a6c7..ccfa0a59ec2 100644 --- a/test/e2e/storage/pvc_label_selector.go +++ b/test/e2e/storage/vsphere/pvc_label_selector.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "time" @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -43,7 +44,7 @@ import ( 9. delete pvc_vvol */ -var _ = SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() { +var _ = utils.SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() { f := framework.NewDefaultFramework("pvclabelselector") var ( c clientset.Interface @@ -68,7 +69,7 @@ var _ = SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() { }) - SIGDescribe("Selector-Label Volume Binding:vsphere", func() { + utils.SIGDescribe("Selector-Label Volume Binding:vsphere", func() { AfterEach(func() { By("Running clean up actions") if framework.ProviderIs("vsphere") { diff --git a/test/e2e/storage/vsphere_scale.go b/test/e2e/storage/vsphere/vsphere_scale.go similarity index 98% rename from test/e2e/storage/vsphere_scale.go rename to test/e2e/storage/vsphere/vsphere_scale.go index ded690e2047..2da0933f87a 100644 --- a/test/e2e/storage/vsphere_scale.go +++ b/test/e2e/storage/vsphere/vsphere_scale.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -30,6 +30,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -52,7 +53,7 @@ type NodeSelector struct { labelValue string } -var _ = SIGDescribe("vcp at scale [Feature:vsphere] ", func() { +var _ = utils.SIGDescribe("vcp at scale [Feature:vsphere] ", func() { f := framework.NewDefaultFramework("vcp-at-scale") var ( diff --git a/test/e2e/storage/vsphere_statefulsets.go b/test/e2e/storage/vsphere/vsphere_statefulsets.go similarity index 98% rename from test/e2e/storage/vsphere_statefulsets.go rename to test/e2e/storage/vsphere/vsphere_statefulsets.go index b0a633ad391..3b5b4596922 100644 --- a/test/e2e/storage/vsphere_statefulsets.go +++ b/test/e2e/storage/vsphere/vsphere_statefulsets.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -49,7 +50,7 @@ const ( storageclassname = "nginx-sc" ) -var _ = SIGDescribe("vsphere statefulset", func() { +var _ = utils.SIGDescribe("vsphere statefulset", func() { f := framework.NewDefaultFramework("vsphere-statefulset") var ( namespace string diff --git a/test/e2e/storage/vsphere_stress.go b/test/e2e/storage/vsphere/vsphere_stress.go similarity index 98% rename from test/e2e/storage/vsphere_stress.go rename to test/e2e/storage/vsphere/vsphere_stress.go index 4be0205e051..2af52b7c94b 100644 --- a/test/e2e/storage/vsphere_stress.go +++ b/test/e2e/storage/vsphere/vsphere_stress.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -31,6 +31,7 @@ import ( k8stype "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -43,7 +44,7 @@ import ( 4. Each instance of routine iterates for n times, where n is read from system env - VCP_STRESS_ITERATIONS 5. Each iteration creates 1 PVC, 1 POD using the provisioned PV, Verify disk is attached to the node, Verify pod can access the volume, delete the pod and finally delete the PVC. */ -var _ = SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", func() { f := framework.NewDefaultFramework("vcp-stress") var ( client clientset.Interface diff --git a/test/e2e/storage/vsphere_utils.go b/test/e2e/storage/vsphere/vsphere_utils.go similarity index 95% rename from test/e2e/storage/vsphere_utils.go rename to test/e2e/storage/vsphere/vsphere_utils.go index b0c87c0cd71..947a424d8cf 100644 --- a/test/e2e/storage/vsphere_utils.go +++ b/test/e2e/storage/vsphere/vsphere_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" "k8s.io/kubernetes/pkg/volume/util/volumehelper" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -250,15 +251,20 @@ func createVSphereVolume(vsp *vsphere.VSphere, volumeOptions *vclib.VolumeOption return volumePath, nil } +// CreateVSphereVolume creates a vmdk volume +func CreateVSphereVolume(vsp *vsphere.VSphere, volumeOptions *vclib.VolumeOptions) (string, error) { + return createVSphereVolume(vsp, volumeOptions) +} + // function to write content to the volume backed by given PVC func writeContentToVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) { - runInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data") + utils.RunInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data") framework.Logf("Done with writing content to volume") } // function to verify content is matching on the volume backed for given PVC func verifyContentOfVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) { - runInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data") + utils.RunInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data") framework.Logf("Successfully verified content of the volume") } @@ -457,3 +463,8 @@ func getVSphere(c clientset.Interface) (*vsphere.VSphere, error) { addNodesToVCP(vsp, c) return vsp, nil } + +// GetVSphere returns vsphere cloud provider +func GetVSphere(c clientset.Interface) (*vsphere.VSphere, error) { + return getVSphere(c) +} diff --git a/test/e2e/storage/vsphere_volume_cluster_ds.go b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go similarity index 96% rename from test/e2e/storage/vsphere_volume_cluster_ds.go rename to test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go index b2ebd29fb16..d200eded4e1 100644 --- a/test/e2e/storage/vsphere_volume_cluster_ds.go +++ b/test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -27,6 +27,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -39,7 +40,7 @@ import ( 1. CLUSTER_DATASTORE which should be set to clustered datastore 2. VSPHERE_SPBM_POLICY_DS_CLUSTER which should be set to a tag based spbm policy tagged to a clustered datastore */ -var _ = SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-provision") var client clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_datastore.go b/test/e2e/storage/vsphere/vsphere_volume_datastore.go similarity index 95% rename from test/e2e/storage/vsphere_volume_datastore.go rename to test/e2e/storage/vsphere/vsphere_volume_datastore.go index df86a96e5ef..70f55ee9f8d 100644 --- a/test/e2e/storage/vsphere_volume_datastore.go +++ b/test/e2e/storage/vsphere/vsphere_volume_datastore.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -44,7 +45,7 @@ const ( 4. Verify the error returned on PVC failure is the correct. */ -var _ = SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-datastore") var ( client clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_diskformat.go b/test/e2e/storage/vsphere/vsphere_volume_diskformat.go similarity index 98% rename from test/e2e/storage/vsphere_volume_diskformat.go rename to test/e2e/storage/vsphere/vsphere_volume_diskformat.go index b805eb4d5a1..f4a9fa9a7a5 100644 --- a/test/e2e/storage/vsphere_volume_diskformat.go +++ b/test/e2e/storage/vsphere/vsphere_volume_diskformat.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "os" @@ -32,6 +32,7 @@ import ( clientset "k8s.io/client-go/kubernetes" vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -52,7 +53,7 @@ import ( 11. Delete PVC, PV and Storage Class */ -var _ = SIGDescribe("Volume Disk Format [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Volume Disk Format [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-disk-format") var ( client clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_disksize.go b/test/e2e/storage/vsphere/vsphere_volume_disksize.go similarity index 96% rename from test/e2e/storage/vsphere_volume_disksize.go rename to test/e2e/storage/vsphere/vsphere_volume_disksize.go index f69983accdc..a84692a2fe2 100644 --- a/test/e2e/storage/vsphere_volume_disksize.go +++ b/test/e2e/storage/vsphere/vsphere_volume_disksize.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -44,7 +45,7 @@ const ( 4. Verify the error returned on PVC failure is the correct. */ -var _ = SIGDescribe("Volume Disk Size [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Volume Disk Size [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-disksize") var ( client clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_fstype.go b/test/e2e/storage/vsphere/vsphere_volume_fstype.go similarity index 98% rename from test/e2e/storage/vsphere_volume_fstype.go rename to test/e2e/storage/vsphere/vsphere_volume_fstype.go index 352b6dd3935..6ec152983a8 100644 --- a/test/e2e/storage/vsphere_volume_fstype.go +++ b/test/e2e/storage/vsphere/vsphere_volume_fstype.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "strings" @@ -28,6 +28,7 @@ import ( clientset "k8s.io/client-go/kubernetes" vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -63,7 +64,7 @@ const ( 7. Verify if the MountVolume.MountDevice fails because it is unable to find the file system executable file on the node. */ -var _ = SIGDescribe("Volume FStype [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Volume FStype [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-fstype") var ( client clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_master_restart.go b/test/e2e/storage/vsphere/vsphere_volume_master_restart.go similarity index 97% rename from test/e2e/storage/vsphere_volume_master_restart.go rename to test/e2e/storage/vsphere/vsphere_volume_master_restart.go index 5ba86162517..cb12e9d0edd 100644 --- a/test/e2e/storage/vsphere_volume_master_restart.go +++ b/test/e2e/storage/vsphere/vsphere_volume_master_restart.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -41,7 +42,7 @@ import ( 6. Delete the pod and wait for the volume to be detached 7. Delete the volume */ -var _ = SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disruptive]", func() { +var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disruptive]", func() { f := framework.NewDefaultFramework("restart-master") const labelKey = "vsphere_e2e_label" diff --git a/test/e2e/storage/vsphere_volume_node_poweroff.go b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go similarity index 98% rename from test/e2e/storage/vsphere_volume_node_poweroff.go rename to test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go index 1a2f37e7560..c0534a73d35 100644 --- a/test/e2e/storage/vsphere_volume_node_poweroff.go +++ b/test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -36,6 +36,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -43,7 +44,7 @@ import ( 1. Verify the pod got provisioned on a different node with volume attached to it 2. Verify the volume is detached from the powered off node */ -var _ = SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", func() { +var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", func() { f := framework.NewDefaultFramework("node-poweroff") var ( client clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_ops_storm.go b/test/e2e/storage/vsphere/vsphere_volume_ops_storm.go similarity index 96% rename from test/e2e/storage/vsphere_volume_ops_storm.go rename to test/e2e/storage/vsphere/vsphere_volume_ops_storm.go index b1b6516d0d9..80a07ceeec0 100644 --- a/test/e2e/storage/vsphere_volume_ops_storm.go +++ b/test/e2e/storage/vsphere/vsphere_volume_ops_storm.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -29,6 +29,7 @@ import ( clientset "k8s.io/client-go/kubernetes" vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* @@ -47,7 +48,7 @@ import ( 10. Delete storage class. */ -var _ = SIGDescribe("Volume Operations Storm [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Volume Operations Storm [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-ops-storm") const DEFAULT_VOLUME_OPS_SCALE = 30 var ( diff --git a/test/e2e/storage/vsphere_volume_perf.go b/test/e2e/storage/vsphere/vsphere_volume_perf.go similarity index 98% rename from test/e2e/storage/vsphere_volume_perf.go rename to test/e2e/storage/vsphere/vsphere_volume_perf.go index 59ca3951782..73c8da97c0b 100644 --- a/test/e2e/storage/vsphere_volume_perf.go +++ b/test/e2e/storage/vsphere/vsphere_volume_perf.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) /* This test calculates latency numbers for volume lifecycle operations @@ -48,7 +49,7 @@ const ( DeleteOp = "DeleteOp" ) -var _ = SIGDescribe("vcp-performance [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("vcp-performance [Feature:vsphere]", func() { f := framework.NewDefaultFramework("vcp-performance") var ( diff --git a/test/e2e/storage/vsphere_volume_placement.go b/test/e2e/storage/vsphere/vsphere_volume_placement.go similarity index 99% rename from test/e2e/storage/vsphere_volume_placement.go rename to test/e2e/storage/vsphere/vsphere_volume_placement.go index 417d0723b15..a0327de9d4a 100644 --- a/test/e2e/storage/vsphere_volume_placement.go +++ b/test/e2e/storage/vsphere/vsphere_volume_placement.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -31,9 +31,10 @@ import ( vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) -var _ = SIGDescribe("Volume Placement", func() { +var _ = utils.SIGDescribe("Volume Placement", func() { f := framework.NewDefaultFramework("volume-placement") var ( c clientset.Interface diff --git a/test/e2e/storage/vsphere_volume_vsan_policy.go b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go similarity index 99% rename from test/e2e/storage/vsphere_volume_vsan_policy.go rename to test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go index f3bef3f1c93..52c86a2e538 100644 --- a/test/e2e/storage/vsphere_volume_vsan_policy.go +++ b/test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package storage +package vsphere import ( "fmt" @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vmware/govmomi/find" "golang.org/x/net/context" "k8s.io/api/core/v1" @@ -34,6 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" ) const ( @@ -90,7 +92,7 @@ const ( */ -var _ = SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsphere]", func() { +var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsphere]", func() { f := framework.NewDefaultFramework("volume-vsan-policy") var ( client clientset.Interface