Merge pull request #50332 from ianchakeres/local-e2e-mntpnt

Automatic merge from submit-queue (batch tested with PRs 50670, 50332)

e2e test for local storage mount point

**What this PR does / why we need it**:

We discovered that kubernetes can treat local directories and actual mountpoints differently. For example, https://github.com/kubernetes/kubernetes/issues/48331. The current local storage e2e tests use directories.

This PR introduces a test that creates a tmpfs and mounts it, and runs one of the local storage e2e tests.

**Which issue this PR fixes**: fixes https://github.com/kubernetes/kubernetes/issues/49126

**Special notes for your reviewer**:

I cherrypicked PR https://github.com/kubernetes/kubernetes/pull/50177, since local storage e2e tests are broken in master on 2017-08-08 due to "no such host" error. This PR replaces NodeExec with SSH commands.

You can run the tests using the following commands:
``` 
$ NUM_NODES=1 KUBE_FEATURE_GATES="PersistentLocalVolumes=true" go run hack/e2e.go -- -v --up
$ go run hack/e2e.go -- -v --test --test_args="--ginkgo.focus=\[Feature:LocalPersistentVolumes\]"
```

Here are the summary of results from my test run:
```
Ran 9 of 651 Specs in 387.905 seconds
SUCCESS! -- 9 Passed | 0 Failed | 0 Pending | 642 Skipped PASS

Ginkgo ran 1 suite in 6m29.369318483s
Test Suite Passed
2017/08/08 11:54:01 util.go:133: Step './hack/ginkgo-e2e.sh --ginkgo.focus=\[Feature:LocalPersistentVolumes\]' finished in 6m32.077462612s
```

**Release note**:
`NONE`
This commit is contained in:
Kubernetes Submit Queue 2017-08-16 01:38:08 -07:00 committed by GitHub
commit bc2f049067

View File

@ -45,6 +45,15 @@ type localTestConfig struct {
scName string
}
type LocalVolumeType string
const (
// default local volume type, aka a directory
DirectoryLocalVolumeType LocalVolumeType = "dir"
// creates a tmpfs and mounts it
TmpfsLocalVolumeType LocalVolumeType = "tmpfs"
)
type localTestVolume struct {
// Node that the volume is on
node *v1.Node
@ -54,6 +63,8 @@ type localTestVolume struct {
pvc *v1.PersistentVolumeClaim
// PV for this volume
pv *v1.PersistentVolume
// Type of local volume
localVolumeType LocalVolumeType
}
const (
@ -97,7 +108,6 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
var (
config *localTestConfig
node0 *v1.Node
scName string
)
@ -107,7 +117,7 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
Expect(len(nodes.Items)).NotTo(BeZero(), "No available nodes for scheduling")
scName = fmt.Sprintf("%v-%v", testSCPrefix, f.Namespace.Name)
// Choose the first node
node0 = &nodes.Items[0]
node0 := &nodes.Items[0]
config = &localTestConfig{
ns: f.Namespace.Name,
@ -123,7 +133,7 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
var testVol *localTestVolume
BeforeEach(func() {
testVol = setupLocalVolumePVCPV(config, node0)
testVol = setupLocalVolumePVCPV(config, DirectoryLocalVolumeType)
})
AfterEach(func() {
@ -132,9 +142,9 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
It("should be able to mount and read from the volume using one-command containers", func() {
By("Creating a pod to read from the PV")
//testFileContent was written during setupLocalVolume
_, readCmd := createWriteAndReadCmds(volumeDir, testFile, "" /*writeTestFileContent*/)
readCmd := createReadCmd(volumeDir, testFile)
podSpec := makeLocalPod(config, testVol, readCmd)
//testFileContent was written during setupLocalVolume
f.TestContainerOutput("pod reads PV", podSpec, 0, []string{testFileContent})
})
@ -154,13 +164,11 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
Expect(pod1NodeNameErr).NotTo(HaveOccurred())
framework.Logf("pod1 %q created on Node %q", pod1.Name, pod1NodeName)
Expect(pod1NodeName).To(Equal(node0.Name))
Expect(pod1NodeName).To(Equal(config.node0.Name))
By("Reading in pod1")
// testFileContent was written during setupLocalVolume
_, readCmd := createWriteAndReadCmds(volumeDir, testFile, "" /*writeTestFileContent*/)
readOut := podRWCmdExec(pod1, readCmd)
Expect(readOut).To(ContainSubstring(testFileContent)) /*aka writeTestFileContents*/
testReadFileContent(volumeDir, testFile, testFileContent, pod1)
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
@ -174,7 +182,10 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
Expect(pod1NodeNameErr).NotTo(HaveOccurred())
framework.Logf("pod1 %q created on Node %q", pod1.Name, pod1NodeName)
Expect(pod1NodeName).To(Equal(node0.Name))
Expect(pod1NodeName).To(Equal(config.node0.Name))
// testFileContent was written during setupLocalVolume
testReadFileContent(volumeDir, testFile, testFileContent, pod1)
By("Writing in pod1")
writeCmd, _ := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
@ -190,7 +201,7 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
var testVol *localTestVolume
BeforeEach(func() {
testVol = setupLocalVolumePVCPV(config, node0)
testVol = setupLocalVolumePVCPV(config, DirectoryLocalVolumeType)
})
AfterEach(func() {
@ -208,91 +219,31 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
podSpec2 := makeLocalPod(config, testVol, readCmd)
f.TestContainerOutput("pod reads PV", podSpec2, 0, []string{testVol.hostDir})
})
})
It("should be able to mount volume in two pods one after other, write from pod1, and read from pod2", func() {
By("Creating pod1")
pod1, pod1Err := createLocalPod(config, testVol)
Expect(pod1Err).NotTo(HaveOccurred())
LocalVolumeTypes := []LocalVolumeType{DirectoryLocalVolumeType, TmpfsLocalVolumeType}
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod1))
pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
Expect(pod1NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod1 %q created on Node %q", pod1.Name, pod1NodeName)
Expect(pod1NodeName).To(Equal(node0.Name))
writeCmd, readCmd := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
By("Creating pod2")
pod2, pod2Err := createLocalPod(config, testVol)
Expect(pod2Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod2))
pod2NodeName, pod2NodeNameErr := podNodeName(config, pod2)
Expect(pod2NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod2 %q created on Node %q", pod2.Name, pod2NodeName)
Expect(pod2NodeName).To(Equal(node0.Name))
By("Reading in pod2")
readOut := podRWCmdExec(pod2, readCmd)
Expect(readOut).To(ContainSubstring(testVol.hostDir)) /*aka writeTestFileContents*/
By("Deleting pod2")
framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
Context("when two pods mount a local volume at the same time", func() {
It("should be able to write from pod1 and read from pod2", func() {
for _, testVolType := range LocalVolumeTypes {
var testVol *localTestVolume
By(fmt.Sprintf("local-volume-type: %s", testVolType))
testVol = setupLocalVolumePVCPV(config, testVolType)
twoPodsReadWriteTest(config, testVol)
cleanupLocalVolume(config, testVol)
}
})
})
Context("when two pods request one prebound PVC at the same time", func() {
var testVol *localTestVolume
BeforeEach(func() {
testVol = setupLocalVolumePVCPV(config, node0)
})
AfterEach(func() {
cleanupLocalVolume(config, testVol)
})
It("should be able to mount volume in two pods at the same time, write from pod1, and read from pod2", func() {
By("Creating pod1 to write to the PV")
pod1, pod1Err := createLocalPod(config, testVol)
Expect(pod1Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod1))
pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
Expect(pod1NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod1 %q created on Node %q", pod1.Name, pod1NodeName)
Expect(pod1NodeName).To(Equal(node0.Name))
By("Creating pod2 to read from the PV")
pod2, pod2Err := createLocalPod(config, testVol)
Expect(pod2Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod2))
pod2NodeName, pod2NodeNameErr := podNodeName(config, pod2)
Expect(pod2NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod2 %q created on Node %q", pod2.Name, pod2NodeName)
Expect(pod2NodeName).To(Equal(node0.Name))
writeCmd, readCmd := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
By("Reading in pod2")
readOut := podRWCmdExec(pod2, readCmd)
Expect(readOut).To(ContainSubstring(testVol.hostDir)) /*aka writeTestFileContents*/
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
By("Deleting pod2")
framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
Context("when two pods mount a local volume one after the other", func() {
It("should be able to write from pod1 and read from pod2", func() {
for _, testVolType := range LocalVolumeTypes {
var testVol *localTestVolume
By(fmt.Sprintf("local-volume-type: %s", testVolType))
testVol = setupLocalVolumePVCPV(config, testVolType)
twoPodsReadWriteSerialTest(config, testVol)
cleanupLocalVolume(config, testVol)
}
})
})
@ -320,7 +271,7 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
By("Creating a directory under discovery path")
framework.Logf("creating local volume under path %q", volumePath)
mkdirCmd := fmt.Sprintf("mkdir %v -m 777", volumePath)
err := framework.IssueSSHCommand(mkdirCmd, framework.TestContext.Provider, node0)
err := framework.IssueSSHCommand(mkdirCmd, framework.TestContext.Provider, config.node0)
Expect(err).NotTo(HaveOccurred())
By("Waiting for a PersitentVolume to be created")
@ -342,7 +293,7 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
// Delete the persistent volume claim: file will be cleaned up and volume be re-created.
By("Deleting the persistent volume claim to clean up persistent volume and re-create one")
writeCmd, _ := createWriteAndReadCmds(volumePath, testFile, testFileContent)
err = framework.IssueSSHCommand(writeCmd, framework.TestContext.Provider, node0)
err = framework.IssueSSHCommand(writeCmd, framework.TestContext.Provider, config.node0)
Expect(err).NotTo(HaveOccurred())
err = config.client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
@ -352,12 +303,96 @@ var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [S
Expect(err).NotTo(HaveOccurred())
Expect(newPV.UID).NotTo(Equal(oldPV.UID))
fileDoesntExistCmd := createFileDoesntExistCmd(volumePath, testFile)
err = framework.IssueSSHCommand(fileDoesntExistCmd, framework.TestContext.Provider, node0)
err = framework.IssueSSHCommand(fileDoesntExistCmd, framework.TestContext.Provider, config.node0)
Expect(err).NotTo(HaveOccurred())
})
})
})
// The tests below are run against multiple mount point types
// Test two pods at the same time, write from pod1, and read from pod2
func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
By("Creating pod1 to write to the PV")
pod1, pod1Err := createLocalPod(config, testVol)
Expect(pod1Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod1))
pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
Expect(pod1NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod1 %q created on Node %q", pod1.Name, pod1NodeName)
Expect(pod1NodeName).To(Equal(config.node0.Name))
// testFileContent was written during setupLocalVolume
testReadFileContent(volumeDir, testFile, testFileContent, pod1)
By("Creating pod2 to read from the PV")
pod2, pod2Err := createLocalPod(config, testVol)
Expect(pod2Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod2))
pod2NodeName, pod2NodeNameErr := podNodeName(config, pod2)
Expect(pod2NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod2 %q created on Node %q", pod2.Name, pod2NodeName)
Expect(pod2NodeName).To(Equal(config.node0.Name))
// testFileContent was written during setupLocalVolume
testReadFileContent(volumeDir, testFile, testFileContent, pod2)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
By("Reading in pod2")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2)
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
By("Deleting pod2")
framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
}
// Test two pods one after other, write from pod1, and read from pod2
func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolume) {
By("Creating pod1")
pod1, pod1Err := createLocalPod(config, testVol)
Expect(pod1Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod1))
pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
Expect(pod1NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod1 %q created on Node %q", pod1.Name, pod1NodeName)
Expect(pod1NodeName).To(Equal(config.node0.Name))
// testFileContent was written during setupLocalVolume
testReadFileContent(volumeDir, testFile, testFileContent, pod1)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
By("Creating pod2")
pod2, pod2Err := createLocalPod(config, testVol)
Expect(pod2Err).NotTo(HaveOccurred())
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod2))
pod2NodeName, pod2NodeNameErr := podNodeName(config, pod2)
Expect(pod2NodeNameErr).NotTo(HaveOccurred())
framework.Logf("Pod2 %q created on Node %q", pod2.Name, pod2NodeName)
Expect(pod2NodeName).To(Equal(config.node0.Name))
By("Reading in pod2")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2)
By("Deleting pod2")
framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
}
// podNode wraps RunKubectl to get node where pod is running
func podNodeName(config *localTestConfig, pod *v1.Pod) (string, error) {
runtimePod, runtimePodErr := config.client.Core().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
@ -365,18 +400,23 @@ func podNodeName(config *localTestConfig, pod *v1.Pod) (string, error) {
}
// setupLocalVolume setups a directory to user for local PV
func setupLocalVolume(config *localTestConfig) *localTestVolume {
func setupLocalVolume(config *localTestConfig, localVolumeType LocalVolumeType) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
if localVolumeType == TmpfsLocalVolumeType {
createAndMountTmpfsLocalVolume(config, hostDir)
}
// populate volume with testFile containing testFileContent
writeCmd, _ := createWriteAndReadCmds(hostDir, testFile, testFileContent)
By(fmt.Sprintf("Creating local volume on node %q at path %q", config.node0.Name, hostDir))
err := framework.IssueSSHCommand(writeCmd, framework.TestContext.Provider, config.node0)
Expect(err).NotTo(HaveOccurred())
return &localTestVolume{
node: config.node0,
hostDir: hostDir,
node: config.node0,
hostDir: hostDir,
localVolumeType: localVolumeType,
}
}
@ -392,6 +432,10 @@ func cleanupLocalVolume(config *localTestConfig, volume *localTestVolume) {
framework.Failf("Failed to delete PV and/or PVC: %v", utilerrors.NewAggregate(errs))
}
if volume.localVolumeType == TmpfsLocalVolumeType {
unmountTmpfsLocalVolume(config, volume.hostDir)
}
By("Removing the test directory")
removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir)
err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, config.node0)
@ -461,15 +505,42 @@ func createLocalPod(config *localTestConfig, volume *localTestVolume) (*v1.Pod,
return framework.CreatePod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "")
}
func createAndMountTmpfsLocalVolume(config *localTestConfig, dir string) {
By(fmt.Sprintf("Creating tmpfs mount point on node %q at path %q", config.node0.Name, dir))
err := framework.IssueSSHCommand(fmt.Sprintf("mkdir -p %q && sudo mount -t tmpfs -o size=1m tmpfs-%q %q", dir, dir, dir), framework.TestContext.Provider, config.node0)
Expect(err).NotTo(HaveOccurred())
}
func unmountTmpfsLocalVolume(config *localTestConfig, dir string) {
By(fmt.Sprintf("Unmount tmpfs mount point on node %q at path %q", config.node0.Name, dir))
err := framework.IssueSSHCommand(fmt.Sprintf("sudo umount %q", dir), framework.TestContext.Provider, config.node0)
Expect(err).NotTo(HaveOccurred())
}
// Create corresponding write and read commands
// to be executed via SSH on the node with the local PV
func createWriteAndReadCmds(testFileDir string, testFile string, writeTestFileContent string) (writeCmd string, readCmd string) {
testFilePath := filepath.Join(testFileDir, testFile)
writeCmd = fmt.Sprintf("mkdir -p %s; echo %s > %s", testFileDir, writeTestFileContent, testFilePath)
readCmd = fmt.Sprintf("cat %s", testFilePath)
writeCmd = createWriteCmd(testFileDir, testFile, writeTestFileContent)
readCmd = createReadCmd(testFileDir, testFile)
return writeCmd, readCmd
}
func createWriteCmd(testFileDir string, testFile string, writeTestFileContent string) string {
testFilePath := filepath.Join(testFileDir, testFile)
return fmt.Sprintf("mkdir -p %s; echo %s > %s", testFileDir, writeTestFileContent, testFilePath)
}
func createReadCmd(testFileDir string, testFile string) string {
testFilePath := filepath.Join(testFileDir, testFile)
return fmt.Sprintf("cat %s", testFilePath)
}
// Read testFile and evaluate whether it contains the testFileContent
func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod) {
readCmd := createReadCmd(volumeDir, testFile)
readOut := podRWCmdExec(pod, readCmd)
Expect(readOut).To(ContainSubstring(testFileContent))
}
// Create command to verify that the file doesn't exist
// to be executed via SSH on the node with the local PV
func createFileDoesntExistCmd(testFileDir string, testFile string) string {
@ -487,9 +558,9 @@ func podRWCmdExec(pod *v1.Pod, cmd string) string {
// Initialize test volume on node
// and create local PVC and PV
func setupLocalVolumePVCPV(config *localTestConfig, node *v1.Node) *localTestVolume {
func setupLocalVolumePVCPV(config *localTestConfig, localVolumeType LocalVolumeType) *localTestVolume {
By("Initializing test volume")
testVol := setupLocalVolume(config)
testVol := setupLocalVolume(config, localVolumeType)
By("Creating local PVC and PV")
createLocalPVCPV(config, testVol)