github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-12 05:21:58 +00:00
Code Issues Projects Releases Wiki Activity

Add e2e tests that check for wrapped volume race

Browse Source 
See #29641 for details.
This commit is contained in:
Ivan Shvedunov 2016-08-16 04:09:48 +03:00
parent fdd2392035
commit 8ff00d17d8
2 changed files with 281 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

330
test/e2e/empty_dir_wrapper.go
View File

@ -18,21 +18,41 @@ package e2e
import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/util/intstr"
"k8s.io/kubernetes/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
"fmt"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
// These numbers are obtained empirically.
// If you make them too low, you'll get flaky
// tests instead of failing ones if the race bug reappears.
// If you make volume counts or pod counts too high,
// the tests may fail because mounting configmap/git_repo
// volumes is not very fast and the tests may time out
// waiting for pods to become Running.
// And of course the higher are the numbers, the
// slower are the tests.
wrappedVolumeRaceConfigMapVolumeCount = 50
wrappedVolumeRaceConfigMapPodCount = 5
wrappedVolumeRaceConfigMapIterationCount = 3
wrappedVolumeRaceGitRepoVolumeCount = 50
wrappedVolumeRaceGitRepoPodCount = 5
wrappedVolumeRaceGitRepoIterationCount = 3
wrappedVolumeRaceRCNamePrefix = "wrapped-volume-race-"
)
// This test will create a pod with a secret volume and gitRepo volume
// Thus requests a secret, a git server pod, and a git server service
var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
f := framework.NewDefaultFramework("emptydir-wrapper")
It("should becomes running", func() {
It("should not conflict", func() {
name := "emptydir-wrapper-test-" + string(uuid.NewUUID())
volumeName := "secret-volume"
volumeMountPath := "/etc/secret-volume"
@ -52,6 +72,115 @@ var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
framework.Failf("unable to create test secret %s: %v", secret.Name, err)
}
gitVolumeName := "git-volume"
gitVolumeMountPath := "/etc/git-volume"
gitURL, gitRepo, gitCleanup := createGitServer(f)
defer gitCleanup()
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "pod-secrets-" + string(uuid.NewUUID()),
},
Spec: api.PodSpec{
Volumes: []api.Volume{
{
Name: volumeName,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: name,
},
},
},
{
Name: gitVolumeName,
VolumeSource: api.VolumeSource{
GitRepo: &api.GitRepoVolumeSource{
Repository: gitURL,
Directory: gitRepo,
},
},
},
},
Containers: []api.Container{
{
Name: "secret-test",
Image: "gcr.io/google_containers/test-webserver:e2e",
VolumeMounts: []api.VolumeMount{
{
Name: volumeName,
MountPath: volumeMountPath,
ReadOnly: true,
},
{
Name: gitVolumeName,
MountPath: gitVolumeMountPath,
},
},
},
},
},
}
pod, err = f.Client.Pods(f.Namespace.Name).Create(pod)
if err != nil {
framework.Failf("unable to create pod %v: %v", pod.Name, err)
}
err = f.WaitForPodRunning(pod.Name)
Expect(err).NotTo(HaveOccurred(), "Failed waiting for pod %s to enter running state", pod.Name)
defer func() {
By("Cleaning up the secret")
if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
framework.Failf("unable to delete secret %v: %v", secret.Name, err)
}
By("Cleaning up the git vol pod")
if err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
framework.Failf("unable to delete git vol pod %v: %v", pod.Name, err)
}
}()
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, pod))
})
// The following two tests check for the problem fixed in #29641.
// In order to reproduce it you need to revert the fix, e.g. via
// git revert -n df1e925143daf34199b55ffb91d0598244888cce
// or
// curl -sL https://github.com/kubernetes/kubernetes/pull/29641.patch | patch -p1 -R
//
// After that these tests will fail because some of the pods
// they create never enter Running state.
//
// They need to be [Serial] and [Slow] because they try to induce
// the race by creating pods with many volumes and container volume mounts,
// which takes considerable time and may interfere with other tests.
//
// Probably should also try making tests for secrets and downwardapi,
// but these cases are harder because tmpfs-based emptyDir
// appears to be less prone to the race problem.
It("should not cause race condition when used for configmaps [Serial] [Slow]", func() {
configMapNames := createConfigmapsForRace(f)
defer deleteConfigMaps(f, configMapNames)
volumes, volumeMounts := makeConfigMapVolumes(configMapNames)
for i := 0; i < wrappedVolumeRaceConfigMapIterationCount; i++ {
testNoWrappedVolumeRace(f, volumes, volumeMounts, wrappedVolumeRaceConfigMapPodCount)
}
})
It("should not cause race condition when used for git_repo [Serial] [Slow]", func() {
gitURL, gitRepo, cleanup := createGitServer(f)
defer cleanup()
volumes, volumeMounts := makeGitRepoVolumes(gitURL, gitRepo)
for i := 0; i < wrappedVolumeRaceGitRepoIterationCount; i++ {
testNoWrappedVolumeRace(f, volumes, volumeMounts, wrappedVolumeRaceGitRepoPodCount)
}
})
})
func createGitServer(f *framework.Framework) (gitURL string, gitRepo string, cleanup func()) {
var err error
gitServerPodName := "git-server-" + string(uuid.NewUUID())
containerPort := 8000
@ -103,79 +232,166 @@ var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
framework.Failf("unable to create test git server service %s: %v", gitServerSvc.Name, err)
}
gitVolumeName := "git-volume"
gitVolumeMountPath := "/etc/git-volume"
gitURL := "http://" + gitServerSvc.Spec.ClusterIP + ":" + strconv.Itoa(httpPort)
gitRepo := "test"
return "http://" + gitServerSvc.Spec.ClusterIP + ":" + strconv.Itoa(httpPort), "test", func() {
By("Cleaning up the git server pod")
if err := f.Client.Pods(f.Namespace.Name).Delete(gitServerPod.Name, api.NewDeleteOptions(0)); err != nil {
framework.Failf("unable to delete git server pod %v: %v", gitServerPod.Name, err)
}
By("Cleaning up the git server svc")
if err := f.Client.Services(f.Namespace.Name).Delete(gitServerSvc.Name); err != nil {
framework.Failf("unable to delete git server svc %v: %v", gitServerSvc.Name, err)
}
}
}
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "pod-secrets-" + string(uuid.NewUUID()),
},
Spec: api.PodSpec{
Volumes: []api.Volume{
{
func makeGitRepoVolumes(gitURL, gitRepo string) (volumes []api.Volume, volumeMounts []api.VolumeMount) {
for i := 0; i < wrappedVolumeRaceGitRepoVolumeCount; i++ {
volumeName := fmt.Sprintf("racey-git-repo-%d", i)
volumes = append(volumes, api.Volume{
Name: volumeName,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: name,
},
},
},
{
Name: gitVolumeName,
VolumeSource: api.VolumeSource{
GitRepo: &api.GitRepoVolumeSource{
Repository: gitURL,
Directory: gitRepo,
},
},
})
volumeMounts = append(volumeMounts, api.VolumeMount{
Name: volumeName,
MountPath: fmt.Sprintf("/etc/git-volume-%d", i),
})
}
return
}
func createConfigmapsForRace(f *framework.Framework) (configMapNames []string) {
By(fmt.Sprintf("Creating %d configmaps", wrappedVolumeRaceConfigMapVolumeCount))
for i := 0; i < wrappedVolumeRaceConfigMapVolumeCount; i++ {
configMapName := fmt.Sprintf("racey-configmap-%d", i)
configMapNames = append(configMapNames, configMapName)
configMap := &api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Namespace: f.Namespace.Name,
Name: configMapName,
},
Data: map[string]string{
"data-1": "value-1",
},
}
_, err := f.Client.ConfigMaps(f.Namespace.Name).Create(configMap)
framework.ExpectNoError(err)
}
return
}
func deleteConfigMaps(f *framework.Framework, configMapNames []string) {
By("Cleaning up the configMaps")
for _, configMapName := range configMapNames {
err := f.Client.ConfigMaps(f.Namespace.Name).Delete(configMapName)
Expect(err).NotTo(HaveOccurred(), "unable to delete configMap %v", configMapName)
}
}
func makeConfigMapVolumes(configMapNames []string) (volumes []api.Volume, volumeMounts []api.VolumeMount) {
for i, configMapName := range configMapNames {
volumeName := fmt.Sprintf("racey-configmap-%d", i)
volumes = append(volumes, api.Volume{
Name: volumeName,
VolumeSource: api.VolumeSource{
ConfigMap: &api.ConfigMapVolumeSource{
LocalObjectReference: api.LocalObjectReference{
Name: configMapName,
},
Items: []api.KeyToPath{
{
Key: "data-1",
Path: "data-1",
},
},
},
},
})
volumeMounts = append(volumeMounts, api.VolumeMount{
Name: volumeName,
MountPath: fmt.Sprintf("/etc/config-%d", i),
})
}
return
}
func testNoWrappedVolumeRace(f *framework.Framework, volumes []api.Volume, volumeMounts []api.VolumeMount, podCount int32) {
rcName := wrappedVolumeRaceRCNamePrefix + string(uuid.NewUUID())
nodeList := framework.GetReadySchedulableNodesOrDie(f.Client)
Expect(len(nodeList.Items)).To(BeNumerically(">", 0))
targetNode := nodeList.Items[0]
By("Creating RC which spawns configmap-volume pods")
affinity := map[string]string{
api.AffinityAnnotationKey: fmt.Sprintf(`
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "In",
"values": ["%s"]
}]
}]
}}}`, targetNode.Name),
}
rc := &api.ReplicationController{
ObjectMeta: api.ObjectMeta{
Name: rcName,
},
Spec: api.ReplicationControllerSpec{
Replicas: podCount,
Selector: map[string]string{
"name": rcName,
},
Template: &api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Annotations: affinity,
Labels: map[string]string{"name": rcName},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "secret-test",
Image: "gcr.io/google_containers/test-webserver:e2e",
VolumeMounts: []api.VolumeMount{
{
Name: volumeName,
MountPath: volumeMountPath,
ReadOnly: true,
},
{
Name: gitVolumeName,
MountPath: gitVolumeMountPath,
Name: "test-container",
Image: "gcr.io/google_containers/busybox:1.24",
Command: []string{"sleep", "10000"},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("10m"),
},
},
VolumeMounts: volumeMounts,
},
},
DNSPolicy: api.DNSDefault,
Volumes: volumes,
},
},
},
}
pod, err = f.Client.Pods(f.Namespace.Name).Create(pod)
if err != nil {
framework.Failf("unable to create pod %v: %v", pod.Name, err)
}
_, err := f.Client.ReplicationControllers(f.Namespace.Name).Create(rc)
Expect(err).NotTo(HaveOccurred(), "error creating replication controller")
defer func() {
By("Cleaning up the secret")
if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
framework.Failf("unable to delete secret %v: %v", secret.Name, err)
}
By("Cleaning up the git server pod")
if err = f.Client.Pods(f.Namespace.Name).Delete(gitServerPod.Name, api.NewDeleteOptions(0)); err != nil {
framework.Failf("unable to delete git server pod %v: %v", gitServerPod.Name, err)
}
By("Cleaning up the git server svc")
if err = f.Client.Services(f.Namespace.Name).Delete(gitServerSvc.Name); err != nil {
framework.Failf("unable to delete git server svc %v: %v", gitServerSvc.Name, err)
}
By("Cleaning up the git vol pod")
if err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
framework.Failf("unable to delete git vol pod %v: %v", pod.Name, err)
}
err := framework.DeleteRCAndPods(f.Client, f.Namespace.Name, rcName)
framework.ExpectNoError(err)
}()
framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, pod))
})
})
pods, err := framework.PodsCreated(f.Client, f.Namespace.Name, rcName, podCount)
By("Ensuring each pod is running")
// Wait for the pods to enter the running state. Waiting loops until the pods
// are running so non-running pods cause a timeout for this test.
for _, pod := range pods.Items {
if pod.DeletionTimestamp != nil {
continue
}
err = f.WaitForPodRunning(pod.Name)
Expect(err).NotTo(HaveOccurred(), "Failed waiting for pod %s to enter running state", pod.Name)
}
}

2
test/test_owners.csv
View File

@ -115,7 +115,7 @@ EmptyDir volumes when FSGroup is specified new files should be created with FSGr
EmptyDir volumes when FSGroup is specified new files should be created with FSGroup ownership when container is root,childsb,1
EmptyDir volumes when FSGroup is specified volume on default medium should have the correct mode using FSGroup,eparis,1
EmptyDir volumes when FSGroup is specified volume on tmpfs should have the correct mode using FSGroup,timothysc,1
EmptyDir wrapper volumes should becomes running,deads2k,1
EmptyDir wrapper volumes should not conflict,deads2k,1
Etcd failure should recover from SIGKILL,pmorie,1
Etcd failure should recover from network partition with master,justinsb,1
Events should be sent by kubelets and the scheduler about pods scheduling and running,zmerlynn,1

1 name owner auto-assigned
115 EmptyDir volumes when FSGroup is specified new files should be created with FSGroup ownership when container is root childsb 1
116 EmptyDir volumes when FSGroup is specified volume on default medium should have the correct mode using FSGroup eparis 1
117 EmptyDir volumes when FSGroup is specified volume on tmpfs should have the correct mode using FSGroup timothysc 1
118 EmptyDir wrapper volumes should becomes running EmptyDir wrapper volumes should not conflict deads2k 1
119 Etcd failure should recover from SIGKILL pmorie 1
120 Etcd failure should recover from network partition with master justinsb 1
121 Events should be sent by kubelets and the scheduler about pods scheduling and running zmerlynn 1