Add e2e tests that check for wrapped volume race

See #29641 for details.

test/e2e/empty_dir_wrapper.go

@@ -18,21 +18,41 @@ package e2e
 
 import (
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/util/intstr"
 	"k8s.io/kubernetes/pkg/util/uuid"
 	"k8s.io/kubernetes/test/e2e/framework"
 
+	"fmt"
 	"strconv"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	// These numbers are obtained empirically.
+	// If you make them too low, you'll get flaky
+	// tests instead of failing ones if the race bug reappears.
+	// If you make volume counts or pod counts too high,
+	// the tests may fail because mounting configmap/git_repo
+	// volumes is not very fast and the tests may time out
+	// waiting for pods to become Running.
+	// And of course the higher are the numbers, the
+	// slower are the tests.
+	wrappedVolumeRaceConfigMapVolumeCount    = 50
+	wrappedVolumeRaceConfigMapPodCount       = 5
+	wrappedVolumeRaceConfigMapIterationCount = 3
+	wrappedVolumeRaceGitRepoVolumeCount      = 50
+	wrappedVolumeRaceGitRepoPodCount         = 5
+	wrappedVolumeRaceGitRepoIterationCount   = 3
+	wrappedVolumeRaceRCNamePrefix            = "wrapped-volume-race-"
 )
 
-// This test will create a pod with a secret volume and gitRepo volume
-// Thus requests a secret, a git server pod, and a git server service
 var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
 	f := framework.NewDefaultFramework("emptydir-wrapper")
 
-	It("should becomes running", func() {
+	It("should not conflict", func() {
 		name := "emptydir-wrapper-test-" + string(uuid.NewUUID())
 		volumeName := "secret-volume"
 		volumeMountPath := "/etc/secret-volume"
@@ -52,61 +72,10 @@ var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
 			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
 		}
 
-		gitServerPodName := "git-server-" + string(uuid.NewUUID())
-		containerPort := 8000
-
-		labels := map[string]string{"name": gitServerPodName}
-
-		gitServerPod := &api.Pod{
-			ObjectMeta: api.ObjectMeta{
-				Name:   gitServerPodName,
-				Labels: labels,
-			},
-			Spec: api.PodSpec{
-				Containers: []api.Container{
-					{
-						Name:            "git-repo",
-						Image:           "gcr.io/google_containers/fakegitserver:0.1",
-						ImagePullPolicy: "IfNotPresent",
-						Ports: []api.ContainerPort{
-							{ContainerPort: int32(containerPort)},
-						},
-					},
-				},
-			},
-		}
-
-		if gitServerPod, err = f.Client.Pods(f.Namespace.Name).Create(gitServerPod); err != nil {
-			framework.Failf("unable to create test git server pod %s: %v", gitServerPod.Name, err)
-		}
-
-		// Portal IP and port
-		httpPort := 2345
-
-		gitServerSvc := &api.Service{
-			ObjectMeta: api.ObjectMeta{
-				Name: "git-server-svc",
-			},
-			Spec: api.ServiceSpec{
-				Selector: labels,
-				Ports: []api.ServicePort{
-					{
-						Name:       "http-portal",
-						Port:       int32(httpPort),
-						TargetPort: intstr.FromInt(containerPort),
-					},
-				},
-			},
-		}
-
-		if gitServerSvc, err = f.Client.Services(f.Namespace.Name).Create(gitServerSvc); err != nil {
-			framework.Failf("unable to create test git server service %s: %v", gitServerSvc.Name, err)
-		}
-
 		gitVolumeName := "git-volume"
 		gitVolumeMountPath := "/etc/git-volume"
-		gitURL := "http://" + gitServerSvc.Spec.ClusterIP + ":" + strconv.Itoa(httpPort)
+		gitURL, gitRepo, gitCleanup := createGitServer(f)
-		gitRepo := "test"
+		defer gitCleanup()
 
 		pod := &api.Pod{
 			ObjectMeta: api.ObjectMeta{
@@ -157,19 +126,14 @@ var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
 			framework.Failf("unable to create pod %v: %v", pod.Name, err)
 		}
 
+		err = f.WaitForPodRunning(pod.Name)
+		Expect(err).NotTo(HaveOccurred(), "Failed waiting for pod %s to enter running state", pod.Name)
+
 		defer func() {
 			By("Cleaning up the secret")
 			if err := f.Client.Secrets(f.Namespace.Name).Delete(secret.Name); err != nil {
 				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
 			}
-			By("Cleaning up the git server pod")
-			if err = f.Client.Pods(f.Namespace.Name).Delete(gitServerPod.Name, api.NewDeleteOptions(0)); err != nil {
-				framework.Failf("unable to delete git server pod %v: %v", gitServerPod.Name, err)
-			}
-			By("Cleaning up the git server svc")
-			if err = f.Client.Services(f.Namespace.Name).Delete(gitServerSvc.Name); err != nil {
-				framework.Failf("unable to delete git server svc %v: %v", gitServerSvc.Name, err)
-			}
 			By("Cleaning up the git vol pod")
 			if err = f.Client.Pods(f.Namespace.Name).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
 				framework.Failf("unable to delete git vol pod %v: %v", pod.Name, err)
@@ -178,4 +142,256 @@ var _ = framework.KubeDescribe("EmptyDir wrapper volumes", func() {
 
 		framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.Client, pod))
 	})
+
+	// The following two tests check for the problem fixed in #29641.
+	// In order to reproduce it you need to revert the fix, e.g. via
+	// git revert -n df1e925143daf34199b55ffb91d0598244888cce
+	// or
+	// curl -sL https://github.com/kubernetes/kubernetes/pull/29641.patch | patch -p1 -R
+	//
+	// After that these tests will fail because some of the pods
+	// they create never enter Running state.
+	//
+	// They need to be [Serial] and [Slow] because they try to induce
+	// the race by creating pods with many volumes and container volume mounts,
+	// which takes considerable time and may interfere with other tests.
+	//
+	// Probably should also try making tests for secrets and downwardapi,
+	// but these cases are harder because tmpfs-based emptyDir
+	// appears to be less prone to the race problem.
+
+	It("should not cause race condition when used for configmaps [Serial] [Slow]", func() {
+		configMapNames := createConfigmapsForRace(f)
+		defer deleteConfigMaps(f, configMapNames)
+		volumes, volumeMounts := makeConfigMapVolumes(configMapNames)
+		for i := 0; i < wrappedVolumeRaceConfigMapIterationCount; i++ {
+			testNoWrappedVolumeRace(f, volumes, volumeMounts, wrappedVolumeRaceConfigMapPodCount)
+		}
+	})
+
+	It("should not cause race condition when used for git_repo [Serial] [Slow]", func() {
+		gitURL, gitRepo, cleanup := createGitServer(f)
+		defer cleanup()
+		volumes, volumeMounts := makeGitRepoVolumes(gitURL, gitRepo)
+		for i := 0; i < wrappedVolumeRaceGitRepoIterationCount; i++ {
+			testNoWrappedVolumeRace(f, volumes, volumeMounts, wrappedVolumeRaceGitRepoPodCount)
+		}
+	})
 })
+
+func createGitServer(f *framework.Framework) (gitURL string, gitRepo string, cleanup func()) {
+	var err error
+	gitServerPodName := "git-server-" + string(uuid.NewUUID())
+	containerPort := 8000
+
+	labels := map[string]string{"name": gitServerPodName}
+
+	gitServerPod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			Name:   gitServerPodName,
+			Labels: labels,
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{
+					Name:            "git-repo",
+					Image:           "gcr.io/google_containers/fakegitserver:0.1",
+					ImagePullPolicy: "IfNotPresent",
+					Ports: []api.ContainerPort{
+						{ContainerPort: int32(containerPort)},
+					},
+				},
+			},
+		},
+	}
+
+	if gitServerPod, err = f.Client.Pods(f.Namespace.Name).Create(gitServerPod); err != nil {
+		framework.Failf("unable to create test git server pod %s: %v", gitServerPod.Name, err)
+	}
+
+	// Portal IP and port
+	httpPort := 2345
+
+	gitServerSvc := &api.Service{
+		ObjectMeta: api.ObjectMeta{
+			Name: "git-server-svc",
+		},
+		Spec: api.ServiceSpec{
+			Selector: labels,
+			Ports: []api.ServicePort{
+				{
+					Name:       "http-portal",
+					Port:       int32(httpPort),
+					TargetPort: intstr.FromInt(containerPort),
+				},
+			},
+		},
+	}
+
+	if gitServerSvc, err = f.Client.Services(f.Namespace.Name).Create(gitServerSvc); err != nil {
+		framework.Failf("unable to create test git server service %s: %v", gitServerSvc.Name, err)
+	}
+
+	return "http://" + gitServerSvc.Spec.ClusterIP + ":" + strconv.Itoa(httpPort), "test", func() {
+		By("Cleaning up the git server pod")
+		if err := f.Client.Pods(f.Namespace.Name).Delete(gitServerPod.Name, api.NewDeleteOptions(0)); err != nil {
+			framework.Failf("unable to delete git server pod %v: %v", gitServerPod.Name, err)
+		}
+		By("Cleaning up the git server svc")
+		if err := f.Client.Services(f.Namespace.Name).Delete(gitServerSvc.Name); err != nil {
+			framework.Failf("unable to delete git server svc %v: %v", gitServerSvc.Name, err)
+		}
+	}
+}
+
+func makeGitRepoVolumes(gitURL, gitRepo string) (volumes []api.Volume, volumeMounts []api.VolumeMount) {
+	for i := 0; i < wrappedVolumeRaceGitRepoVolumeCount; i++ {
+		volumeName := fmt.Sprintf("racey-git-repo-%d", i)
+		volumes = append(volumes, api.Volume{
+			Name: volumeName,
+			VolumeSource: api.VolumeSource{
+				GitRepo: &api.GitRepoVolumeSource{
+					Repository: gitURL,
+					Directory:  gitRepo,
+				},
+			},
+		})
+		volumeMounts = append(volumeMounts, api.VolumeMount{
+			Name:      volumeName,
+			MountPath: fmt.Sprintf("/etc/git-volume-%d", i),
+		})
+	}
+	return
+}
+
+func createConfigmapsForRace(f *framework.Framework) (configMapNames []string) {
+	By(fmt.Sprintf("Creating %d configmaps", wrappedVolumeRaceConfigMapVolumeCount))
+	for i := 0; i < wrappedVolumeRaceConfigMapVolumeCount; i++ {
+		configMapName := fmt.Sprintf("racey-configmap-%d", i)
+		configMapNames = append(configMapNames, configMapName)
+		configMap := &api.ConfigMap{
+			ObjectMeta: api.ObjectMeta{
+				Namespace: f.Namespace.Name,
+				Name:      configMapName,
+			},
+			Data: map[string]string{
+				"data-1": "value-1",
+			},
+		}
+		_, err := f.Client.ConfigMaps(f.Namespace.Name).Create(configMap)
+		framework.ExpectNoError(err)
+	}
+	return
+}
+
+func deleteConfigMaps(f *framework.Framework, configMapNames []string) {
+	By("Cleaning up the configMaps")
+	for _, configMapName := range configMapNames {
+		err := f.Client.ConfigMaps(f.Namespace.Name).Delete(configMapName)
+		Expect(err).NotTo(HaveOccurred(), "unable to delete configMap %v", configMapName)
+	}
+}
+
+func makeConfigMapVolumes(configMapNames []string) (volumes []api.Volume, volumeMounts []api.VolumeMount) {
+	for i, configMapName := range configMapNames {
+		volumeName := fmt.Sprintf("racey-configmap-%d", i)
+		volumes = append(volumes, api.Volume{
+			Name: volumeName,
+			VolumeSource: api.VolumeSource{
+				ConfigMap: &api.ConfigMapVolumeSource{
+					LocalObjectReference: api.LocalObjectReference{
+						Name: configMapName,
+					},
+					Items: []api.KeyToPath{
+						{
+							Key:  "data-1",
+							Path: "data-1",
+						},
+					},
+				},
+			},
+		})
+		volumeMounts = append(volumeMounts, api.VolumeMount{
+			Name:      volumeName,
+			MountPath: fmt.Sprintf("/etc/config-%d", i),
+		})
+	}
+	return
+}
+
+func testNoWrappedVolumeRace(f *framework.Framework, volumes []api.Volume, volumeMounts []api.VolumeMount, podCount int32) {
+	rcName := wrappedVolumeRaceRCNamePrefix + string(uuid.NewUUID())
+	nodeList := framework.GetReadySchedulableNodesOrDie(f.Client)
+	Expect(len(nodeList.Items)).To(BeNumerically(">", 0))
+	targetNode := nodeList.Items[0]
+
+	By("Creating RC which spawns configmap-volume pods")
+	affinity := map[string]string{
+		api.AffinityAnnotationKey: fmt.Sprintf(`
+				{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
+					"nodeSelectorTerms": [{
+						"matchExpressions": [{
+							"key": "kubernetes.io/hostname",
+							"operator": "In",
+							"values": ["%s"]
+					}]
+				}]
+			}}}`, targetNode.Name),
+	}
+
+	rc := &api.ReplicationController{
+		ObjectMeta: api.ObjectMeta{
+			Name: rcName,
+		},
+		Spec: api.ReplicationControllerSpec{
+			Replicas: podCount,
+			Selector: map[string]string{
+				"name": rcName,
+			},
+			Template: &api.PodTemplateSpec{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: affinity,
+					Labels:      map[string]string{"name": rcName},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{
+						{
+							Name:    "test-container",
+							Image:   "gcr.io/google_containers/busybox:1.24",
+							Command: []string{"sleep", "10000"},
+							Resources: api.ResourceRequirements{
+								Requests: api.ResourceList{
+									api.ResourceCPU: resource.MustParse("10m"),
+								},
+							},
+							VolumeMounts: volumeMounts,
+						},
+					},
+					DNSPolicy: api.DNSDefault,
+					Volumes:   volumes,
+				},
+			},
+		},
+	}
+	_, err := f.Client.ReplicationControllers(f.Namespace.Name).Create(rc)
+	Expect(err).NotTo(HaveOccurred(), "error creating replication controller")
+
+	defer func() {
+		err := framework.DeleteRCAndPods(f.Client, f.Namespace.Name, rcName)
+		framework.ExpectNoError(err)
+	}()
+
+	pods, err := framework.PodsCreated(f.Client, f.Namespace.Name, rcName, podCount)
+
+	By("Ensuring each pod is running")
+
+	// Wait for the pods to enter the running state. Waiting loops until the pods
+	// are running so non-running pods cause a timeout for this test.
+	for _, pod := range pods.Items {
+		if pod.DeletionTimestamp != nil {
+			continue
+		}
+		err = f.WaitForPodRunning(pod.Name)
+		Expect(err).NotTo(HaveOccurred(), "Failed waiting for pod %s to enter running state", pod.Name)
+	}
+}

test/test_owners.csv

@@ -115,7 +115,7 @@ EmptyDir volumes when FSGroup is specified new files should be created with FSGr
 EmptyDir volumes when FSGroup is specified new files should be created with FSGroup ownership when container is root,childsb,1
 EmptyDir volumes when FSGroup is specified volume on default medium should have the correct mode using FSGroup,eparis,1
 EmptyDir volumes when FSGroup is specified volume on tmpfs should have the correct mode using FSGroup,timothysc,1
-EmptyDir wrapper volumes should becomes running,deads2k,1
+EmptyDir wrapper volumes should not conflict,deads2k,1
 Etcd failure should recover from SIGKILL,pmorie,1
 Etcd failure should recover from network partition with master,justinsb,1
 Events should be sent by kubelets and the scheduler about pods scheduling and running,zmerlynn,1