diff --git a/test/e2e/common/util.go b/test/e2e/common/util.go index 67824d0661b..cc4f64efb04 100644 --- a/test/e2e/common/util.go +++ b/test/e2e/common/util.go @@ -16,6 +16,10 @@ limitations under the License. package common +import ( + "k8s.io/kubernetes/pkg/util/sets" +) + type Suite string const ( @@ -24,3 +28,20 @@ const ( ) var CurrentSuite Suite + +// CommonImageWhiteList is the list of images used in common test. These images should be prepulled +// before a tests starts, so that the tests won't fail due image pulling flakes. Currently, this is +// only used by node e2e test. +// TODO(random-liu): Change the image puller pod to use similar mechanism. +var CommonImageWhiteList = sets.NewString( + "gcr.io/google_containers/busybox:1.24", + "gcr.io/google_containers/eptest:0.1", + "gcr.io/google_containers/liveness:e2e", + "gcr.io/google_containers/mounttest:0.7", + "gcr.io/google_containers/mounttest-user:0.3", + "gcr.io/google_containers/netexec:1.4", + "gcr.io/google_containers/nginx-slim:0.7", + "gcr.io/google_containers/serve_hostname:v1.4", + "gcr.io/google_containers/test-webserver:e2e", + "gcr.io/google_containers/hostexec:1.2", +) diff --git a/test/e2e/framework/pods.go b/test/e2e/framework/pods.go index d82881d6e1b..70fedc26090 100644 --- a/test/e2e/framework/pods.go +++ b/test/e2e/framework/pods.go @@ -24,12 +24,18 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/wait" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) +// ImageWhiteList is the images used in the current test suite. It should be initialized in test suite and +// the images in the white list should be pre-pulled in the test suite. Currently, this is only used by +// node e2e test. +var ImageWhiteList sets.String + // Convenience method for getting a pod client interface in the framework's namespace, // possibly applying test-suite specific transformations to the pod spec, e.g. for // node e2e pod scheduling. @@ -107,6 +113,26 @@ func (c *PodClient) mungeSpec(pod *api.Pod) { if TestContext.NodeName != "" { Expect(pod.Spec.NodeName).To(Or(BeZero(), Equal(TestContext.NodeName)), "Test misconfigured") pod.Spec.NodeName = TestContext.NodeName + if !TestContext.PrepullImages { + return + } + // If prepull is enabled, munge the container spec to make sure the images are not pulled + // during the test. + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + if c.ImagePullPolicy == api.PullAlways { + // If the image pull policy is PullAlways, the image doesn't need to be in + // the white list or pre-pulled, because the image is expected to be pulled + // in the test anyway. + continue + } + // If the image policy is not PullAlways, the image must be in the white list and + // pre-pulled. + Expect(ImageWhiteList.Has(c.Image)).To(BeTrue(), "Image %q is not in the white list, consider adding it to CommonImageWhiteList in test/e2e/common/util.go or NodeImageWhiteList in test/e2e_node/image_list.go", c.Image) + // Do not pull images during the tests because the images in white list should have + // been prepulled. + c.ImagePullPolicy = api.PullNever + } } } diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index c49af1613da..7b61d877f86 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -92,6 +92,8 @@ type NodeTestContextType struct { EvictionHard string // ManifestPath is the static pod manifest path. ManifestPath string + // PrepullImages indicates whether node e2e framework should prepull images. + PrepullImages bool } type CloudConfig struct { @@ -183,6 +185,7 @@ func RegisterNodeFlags() { //flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.") flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi,imagefs.available<10%", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.") flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.") + flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") } // Enable viper configuration management of flags. diff --git a/test/e2e_node/apparmor_test.go b/test/e2e_node/apparmor_test.go index f5e87b56764..4d7ad3ad6d9 100644 --- a/test/e2e_node/apparmor_test.go +++ b/test/e2e_node/apparmor_test.go @@ -165,7 +165,7 @@ func createPodWithAppArmor(f *framework.Framework, profile string) *api.Pod { Spec: api.PodSpec{ Containers: []api.Container{{ Name: "test", - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Command: []string{"touch", "foo"}, }}, RestartPolicy: api.RestartPolicyNever, diff --git a/test/e2e_node/cgroup_manager_test.go b/test/e2e_node/cgroup_manager_test.go index 52fffc14e17..8f23c829682 100644 --- a/test/e2e_node/cgroup_manager_test.go +++ b/test/e2e_node/cgroup_manager_test.go @@ -45,7 +45,7 @@ var _ = framework.KubeDescribe("Kubelet Cgroup Manager [Skip]", func() { RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Name: contName, Command: []string{"sh", "-c", "if [ -d /tmp/memory/Burstable ] && [ -d /tmp/memory/BestEffort ]; then exit 0; else exit 1; fi"}, VolumeMounts: []api.VolumeMount{ diff --git a/test/e2e_node/container_manager_test.go b/test/e2e_node/container_manager_test.go index e5fd987c485..3cea30426b7 100644 --- a/test/e2e_node/container_manager_test.go +++ b/test/e2e_node/container_manager_test.go @@ -104,7 +104,7 @@ var _ = framework.KubeDescribe("Kubelet Container Manager [Serial]", func() { Spec: api.PodSpec{ Containers: []api.Container{ { - Image: ImageRegistry[serveHostnameImage], + Image: "gcr.io/google_containers/serve_hostname:v1.4", Name: podName, }, }, @@ -148,7 +148,7 @@ var _ = framework.KubeDescribe("Kubelet Container Manager [Serial]", func() { Spec: api.PodSpec{ Containers: []api.Container{ { - Image: ImageRegistry[nginxImage], + Image: "gcr.io/google_containers/nginx-slim:0.7", Name: podName, Resources: api.ResourceRequirements{ Limits: api.ResourceList{ @@ -189,7 +189,7 @@ var _ = framework.KubeDescribe("Kubelet Container Manager [Serial]", func() { Spec: api.PodSpec{ Containers: []api.Container{ { - Image: ImageRegistry[testWebServer], + Image: "gcr.io/google_containers/test-webserver:e2e", Name: podName, Resources: api.ResourceRequirements{ Requests: api.ResourceList{ diff --git a/test/e2e_node/density_test.go b/test/e2e_node/density_test.go index f598f8e56fa..f68053afd51 100644 --- a/test/e2e_node/density_test.go +++ b/test/e2e_node/density_test.go @@ -324,7 +324,7 @@ func runDensityBatchTest(f *framework.Framework, rc *ResourceCollector, testArg ) // create test pod data structure - pods := newTestPods(testArg.podsNr, ImageRegistry[pauseImage], podType) + pods := newTestPods(testArg.podsNr, framework.GetPauseImageNameForHostArch(), podType) // the controller watches the change of pod status controller := newInformerWatchPod(f, mutex, watchTimes, podType) @@ -403,8 +403,8 @@ func runDensitySeqTest(f *framework.Framework, rc *ResourceCollector, testArg de podType = "density_test_pod" sleepBeforeCreatePods = 30 * time.Second ) - bgPods := newTestPods(testArg.bgPodsNr, ImageRegistry[pauseImage], "background_pod") - testPods := newTestPods(testArg.podsNr, ImageRegistry[pauseImage], podType) + bgPods := newTestPods(testArg.bgPodsNr, framework.GetPauseImageNameForHostArch(), "background_pod") + testPods := newTestPods(testArg.podsNr, framework.GetPauseImageNameForHostArch(), podType) By("Creating a batch of background pods") diff --git a/test/e2e_node/disk_eviction_test.go b/test/e2e_node/disk_eviction_test.go index b8fc9ee9307..3285f889c39 100644 --- a/test/e2e_node/disk_eviction_test.go +++ b/test/e2e_node/disk_eviction_test.go @@ -84,7 +84,7 @@ var _ = framework.KubeDescribe("Kubelet Eviction Manager [Serial] [Disruptive]", RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Name: busyPodName, // Filling the disk Command: []string{"sh", "-c", @@ -190,7 +190,7 @@ func createIdlePod(podName string, podClient *framework.PodClient) { RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[pauseImage], + Image: framework.GetPauseImageNameForHostArch(), Name: podName, }, }, diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index becc40812f3..3c0c4bb5589 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -47,7 +47,6 @@ import ( var e2es *services.E2EServices -var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.") var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.") func init() { @@ -94,7 +93,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { } // Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue // This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling. - if *prePullImages { + if framework.TestContext.PrepullImages { glog.Infof("Pre-pulling images so that they are cached for the tests.") err := PrePullAllImages() Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e_node/image_conformance_test.go b/test/e2e_node/image_conformance_test.go index a5811df2a24..dea6522c0c8 100644 --- a/test/e2e_node/image_conformance_test.go +++ b/test/e2e_node/image_conformance_test.go @@ -36,8 +36,8 @@ var _ = Describe("Image Container Conformance Test", func() { var conformImages []ConformanceImage BeforeEach(func() { existImageTags := []string{ - NoPullImageRegistry[pullTestExecHealthz], - NoPullImageRegistry[pullTestAlpineWithBash], + "gcr.io/google_containers/exechealthz:1.0", + "gcr.io/google_containers/alpine-with-bash:1.0", } for _, existImageTag := range existImageTags { conformImage, _ := NewConformanceImage("docker", existImageTag) diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index 758b542ca76..5ffe193cc6e 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -23,6 +23,8 @@ import ( "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/sets" + commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" ) @@ -31,57 +33,33 @@ const ( maxImagePullRetries = 5 // Sleep duration between image pull retry attempts. imagePullRetryDelay = time.Second - - busyBoxImage = iota - epTestImage - hostExecImage - livenessImage - mountTestImage5 - mountTestImage6 - mountTestImage7 - mountTestUserImage - netExecImage - nginxImage - pauseImage - serveHostnameImage - testWebServer - - // Images just used for explicitly testing pulling of images - pullTestAlpine - pullTestAlpineWithBash - pullTestAuthenticatedAlpine - pullTestExecHealthz ) -var ImageRegistry = map[int]string{ - busyBoxImage: "gcr.io/google_containers/busybox:1.24", - epTestImage: "gcr.io/google_containers/eptest:0.1", - hostExecImage: "gcr.io/google_containers/hostexec:1.2", - livenessImage: "gcr.io/google_containers/liveness:e2e", - mountTestImage7: "gcr.io/google_containers/mounttest:0.7", - mountTestUserImage: "gcr.io/google_containers/mounttest-user:0.3", - netExecImage: "gcr.io/google_containers/netexec:1.4", - nginxImage: "gcr.io/google_containers/nginx-slim:0.7", - pauseImage: framework.GetPauseImageNameForHostArch(), - serveHostnameImage: "gcr.io/google_containers/serve_hostname:v1.4", - testWebServer: "gcr.io/google_containers/test-webserver:e2e", +// NodeImageWhiteList is a list of images used in node e2e test. These images will be prepulled +// before test running so that the image pulling won't fail in actual test. +var NodeImageWhiteList = sets.NewString( + "google/cadvisor:latest", + "gcr.io/google-containers/stress:v1", + "gcr.io/google_containers/busybox:1.24", + "gcr.io/google_containers/nginx-slim:0.7", + "gcr.io/google_containers/serve_hostname:v1.4", + framework.GetPauseImageNameForHostArch(), +) + +func init() { + // Union NodeImageWhiteList and CommonImageWhiteList into the framework image white list. + framework.ImageWhiteList = NodeImageWhiteList.Union(commontest.CommonImageWhiteList) } -// These are used by tests that explicitly test the ability to pull images -var NoPullImageRegistry = map[int]string{ - pullTestExecHealthz: "gcr.io/google_containers/exechealthz:1.0", - pullTestAlpine: "alpine:3.1", - pullTestAlpineWithBash: "gcr.io/google_containers/alpine-with-bash:1.0", - pullTestAuthenticatedAlpine: "gcr.io/authenticated-image-pulling/alpine:3.1", -} - -// Pre-fetch all images tests depend on so that we don't fail in an actual test +// Pre-fetch all images tests depend on so that we don't fail in an actual test. func PrePullAllImages() error { usr, err := user.Current() if err != nil { return err } - for _, image := range ImageRegistry { + images := framework.ImageWhiteList.List() + glog.V(4).Infof("Pre-pulling images %+v", images) + for _, image := range images { var ( err error output []byte @@ -90,6 +68,7 @@ func PrePullAllImages() error { if i > 0 { time.Sleep(imagePullRetryDelay) } + // TODO(random-liu): Use docker client to get rid of docker binary dependency. if output, err = exec.Command("docker", "pull", image).CombinedOutput(); err == nil { break } diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go index 445dc52196b..9202e1e0797 100644 --- a/test/e2e_node/kubelet_test.go +++ b/test/e2e_node/kubelet_test.go @@ -55,7 +55,7 @@ var _ = framework.KubeDescribe("Kubelet", func() { RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Name: podName, Command: []string{"sh", "-c", "echo 'Hello World' ; sleep 240"}, }, @@ -89,7 +89,7 @@ var _ = framework.KubeDescribe("Kubelet", func() { RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Name: podName, Command: []string{"/bin/false"}, }, @@ -136,7 +136,7 @@ var _ = framework.KubeDescribe("Kubelet", func() { RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Name: podName, Command: []string{"sh", "-c", "echo test > /file; sleep 240"}, SecurityContext: &api.SecurityContext{ @@ -221,7 +221,7 @@ func createSummaryTestPods(podClient *framework.PodClient, podNamePrefix string, RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Command: []string{"sh", "-c", "while true; do echo 'hello world' | tee /test-empty-dir-mnt/file ; sleep 1; done"}, Name: podName + containerSuffix, VolumeMounts: []api.VolumeMount{ diff --git a/test/e2e_node/memory_eviction_test.go b/test/e2e_node/memory_eviction_test.go index 4762aeda69d..37639ec6411 100644 --- a/test/e2e_node/memory_eviction_test.go +++ b/test/e2e_node/memory_eviction_test.go @@ -116,7 +116,7 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial] [Disruptive]", fu RestartPolicy: api.RestartPolicyNever, Containers: []api.Container{ { - Image: ImageRegistry[pauseImage], + Image: framework.GetPauseImageNameForHostArch(), Name: podName, }, }, diff --git a/test/e2e_node/mirror_pod_test.go b/test/e2e_node/mirror_pod_test.go index 9aa9c9ff37d..b7315042c85 100644 --- a/test/e2e_node/mirror_pod_test.go +++ b/test/e2e_node/mirror_pod_test.go @@ -44,7 +44,8 @@ var _ = framework.KubeDescribe("MirrorPod", func() { mirrorPodName = staticPodName + "-" + framework.TestContext.NodeName By("create the static pod") - err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, ImageRegistry[nginxImage], api.RestartPolicyAlways) + err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, + "gcr.io/google_containers/nginx-slim:0.7", api.RestartPolicyAlways) Expect(err).ShouldNot(HaveOccurred()) By("wait for the mirror pod to be running") @@ -59,7 +60,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() { uid := pod.UID By("update the static pod container image") - image := ImageRegistry[pauseImage] + image := framework.GetPauseImageNameForHostArch() err = createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, image, api.RestartPolicyAlways) Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e_node/resource_usage_test.go b/test/e2e_node/resource_usage_test.go index 2ebfbe1f214..f9a6a05d0d7 100644 --- a/test/e2e_node/resource_usage_test.go +++ b/test/e2e_node/resource_usage_test.go @@ -139,7 +139,7 @@ func runResourceUsageTest(f *framework.Framework, rc *ResourceCollector, testArg // sleep for an interval here to measure steady data sleepAfterCreatePods = 10 * time.Second ) - pods := newTestPods(testArg.podsNr, ImageRegistry[pauseImage], "test_pod") + pods := newTestPods(testArg.podsNr, framework.GetPauseImageNameForHostArch(), "test_pod") rc.Start() // Explicitly delete pods to prevent namespace controller cleanning up timeout diff --git a/test/e2e_node/runtime_conformance_test.go b/test/e2e_node/runtime_conformance_test.go index d0792f59fa9..697cc2839b4 100644 --- a/test/e2e_node/runtime_conformance_test.go +++ b/test/e2e_node/runtime_conformance_test.go @@ -45,7 +45,7 @@ var _ = framework.KubeDescribe("Container Runtime Conformance Test", func() { restartCountVolumeName := "restart-count" restartCountVolumePath := "/restart-count" testContainer := api.Container{ - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", VolumeMounts: []api.VolumeMount{ { MountPath: restartCountVolumePath, @@ -136,7 +136,7 @@ while true; do sleep 1; done c := ConformanceContainer{ PodClient: f.PodClient(), Container: api.Container{ - Image: ImageRegistry[busyBoxImage], + Image: "gcr.io/google_containers/busybox:1.24", Name: name, Command: []string{"/bin/sh", "-c"}, Args: []string{fmt.Sprintf("/bin/echo -n %s > %s", terminationMessage, terminationMessagePath)}, @@ -185,6 +185,9 @@ while true; do sleep 1; done Data: map[string][]byte{api.DockerConfigJsonKey: []byte(auth)}, Type: api.SecretTypeDockerConfigJson, } + // The following images are not added into NodeImageWhiteList, because this test is + // testing image pulling, these images don't need to be prepulled. The ImagePullPolicy + // is api.PullAlways, so it won't be blocked by framework image white list check. for _, testCase := range []struct { description string image string @@ -206,25 +209,25 @@ while true; do sleep 1; done }, { description: "should be able to pull image from gcr.io", - image: NoPullImageRegistry[pullTestAlpineWithBash], + image: "gcr.io/google_containers/alpine-with-bash:1.0", phase: api.PodRunning, waiting: false, }, { description: "should be able to pull image from docker hub", - image: NoPullImageRegistry[pullTestAlpine], + image: "alpine:3.1", phase: api.PodRunning, waiting: false, }, { description: "should not be able to pull from private registry without secret", - image: NoPullImageRegistry[pullTestAuthenticatedAlpine], + image: "gcr.io/authenticated-image-pulling/alpine:3.1", phase: api.PodPending, waiting: true, }, { description: "should be able to pull from private registry with secret", - image: NoPullImageRegistry[pullTestAuthenticatedAlpine], + image: "gcr.io/authenticated-image-pulling/alpine:3.1", secret: true, phase: api.PodRunning, waiting: false,