Merge pull request #36299 from Random-Liu/mark-more-conformance-test

Automatic merge from submit-queue

Node Conformance Test: Mark more conformance test

For https://github.com/kubernetes/kubernetes/issues/30122.

This PR:
1) Removes unused image test.
2) Marks more conformance tests based on https://docs.google.com/spreadsheets/d/1yib6ypfdWuq8Ikyo-rTcBGHe76Xur7tGqCKD9dkzx0Y/edit?usp=sharing.

Notice that 2 tests are not marked conformance for now:
1. **OOM score test:** The test is serial and is verifying host PID directly. The test should start a pod with PID=host and verify inside the pod. @vishh 
2. **Summary api test:** The assumption made in the test doesn't always make sense for arbitrary image, for example: The fs capacity bounds is only [(100mb, 100gb)](https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/summary_test.go#L62). @timstclair 
3. We should consider mark **[cgroup manager test](https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/cgroup_manager_test.go)** as conformance test. 

@dchen1107 @vishh @timstclair 
/cc @kubernetes/sig-node

test/e2e/common/configmap.go

@@ -191,7 +191,7 @@ var _ = framework.KubeDescribe("ConfigMap", func() {
 		})
 	})
 
-	It("should be consumable in multiple volumes in the same pod", func() {
+	It("should be consumable in multiple volumes in the same pod [Conformance]", func() {
 		var (
 			name             = "configmap-test-volume-" + string(uuid.NewUUID())
 			volumeName       = "configmap-volume"

test/e2e/common/downward_api.go

@@ -61,7 +61,7 @@ var _ = framework.KubeDescribe("Downward API", func() {
 		testDownwardAPI(f, podName, env, expectations)
 	})
 
-	It("should provide pod IP as an env var", func() {
+	It("should provide pod IP as an env var [Conformance]", func() {
 		podName := "downward-api-" + string(uuid.NewUUID())
 		env := []api.EnvVar{
 			{
@@ -82,7 +82,7 @@ var _ = framework.KubeDescribe("Downward API", func() {
 		testDownwardAPI(f, podName, env, expectations)
 	})
 
-	It("should provide container's limits.cpu/memory and requests.cpu/memory as env vars", func() {
+	It("should provide container's limits.cpu/memory and requests.cpu/memory as env vars [Conformance]", func() {
 		podName := "downward-api-" + string(uuid.NewUUID())
 		env := []api.EnvVar{
 			{
@@ -128,7 +128,7 @@ var _ = framework.KubeDescribe("Downward API", func() {
 		testDownwardAPI(f, podName, env, expectations)
 	})
 
-	It("should provide default limits.cpu/memory from node allocatable", func() {
+	It("should provide default limits.cpu/memory from node allocatable [Conformance]", func() {
 		podName := "downward-api-" + string(uuid.NewUUID())
 		env := []api.EnvVar{
 			{

test/e2e/common/downwardapi_volume.go

@@ -137,7 +137,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() {
 			podLogTimeout, framework.Poll).Should(ContainSubstring("builder=\"foo\"\n"))
 	})
 
-	It("should provide container's cpu limit", func() {
+	It("should provide container's cpu limit [Conformance]", func() {
 		podName := "downwardapi-volume-" + string(uuid.NewUUID())
 		pod := downwardAPIVolumeForContainerResources(podName, "/etc/cpu_limit")
 
@@ -146,7 +146,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() {
 		})
 	})
 
-	It("should provide container's memory limit", func() {
+	It("should provide container's memory limit [Conformance]", func() {
 		podName := "downwardapi-volume-" + string(uuid.NewUUID())
 		pod := downwardAPIVolumeForContainerResources(podName, "/etc/memory_limit")
 
@@ -155,7 +155,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() {
 		})
 	})
 
-	It("should provide container's cpu request", func() {
+	It("should provide container's cpu request [Conformance]", func() {
 		podName := "downwardapi-volume-" + string(uuid.NewUUID())
 		pod := downwardAPIVolumeForContainerResources(podName, "/etc/cpu_request")
 
@@ -164,7 +164,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() {
 		})
 	})
 
-	It("should provide container's memory request", func() {
+	It("should provide container's memory request [Conformance]", func() {
 		podName := "downwardapi-volume-" + string(uuid.NewUUID())
 		pod := downwardAPIVolumeForContainerResources(podName, "/etc/memory_request")
 
@@ -173,14 +173,14 @@ var _ = framework.KubeDescribe("Downward API volume", func() {
 		})
 	})
 
-	It("should provide node allocatable (cpu) as default cpu limit if the limit is not set", func() {
+	It("should provide node allocatable (cpu) as default cpu limit if the limit is not set [Conformance]", func() {
 		podName := "downwardapi-volume-" + string(uuid.NewUUID())
 		pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/cpu_limit")
 
 		f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"})
 	})
 
-	It("should provide node allocatable (memory) as default memory limit if the limit is not set", func() {
+	It("should provide node allocatable (memory) as default memory limit if the limit is not set [Conformance]", func() {
 		podName := "downwardapi-volume-" + string(uuid.NewUUID())
 		pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/memory_limit")
 

test/e2e/common/kubelet_etc_hosts.go

@@ -45,7 +45,7 @@ var _ = framework.KubeDescribe("KubeletManagedEtcHosts", func() {
 		f: f,
 	}
 
-	It("should test kubelet managed /etc/hosts file", func() {
+	It("should test kubelet managed /etc/hosts file [Conformance]", func() {
 		By("Setting up the test")
 		config.setup()
 

test/e2e_node/BUILD

@@ -16,7 +16,6 @@ go_library(
         "benchmark_util.go",
         "container.go",
         "doc.go",
-        "image.go",
         "image_list.go",
         "resource_collector.go",
         "simple_mount.go",
@@ -30,9 +29,6 @@ go_library(
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
-        "//pkg/kubelet/container:go_default_library",
-        "//pkg/kubelet/dockertools:go_default_library",
-        "//pkg/kubelet/pod:go_default_library",
         "//pkg/labels:go_default_library",
         "//pkg/util/procfs:go_default_library",
         "//pkg/util/runtime:go_default_library",
@@ -61,7 +57,6 @@ go_test(
         "disk_eviction_test.go",
         "dynamic_kubelet_configuration_test.go",
         "e2e_node_suite_test.go",
-        "image_conformance_test.go",
         "image_id_test.go",
         "kubelet_test.go",
         "lifecycle_hook_test.go",

test/e2e_node/image.go

@@ -1,82 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e_node
-
-import (
-	"fmt"
-
-	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
-	"k8s.io/kubernetes/pkg/kubelet/dockertools"
-	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
-)
-
-type ConformanceImage struct {
-	Image   kubecontainer.ImageSpec
-	Runtime kubecontainer.Runtime
-}
-
-func NewConformanceImage(containerRuntime string, image string) (ci ConformanceImage, err error) {
-	ci.Image = kubecontainer.ImageSpec{Image: image}
-	if containerRuntime == "docker" {
-		ci.Runtime = dockerRuntime()
-		return ci, nil
-	}
-	return ci, fmt.Errorf("Unsupported runtime : %s.", containerRuntime)
-}
-
-//TODO: do not expose kubelet implementation details after we refactor the runtime API.
-func dockerRuntime() kubecontainer.Runtime {
-	dockerClient := dockertools.ConnectToDockerOrDie("", 0)
-	pm := kubepod.NewBasicPodManager(nil)
-	dm := dockertools.NewDockerManager(
-		dockerClient,
-		nil, nil, nil, pm, nil,
-		"", 0, 0, "",
-		nil, nil, nil, nil, nil, nil, nil,
-		false, nil, true, false, false, "",
-	)
-
-	return dm
-}
-
-func (ci *ConformanceImage) Pull() error {
-	return ci.Runtime.PullImage(ci.Image, nil)
-}
-
-func (ci *ConformanceImage) Present() (bool, error) {
-	return ci.Runtime.IsImagePresent(ci.Image)
-}
-
-func (ci *ConformanceImage) List() ([]string, error) {
-	if images, err := ci.Runtime.ListImages(); err != nil {
-		return nil, err
-	} else {
-		var tags []string
-		for _, image := range images {
-			tags = append(tags, image.RepoTags...)
-		}
-		return tags, nil
-	}
-}
-
-func (ci *ConformanceImage) Remove() error {
-	return ci.Runtime.RemoveImage(ci.Image)
-}
-
-func (ci *ConformanceImage) GetTag() string {
-	return ci.Image.Image
-}

test/e2e_node/image_conformance_test.go

@@ -1,120 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e_node
-
-import (
-	"time"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-const (
-	imageRetryTimeout         = time.Minute * 10 // Image pulls can take a long time and shouldn't cause flakes
-	imagePullInterval         = time.Second * 15
-	imageConsistentlyTimeout  = time.Second * 30
-	imageConsistentlyInterval = time.Second * 5
-)
-
-var _ = Describe("Image Container Conformance Test", func() {
-	Describe("[Flaky] image conformance blackbox test", func() {
-		Context("when testing images that exist", func() {
-			var conformImages []ConformanceImage
-			BeforeEach(func() {
-				existImageTags := []string{
-					"gcr.io/google_containers/exechealthz:1.0",
-					"gcr.io/google_containers/alpine-with-bash:1.0",
-				}
-				for _, existImageTag := range existImageTags {
-					conformImage, _ := NewConformanceImage("docker", existImageTag)
-					// Pulling images from gcr.io is flaky, so retry failures
-					Eventually(func() error {
-						return conformImage.Pull()
-					}, imageRetryTimeout, imagePullInterval).ShouldNot(HaveOccurred())
-					conformImages = append(conformImages, conformImage)
-				}
-			})
-
-			It("It should present successfully", func() {
-				for _, conformImage := range conformImages {
-					present, err := conformImage.Present()
-					Expect(err).ShouldNot(HaveOccurred())
-					Expect(present).To(BeTrue())
-				}
-			})
-
-			It("should list pulled images", func() {
-				image, _ := NewConformanceImage("docker", "")
-				tags, err := image.List()
-				Expect(err).ShouldNot(HaveOccurred())
-				for _, conformImage := range conformImages {
-					Expect(tags).To(ContainElement(conformImage.GetTag()))
-				}
-			})
-
-			AfterEach(func() {
-				for _, conformImage := range conformImages {
-					conformImage.Remove()
-				}
-				conformImages = []ConformanceImage{}
-			})
-
-		})
-		Context("when testing image that does not exist", func() {
-			var conformImages []ConformanceImage
-			invalidImageTags := []string{
-				// nonexistent image registry
-				"foo.com/foo/fooimage",
-				// nonexistent image
-				"gcr.io/google_containers/not_exist",
-				// TODO(random-liu): Add test for image pulling credential
-			}
-			It("should ignore pull failures", func() {
-				for _, invalidImageTag := range invalidImageTags {
-					conformImage, _ := NewConformanceImage("docker", invalidImageTag)
-					// Pulling images from gcr.io is flaky, so retry to make sure failure is not caused by flaky.
-					Expect(conformImage.Pull()).Should(HaveOccurred())
-					conformImages = append(conformImages, conformImage)
-				}
-
-				By("not presenting images", func() {
-					for _, conformImage := range conformImages {
-						present, err := conformImage.Present()
-						Expect(err).ShouldNot(HaveOccurred())
-						Expect(present).To(BeFalse())
-					}
-				})
-
-				By("not listing pulled images", func() {
-					image, _ := NewConformanceImage("docker", "")
-					tags, err := image.List()
-					Expect(err).ShouldNot(HaveOccurred())
-					for _, conformImage := range conformImages {
-						Expect(tags).NotTo(ContainElement(conformImage.GetTag()))
-					}
-				})
-
-				By("not removing non-exist images", func() {
-					for _, conformImage := range conformImages {
-						err := conformImage.Remove()
-						Expect(err).Should(HaveOccurred())
-					}
-				})
-			})
-		})
-	})
-})

test/e2e_node/kubelet_test.go

@@ -38,7 +38,7 @@ var _ = framework.KubeDescribe("Kubelet", func() {
 	})
 	Context("when scheduling a busybox command in a pod", func() {
 		podName := "busybox-scheduling-" + string(uuid.NewUUID())
-		It("it should print the output to logs", func() {
+		It("it should print the output to logs [Conformance]", func() {
 			podClient.CreateSync(&api.Pod{
 				ObjectMeta: api.ObjectMeta{
 					Name: podName,
@@ -118,7 +118,7 @@ var _ = framework.KubeDescribe("Kubelet", func() {
 	})
 	Context("when scheduling a read only busybox container", func() {
 		podName := "busybox-readonly-fs" + string(uuid.NewUUID())
-		It("it should not write to root filesystem", func() {
+		It("it should not write to root filesystem [Conformance]", func() {
 			isReadOnly := true
 			podClient.CreateSync(&api.Pod{
 				ObjectMeta: api.ObjectMeta{

test/e2e_node/lifecycle_hook_test.go

@@ -71,7 +71,7 @@ var _ = framework.KubeDescribe("Container Lifecycle Hook", func() {
 		})
 
 		Context("when it is exec hook", func() {
-			It("should execute poststart exec hook properly", func() {
+			It("should execute poststart exec hook properly [Conformance]", func() {
 				podWithHook := getLifecycleHookTestPod("pod-with-poststart-exec-hook",
 					// Block forever
 					[]string{"tail", "-f", "/dev/null"},
@@ -84,7 +84,7 @@ var _ = framework.KubeDescribe("Container Lifecycle Hook", func() {
 				testPodWithHook(podWithHook)
 			})
 
-			It("should execute prestop exec hook properly", func() {
+			It("should execute prestop exec hook properly [Conformance]", func() {
 				podWithHook := getLifecycleHookTestPod("pod-with-prestop-exec-hook",
 					// Block forever
 					[]string{"tail", "-f", "/dev/null"},
@@ -117,7 +117,7 @@ var _ = framework.KubeDescribe("Container Lifecycle Hook", func() {
 				podHandleHookRequest = podClient.CreateSync(podHandleHookRequest)
 				targetIP = podHandleHookRequest.Status.PodIP
 			})
-			It("should execute poststart http hook properly", func() {
+			It("should execute poststart http hook properly [Conformance]", func() {
 				podWithHook := getLifecycleHookTestPod("pod-with-poststart-http-hook",
 					// Block forever
 					[]string{"tail", "-f", "/dev/null"},
@@ -132,7 +132,7 @@ var _ = framework.KubeDescribe("Container Lifecycle Hook", func() {
 				}
 				testPodWithHook(podWithHook)
 			})
-			It("should execute prestop http hook properly", func() {
+			It("should execute prestop http hook properly [Conformance]", func() {
 				podWithHook := getLifecycleHookTestPod("pod-with-prestop-http-hook",
 					// Block forever
 					[]string{"tail", "-f", "/dev/null"},

test/e2e_node/mirror_pod_test.go

@@ -53,7 +53,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
 				return checkMirrorPodRunning(f.ClientSet, mirrorPodName, ns)
 			}, 2*time.Minute, time.Second*4).Should(BeNil())
 		})
-		It("should be updated when static pod updated", func() {
+		It("should be updated when static pod updated [Conformance]", func() {
 			By("get mirror pod uid")
 			pod, err := f.ClientSet.Core().Pods(ns).Get(mirrorPodName)
 			Expect(err).ShouldNot(HaveOccurred())
@@ -75,7 +75,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
 			Expect(len(pod.Spec.Containers)).Should(Equal(1))
 			Expect(pod.Spec.Containers[0].Image).Should(Equal(image))
 		})
-		It("should be recreated when mirror pod gracefully deleted", func() {
+		It("should be recreated when mirror pod gracefully deleted [Conformance]", func() {
 			By("get mirror pod uid")
 			pod, err := f.ClientSet.Core().Pods(ns).Get(mirrorPodName)
 			Expect(err).ShouldNot(HaveOccurred())
@@ -90,7 +90,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
 				return checkMirrorPodRecreatedAndRunnig(f.ClientSet, mirrorPodName, ns, uid)
 			}, 2*time.Minute, time.Second*4).Should(BeNil())
 		})
-		It("should be recreated when mirror pod forcibly deleted", func() {
+		It("should be recreated when mirror pod forcibly deleted [Conformance]", func() {
 			By("get mirror pod uid")
 			pod, err := f.ClientSet.Core().Pods(ns).Get(mirrorPodName)
 			Expect(err).ShouldNot(HaveOccurred())

test/e2e_node/runtime_conformance_test.go

@@ -234,7 +234,7 @@ while true; do sleep 1; done
 				},
 			} {
 				testCase := testCase
-				It(testCase.description, func() {
+				It(testCase.description+" [Conformance]", func() {
 					name := "image-pull-test"
 					command := []string{"/bin/sh", "-c", "while true; do sleep 1; done"}
 					container := ConformanceContainer{