diff --git a/test/e2e/common/container_probe.go b/test/e2e/common/container_probe.go index e9907eaef67..6ae08c51f6b 100644 --- a/test/e2e/common/container_probe.go +++ b/test/e2e/common/container_probe.go @@ -90,6 +90,145 @@ var _ = framework.KubeDescribe("Probing container", func() { Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount) }) + It("should be restarted with a exec \"cat /tmp/health\" liveness probe [Conformance]", func() { + runLivenessTest(f, &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "liveness-exec", + Labels: map[string]string{"test": "liveness"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "liveness", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; rm -rf /tmp/health; sleep 600"}, + LivenessProbe: &api.Probe{ + Handler: api.Handler{ + Exec: &api.ExecAction{ + Command: []string{"cat", "/tmp/health"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + }, + }, + }, + }, + }, 1, defaultObservationTimeout) + }) + + It("should *not* be restarted with a exec \"cat /tmp/health\" liveness probe [Conformance]", func() { + runLivenessTest(f, &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "liveness-exec", + Labels: map[string]string{"test": "liveness"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "liveness", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 600"}, + LivenessProbe: &api.Probe{ + Handler: api.Handler{ + Exec: &api.ExecAction{ + Command: []string{"cat", "/tmp/health"}, + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + }, + }, + }, + }, + }, 0, defaultObservationTimeout) + }) + + It("should be restarted with a /healthz http liveness probe [Conformance]", func() { + runLivenessTest(f, &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "liveness-http", + Labels: map[string]string{"test": "liveness"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "liveness", + Image: "gcr.io/google_containers/liveness:e2e", + Command: []string{"/server"}, + LivenessProbe: &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + }, + }, + }, + }, + }, 1, defaultObservationTimeout) + }) + + // Slow by design (5 min) + It("should have monotonically increasing restart count [Conformance] [Slow]", func() { + runLivenessTest(f, &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "liveness-http", + Labels: map[string]string{"test": "liveness"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "liveness", + Image: "gcr.io/google_containers/liveness:e2e", + Command: []string{"/server"}, + LivenessProbe: &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8080), + }, + }, + InitialDelaySeconds: 5, + FailureThreshold: 1, + }, + }, + }, + }, + }, 5, time.Minute*5) + }) + + It("should *not* be restarted with a /healthz http liveness probe [Conformance]", func() { + runLivenessTest(f, &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "liveness-http", + Labels: map[string]string{"test": "liveness"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "liveness", + Image: "gcr.io/google_containers/nginx-slim:0.7", + Ports: []api.ContainerPort{{ContainerPort: 80}}, + LivenessProbe: &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Path: "/", + Port: intstr.FromInt(80), + }, + }, + InitialDelaySeconds: 15, + FailureThreshold: 1, + }, + }, + }, + }, + }, 0, defaultObservationTimeout) + }) + }) func getContainerStartedTime(p *api.Pod, containerName string) (time.Time, error) { @@ -171,3 +310,63 @@ func (b webserverProbeBuilder) build() *api.Probe { } return probe } + +func runLivenessTest(f *framework.Framework, pod *api.Pod, expectNumRestarts int, timeout time.Duration) { + podClient := f.PodClient() + ns := f.Namespace.Name + Expect(pod.Spec.Containers).NotTo(BeEmpty()) + containerName := pod.Spec.Containers[0].Name + // At the end of the test, clean up by removing the pod. + defer func() { + By("deleting the pod") + podClient.Delete(pod.Name, api.NewDeleteOptions(0)) + }() + By(fmt.Sprintf("Creating pod %s in namespace %s", pod.Name, ns)) + podClient.Create(pod) + + // Wait until the pod is not pending. (Here we need to check for something other than + // 'Pending' other than checking for 'Running', since when failures occur, we go to + // 'Terminated' which can cause indefinite blocking.) + framework.ExpectNoError(framework.WaitForPodNotPending(f.Client, ns, pod.Name, pod.ResourceVersion), + fmt.Sprintf("starting pod %s in namespace %s", pod.Name, ns)) + framework.Logf("Started pod %s in namespace %s", pod.Name, ns) + + // Check the pod's current state and verify that restartCount is present. + By("checking the pod's current state and verifying that restartCount is present") + pod, err := podClient.Get(pod.Name) + framework.ExpectNoError(err, fmt.Sprintf("getting pod %s in namespace %s", pod.Name, ns)) + initialRestartCount := api.GetExistingContainerStatus(pod.Status.ContainerStatuses, containerName).RestartCount + framework.Logf("Initial restart count of pod %s is %d", pod.Name, initialRestartCount) + + // Wait for the restart state to be as desired. + deadline := time.Now().Add(timeout) + lastRestartCount := initialRestartCount + observedRestarts := int32(0) + for start := time.Now(); time.Now().Before(deadline); time.Sleep(2 * time.Second) { + pod, err = podClient.Get(pod.Name) + framework.ExpectNoError(err, fmt.Sprintf("getting pod %s", pod.Name)) + restartCount := api.GetExistingContainerStatus(pod.Status.ContainerStatuses, containerName).RestartCount + if restartCount != lastRestartCount { + framework.Logf("Restart count of pod %s/%s is now %d (%v elapsed)", + ns, pod.Name, restartCount, time.Since(start)) + if restartCount < lastRestartCount { + framework.Failf("Restart count should increment monotonically: restart cont of pod %s/%s changed from %d to %d", + ns, pod.Name, lastRestartCount, restartCount) + } + } + observedRestarts = restartCount - initialRestartCount + if expectNumRestarts > 0 && int(observedRestarts) >= expectNumRestarts { + // Stop if we have observed more than expectNumRestarts restarts. + break + } + lastRestartCount = restartCount + } + + // If we expected 0 restarts, fail if observed any restart. + // If we expected n restarts (n > 0), fail if we observed < n restarts. + if (expectNumRestarts == 0 && observedRestarts > 0) || (expectNumRestarts > 0 && + int(observedRestarts) < expectNumRestarts) { + framework.Failf("pod %s/%s - expected number of restarts: %d, found restarts: %d", + ns, pod.Name, expectNumRestarts, observedRestarts) + } +} diff --git a/test/e2e/common/init_container.go b/test/e2e/common/init_container.go new file mode 100644 index 00000000000..6098a3b75ed --- /dev/null +++ b/test/e2e/common/init_container.go @@ -0,0 +1,379 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "fmt" + "strconv" + "time" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/util/uuid" + "k8s.io/kubernetes/pkg/watch" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = framework.KubeDescribe("InitContainer", func() { + f := framework.NewDefaultFramework("init-container") + var podClient *framework.PodClient + BeforeEach(func() { + podClient = f.PodClient() + }) + + It("should invoke init containers on a RestartNever pod", func() { + framework.SkipIfContainerRuntimeIs("rkt") // #25988 + + By("creating the pod") + name := "pod-init-" + string(uuid.NewUUID()) + value := strconv.Itoa(time.Now().Nanosecond()) + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": "foo", + "time": value, + }, + }, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyNever, + InitContainers: []api.Container{ + { + Name: "init1", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + { + Name: "init2", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + }, + Containers: []api.Container{ + { + Name: "run1", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + }, + }, + } + defer podClient.Delete(pod.Name, nil) + startedPod := podClient.Create(pod) + w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) + Expect(err).NotTo(HaveOccurred(), "error watching a pod") + wr := watch.NewRecorder(w) + event, err := watch.Until(framework.PodStartTimeout, wr, client.PodCompleted) + Expect(err).To(BeNil()) + framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) + endPod := event.Object.(*api.Pod) + + Expect(endPod.Status.Phase).To(Equal(api.PodSucceeded)) + _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) + Expect(init).NotTo(BeNil()) + Expect(init.Status).To(Equal(api.ConditionTrue)) + + Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) + for _, status := range endPod.Status.InitContainerStatuses { + Expect(status.Ready).To(BeTrue()) + Expect(status.State.Terminated).NotTo(BeNil()) + Expect(status.State.Terminated.ExitCode).To(BeZero()) + } + }) + + It("should invoke init containers on a RestartAlways pod", func() { + framework.SkipIfContainerRuntimeIs("rkt") // #25988 + + By("creating the pod") + name := "pod-init-" + string(uuid.NewUUID()) + value := strconv.Itoa(time.Now().Nanosecond()) + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": "foo", + "time": value, + }, + }, + Spec: api.PodSpec{ + InitContainers: []api.Container{ + { + Name: "init1", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + { + Name: "init2", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + }, + Containers: []api.Container{ + { + Name: "run1", + Image: framework.GetPauseImageName(f.Client), + Resources: api.ResourceRequirements{ + Limits: api.ResourceList{ + api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(10*1024*1024, resource.DecimalSI), + }, + }, + }, + }, + }, + } + defer podClient.Delete(pod.Name, nil) + startedPod := podClient.Create(pod) + w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) + Expect(err).NotTo(HaveOccurred(), "error watching a pod") + wr := watch.NewRecorder(w) + event, err := watch.Until(framework.PodStartTimeout, wr, client.PodRunning) + Expect(err).To(BeNil()) + framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) + endPod := event.Object.(*api.Pod) + + Expect(endPod.Status.Phase).To(Equal(api.PodRunning)) + _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) + Expect(init).NotTo(BeNil()) + Expect(init.Status).To(Equal(api.ConditionTrue)) + + Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) + for _, status := range endPod.Status.InitContainerStatuses { + Expect(status.Ready).To(BeTrue()) + Expect(status.State.Terminated).NotTo(BeNil()) + Expect(status.State.Terminated.ExitCode).To(BeZero()) + } + }) + + It("should not start app containers if init containers fail on a RestartAlways pod", func() { + framework.SkipIfContainerRuntimeIs("rkt") // #25988 + + By("creating the pod") + name := "pod-init-" + string(uuid.NewUUID()) + value := strconv.Itoa(time.Now().Nanosecond()) + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": "foo", + "time": value, + }, + }, + Spec: api.PodSpec{ + InitContainers: []api.Container{ + { + Name: "init1", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/false"}, + }, + { + Name: "init2", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + }, + Containers: []api.Container{ + { + Name: "run1", + Image: framework.GetPauseImageName(f.Client), + Resources: api.ResourceRequirements{ + Limits: api.ResourceList{ + api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(10*1024*1024, resource.DecimalSI), + }, + }, + }, + }, + }, + } + defer podClient.Delete(pod.Name, nil) + startedPod := podClient.Create(pod) + w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) + Expect(err).NotTo(HaveOccurred(), "error watching a pod") + + wr := watch.NewRecorder(w) + event, err := watch.Until( + framework.PodStartTimeout, wr, + // check for the first container to fail at least once + func(evt watch.Event) (bool, error) { + switch t := evt.Object.(type) { + case *api.Pod: + for _, status := range t.Status.ContainerStatuses { + if status.State.Waiting == nil { + return false, fmt.Errorf("container %q should not be out of waiting: %#v", status.Name, status) + } + if status.State.Waiting.Reason != "PodInitializing" { + return false, fmt.Errorf("container %q should have reason PodInitializing: %#v", status.Name, status) + } + } + if len(t.Status.InitContainerStatuses) != 2 { + return false, nil + } + status := t.Status.InitContainerStatuses[1] + if status.State.Waiting == nil { + return false, fmt.Errorf("second init container should not be out of waiting: %#v", status) + } + if status.State.Waiting.Reason != "PodInitializing" { + return false, fmt.Errorf("second init container should have reason PodInitializing: %#v", status) + } + status = t.Status.InitContainerStatuses[0] + if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 { + return false, fmt.Errorf("first init container should have exitCode != 0: %#v", status) + } + // continue until we see an attempt to restart the pod + return status.LastTerminationState.Terminated != nil, nil + default: + return false, fmt.Errorf("unexpected object: %#v", t) + } + }, + // verify we get two restarts + func(evt watch.Event) (bool, error) { + switch t := evt.Object.(type) { + case *api.Pod: + status := t.Status.InitContainerStatuses[0] + if status.RestartCount < 3 { + return false, nil + } + framework.Logf("init container has failed twice: %#v", t) + // TODO: more conditions + return true, nil + default: + return false, fmt.Errorf("unexpected object: %#v", t) + } + }, + ) + Expect(err).To(BeNil()) + framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) + endPod := event.Object.(*api.Pod) + + Expect(endPod.Status.Phase).To(Equal(api.PodPending)) + _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) + Expect(init).NotTo(BeNil()) + Expect(init.Status).To(Equal(api.ConditionFalse)) + Expect(init.Reason).To(Equal("ContainersNotInitialized")) + Expect(init.Message).To(Equal("containers with incomplete status: [init1 init2]")) + Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) + }) + + It("should not start app containers and fail the pod if init containers fail on a RestartNever pod", func() { + framework.SkipIfContainerRuntimeIs("rkt") // #25988 + + By("creating the pod") + name := "pod-init-" + string(uuid.NewUUID()) + value := strconv.Itoa(time.Now().Nanosecond()) + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": "foo", + "time": value, + }, + }, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyNever, + InitContainers: []api.Container{ + { + Name: "init1", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + }, + { + Name: "init2", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/false"}, + }, + }, + Containers: []api.Container{ + { + Name: "run1", + Image: "gcr.io/google_containers/busybox:1.24", + Command: []string{"/bin/true"}, + Resources: api.ResourceRequirements{ + Limits: api.ResourceList{ + api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), + api.ResourceMemory: *resource.NewQuantity(10*1024*1024, resource.DecimalSI), + }, + }, + }, + }, + }, + } + defer podClient.Delete(pod.Name, nil) + startedPod := podClient.Create(pod) + w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) + Expect(err).NotTo(HaveOccurred(), "error watching a pod") + + wr := watch.NewRecorder(w) + event, err := watch.Until( + framework.PodStartTimeout, wr, + // check for the second container to fail at least once + func(evt watch.Event) (bool, error) { + switch t := evt.Object.(type) { + case *api.Pod: + for _, status := range t.Status.ContainerStatuses { + if status.State.Waiting == nil { + return false, fmt.Errorf("container %q should not be out of waiting: %#v", status.Name, status) + } + if status.State.Waiting.Reason != "PodInitializing" { + return false, fmt.Errorf("container %q should have reason PodInitializing: %#v", status.Name, status) + } + } + if len(t.Status.InitContainerStatuses) != 2 { + return false, nil + } + status := t.Status.InitContainerStatuses[0] + if status.State.Terminated == nil { + if status.State.Waiting != nil && status.State.Waiting.Reason != "PodInitializing" { + return false, fmt.Errorf("second init container should have reason PodInitializing: %#v", status) + } + return false, nil + } + if status.State.Terminated != nil && status.State.Terminated.ExitCode != 0 { + return false, fmt.Errorf("first init container should have exitCode != 0: %#v", status) + } + status = t.Status.InitContainerStatuses[1] + if status.State.Terminated == nil { + return false, nil + } + if status.State.Terminated.ExitCode == 0 { + return false, fmt.Errorf("second init container should have failed: %#v", status) + } + return true, nil + default: + return false, fmt.Errorf("unexpected object: %#v", t) + } + }, + client.PodCompleted, + ) + Expect(err).To(BeNil()) + framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) + endPod := event.Object.(*api.Pod) + + Expect(endPod.Status.Phase).To(Equal(api.PodFailed)) + _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) + Expect(init).NotTo(BeNil()) + Expect(init.Status).To(Equal(api.ConditionFalse)) + Expect(init.Reason).To(Equal("ContainersNotInitialized")) + Expect(init.Message).To(Equal("containers with incomplete status: [init2]")) + Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) + Expect(endPod.Status.ContainerStatuses[0].State.Waiting).ToNot(BeNil()) + }) +}) diff --git a/test/e2e/common/pods.go b/test/e2e/common/pods.go index 2fb1f5fcd7b..26270271e5f 100644 --- a/test/e2e/common/pods.go +++ b/test/e2e/common/pods.go @@ -27,8 +27,6 @@ import ( "golang.org/x/net/websocket" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/resource" - client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util/intstr" @@ -51,66 +49,6 @@ var ( maxBackOffTolerance = time.Duration(1.3 * float64(kubelet.MaxContainerBackOff)) ) -func runLivenessTest(f *framework.Framework, pod *api.Pod, expectNumRestarts int, timeout time.Duration) { - podClient := f.PodClient() - ns := f.Namespace.Name - Expect(pod.Spec.Containers).NotTo(BeEmpty()) - containerName := pod.Spec.Containers[0].Name - // At the end of the test, clean up by removing the pod. - defer func() { - By("deleting the pod") - podClient.Delete(pod.Name, api.NewDeleteOptions(0)) - }() - By(fmt.Sprintf("Creating pod %s in namespace %s", pod.Name, ns)) - podClient.Create(pod) - - // Wait until the pod is not pending. (Here we need to check for something other than - // 'Pending' other than checking for 'Running', since when failures occur, we go to - // 'Terminated' which can cause indefinite blocking.) - framework.ExpectNoError(framework.WaitForPodNotPending(f.Client, ns, pod.Name, pod.ResourceVersion), - fmt.Sprintf("starting pod %s in namespace %s", pod.Name, ns)) - framework.Logf("Started pod %s in namespace %s", pod.Name, ns) - - // Check the pod's current state and verify that restartCount is present. - By("checking the pod's current state and verifying that restartCount is present") - pod, err := podClient.Get(pod.Name) - framework.ExpectNoError(err, fmt.Sprintf("getting pod %s in namespace %s", pod.Name, ns)) - initialRestartCount := api.GetExistingContainerStatus(pod.Status.ContainerStatuses, containerName).RestartCount - framework.Logf("Initial restart count of pod %s is %d", pod.Name, initialRestartCount) - - // Wait for the restart state to be as desired. - deadline := time.Now().Add(timeout) - lastRestartCount := initialRestartCount - observedRestarts := int32(0) - for start := time.Now(); time.Now().Before(deadline); time.Sleep(2 * time.Second) { - pod, err = podClient.Get(pod.Name) - framework.ExpectNoError(err, fmt.Sprintf("getting pod %s", pod.Name)) - restartCount := api.GetExistingContainerStatus(pod.Status.ContainerStatuses, containerName).RestartCount - if restartCount != lastRestartCount { - framework.Logf("Restart count of pod %s/%s is now %d (%v elapsed)", - ns, pod.Name, restartCount, time.Since(start)) - if restartCount < lastRestartCount { - framework.Failf("Restart count should increment monotonically: restart cont of pod %s/%s changed from %d to %d", - ns, pod.Name, lastRestartCount, restartCount) - } - } - observedRestarts = restartCount - initialRestartCount - if expectNumRestarts > 0 && int(observedRestarts) >= expectNumRestarts { - // Stop if we have observed more than expectNumRestarts restarts. - break - } - lastRestartCount = restartCount - } - - // If we expected 0 restarts, fail if observed any restart. - // If we expected n restarts (n > 0), fail if we observed < n restarts. - if (expectNumRestarts == 0 && observedRestarts > 0) || (expectNumRestarts > 0 && - int(observedRestarts) < expectNumRestarts) { - framework.Failf("pod %s/%s - expected number of restarts: %d, found restarts: %d", - ns, pod.Name, expectNumRestarts, observedRestarts) - } -} - // testHostIP tests that a pod gets a host IP func testHostIP(podClient *framework.PodClient, pod *api.Pod) { By("creating pod") @@ -510,487 +448,6 @@ var _ = framework.KubeDescribe("Pods", func() { }) }) - It("should invoke init containers on a RestartNever pod", func() { - framework.SkipIfContainerRuntimeIs("rkt") // #25988 - - podClient := f.PodClient() - By("creating the pod") - name := "pod-init-" + string(uuid.NewUUID()) - value := strconv.Itoa(time.Now().Nanosecond()) - pod := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "name": "foo", - "time": value, - }, - }, - Spec: api.PodSpec{ - RestartPolicy: api.RestartPolicyNever, - InitContainers: []api.Container{ - { - Name: "init1", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - { - Name: "init2", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - }, - Containers: []api.Container{ - { - Name: "run1", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - }, - }, - } - defer podClient.Delete(pod.Name, nil) - startedPod := podClient.Create(pod) - w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") - wr := watch.NewRecorder(w) - event, err := watch.Until(framework.PodStartTimeout, wr, client.PodCompleted) - Expect(err).To(BeNil()) - framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) - endPod := event.Object.(*api.Pod) - - Expect(endPod.Status.Phase).To(Equal(api.PodSucceeded)) - _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) - Expect(init).NotTo(BeNil()) - Expect(init.Status).To(Equal(api.ConditionTrue)) - - Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) - for _, status := range endPod.Status.InitContainerStatuses { - Expect(status.Ready).To(BeTrue()) - Expect(status.State.Terminated).NotTo(BeNil()) - Expect(status.State.Terminated.ExitCode).To(BeZero()) - } - }) - - It("should invoke init containers on a RestartAlways pod", func() { - framework.SkipIfContainerRuntimeIs("rkt") // #25988 - - podClient := f.PodClient() - By("creating the pod") - name := "pod-init-" + string(uuid.NewUUID()) - value := strconv.Itoa(time.Now().Nanosecond()) - pod := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "name": "foo", - "time": value, - }, - }, - Spec: api.PodSpec{ - InitContainers: []api.Container{ - { - Name: "init1", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - { - Name: "init2", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - }, - Containers: []api.Container{ - { - Name: "run1", - Image: framework.GetPauseImageName(f.Client), - Resources: api.ResourceRequirements{ - Limits: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(10*1024*1024, resource.DecimalSI), - }, - }, - }, - }, - }, - } - defer podClient.Delete(pod.Name, nil) - startedPod := podClient.Create(pod) - w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") - wr := watch.NewRecorder(w) - event, err := watch.Until(framework.PodStartTimeout, wr, client.PodRunning) - Expect(err).To(BeNil()) - framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) - endPod := event.Object.(*api.Pod) - - Expect(endPod.Status.Phase).To(Equal(api.PodRunning)) - _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) - Expect(init).NotTo(BeNil()) - Expect(init.Status).To(Equal(api.ConditionTrue)) - - Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) - for _, status := range endPod.Status.InitContainerStatuses { - Expect(status.Ready).To(BeTrue()) - Expect(status.State.Terminated).NotTo(BeNil()) - Expect(status.State.Terminated.ExitCode).To(BeZero()) - } - }) - - It("should not start app containers if init containers fail on a RestartAlways pod", func() { - framework.SkipIfContainerRuntimeIs("rkt") // #25988 - - podClient := f.PodClient() - By("creating the pod") - name := "pod-init-" + string(uuid.NewUUID()) - value := strconv.Itoa(time.Now().Nanosecond()) - pod := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "name": "foo", - "time": value, - }, - }, - Spec: api.PodSpec{ - InitContainers: []api.Container{ - { - Name: "init1", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/false"}, - }, - { - Name: "init2", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - }, - Containers: []api.Container{ - { - Name: "run1", - Image: framework.GetPauseImageName(f.Client), - Resources: api.ResourceRequirements{ - Limits: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(10*1024*1024, resource.DecimalSI), - }, - }, - }, - }, - }, - } - defer podClient.Delete(pod.Name, nil) - startedPod := podClient.Create(pod) - w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") - - wr := watch.NewRecorder(w) - event, err := watch.Until( - framework.PodStartTimeout, wr, - // check for the first container to fail at least once - func(evt watch.Event) (bool, error) { - switch t := evt.Object.(type) { - case *api.Pod: - for _, status := range t.Status.ContainerStatuses { - if status.State.Waiting == nil { - return false, fmt.Errorf("container %q should not be out of waiting: %#v", status.Name, status) - } - if status.State.Waiting.Reason != "PodInitializing" { - return false, fmt.Errorf("container %q should have reason PodInitializing: %#v", status.Name, status) - } - } - if len(t.Status.InitContainerStatuses) != 2 { - return false, nil - } - status := t.Status.InitContainerStatuses[1] - if status.State.Waiting == nil { - return false, fmt.Errorf("second init container should not be out of waiting: %#v", status) - } - if status.State.Waiting.Reason != "PodInitializing" { - return false, fmt.Errorf("second init container should have reason PodInitializing: %#v", status) - } - status = t.Status.InitContainerStatuses[0] - if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 { - return false, fmt.Errorf("first init container should have exitCode != 0: %#v", status) - } - // continue until we see an attempt to restart the pod - return status.LastTerminationState.Terminated != nil, nil - default: - return false, fmt.Errorf("unexpected object: %#v", t) - } - }, - // verify we get two restarts - func(evt watch.Event) (bool, error) { - switch t := evt.Object.(type) { - case *api.Pod: - status := t.Status.InitContainerStatuses[0] - if status.RestartCount < 3 { - return false, nil - } - framework.Logf("init container has failed twice: %#v", t) - // TODO: more conditions - return true, nil - default: - return false, fmt.Errorf("unexpected object: %#v", t) - } - }, - ) - Expect(err).To(BeNil()) - framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) - endPod := event.Object.(*api.Pod) - - Expect(endPod.Status.Phase).To(Equal(api.PodPending)) - _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) - Expect(init).NotTo(BeNil()) - Expect(init.Status).To(Equal(api.ConditionFalse)) - Expect(init.Reason).To(Equal("ContainersNotInitialized")) - Expect(init.Message).To(Equal("containers with incomplete status: [init1 init2]")) - Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) - }) - - It("should not start app containers and fail the pod if init containers fail on a RestartNever pod", func() { - framework.SkipIfContainerRuntimeIs("rkt") // #25988 - - podClient := f.PodClient() - By("creating the pod") - name := "pod-init-" + string(uuid.NewUUID()) - value := strconv.Itoa(time.Now().Nanosecond()) - pod := &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "name": "foo", - "time": value, - }, - }, - Spec: api.PodSpec{ - RestartPolicy: api.RestartPolicyNever, - InitContainers: []api.Container{ - { - Name: "init1", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - }, - { - Name: "init2", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/false"}, - }, - }, - Containers: []api.Container{ - { - Name: "run1", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/true"}, - Resources: api.ResourceRequirements{ - Limits: api.ResourceList{ - api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - api.ResourceMemory: *resource.NewQuantity(10*1024*1024, resource.DecimalSI), - }, - }, - }, - }, - }, - } - defer podClient.Delete(pod.Name, nil) - startedPod := podClient.Create(pod) - w, err := podClient.Watch(api.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") - - wr := watch.NewRecorder(w) - event, err := watch.Until( - framework.PodStartTimeout, wr, - // check for the second container to fail at least once - func(evt watch.Event) (bool, error) { - switch t := evt.Object.(type) { - case *api.Pod: - for _, status := range t.Status.ContainerStatuses { - if status.State.Waiting == nil { - return false, fmt.Errorf("container %q should not be out of waiting: %#v", status.Name, status) - } - if status.State.Waiting.Reason != "PodInitializing" { - return false, fmt.Errorf("container %q should have reason PodInitializing: %#v", status.Name, status) - } - } - if len(t.Status.InitContainerStatuses) != 2 { - return false, nil - } - status := t.Status.InitContainerStatuses[0] - if status.State.Terminated == nil { - if status.State.Waiting != nil && status.State.Waiting.Reason != "PodInitializing" { - return false, fmt.Errorf("second init container should have reason PodInitializing: %#v", status) - } - return false, nil - } - if status.State.Terminated != nil && status.State.Terminated.ExitCode != 0 { - return false, fmt.Errorf("first init container should have exitCode != 0: %#v", status) - } - status = t.Status.InitContainerStatuses[1] - if status.State.Terminated == nil { - return false, nil - } - if status.State.Terminated.ExitCode == 0 { - return false, fmt.Errorf("second init container should have failed: %#v", status) - } - return true, nil - default: - return false, fmt.Errorf("unexpected object: %#v", t) - } - }, - client.PodCompleted, - ) - Expect(err).To(BeNil()) - framework.CheckInvariants(wr.Events(), framework.ContainerInitInvariant) - endPod := event.Object.(*api.Pod) - - Expect(endPod.Status.Phase).To(Equal(api.PodFailed)) - _, init := api.GetPodCondition(&endPod.Status, api.PodInitialized) - Expect(init).NotTo(BeNil()) - Expect(init.Status).To(Equal(api.ConditionFalse)) - Expect(init.Reason).To(Equal("ContainersNotInitialized")) - Expect(init.Message).To(Equal("containers with incomplete status: [init2]")) - Expect(len(endPod.Status.InitContainerStatuses)).To(Equal(2)) - Expect(endPod.Status.ContainerStatuses[0].State.Waiting).ToNot(BeNil()) - }) - - It("should be restarted with a exec \"cat /tmp/health\" liveness probe [Conformance]", func() { - runLivenessTest(f, &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "liveness-exec", - Labels: map[string]string{"test": "liveness"}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "liveness", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 10; rm -rf /tmp/health; sleep 600"}, - LivenessProbe: &api.Probe{ - Handler: api.Handler{ - Exec: &api.ExecAction{ - Command: []string{"cat", "/tmp/health"}, - }, - }, - InitialDelaySeconds: 15, - FailureThreshold: 1, - }, - }, - }, - }, - }, 1, defaultObservationTimeout) - }) - - It("should *not* be restarted with a exec \"cat /tmp/health\" liveness probe [Conformance]", func() { - runLivenessTest(f, &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "liveness-exec", - Labels: map[string]string{"test": "liveness"}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "liveness", - Image: "gcr.io/google_containers/busybox:1.24", - Command: []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 600"}, - LivenessProbe: &api.Probe{ - Handler: api.Handler{ - Exec: &api.ExecAction{ - Command: []string{"cat", "/tmp/health"}, - }, - }, - InitialDelaySeconds: 15, - FailureThreshold: 1, - }, - }, - }, - }, - }, 0, defaultObservationTimeout) - }) - - It("should be restarted with a /healthz http liveness probe [Conformance]", func() { - runLivenessTest(f, &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "liveness-http", - Labels: map[string]string{"test": "liveness"}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "liveness", - Image: "gcr.io/google_containers/liveness:e2e", - Command: []string{"/server"}, - LivenessProbe: &api.Probe{ - Handler: api.Handler{ - HTTPGet: &api.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(8080), - }, - }, - InitialDelaySeconds: 15, - FailureThreshold: 1, - }, - }, - }, - }, - }, 1, defaultObservationTimeout) - }) - - // Slow by design (5 min) - It("should have monotonically increasing restart count [Conformance] [Slow]", func() { - runLivenessTest(f, &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "liveness-http", - Labels: map[string]string{"test": "liveness"}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "liveness", - Image: "gcr.io/google_containers/liveness:e2e", - Command: []string{"/server"}, - LivenessProbe: &api.Probe{ - Handler: api.Handler{ - HTTPGet: &api.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(8080), - }, - }, - InitialDelaySeconds: 5, - FailureThreshold: 1, - }, - }, - }, - }, - }, 5, time.Minute*5) - }) - - It("should *not* be restarted with a /healthz http liveness probe [Conformance]", func() { - runLivenessTest(f, &api.Pod{ - ObjectMeta: api.ObjectMeta{ - Name: "liveness-http", - Labels: map[string]string{"test": "liveness"}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: "liveness", - Image: "gcr.io/google_containers/nginx-slim:0.7", - Ports: []api.ContainerPort{{ContainerPort: 80}}, - LivenessProbe: &api.Probe{ - Handler: api.Handler{ - HTTPGet: &api.HTTPGetAction{ - Path: "/", - Port: intstr.FromInt(80), - }, - }, - InitialDelaySeconds: 15, - FailureThreshold: 1, - }, - }, - }, - }, - }, 0, defaultObservationTimeout) - }) - It("should support remote command execution over websockets", func() { config, err := framework.LoadConfig() Expect(err).NotTo(HaveOccurred(), "unable to get base config")