From 6ceb221f8e62bf1752f2e173fcce4bb3b1cb7bd0 Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Fri, 19 Feb 2016 15:30:40 -0800 Subject: [PATCH 1/2] Log pods of RS when deployment e2e test fails --- pkg/util/deployment/deployment.go | 43 ++++++++++++++++++------------- test/e2e/util.go | 14 ++++++++++ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/pkg/util/deployment/deployment.go b/pkg/util/deployment/deployment.go index c38299eef89..2a310e6e6d1 100644 --- a/pkg/util/deployment/deployment.go +++ b/pkg/util/deployment/deployment.go @@ -179,7 +179,7 @@ func GetReplicaCountForReplicaSets(replicaSets []*extensions.ReplicaSet) int { // Returns the number of available pods corresponding to the given replica sets. func GetAvailablePodsForReplicaSets(c clientset.Interface, rss []*extensions.ReplicaSet, minReadySeconds int) (int, error) { - allPods, err := getPodsForReplicaSets(c, rss) + allPods, err := GetPodsForReplicaSets(c, rss) if err != nil { return 0, err } @@ -189,28 +189,35 @@ func GetAvailablePodsForReplicaSets(c clientset.Interface, rss []*extensions.Rep func getReadyPodsCount(pods []api.Pod, minReadySeconds int) int { readyPodCount := 0 for _, pod := range pods { - if api.IsPodReady(&pod) { - // Check if we've passed minReadySeconds since LastTransitionTime - // If so, this pod is ready - for _, c := range pod.Status.Conditions { - // we only care about pod ready conditions - if c.Type == api.PodReady { - // 2 cases that this ready condition is valid (passed minReadySeconds, i.e. the pod is ready): - // 1. minReadySeconds <= 0 - // 2. LastTransitionTime (is set) + minReadySeconds (>0) < current time - minReadySecondsDuration := time.Duration(minReadySeconds) * time.Second - if minReadySeconds <= 0 || !c.LastTransitionTime.IsZero() && c.LastTransitionTime.Add(minReadySecondsDuration).Before(time.Now()) { - readyPodCount++ - break - } - } - } + if IsPodAvailable(&pod, minReadySeconds) { + readyPodCount++ } } return readyPodCount } -func getPodsForReplicaSets(c clientset.Interface, replicaSets []*extensions.ReplicaSet) ([]api.Pod, error) { +func IsPodAvailable(pod *api.Pod, minReadySeconds int) bool { + if !api.IsPodReady(pod) { + return false + } + // Check if we've passed minReadySeconds since LastTransitionTime + // If so, this pod is ready + for _, c := range pod.Status.Conditions { + // we only care about pod ready conditions + if c.Type == api.PodReady { + // 2 cases that this ready condition is valid (passed minReadySeconds, i.e. the pod is ready): + // 1. minReadySeconds <= 0 + // 2. LastTransitionTime (is set) + minReadySeconds (>0) < current time + minReadySecondsDuration := time.Duration(minReadySeconds) * time.Second + if minReadySeconds <= 0 || !c.LastTransitionTime.IsZero() && c.LastTransitionTime.Add(minReadySecondsDuration).Before(time.Now()) { + return true + } + } + } + return false +} + +func GetPodsForReplicaSets(c clientset.Interface, replicaSets []*extensions.ReplicaSet) ([]api.Pod, error) { allPods := []api.Pod{} for _, rs := range replicaSets { selector, err := unversioned.LabelSelectorAsSelector(rs.Spec.Selector) diff --git a/test/e2e/util.go b/test/e2e/util.go index 76eca0be8a1..b7c4f0242ee 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -2128,6 +2128,7 @@ func waitForDeploymentStatus(c clientset.Interface, ns, deploymentName string, d } if totalAvailable < minAvailable { logReplicaSetsOfDeployment(deploymentName, oldRSs, newRS) + logPodsOfReplicaSets(c, allRSs, minReadySeconds) return false, fmt.Errorf("total pods available: %d, less than the min required: %d", totalAvailable, minAvailable) } @@ -2170,6 +2171,19 @@ func logReplicaSetsOfDeployment(deploymentName string, oldRSs []*extensions.Repl Logf("New ReplicaSet of deployment %s: %+v", deploymentName, newRS) } +func logPodsOfReplicaSets(c clientset.Interface, rss []*extensions.ReplicaSet, minReadySeconds int) { + allPods, err := deploymentutil.GetPodsForReplicaSets(c, rss) + if err == nil { + for _, pod := range allPods { + availability := "not available" + if deploymentutil.IsPodAvailable(&pod, minReadySeconds) { + availability = "available" + } + Logf("Pod %s is %s: %+v", pod.Name, availability, pod) + } + } +} + // Waits for the number of events on the given object to reach a desired count. func waitForEvents(c *client.Client, ns string, objOrRef runtime.Object, desiredEventsCount int) error { return wait.Poll(poll, 5*time.Minute, func() (bool, error) { From b267ede42cad2d514caeee7fd49d6ce35c143f7d Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Fri, 19 Feb 2016 15:53:34 -0800 Subject: [PATCH 2/2] Address comments --- pkg/util/deployment/deployment.go | 5 +---- test/e2e/util.go | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/util/deployment/deployment.go b/pkg/util/deployment/deployment.go index 2a310e6e6d1..5f5baa782e3 100644 --- a/pkg/util/deployment/deployment.go +++ b/pkg/util/deployment/deployment.go @@ -197,14 +197,11 @@ func getReadyPodsCount(pods []api.Pod, minReadySeconds int) int { } func IsPodAvailable(pod *api.Pod, minReadySeconds int) bool { - if !api.IsPodReady(pod) { - return false - } // Check if we've passed minReadySeconds since LastTransitionTime // If so, this pod is ready for _, c := range pod.Status.Conditions { // we only care about pod ready conditions - if c.Type == api.PodReady { + if c.Type == api.PodReady && c.Status == api.ConditionTrue { // 2 cases that this ready condition is valid (passed minReadySeconds, i.e. the pod is ready): // 1. minReadySeconds <= 0 // 2. LastTransitionTime (is set) + minReadySeconds (>0) < current time diff --git a/test/e2e/util.go b/test/e2e/util.go index b7c4f0242ee..0913b8d6a03 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -2124,6 +2124,7 @@ func waitForDeploymentStatus(c clientset.Interface, ns, deploymentName string, d } if totalCreated > maxCreated { logReplicaSetsOfDeployment(deploymentName, oldRSs, newRS) + logPodsOfReplicaSets(c, allRSs, minReadySeconds) return false, fmt.Errorf("total pods created: %d, more than the max allowed: %d", totalCreated, maxCreated) } if totalAvailable < minAvailable { @@ -2137,10 +2138,12 @@ func waitForDeploymentStatus(c clientset.Interface, ns, deploymentName string, d // Verify replica sets. if deploymentutil.GetReplicaCountForReplicaSets(oldRSs) != 0 { logReplicaSetsOfDeployment(deploymentName, oldRSs, newRS) + logPodsOfReplicaSets(c, allRSs, minReadySeconds) return false, fmt.Errorf("old replica sets are not fully scaled down") } if deploymentutil.GetReplicaCountForReplicaSets([]*extensions.ReplicaSet{newRS}) != desiredUpdatedReplicas { logReplicaSetsOfDeployment(deploymentName, oldRSs, newRS) + logPodsOfReplicaSets(c, allRSs, minReadySeconds) return false, fmt.Errorf("new replica sets is not fully scaled up") } return true, nil