From 3c7e0bba1d4cc8247d248756dcfef884bc406992 Mon Sep 17 00:00:00 2001
From: Peter Pan <Peter.Pan@daocloud.io>
Date: Wed, 19 Apr 2023 23:43:40 -0400
Subject: [PATCH] chore: analyze Pod ReadinessProbe faliure

Signed-off-by: Peter Pan <Peter.Pan@daocloud.io>
---
 pkg/analyzer/pod.go      | 19 ++++++++++-
 pkg/analyzer/pod_test.go | 72 +++++++++++++++++++++++++++++++---------
 2 files changed, 75 insertions(+), 16 deletions(-)

diff --git a/pkg/analyzer/pod.go b/pkg/analyzer/pod.go
index 2d413549..c853e1e8 100644
--- a/pkg/analyzer/pod.go
+++ b/pkg/analyzer/pod.go
@@ -44,7 +44,7 @@ func (PodAnalyzer) Analyze(a common.Analyzer) ([]common.Result, error) {
 			}
 		}
 
-		// Check through container status to check for crashes
+		// Check through container status to check for crashes or unready
 		for _, containerStatus := range pod.Status.ContainerStatuses {
 			if containerStatus.State.Waiting != nil {
 				if containerStatus.State.Waiting.Reason == "CrashLoopBackOff" || containerStatus.State.Waiting.Reason == "ImagePullBackOff" {
@@ -70,6 +70,23 @@ func (PodAnalyzer) Analyze(a common.Analyzer) ([]common.Result, error) {
 						})
 					}
 				}
+			} else {
+				// when pod is Running but its ReadinessProbe fails
+				if containerStatus.Ready == false && pod.Status.Phase == "Running" {
+					// parse the event log and append details
+					evt, err := FetchLatestEvent(a.Context, a.Client, pod.Namespace, pod.Name)
+					if err != nil || evt == nil {
+						continue
+					}
+					if evt.Reason == "Unhealthy" && evt.Message != "" {
+						failures = append(failures, common.Failure{
+							Text:      evt.Message,
+							Sensitive: []common.Sensitive{},
+						})
+
+					}
+
+				}
 			}
 		}
 		if len(failures) > 0 {
diff --git a/pkg/analyzer/pod_test.go b/pkg/analyzer/pod_test.go
index 03e11b91..32470d81 100644
--- a/pkg/analyzer/pod_test.go
+++ b/pkg/analyzer/pod_test.go
@@ -14,23 +14,65 @@ import (
 
 func TestPodAnalyzer(t *testing.T) {
 
-	clientset := fake.NewSimpleClientset(&v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:        "example",
-			Namespace:   "default",
-			Annotations: map[string]string{},
-		},
-		Status: v1.PodStatus{
-			Phase: v1.PodPending,
-			Conditions: []v1.PodCondition{
-				{
-					Type:    v1.PodScheduled,
-					Reason:  "Unschedulable",
-					Message: "0/1 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.",
+	clientset := fake.NewSimpleClientset(
+		&v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        "example",
+				Namespace:   "default",
+				Annotations: map[string]string{},
+			},
+			Status: v1.PodStatus{
+				Phase: v1.PodPending,
+				Conditions: []v1.PodCondition{
+					{
+						Type:    v1.PodScheduled,
+						Reason:  "Unschedulable",
+						Message: "0/1 nodes are available: 1 node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate.",
+					},
 				},
 			},
 		},
-	})
+		&v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "example2",
+				Namespace: "default",
+			},
+			Status: v1.PodStatus{
+				Phase: v1.PodRunning,
+				ContainerStatuses: []v1.ContainerStatus{
+					{
+						Name:  "example2",
+						Ready: false,
+					},
+				},
+				Conditions: []v1.PodCondition{
+					{
+						Type:    v1.ContainersReady,
+						Reason:  "ContainersNotReady",
+						Message: "containers with unready status: [example2]",
+					},
+				},
+			},
+		},
+		// simulate event: 30s         Warning   Unhealthy              pod/my-nginx-7fb4dbcf47-4ch4w                         Readiness probe failed: bash: xxxx: command not found
+		&v1.Event{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "foo",
+				Namespace: "default",
+			},
+			InvolvedObject: v1.ObjectReference{
+				Kind:       "Pod",
+				Name:       "example2",
+				Namespace:  "default",
+				UID:        "differentUid",
+				APIVersion: "v1",
+			},
+			Reason:  "Unhealthy",
+			Message: "readiness probe failed: the detail reason here ...",
+			Source:  v1.EventSource{Component: "eventTest"},
+			Count:   1,
+			Type:    v1.EventTypeWarning,
+		})
 
 	config := common.Analyzer{
 		Client: &kubernetes.Client{
@@ -45,7 +87,7 @@ func TestPodAnalyzer(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	assert.Equal(t, len(analysisResults), 1)
+	assert.Equal(t, len(analysisResults), 2)
 }
 
 func TestPodAnalyzerNamespaceFiltering(t *testing.T) {