From d46cdbed6c913914b3b5656005c2db025e7e60ad Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Fri, 1 Jun 2018 18:04:09 -0700 Subject: [PATCH] Generate pod ready status with readiness gates --- pkg/api/v1/pod/util.go | 15 ++- pkg/kubelet/kubelet_pods.go | 8 +- pkg/kubelet/status/generate.go | 32 ++++++- pkg/kubelet/status/generate_test.go | 143 ++++++++++++++++++++++++---- 4 files changed, 170 insertions(+), 28 deletions(-) diff --git a/pkg/api/v1/pod/util.go b/pkg/api/v1/pod/util.go index 74f5f883221..e84295092e4 100644 --- a/pkg/api/v1/pod/util.go +++ b/pkg/api/v1/pod/util.go @@ -261,9 +261,18 @@ func GetPodCondition(status *v1.PodStatus, conditionType v1.PodConditionType) (i if status == nil { return -1, nil } - for i := range status.Conditions { - if status.Conditions[i].Type == conditionType { - return i, &status.Conditions[i] + return GetPodConditionFromList(status.Conditions, conditionType) +} + +// GetPodConditionFromList extracts the provided condition from the given list of condition and +// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. +func GetPodConditionFromList(conditions []v1.PodCondition, conditionType v1.PodConditionType) (int, *v1.PodCondition) { + if conditions == nil { + return -1, nil + } + for i := range conditions { + if conditions[i].Type == conditionType { + return i, &conditions[i] } } return -1, nil diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index d62c57e8302..382e820e901 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -1354,7 +1354,7 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po } kl.probeManager.UpdatePodStatus(pod.UID, s) s.Conditions = append(s.Conditions, status.GeneratePodInitializedCondition(spec, s.InitContainerStatuses, s.Phase)) - s.Conditions = append(s.Conditions, status.GeneratePodReadyCondition(spec, s.ContainerStatuses, s.Phase)) + s.Conditions = append(s.Conditions, status.GeneratePodReadyCondition(spec, s.Conditions, s.ContainerStatuses, s.Phase)) // Status manager will take care of the LastTransitionTimestamp, either preserve // the timestamp from apiserver, or set a new one. When kubelet sees the pod, // `PodScheduled` condition must be true. @@ -1407,6 +1407,12 @@ func (kl *Kubelet) convertStatusToAPIStatus(pod *v1.Pod, podStatus *kubecontaine true, ) + // Preserves conditions not controlled by kubelet + for _, c := range pod.Status.Conditions { + if !kubetypes.PodConditionByKubelet(c.Type) { + apiPodStatus.Conditions = append(apiPodStatus.Conditions, c) + } + } return &apiPodStatus } diff --git a/pkg/kubelet/status/generate.go b/pkg/kubelet/status/generate.go index c094a95d8a5..30ec57b92a4 100644 --- a/pkg/kubelet/status/generate.go +++ b/pkg/kubelet/status/generate.go @@ -29,11 +29,13 @@ const ( PodCompleted = "PodCompleted" ContainersNotReady = "ContainersNotReady" ContainersNotInitialized = "ContainersNotInitialized" + ReadinessGatesNotReady = "ReadinessGatesNotReady" ) -// GeneratePodReadyCondition returns ready condition if all containers in a pod are ready, else it -// returns an unready condition. -func GeneratePodReadyCondition(spec *v1.PodSpec, containerStatuses []v1.ContainerStatus, podPhase v1.PodPhase) v1.PodCondition { +// GeneratePodReadyCondition returns "Ready" condition of a pod. +// The status of "Ready" condition is "True", if all containers in a pod are ready +// AND all matching conditions specified in the ReadinessGates have status equal to "True". +func GeneratePodReadyCondition(spec *v1.PodSpec, conditions []v1.PodCondition, containerStatuses []v1.ContainerStatus, podPhase v1.PodPhase) v1.PodCondition { // Find if all containers are ready or not. if containerStatuses == nil { return v1.PodCondition{ @@ -63,6 +65,7 @@ func GeneratePodReadyCondition(spec *v1.PodSpec, containerStatuses []v1.Containe } } + // Generate message for containers in unknown condition. unreadyMessages := []string{} if len(unknownContainers) > 0 { unreadyMessages = append(unreadyMessages, fmt.Sprintf("containers with unknown status: %s", unknownContainers)) @@ -80,6 +83,29 @@ func GeneratePodReadyCondition(spec *v1.PodSpec, containerStatuses []v1.Containe } } + // Evaluate corresponding conditions specified in readiness gate + // Generate message if any readiness gate is not satisfied. + unreadyMessages = []string{} + for _, rg := range spec.ReadinessGates { + _, c := podutil.GetPodConditionFromList(conditions, rg.ConditionType) + if c == nil { + unreadyMessages = append(unreadyMessages, fmt.Sprintf("corresponding condition of pod readiness gate %q does not exist.", string(rg.ConditionType))) + } else if c.Status != v1.ConditionTrue { + unreadyMessages = append(unreadyMessages, fmt.Sprintf("the status of pod readiness gate %q is not \"True\", but %v", string(rg.ConditionType), c.Status)) + } + } + + // Set "Ready" condition to "False" if any readiness gate is not ready. + if len(unreadyMessages) != 0 { + unreadyMessage := strings.Join(unreadyMessages, ", ") + return v1.PodCondition{ + Type: v1.PodReady, + Status: v1.ConditionFalse, + Reason: ReadinessGatesNotReady, + Message: unreadyMessage, + } + } + return v1.PodCondition{ Type: v1.PodReady, Status: v1.ConditionTrue, diff --git a/pkg/kubelet/status/generate_test.go b/pkg/kubelet/status/generate_test.go index 9b0b61cdb40..46ea7df97fd 100644 --- a/pkg/kubelet/status/generate_test.go +++ b/pkg/kubelet/status/generate_test.go @@ -27,21 +27,24 @@ import ( func TestGeneratePodReadyCondition(t *testing.T) { tests := []struct { spec *v1.PodSpec + conditions []v1.PodCondition containerStatuses []v1.ContainerStatus podPhase v1.PodPhase - expected v1.PodCondition + expectReady v1.PodCondition }{ { spec: nil, + conditions: nil, containerStatuses: nil, podPhase: v1.PodRunning, - expected: getReadyCondition(false, UnknownContainerStatuses, ""), + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, UnknownContainerStatuses, ""), }, { spec: &v1.PodSpec{}, + conditions: nil, containerStatuses: []v1.ContainerStatus{}, podPhase: v1.PodRunning, - expected: getReadyCondition(true, "", ""), + expectReady: getPodCondition(v1.PodReady, v1.ConditionTrue, "", ""), }, { spec: &v1.PodSpec{ @@ -49,9 +52,10 @@ func TestGeneratePodReadyCondition(t *testing.T) { {Name: "1234"}, }, }, + conditions: nil, containerStatuses: []v1.ContainerStatus{}, podPhase: v1.PodRunning, - expected: getReadyCondition(false, ContainersNotReady, "containers with unknown status: [1234]"), + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ContainersNotReady, "containers with unknown status: [1234]"), }, { spec: &v1.PodSpec{ @@ -60,12 +64,13 @@ func TestGeneratePodReadyCondition(t *testing.T) { {Name: "5678"}, }, }, + conditions: nil, containerStatuses: []v1.ContainerStatus{ getReadyStatus("1234"), getReadyStatus("5678"), }, - podPhase: v1.PodRunning, - expected: getReadyCondition(true, "", ""), + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionTrue, "", ""), }, { spec: &v1.PodSpec{ @@ -74,11 +79,12 @@ func TestGeneratePodReadyCondition(t *testing.T) { {Name: "5678"}, }, }, + conditions: nil, containerStatuses: []v1.ContainerStatus{ getReadyStatus("1234"), }, - podPhase: v1.PodRunning, - expected: getReadyCondition(false, ContainersNotReady, "containers with unknown status: [5678]"), + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ContainersNotReady, "containers with unknown status: [5678]"), }, { spec: &v1.PodSpec{ @@ -87,12 +93,13 @@ func TestGeneratePodReadyCondition(t *testing.T) { {Name: "5678"}, }, }, + conditions: nil, containerStatuses: []v1.ContainerStatus{ getReadyStatus("1234"), getNotReadyStatus("5678"), }, - podPhase: v1.PodRunning, - expected: getReadyCondition(false, ContainersNotReady, "containers with unready status: [5678]"), + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ContainersNotReady, "containers with unready status: [5678]"), }, { spec: &v1.PodSpec{ @@ -100,18 +107,116 @@ func TestGeneratePodReadyCondition(t *testing.T) { {Name: "1234"}, }, }, + conditions: nil, containerStatuses: []v1.ContainerStatus{ getNotReadyStatus("1234"), }, - podPhase: v1.PodSucceeded, - expected: getReadyCondition(false, PodCompleted, ""), + podPhase: v1.PodSucceeded, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, PodCompleted, ""), + }, + { + spec: &v1.PodSpec{ + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + }, + }, + conditions: nil, + containerStatuses: []v1.ContainerStatus{}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ReadinessGatesNotReady, `corresponding condition of pod readiness gate "gate1" does not exist.`), + }, + { + spec: &v1.PodSpec{ + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + }, + }, + conditions: []v1.PodCondition{ + getPodCondition("gate1", v1.ConditionFalse, "", ""), + }, + containerStatuses: []v1.ContainerStatus{}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ReadinessGatesNotReady, `the status of pod readiness gate "gate1" is not "True", but False`), + }, + { + spec: &v1.PodSpec{ + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + }, + }, + conditions: []v1.PodCondition{ + getPodCondition("gate1", v1.ConditionTrue, "", ""), + }, + containerStatuses: []v1.ContainerStatus{}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionTrue, "", ""), + }, + { + spec: &v1.PodSpec{ + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + {ConditionType: v1.PodConditionType("gate2")}, + }, + }, + conditions: []v1.PodCondition{ + getPodCondition("gate1", v1.ConditionTrue, "", ""), + }, + containerStatuses: []v1.ContainerStatus{}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ReadinessGatesNotReady, `corresponding condition of pod readiness gate "gate2" does not exist.`), + }, + { + spec: &v1.PodSpec{ + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + {ConditionType: v1.PodConditionType("gate2")}, + }, + }, + conditions: []v1.PodCondition{ + getPodCondition("gate1", v1.ConditionTrue, "", ""), + getPodCondition("gate2", v1.ConditionFalse, "", ""), + }, + containerStatuses: []v1.ContainerStatus{}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ReadinessGatesNotReady, `the status of pod readiness gate "gate2" is not "True", but False`), + }, + { + spec: &v1.PodSpec{ + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + {ConditionType: v1.PodConditionType("gate2")}, + }, + }, + conditions: []v1.PodCondition{ + getPodCondition("gate1", v1.ConditionTrue, "", ""), + getPodCondition("gate2", v1.ConditionTrue, "", ""), + }, + containerStatuses: []v1.ContainerStatus{}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionTrue, "", ""), + }, + { + spec: &v1.PodSpec{ + Containers: []v1.Container{ + {Name: "1234"}, + }, + ReadinessGates: []v1.PodReadinessGate{ + {ConditionType: v1.PodConditionType("gate1")}, + }, + }, + conditions: []v1.PodCondition{ + getPodCondition("gate1", v1.ConditionTrue, "", ""), + }, + containerStatuses: []v1.ContainerStatus{getNotReadyStatus("1234")}, + podPhase: v1.PodRunning, + expectReady: getPodCondition(v1.PodReady, v1.ConditionFalse, ContainersNotReady, "containers with unready status: [1234]"), }, } for i, test := range tests { - condition := GeneratePodReadyCondition(test.spec, test.containerStatuses, test.podPhase) - if !reflect.DeepEqual(condition, test.expected) { - t.Errorf("On test case %v, expected:\n%+v\ngot\n%+v\n", i, test.expected, condition) + ready := GeneratePodReadyCondition(test.spec, test.conditions, test.containerStatuses, test.podPhase) + if !reflect.DeepEqual(ready, test.expectReady) { + t.Errorf("On test case %v, expectReady:\n%+v\ngot\n%+v\n", i, test.expectReady, ready) } } } @@ -220,13 +325,9 @@ func TestGeneratePodInitializedCondition(t *testing.T) { } } -func getReadyCondition(ready bool, reason, message string) v1.PodCondition { - status := v1.ConditionFalse - if ready { - status = v1.ConditionTrue - } +func getPodCondition(conditionType v1.PodConditionType, status v1.ConditionStatus, reason, message string) v1.PodCondition { return v1.PodCondition{ - Type: v1.PodReady, + Type: conditionType, Status: status, Reason: reason, Message: message,