From ec4182d0032d5993f5724b0393d795f0c52c3e23 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Thu, 11 Mar 2021 18:27:32 +0100 Subject: [PATCH 1/2] Add last restart time to 'RESTARTS' column to 'kubectl get pods' output This commit adds the last time since a container has restarted in a pod to the 'RESTARTS' column to the 'kubectl get pods' output Signed-off-by: Jordan Jacobelli --- pkg/printers/internalversion/printers.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/printers/internalversion/printers.go b/pkg/printers/internalversion/printers.go index 081a2494bca..97626c2c119 100644 --- a/pkg/printers/internalversion/printers.go +++ b/pkg/printers/internalversion/printers.go @@ -90,7 +90,7 @@ func AddHandlers(h printers.PrintHandler) { {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "Ready", Type: "string", Description: "The aggregate readiness state of this pod for accepting traffic."}, {Name: "Status", Type: "string", Description: "The aggregate status of the containers in this pod."}, - {Name: "Restarts", Type: "integer", Description: "The number of times the containers in this pod have been restarted."}, + {Name: "Restarts", Type: "string", Description: "The number of times the containers in this pod have been restarted and when the last container in this pod has restarted."}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, {Name: "IP", Type: "string", Priority: 1, Description: apiv1.PodStatus{}.SwaggerDoc()["podIP"]}, {Name: "Node", Type: "string", Priority: 1, Description: apiv1.PodSpec{}.SwaggerDoc()["nodeName"]}, @@ -741,6 +741,7 @@ func printPod(pod *api.Pod, options printers.GenerateOptions) ([]metav1.TableRow restarts := 0 totalContainers := len(pod.Spec.Containers) readyContainers := 0 + lastRestartDate := metav1.NewTime(time.Time{}) reason := string(pod.Status.Phase) if pod.Status.Reason != "" { @@ -762,6 +763,12 @@ func printPod(pod *api.Pod, options printers.GenerateOptions) ([]metav1.TableRow for i := range pod.Status.InitContainerStatuses { container := pod.Status.InitContainerStatuses[i] restarts += int(container.RestartCount) + if container.LastTerminationState.Terminated != nil { + terminatedDate := container.LastTerminationState.Terminated.FinishedAt + if lastRestartDate.Before(&terminatedDate) { + lastRestartDate = terminatedDate + } + } switch { case container.State.Terminated != nil && container.State.Terminated.ExitCode == 0: continue @@ -793,6 +800,12 @@ func printPod(pod *api.Pod, options printers.GenerateOptions) ([]metav1.TableRow container := pod.Status.ContainerStatuses[i] restarts += int(container.RestartCount) + if container.LastTerminationState.Terminated != nil { + terminatedDate := container.LastTerminationState.Terminated.FinishedAt + if lastRestartDate.Before(&terminatedDate) { + lastRestartDate = terminatedDate + } + } if container.State.Waiting != nil && container.State.Waiting.Reason != "" { reason = container.State.Waiting.Reason } else if container.State.Terminated != nil && container.State.Terminated.Reason != "" { @@ -825,7 +838,12 @@ func printPod(pod *api.Pod, options printers.GenerateOptions) ([]metav1.TableRow reason = "Terminating" } - row.Cells = append(row.Cells, pod.Name, fmt.Sprintf("%d/%d", readyContainers, totalContainers), reason, int64(restarts), translateTimestampSince(pod.CreationTimestamp)) + restartsStr := strconv.Itoa(restarts) + if !lastRestartDate.IsZero() { + restartsStr = fmt.Sprintf("%d (%s ago)", restarts, translateTimestampSince(lastRestartDate)) + } + + row.Cells = append(row.Cells, pod.Name, fmt.Sprintf("%d/%d", readyContainers, totalContainers), reason, restartsStr, translateTimestampSince(pod.CreationTimestamp)) if options.Wide { nodeName := pod.Spec.NodeName nominatedNodeName := pod.Status.NominatedNodeName From 9eea445bcce11b17f3d009f0528f0003fd9f5a37 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Thu, 11 Mar 2021 18:27:48 +0100 Subject: [PATCH 2/2] Update test cases for 'RESTARTS' column in 'kubectl get pods' Signed-off-by: Jordan Jacobelli --- pkg/printers/internalversion/printers_test.go | 248 ++++++++++++++++-- pkg/registry/core/pod/storage/storage_test.go | 8 +- 2 files changed, 233 insertions(+), 23 deletions(-) diff --git a/pkg/printers/internalversion/printers_test.go b/pkg/printers/internalversion/printers_test.go index cf76ac02715..1ac2f635d9f 100644 --- a/pkg/printers/internalversion/printers_test.go +++ b/pkg/printers/internalversion/printers_test.go @@ -1190,7 +1190,7 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", "6", ""}}}, }, { // Test container error overwrites pod phase @@ -1205,7 +1205,7 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test2", "1/2", "ContainerWaitingReason", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test2", "1/2", "ContainerWaitingReason", "6", ""}}}, }, { // Test the same as the above but with Terminated state and the first container overwrites the rest @@ -1220,7 +1220,7 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test3", "0/2", "ContainerWaitingReason", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test3", "0/2", "ContainerWaitingReason", "6", ""}}}, }, { // Test ready is not enough for reporting running @@ -1235,7 +1235,7 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test4", "1/2", "podPhase", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test4", "1/2", "podPhase", "6", ""}}}, }, { // Test ready is not enough for reporting running @@ -1251,7 +1251,7 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test5", "1/2", "podReason", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test5", "1/2", "podReason", "6", ""}}}, }, { // Test pod has 2 containers, one is running and the other is completed, w/o ready condition @@ -1267,7 +1267,7 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test6", "1/2", "NotReady", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test6", "1/2", "NotReady", "6", ""}}}, }, { // Test pod has 2 containers, one is running and the other is completed, with ready condition @@ -1286,7 +1286,217 @@ func TestPrintPod(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test6", "1/2", "Running", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test6", "1/2", "Running", "6", ""}}}, + }, + { + // Test pod has 1 init container restarting and 1 container not running + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test7"}, + Spec: api.PodSpec{InitContainers: make([]api.Container, 1), Containers: make([]api.Container, 1)}, + Status: api.PodStatus{ + Phase: "podPhase", + InitContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-10 * time.Second))}}, + }, + }, + ContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + RestartCount: 0, + State: api.ContainerState{Waiting: &api.ContainerStateWaiting{}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test7", "0/1", "Init:0/1", "3 (10s ago)", ""}}}, + }, + { + // Test pod has 2 init containers, one restarting and the other not running, and 1 container not running + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test8"}, + Spec: api.PodSpec{InitContainers: make([]api.Container, 2), Containers: make([]api.Container, 1)}, + Status: api.PodStatus{ + Phase: "podPhase", + InitContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-10 * time.Second))}}, + }, + { + Ready: false, + State: api.ContainerState{Waiting: &api.ContainerStateWaiting{}}, + }, + }, + ContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + State: api.ContainerState{Waiting: &api.ContainerStateWaiting{}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test8", "0/1", "Init:0/2", "3 (10s ago)", ""}}}, + }, + { + // Test pod has 2 init containers, one completed without restarts and the other restarting, and 1 container not running + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test9"}, + Spec: api.PodSpec{InitContainers: make([]api.Container, 2), Containers: make([]api.Container, 1)}, + Status: api.PodStatus{ + Phase: "podPhase", + InitContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + State: api.ContainerState{Terminated: &api.ContainerStateTerminated{}}, + }, + { + Ready: false, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-10 * time.Second))}}, + }, + }, + ContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + State: api.ContainerState{Waiting: &api.ContainerStateWaiting{}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test9", "0/1", "Init:1/2", "3 (10s ago)", ""}}}, + }, + { + // Test pod has 2 init containers, one completed with restarts and the other restarting, and 1 container not running + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test10"}, + Spec: api.PodSpec{InitContainers: make([]api.Container, 2), Containers: make([]api.Container, 1)}, + Status: api.PodStatus{ + Phase: "podPhase", + InitContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + RestartCount: 2, + State: api.ContainerState{Terminated: &api.ContainerStateTerminated{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-2 * time.Minute))}}, + }, + { + Ready: false, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-10 * time.Second))}}, + }, + }, + ContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + State: api.ContainerState{Waiting: &api.ContainerStateWaiting{}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test10", "0/1", "Init:1/2", "5 (10s ago)", ""}}}, + }, + { + // Test pod has 1 init container completed with restarts and one container restarting + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test11"}, + Spec: api.PodSpec{InitContainers: make([]api.Container, 1), Containers: make([]api.Container, 1)}, + Status: api.PodStatus{ + Phase: "Running", + InitContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + RestartCount: 2, + State: api.ContainerState{Terminated: &api.ContainerStateTerminated{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-2 * time.Minute))}}, + }, + }, + ContainerStatuses: []api.ContainerStatus{ + { + Ready: false, + RestartCount: 4, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-20 * time.Second))}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test11", "0/1", "Running", "4 (20s ago)", ""}}}, + }, + { + // Test pod has 1 container that restarted 5d ago + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test12"}, + Spec: api.PodSpec{Containers: make([]api.Container, 1)}, + Status: api.PodStatus{ + Phase: "Running", + ContainerStatuses: []api.ContainerStatus{ + { + Ready: true, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-5 * 24 * time.Hour))}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test12", "1/1", "Running", "3 (5d ago)", ""}}}, + }, + { + // Test pod has 2 containers, one has never restarted and the other has restarted 10d ago + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test13"}, + Spec: api.PodSpec{Containers: make([]api.Container, 2)}, + Status: api.PodStatus{ + Phase: "Running", + ContainerStatuses: []api.ContainerStatus{ + { + Ready: true, + RestartCount: 0, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + }, + { + Ready: true, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-10 * 24 * time.Hour))}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test13", "2/2", "Running", "3 (10d ago)", ""}}}, + }, + { + // Test pod has 2 containers, one restarted 5d ago and the other restarted 20d ago + api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test14"}, + Spec: api.PodSpec{Containers: make([]api.Container, 2)}, + Status: api.PodStatus{ + Phase: "Running", + ContainerStatuses: []api.ContainerStatus{ + { + Ready: true, + RestartCount: 6, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-5 * 24 * time.Hour))}}, + }, + { + Ready: true, + RestartCount: 3, + State: api.ContainerState{Running: &api.ContainerStateRunning{}}, + LastTerminationState: api.ContainerState{Terminated: &api.ContainerStateTerminated{FinishedAt: metav1.NewTime(time.Now().Add(-20 * 24 * time.Hour))}}, + }, + }, + }, + }, + []metav1.TableRow{{Cells: []interface{}{"test14", "2/2", "Running", "9 (5d ago)", ""}}}, }, } @@ -1351,7 +1561,7 @@ func TestPrintPodwide(t *testing.T) { NominatedNodeName: "node1", }, }, - []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", int64(6), "", "1.1.1.1", "test1", "node1", "1/3"}}}, + []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", "6", "", "1.1.1.1", "test1", "node1", "1/3"}}}, }, { // Test when the NodeName and PodIP are not none @@ -1392,7 +1602,7 @@ func TestPrintPodwide(t *testing.T) { NominatedNodeName: "node1", }, }, - []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", int64(6), "", "1.1.1.1", "test1", "node1", "1/3"}}}, + []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "podPhase", "6", "", "1.1.1.1", "test1", "node1", "1/3"}}}, }, { // Test when the NodeName and PodIP are none @@ -1410,7 +1620,7 @@ func TestPrintPodwide(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test2", "1/2", "ContainerWaitingReason", int64(6), "", "", "", "", ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test2", "1/2", "ContainerWaitingReason", "6", "", "", "", "", ""}}}, }, } @@ -1470,7 +1680,7 @@ func TestPrintPodConditions(t *testing.T) { { pod: runningPod, // Columns: Name, Ready, Reason, Restarts, Age - expect: []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "Running", int64(6), ""}}}, + expect: []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "Running", "6", ""}}}, }, // Should have TableRowCondition: podSuccessConditions { @@ -1478,7 +1688,7 @@ func TestPrintPodConditions(t *testing.T) { expect: []metav1.TableRow{ { // Columns: Name, Ready, Reason, Restarts, Age - Cells: []interface{}{"test1", "1/2", "Succeeded", int64(6), ""}, + Cells: []interface{}{"test1", "1/2", "Succeeded", "6", ""}, Conditions: podSuccessConditions, }, }, @@ -1489,7 +1699,7 @@ func TestPrintPodConditions(t *testing.T) { expect: []metav1.TableRow{ { // Columns: Name, Ready, Reason, Restarts, Age - Cells: []interface{}{"test2", "1/2", "Failed", int64(6), ""}, + Cells: []interface{}{"test2", "1/2", "Failed", "6", ""}, Conditions: podFailedConditions, }, }, @@ -1542,7 +1752,7 @@ func TestPrintPodList(t *testing.T) { }, }, }, - []metav1.TableRow{{Cells: []interface{}{"test1", "2/2", "podPhase", int64(6), ""}}, {Cells: []interface{}{"test2", "1/1", "podPhase", int64(1), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test1", "2/2", "podPhase", "6", ""}}, {Cells: []interface{}{"test2", "1/1", "podPhase", "1", ""}}}, }, } @@ -1580,7 +1790,7 @@ func TestPrintNonTerminatedPod(t *testing.T) { }, }, // Columns: Name, Ready, Reason, Restarts, Age - []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "Running", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test1", "1/2", "Running", "6", ""}}}, }, { // Test pod phase Pending should be printed @@ -1596,7 +1806,7 @@ func TestPrintNonTerminatedPod(t *testing.T) { }, }, // Columns: Name, Ready, Reason, Restarts, Age - []metav1.TableRow{{Cells: []interface{}{"test2", "1/2", "Pending", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test2", "1/2", "Pending", "6", ""}}}, }, { // Test pod phase Unknown should be printed @@ -1612,7 +1822,7 @@ func TestPrintNonTerminatedPod(t *testing.T) { }, }, // Columns: Name, Ready, Reason, Restarts, Age - []metav1.TableRow{{Cells: []interface{}{"test3", "1/2", "Unknown", int64(6), ""}}}, + []metav1.TableRow{{Cells: []interface{}{"test3", "1/2", "Unknown", "6", ""}}}, }, { // Test pod phase Succeeded should be printed @@ -1630,7 +1840,7 @@ func TestPrintNonTerminatedPod(t *testing.T) { // Columns: Name, Ready, Reason, Restarts, Age []metav1.TableRow{ { - Cells: []interface{}{"test4", "1/2", "Succeeded", int64(6), ""}, + Cells: []interface{}{"test4", "1/2", "Succeeded", "6", ""}, Conditions: podSuccessConditions, }, }, @@ -1651,7 +1861,7 @@ func TestPrintNonTerminatedPod(t *testing.T) { // Columns: Name, Ready, Reason, Restarts, Age []metav1.TableRow{ { - Cells: []interface{}{"test5", "1/2", "Failed", int64(6), ""}, + Cells: []interface{}{"test5", "1/2", "Failed", "6", ""}, Conditions: podFailedConditions, }, }, diff --git a/pkg/registry/core/pod/storage/storage_test.go b/pkg/registry/core/pod/storage/storage_test.go index e9cae9da4a5..85f62ffadaf 100644 --- a/pkg/registry/core/pod/storage/storage_test.go +++ b/pkg/registry/core/pod/storage/storage_test.go @@ -465,7 +465,7 @@ func TestConvertToTableList(t *testing.T) { {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "Ready", Type: "string", Description: "The aggregate readiness state of this pod for accepting traffic."}, {Name: "Status", Type: "string", Description: "The aggregate status of the containers in this pod."}, - {Name: "Restarts", Type: "integer", Description: "The number of times the containers in this pod have been restarted."}, + {Name: "Restarts", Type: "string", Description: "The number of times the containers in this pod have been restarted and when the last container in this pod has restarted."}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, {Name: "IP", Type: "string", Priority: 1, Description: v1.PodStatus{}.SwaggerDoc()["podIP"]}, {Name: "Node", Type: "string", Priority: 1, Description: v1.PodSpec{}.SwaggerDoc()["nodeName"]}, @@ -565,7 +565,7 @@ func TestConvertToTableList(t *testing.T) { out: &metav1.Table{ ColumnDefinitions: columns, Rows: []metav1.TableRow{ - {Cells: []interface{}{"", "0/0", "", int64(0), "", "", "", "", ""}, Object: runtime.RawExtension{Object: &api.Pod{}}}, + {Cells: []interface{}{"", "0/0", "", "0", "", "", "", "", ""}, Object: runtime.RawExtension{Object: &api.Pod{}}}, }, }, }, @@ -574,7 +574,7 @@ func TestConvertToTableList(t *testing.T) { out: &metav1.Table{ ColumnDefinitions: columns, Rows: []metav1.TableRow{ - {Cells: []interface{}{"foo", "1/2", "Pending", int64(10), "370d", "10.1.2.3", "test-node", "nominated-node", "1/2"}, Object: runtime.RawExtension{Object: pod1}}, + {Cells: []interface{}{"foo", "1/2", "Pending", "10", "370d", "10.1.2.3", "test-node", "nominated-node", "1/2"}, Object: runtime.RawExtension{Object: pod1}}, }, }, }, @@ -587,7 +587,7 @@ func TestConvertToTableList(t *testing.T) { out: &metav1.Table{ ColumnDefinitions: columns, Rows: []metav1.TableRow{ - {Cells: []interface{}{"foo", "1/2", "Pending", int64(10), "370d", "10.1.2.3", "test-node", "nominated-node", "1/2"}, Object: runtime.RawExtension{Object: multiIPsPod}}, + {Cells: []interface{}{"foo", "1/2", "Pending", "10", "370d", "10.1.2.3", "test-node", "nominated-node", "1/2"}, Object: runtime.RawExtension{Object: multiIPsPod}}, }, }, },