diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 055e8e0815f..27490aa2cfb 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -590,6 +590,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, // podManager is also responsible for keeping secretManager and configMapManager contents up-to-date. klet.podManager = kubepod.NewBasicPodManager(kubepod.NewBasicMirrorClient(klet.kubeClient), secretManager, configMapManager, checkpointManager) + klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet) + if remoteRuntimeEndpoint != "" { // remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified if remoteImageEndpoint == "" { @@ -705,7 +707,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.resourceAnalyzer, klet.podManager, klet.runtimeCache, - klet.containerRuntime) + klet.containerRuntime, + klet.statusManager) } else { klet.StatsProvider = stats.NewCRIStatsProvider( klet.cadvisor, @@ -754,8 +757,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, klet.containerLogManager = logs.NewStubContainerLogManager() } - klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet) - if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory) if err != nil { diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 66a41472221..192060bda84 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -264,7 +264,8 @@ func newTestKubeletWithImageList( kubelet.resourceAnalyzer, kubelet.podManager, kubelet.runtimeCache, - fakeRuntime) + fakeRuntime, + kubelet.statusManager) fakeImageGCPolicy := images.ImageGCPolicy{ HighThresholdPercent: 90, LowThresholdPercent: 80, diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD index 07ca1b54540..ca3efe32f00 100644 --- a/pkg/kubelet/stats/BUILD +++ b/pkg/kubelet/stats/BUILD @@ -24,6 +24,7 @@ go_library( "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/pod:go_default_library", "//pkg/kubelet/server/stats:go_default_library", + "//pkg/kubelet/status:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/volume:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -71,8 +72,10 @@ go_test( "//pkg/kubelet/leaky:go_default_library", "//pkg/kubelet/pod/testing:go_default_library", "//pkg/kubelet/server/stats:go_default_library", + "//pkg/kubelet/status/testing:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/volume:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 9549c12a1a6..a8abfb26646 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -33,6 +33,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/leaky" "k8s.io/kubernetes/pkg/kubelet/server/stats" + "k8s.io/kubernetes/pkg/kubelet/status" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -47,6 +48,8 @@ type cadvisorStatsProvider struct { resourceAnalyzer stats.ResourceAnalyzer // imageService is used to get the stats of the image filesystem. imageService kubecontainer.ImageService + // statusProvider is used to get pod metadata + statusProvider status.PodStatusProvider } // newCadvisorStatsProvider returns a containerStatsProvider that provides @@ -55,11 +58,13 @@ func newCadvisorStatsProvider( cadvisor cadvisor.Interface, resourceAnalyzer stats.ResourceAnalyzer, imageService kubecontainer.ImageService, + statusProvider status.PodStatusProvider, ) containerStatsProvider { return &cadvisorStatsProvider{ cadvisor: cadvisor, resourceAnalyzer: resourceAnalyzer, imageService: imageService, + statusProvider: statusProvider, } } @@ -114,7 +119,6 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { // Special case for infrastructure container which is hidden from // the user and has network stats. podStats.Network = cadvisorInfoToNetworkStats("pod:"+ref.Namespace+"_"+ref.Name, &cinfo) - podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime) } else { podStats.Containers = append(podStats.Containers, *cadvisorInfoToContainerStats(containerName, &cinfo, &rootFsInfo, &imageFsInfo)) } @@ -139,7 +143,13 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { podStats.CPU = cpu podStats.Memory = memory } - result = append(result, *podStats) + + status, found := p.statusProvider.GetPodStatus(podUID) + if found && status.StartTime != nil && !status.StartTime.IsZero() { + podStats.StartTime = *status.StartTime + // only append stats if we were able to get the start time of the pod + result = append(result, *podStats) + } } return result, nil diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index 68a41acb1b9..9c4b6ddd78a 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -22,12 +22,16 @@ import ( cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/leaky" serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" + statustest "k8s.io/kubernetes/pkg/kubelet/status/testing" ) func TestRemoveTerminatedContainerInfo(t *testing.T) { @@ -196,10 +200,17 @@ func TestCadvisorListPodStats(t *testing.T) { EphemeralVolumes: ephemeralVolumes, PersistentVolumes: persistentVolumes, } + p0Time := metav1.Now() + p1Time := metav1.Now() + p2Time := metav1.Now() + mockStatus := new(statustest.MockStatusProvider) + mockStatus.On("GetPodStatus", types.UID("UID"+pName0)).Return(v1.PodStatus{StartTime: &p0Time}, true) + mockStatus.On("GetPodStatus", types.UID("UID"+pName1)).Return(v1.PodStatus{StartTime: &p1Time}, true) + mockStatus.On("GetPodStatus", types.UID("UID"+pName2)).Return(v1.PodStatus{StartTime: &p2Time}, true) resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats} - p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime) + p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime, mockStatus) pods, err := p.ListPodStats() assert.NoError(t, err) @@ -227,7 +238,7 @@ func TestCadvisorListPodStats(t *testing.T) { checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU) checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory) - assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) + assert.EqualValues(t, p0Time.Unix(), ps.StartTime.Time.Unix()) checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network) checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, ps.EphemeralStorage) if ps.CPU != nil { @@ -349,7 +360,7 @@ func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) { resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats} - p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil) + p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil, nil) pods, err := p.ListPodCPUAndMemoryStats() assert.NoError(t, err) @@ -435,7 +446,7 @@ func TestCadvisorImagesFsStats(t *testing.T) { mockCadvisor.On("ImagesFsInfo").Return(imageFsInfo, nil) mockRuntime.On("ImageStats").Return(imageStats, nil) - provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime) + provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil) stats, err := provider.ImageFsStats() assert.NoError(err) diff --git a/pkg/kubelet/stats/stats_provider.go b/pkg/kubelet/stats/stats_provider.go index 903f8678a64..b06c7a8ad8a 100644 --- a/pkg/kubelet/stats/stats_provider.go +++ b/pkg/kubelet/stats/stats_provider.go @@ -28,6 +28,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" "k8s.io/kubernetes/pkg/kubelet/server/stats" + "k8s.io/kubernetes/pkg/kubelet/status" ) // NewCRIStatsProvider returns a StatsProvider that provides the node stats @@ -52,8 +53,9 @@ func NewCadvisorStatsProvider( podManager kubepod.Manager, runtimeCache kubecontainer.RuntimeCache, imageService kubecontainer.ImageService, + statusProvider status.PodStatusProvider, ) *StatsProvider { - return newStatsProvider(cadvisor, podManager, runtimeCache, newCadvisorStatsProvider(cadvisor, resourceAnalyzer, imageService)) + return newStatsProvider(cadvisor, podManager, runtimeCache, newCadvisorStatsProvider(cadvisor, resourceAnalyzer, imageService, statusProvider)) } // newStatsProvider returns a new StatsProvider that provides node stats from diff --git a/pkg/kubelet/status/testing/BUILD b/pkg/kubelet/status/testing/BUILD index 4bc79aca4ec..c335925ad29 100644 --- a/pkg/kubelet/status/testing/BUILD +++ b/pkg/kubelet/status/testing/BUILD @@ -7,9 +7,16 @@ load( go_library( name = "go_default_library", - srcs = ["fake_pod_deletion_safety.go"], + srcs = [ + "fake_pod_deletion_safety.go", + "mock_pod_status_provider.go", + ], importpath = "k8s.io/kubernetes/pkg/kubelet/status/testing", - deps = ["//staging/src/k8s.io/api/core/v1:go_default_library"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/github.com/stretchr/testify/mock:go_default_library", + ], ) filegroup( diff --git a/pkg/kubelet/status/testing/mock_pod_status_provider.go b/pkg/kubelet/status/testing/mock_pod_status_provider.go new file mode 100644 index 00000000000..cef85222ada --- /dev/null +++ b/pkg/kubelet/status/testing/mock_pod_status_provider.go @@ -0,0 +1,33 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "github.com/stretchr/testify/mock" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +type MockStatusProvider struct { + mock.Mock +} + +func (m *MockStatusProvider) GetPodStatus(uid types.UID) (v1.PodStatus, bool) { + args := m.Called(uid) + return args.Get(0).(v1.PodStatus), args.Bool(1) +}