From a6718f59691d7b3989c4222fb21e8daeb2ff84cc Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Thu, 14 Apr 2016 18:01:40 -0700 Subject: [PATCH] rkt: Implement pod `FinishedAt` This is implemented via touching a file on stop as a hook in the systemd unit. The ctime of this file is then used to get the `finishedAt` time in the future. In addition, this changes the `startedAt` and `createdAt` to use the api server's results rather than the annotations it previously used. It's possible we might want to move this into the api in the future. Fixes #23887 --- pkg/kubelet/container/helpers.go | 2 + pkg/kubelet/container/os.go | 6 + pkg/kubelet/container/testing/os.go | 15 ++- pkg/kubelet/dockertools/manager_test.go | 4 + pkg/kubelet/kubelet.go | 11 +- pkg/kubelet/rkt/fake_rkt_interface_test.go | 5 + pkg/kubelet/rkt/mock_os/mockfileinfo.go | 109 +++++++++++++++++++ pkg/kubelet/rkt/rkt.go | 118 +++++++++++++------- pkg/kubelet/rkt/rkt_test.go | 121 ++++++++++++++------- 9 files changed, 306 insertions(+), 85 deletions(-) create mode 100644 pkg/kubelet/rkt/mock_os/mockfileinfo.go diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index d70f3516872..781080b6669 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/types" hashutil "k8s.io/kubernetes/pkg/util/hash" "k8s.io/kubernetes/third_party/golang/expansion" @@ -42,6 +43,7 @@ type RuntimeHelper interface { GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*RunContainerOptions, error) GetClusterDNS(pod *api.Pod) (dnsServers []string, dnsSearches []string, err error) GeneratePodHostNameAndDomain(pod *api.Pod) (hostname string, hostDomain string) + GetPodDir(podUID types.UID) string } // ShouldContainerBeRestarted checks whether a container needs to be restarted. diff --git a/pkg/kubelet/container/os.go b/pkg/kubelet/container/os.go index 37a1abeac41..7fedd5703f2 100644 --- a/pkg/kubelet/container/os.go +++ b/pkg/kubelet/container/os.go @@ -25,6 +25,7 @@ import ( type OSInterface interface { Mkdir(path string, perm os.FileMode) error Symlink(oldname string, newname string) error + Stat(path string) (os.FileInfo, error) } // RealOS is used to dispatch the real system level operaitons. @@ -39,3 +40,8 @@ func (RealOS) Mkdir(path string, perm os.FileMode) error { func (RealOS) Symlink(oldname string, newname string) error { return os.Symlink(oldname, newname) } + +// Stat will call os.Stat to get the FileInfo for a given path +func (RealOS) Stat(path string) (os.FileInfo, error) { + return os.Stat(path) +} diff --git a/pkg/kubelet/container/testing/os.go b/pkg/kubelet/container/testing/os.go index fd379c2e2af..97dd73305c9 100644 --- a/pkg/kubelet/container/testing/os.go +++ b/pkg/kubelet/container/testing/os.go @@ -17,12 +17,17 @@ limitations under the License. package testing import ( + "errors" "os" ) // FakeOS mocks out certain OS calls to avoid perturbing the filesystem // on the test machine. -type FakeOS struct{} +// If a member of the form `*Fn` is set, that function will be called in place +// of the real call. +type FakeOS struct { + StatFn func(string) (os.FileInfo, error) +} // Mkdir is a fake call that just returns nil. func (FakeOS) Mkdir(path string, perm os.FileMode) error { @@ -33,3 +38,11 @@ func (FakeOS) Mkdir(path string, perm os.FileMode) error { func (FakeOS) Symlink(oldname string, newname string) error { return nil } + +// Stat is a fake that returns an error +func (f FakeOS) Stat(path string) (os.FileInfo, error) { + if f.StatFn != nil { + return f.StatFn(path) + } + return nil, errors.New("unimplemented testing mock") +} diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index c5041dccf46..28c4e1f2fb9 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -93,6 +93,10 @@ func (f *fakeRuntimeHelper) GeneratePodHostNameAndDomain(pod *api.Pod) (string, return "", "" } +func (f *fakeRuntimeHelper) GetPodDir(types.UID) string { + return "" +} + func createTestDockerManager(fakeHTTPClient *fakeHTTP, fakeDocker *FakeDockerClient) (*DockerManager, *FakeDockerClient) { if fakeHTTPClient == nil { fakeHTTPClient = &fakeHTTP{} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index c52eb671028..3c548c2b043 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -72,6 +72,7 @@ import ( "k8s.io/kubernetes/pkg/util/atomic" "k8s.io/kubernetes/pkg/util/bandwidth" utilerrors "k8s.io/kubernetes/pkg/util/errors" + utilexec "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/flowcontrol" kubeio "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" @@ -426,6 +427,8 @@ func NewMainKubelet( klet.livenessManager, klet.volumeManager, klet.httpClient, + utilexec.New(), + kubecontainer.RealOS{}, imageBackOff, serializeImagePulls, ) @@ -830,8 +833,12 @@ func (kl *Kubelet) getPluginDir(pluginName string) string { return path.Join(kl.getPluginsDir(), pluginName) } -// getPodDir returns the full path to the per-pod data directory for the -// specified pod. This directory may not exist if the pod does not exist. +// GetPodDir returns the full path to the per-pod data directory for the +// specified pod. This directory may not exist if the pod does not exist. +func (kl *Kubelet) GetPodDir(podUID types.UID) string { + return kl.getPodDir(podUID) +} + func (kl *Kubelet) getPodDir(podUID types.UID) string { // Backwards compat. The "old" stuff should be removed before 1.0 // release. The thinking here is this: diff --git a/pkg/kubelet/rkt/fake_rkt_interface_test.go b/pkg/kubelet/rkt/fake_rkt_interface_test.go index d2bbf1b094d..d028d9efc30 100644 --- a/pkg/kubelet/rkt/fake_rkt_interface_test.go +++ b/pkg/kubelet/rkt/fake_rkt_interface_test.go @@ -22,6 +22,7 @@ import ( "sync" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/types" "github.com/coreos/go-systemd/dbus" rktapi "github.com/coreos/rkt/api/v1alpha" @@ -166,3 +167,7 @@ func (f *fakeRuntimeHelper) GetClusterDNS(pod *api.Pod) ([]string, []string, err func (f *fakeRuntimeHelper) GeneratePodHostNameAndDomain(pod *api.Pod) (string, string) { return f.hostName, f.hostDomain } + +func (f *fakeRuntimeHelper) GetPodDir(podUID types.UID) string { + return "/poddir/" + string(podUID) +} diff --git a/pkg/kubelet/rkt/mock_os/mockfileinfo.go b/pkg/kubelet/rkt/mock_os/mockfileinfo.go new file mode 100644 index 00000000000..16761e2586b --- /dev/null +++ b/pkg/kubelet/rkt/mock_os/mockfileinfo.go @@ -0,0 +1,109 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Generated via: mockgen os FileInfo +// Edited to include required boilerplate +// Source: os (interfaces: FileInfo) + +package mock_os + +import ( + os "os" + time "time" + + gomock "github.com/golang/mock/gomock" +) + +// Mock of FileInfo interface +type MockFileInfo struct { + ctrl *gomock.Controller + recorder *_MockFileInfoRecorder +} + +// Recorder for MockFileInfo (not exported) +type _MockFileInfoRecorder struct { + mock *MockFileInfo +} + +func NewMockFileInfo(ctrl *gomock.Controller) *MockFileInfo { + mock := &MockFileInfo{ctrl: ctrl} + mock.recorder = &_MockFileInfoRecorder{mock} + return mock +} + +func (_m *MockFileInfo) EXPECT() *_MockFileInfoRecorder { + return _m.recorder +} + +func (_m *MockFileInfo) IsDir() bool { + ret := _m.ctrl.Call(_m, "IsDir") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockFileInfoRecorder) IsDir() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "IsDir") +} + +func (_m *MockFileInfo) ModTime() time.Time { + ret := _m.ctrl.Call(_m, "ModTime") + ret0, _ := ret[0].(time.Time) + return ret0 +} + +func (_mr *_MockFileInfoRecorder) ModTime() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "ModTime") +} + +func (_m *MockFileInfo) Mode() os.FileMode { + ret := _m.ctrl.Call(_m, "Mode") + ret0, _ := ret[0].(os.FileMode) + return ret0 +} + +func (_mr *_MockFileInfoRecorder) Mode() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Mode") +} + +func (_m *MockFileInfo) Name() string { + ret := _m.ctrl.Call(_m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +func (_mr *_MockFileInfoRecorder) Name() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") +} + +func (_m *MockFileInfo) Size() int64 { + ret := _m.ctrl.Call(_m, "Size") + ret0, _ := ret[0].(int64) + return ret0 +} + +func (_mr *_MockFileInfoRecorder) Size() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Size") +} + +func (_m *MockFileInfo) Sys() interface{} { + ret := _m.ctrl.Call(_m, "Sys") + ret0, _ := ret[0].(interface{}) + return ret0 +} + +func (_mr *_MockFileInfoRecorder) Sys() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Sys") +} diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 2e2e701879b..d9af816d047 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "sync" @@ -76,13 +77,11 @@ const ( unitRktID = "RktID" unitRestartCount = "RestartCount" - k8sRktKubeletAnno = "rkt.kubernetes.io/managed-by-kubelet" - k8sRktKubeletAnnoValue = "true" - k8sRktUIDAnno = "rkt.kubernetes.io/uid" - k8sRktNameAnno = "rkt.kubernetes.io/name" - k8sRktNamespaceAnno = "rkt.kubernetes.io/namespace" - //TODO: remove the creation time annotation once this is closed: https://github.com/coreos/rkt/issues/1789 - k8sRktCreationTimeAnno = "rkt.kubernetes.io/created" + k8sRktKubeletAnno = "rkt.kubernetes.io/managed-by-kubelet" + k8sRktKubeletAnnoValue = "true" + k8sRktUIDAnno = "rkt.kubernetes.io/uid" + k8sRktNameAnno = "rkt.kubernetes.io/name" + k8sRktNamespaceAnno = "rkt.kubernetes.io/namespace" k8sRktContainerHashAnno = "rkt.kubernetes.io/container-hash" k8sRktRestartCountAnno = "rkt.kubernetes.io/restart-count" k8sRktTerminationMessagePathAnno = "rkt.kubernetes.io/termination-message-path" @@ -128,6 +127,11 @@ type Runtime struct { volumeGetter volumeGetter imagePuller kubecontainer.ImagePuller runner kubecontainer.HandlerRunner + execer utilexec.Interface + os kubecontainer.OSInterface + + // used for a systemd Exec, which requires the full path. + touchPath string versions versions } @@ -151,6 +155,8 @@ func New( livenessManager proberesults.Manager, volumeGetter volumeGetter, httpClient kubetypes.HttpGetter, + execer utilexec.Interface, + os kubecontainer.OSInterface, imageBackOff *flowcontrol.Backoff, serializeImagePulls bool, ) (*Runtime, error) { @@ -170,12 +176,17 @@ func New( if config.Path == "" { // No default rkt path was set, so try to find one in $PATH. var err error - config.Path, err = exec.LookPath("rkt") + config.Path, err = execer.LookPath("rkt") if err != nil { return nil, fmt.Errorf("cannot find rkt binary: %v", err) } } + touchPath, err := execer.LookPath("touch") + if err != nil { + return nil, fmt.Errorf("cannot find touch binary: %v", err) + } + rkt := &Runtime{ systemd: systemd, apisvcConn: apisvcConn, @@ -187,6 +198,8 @@ func New( recorder: recorder, livenessManager: livenessManager, volumeGetter: volumeGetter, + execer: execer, + touchPath: touchPath, } rkt.config, err = rkt.getConfig(rkt.config) @@ -541,7 +554,6 @@ func (r *Runtime) makePodManifest(pod *api.Pod, pullSecrets []api.Secret) (*appc manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktUIDAnno), string(pod.UID)) manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktNameAnno), pod.Name) manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktNamespaceAnno), pod.Namespace) - manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktCreationTimeAnno), strconv.FormatInt(time.Now().Unix(), 10)) manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktRestartCountAnno), strconv.Itoa(restartCount)) for _, c := range pod.Spec.Containers { @@ -587,6 +599,34 @@ func makeHostNetworkMount(opts *kubecontainer.RunContainerOptions) (*kubecontain return &hostsMount, &resolvMount } +// podFinishedMarkerPath returns the path to a file which should be used to +// indicate the pod exiting, and the time thereof. +// If the file at the path does not exist, the pod should not be exited. If it +// does exist, then the ctime of the file should indicate the time the pod +// exited. +func podFinishedMarkerPath(podDir string, rktUID string) string { + return filepath.Join(podDir, "finished-"+rktUID) +} + +func podFinishedMarkCommand(touchPath, podDir, rktUID string) string { + // TODO, if the path has a `'` character in it, this breaks. + return touchPath + " " + podFinishedMarkerPath(podDir, rktUID) +} + +// podFinishedAt returns the time that a pod exited, or a zero time if it has +// not. +func (r *Runtime) podFinishedAt(podUID types.UID, rktUID string) time.Time { + markerFile := podFinishedMarkerPath(r.runtimeHelper.GetPodDir(podUID), rktUID) + stat, err := r.os.Stat(markerFile) + if err != nil { + if !os.IsNotExist(err) { + glog.Warningf("rkt: unexpected fs error checking pod finished marker: %v", err) + } + return time.Time{} + } + return stat.ModTime() +} + func makeContainerLogMount(opts *kubecontainer.RunContainerOptions, container *api.Container) (*kubecontainer.Mount, error) { if opts.PodContainerDir == "" || container.TerminationMessagePath == "" { return nil, nil @@ -884,8 +924,11 @@ func (r *Runtime) preparePod(pod *api.Pod, pullSecrets []api.Secret) (string, *k // TODO handle pod.Spec.HostPID // TODO handle pod.Spec.HostIPC + // TODO per container finishedAt, not just per pod + markPodFinished := podFinishedMarkCommand(r.touchPath, r.runtimeHelper.GetPodDir(pod.UID), uuid) units := []*unit.UnitOption{ newUnitOption("Service", "ExecStart", runPrepared), + newUnitOption("Service", "ExecStopPost", markPodFinished), // This enables graceful stop. newUnitOption("Service", "KillMode", "mixed"), } @@ -1045,14 +1088,6 @@ func (r *Runtime) convertRktPod(rktpod *rktapi.Pod) (*kubecontainer.Pod, error) if !ok { return nil, fmt.Errorf("pod is missing annotation %s", k8sRktNamespaceAnno) } - podCreatedString, ok := manifest.Annotations.Get(k8sRktCreationTimeAnno) - if !ok { - return nil, fmt.Errorf("pod is missing annotation %s", k8sRktCreationTimeAnno) - } - podCreated, err := strconv.ParseInt(podCreatedString, 10, 64) - if err != nil { - return nil, fmt.Errorf("couldn't parse pod creation timestamp: %v", err) - } kubepod := &kubecontainer.Pod{ ID: types.UID(podUID), @@ -1078,8 +1113,8 @@ func (r *Runtime) convertRktPod(rktpod *rktapi.Pod) (*kubecontainer.Pod, error) // By default, the version returned by rkt API service will be "latest" if not specified. Image: fmt.Sprintf("%s:%s", app.Image.Name, app.Image.Version), Hash: containerHash, - Created: podCreated, State: appStateToContainerState(app.State), + Created: time.Unix(0, rktpod.CreatedAt).Unix(), // convert ns to s }) } @@ -1526,7 +1561,7 @@ func appStateToContainerState(state rktapi.AppState) kubecontainer.ContainerStat } // getPodInfo returns the pod manifest, creation time and restart count of the pod. -func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, creationTime time.Time, restartCount int, err error) { +func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, restartCount int, err error) { // TODO(yifan): The manifest is only used for getting the annotations. // Consider to let the server to unmarshal the annotations. var manifest appcschema.PodManifest @@ -1534,16 +1569,6 @@ func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, creationT return } - creationTimeStr, ok := manifest.Annotations.Get(k8sRktCreationTimeAnno) - if !ok { - err = fmt.Errorf("no creation timestamp in pod manifest") - return - } - unixSec, err := strconv.ParseInt(creationTimeStr, 10, 64) - if err != nil { - return - } - if countString, ok := manifest.Annotations.Get(k8sRktRestartCountAnno); ok { restartCount, err = strconv.Atoi(countString) if err != nil { @@ -1551,11 +1576,11 @@ func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, creationT } } - return &manifest, time.Unix(unixSec, 0), restartCount, nil + return &manifest, restartCount, nil } // populateContainerStatus fills the container status according to the app's information. -func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcschema.RuntimeApp, restartCount int, creationTime time.Time) (*kubecontainer.ContainerStatus, error) { +func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcschema.RuntimeApp, restartCount int, finishedTime time.Time) (*kubecontainer.ContainerStatus, error) { hashStr, ok := runtimeApp.Annotations.Get(k8sRktContainerHashAnno) if !ok { return nil, fmt.Errorf("No container hash in pod manifest") @@ -1584,14 +1609,17 @@ func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcsche } } + createdTime := time.Unix(0, pod.CreatedAt) + startedTime := time.Unix(0, pod.StartedAt) + return &kubecontainer.ContainerStatus{ - ID: buildContainerID(&containerID{uuid: pod.Id, appName: app.Name}), - Name: app.Name, - State: appStateToContainerState(app.State), - // TODO(yifan): Use the creation/start/finished timestamp when it's implemented. - CreatedAt: creationTime, - StartedAt: creationTime, - ExitCode: int(app.ExitCode), + ID: buildContainerID(&containerID{uuid: pod.Id, appName: app.Name}), + Name: app.Name, + State: appStateToContainerState(app.State), + CreatedAt: createdTime, + StartedAt: startedTime, + FinishedAt: finishedTime, + ExitCode: int(app.ExitCode), // By default, the version returned by rkt API service will be "latest" if not specified. Image: fmt.Sprintf("%s:%s", app.Image.Name, app.Image.Version), ImageID: "rkt://" + app.Image.Id, // TODO(yifan): Add the prefix only in api.PodStatus. @@ -1605,6 +1633,13 @@ func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcsche }, nil } +// GetPodStatus returns the status for a pod specified by a given UID, name, +// and namespace. It will attempt to find pod's information via a request to +// the rkt api server. +// An error will be returned if the api server returns an error. If the api +// server doesn't error, but doesn't provide meaningful information about the +// pod, a status with no information (other than the passed in arguments) is +// returned anyways. func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecontainer.PodStatus, error) { podStatus := &kubecontainer.PodStatus{ ID: uid, @@ -1626,7 +1661,7 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont // In this loop, we group all containers from all pods together, // also we try to find the latest pod, so we can fill other info of the pod below. for _, pod := range listResp.Pods { - manifest, creationTime, restartCount, err := getPodInfo(pod) + manifest, restartCount, err := getPodInfo(pod) if err != nil { glog.Warningf("rkt: Couldn't get necessary info from the rkt pod, (uuid %q): %v", pod.Id, err) continue @@ -1637,11 +1672,10 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont latestRestartCount = restartCount } + finishedTime := r.podFinishedAt(uid, pod.Id) for i, app := range pod.Apps { // The order of the apps is determined by the rkt pod manifest. - // TODO(yifan): Save creationTime, restartCount in each app's annotation, - // so we don't need to pass them. - cs, err := populateContainerStatus(*pod, *app, manifest.Apps[i], restartCount, creationTime) + cs, err := populateContainerStatus(*pod, *app, manifest.Apps[i], restartCount, finishedTime) if err != nil { glog.Warningf("rkt: Failed to populate container status(uuid %q, app %q): %v", pod.Id, app.Name, err) continue diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index baebbe1d1a4..a721dcb7833 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -27,11 +27,14 @@ import ( appcschema "github.com/appc/spec/schema" appctypes "github.com/appc/spec/schema/types" rktapi "github.com/coreos/rkt/api/v1alpha" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + containertesting "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/lifecycle" + "k8s.io/kubernetes/pkg/kubelet/rkt/mock_os" "k8s.io/kubernetes/pkg/util/errors" utiltesting "k8s.io/kubernetes/pkg/util/testing" ) @@ -62,9 +65,10 @@ func mustRktHash(hash string) *appctypes.Hash { func makeRktPod(rktPodState rktapi.PodState, rktPodID, podUID, podName, podNamespace, - podIP, podCreationTs, podRestartCount string, - appNames, imgIDs, imgNames, containerHashes []string, - appStates []rktapi.AppState, exitcodes []int32) *rktapi.Pod { + podIP string, podCreatedAt, podStartedAt int64, + podRestartCount string, appNames, imgIDs, imgNames, + containerHashes []string, appStates []rktapi.AppState, + exitcodes []int32) *rktapi.Pod { podManifest := &appcschema.PodManifest{ ACKind: appcschema.PodManifestKind, @@ -86,10 +90,6 @@ func makeRktPod(rktPodState rktapi.PodState, Name: *appctypes.MustACIdentifier(k8sRktNamespaceAnno), Value: podNamespace, }, - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(k8sRktCreationTimeAnno), - Value: podCreationTs, - }, appctypes.Annotation{ Name: *appctypes.MustACIdentifier(k8sRktRestartCountAnno), Value: podRestartCount, @@ -143,11 +143,13 @@ func makeRktPod(rktPodState rktapi.PodState, } return &rktapi.Pod{ - Id: rktPodID, - State: rktPodState, - Networks: []*rktapi.Network{{Name: defaultNetworkName, Ipv4: podIP}}, - Apps: apps, - Manifest: mustMarshalPodManifest(podManifest), + Id: rktPodID, + State: rktPodState, + Networks: []*rktapi.Network{{Name: defaultNetworkName, Ipv4: podIP}}, + Apps: apps, + Manifest: mustMarshalPodManifest(podManifest), + StartedAt: podStartedAt, + CreatedAt: podCreatedAt, } } @@ -346,6 +348,10 @@ func TestGetPods(t *testing.T) { fs := newFakeSystemd() r := &Runtime{apisvc: fr, systemd: fs} + ns := func(seconds int64) int64 { + return seconds * 1e9 + } + tests := []struct { pods []*rktapi.Pod result []*kubecontainer.Pod @@ -357,7 +363,7 @@ func TestGetPods(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_RUNNING, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", "100000", "7", + "10.10.10.42", ns(10), ns(10), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -377,7 +383,7 @@ func TestGetPods(t *testing.T) { Name: "app-1", Image: "img-name-1:latest", Hash: 1001, - Created: 100000, + Created: 10, State: "running", }, { @@ -385,7 +391,7 @@ func TestGetPods(t *testing.T) { Name: "app-2", Image: "img-name-2:latest", Hash: 1002, - Created: 100000, + Created: 10, State: "exited", }, }, @@ -397,7 +403,7 @@ func TestGetPods(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_RUNNING, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", "100000", "7", + "10.10.10.42", ns(10), ns(20), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -407,7 +413,7 @@ func TestGetPods(t *testing.T) { ), makeRktPod(rktapi.PodState_POD_STATE_EXITED, "uuid-4003", "43", "guestbook", "default", - "10.10.10.43", "90000", "7", + "10.10.10.43", ns(30), ns(40), "7", []string{"app-11", "app-22"}, []string{"img-id-11", "img-id-22"}, []string{"img-name-11", "img-name-22"}, @@ -417,7 +423,7 @@ func TestGetPods(t *testing.T) { ), makeRktPod(rktapi.PodState_POD_STATE_EXITED, "uuid-4004", "43", "guestbook", "default", - "10.10.10.44", "100000", "8", + "10.10.10.44", ns(50), ns(60), "8", []string{"app-11", "app-22"}, []string{"img-id-11", "img-id-22"}, []string{"img-name-11", "img-name-22"}, @@ -437,7 +443,7 @@ func TestGetPods(t *testing.T) { Name: "app-1", Image: "img-name-1:latest", Hash: 1001, - Created: 100000, + Created: 10, State: "running", }, { @@ -445,7 +451,7 @@ func TestGetPods(t *testing.T) { Name: "app-2", Image: "img-name-2:latest", Hash: 1002, - Created: 100000, + Created: 10, State: "exited", }, }, @@ -460,7 +466,7 @@ func TestGetPods(t *testing.T) { Name: "app-11", Image: "img-name-11:latest", Hash: 10011, - Created: 90000, + Created: 30, State: "exited", }, { @@ -468,7 +474,7 @@ func TestGetPods(t *testing.T) { Name: "app-22", Image: "img-name-22:latest", Hash: 10022, - Created: 90000, + Created: 30, State: "exited", }, { @@ -476,7 +482,7 @@ func TestGetPods(t *testing.T) { Name: "app-11", Image: "img-name-11:latest", Hash: 10011, - Created: 100000, + Created: 50, State: "running", }, { @@ -484,7 +490,7 @@ func TestGetPods(t *testing.T) { Name: "app-22", Image: "img-name-22:latest", Hash: 10022, - Created: 100000, + Created: 50, State: "running", }, }, @@ -557,7 +563,18 @@ func TestGetPodsFilters(t *testing.T) { func TestGetPodStatus(t *testing.T) { fr := newFakeRktInterface() fs := newFakeSystemd() - r := &Runtime{apisvc: fr, systemd: fs} + fos := &containertesting.FakeOS{} + frh := &fakeRuntimeHelper{} + r := &Runtime{ + apisvc: fr, + systemd: fs, + runtimeHelper: frh, + os: fos, + } + + ns := func(seconds int64) int64 { + return seconds * 1e9 + } tests := []struct { pods []*rktapi.Pod @@ -573,7 +590,7 @@ func TestGetPodStatus(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_RUNNING, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", "100000", "7", + "10.10.10.42", ns(10), ns(20), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -592,8 +609,9 @@ func TestGetPodStatus(t *testing.T) { ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), Name: "app-1", State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(100000, 0), - StartedAt: time.Unix(100000, 0), + CreatedAt: time.Unix(10, 0), + StartedAt: time.Unix(20, 0), + FinishedAt: time.Unix(0, 30), Image: "img-name-1:latest", ImageID: "rkt://img-id-1", Hash: 1001, @@ -603,8 +621,9 @@ func TestGetPodStatus(t *testing.T) { ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), Name: "app-2", State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(100000, 0), - StartedAt: time.Unix(100000, 0), + CreatedAt: time.Unix(10, 0), + StartedAt: time.Unix(20, 0), + FinishedAt: time.Unix(0, 30), Image: "img-name-2:latest", ImageID: "rkt://img-id-2", Hash: 1002, @@ -619,7 +638,7 @@ func TestGetPodStatus(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_EXITED, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", "90000", "7", + "10.10.10.42", ns(10), ns(20), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -629,7 +648,7 @@ func TestGetPodStatus(t *testing.T) { ), makeRktPod(rktapi.PodState_POD_STATE_RUNNING, // The latest pod is running. "uuid-4003", "42", "guestbook", "default", - "10.10.10.42", "100000", "10", + "10.10.10.42", ns(10), ns(20), "10", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -649,8 +668,9 @@ func TestGetPodStatus(t *testing.T) { ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), Name: "app-1", State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(90000, 0), - StartedAt: time.Unix(90000, 0), + CreatedAt: time.Unix(10, 0), + StartedAt: time.Unix(20, 0), + FinishedAt: time.Unix(0, 30), Image: "img-name-1:latest", ImageID: "rkt://img-id-1", Hash: 1001, @@ -660,8 +680,9 @@ func TestGetPodStatus(t *testing.T) { ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), Name: "app-2", State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(90000, 0), - StartedAt: time.Unix(90000, 0), + CreatedAt: time.Unix(10, 0), + StartedAt: time.Unix(20, 0), + FinishedAt: time.Unix(0, 30), Image: "img-name-2:latest", ImageID: "rkt://img-id-2", Hash: 1002, @@ -672,8 +693,9 @@ func TestGetPodStatus(t *testing.T) { ID: kubecontainer.BuildContainerID("rkt", "uuid-4003:app-1"), Name: "app-1", State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(100000, 0), - StartedAt: time.Unix(100000, 0), + CreatedAt: time.Unix(10, 0), + StartedAt: time.Unix(20, 0), + FinishedAt: time.Unix(0, 30), Image: "img-name-1:latest", ImageID: "rkt://img-id-1", Hash: 1001, @@ -683,8 +705,9 @@ func TestGetPodStatus(t *testing.T) { ID: kubecontainer.BuildContainerID("rkt", "uuid-4003:app-2"), Name: "app-2", State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(100000, 0), - StartedAt: time.Unix(100000, 0), + CreatedAt: time.Unix(10, 0), + StartedAt: time.Unix(20, 0), + FinishedAt: time.Unix(0, 30), Image: "img-name-2:latest", ImageID: "rkt://img-id-2", Hash: 1002, @@ -697,10 +720,28 @@ func TestGetPodStatus(t *testing.T) { }, } + ctrl := gomock.NewController(t) + defer ctrl.Finish() + for i, tt := range tests { testCaseHint := fmt.Sprintf("test case #%d", i) fr.pods = tt.pods + podTimes := map[string]time.Time{} + for _, pod := range tt.pods { + podTimes[podFinishedMarkerPath(r.runtimeHelper.GetPodDir(tt.result.ID), pod.Id)] = tt.result.ContainerStatuses[0].FinishedAt + } + + r.os.(*containertesting.FakeOS).StatFn = func(name string) (os.FileInfo, error) { + podTime, ok := podTimes[name] + if !ok { + t.Errorf("osStat called with %v, but only knew about %#v", name, podTimes) + } + mockFI := mock_os.NewMockFileInfo(ctrl) + mockFI.EXPECT().ModTime().Return(podTime) + return mockFI, nil + } + status, err := r.GetPodStatus("42", "guestbook", "default") if err != nil { t.Errorf("test case #%d: unexpected error: %v", i, err)