From 4bd9dbf6ad299e3afaee98edf71bd85559b5b84d Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Tue, 1 Nov 2016 21:39:46 -0700 Subject: [PATCH] Add RuntimeStatus in container/runtime.go --- pkg/kubelet/container/runtime.go | 58 ++++++++++++++++++- pkg/kubelet/container/testing/fake_runtime.go | 6 +- pkg/kubelet/container/testing/runtime_mock.go | 4 +- pkg/kubelet/dockershim/docker_service.go | 2 +- pkg/kubelet/dockertools/docker_manager.go | 4 +- pkg/kubelet/kubelet.go | 26 ++++++++- pkg/kubelet/kubelet_node_status_test.go | 32 ++++++++++ pkg/kubelet/kuberuntime/helpers.go | 19 +++--- .../kuberuntime/kuberuntime_manager.go | 21 ++----- pkg/kubelet/rkt/rkt.go | 4 +- 10 files changed, 140 insertions(+), 36 deletions(-) diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index f9ed18ee638..1020323d81d 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -70,8 +70,9 @@ type Runtime interface { // This may be different from the runtime engine's version. // TODO(random-liu): We should fold this into Version() APIVersion() (Version, error) - // Status returns error if the runtime is unhealthy; nil otherwise. - Status() error + // Status returns the status of the runtime. An error is returned if the Status + // function itself fails, nil otherwise. + Status() (*RuntimeStatus, error) // GetPods returns a list of containers grouped by pods. The boolean parameter // specifies whether the runtime returns all containers including those already // exited and dead containers (used for garbage collection). @@ -446,6 +447,59 @@ type VolumeInfo struct { type VolumeMap map[string]VolumeInfo +// RuntimeConditionType is the types of required runtime conditions. +type RuntimeConditionType string + +const ( + // RuntimeReady means the runtime is up and ready to accept basic containers. + RuntimeReady RuntimeConditionType = "RuntimeReady" + // NetworkReady means the runtime network is up and ready to accept containers which require network. + NetworkReady RuntimeConditionType = "NetworkReady" +) + +// RuntimeStatus contains the status of the runtime. +type RuntimeStatus struct { + // Conditions is an array of current observed runtime conditions. + Conditions []RuntimeCondition +} + +// GetRuntimeCondition gets a specified runtime condition from the runtime status. +func (r *RuntimeStatus) GetRuntimeCondition(t RuntimeConditionType) *RuntimeCondition { + for i := range r.Conditions { + c := &r.Conditions[i] + if c.Type == t { + return c + } + } + return nil +} + +// String formats the runtime status into human readable string. +func (s *RuntimeStatus) String() string { + var ss []string + for _, c := range s.Conditions { + ss = append(ss, c.String()) + } + return fmt.Sprintf("Runtime Conditions: %s", strings.Join(ss, ", ")) +} + +// RuntimeCondition contains condition information for the runtime. +type RuntimeCondition struct { + // Type of runtime condition. + Type RuntimeConditionType + // Status of the condition, one of true/false. + Status bool + // Reason is brief reason for the condition's last transition. + Reason string + // Message is human readable message indicating details about last transition. + Message string +} + +// String formats the runtime condition into human readable string. +func (c *RuntimeCondition) String() string { + return fmt.Sprintf("%s=%t reason:%s message:%s", c.Type, c.Status, c.Reason, c.Message) +} + type Pods []*Pod // FindPodByID finds and returns a pod in the pod list by UID. It will return an empty pod diff --git a/pkg/kubelet/container/testing/fake_runtime.go b/pkg/kubelet/container/testing/fake_runtime.go index a79c65f33cc..1e74f6c1f3e 100644 --- a/pkg/kubelet/container/testing/fake_runtime.go +++ b/pkg/kubelet/container/testing/fake_runtime.go @@ -50,6 +50,7 @@ type FakeRuntime struct { KilledPods []string StartedContainers []string KilledContainers []string + RuntimeStatus *RuntimeStatus VersionInfo string APIVersionInfo string RuntimeType string @@ -137,6 +138,7 @@ func (f *FakeRuntime) ClearCalls() { f.KilledPods = []string{} f.StartedContainers = []string{} f.KilledContainers = []string{} + f.RuntimeStatus = nil f.VersionInfo = "" f.RuntimeType = "" f.Err = nil @@ -207,12 +209,12 @@ func (f *FakeRuntime) APIVersion() (Version, error) { return &FakeVersion{Version: f.APIVersionInfo}, f.Err } -func (f *FakeRuntime) Status() error { +func (f *FakeRuntime) Status() (*RuntimeStatus, error) { f.Lock() defer f.Unlock() f.CalledFunctions = append(f.CalledFunctions, "Status") - return f.StatusErr + return f.RuntimeStatus, f.StatusErr } func (f *FakeRuntime) GetPods(all bool) ([]*Pod, error) { diff --git a/pkg/kubelet/container/testing/runtime_mock.go b/pkg/kubelet/container/testing/runtime_mock.go index b00ef98c681..c2a1e0e97a8 100644 --- a/pkg/kubelet/container/testing/runtime_mock.go +++ b/pkg/kubelet/container/testing/runtime_mock.go @@ -54,9 +54,9 @@ func (r *Mock) APIVersion() (Version, error) { return args.Get(0).(Version), args.Error(1) } -func (r *Mock) Status() error { +func (r *Mock) Status() (*RuntimeStatus, error) { args := r.Called() - return args.Error(0) + return args.Get(0).(*RuntimeStatus), args.Error(0) } func (r *Mock) GetPods(all bool) ([]*Pod, error) { diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 4399bc398a4..6e61573fb6d 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -60,7 +60,7 @@ const ( sandboxIDLabelKey = "io.kubernetes.sandbox.id" ) -// NetworkPluginArgs is the subset of kubelet runtime args we pass +// NetworkPluginSettings is the subset of kubelet runtime args we pass // to the container runtime shim so it can probe for network plugins. // In the future we will feed these directly to a standalone container // runtime process. diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index 63a0f290c0e..f0d85f56117 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -1107,8 +1107,8 @@ func (dm *DockerManager) APIVersion() (kubecontainer.Version, error) { // Now we do this by checking whether: // 1) `docker version` works // 2) docker version is compatible with minimum requirement -func (dm *DockerManager) Status() error { - return dm.checkVersionCompatibility() +func (dm *DockerManager) Status() (*kubecontainer.RuntimeStatus, error) { + return nil, dm.checkVersionCompatibility() } func (dm *DockerManager) checkVersionCompatibility() error { diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 3ddf5edf1ba..fa8a612db08 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1944,10 +1944,34 @@ func (kl *Kubelet) PLEGHealthCheck() (bool, error) { // and returns an error if the status check fails. If the status check is OK, // update the container runtime uptime in the kubelet runtimeState. func (kl *Kubelet) updateRuntimeUp() { - if err := kl.containerRuntime.Status(); err != nil { + s, err := kl.containerRuntime.Status() + if err != nil { glog.Errorf("Container runtime sanity check failed: %v", err) return } + // Only check specific conditions when runtime integration type is cri, + // because the old integration doesn't populate any runtime condition. + // TODO(random-liu): Add runtime error in runtimeState, and update it + // when when runtime is not ready, so that the information in RuntimeReady + // condition will be propagated to NodeReady condition. + // TODO(random-liu): Update network state according to NetworkReady runtime + // condition once it is implemented in dockershim. + if kl.kubeletConfiguration.ExperimentalRuntimeIntegrationType == "cri" { + if s == nil { + glog.Errorf("Container runtime status is nil") + return + } + runtimeReady := s.GetRuntimeCondition(kubecontainer.RuntimeReady) + // Periodically log the whole runtime status for debugging. + // TODO(random-liu): Consider to send node event when optional + // condition is unmet. + glog.V(4).Infof("Container runtime status: %v", s) + // If RuntimeReady is not set or is false, report an error. + if runtimeReady == nil || !runtimeReady.Status { + glog.Errorf("Container runtime not ready: %v", runtimeReady) + return + } + } kl.oneTimeInitializer.Do(kl.initializeRuntimeDependentModules) kl.runtimeState.setRuntimeSync(kl.clock.Now()) } diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index 1fba8300b35..c32f0109b51 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -865,6 +865,38 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) { clock.SetTime(time.Now()) kubelet.updateRuntimeUp() checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage) + + // Test cri integration. + kubelet.kubeletConfiguration.ExperimentalRuntimeIntegrationType = "cri" + fakeRuntime.StatusErr = nil + + // Should report node not ready if runtime status is nil. + fakeRuntime.RuntimeStatus = nil + kubelet.updateRuntimeUp() + checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage) + + // Should report node not ready if runtime status is empty. + fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{} + kubelet.updateRuntimeUp() + checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage) + + // Should report node not ready if RuntimeReady is false. + fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{ + Conditions: []kubecontainer.RuntimeCondition{ + {Type: kubecontainer.RuntimeReady, Status: false}, + }, + } + kubelet.updateRuntimeUp() + checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage) + + // Should report node ready if RuntimeReady is true. + fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{ + Conditions: []kubecontainer.RuntimeCondition{ + {Type: kubecontainer.RuntimeReady, Status: true}, + }, + } + kubelet.updateRuntimeUp() + checkNodeStatus(api.ConditionTrue, "KubeletReady", readyMessage) } func TestUpdateNodeStatusError(t *testing.T) { diff --git a/pkg/kubelet/kuberuntime/helpers.go b/pkg/kubelet/kuberuntime/helpers.go index caef101b6b8..f17853e0535 100644 --- a/pkg/kubelet/kuberuntime/helpers.go +++ b/pkg/kubelet/kuberuntime/helpers.go @@ -217,13 +217,16 @@ func buildPodLogsDirectory(podUID types.UID) string { return filepath.Join(podLogsRootDirectory, string(podUID)) } -// getRuntimeCondition gets specified runtime condition from the runtime status. -func getRuntimeCondition(status *runtimeApi.RuntimeStatus, t string) *runtimeApi.RuntimeCondition { - conditions := status.GetConditions() - for _, condition := range conditions { - if condition.GetType() == t { - return condition - } +// toKubeRuntimeStatus converts the runtimeApi.RuntimeStatus to kubecontainer.RuntimeStatus. +func toKubeRuntimeStatus(status *runtimeApi.RuntimeStatus) *kubecontainer.RuntimeStatus { + conditions := []kubecontainer.RuntimeCondition{} + for _, c := range status.GetConditions() { + conditions = append(conditions, kubecontainer.RuntimeCondition{ + Type: kubecontainer.RuntimeConditionType(c.GetType()), + Status: c.GetStatus(), + Reason: c.GetReason(), + Message: c.GetMessage(), + }) } - return nil + return &kubecontainer.RuntimeStatus{Conditions: conditions} } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index d3db63f989a..8fc9dea78e0 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -268,25 +268,14 @@ func (m *kubeGenericRuntimeManager) APIVersion() (kubecontainer.Version, error) return newRuntimeVersion(typedVersion.GetRuntimeApiVersion()) } -// Status returns error if the runtime is unhealthy; nil otherwise. -// TODO(random-liu): Change the Status in runtime interface to return runtime status. -// TODO(random-liu): Add unit test for this function after addressing the TODO above. -func (m *kubeGenericRuntimeManager) Status() error { +// Status returns the status of the runtime. An error is returned if the Status +// function itself fails, nil otherwise. +func (m *kubeGenericRuntimeManager) Status() (*kubecontainer.RuntimeStatus, error) { status, err := m.runtimeService.Status() if err != nil { - return fmt.Errorf("failed to checkout runtime status: %v", err) + return nil, err } - networkReady := getRuntimeCondition(status, runtimeApi.NetworkReady) - if networkReady == nil || !networkReady.GetStatus() { - return fmt.Errorf("runtime network not ready: reason: %q, message: %q", - networkReady.GetReason(), networkReady.GetMessage()) - } - runtimeReady := getRuntimeCondition(status, runtimeApi.RuntimeReady) - if runtimeReady == nil || !runtimeReady.GetStatus() { - return fmt.Errorf("runtime not ready: reason: %q, message: %q", - runtimeReady.GetReason(), runtimeReady.GetMessage()) - } - return nil + return toKubeRuntimeStatus(status), nil } // GetPods returns a list of containers grouped by pods. The boolean parameter diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 62c9172c8ab..b39f26855e0 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -1701,8 +1701,8 @@ func (r *Runtime) APIVersion() (kubecontainer.Version, error) { } // Status returns error if rkt is unhealthy, nil otherwise. -func (r *Runtime) Status() error { - return r.checkVersion(minimumRktBinVersion, minimumRktApiVersion, minimumSystemdVersion) +func (r *Runtime) Status() (*kubecontainer.RuntimeStatus, error) { + return nil, r.checkVersion(minimumRktBinVersion, minimumRktApiVersion, minimumSystemdVersion) } // SyncPod syncs the running pod to match the specified desired pod.