From b2ce446a8811ff65044a30173081df13696d1cce Mon Sep 17 00:00:00 2001 From: WanLinghao Date: Tue, 5 Mar 2019 15:21:29 +0800 Subject: [PATCH] Migrate oom watcher not relying on cAdviosr any more, it is part of plan of removing cAdvisor from k8s. For more informations about this plan, please refer: https://github.com/kubernetes/kubernetes/issues/68522 --- pkg/kubelet/BUILD | 5 +- pkg/kubelet/kubelet.go | 5 +- pkg/kubelet/oom/BUILD | 82 +++++++++++++++++++ .../oom_watcher_linux.go} | 43 ++++------ .../oom_watcher_linux_test.go} | 6 +- pkg/kubelet/oom/oom_watcher_unsupported.go | 35 ++++++++ pkg/kubelet/oom/types.go | 25 ++++++ 7 files changed, 167 insertions(+), 34 deletions(-) create mode 100644 pkg/kubelet/oom/BUILD rename pkg/kubelet/{oom_watcher.go => oom/oom_watcher_linux.go} (52%) rename pkg/kubelet/{oom_watcher_test.go => oom/oom_watcher_linux_test.go} (84%) create mode 100644 pkg/kubelet/oom/oom_watcher_unsupported.go create mode 100644 pkg/kubelet/oom/types.go diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index b824a8dbbb5..571d612a12f 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -21,7 +21,6 @@ go_library( "kubelet_pods.go", "kubelet_resources.go", "kubelet_volumes.go", - "oom_watcher.go", "pod_container_deletor.go", "pod_workers.go", "reason_cache.go", @@ -72,6 +71,7 @@ go_library( "//pkg/kubelet/network/dns:go_default_library", "//pkg/kubelet/nodelease:go_default_library", "//pkg/kubelet/nodestatus:go_default_library", + "//pkg/kubelet/oom:go_default_library", "//pkg/kubelet/pleg:go_default_library", "//pkg/kubelet/pod:go_default_library", "//pkg/kubelet/preemption:go_default_library", @@ -144,7 +144,6 @@ go_library( "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//third_party/forked/golang/expansion:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", - "//vendor/github.com/google/cadvisor/events:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/k8s.io/klog:go_default_library", @@ -168,7 +167,6 @@ go_test( "kubelet_test.go", "kubelet_volumes_linux_test.go", "kubelet_volumes_test.go", - "oom_watcher_test.go", "pod_container_deletor_test.go", "pod_workers_test.go", "reason_cache_test.go", @@ -297,6 +295,7 @@ filegroup( "//pkg/kubelet/network:all-srcs", "//pkg/kubelet/nodelease:all-srcs", "//pkg/kubelet/nodestatus:all-srcs", + "//pkg/kubelet/oom:all-srcs", "//pkg/kubelet/pleg:all-srcs", "//pkg/kubelet/pod:all-srcs", "//pkg/kubelet/preemption:all-srcs", diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 16d98618682..b33282bb7b2 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -80,6 +80,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics/collectors" "k8s.io/kubernetes/pkg/kubelet/network/dns" "k8s.io/kubernetes/pkg/kubelet/nodelease" + oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom" "k8s.io/kubernetes/pkg/kubelet/pleg" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" "k8s.io/kubernetes/pkg/kubelet/preemption" @@ -469,7 +470,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, containerRefManager := kubecontainer.NewRefManager() - oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder) + oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder) clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS)) for _, ipEntry := range kubeCfg.ClusterDNS { @@ -1088,7 +1089,7 @@ type Kubelet struct { os kubecontainer.OSInterface // Watcher of out of memory events. - oomWatcher OOMWatcher + oomWatcher oomwatcher.OOMWatcher // Monitor resource usage resourceAnalyzer serverstats.ResourceAnalyzer diff --git a/pkg/kubelet/oom/BUILD b/pkg/kubelet/oom/BUILD new file mode 100644 index 00000000000..0e852f1fce7 --- /dev/null +++ b/pkg/kubelet/oom/BUILD @@ -0,0 +1,82 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "oom_watcher_linux.go", + "oom_watcher_unsupported.go", + "types.go", + ], + importpath = "k8s.io/kubernetes/pkg/kubelet/oom", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:darwin": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:dragonfly": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:freebsd": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + "//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], + "@io_bazel_rules_go//go/platform:nacl": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:netbsd": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:openbsd": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:plan9": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:solaris": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows": [ + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = ["oom_watcher_linux_test.go"], + embed = [":go_default_library"], + deps = select({ + "@io_bazel_rules_go//go/platform:linux": [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/tools/record:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], + "//conditions:default": [], + }), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubelet/oom_watcher.go b/pkg/kubelet/oom/oom_watcher_linux.go similarity index 52% rename from pkg/kubelet/oom_watcher.go rename to pkg/kubelet/oom/oom_watcher_linux.go index 1ca014b4bab..77c75bf65cb 100644 --- a/pkg/kubelet/oom_watcher.go +++ b/pkg/kubelet/oom/oom_watcher_linux.go @@ -1,3 +1,5 @@ +// +build linux + /* Copyright 2015 The Kubernetes Authors. @@ -14,61 +16,52 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubelet +package oom import ( - "github.com/google/cadvisor/events" - cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/record" "k8s.io/klog" - "k8s.io/kubernetes/pkg/kubelet/cadvisor" + + "github.com/google/cadvisor/utils/oomparser" ) -// OOMWatcher defines the interface of OOM watchers. -type OOMWatcher interface { - Start(ref *v1.ObjectReference) error -} - type realOOMWatcher struct { - cadvisor cadvisor.Interface recorder record.EventRecorder } +var _ OOMWatcher = &realOOMWatcher{} + // NewOOMWatcher creates and initializes a OOMWatcher based on parameters. -func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher { +func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher { return &realOOMWatcher{ - cadvisor: cadvisor, recorder: recorder, } } const systemOOMEvent = "SystemOOM" -// Watches cadvisor for system oom's and records an event for every system oom encountered. +// Watches for system oom's and records an event for every system oom encountered. func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error { - request := events.Request{ - EventType: map[cadvisorapi.EventType]bool{ - cadvisorapi.EventOom: true, - }, - ContainerName: "/", - IncludeSubcontainers: false, - } - eventChannel, err := ow.cadvisor.WatchEvents(&request) + oomLog, err := oomparser.New() if err != nil { return err } + outStream := make(chan *oomparser.OomInstance, 10) + go oomLog.StreamOoms(outStream) go func() { defer runtime.HandleCrash() - for event := range eventChannel.GetChannel() { - klog.V(2).Infof("Got sys oom event from cadvisor: %v", event) - ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered") + for event := range outStream { + if event.ContainerName == "/" { + klog.V(1).Infof("Got sys oom event: %v", event) + ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered") + } } - klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor") + klog.Errorf("Unexpectedly stopped receiving OOM notifications") }() return nil } diff --git a/pkg/kubelet/oom_watcher_test.go b/pkg/kubelet/oom/oom_watcher_linux_test.go similarity index 84% rename from pkg/kubelet/oom_watcher_test.go rename to pkg/kubelet/oom/oom_watcher_linux_test.go index 6fd0287e571..5aba6e41424 100644 --- a/pkg/kubelet/oom_watcher_test.go +++ b/pkg/kubelet/oom/oom_watcher_linux_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubelet +package oom import ( "testing" @@ -23,14 +23,12 @@ import ( "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" - cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" ) func TestBasic(t *testing.T) { fakeRecorder := &record.FakeRecorder{} - mockCadvisor := &cadvisortest.Fake{} node := &v1.ObjectReference{} - oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder) + oomWatcher := NewOOMWatcher(fakeRecorder) assert.NoError(t, oomWatcher.Start(node)) // TODO: Improve this test once cadvisor exports events.EventChannel as an interface diff --git a/pkg/kubelet/oom/oom_watcher_unsupported.go b/pkg/kubelet/oom/oom_watcher_unsupported.go new file mode 100644 index 00000000000..aea98ae71ca --- /dev/null +++ b/pkg/kubelet/oom/oom_watcher_unsupported.go @@ -0,0 +1,35 @@ +// +build !linux + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package oom + +import ( + "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" +) + +type oomWatcherUnsupported struct{} + +var _ OOMWatcher = new(oomWatcherUnsupported) + +func NewOOMWatcher(_ record.EventRecorder) OOMWatcher { + return &oomWatcherUnsupported{} +} + +func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error { + return nil +} diff --git a/pkg/kubelet/oom/types.go b/pkg/kubelet/oom/types.go new file mode 100644 index 00000000000..222ef8d9830 --- /dev/null +++ b/pkg/kubelet/oom/types.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package oom + +import ( + "k8s.io/api/core/v1" +) + +// OOMWatcher defines the interface of OOM watchers. +type OOMWatcher interface { + Start(ref *v1.ObjectReference) error +}