Migrate oom watcher not relying on cAdviosr any more, it is part of plan

of removing cAdvisor from k8s.
For more informations about this plan, please refer:
https://github.com/kubernetes/kubernetes/issues/68522
This commit is contained in:
WanLinghao 2019-03-05 15:21:29 +08:00
parent 6af2e9afea
commit b2ce446a88
7 changed files with 167 additions and 34 deletions

View File

@ -21,7 +21,6 @@ go_library(
"kubelet_pods.go", "kubelet_pods.go",
"kubelet_resources.go", "kubelet_resources.go",
"kubelet_volumes.go", "kubelet_volumes.go",
"oom_watcher.go",
"pod_container_deletor.go", "pod_container_deletor.go",
"pod_workers.go", "pod_workers.go",
"reason_cache.go", "reason_cache.go",
@ -72,6 +71,7 @@ go_library(
"//pkg/kubelet/network/dns:go_default_library", "//pkg/kubelet/network/dns:go_default_library",
"//pkg/kubelet/nodelease:go_default_library", "//pkg/kubelet/nodelease:go_default_library",
"//pkg/kubelet/nodestatus:go_default_library", "//pkg/kubelet/nodestatus:go_default_library",
"//pkg/kubelet/oom:go_default_library",
"//pkg/kubelet/pleg:go_default_library", "//pkg/kubelet/pleg:go_default_library",
"//pkg/kubelet/pod:go_default_library", "//pkg/kubelet/pod:go_default_library",
"//pkg/kubelet/preemption:go_default_library", "//pkg/kubelet/preemption:go_default_library",
@ -144,7 +144,6 @@ go_library(
"//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
"//third_party/forked/golang/expansion:go_default_library", "//third_party/forked/golang/expansion:go_default_library",
"//vendor/github.com/golang/groupcache/lru:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library",
"//vendor/github.com/google/cadvisor/events:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
"//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/klog:go_default_library",
@ -168,7 +167,6 @@ go_test(
"kubelet_test.go", "kubelet_test.go",
"kubelet_volumes_linux_test.go", "kubelet_volumes_linux_test.go",
"kubelet_volumes_test.go", "kubelet_volumes_test.go",
"oom_watcher_test.go",
"pod_container_deletor_test.go", "pod_container_deletor_test.go",
"pod_workers_test.go", "pod_workers_test.go",
"reason_cache_test.go", "reason_cache_test.go",
@ -297,6 +295,7 @@ filegroup(
"//pkg/kubelet/network:all-srcs", "//pkg/kubelet/network:all-srcs",
"//pkg/kubelet/nodelease:all-srcs", "//pkg/kubelet/nodelease:all-srcs",
"//pkg/kubelet/nodestatus:all-srcs", "//pkg/kubelet/nodestatus:all-srcs",
"//pkg/kubelet/oom:all-srcs",
"//pkg/kubelet/pleg:all-srcs", "//pkg/kubelet/pleg:all-srcs",
"//pkg/kubelet/pod:all-srcs", "//pkg/kubelet/pod:all-srcs",
"//pkg/kubelet/preemption:all-srcs", "//pkg/kubelet/preemption:all-srcs",

View File

@ -80,6 +80,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors" "k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
"k8s.io/kubernetes/pkg/kubelet/network/dns" "k8s.io/kubernetes/pkg/kubelet/network/dns"
"k8s.io/kubernetes/pkg/kubelet/nodelease" "k8s.io/kubernetes/pkg/kubelet/nodelease"
oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom"
"k8s.io/kubernetes/pkg/kubelet/pleg" "k8s.io/kubernetes/pkg/kubelet/pleg"
kubepod "k8s.io/kubernetes/pkg/kubelet/pod" kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
"k8s.io/kubernetes/pkg/kubelet/preemption" "k8s.io/kubernetes/pkg/kubelet/preemption"
@ -469,7 +470,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
containerRefManager := kubecontainer.NewRefManager() containerRefManager := kubecontainer.NewRefManager()
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder) oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder)
clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS)) clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
for _, ipEntry := range kubeCfg.ClusterDNS { for _, ipEntry := range kubeCfg.ClusterDNS {
@ -1088,7 +1089,7 @@ type Kubelet struct {
os kubecontainer.OSInterface os kubecontainer.OSInterface
// Watcher of out of memory events. // Watcher of out of memory events.
oomWatcher OOMWatcher oomWatcher oomwatcher.OOMWatcher
// Monitor resource usage // Monitor resource usage
resourceAnalyzer serverstats.ResourceAnalyzer resourceAnalyzer serverstats.ResourceAnalyzer

82
pkg/kubelet/oom/BUILD Normal file
View File

@ -0,0 +1,82 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"oom_watcher_linux.go",
"oom_watcher_unsupported.go",
"types.go",
],
importpath = "k8s.io/kubernetes/pkg/kubelet/oom",
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
],
"//conditions:default": [],
}),
)
go_test(
name = "go_default_test",
srcs = ["oom_watcher_linux_test.go"],
embed = [":go_default_library"],
deps = select({
"@io_bazel_rules_go//go/platform:linux": [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -1,3 +1,5 @@
// +build linux
/* /*
Copyright 2015 The Kubernetes Authors. Copyright 2015 The Kubernetes Authors.
@ -14,61 +16,52 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package kubelet package oom
import ( import (
"github.com/google/cadvisor/events"
cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
"k8s.io/klog" "k8s.io/klog"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"github.com/google/cadvisor/utils/oomparser"
) )
// OOMWatcher defines the interface of OOM watchers.
type OOMWatcher interface {
Start(ref *v1.ObjectReference) error
}
type realOOMWatcher struct { type realOOMWatcher struct {
cadvisor cadvisor.Interface
recorder record.EventRecorder recorder record.EventRecorder
} }
var _ OOMWatcher = &realOOMWatcher{}
// NewOOMWatcher creates and initializes a OOMWatcher based on parameters. // NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher { func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher {
return &realOOMWatcher{ return &realOOMWatcher{
cadvisor: cadvisor,
recorder: recorder, recorder: recorder,
} }
} }
const systemOOMEvent = "SystemOOM" const systemOOMEvent = "SystemOOM"
// Watches cadvisor for system oom's and records an event for every system oom encountered. // Watches for system oom's and records an event for every system oom encountered.
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error { func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
request := events.Request{ oomLog, err := oomparser.New()
EventType: map[cadvisorapi.EventType]bool{
cadvisorapi.EventOom: true,
},
ContainerName: "/",
IncludeSubcontainers: false,
}
eventChannel, err := ow.cadvisor.WatchEvents(&request)
if err != nil { if err != nil {
return err return err
} }
outStream := make(chan *oomparser.OomInstance, 10)
go oomLog.StreamOoms(outStream)
go func() { go func() {
defer runtime.HandleCrash() defer runtime.HandleCrash()
for event := range eventChannel.GetChannel() { for event := range outStream {
klog.V(2).Infof("Got sys oom event from cadvisor: %v", event) if event.ContainerName == "/" {
ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered") klog.V(1).Infof("Got sys oom event: %v", event)
ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
} }
klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor") }
klog.Errorf("Unexpectedly stopped receiving OOM notifications")
}() }()
return nil return nil
} }

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package kubelet package oom
import ( import (
"testing" "testing"
@ -23,14 +23,12 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
) )
func TestBasic(t *testing.T) { func TestBasic(t *testing.T) {
fakeRecorder := &record.FakeRecorder{} fakeRecorder := &record.FakeRecorder{}
mockCadvisor := &cadvisortest.Fake{}
node := &v1.ObjectReference{} node := &v1.ObjectReference{}
oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder) oomWatcher := NewOOMWatcher(fakeRecorder)
assert.NoError(t, oomWatcher.Start(node)) assert.NoError(t, oomWatcher.Start(node))
// TODO: Improve this test once cadvisor exports events.EventChannel as an interface // TODO: Improve this test once cadvisor exports events.EventChannel as an interface

View File

@ -0,0 +1,35 @@
// +build !linux
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oom
import (
"k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
)
type oomWatcherUnsupported struct{}
var _ OOMWatcher = new(oomWatcherUnsupported)
func NewOOMWatcher(_ record.EventRecorder) OOMWatcher {
return &oomWatcherUnsupported{}
}
func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error {
return nil
}

25
pkg/kubelet/oom/types.go Normal file
View File

@ -0,0 +1,25 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oom
import (
"k8s.io/api/core/v1"
)
// OOMWatcher defines the interface of OOM watchers.
type OOMWatcher interface {
Start(ref *v1.ObjectReference) error
}