mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-31 23:37:01 +00:00
Migrate oom watcher not relying on cAdviosr any more, it is part of plan
of removing cAdvisor from k8s. For more informations about this plan, please refer: https://github.com/kubernetes/kubernetes/issues/68522
This commit is contained in:
parent
6af2e9afea
commit
b2ce446a88
@ -21,7 +21,6 @@ go_library(
|
|||||||
"kubelet_pods.go",
|
"kubelet_pods.go",
|
||||||
"kubelet_resources.go",
|
"kubelet_resources.go",
|
||||||
"kubelet_volumes.go",
|
"kubelet_volumes.go",
|
||||||
"oom_watcher.go",
|
|
||||||
"pod_container_deletor.go",
|
"pod_container_deletor.go",
|
||||||
"pod_workers.go",
|
"pod_workers.go",
|
||||||
"reason_cache.go",
|
"reason_cache.go",
|
||||||
@ -72,6 +71,7 @@ go_library(
|
|||||||
"//pkg/kubelet/network/dns:go_default_library",
|
"//pkg/kubelet/network/dns:go_default_library",
|
||||||
"//pkg/kubelet/nodelease:go_default_library",
|
"//pkg/kubelet/nodelease:go_default_library",
|
||||||
"//pkg/kubelet/nodestatus:go_default_library",
|
"//pkg/kubelet/nodestatus:go_default_library",
|
||||||
|
"//pkg/kubelet/oom:go_default_library",
|
||||||
"//pkg/kubelet/pleg:go_default_library",
|
"//pkg/kubelet/pleg:go_default_library",
|
||||||
"//pkg/kubelet/pod:go_default_library",
|
"//pkg/kubelet/pod:go_default_library",
|
||||||
"//pkg/kubelet/preemption:go_default_library",
|
"//pkg/kubelet/preemption:go_default_library",
|
||||||
@ -144,7 +144,6 @@ go_library(
|
|||||||
"//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
|
"//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
|
||||||
"//third_party/forked/golang/expansion:go_default_library",
|
"//third_party/forked/golang/expansion:go_default_library",
|
||||||
"//vendor/github.com/golang/groupcache/lru:go_default_library",
|
"//vendor/github.com/golang/groupcache/lru:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/events:go_default_library",
|
|
||||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
|
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
|
||||||
"//vendor/k8s.io/klog:go_default_library",
|
"//vendor/k8s.io/klog:go_default_library",
|
||||||
@ -168,7 +167,6 @@ go_test(
|
|||||||
"kubelet_test.go",
|
"kubelet_test.go",
|
||||||
"kubelet_volumes_linux_test.go",
|
"kubelet_volumes_linux_test.go",
|
||||||
"kubelet_volumes_test.go",
|
"kubelet_volumes_test.go",
|
||||||
"oom_watcher_test.go",
|
|
||||||
"pod_container_deletor_test.go",
|
"pod_container_deletor_test.go",
|
||||||
"pod_workers_test.go",
|
"pod_workers_test.go",
|
||||||
"reason_cache_test.go",
|
"reason_cache_test.go",
|
||||||
@ -297,6 +295,7 @@ filegroup(
|
|||||||
"//pkg/kubelet/network:all-srcs",
|
"//pkg/kubelet/network:all-srcs",
|
||||||
"//pkg/kubelet/nodelease:all-srcs",
|
"//pkg/kubelet/nodelease:all-srcs",
|
||||||
"//pkg/kubelet/nodestatus:all-srcs",
|
"//pkg/kubelet/nodestatus:all-srcs",
|
||||||
|
"//pkg/kubelet/oom:all-srcs",
|
||||||
"//pkg/kubelet/pleg:all-srcs",
|
"//pkg/kubelet/pleg:all-srcs",
|
||||||
"//pkg/kubelet/pod:all-srcs",
|
"//pkg/kubelet/pod:all-srcs",
|
||||||
"//pkg/kubelet/preemption:all-srcs",
|
"//pkg/kubelet/preemption:all-srcs",
|
||||||
|
@ -80,6 +80,7 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
|
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/network/dns"
|
"k8s.io/kubernetes/pkg/kubelet/network/dns"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/nodelease"
|
"k8s.io/kubernetes/pkg/kubelet/nodelease"
|
||||||
|
oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/pleg"
|
"k8s.io/kubernetes/pkg/kubelet/pleg"
|
||||||
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
|
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/preemption"
|
"k8s.io/kubernetes/pkg/kubelet/preemption"
|
||||||
@ -469,7 +470,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
|
|||||||
|
|
||||||
containerRefManager := kubecontainer.NewRefManager()
|
containerRefManager := kubecontainer.NewRefManager()
|
||||||
|
|
||||||
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
|
oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder)
|
||||||
|
|
||||||
clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
|
clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
|
||||||
for _, ipEntry := range kubeCfg.ClusterDNS {
|
for _, ipEntry := range kubeCfg.ClusterDNS {
|
||||||
@ -1088,7 +1089,7 @@ type Kubelet struct {
|
|||||||
os kubecontainer.OSInterface
|
os kubecontainer.OSInterface
|
||||||
|
|
||||||
// Watcher of out of memory events.
|
// Watcher of out of memory events.
|
||||||
oomWatcher OOMWatcher
|
oomWatcher oomwatcher.OOMWatcher
|
||||||
|
|
||||||
// Monitor resource usage
|
// Monitor resource usage
|
||||||
resourceAnalyzer serverstats.ResourceAnalyzer
|
resourceAnalyzer serverstats.ResourceAnalyzer
|
||||||
|
82
pkg/kubelet/oom/BUILD
Normal file
82
pkg/kubelet/oom/BUILD
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||||
|
|
||||||
|
go_library(
|
||||||
|
name = "go_default_library",
|
||||||
|
srcs = [
|
||||||
|
"oom_watcher_linux.go",
|
||||||
|
"oom_watcher_unsupported.go",
|
||||||
|
"types.go",
|
||||||
|
],
|
||||||
|
importpath = "k8s.io/kubernetes/pkg/kubelet/oom",
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
|
] + select({
|
||||||
|
"@io_bazel_rules_go//go/platform:android": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:darwin": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:dragonfly": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:freebsd": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:linux": [
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
"//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library",
|
||||||
|
"//vendor/k8s.io/klog:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:nacl": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:netbsd": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:openbsd": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:plan9": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:solaris": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"@io_bazel_rules_go//go/platform:windows": [
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
],
|
||||||
|
"//conditions:default": [],
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
go_test(
|
||||||
|
name = "go_default_test",
|
||||||
|
srcs = ["oom_watcher_linux_test.go"],
|
||||||
|
embed = [":go_default_library"],
|
||||||
|
deps = select({
|
||||||
|
"@io_bazel_rules_go//go/platform:linux": [
|
||||||
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
|
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||||
|
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||||
|
],
|
||||||
|
"//conditions:default": [],
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "package-srcs",
|
||||||
|
srcs = glob(["**"]),
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
filegroup(
|
||||||
|
name = "all-srcs",
|
||||||
|
srcs = [":package-srcs"],
|
||||||
|
tags = ["automanaged"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
@ -1,3 +1,5 @@
|
|||||||
|
// +build linux
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Copyright 2015 The Kubernetes Authors.
|
Copyright 2015 The Kubernetes Authors.
|
||||||
|
|
||||||
@ -14,61 +16,52 @@ See the License for the specific language governing permissions and
|
|||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package kubelet
|
package oom
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/google/cadvisor/events"
|
|
||||||
cadvisorapi "github.com/google/cadvisor/info/v1"
|
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
"k8s.io/apimachinery/pkg/util/runtime"
|
"k8s.io/apimachinery/pkg/util/runtime"
|
||||||
"k8s.io/client-go/tools/record"
|
"k8s.io/client-go/tools/record"
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
|
||||||
|
"github.com/google/cadvisor/utils/oomparser"
|
||||||
)
|
)
|
||||||
|
|
||||||
// OOMWatcher defines the interface of OOM watchers.
|
|
||||||
type OOMWatcher interface {
|
|
||||||
Start(ref *v1.ObjectReference) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type realOOMWatcher struct {
|
type realOOMWatcher struct {
|
||||||
cadvisor cadvisor.Interface
|
|
||||||
recorder record.EventRecorder
|
recorder record.EventRecorder
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var _ OOMWatcher = &realOOMWatcher{}
|
||||||
|
|
||||||
// NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
|
// NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
|
||||||
func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher {
|
func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher {
|
||||||
return &realOOMWatcher{
|
return &realOOMWatcher{
|
||||||
cadvisor: cadvisor,
|
|
||||||
recorder: recorder,
|
recorder: recorder,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const systemOOMEvent = "SystemOOM"
|
const systemOOMEvent = "SystemOOM"
|
||||||
|
|
||||||
// Watches cadvisor for system oom's and records an event for every system oom encountered.
|
// Watches for system oom's and records an event for every system oom encountered.
|
||||||
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
|
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
|
||||||
request := events.Request{
|
oomLog, err := oomparser.New()
|
||||||
EventType: map[cadvisorapi.EventType]bool{
|
|
||||||
cadvisorapi.EventOom: true,
|
|
||||||
},
|
|
||||||
ContainerName: "/",
|
|
||||||
IncludeSubcontainers: false,
|
|
||||||
}
|
|
||||||
eventChannel, err := ow.cadvisor.WatchEvents(&request)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
outStream := make(chan *oomparser.OomInstance, 10)
|
||||||
|
go oomLog.StreamOoms(outStream)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
defer runtime.HandleCrash()
|
defer runtime.HandleCrash()
|
||||||
|
|
||||||
for event := range eventChannel.GetChannel() {
|
for event := range outStream {
|
||||||
klog.V(2).Infof("Got sys oom event from cadvisor: %v", event)
|
if event.ContainerName == "/" {
|
||||||
ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
|
klog.V(1).Infof("Got sys oom event: %v", event)
|
||||||
|
ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
|
||||||
}
|
}
|
||||||
klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor")
|
}
|
||||||
|
klog.Errorf("Unexpectedly stopped receiving OOM notifications")
|
||||||
}()
|
}()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
|||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package kubelet
|
package oom
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
@ -23,14 +23,12 @@ import (
|
|||||||
|
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
"k8s.io/client-go/tools/record"
|
"k8s.io/client-go/tools/record"
|
||||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBasic(t *testing.T) {
|
func TestBasic(t *testing.T) {
|
||||||
fakeRecorder := &record.FakeRecorder{}
|
fakeRecorder := &record.FakeRecorder{}
|
||||||
mockCadvisor := &cadvisortest.Fake{}
|
|
||||||
node := &v1.ObjectReference{}
|
node := &v1.ObjectReference{}
|
||||||
oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder)
|
oomWatcher := NewOOMWatcher(fakeRecorder)
|
||||||
assert.NoError(t, oomWatcher.Start(node))
|
assert.NoError(t, oomWatcher.Start(node))
|
||||||
|
|
||||||
// TODO: Improve this test once cadvisor exports events.EventChannel as an interface
|
// TODO: Improve this test once cadvisor exports events.EventChannel as an interface
|
35
pkg/kubelet/oom/oom_watcher_unsupported.go
Normal file
35
pkg/kubelet/oom/oom_watcher_unsupported.go
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
// +build !linux
|
||||||
|
|
||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
package oom
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/api/core/v1"
|
||||||
|
"k8s.io/client-go/tools/record"
|
||||||
|
)
|
||||||
|
|
||||||
|
type oomWatcherUnsupported struct{}
|
||||||
|
|
||||||
|
var _ OOMWatcher = new(oomWatcherUnsupported)
|
||||||
|
|
||||||
|
func NewOOMWatcher(_ record.EventRecorder) OOMWatcher {
|
||||||
|
return &oomWatcherUnsupported{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error {
|
||||||
|
return nil
|
||||||
|
}
|
25
pkg/kubelet/oom/types.go
Normal file
25
pkg/kubelet/oom/types.go
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2019 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
package oom
|
||||||
|
|
||||||
|
import (
|
||||||
|
"k8s.io/api/core/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OOMWatcher defines the interface of OOM watchers.
|
||||||
|
type OOMWatcher interface {
|
||||||
|
Start(ref *v1.ObjectReference) error
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user