diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 187d8730147..477acc14db1 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -55,6 +55,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/server" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -183,6 +184,11 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { return nil, err } + thresholds, err := eviction.ParseThresholdConfig(s.EvictionHard, s.EvictionSoft, s.EvictionSoftGracePeriod) + if err != nil { + return nil, err + } + return &KubeletConfig{ Address: net.ParseIP(s.Address), AllowPrivileged: s.AllowPrivileged, @@ -260,7 +266,8 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) { HairpinMode: s.HairpinMode, BabysitDaemons: s.BabysitDaemons, ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay, - NodeIP: net.ParseIP(s.NodeIP), + NodeIP: net.ParseIP(s.NodeIP), + Thresholds: thresholds, }, nil } @@ -773,6 +780,7 @@ type KubeletConfig struct { Writer io.Writer VolumePlugins []volume.VolumePlugin OutOfDiskTransitionFrequency time.Duration + Thresholds []eviction.Threshold ExperimentalFlannelOverlay bool NodeIP net.IP @@ -868,6 +876,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.ContainerRuntimeOptions, kc.HairpinMode, kc.BabysitDaemons, + kc.Thresholds, kc.Options, ) diff --git a/pkg/kubelet/eviction/doc.go b/pkg/kubelet/eviction/doc.go new file mode 100644 index 00000000000..d46bb277968 --- /dev/null +++ b/pkg/kubelet/eviction/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package eviction is responsible for enforcing eviction thresholds to maintain +// node stability. +package eviction diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go new file mode 100644 index 00000000000..4b689c967b6 --- /dev/null +++ b/pkg/kubelet/eviction/helpers.go @@ -0,0 +1,164 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eviction + +import ( + "fmt" + "strings" + "time" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/util/sets" +) + +const ( + unsupportedEvictionSignal = "unsupported eviction signal %v" +) + +// signalToResource maps a Signal to its associated Resource. +var signalToResource = map[Signal]api.ResourceName{ + SignalMemoryAvailable: api.ResourceMemory, +} + +// validSignal returns true if the signal is supported. +func validSignal(signal Signal) bool { + _, found := signalToResource[signal] + return found +} + +// ParseThresholdConfig parses the flags for thresholds. +func ParseThresholdConfig(evictionHard, evictionSoft, evictionSoftGracePeriod string) ([]Threshold, error) { + results := []Threshold{} + + hardThresholds, err := parseThresholdStatements(evictionHard) + if err != nil { + return nil, err + } + results = append(results, hardThresholds...) + + softThresholds, err := parseThresholdStatements(evictionSoft) + if err != nil { + return nil, err + } + gracePeriods, err := parseGracePeriods(evictionSoftGracePeriod) + if err != nil { + return nil, err + } + for i := range softThresholds { + signal := softThresholds[i].Signal + period, found := gracePeriods[signal] + if !found { + return nil, fmt.Errorf("grace period must be specified for the soft eviction threshold %v", signal) + } + softThresholds[i].GracePeriod = period + } + results = append(results, softThresholds...) + return results, nil +} + +// parseThresholdStatements parses the input statements into a list of Threshold objects. +func parseThresholdStatements(expr string) ([]Threshold, error) { + if len(expr) == 0 { + return nil, nil + } + results := []Threshold{} + statements := strings.Split(expr, ",") + signalsFound := sets.NewString() + for _, statement := range statements { + result, err := parseThresholdStatement(statement) + if err != nil { + return nil, err + } + if signalsFound.Has(string(result.Signal)) { + return nil, fmt.Errorf("found duplicate eviction threshold for signal %v", result.Signal) + } + signalsFound.Insert(string(result.Signal)) + results = append(results, result) + } + return results, nil +} + +// parseThresholdStatement parses a threshold statement. +func parseThresholdStatement(statement string) (Threshold, error) { + tokens2Operator := map[string]ThresholdOperator{ + "<": OpLessThan, + } + var ( + operator ThresholdOperator + parts []string + ) + for token := range tokens2Operator { + parts = strings.Split(statement, token) + // if we got a token, we know this was the operator... + if len(parts) > 1 { + operator = tokens2Operator[token] + break + } + } + if len(operator) == 0 || len(parts) != 2 { + return Threshold{}, fmt.Errorf("invalid eviction threshold syntax %v, expected ", statement) + } + signal := Signal(parts[0]) + if !validSignal(signal) { + return Threshold{}, fmt.Errorf(unsupportedEvictionSignal, signal) + } + + quantity, err := resource.ParseQuantity(parts[1]) + if err != nil { + return Threshold{}, err + } + return Threshold{ + Signal: signal, + Operator: operator, + Value: *quantity, + }, nil +} + +// parseGracePeriods parses the grace period statements +func parseGracePeriods(expr string) (map[Signal]time.Duration, error) { + if len(expr) == 0 { + return nil, nil + } + results := map[Signal]time.Duration{} + statements := strings.Split(expr, ",") + for _, statement := range statements { + parts := strings.Split(statement, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid eviction grace period syntax %v, expected =", statement) + } + signal := Signal(parts[0]) + if !validSignal(signal) { + return nil, fmt.Errorf(unsupportedEvictionSignal, signal) + } + + gracePeriod, err := time.ParseDuration(parts[1]) + if err != nil { + return nil, err + } + if gracePeriod < 0 { + return nil, fmt.Errorf("invalid eviction grace period specified: %v, must be a positive value", parts[1]) + } + + // check against duplicate statements + if _, found := results[signal]; found { + return nil, fmt.Errorf("duplicate eviction grace period specified for %v", signal) + } + results[signal] = gracePeriod + } + return results, nil +} diff --git a/pkg/kubelet/eviction/helpers_test.go b/pkg/kubelet/eviction/helpers_test.go new file mode 100644 index 00000000000..ad0983a2195 --- /dev/null +++ b/pkg/kubelet/eviction/helpers_test.go @@ -0,0 +1,142 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eviction + +import ( + "testing" + "time" + + "k8s.io/kubernetes/pkg/api/resource" +) + +func TestParseThresholdConfig(t *testing.T) { + gracePeriod, _ := time.ParseDuration("30s") + testCases := map[string]struct { + evictionHard string + evictionSoft string + evictionSoftGracePeriod string + expectErr bool + expectThresholds []Threshold + }{ + "no values": { + evictionHard: "", + evictionSoft: "", + evictionSoftGracePeriod: "", + expectErr: false, + expectThresholds: []Threshold{}, + }, + "all flag values": { + evictionHard: "memory.available<150Mi", + evictionSoft: "memory.available<300Mi", + evictionSoftGracePeriod: "memory.available=30s", + expectErr: false, + expectThresholds: []Threshold{ + { + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: resource.MustParse("150Mi"), + }, + { + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: resource.MustParse("300Mi"), + GracePeriod: gracePeriod, + }, + }, + }, + "invalid-signal": { + evictionHard: "mem.available<150Mi", + evictionSoft: "", + evictionSoftGracePeriod: "", + expectErr: true, + expectThresholds: []Threshold{}, + }, + "duplicate-signal": { + evictionHard: "memory.available<150Mi,memory.available<100Mi", + evictionSoft: "", + evictionSoftGracePeriod: "", + expectErr: true, + expectThresholds: []Threshold{}, + }, + "valid-and-invalid-signal": { + evictionHard: "memory.available<150Mi,invalid.foo<150Mi", + evictionSoft: "", + evictionSoftGracePeriod: "", + expectErr: true, + expectThresholds: []Threshold{}, + }, + "soft-no-grace-period": { + evictionHard: "", + evictionSoft: "memory.available<150Mi", + evictionSoftGracePeriod: "", + expectErr: true, + expectThresholds: []Threshold{}, + }, + "soft-neg-grace-period": { + evictionHard: "", + evictionSoft: "memory.available<150Mi", + evictionSoftGracePeriod: "memory.available=-30s", + expectErr: true, + expectThresholds: []Threshold{}, + }, + } + for testName, testCase := range testCases { + thresholds, err := ParseThresholdConfig(testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod) + if testCase.expectErr != (err != nil) { + t.Errorf("Err not as expected, test: %v, error expected: %v, actual: %v", testName, testCase.expectErr, err) + } + if !thresholdsEqual(testCase.expectThresholds, thresholds) { + t.Errorf("thresholds not as expected, test: %v, expected: %v, actual: %v", testName, testCase.expectThresholds, thresholds) + } + } +} + +func thresholdsEqual(expected []Threshold, actual []Threshold) bool { + if len(expected) != len(actual) { + return false + } + for _, aThreshold := range expected { + equal := false + for _, bThreshold := range actual { + if thresholdEqual(aThreshold, bThreshold) { + equal = true + } + } + if !equal { + return false + } + } + for _, aThreshold := range actual { + equal := false + for _, bThreshold := range expected { + if thresholdEqual(aThreshold, bThreshold) { + equal = true + } + } + if !equal { + return false + } + } + return true +} + +func thresholdEqual(a Threshold, b Threshold) bool { + return a.GracePeriod == b.GracePeriod && + a.Operator == b.Operator && + a.Signal == b.Signal && + a.Value.Cmp(b.Value) == 0 +} diff --git a/pkg/kubelet/eviction/types.go b/pkg/kubelet/eviction/types.go new file mode 100644 index 00000000000..63d39fe875f --- /dev/null +++ b/pkg/kubelet/eviction/types.go @@ -0,0 +1,51 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package eviction + +import ( + "time" + + "k8s.io/kubernetes/pkg/api/resource" +) + +// Signal defines a signal that can trigger eviction of pods on a node. +type Signal string + +const ( + // SignalMemoryAvailable is memory available (i.e. capacity - workingSet), in bytes. + SignalMemoryAvailable Signal = "memory.available" +) + +// ThresholdOperator is the operator used to express a Threshold. +type ThresholdOperator string + +const ( + // OpLessThan is the operator that expresses a less than operator. + OpLessThan ThresholdOperator = "LessThan" +) + +// Threshold defines a metric for when eviction should occur. +type Threshold struct { + // Signal defines the entity that was measured. + Signal Signal + // Operator represents a relationship of a signal to a value. + Operator ThresholdOperator + // value is a quantity associated with the signal that is evaluated against the specified operator. + Value resource.Quantity + // GracePeriod represents the amount of time that a threshold must be met before eviction is triggered. + GracePeriod time.Duration +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 2d7afceba82..70e72d09fcb 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -52,6 +52,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/envvars" + "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/network" @@ -220,6 +221,7 @@ func NewMainKubelet( containerRuntimeOptions []kubecontainer.Option, hairpinMode string, babysitDaemons bool, + thresholds []eviction.Threshold, kubeOptions []Option, ) (*Kubelet, error) { if rootDirectory == "" { @@ -927,6 +929,7 @@ func (kl *Kubelet) initializeModules() error { // Step 7: Start resource analyzer kl.resourceAnalyzer.Start() + return nil } @@ -965,6 +968,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) { // Start component sync loops. kl.statusManager.Start() kl.probeManager.Start() + // Start the pod lifecycle event generator. kl.pleg.Start() kl.syncLoop(updates, kl)