Merge pull request #24750 from derekwaynecarr/kubelet_eviction_flag_parsing

Automatic merge from submit-queue

Kubelet eviction flag parsers and tests

The first two commits are from https://github.com/kubernetes/kubernetes/pull/24559 that have achieved LGTM.  

The last commit is only part that is interesting, it adds the parsing logic to handle the flags, and reserves `pkg/kubelet/eviction` for eviction manager logic.
This commit is contained in:
k8s-merge-robot 2016-05-09 04:15:04 -07:00
commit 2cf511b1f5
6 changed files with 390 additions and 1 deletions

View File

@ -55,6 +55,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/eviction"
"k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/kubelet/server"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
@ -183,6 +184,11 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
return nil, err
}
thresholds, err := eviction.ParseThresholdConfig(s.EvictionHard, s.EvictionSoft, s.EvictionSoftGracePeriod)
if err != nil {
return nil, err
}
return &KubeletConfig{
Address: net.ParseIP(s.Address),
AllowPrivileged: s.AllowPrivileged,
@ -260,7 +266,8 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
HairpinMode: s.HairpinMode,
BabysitDaemons: s.BabysitDaemons,
ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay,
NodeIP: net.ParseIP(s.NodeIP),
NodeIP: net.ParseIP(s.NodeIP),
Thresholds: thresholds,
}, nil
}
@ -773,6 +780,7 @@ type KubeletConfig struct {
Writer io.Writer
VolumePlugins []volume.VolumePlugin
OutOfDiskTransitionFrequency time.Duration
Thresholds []eviction.Threshold
ExperimentalFlannelOverlay bool
NodeIP net.IP
@ -868,6 +876,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
kc.ContainerRuntimeOptions,
kc.HairpinMode,
kc.BabysitDaemons,
kc.Thresholds,
kc.Options,
)

View File

@ -0,0 +1,19 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package eviction is responsible for enforcing eviction thresholds to maintain
// node stability.
package eviction

View File

@ -0,0 +1,164 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package eviction
import (
"fmt"
"strings"
"time"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/util/sets"
)
const (
unsupportedEvictionSignal = "unsupported eviction signal %v"
)
// signalToResource maps a Signal to its associated Resource.
var signalToResource = map[Signal]api.ResourceName{
SignalMemoryAvailable: api.ResourceMemory,
}
// validSignal returns true if the signal is supported.
func validSignal(signal Signal) bool {
_, found := signalToResource[signal]
return found
}
// ParseThresholdConfig parses the flags for thresholds.
func ParseThresholdConfig(evictionHard, evictionSoft, evictionSoftGracePeriod string) ([]Threshold, error) {
results := []Threshold{}
hardThresholds, err := parseThresholdStatements(evictionHard)
if err != nil {
return nil, err
}
results = append(results, hardThresholds...)
softThresholds, err := parseThresholdStatements(evictionSoft)
if err != nil {
return nil, err
}
gracePeriods, err := parseGracePeriods(evictionSoftGracePeriod)
if err != nil {
return nil, err
}
for i := range softThresholds {
signal := softThresholds[i].Signal
period, found := gracePeriods[signal]
if !found {
return nil, fmt.Errorf("grace period must be specified for the soft eviction threshold %v", signal)
}
softThresholds[i].GracePeriod = period
}
results = append(results, softThresholds...)
return results, nil
}
// parseThresholdStatements parses the input statements into a list of Threshold objects.
func parseThresholdStatements(expr string) ([]Threshold, error) {
if len(expr) == 0 {
return nil, nil
}
results := []Threshold{}
statements := strings.Split(expr, ",")
signalsFound := sets.NewString()
for _, statement := range statements {
result, err := parseThresholdStatement(statement)
if err != nil {
return nil, err
}
if signalsFound.Has(string(result.Signal)) {
return nil, fmt.Errorf("found duplicate eviction threshold for signal %v", result.Signal)
}
signalsFound.Insert(string(result.Signal))
results = append(results, result)
}
return results, nil
}
// parseThresholdStatement parses a threshold statement.
func parseThresholdStatement(statement string) (Threshold, error) {
tokens2Operator := map[string]ThresholdOperator{
"<": OpLessThan,
}
var (
operator ThresholdOperator
parts []string
)
for token := range tokens2Operator {
parts = strings.Split(statement, token)
// if we got a token, we know this was the operator...
if len(parts) > 1 {
operator = tokens2Operator[token]
break
}
}
if len(operator) == 0 || len(parts) != 2 {
return Threshold{}, fmt.Errorf("invalid eviction threshold syntax %v, expected <signal><operator><value>", statement)
}
signal := Signal(parts[0])
if !validSignal(signal) {
return Threshold{}, fmt.Errorf(unsupportedEvictionSignal, signal)
}
quantity, err := resource.ParseQuantity(parts[1])
if err != nil {
return Threshold{}, err
}
return Threshold{
Signal: signal,
Operator: operator,
Value: *quantity,
}, nil
}
// parseGracePeriods parses the grace period statements
func parseGracePeriods(expr string) (map[Signal]time.Duration, error) {
if len(expr) == 0 {
return nil, nil
}
results := map[Signal]time.Duration{}
statements := strings.Split(expr, ",")
for _, statement := range statements {
parts := strings.Split(statement, "=")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid eviction grace period syntax %v, expected <signal>=<duration>", statement)
}
signal := Signal(parts[0])
if !validSignal(signal) {
return nil, fmt.Errorf(unsupportedEvictionSignal, signal)
}
gracePeriod, err := time.ParseDuration(parts[1])
if err != nil {
return nil, err
}
if gracePeriod < 0 {
return nil, fmt.Errorf("invalid eviction grace period specified: %v, must be a positive value", parts[1])
}
// check against duplicate statements
if _, found := results[signal]; found {
return nil, fmt.Errorf("duplicate eviction grace period specified for %v", signal)
}
results[signal] = gracePeriod
}
return results, nil
}

View File

@ -0,0 +1,142 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package eviction
import (
"testing"
"time"
"k8s.io/kubernetes/pkg/api/resource"
)
func TestParseThresholdConfig(t *testing.T) {
gracePeriod, _ := time.ParseDuration("30s")
testCases := map[string]struct {
evictionHard string
evictionSoft string
evictionSoftGracePeriod string
expectErr bool
expectThresholds []Threshold
}{
"no values": {
evictionHard: "",
evictionSoft: "",
evictionSoftGracePeriod: "",
expectErr: false,
expectThresholds: []Threshold{},
},
"all flag values": {
evictionHard: "memory.available<150Mi",
evictionSoft: "memory.available<300Mi",
evictionSoftGracePeriod: "memory.available=30s",
expectErr: false,
expectThresholds: []Threshold{
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: resource.MustParse("150Mi"),
},
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: resource.MustParse("300Mi"),
GracePeriod: gracePeriod,
},
},
},
"invalid-signal": {
evictionHard: "mem.available<150Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
expectErr: true,
expectThresholds: []Threshold{},
},
"duplicate-signal": {
evictionHard: "memory.available<150Mi,memory.available<100Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
expectErr: true,
expectThresholds: []Threshold{},
},
"valid-and-invalid-signal": {
evictionHard: "memory.available<150Mi,invalid.foo<150Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
expectErr: true,
expectThresholds: []Threshold{},
},
"soft-no-grace-period": {
evictionHard: "",
evictionSoft: "memory.available<150Mi",
evictionSoftGracePeriod: "",
expectErr: true,
expectThresholds: []Threshold{},
},
"soft-neg-grace-period": {
evictionHard: "",
evictionSoft: "memory.available<150Mi",
evictionSoftGracePeriod: "memory.available=-30s",
expectErr: true,
expectThresholds: []Threshold{},
},
}
for testName, testCase := range testCases {
thresholds, err := ParseThresholdConfig(testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod)
if testCase.expectErr != (err != nil) {
t.Errorf("Err not as expected, test: %v, error expected: %v, actual: %v", testName, testCase.expectErr, err)
}
if !thresholdsEqual(testCase.expectThresholds, thresholds) {
t.Errorf("thresholds not as expected, test: %v, expected: %v, actual: %v", testName, testCase.expectThresholds, thresholds)
}
}
}
func thresholdsEqual(expected []Threshold, actual []Threshold) bool {
if len(expected) != len(actual) {
return false
}
for _, aThreshold := range expected {
equal := false
for _, bThreshold := range actual {
if thresholdEqual(aThreshold, bThreshold) {
equal = true
}
}
if !equal {
return false
}
}
for _, aThreshold := range actual {
equal := false
for _, bThreshold := range expected {
if thresholdEqual(aThreshold, bThreshold) {
equal = true
}
}
if !equal {
return false
}
}
return true
}
func thresholdEqual(a Threshold, b Threshold) bool {
return a.GracePeriod == b.GracePeriod &&
a.Operator == b.Operator &&
a.Signal == b.Signal &&
a.Value.Cmp(b.Value) == 0
}

View File

@ -0,0 +1,51 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package eviction
import (
"time"
"k8s.io/kubernetes/pkg/api/resource"
)
// Signal defines a signal that can trigger eviction of pods on a node.
type Signal string
const (
// SignalMemoryAvailable is memory available (i.e. capacity - workingSet), in bytes.
SignalMemoryAvailable Signal = "memory.available"
)
// ThresholdOperator is the operator used to express a Threshold.
type ThresholdOperator string
const (
// OpLessThan is the operator that expresses a less than operator.
OpLessThan ThresholdOperator = "LessThan"
)
// Threshold defines a metric for when eviction should occur.
type Threshold struct {
// Signal defines the entity that was measured.
Signal Signal
// Operator represents a relationship of a signal to a value.
Operator ThresholdOperator
// value is a quantity associated with the signal that is evaluated against the specified operator.
Value resource.Quantity
// GracePeriod represents the amount of time that a threshold must be met before eviction is triggered.
GracePeriod time.Duration
}

View File

@ -53,6 +53,7 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/envvars"
"k8s.io/kubernetes/pkg/kubelet/eviction"
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/network"
@ -223,6 +224,7 @@ func NewMainKubelet(
containerRuntimeOptions []kubecontainer.Option,
hairpinMode string,
babysitDaemons bool,
thresholds []eviction.Threshold,
kubeOptions []Option,
) (*Kubelet, error) {
if rootDirectory == "" {
@ -930,6 +932,7 @@ func (kl *Kubelet) initializeModules() error {
// Step 7: Start resource analyzer
kl.resourceAnalyzer.Start()
return nil
}
@ -968,6 +971,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
// Start component sync loops.
kl.statusManager.Start()
kl.probeManager.Start()
// Start the pod lifecycle event generator.
kl.pleg.Start()
kl.syncLoop(updates, kl)