Merge pull request #24750 from derekwaynecarr/kubelet_eviction_flag_parsing

Automatic merge from submit-queue

Kubelet eviction flag parsers and tests

The first two commits are from https://github.com/kubernetes/kubernetes/pull/24559 that have achieved LGTM.  

The last commit is only part that is interesting, it adds the parsing logic to handle the flags, and reserves `pkg/kubelet/eviction` for eviction manager logic.

cmd/kubelet/app/server.go

@@ -55,6 +55,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
+	"k8s.io/kubernetes/pkg/kubelet/eviction"
 	"k8s.io/kubernetes/pkg/kubelet/network"
 	"k8s.io/kubernetes/pkg/kubelet/server"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
@@ -183,6 +184,11 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		return nil, err
 	}
 
+	thresholds, err := eviction.ParseThresholdConfig(s.EvictionHard, s.EvictionSoft, s.EvictionSoftGracePeriod)
+	if err != nil {
+		return nil, err
+	}
+
 	return &KubeletConfig{
 		Address:                   net.ParseIP(s.Address),
 		AllowPrivileged:           s.AllowPrivileged,
@@ -261,6 +267,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		BabysitDaemons:                 s.BabysitDaemons,
 		ExperimentalFlannelOverlay:     s.ExperimentalFlannelOverlay,
 		NodeIP:     net.ParseIP(s.NodeIP),
+		Thresholds: thresholds,
 	}, nil
 }
 
@@ -773,6 +780,7 @@ type KubeletConfig struct {
 	Writer                         io.Writer
 	VolumePlugins                  []volume.VolumePlugin
 	OutOfDiskTransitionFrequency   time.Duration
+	Thresholds                     []eviction.Threshold
 
 	ExperimentalFlannelOverlay bool
 	NodeIP                     net.IP
@@ -868,6 +876,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.ContainerRuntimeOptions,
 		kc.HairpinMode,
 		kc.BabysitDaemons,
+		kc.Thresholds,
 		kc.Options,
 	)
 

pkg/kubelet/eviction/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package eviction is responsible for enforcing eviction thresholds to maintain
+// node stability.
+package eviction

pkg/kubelet/eviction/helpers.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eviction
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/util/sets"
+)
+
+const (
+	unsupportedEvictionSignal = "unsupported eviction signal %v"
+)
+
+// signalToResource maps a Signal to its associated Resource.
+var signalToResource = map[Signal]api.ResourceName{
+	SignalMemoryAvailable: api.ResourceMemory,
+}
+
+// validSignal returns true if the signal is supported.
+func validSignal(signal Signal) bool {
+	_, found := signalToResource[signal]
+	return found
+}
+
+// ParseThresholdConfig parses the flags for thresholds.
+func ParseThresholdConfig(evictionHard, evictionSoft, evictionSoftGracePeriod string) ([]Threshold, error) {
+	results := []Threshold{}
+
+	hardThresholds, err := parseThresholdStatements(evictionHard)
+	if err != nil {
+		return nil, err
+	}
+	results = append(results, hardThresholds...)
+
+	softThresholds, err := parseThresholdStatements(evictionSoft)
+	if err != nil {
+		return nil, err
+	}
+	gracePeriods, err := parseGracePeriods(evictionSoftGracePeriod)
+	if err != nil {
+		return nil, err
+	}
+	for i := range softThresholds {
+		signal := softThresholds[i].Signal
+		period, found := gracePeriods[signal]
+		if !found {
+			return nil, fmt.Errorf("grace period must be specified for the soft eviction threshold %v", signal)
+		}
+		softThresholds[i].GracePeriod = period
+	}
+	results = append(results, softThresholds...)
+	return results, nil
+}
+
+// parseThresholdStatements parses the input statements into a list of Threshold objects.
+func parseThresholdStatements(expr string) ([]Threshold, error) {
+	if len(expr) == 0 {
+		return nil, nil
+	}
+	results := []Threshold{}
+	statements := strings.Split(expr, ",")
+	signalsFound := sets.NewString()
+	for _, statement := range statements {
+		result, err := parseThresholdStatement(statement)
+		if err != nil {
+			return nil, err
+		}
+		if signalsFound.Has(string(result.Signal)) {
+			return nil, fmt.Errorf("found duplicate eviction threshold for signal %v", result.Signal)
+		}
+		signalsFound.Insert(string(result.Signal))
+		results = append(results, result)
+	}
+	return results, nil
+}
+
+// parseThresholdStatement parses a threshold statement.
+func parseThresholdStatement(statement string) (Threshold, error) {
+	tokens2Operator := map[string]ThresholdOperator{
+		"<": OpLessThan,
+	}
+	var (
+		operator ThresholdOperator
+		parts    []string
+	)
+	for token := range tokens2Operator {
+		parts = strings.Split(statement, token)
+		// if we got a token, we know this was the operator...
+		if len(parts) > 1 {
+			operator = tokens2Operator[token]
+			break
+		}
+	}
+	if len(operator) == 0 || len(parts) != 2 {
+		return Threshold{}, fmt.Errorf("invalid eviction threshold syntax %v, expected <signal><operator><value>", statement)
+	}
+	signal := Signal(parts[0])
+	if !validSignal(signal) {
+		return Threshold{}, fmt.Errorf(unsupportedEvictionSignal, signal)
+	}
+
+	quantity, err := resource.ParseQuantity(parts[1])
+	if err != nil {
+		return Threshold{}, err
+	}
+	return Threshold{
+		Signal:   signal,
+		Operator: operator,
+		Value:    *quantity,
+	}, nil
+}
+
+// parseGracePeriods parses the grace period statements
+func parseGracePeriods(expr string) (map[Signal]time.Duration, error) {
+	if len(expr) == 0 {
+		return nil, nil
+	}
+	results := map[Signal]time.Duration{}
+	statements := strings.Split(expr, ",")
+	for _, statement := range statements {
+		parts := strings.Split(statement, "=")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid eviction grace period syntax %v, expected <signal>=<duration>", statement)
+		}
+		signal := Signal(parts[0])
+		if !validSignal(signal) {
+			return nil, fmt.Errorf(unsupportedEvictionSignal, signal)
+		}
+
+		gracePeriod, err := time.ParseDuration(parts[1])
+		if err != nil {
+			return nil, err
+		}
+		if gracePeriod < 0 {
+			return nil, fmt.Errorf("invalid eviction grace period specified: %v, must be a positive value", parts[1])
+		}
+
+		// check against duplicate statements
+		if _, found := results[signal]; found {
+			return nil, fmt.Errorf("duplicate eviction grace period specified for %v", signal)
+		}
+		results[signal] = gracePeriod
+	}
+	return results, nil
+}

pkg/kubelet/eviction/helpers_test.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eviction
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/resource"
+)
+
+func TestParseThresholdConfig(t *testing.T) {
+	gracePeriod, _ := time.ParseDuration("30s")
+	testCases := map[string]struct {
+		evictionHard            string
+		evictionSoft            string
+		evictionSoftGracePeriod string
+		expectErr               bool
+		expectThresholds        []Threshold
+	}{
+		"no values": {
+			evictionHard:            "",
+			evictionSoft:            "",
+			evictionSoftGracePeriod: "",
+			expectErr:               false,
+			expectThresholds:        []Threshold{},
+		},
+		"all flag values": {
+			evictionHard:            "memory.available<150Mi",
+			evictionSoft:            "memory.available<300Mi",
+			evictionSoftGracePeriod: "memory.available=30s",
+			expectErr:               false,
+			expectThresholds: []Threshold{
+				{
+					Signal:   SignalMemoryAvailable,
+					Operator: OpLessThan,
+					Value:    resource.MustParse("150Mi"),
+				},
+				{
+					Signal:      SignalMemoryAvailable,
+					Operator:    OpLessThan,
+					Value:       resource.MustParse("300Mi"),
+					GracePeriod: gracePeriod,
+				},
+			},
+		},
+		"invalid-signal": {
+			evictionHard:            "mem.available<150Mi",
+			evictionSoft:            "",
+			evictionSoftGracePeriod: "",
+			expectErr:               true,
+			expectThresholds:        []Threshold{},
+		},
+		"duplicate-signal": {
+			evictionHard:            "memory.available<150Mi,memory.available<100Mi",
+			evictionSoft:            "",
+			evictionSoftGracePeriod: "",
+			expectErr:               true,
+			expectThresholds:        []Threshold{},
+		},
+		"valid-and-invalid-signal": {
+			evictionHard:            "memory.available<150Mi,invalid.foo<150Mi",
+			evictionSoft:            "",
+			evictionSoftGracePeriod: "",
+			expectErr:               true,
+			expectThresholds:        []Threshold{},
+		},
+		"soft-no-grace-period": {
+			evictionHard:            "",
+			evictionSoft:            "memory.available<150Mi",
+			evictionSoftGracePeriod: "",
+			expectErr:               true,
+			expectThresholds:        []Threshold{},
+		},
+		"soft-neg-grace-period": {
+			evictionHard:            "",
+			evictionSoft:            "memory.available<150Mi",
+			evictionSoftGracePeriod: "memory.available=-30s",
+			expectErr:               true,
+			expectThresholds:        []Threshold{},
+		},
+	}
+	for testName, testCase := range testCases {
+		thresholds, err := ParseThresholdConfig(testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod)
+		if testCase.expectErr != (err != nil) {
+			t.Errorf("Err not as expected, test: %v, error expected: %v, actual: %v", testName, testCase.expectErr, err)
+		}
+		if !thresholdsEqual(testCase.expectThresholds, thresholds) {
+			t.Errorf("thresholds not as expected, test: %v, expected: %v, actual: %v", testName, testCase.expectThresholds, thresholds)
+		}
+	}
+}
+
+func thresholdsEqual(expected []Threshold, actual []Threshold) bool {
+	if len(expected) != len(actual) {
+		return false
+	}
+	for _, aThreshold := range expected {
+		equal := false
+		for _, bThreshold := range actual {
+			if thresholdEqual(aThreshold, bThreshold) {
+				equal = true
+			}
+		}
+		if !equal {
+			return false
+		}
+	}
+	for _, aThreshold := range actual {
+		equal := false
+		for _, bThreshold := range expected {
+			if thresholdEqual(aThreshold, bThreshold) {
+				equal = true
+			}
+		}
+		if !equal {
+			return false
+		}
+	}
+	return true
+}
+
+func thresholdEqual(a Threshold, b Threshold) bool {
+	return a.GracePeriod == b.GracePeriod &&
+		a.Operator == b.Operator &&
+		a.Signal == b.Signal &&
+		a.Value.Cmp(b.Value) == 0
+}

pkg/kubelet/eviction/types.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eviction
+
+import (
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/resource"
+)
+
+// Signal defines a signal that can trigger eviction of pods on a node.
+type Signal string
+
+const (
+	// SignalMemoryAvailable is memory available (i.e. capacity - workingSet), in bytes.
+	SignalMemoryAvailable Signal = "memory.available"
+)
+
+// ThresholdOperator is the operator used to express a Threshold.
+type ThresholdOperator string
+
+const (
+	// OpLessThan is the operator that expresses a less than operator.
+	OpLessThan ThresholdOperator = "LessThan"
+)
+
+// Threshold defines a metric for when eviction should occur.
+type Threshold struct {
+	// Signal defines the entity that was measured.
+	Signal Signal
+	// Operator represents a relationship of a signal to a value.
+	Operator ThresholdOperator
+	// value is a quantity associated with the signal that is evaluated against the specified operator.
+	Value resource.Quantity
+	// GracePeriod represents the amount of time that a threshold must be met before eviction is triggered.
+	GracePeriod time.Duration
+}

pkg/kubelet/kubelet.go

@@ -53,6 +53,7 @@ import (
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
 	"k8s.io/kubernetes/pkg/kubelet/envvars"
+	"k8s.io/kubernetes/pkg/kubelet/eviction"
 	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/network"
@@ -223,6 +224,7 @@ func NewMainKubelet(
 	containerRuntimeOptions []kubecontainer.Option,
 	hairpinMode string,
 	babysitDaemons bool,
+	thresholds []eviction.Threshold,
 	kubeOptions []Option,
 ) (*Kubelet, error) {
 	if rootDirectory == "" {
@@ -930,6 +932,7 @@ func (kl *Kubelet) initializeModules() error {
 
 	// Step 7: Start resource analyzer
 	kl.resourceAnalyzer.Start()
+
 	return nil
 }
 
@@ -968,6 +971,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 	// Start component sync loops.
 	kl.statusManager.Start()
 	kl.probeManager.Start()
+
 	// Start the pod lifecycle event generator.
 	kl.pleg.Start()
 	kl.syncLoop(updates, kl)