Merge pull request #27749 from dubstack/dubstack-refactor-qos

Automatic merge from submit-queue

[Refactor] QOS to have QOS Class type for QoS classes

This PR adds a QOSClass type and initializes QOSclass constants for the three QoS classes.
It would be good to use this in all future QOS related features. 
This would be good to have for the (Pod level cgroups isolation proposal)[https://github.com/kubernetes/kubernetes/pull/26751] that i am working on aswell.
@vishh PTAL
 
Signed-off-by: Buddha Prakash <buddhap@google.com>

pkg/kubectl/describe.go

@@ -44,7 +44,7 @@ import (
 	adapter "k8s.io/kubernetes/pkg/client/unversioned/adapters/internalclientset"
 	"k8s.io/kubernetes/pkg/fieldpath"
 	"k8s.io/kubernetes/pkg/fields"
-	qosutil "k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/labels"
 	"k8s.io/kubernetes/pkg/types"
 	deploymentutil "k8s.io/kubernetes/pkg/util/deployment"
@@ -540,7 +540,7 @@ func describePod(pod *api.Pod, events *api.EventList) (string, error) {
 			}
 		}
 		describeVolumes(pod.Spec.Volumes, out, "")
-		fmt.Fprintf(out, "QoS Tier:\t%s\n", qosutil.GetPodQos(pod))
+		fmt.Fprintf(out, "QoS Class:\t%s\n", qos.GetPodQos(pod))
 		if events != nil {
 			DescribeEvents(events, out)
 		}

pkg/kubectl/sorted_resource_name_list.go

@@ -20,7 +20,7 @@ import (
 	"sort"
 
 	"k8s.io/kubernetes/pkg/api"
-	qosutil "k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/qos"
 )
 
 type SortableResourceNames []api.ResourceName
@@ -62,7 +62,7 @@ func (list SortableResourceQuotas) Less(i, j int) bool {
 }
 
 // SortedQoSResourceNames returns the sorted resource names of a QoS list.
-func SortedQoSResourceNames(list qosutil.QoSList) []api.ResourceName {
+func SortedQoSResourceNames(list qos.QoSList) []api.ResourceName {
 	resources := make([]api.ResourceName, 0, len(list))
 	for res := range list {
 		resources = append(resources, res)

pkg/kubelet/eviction/eviction_manager.go

@@ -25,7 +25,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/client/record"
 	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
-	qosutil "k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	"k8s.io/kubernetes/pkg/util"
@@ -87,7 +87,7 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
 	if len(m.nodeConditions) == 0 {
 		return lifecycle.PodAdmitResult{Admit: true}
 	}
-	notBestEffort := qosutil.BestEffort != qosutil.GetPodQos(attrs.Pod)
+	notBestEffort := qos.BestEffort != qos.GetPodQos(attrs.Pod)
 	if notBestEffort {
 		return lifecycle.PodAdmitResult{Admit: true}
 	}

pkg/kubelet/eviction/helpers.go

@@ -27,7 +27,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
-	qosutil "k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 	"k8s.io/kubernetes/pkg/quota/evaluator/core"
 	"k8s.io/kubernetes/pkg/util/sets"
@@ -300,20 +300,20 @@ func (ms *multiSorter) Less(i, j int) bool {
 }
 
 // qos compares pods by QoS (BestEffort < Burstable < Guaranteed)
-func qos(p1, p2 *api.Pod) int {
+func qosComparator(p1, p2 *api.Pod) int {
-	qosP1 := qosutil.GetPodQos(p1)
+	qosP1 := qos.GetPodQos(p1)
-	qosP2 := qosutil.GetPodQos(p2)
+	qosP2 := qos.GetPodQos(p2)
 	// its a tie
 	if qosP1 == qosP2 {
 		return 0
 	}
 	// if p1 is best effort, we know p2 is burstable or guaranteed
-	if qosP1 == qosutil.BestEffort {
+	if qosP1 == qos.BestEffort {
 		return -1
 	}
 	// we know p1 and p2 are not besteffort, so if p1 is burstable, p2 must be guaranteed
-	if qosP1 == qosutil.Burstable {
+	if qosP1 == qos.Burstable {
-		if qosP2 == qosutil.Guaranteed {
+		if qosP2 == qos.Guaranteed {
 			return -1
 		}
 		return 1
@@ -397,12 +397,12 @@ func disk(stats statsFunc) cmpFunc {
 
 // rankMemoryPressure orders the input pods for eviction in response to memory pressure.
 func rankMemoryPressure(pods []*api.Pod, stats statsFunc) {
-	orderedBy(qos, memory(stats)).Sort(pods)
+	orderedBy(qosComparator, memory(stats)).Sort(pods)
 }
 
 // rankDiskPressure orders the input pods for eviction in response to disk pressure.
 func rankDiskPressure(pods []*api.Pod, stats statsFunc) {
-	orderedBy(qos, disk(stats)).Sort(pods)
+	orderedBy(qosComparator, disk(stats)).Sort(pods)
 }
 
 // byEvictionPriority implements sort.Interface for []api.ResourceName.

pkg/kubelet/eviction/helpers_test.go

@@ -166,7 +166,7 @@ func TestOrderedByQoS(t *testing.T) {
 	})
 
 	pods := []*api.Pod{guaranteed, burstable, bestEffort}
-	orderedBy(qos).Sort(pods)
+	orderedBy(qosComparator).Sort(pods)
 
 	expected := []*api.Pod{bestEffort, burstable, guaranteed}
 	for i := range expected {
@@ -218,7 +218,7 @@ func TestOrderedByMemory(t *testing.T) {
 	}
 }
 
-// TestOrderedByQoSMemory ensures we order by qos and then memory consumption relative to request.
+// TestOrderedByQoSMemory ensures we order by qosComparator and then memory consumption relative to request.
 func TestOrderedByQoSMemory(t *testing.T) {
 	pod1 := newPod("best-effort-high", []api.Container{
 		newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")),
@@ -252,7 +252,7 @@ func TestOrderedByQoSMemory(t *testing.T) {
 	}
 	pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
 	expected := []*api.Pod{pod1, pod2, pod4, pod3, pod5, pod6}
-	orderedBy(qos, memory(statsFn)).Sort(pods)
+	orderedBy(qosComparator, memory(statsFn)).Sort(pods)
 	for i := range expected {
 		if pods[i] != expected[i] {
 			t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)

pkg/kubelet/qos/policy.go

@@ -18,7 +18,6 @@ package qos
 
 import (
 	"k8s.io/kubernetes/pkg/api"
-	"k8s.io/kubernetes/pkg/kubelet/qos/util"
 )
 
 const (
@@ -36,11 +35,11 @@ const (
 // and 1000. Containers with higher OOM scores are killed if the system runs out of memory.
 // See https://lwn.net/Articles/391222/ for more information.
 func GetContainerOOMScoreAdjust(pod *api.Pod, container *api.Container, memoryCapacity int64) int {
-	switch util.GetPodQos(pod) {
+	switch GetPodQos(pod) {
-	case util.Guaranteed:
+	case Guaranteed:
 		// Guaranteed containers should be the last to get killed.
 		return guaranteedOOMScoreAdj
-	case util.BestEffort:
+	case BestEffort:
 		return besteffortOOMScoreAdj
 	}
 

pkg/kubelet/qos/qos.go

@@ -14,19 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package qos
 
 import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 )
 
-const (
-	Guaranteed = "Guaranteed"
-	Burstable  = "Burstable"
-	BestEffort = "BestEffort"
-)
-
 // isResourceGuaranteed returns true if the container's resource requirements are Guaranteed.
 func isResourceGuaranteed(container *api.Container, resource api.ResourceName) bool {
 	// A container resource is guaranteed if its request == limit.
@@ -51,7 +45,7 @@ func isResourceBestEffort(container *api.Container, resource api.ResourceName) b
 // A pod is besteffort if none of its containers have specified any requests or limits.
 // A pod is guaranteed only when requests and limits are specified for all the containers and they are equal.
 // A pod is burstable if limits and requests do not match across all containers.
-func GetPodQos(pod *api.Pod) string {
+func GetPodQos(pod *api.Pod) QOSClass {
 	requests := api.ResourceList{}
 	limits := api.ResourceList{}
 	zeroQuantity := resource.MustParse("0")
@@ -106,7 +100,7 @@ func GetPodQos(pod *api.Pod) string {
 }
 
 // QoSList is a set of (resource name, QoS class) pairs.
-type QoSList map[api.ResourceName]string
+type QoSList map[api.ResourceName]QOSClass
 
 // GetQoS returns a mapping of resource name to QoS class of a container
 func GetQoS(container *api.Container) QoSList {

pkg/kubelet/qos/qos_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package qos
 
 import (
 	"testing"
@@ -62,7 +62,7 @@ func newPod(name string, containers []api.Container) *api.Pod {
 func TestGetPodQos(t *testing.T) {
 	testCases := []struct {
 		pod      *api.Pod
-		expected string
+		expected QOSClass
 	}{
 		{
 			pod: newPod("guaranteed", []api.Container{

pkg/kubelet/qos/types.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package qos
+
+// QOSClass defines the supported qos classes of Pods/Containers.
+type QOSClass string
+
+const (
+	// Guaranteed is the Guaranteed qos class.
+	Guaranteed QOSClass = "Guaranteed"
+	// Burstable is the Burstable qos class.
+	Burstable QOSClass = "Burstable"
+	// BestEffort is the BestEffort qos class.
+	BestEffort QOSClass = "BestEffort"
+)

pkg/quota/evaluator/core/pods.go

@@ -25,7 +25,7 @@ import (
 	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/api/validation"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
-	"k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/quota"
 	"k8s.io/kubernetes/pkg/quota/generic"
 	"k8s.io/kubernetes/pkg/runtime"
@@ -172,7 +172,7 @@ func PodMatchesScopeFunc(scope api.ResourceQuotaScope, object runtime.Object) bo
 }
 
 func isBestEffort(pod *api.Pod) bool {
-	return util.GetPodQos(pod) == util.BestEffort
+	return qos.GetPodQos(pod) == qos.BestEffort
 }
 
 func isTerminating(pod *api.Pod) bool {

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -26,7 +26,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/client/cache"
-	qosutil "k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/labels"
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
@@ -1020,7 +1020,7 @@ func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint
 
 // Determine if a pod is scheduled with best-effort QoS
 func isPodBestEffort(pod *api.Pod) bool {
-	return qosutil.GetPodQos(pod) == qosutil.BestEffort
+	return qos.GetPodQos(pod) == qos.BestEffort
 }
 
 // CheckNodeMemoryPressurePredicate checks if a pod can be scheduled on a node

test/e2e/kubectl.go

@@ -509,7 +509,7 @@ var _ = framework.KubeDescribe("Kubectl client", func() {
 					{"Controllers:", "ReplicationController/redis-master"},
 					{"Image:", redisImage},
 					{"State:", "Running"},
-					{"QoS Tier:", "BestEffort"},
+					{"QoS Class:", "BestEffort"},
 				}
 				checkOutput(output, requiredStrings)
 			})