From 83b7f77ee2e06ad1a0af7495afd02a7742928678 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Sat, 20 May 2017 07:16:53 -0400 Subject: [PATCH 1/2] Moved qos to api.helpers. --- pkg/api/helper/BUILD | 5 +- pkg/api/helper/qos/BUILD | 32 +++++++ pkg/api/helper/qos/qos.go | 94 +++++++++++++++++++ pkg/api/v1/helper/BUILD | 5 +- pkg/api/v1/helper/qos/BUILD | 47 ++++++++++ pkg/{kubelet => api/v1/helper}/qos/qos.go | 78 +-------------- .../v1/helper}/qos/qos_test.go | 92 ++++++++++-------- pkg/kubelet/BUILD | 2 +- pkg/kubelet/cm/BUILD | 1 + pkg/kubelet/cm/helpers_linux.go | 4 +- pkg/kubelet/cm/pod_container_manager_linux.go | 4 +- pkg/kubelet/cm/qos_container_manager_linux.go | 6 +- pkg/kubelet/eviction/BUILD | 2 +- pkg/kubelet/eviction/eviction_manager.go | 4 +- pkg/kubelet/eviction/helpers.go | 6 +- pkg/kubelet/kubelet_pods.go | 4 +- pkg/kubelet/preemption/BUILD | 2 +- pkg/kubelet/preemption/preemption.go | 4 +- pkg/kubelet/qos/BUILD | 11 +-- pkg/kubelet/qos/policy.go | 7 +- pkg/printers/internalversion/BUILD | 2 +- pkg/printers/internalversion/describe.go | 4 +- pkg/quota/evaluator/core/BUILD | 2 +- pkg/quota/evaluator/core/pods.go | 4 +- pkg/registry/core/pod/BUILD | 2 +- pkg/registry/core/pod/strategy.go | 4 +- .../pkg/scheduler/algorithm/predicates/BUILD | 2 +- .../algorithm/predicates/predicates.go | 4 +- 28 files changed, 276 insertions(+), 158 deletions(-) create mode 100644 pkg/api/helper/qos/BUILD create mode 100644 pkg/api/helper/qos/qos.go create mode 100644 pkg/api/v1/helper/qos/BUILD rename pkg/{kubelet => api/v1/helper}/qos/qos.go (57%) rename pkg/{kubelet => api/v1/helper}/qos/qos_test.go (93%) diff --git a/pkg/api/helper/BUILD b/pkg/api/helper/BUILD index 9424c759411..160bb5865eb 100644 --- a/pkg/api/helper/BUILD +++ b/pkg/api/helper/BUILD @@ -46,6 +46,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//pkg/api/helper/qos:all-srcs", + ], tags = ["automanaged"], ) diff --git a/pkg/api/helper/qos/BUILD b/pkg/api/helper/qos/BUILD new file mode 100644 index 00000000000..9dfe7de8549 --- /dev/null +++ b/pkg/api/helper/qos/BUILD @@ -0,0 +1,32 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["qos.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/api/helper/qos/qos.go b/pkg/api/helper/qos/qos.go new file mode 100644 index 00000000000..6edd0e1b25a --- /dev/null +++ b/pkg/api/helper/qos/qos.go @@ -0,0 +1,94 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// NOTE: DO NOT use those helper functions through client-go, the +// package path will be changed in the future. +package qos + +import ( + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/api" +) + +// supportedComputeResources is the list of compute resources for with QoS is supported. +var supportedQoSComputeResources = sets.NewString(string(api.ResourceCPU), string(api.ResourceMemory)) + +// GetPodQOS returns the QoS class of a pod. +// A pod is besteffort if none of its containers have specified any requests or limits. +// A pod is guaranteed only when requests and limits are specified for all the containers and they are equal. +// A pod is burstable if limits and requests do not match across all containers. +func GetPodQOS(pod *api.Pod) api.PodQOSClass { + requests := api.ResourceList{} + limits := api.ResourceList{} + zeroQuantity := resource.MustParse("0") + isGuaranteed := true + for _, container := range pod.Spec.Containers { + // process requests + for name, quantity := range container.Resources.Requests { + if !supportedQoSComputeResources.Has(string(name)) { + continue + } + if quantity.Cmp(zeroQuantity) == 1 { + delta := quantity.Copy() + if _, exists := requests[name]; !exists { + requests[name] = *delta + } else { + delta.Add(requests[name]) + requests[name] = *delta + } + } + } + // process limits + qosLimitsFound := sets.NewString() + for name, quantity := range container.Resources.Limits { + if !supportedQoSComputeResources.Has(string(name)) { + continue + } + if quantity.Cmp(zeroQuantity) == 1 { + qosLimitsFound.Insert(string(name)) + delta := quantity.Copy() + if _, exists := limits[name]; !exists { + limits[name] = *delta + } else { + delta.Add(limits[name]) + limits[name] = *delta + } + } + } + + if len(qosLimitsFound) != len(supportedQoSComputeResources) { + isGuaranteed = false + } + } + if len(requests) == 0 && len(limits) == 0 { + return api.PodQOSBestEffort + } + // Check is requests match limits for all resources. + if isGuaranteed { + for name, req := range requests { + if lim, exists := limits[name]; !exists || lim.Cmp(req) != 0 { + isGuaranteed = false + break + } + } + } + if isGuaranteed && + len(requests) == len(limits) { + return api.PodQOSGuaranteed + } + return api.PodQOSBurstable +} diff --git a/pkg/api/v1/helper/BUILD b/pkg/api/v1/helper/BUILD index 4a2ec81e511..e8aefb61f05 100644 --- a/pkg/api/v1/helper/BUILD +++ b/pkg/api/v1/helper/BUILD @@ -43,6 +43,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//pkg/api/v1/helper/qos:all-srcs", + ], tags = ["automanaged"], ) diff --git a/pkg/api/v1/helper/qos/BUILD b/pkg/api/v1/helper/qos/BUILD new file mode 100644 index 00000000000..b45e5cfe1a0 --- /dev/null +++ b/pkg/api/v1/helper/qos/BUILD @@ -0,0 +1,47 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["qos_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/helper/qos:go_default_library", + "//pkg/api/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = ["qos.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/kubelet/qos/qos.go b/pkg/api/v1/helper/qos/qos.go similarity index 57% rename from pkg/kubelet/qos/qos.go rename to pkg/api/v1/helper/qos/qos.go index 5c166195a96..2b7a3bbea5c 100644 --- a/pkg/kubelet/qos/qos.go +++ b/pkg/api/v1/helper/qos/qos.go @@ -19,10 +19,14 @@ package qos import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" ) +// QOSList is a set of (resource name, QoS class) pairs. +type QOSList map[v1.ResourceName]v1.PodQOSClass + +var supportedQoSComputeResources = sets.NewString(string(v1.ResourceCPU), string(v1.ResourceMemory)) + // GetPodQOS returns the QoS class of a pod. // A pod is besteffort if none of its containers have specified any requests or limits. // A pod is guaranteed only when requests and limits are specified for all the containers and they are equal. @@ -88,75 +92,3 @@ func GetPodQOS(pod *v1.Pod) v1.PodQOSClass { } return v1.PodQOSBurstable } - -// InternalGetPodQOS returns the QoS class of a pod. -// A pod is besteffort if none of its containers have specified any requests or limits. -// A pod is guaranteed only when requests and limits are specified for all the containers and they are equal. -// A pod is burstable if limits and requests do not match across all containers. -func InternalGetPodQOS(pod *api.Pod) api.PodQOSClass { - requests := api.ResourceList{} - limits := api.ResourceList{} - zeroQuantity := resource.MustParse("0") - isGuaranteed := true - for _, container := range pod.Spec.Containers { - // process requests - for name, quantity := range container.Resources.Requests { - if !supportedQoSComputeResources.Has(string(name)) { - continue - } - if quantity.Cmp(zeroQuantity) == 1 { - delta := quantity.Copy() - if _, exists := requests[name]; !exists { - requests[name] = *delta - } else { - delta.Add(requests[name]) - requests[name] = *delta - } - } - } - // process limits - qosLimitsFound := sets.NewString() - for name, quantity := range container.Resources.Limits { - if !supportedQoSComputeResources.Has(string(name)) { - continue - } - if quantity.Cmp(zeroQuantity) == 1 { - qosLimitsFound.Insert(string(name)) - delta := quantity.Copy() - if _, exists := limits[name]; !exists { - limits[name] = *delta - } else { - delta.Add(limits[name]) - limits[name] = *delta - } - } - } - - if len(qosLimitsFound) != len(supportedQoSComputeResources) { - isGuaranteed = false - } - } - if len(requests) == 0 && len(limits) == 0 { - return api.PodQOSBestEffort - } - // Check is requests match limits for all resources. - if isGuaranteed { - for name, req := range requests { - if lim, exists := limits[name]; !exists || lim.Cmp(req) != 0 { - isGuaranteed = false - break - } - } - } - if isGuaranteed && - len(requests) == len(limits) { - return api.PodQOSGuaranteed - } - return api.PodQOSBurstable -} - -// QOSList is a set of (resource name, QoS class) pairs. -type QOSList map[v1.ResourceName]v1.PodQOSClass - -// supportedComputeResources is the list of compute resources for with QoS is supported. -var supportedQoSComputeResources = sets.NewString(string(v1.ResourceCPU), string(v1.ResourceMemory)) diff --git a/pkg/kubelet/qos/qos_test.go b/pkg/api/v1/helper/qos/qos_test.go similarity index 93% rename from pkg/kubelet/qos/qos_test.go rename to pkg/api/v1/helper/qos/qos_test.go index 63a2dfcf5cb..137a7e31554 100644 --- a/pkg/kubelet/qos/qos_test.go +++ b/pkg/api/v1/helper/qos/qos_test.go @@ -21,50 +21,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/helper/qos" "k8s.io/kubernetes/pkg/api/v1" ) -func getResourceList(cpu, memory string) v1.ResourceList { - res := v1.ResourceList{} - if cpu != "" { - res[v1.ResourceCPU] = resource.MustParse(cpu) - } - if memory != "" { - res[v1.ResourceMemory] = resource.MustParse(memory) - } - return res -} - -func addResource(rName, value string, rl v1.ResourceList) v1.ResourceList { - rl[v1.ResourceName(rName)] = resource.MustParse(value) - return rl -} - -func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements { - res := v1.ResourceRequirements{} - res.Requests = requests - res.Limits = limits - return res -} - -func newContainer(name string, requests v1.ResourceList, limits v1.ResourceList) v1.Container { - return v1.Container{ - Name: name, - Resources: getResourceRequirements(requests, limits), - } -} - -func newPod(name string, containers []v1.Container) *v1.Pod { - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: v1.PodSpec{ - Containers: containers, - }, - } -} - func TestGetPodQOS(t *testing.T) { testCases := []struct { pod *v1.Pod @@ -173,5 +134,54 @@ func TestGetPodQOS(t *testing.T) { if actual := GetPodQOS(testCase.pod); testCase.expected != actual { t.Errorf("[%d]: invalid qos pod %s, expected: %s, actual: %s", id, testCase.pod.Name, testCase.expected, actual) } + + // Convert v1.Pod to api.Pod, and then check against `api.helper.GetPodQOS`. + pod := api.Pod{} + v1.Convert_v1_Pod_To_api_Pod(testCase.pod, &pod, nil) + + if actual := qos.GetPodQOS(&pod); api.PodQOSClass(testCase.expected) != actual { + t.Errorf("[%d]: invalid qos pod %s, expected: %s, actual: %s", id, testCase.pod.Name, testCase.expected, actual) + } + } +} + +func getResourceList(cpu, memory string) v1.ResourceList { + res := v1.ResourceList{} + if cpu != "" { + res[v1.ResourceCPU] = resource.MustParse(cpu) + } + if memory != "" { + res[v1.ResourceMemory] = resource.MustParse(memory) + } + return res +} + +func addResource(rName, value string, rl v1.ResourceList) v1.ResourceList { + rl[v1.ResourceName(rName)] = resource.MustParse(value) + return rl +} + +func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements { + res := v1.ResourceRequirements{} + res.Requests = requests + res.Limits = limits + return res +} + +func newContainer(name string, requests v1.ResourceList, limits v1.ResourceList) v1.Container { + return v1.Container{ + Name: name, + Resources: getResourceRequirements(requests, limits), + } +} + +func newPod(name string, containers []v1.Container) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.PodSpec{ + Containers: containers, + }, } } diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 3bb4a3443f0..71d68e170f2 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -38,6 +38,7 @@ go_library( "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/api/v1/helper:go_default_library", + "//pkg/api/v1/helper/qos:go_default_library", "//pkg/api/v1/pod:go_default_library", "//pkg/api/v1/resource:go_default_library", "//pkg/api/v1/validation:go_default_library", @@ -72,7 +73,6 @@ go_library( "//pkg/kubelet/preemption:go_default_library", "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/prober/results:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/remote:go_default_library", "//pkg/kubelet/rkt:go_default_library", "//pkg/kubelet/secret:go_default_library", diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 367fd7c86e1..ce6fdc59d8d 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -25,6 +25,7 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/api/v1:go_default_library", + "//pkg/api/v1/helper/qos:go_default_library", "//pkg/api/v1/resource:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", diff --git a/pkg/kubelet/cm/helpers_linux.go b/pkg/kubelet/cm/helpers_linux.go index ca2f7235ca0..a535c6b6334 100644 --- a/pkg/kubelet/cm/helpers_linux.go +++ b/pkg/kubelet/cm/helpers_linux.go @@ -26,8 +26,8 @@ import ( libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" "k8s.io/kubernetes/pkg/api/v1" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" "k8s.io/kubernetes/pkg/api/v1/resource" - "k8s.io/kubernetes/pkg/kubelet/qos" ) const ( @@ -121,7 +121,7 @@ func ResourceConfigForPod(pod *v1.Pod) *ResourceConfig { } // determine the qos class - qosClass := qos.GetPodQOS(pod) + qosClass := v1qos.GetPodQOS(pod) // build the result result := &ResourceConfig{} diff --git a/pkg/kubelet/cm/pod_container_manager_linux.go b/pkg/kubelet/cm/pod_container_manager_linux.go index f59495f5f0f..d45ab29e792 100644 --- a/pkg/kubelet/cm/pod_container_manager_linux.go +++ b/pkg/kubelet/cm/pod_container_manager_linux.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/kubelet/qos" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" ) const ( @@ -93,7 +93,7 @@ func (m *podContainerManagerImpl) EnsureExists(pod *v1.Pod) error { // GetPodContainerName returns the CgroupName identifer, and its literal cgroupfs form on the host. func (m *podContainerManagerImpl) GetPodContainerName(pod *v1.Pod) (CgroupName, string) { - podQOS := qos.GetPodQOS(pod) + podQOS := v1qos.GetPodQOS(pod) // Get the parent QOS container name var parentContainer string switch podQOS { diff --git a/pkg/kubelet/cm/qos_container_manager_linux.go b/pkg/kubelet/cm/qos_container_manager_linux.go index 8039aa12f90..6238531ebd5 100644 --- a/pkg/kubelet/cm/qos_container_manager_linux.go +++ b/pkg/kubelet/cm/qos_container_manager_linux.go @@ -28,8 +28,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/api/v1" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" "k8s.io/kubernetes/pkg/api/v1/resource" - "k8s.io/kubernetes/pkg/kubelet/qos" ) const ( @@ -143,7 +143,7 @@ func (m *qosContainerManagerImpl) setCPUCgroupConfig(configs map[v1.PodQOSClass] burstablePodCPURequest := int64(0) for i := range pods { pod := pods[i] - qosClass := qos.GetPodQOS(pod) + qosClass := v1qos.GetPodQOS(pod) if qosClass != v1.PodQOSBurstable { // we only care about the burstable qos tier continue @@ -183,7 +183,7 @@ func (m *qosContainerManagerImpl) setMemoryReserve(configs map[v1.PodQOSClass]*C pods := m.activePods() for _, pod := range pods { podMemoryRequest := int64(0) - qosClass := qos.GetPodQOS(pod) + qosClass := v1qos.GetPodQOS(pod) if qosClass == v1.PodQOSBestEffort { // limits are not set for Best Effort pods continue diff --git a/pkg/kubelet/eviction/BUILD b/pkg/kubelet/eviction/BUILD index 511b9462e52..2202e635bd8 100644 --- a/pkg/kubelet/eviction/BUILD +++ b/pkg/kubelet/eviction/BUILD @@ -61,6 +61,7 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", + "//pkg/api/v1/helper/qos:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", @@ -68,7 +69,6 @@ go_library( "//pkg/kubelet/lifecycle:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/pod:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index 6cdb87a5630..ec3e1a13513 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -31,13 +31,13 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/clock" "k8s.io/kubernetes/pkg/api/v1" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/metrics" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/server/stats" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" @@ -119,7 +119,7 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd } // the node has memory pressure, admit if not best-effort if hasNodeCondition(m.nodeConditions, v1.NodeMemoryPressure) { - notBestEffort := v1.PodQOSBestEffort != qos.GetPodQOS(attrs.Pod) + notBestEffort := v1.PodQOSBestEffort != v1qos.GetPodQOS(attrs.Pod) if notBestEffort { return lifecycle.PodAdmitResult{Admit: true} } diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 72aa9d30b4f..2f37b19cd08 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -28,10 +28,10 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/quota/evaluator/core" ) @@ -513,8 +513,8 @@ func (ms *multiSorter) Less(i, j int) bool { // qosComparator compares pods by QoS (BestEffort < Burstable < Guaranteed) func qosComparator(p1, p2 *v1.Pod) int { - qosP1 := qos.GetPodQOS(p1) - qosP2 := qos.GetPodQOS(p2) + qosP1 := v1qos.GetPodQOS(p1) + qosP2 := v1qos.GetPodQOS(p2) // its a tie if qosP1 == qosP2 { return 0 diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 0af466cbcad..62a2afcad32 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -44,6 +44,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" v1helper "k8s.io/kubernetes/pkg/api/v1/helper" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/pkg/api/v1/validation" @@ -52,7 +53,6 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/envvars" "k8s.io/kubernetes/pkg/kubelet/images" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/server/portforward" remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/kubelet/status" @@ -1225,7 +1225,7 @@ func (kl *Kubelet) convertStatusToAPIStatus(pod *v1.Pod, podStatus *kubecontaine var apiPodStatus v1.PodStatus apiPodStatus.PodIP = podStatus.IP // set status for Pods created on versions of kube older than 1.6 - apiPodStatus.QOSClass = qos.GetPodQOS(pod) + apiPodStatus.QOSClass = v1qos.GetPodQOS(pod) apiPodStatus.ContainerStatuses = kl.convertToAPIContainerStatuses( pod, podStatus, diff --git a/pkg/kubelet/preemption/BUILD b/pkg/kubelet/preemption/BUILD index 30c1d012afc..792263d8321 100644 --- a/pkg/kubelet/preemption/BUILD +++ b/pkg/kubelet/preemption/BUILD @@ -14,12 +14,12 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/api/v1:go_default_library", + "//pkg/api/v1/helper/qos:go_default_library", "//pkg/api/v1/resource:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/eviction:go_default_library", "//pkg/kubelet/lifecycle:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//plugin/pkg/scheduler/algorithm:go_default_library", diff --git a/pkg/kubelet/preemption/preemption.go b/pkg/kubelet/preemption/preemption.go index c84abfc1a0e..7343a306306 100644 --- a/pkg/kubelet/preemption/preemption.go +++ b/pkg/kubelet/preemption/preemption.go @@ -24,12 +24,12 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/v1" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" - "k8s.io/kubernetes/pkg/kubelet/qos" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" @@ -231,7 +231,7 @@ func (a admissionRequirementList) toString() string { func sortPodsByQOS(pods []*v1.Pod) (bestEffort, burstable, guaranteed []*v1.Pod) { for _, pod := range pods { if !kubetypes.IsCriticalPod(pod) { - switch qos.GetPodQOS(pod) { + switch v1qos.GetPodQOS(pod) { case v1.PodQOSBestEffort: bestEffort = append(bestEffort, pod) case v1.PodQOSBurstable: diff --git a/pkg/kubelet/qos/BUILD b/pkg/kubelet/qos/BUILD index 283a0dd5c3e..386a9ba3cb6 100644 --- a/pkg/kubelet/qos/BUILD +++ b/pkg/kubelet/qos/BUILD @@ -10,16 +10,12 @@ load( go_test( name = "go_default_test", - srcs = [ - "policy_test.go", - "qos_test.go", - ], + srcs = ["policy_test.go"], library = ":go_default_library", tags = ["automanaged"], deps = [ "//pkg/api/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ], ) @@ -28,14 +24,11 @@ go_library( srcs = [ "doc.go", "policy.go", - "qos.go", ], tags = ["automanaged"], deps = [ - "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//pkg/api/v1/helper/qos:go_default_library", ], ) diff --git a/pkg/kubelet/qos/policy.go b/pkg/kubelet/qos/policy.go index 8eb97b80093..8c3d8f76e0f 100644 --- a/pkg/kubelet/qos/policy.go +++ b/pkg/kubelet/qos/policy.go @@ -16,7 +16,10 @@ limitations under the License. package qos -import "k8s.io/kubernetes/pkg/api/v1" +import ( + "k8s.io/kubernetes/pkg/api/v1" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" +) const ( // PodInfraOOMAdj is very docker specific. For arbitrary runtime, it may not make @@ -38,7 +41,7 @@ const ( // and 1000. Containers with higher OOM scores are killed if the system runs out of memory. // See https://lwn.net/Articles/391222/ for more information. func GetContainerOOMScoreAdjust(pod *v1.Pod, container *v1.Container, memoryCapacity int64) int { - switch GetPodQOS(pod) { + switch v1qos.GetPodQOS(pod) { case v1.PodQOSGuaranteed: // Guaranteed containers should be the last to get killed. return guaranteedOOMScoreAdj diff --git a/pkg/printers/internalversion/BUILD b/pkg/printers/internalversion/BUILD index cbd60408da5..05b07b46601 100644 --- a/pkg/printers/internalversion/BUILD +++ b/pkg/printers/internalversion/BUILD @@ -62,6 +62,7 @@ go_library( "//pkg/api:go_default_library", "//pkg/api/events:go_default_library", "//pkg/api/helper:go_default_library", + "//pkg/api/helper/qos:go_default_library", "//pkg/api/ref:go_default_library", "//pkg/api/resource:go_default_library", "//pkg/apis/apps:go_default_library", @@ -83,7 +84,6 @@ go_library( "//pkg/controller:go_default_library", "//pkg/controller/deployment/util:go_default_library", "//pkg/fieldpath:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/printers:go_default_library", "//pkg/util/node:go_default_library", "//pkg/util/slice:go_default_library", diff --git a/pkg/printers/internalversion/describe.go b/pkg/printers/internalversion/describe.go index ec60fbf2aaa..1227223489f 100644 --- a/pkg/printers/internalversion/describe.go +++ b/pkg/printers/internalversion/describe.go @@ -49,6 +49,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/events" "k8s.io/kubernetes/pkg/api/helper" + "k8s.io/kubernetes/pkg/api/helper/qos" "k8s.io/kubernetes/pkg/api/ref" resourcehelper "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/apis/apps" @@ -68,7 +69,6 @@ import ( "k8s.io/kubernetes/pkg/controller" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" "k8s.io/kubernetes/pkg/fieldpath" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/printers" "k8s.io/kubernetes/pkg/util/slice" ) @@ -649,7 +649,7 @@ func describePod(pod *api.Pod, events *api.EventList) (string, error) { if pod.Status.QOSClass != "" { w.Write(LEVEL_0, "QoS Class:\t%s\n", pod.Status.QOSClass) } else { - w.Write(LEVEL_0, "QoS Class:\t%s\n", qos.InternalGetPodQOS(pod)) + w.Write(LEVEL_0, "QoS Class:\t%s\n", qos.GetPodQOS(pod)) } printLabelsMultiline(w, "Node-Selectors", pod.Spec.NodeSelector) printPodTolerationsMultiline(w, "Tolerations", pod.Spec.Tolerations) diff --git a/pkg/quota/evaluator/core/BUILD b/pkg/quota/evaluator/core/BUILD index b6f5363b658..82c5d489754 100644 --- a/pkg/quota/evaluator/core/BUILD +++ b/pkg/quota/evaluator/core/BUILD @@ -25,11 +25,11 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/api/helper:go_default_library", + "//pkg/api/helper/qos:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/api/validation:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/client/informers/informers_generated/externalversions:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//pkg/quota:go_default_library", "//pkg/quota/generic:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/pkg/quota/evaluator/core/pods.go b/pkg/quota/evaluator/core/pods.go index 403238f0265..85ca0279bea 100644 --- a/pkg/quota/evaluator/core/pods.go +++ b/pkg/quota/evaluator/core/pods.go @@ -28,11 +28,11 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/admission" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/helper/qos" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/api/validation" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/quota" "k8s.io/kubernetes/pkg/quota/generic" ) @@ -257,7 +257,7 @@ func PodUsageFunc(obj runtime.Object) (api.ResourceList, error) { } func isBestEffort(pod *api.Pod) bool { - return qos.InternalGetPodQOS(pod) == api.PodQOSBestEffort + return qos.GetPodQOS(pod) == api.PodQOSBestEffort } func isTerminating(pod *api.Pod) bool { diff --git a/pkg/registry/core/pod/BUILD b/pkg/registry/core/pod/BUILD index a18862a2145..429a1a9d672 100644 --- a/pkg/registry/core/pod/BUILD +++ b/pkg/registry/core/pod/BUILD @@ -17,9 +17,9 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/api:go_default_library", + "//pkg/api/helper/qos:go_default_library", "//pkg/api/validation:go_default_library", "//pkg/kubelet/client:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", diff --git a/pkg/registry/core/pod/strategy.go b/pkg/registry/core/pod/strategy.go index 6d73e1c7835..60fd93fed6e 100644 --- a/pkg/registry/core/pod/strategy.go +++ b/pkg/registry/core/pod/strategy.go @@ -38,9 +38,9 @@ import ( "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/helper/qos" "k8s.io/kubernetes/pkg/api/validation" "k8s.io/kubernetes/pkg/kubelet/client" - "k8s.io/kubernetes/pkg/kubelet/qos" ) // podStrategy implements behavior for Pods @@ -63,7 +63,7 @@ func (podStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.O pod := obj.(*api.Pod) pod.Status = api.PodStatus{ Phase: api.PodPending, - QOSClass: qos.InternalGetPodQOS(pod), + QOSClass: qos.GetPodQOS(pod), } } diff --git a/plugin/pkg/scheduler/algorithm/predicates/BUILD b/plugin/pkg/scheduler/algorithm/predicates/BUILD index 1fab4ae2294..77880a33167 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/BUILD +++ b/plugin/pkg/scheduler/algorithm/predicates/BUILD @@ -20,8 +20,8 @@ go_library( deps = [ "//pkg/api/v1:go_default_library", "//pkg/api/v1/helper:go_default_library", + "//pkg/api/v1/helper/qos:go_default_library", "//pkg/client/listers/core/v1:go_default_library", - "//pkg/kubelet/qos:go_default_library", "//plugin/pkg/scheduler/algorithm:go_default_library", "//plugin/pkg/scheduler/algorithm/priorities/util:go_default_library", "//plugin/pkg/scheduler/schedulercache:go_default_library", diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 6fe27210025..6a86bebaf6a 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -32,8 +32,8 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/kubernetes/pkg/api/v1" v1helper "k8s.io/kubernetes/pkg/api/v1/helper" + v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1" - "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" priorityutil "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities/util" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" @@ -1230,7 +1230,7 @@ func PodToleratesNodeTaints(pod *v1.Pod, meta interface{}, nodeInfo *schedulerca // Determine if a pod is scheduled with best-effort QoS func isPodBestEffort(pod *v1.Pod) bool { - return qos.GetPodQOS(pod) == v1.PodQOSBestEffort + return v1qos.GetPodQOS(pod) == v1.PodQOSBestEffort } // CheckNodeMemoryPressurePredicate checks if a pod can be scheduled on a node From fd0190fd685d648ab2c4ddc5802d14ad8813d07f Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Sat, 20 May 2017 07:49:53 -0400 Subject: [PATCH 2/2] generated client-go. --- .../k8s.io/client-go/pkg/api/helper/qos/BUILD | 20 ++++ .../client-go/pkg/api/helper/qos/qos.go | 94 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 staging/src/k8s.io/client-go/pkg/api/helper/qos/BUILD create mode 100644 staging/src/k8s.io/client-go/pkg/api/helper/qos/qos.go diff --git a/staging/src/k8s.io/client-go/pkg/api/helper/qos/BUILD b/staging/src/k8s.io/client-go/pkg/api/helper/qos/BUILD new file mode 100644 index 00000000000..d3b18aacaea --- /dev/null +++ b/staging/src/k8s.io/client-go/pkg/api/helper/qos/BUILD @@ -0,0 +1,20 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["qos.go"], + tags = ["automanaged"], + visibility = ["//visibility:private"], + deps = [ + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/client-go/pkg/api:go_default_library", + ], +) diff --git a/staging/src/k8s.io/client-go/pkg/api/helper/qos/qos.go b/staging/src/k8s.io/client-go/pkg/api/helper/qos/qos.go new file mode 100644 index 00000000000..adc19c68bde --- /dev/null +++ b/staging/src/k8s.io/client-go/pkg/api/helper/qos/qos.go @@ -0,0 +1,94 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// NOTE: DO NOT use those helper functions through client-go, the +// package path will be changed in the future. +package qos + +import ( + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/pkg/api" +) + +// supportedComputeResources is the list of compute resources for with QoS is supported. +var supportedQoSComputeResources = sets.NewString(string(api.ResourceCPU), string(api.ResourceMemory)) + +// GetPodQOS returns the QoS class of a pod. +// A pod is besteffort if none of its containers have specified any requests or limits. +// A pod is guaranteed only when requests and limits are specified for all the containers and they are equal. +// A pod is burstable if limits and requests do not match across all containers. +func GetPodQOS(pod *api.Pod) api.PodQOSClass { + requests := api.ResourceList{} + limits := api.ResourceList{} + zeroQuantity := resource.MustParse("0") + isGuaranteed := true + for _, container := range pod.Spec.Containers { + // process requests + for name, quantity := range container.Resources.Requests { + if !supportedQoSComputeResources.Has(string(name)) { + continue + } + if quantity.Cmp(zeroQuantity) == 1 { + delta := quantity.Copy() + if _, exists := requests[name]; !exists { + requests[name] = *delta + } else { + delta.Add(requests[name]) + requests[name] = *delta + } + } + } + // process limits + qosLimitsFound := sets.NewString() + for name, quantity := range container.Resources.Limits { + if !supportedQoSComputeResources.Has(string(name)) { + continue + } + if quantity.Cmp(zeroQuantity) == 1 { + qosLimitsFound.Insert(string(name)) + delta := quantity.Copy() + if _, exists := limits[name]; !exists { + limits[name] = *delta + } else { + delta.Add(limits[name]) + limits[name] = *delta + } + } + } + + if len(qosLimitsFound) != len(supportedQoSComputeResources) { + isGuaranteed = false + } + } + if len(requests) == 0 && len(limits) == 0 { + return api.PodQOSBestEffort + } + // Check is requests match limits for all resources. + if isGuaranteed { + for name, req := range requests { + if lim, exists := limits[name]; !exists || lim.Cmp(req) != 0 { + isGuaranteed = false + break + } + } + } + if isGuaranteed && + len(requests) == len(limits) { + return api.PodQOSGuaranteed + } + return api.PodQOSBurstable +}