From dab7280ae4656d9a05207571a71c56b020aa35c8 Mon Sep 17 00:00:00 2001 From: dinghaiyang Date: Fri, 24 Jul 2015 09:27:29 +0800 Subject: [PATCH] Improve clarity around PodFitsResource by showing pods limits in `kubectl describe node` --- pkg/kubectl/describe.go | 27 +++++++++++++++-- pkg/kubelet/kubelet.go | 26 ++++++++++------ .../algorithm/predicates/predicates.go | 30 +++++++++++++++---- plugin/pkg/scheduler/generic_scheduler.go | 5 ++++ 4 files changed, 71 insertions(+), 17 deletions(-) diff --git a/pkg/kubectl/describe.go b/pkg/kubectl/describe.go index 083bd45dbb8..0639f4d624f 100644 --- a/pkg/kubectl/describe.go +++ b/pkg/kubectl/describe.go @@ -878,10 +878,33 @@ func describeNode(node *api.Node, pods []*api.Pod, events *api.EventList) (strin fmt.Fprintf(out, "ExternalID:\t%s\n", node.Spec.ExternalID) } fmt.Fprintf(out, "Pods:\t(%d in total)\n", len(pods)) - fmt.Fprint(out, " Namespace\tName\n") + fmt.Fprint(out, " Namespace\tName\t\tCPU(milliCPU)\t\tMemory(bytes)\n") + totalMilliCPU := int64(0) + totalMemory := int64(0) + fractionPodCPU := float64(0) + fractionPodMemory := float64(0) + fractionTotalCPU := float64(0) + fractionTotalMemory := float64(0) for _, pod := range pods { - fmt.Fprintf(out, " %s\t%s\n", pod.Namespace, pod.Name) + podTotalMilliCPU := int64(0) + podTotalMemory := int64(0) + + for ix := range pod.Spec.Containers { + limits := pod.Spec.Containers[ix].Resources.Limits + podTotalMilliCPU += limits.Cpu().MilliValue() + podTotalMemory += limits.Memory().Value() + } + totalMilliCPU += podTotalMilliCPU + totalMemory += podTotalMemory + fractionPodCPU = float64(podTotalMilliCPU) / float64(node.Status.Capacity.Cpu().MilliValue()) * 100 + fractionPodMemory = float64(podTotalMemory) / float64(node.Status.Capacity.Memory().Value()) * 100 + fmt.Fprintf(out, " %s\t%s\t\t%d (%d%% of total)\t\t%d (%d%% of total)\n", pod.Namespace, pod.Name, podTotalMilliCPU, int64(fractionPodCPU), podTotalMemory, int64(fractionPodMemory)) } + fmt.Fprint(out, "TotalResourceLimits:\n") + fractionTotalCPU = float64(totalMilliCPU) / float64(node.Status.Capacity.Cpu().MilliValue()) * 100 + fractionTotalMemory = float64(totalMemory) / float64(node.Status.Capacity.Memory().Value()) * 100 + fmt.Fprintf(out, " CPU(milliCPU):\t\t%d (%d%% of total)\n", totalMilliCPU, int64(fractionTotalCPU)) + fmt.Fprintf(out, " Memory(bytes):\t\t%d (%d%% of total)\n", totalMemory, int64(fractionTotalMemory)) if events != nil { DescribeEvents(events, out) } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 179a89200cc..a2fa37ab291 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1568,19 +1568,19 @@ func checkHostPortConflicts(pods []*api.Pod) (fitting []*api.Pod, notFitting []* return } -// checkCapacityExceeded detects pods that exceeds node's resources. -func (kl *Kubelet) checkCapacityExceeded(pods []*api.Pod) (fitting []*api.Pod, notFitting []*api.Pod) { +// checkSufficientfFreeResources detects pods that exceeds node's resources. +func (kl *Kubelet) checkSufficientfFreeResources(pods []*api.Pod) (fitting []*api.Pod, notFittingCPU, notFittingMemory []*api.Pod) { info, err := kl.GetCachedMachineInfo() if err != nil { glog.Errorf("error getting machine info: %v", err) - return pods, nil + return pods, nil, nil } // Respect the pod creation order when resolving conflicts. sort.Sort(podsByCreationTime(pods)) capacity := CapacityFromMachineInfo(info) - return predicates.CheckPodsExceedingCapacity(pods, capacity) + return predicates.CheckPodsExceedingFreeResources(pods, capacity) } // handleOutOfDisk detects if pods can't fit due to lack of disk space. @@ -1673,14 +1673,22 @@ func (kl *Kubelet) handleNotFittingPods(pods []*api.Pod) []*api.Pod { Reason: reason, Message: "Pod cannot be started due to node selector mismatch"}) } - fitting, notFitting = kl.checkCapacityExceeded(fitting) - for _, pod := range notFitting { - reason := "CapacityExceeded" - kl.recorder.Eventf(pod, reason, "Cannot start the pod due to exceeded capacity.") + fitting, notFittingCPU, notFittingMemory := kl.checkSufficientfFreeResources(fitting) + for _, pod := range notFittingCPU { + reason := "InsufficientFreeCPU" + kl.recorder.Eventf(pod, reason, "Cannot start the pod due to insufficient free CPU.") kl.statusManager.SetPodStatus(pod, api.PodStatus{ Phase: api.PodFailed, Reason: reason, - Message: "Pod cannot be started due to exceeded capacity"}) + Message: "Pod cannot be started due to insufficient free CPU"}) + } + for _, pod := range notFittingMemory { + reason := "InsufficientFreeMemory" + kl.recorder.Eventf(pod, reason, "Cannot start the pod due to insufficient free memory.") + kl.statusManager.SetPodStatus(pod, api.PodStatus{ + Phase: api.PodFailed, + Reason: reason, + Message: "Pod cannot be started due to insufficient free memory"}) } return fitting } diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index ca4e2e336a9..ac557120ac6 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -105,6 +105,8 @@ type resourceRequest struct { memory int64 } +var FailedResourceType string + func getResourceRequest(pod *api.Pod) resourceRequest { result := resourceRequest{} for ix := range pod.Spec.Containers { @@ -115,7 +117,7 @@ func getResourceRequest(pod *api.Pod) resourceRequest { return result } -func CheckPodsExceedingCapacity(pods []*api.Pod, capacity api.ResourceList) (fitting []*api.Pod, notFitting []*api.Pod) { +func CheckPodsExceedingFreeResources(pods []*api.Pod, capacity api.ResourceList) (fitting []*api.Pod, notFittingCPU, notFittingMemory []*api.Pod) { totalMilliCPU := capacity.Cpu().MilliValue() totalMemory := capacity.Memory().Value() milliCPURequested := int64(0) @@ -124,9 +126,14 @@ func CheckPodsExceedingCapacity(pods []*api.Pod, capacity api.ResourceList) (fit podRequest := getResourceRequest(pod) fitsCPU := totalMilliCPU == 0 || (totalMilliCPU-milliCPURequested) >= podRequest.milliCPU fitsMemory := totalMemory == 0 || (totalMemory-memoryRequested) >= podRequest.memory - if !fitsCPU || !fitsMemory { - // the pod doesn't fit - notFitting = append(notFitting, pod) + if !fitsCPU { + // the pod doesn't fit due to CPU limit + notFittingCPU = append(notFittingCPU, pod) + continue + } + if !fitsMemory { + // the pod doesn't fit due to Memory limit + notFittingMemory = append(notFittingMemory, pod) continue } // the pod fits @@ -150,9 +157,20 @@ func (r *ResourceFit) PodFitsResources(pod *api.Pod, existingPods []*api.Pod, no pods := []*api.Pod{} copy(pods, existingPods) pods = append(existingPods, pod) - _, exceeding := CheckPodsExceedingCapacity(pods, info.Status.Capacity) - if len(exceeding) > 0 || int64(len(pods)) > info.Status.Capacity.Pods().Value() { + _, exceedingCPU, exceedingMemory := CheckPodsExceedingFreeResources(pods, info.Status.Capacity) + if int64(len(pods)) > info.Status.Capacity.Pods().Value() { glog.V(4).Infof("Cannot schedule Pod %v, because Node %v is full, running %v out of %v Pods.", pod, node, len(pods)-1, info.Status.Capacity.Pods().Value()) + FailedResourceType = "PodExceedsMaxPodNumber" + return false, nil + } + if len(exceedingCPU) > 0 { + glog.V(4).Infof("Cannot schedule Pod %v, because Node does not have sufficient CPU", pod) + FailedResourceType = "PodExceedsFreeCPU" + return false, nil + } + if len(exceedingMemory) > 0 { + glog.V(4).Infof("Cannot schedule Pod %v, because Node does not have sufficient Memory", pod) + FailedResourceType = "PodExceedsFreeMemory" return false, nil } glog.V(4).Infof("Schedule Pod %v on Node %v is allowed, Node is running only %v out of %v Pods.", pod, node, len(pods)-1, info.Status.Capacity.Pods().Value()) diff --git a/plugin/pkg/scheduler/generic_scheduler.go b/plugin/pkg/scheduler/generic_scheduler.go index cf0eb213d26..07a2e855162 100644 --- a/plugin/pkg/scheduler/generic_scheduler.go +++ b/plugin/pkg/scheduler/generic_scheduler.go @@ -116,6 +116,7 @@ func findNodesThatFit(pod *api.Pod, podLister algorithm.PodLister, predicateFunc for _, node := range nodes.Items { fits := true for name, predicate := range predicateFuncs { + predicates.FailedResourceType = "" fit, err := predicate(pod, machineToPods[node.Name], node.Name) if err != nil { return api.NodeList{}, FailedPredicateMap{}, err @@ -125,6 +126,10 @@ func findNodesThatFit(pod *api.Pod, podLister algorithm.PodLister, predicateFunc if _, found := failedPredicateMap[node.Name]; !found { failedPredicateMap[node.Name] = util.StringSet{} } + if predicates.FailedResourceType != "" { + failedPredicateMap[node.Name].Insert(predicates.FailedResourceType) + break + } failedPredicateMap[node.Name].Insert(name) break }