mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-28 14:07:14 +00:00
Added static cpumanager policy.
This commit is contained in:
parent
e03a6435bb
commit
d0bcbbb437
@ -327,7 +327,7 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
|
|||||||
fs.BoolVar(&c.CgroupsPerQOS, "cgroups-per-qos", c.CgroupsPerQOS, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
|
fs.BoolVar(&c.CgroupsPerQOS, "cgroups-per-qos", c.CgroupsPerQOS, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
|
||||||
fs.StringVar(&c.CgroupDriver, "cgroup-driver", c.CgroupDriver, "Driver that the kubelet uses to manipulate cgroups on the host. Possible values: 'cgroupfs', 'systemd'")
|
fs.StringVar(&c.CgroupDriver, "cgroup-driver", c.CgroupDriver, "Driver that the kubelet uses to manipulate cgroups on the host. Possible values: 'cgroupfs', 'systemd'")
|
||||||
fs.StringVar(&c.CgroupRoot, "cgroup-root", c.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.")
|
fs.StringVar(&c.CgroupRoot, "cgroup-root", c.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.")
|
||||||
fs.StringVar(&c.CPUManagerPolicy, "cpu-manager-policy", c.CPUManagerPolicy, "<Warning: Alpha feature> CPU Manager policy to use. Possible values: 'none'. Default: 'none'")
|
fs.StringVar(&c.CPUManagerPolicy, "cpu-manager-policy", c.CPUManagerPolicy, "<Warning: Alpha feature> CPU Manager policy to use. Possible values: 'none', 'static'. Default: 'none'")
|
||||||
fs.DurationVar(&c.CPUManagerReconcilePeriod.Duration, "cpu-manager-reconcile-period", c.CPUManagerReconcilePeriod.Duration, "<Warning: Alpha feature> CPU Manager reconciliation period. Examples: '10s', or '1m'. If not supplied, defaults to `NodeStatusUpdateFrequency`")
|
fs.DurationVar(&c.CPUManagerReconcilePeriod.Duration, "cpu-manager-reconcile-period", c.CPUManagerReconcilePeriod.Duration, "<Warning: Alpha feature> CPU Manager reconciliation period. Examples: '10s', or '1m'. If not supplied, defaults to `NodeStatusUpdateFrequency`")
|
||||||
fs.StringVar(&c.ContainerRuntime, "container-runtime", c.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'.")
|
fs.StringVar(&c.ContainerRuntime, "container-runtime", c.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'.")
|
||||||
fs.DurationVar(&c.RuntimeRequestTimeout.Duration, "runtime-request-timeout", c.RuntimeRequestTimeout.Duration, "Timeout of all runtime requests except long running request - pull, logs, exec and attach. When timeout exceeded, kubelet will cancel the request, throw out an error and retry later.")
|
fs.DurationVar(&c.RuntimeRequestTimeout.Duration, "runtime-request-timeout", c.RuntimeRequestTimeout.Duration, "Timeout of all runtime requests except long running request - pull, logs, exec and attach. When timeout exceeded, kubelet will cancel the request, throw out an error and retry later.")
|
||||||
|
@ -3,15 +3,19 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
|||||||
go_library(
|
go_library(
|
||||||
name = "go_default_library",
|
name = "go_default_library",
|
||||||
srcs = [
|
srcs = [
|
||||||
|
"cpu_assignment.go",
|
||||||
"cpu_manager.go",
|
"cpu_manager.go",
|
||||||
"fake_cpu_manager.go",
|
"fake_cpu_manager.go",
|
||||||
"policy.go",
|
"policy.go",
|
||||||
"policy_none.go",
|
"policy_none.go",
|
||||||
|
"policy_static.go",
|
||||||
],
|
],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
deps = [
|
deps = [
|
||||||
|
"//pkg/api/v1/helper/qos:go_default_library",
|
||||||
"//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
|
"//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
|
||||||
"//pkg/kubelet/cm/cpumanager/state:go_default_library",
|
"//pkg/kubelet/cm/cpumanager/state:go_default_library",
|
||||||
|
"//pkg/kubelet/cm/cpumanager/topology:go_default_library",
|
||||||
"//pkg/kubelet/cm/cpuset:go_default_library",
|
"//pkg/kubelet/cm/cpuset:go_default_library",
|
||||||
"//pkg/kubelet/container:go_default_library",
|
"//pkg/kubelet/container:go_default_library",
|
||||||
"//pkg/kubelet/status:go_default_library",
|
"//pkg/kubelet/status:go_default_library",
|
||||||
|
@ -18,6 +18,7 @@ package cpumanager
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -28,6 +29,7 @@ import (
|
|||||||
|
|
||||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
|
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/status"
|
"k8s.io/kubernetes/pkg/kubelet/status"
|
||||||
@ -105,8 +107,36 @@ func NewManager(
|
|||||||
case PolicyNone:
|
case PolicyNone:
|
||||||
policy = NewNonePolicy()
|
policy = NewNonePolicy()
|
||||||
|
|
||||||
|
case PolicyStatic:
|
||||||
|
topo, err := topology.Discover(machineInfo)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
glog.Infof("[cpumanager] detected CPU topology: %v", topo)
|
||||||
|
reservedCPUs, ok := nodeAllocatableReservation[v1.ResourceCPU]
|
||||||
|
if !ok {
|
||||||
|
// The static policy cannot initialize without this information. Panic!
|
||||||
|
panic("[cpumanager] unable to determine reserved CPU resources for static policy")
|
||||||
|
}
|
||||||
|
if reservedCPUs.IsZero() {
|
||||||
|
// Panic!
|
||||||
|
//
|
||||||
|
// The static policy requires this to be nonzero. Zero CPU reservation
|
||||||
|
// would allow the shared pool to be completely exhausted. At that point
|
||||||
|
// either we would violate our guarantee of exclusivity or need to evict
|
||||||
|
// any pod that has at least one container that requires zero CPUs.
|
||||||
|
// See the comments in policy_static.go for more details.
|
||||||
|
panic("[cpumanager] the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Take the ceiling of the reservation, since fractional CPUs cannot be
|
||||||
|
// exclusively allocated.
|
||||||
|
reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
|
||||||
|
numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
|
||||||
|
policy = NewStaticPolicy(topo, numReservedCPUs)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
glog.Warningf("[cpumanager] Unknown policy (\"%s\"), falling back to \"%s\" policy (\"%s\")", cpuPolicyName, PolicyNone)
|
glog.Errorf("[cpumanager] Unknown policy \"%s\", falling back to default policy \"%s\"", cpuPolicyName, PolicyNone)
|
||||||
policy = NewNonePolicy()
|
policy = NewNonePolicy()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
172
pkg/kubelet/cm/cpumanager/policy_static.go
Normal file
172
pkg/kubelet/cm/cpumanager/policy_static.go
Normal file
@ -0,0 +1,172 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2017 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package cpumanager
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/golang/glog"
|
||||||
|
"k8s.io/api/core/v1"
|
||||||
|
v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PolicyStatic is the name of the static policy
|
||||||
|
const PolicyStatic policyName = "static"
|
||||||
|
|
||||||
|
var _ Policy = &staticPolicy{}
|
||||||
|
|
||||||
|
// staticPolicy is a CPU manager policy that does not change CPU
|
||||||
|
// assignments for exclusively pinned guaranteed containers after the main
|
||||||
|
// container process starts.
|
||||||
|
//
|
||||||
|
// This policy allocates CPUs exclusively for a container if all the following
|
||||||
|
// conditions are met:
|
||||||
|
//
|
||||||
|
// - The pod QoS class is Guaranteed.
|
||||||
|
// - The CPU request is a positive integer.
|
||||||
|
//
|
||||||
|
// The static policy maintains the following sets of logical CPUs:
|
||||||
|
//
|
||||||
|
// - SHARED: Burstable, BestEffort, and non-integral Guaranteed containers
|
||||||
|
// run here. Initially this contains all CPU IDs on the system. As
|
||||||
|
// exclusive allocations are created and destroyed, this CPU set shrinks
|
||||||
|
// and grows, accordingly. This is stored in the state as the default
|
||||||
|
// CPU set.
|
||||||
|
//
|
||||||
|
// - RESERVED: A subset of the shared pool which is not exclusively
|
||||||
|
// allocatable. The membership of this pool is static for the lifetime of
|
||||||
|
// the Kubelet. The size of the reserved pool is
|
||||||
|
// ceil(systemreserved.cpu + kubereserved.cpu).
|
||||||
|
// Reserved CPUs are taken topologically starting with lowest-indexed
|
||||||
|
// physical core, as reported by cAdvisor.
|
||||||
|
//
|
||||||
|
// - ASSIGNABLE: Equal to SHARED - RESERVED. Exclusive CPUs are allocated
|
||||||
|
// from this pool.
|
||||||
|
//
|
||||||
|
// - EXCLUSIVE ALLOCATIONS: CPU sets assigned exclusively to one container.
|
||||||
|
// These are stored as explicit assignments in the state.
|
||||||
|
//
|
||||||
|
// When an exclusive allocation is made, the static policy also updates the
|
||||||
|
// default cpuset in the state abstraction. The CPU manager's periodic
|
||||||
|
// reconcile loop takes care of rewriting the cpuset in cgroupfs for any
|
||||||
|
// containers that may be running in the shared pool. For this reason,
|
||||||
|
// applications running within exclusively-allocated containers must tolerate
|
||||||
|
// potentially sharing their allocated CPUs for up to the CPU manager
|
||||||
|
// reconcile period.
|
||||||
|
type staticPolicy struct {
|
||||||
|
// cpu socket topology
|
||||||
|
topology *topology.CPUTopology
|
||||||
|
// set of CPUs that is not available for exclusive assignment
|
||||||
|
reserved cpuset.CPUSet
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure staticPolicy implements Policy interface
|
||||||
|
var _ Policy = &staticPolicy{}
|
||||||
|
|
||||||
|
// NewStaticPolicy returns a CPU manager policy that does not change CPU
|
||||||
|
// assignments for exclusively pinned guaranteed containers after the main
|
||||||
|
// container process starts.
|
||||||
|
func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int) Policy {
|
||||||
|
allCPUs := topology.CPUDetails.CPUs()
|
||||||
|
// takeByTopology allocates CPUs associated with low-numbered cores from
|
||||||
|
// allCPUs.
|
||||||
|
//
|
||||||
|
// For example: Given a system with 8 CPUs available and HT enabled,
|
||||||
|
// if numReservedCPUs=2, then reserved={0,4}
|
||||||
|
reserved, _ := takeByTopology(topology, allCPUs, numReservedCPUs)
|
||||||
|
|
||||||
|
if reserved.Size() != numReservedCPUs {
|
||||||
|
panic(fmt.Sprintf("[cpumanager] unable to reserve the required amount of CPUs (size of %s did not equal %d)", reserved, numReservedCPUs))
|
||||||
|
}
|
||||||
|
|
||||||
|
glog.Infof("[cpumanager] reserved %d CPUs (\"%s\") not available for exclusive assignment", reserved.Size(), reserved)
|
||||||
|
|
||||||
|
return &staticPolicy{
|
||||||
|
topology: topology,
|
||||||
|
reserved: reserved,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *staticPolicy) Name() string {
|
||||||
|
return string(PolicyStatic)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *staticPolicy) Start(s state.State) {
|
||||||
|
// Configure the shared pool to include all detected CPU IDs.
|
||||||
|
allCPUs := p.topology.CPUDetails.CPUs()
|
||||||
|
s.SetDefaultCPUSet(allCPUs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignableCPUs returns the set of unassigned CPUs minus the reserved set.
|
||||||
|
func (p *staticPolicy) assignableCPUs(s state.State) cpuset.CPUSet {
|
||||||
|
return s.GetDefaultCPUSet().Difference(p.reserved)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *staticPolicy) AddContainer(s state.State, pod *v1.Pod, container *v1.Container, containerID string) error {
|
||||||
|
glog.Infof("[cpumanager] static policy: AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID)
|
||||||
|
if numCPUs := guaranteedCPUs(pod, container); numCPUs != 0 {
|
||||||
|
// container belongs in an exclusively allocated pool
|
||||||
|
cpuset, err := p.allocateCPUs(s, numCPUs)
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("[cpumanager] unable to allocate %d CPUs (container id: %s, error: %v)", numCPUs, containerID, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.SetCPUSet(containerID, cpuset)
|
||||||
|
}
|
||||||
|
// container belongs in the shared pool (nothing to do; use default cpuset)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *staticPolicy) RemoveContainer(s state.State, containerID string) error {
|
||||||
|
glog.Infof("[cpumanager] static policy: RemoveContainer (container id: %s)", containerID)
|
||||||
|
if toRelease, ok := s.GetCPUSet(containerID); ok {
|
||||||
|
s.Delete(containerID)
|
||||||
|
// Mutate the shared pool, adding released cpus.
|
||||||
|
s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(toRelease))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int) (cpuset.CPUSet, error) {
|
||||||
|
glog.Infof("[cpumanager] allocateCpus: (numCPUs: %d)", numCPUs)
|
||||||
|
result, err := takeByTopology(p.topology, p.assignableCPUs(s), numCPUs)
|
||||||
|
if err != nil {
|
||||||
|
return cpuset.NewCPUSet(), err
|
||||||
|
}
|
||||||
|
// Remove allocated CPUs from the shared CPUSet.
|
||||||
|
s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result))
|
||||||
|
|
||||||
|
glog.Infof("[cpumanager] allocateCPUs: returning \"%v\"", result)
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func guaranteedCPUs(pod *v1.Pod, container *v1.Container) int {
|
||||||
|
if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
|
||||||
|
if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// Safe downcast to do for all systems with < 2.1 billion CPUs.
|
||||||
|
// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
|
||||||
|
// https://golang.org/ref/spec#Numeric_types
|
||||||
|
return int(cpuQuantity.Value())
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user