mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-30 23:15:14 +00:00
Merge pull request #111278 from arpitsardhana/master
KEP-3327: Add CPUManager policy option to align CPUs by Socket instead of by NUMA node
This commit is contained in:
commit
9fb1f67af7
@ -23,16 +23,20 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
kubefeatures "k8s.io/kubernetes/pkg/features"
|
kubefeatures "k8s.io/kubernetes/pkg/features"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
FullPCPUsOnlyOption string = "full-pcpus-only"
|
FullPCPUsOnlyOption string = "full-pcpus-only"
|
||||||
DistributeCPUsAcrossNUMAOption string = "distribute-cpus-across-numa"
|
DistributeCPUsAcrossNUMAOption string = "distribute-cpus-across-numa"
|
||||||
|
AlignBySocketOption string = "align-by-socket"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
alphaOptions = sets.NewString(
|
alphaOptions = sets.NewString(
|
||||||
DistributeCPUsAcrossNUMAOption,
|
DistributeCPUsAcrossNUMAOption,
|
||||||
|
AlignBySocketOption,
|
||||||
)
|
)
|
||||||
betaOptions = sets.NewString(
|
betaOptions = sets.NewString(
|
||||||
FullPCPUsOnlyOption,
|
FullPCPUsOnlyOption,
|
||||||
@ -69,6 +73,9 @@ type StaticPolicyOptions struct {
|
|||||||
// Flag to evenly distribute CPUs across NUMA nodes in cases where more
|
// Flag to evenly distribute CPUs across NUMA nodes in cases where more
|
||||||
// than one NUMA node is required to satisfy the allocation.
|
// than one NUMA node is required to satisfy the allocation.
|
||||||
DistributeCPUsAcrossNUMA bool
|
DistributeCPUsAcrossNUMA bool
|
||||||
|
// Flag to ensure CPUs are considered aligned at socket boundary rather than
|
||||||
|
// NUMA boundary
|
||||||
|
AlignBySocket bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStaticPolicyOptions(policyOptions map[string]string) (StaticPolicyOptions, error) {
|
func NewStaticPolicyOptions(policyOptions map[string]string) (StaticPolicyOptions, error) {
|
||||||
@ -91,6 +98,12 @@ func NewStaticPolicyOptions(policyOptions map[string]string) (StaticPolicyOption
|
|||||||
return opts, fmt.Errorf("bad value for option %q: %w", name, err)
|
return opts, fmt.Errorf("bad value for option %q: %w", name, err)
|
||||||
}
|
}
|
||||||
opts.DistributeCPUsAcrossNUMA = optValue
|
opts.DistributeCPUsAcrossNUMA = optValue
|
||||||
|
case AlignBySocketOption:
|
||||||
|
optValue, err := strconv.ParseBool(value)
|
||||||
|
if err != nil {
|
||||||
|
return opts, fmt.Errorf("bad value for option %q: %w", name, err)
|
||||||
|
}
|
||||||
|
opts.AlignBySocket = optValue
|
||||||
default:
|
default:
|
||||||
// this should never be reached, we already detect unknown options,
|
// this should never be reached, we already detect unknown options,
|
||||||
// but we keep it as further safety.
|
// but we keep it as further safety.
|
||||||
@ -99,3 +112,17 @@ func NewStaticPolicyOptions(policyOptions map[string]string) (StaticPolicyOption
|
|||||||
}
|
}
|
||||||
return opts, nil
|
return opts, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ValidateStaticPolicyOptions(opts StaticPolicyOptions, topology *topology.CPUTopology, topologyManager topologymanager.Store) error {
|
||||||
|
if opts.AlignBySocket {
|
||||||
|
// Not compatible with topology manager single-numa-node policy option.
|
||||||
|
if topologyManager.GetPolicy().Name() == topologymanager.PolicySingleNumaNode {
|
||||||
|
return fmt.Errorf("Topolgy manager %s policy is incompatible with CPUManager %s policy option", topologymanager.PolicySingleNumaNode, AlignBySocketOption)
|
||||||
|
}
|
||||||
|
// Not compatible with topology when number of sockets are more than number of NUMA nodes.
|
||||||
|
if topology.NumSockets > topology.NumNUMANodes {
|
||||||
|
return fmt.Errorf("Align by socket is not compatible with hardware where number of sockets are more than number of NUMA")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
@ -23,6 +23,8 @@ import (
|
|||||||
"k8s.io/component-base/featuregate"
|
"k8s.io/component-base/featuregate"
|
||||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
pkgfeatures "k8s.io/kubernetes/pkg/features"
|
pkgfeatures "k8s.io/kubernetes/pkg/features"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
||||||
)
|
)
|
||||||
|
|
||||||
type optionAvailTest struct {
|
type optionAvailTest struct {
|
||||||
@ -54,7 +56,7 @@ func TestPolicyDefaultsAvailable(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPolicyBetaOptionsAvailable(t *testing.T) {
|
func TestPolicyOptionsAvailable(t *testing.T) {
|
||||||
testCases := []optionAvailTest{
|
testCases := []optionAvailTest{
|
||||||
{
|
{
|
||||||
option: "this-option-does-not-exist",
|
option: "this-option-does-not-exist",
|
||||||
@ -80,6 +82,18 @@ func TestPolicyBetaOptionsAvailable(t *testing.T) {
|
|||||||
featureGateEnable: false,
|
featureGateEnable: false,
|
||||||
expectedAvailable: false,
|
expectedAvailable: false,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
option: AlignBySocketOption,
|
||||||
|
featureGate: pkgfeatures.CPUManagerPolicyAlphaOptions,
|
||||||
|
featureGateEnable: true,
|
||||||
|
expectedAvailable: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
option: AlignBySocketOption,
|
||||||
|
featureGate: pkgfeatures.CPUManagerPolicyBetaOptions,
|
||||||
|
featureGateEnable: true,
|
||||||
|
expectedAvailable: false,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, testCase := range testCases {
|
for _, testCase := range testCases {
|
||||||
t.Run(testCase.option, func(t *testing.T) {
|
t.Run(testCase.option, func(t *testing.T) {
|
||||||
@ -92,3 +106,74 @@ func TestPolicyBetaOptionsAvailable(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateStaticPolicyOptions(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
description string
|
||||||
|
policyOption map[string]string
|
||||||
|
topology *topology.CPUTopology
|
||||||
|
topoMgrPolicy string
|
||||||
|
expectedErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
description: "Align by socket not enabled",
|
||||||
|
policyOption: map[string]string{FullPCPUsOnlyOption: "true"},
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
topoMgrPolicy: topologymanager.PolicySingleNumaNode,
|
||||||
|
expectedErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Align by socket enabled with topology manager single numa node",
|
||||||
|
policyOption: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
topoMgrPolicy: topologymanager.PolicySingleNumaNode,
|
||||||
|
expectedErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Align by socket enabled with num_sockets > num_numa",
|
||||||
|
policyOption: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
topology: fakeTopoMultiSocketDualSocketPerNumaHT,
|
||||||
|
topoMgrPolicy: topologymanager.PolicyNone,
|
||||||
|
expectedErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Align by socket enabled: with topology manager None policy",
|
||||||
|
policyOption: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
topoMgrPolicy: topologymanager.PolicyNone,
|
||||||
|
expectedErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Align by socket enabled: with topology manager best-effort policy",
|
||||||
|
policyOption: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
topoMgrPolicy: topologymanager.PolicyBestEffort,
|
||||||
|
expectedErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Align by socket enabled: with topology manager restricted policy",
|
||||||
|
policyOption: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
topoMgrPolicy: topologymanager.PolicyRestricted,
|
||||||
|
expectedErr: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
t.Run(testCase.description, func(t *testing.T) {
|
||||||
|
topoMgrPolicy := topologymanager.NewNonePolicy()
|
||||||
|
if testCase.topoMgrPolicy == topologymanager.PolicySingleNumaNode {
|
||||||
|
topoMgrPolicy = topologymanager.NewSingleNumaNodePolicy(nil)
|
||||||
|
|
||||||
|
}
|
||||||
|
topoMgrStore := topologymanager.NewFakeManagerWithPolicy(topoMgrPolicy)
|
||||||
|
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true)()
|
||||||
|
policyOpt, _ := NewStaticPolicyOptions(testCase.policyOption)
|
||||||
|
err := ValidateStaticPolicyOptions(policyOpt, testCase.topology, topoMgrStore)
|
||||||
|
gotError := (err != nil)
|
||||||
|
if gotError != testCase.expectedErr {
|
||||||
|
t.Errorf("testCase %q failed, got %v expected %v", testCase.description, gotError, testCase.expectedErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -115,6 +115,10 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
err = ValidateStaticPolicyOptions(opts, topology, affinity)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
klog.InfoS("Static policy created with configuration", "options", opts)
|
klog.InfoS("Static policy created with configuration", "options", opts)
|
||||||
|
|
||||||
@ -325,10 +329,7 @@ func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bit
|
|||||||
// If there are aligned CPUs in numaAffinity, attempt to take those first.
|
// If there are aligned CPUs in numaAffinity, attempt to take those first.
|
||||||
result := cpuset.NewCPUSet()
|
result := cpuset.NewCPUSet()
|
||||||
if numaAffinity != nil {
|
if numaAffinity != nil {
|
||||||
alignedCPUs := cpuset.NewCPUSet()
|
alignedCPUs := p.getAlignedCPUs(numaAffinity, allocatableCPUs)
|
||||||
for _, numaNodeID := range numaAffinity.GetBits() {
|
|
||||||
alignedCPUs = alignedCPUs.Union(allocatableCPUs.Intersection(p.topology.CPUDetails.CPUsInNUMANodes(numaNodeID)))
|
|
||||||
}
|
|
||||||
|
|
||||||
numAlignedToAlloc := alignedCPUs.Size()
|
numAlignedToAlloc := alignedCPUs.Size()
|
||||||
if numCPUs < numAlignedToAlloc {
|
if numCPUs < numAlignedToAlloc {
|
||||||
@ -571,6 +572,10 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
|
|||||||
// to the minAffinitySize. Only those with an equal number of bits set (and
|
// to the minAffinitySize. Only those with an equal number of bits set (and
|
||||||
// with a minimal set of numa nodes) will be considered preferred.
|
// with a minimal set of numa nodes) will be considered preferred.
|
||||||
for i := range hints {
|
for i := range hints {
|
||||||
|
if p.options.AlignBySocket && p.isHintSocketAligned(hints[i], minAffinitySize) {
|
||||||
|
hints[i].Preferred = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
|
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
|
||||||
hints[i].Preferred = true
|
hints[i].Preferred = true
|
||||||
}
|
}
|
||||||
@ -578,3 +583,39 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
|
|||||||
|
|
||||||
return hints
|
return hints
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isHintSocketAligned function return true if numa nodes in hint are socket aligned.
|
||||||
|
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
|
||||||
|
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
|
||||||
|
numaNodesPerSocket := p.topology.NumNUMANodes / p.topology.NumSockets
|
||||||
|
if numaNodesPerSocket == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// minSockets refers to minimum number of socket required to satify allocation.
|
||||||
|
// A hint is considered socket aligned if sockets across which numa nodes span is equal to minSockets
|
||||||
|
minSockets := (minAffinitySize + numaNodesPerSocket - 1) / numaNodesPerSocket
|
||||||
|
return p.topology.CPUDetails.SocketsInNUMANodes(numaNodesBitMask...).Size() == minSockets
|
||||||
|
}
|
||||||
|
|
||||||
|
// getAlignedCPUs return set of aligned CPUs based on numa affinity mask and configured policy options.
|
||||||
|
func (p *staticPolicy) getAlignedCPUs(numaAffinity bitmask.BitMask, allocatableCPUs cpuset.CPUSet) cpuset.CPUSet {
|
||||||
|
alignedCPUs := cpuset.NewCPUSet()
|
||||||
|
numaBits := numaAffinity.GetBits()
|
||||||
|
|
||||||
|
// If align-by-socket policy option is enabled, NUMA based hint is expanded to
|
||||||
|
// socket aligned hint. It will ensure that first socket aligned available CPUs are
|
||||||
|
// allocated before we try to find CPUs across socket to satisfy allocation request.
|
||||||
|
if p.options.AlignBySocket {
|
||||||
|
socketBits := p.topology.CPUDetails.SocketsInNUMANodes(numaBits...).ToSliceNoSort()
|
||||||
|
for _, socketID := range socketBits {
|
||||||
|
alignedCPUs = alignedCPUs.Union(allocatableCPUs.Intersection(p.topology.CPUDetails.CPUsInSockets(socketID)))
|
||||||
|
}
|
||||||
|
return alignedCPUs
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, numaNodeID := range numaBits {
|
||||||
|
alignedCPUs = alignedCPUs.Union(allocatableCPUs.Intersection(p.topology.CPUDetails.CPUsInNUMANodes(numaNodeID)))
|
||||||
|
}
|
||||||
|
|
||||||
|
return alignedCPUs
|
||||||
|
}
|
||||||
|
@ -22,6 +22,9 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
|
pkgfeatures "k8s.io/kubernetes/pkg/features"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
||||||
@ -39,6 +42,7 @@ type staticPolicyTest struct {
|
|||||||
stAssignments state.ContainerCPUAssignments
|
stAssignments state.ContainerCPUAssignments
|
||||||
stDefaultCPUSet cpuset.CPUSet
|
stDefaultCPUSet cpuset.CPUSet
|
||||||
pod *v1.Pod
|
pod *v1.Pod
|
||||||
|
topologyHint *topologymanager.TopologyHint
|
||||||
expErr error
|
expErr error
|
||||||
expCPUAlloc bool
|
expCPUAlloc bool
|
||||||
expCSet cpuset.CPUSet
|
expCSet cpuset.CPUSet
|
||||||
@ -190,6 +194,7 @@ func TestStaticPolicyAdd(t *testing.T) {
|
|||||||
|
|
||||||
// these are the cases which must behave the same regardless the policy options.
|
// these are the cases which must behave the same regardless the policy options.
|
||||||
// So we will permutate the options to ensure this holds true.
|
// So we will permutate the options to ensure this holds true.
|
||||||
|
|
||||||
optionsInsensitiveTestCases := []staticPolicyTest{
|
optionsInsensitiveTestCases := []staticPolicyTest{
|
||||||
{
|
{
|
||||||
description: "GuPodSingleCore, SingleSocketHT, ExpectError",
|
description: "GuPodSingleCore, SingleSocketHT, ExpectError",
|
||||||
@ -493,6 +498,42 @@ func TestStaticPolicyAdd(t *testing.T) {
|
|||||||
expCSet: cpuset.NewCPUSet(),
|
expCSet: cpuset.NewCPUSet(),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
newNUMAAffinity := func(bits ...int) bitmask.BitMask {
|
||||||
|
affinity, _ := bitmask.NewBitMask(bits...)
|
||||||
|
return affinity
|
||||||
|
}
|
||||||
|
alignBySocketOptionTestCases := []staticPolicyTest{
|
||||||
|
{
|
||||||
|
description: "Align by socket: true, cpu's within same socket of numa in hint are part of allocation",
|
||||||
|
topo: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
options: map[string]string{
|
||||||
|
AlignBySocketOption: "true",
|
||||||
|
},
|
||||||
|
numReservedCPUs: 1,
|
||||||
|
stAssignments: state.ContainerCPUAssignments{},
|
||||||
|
stDefaultCPUSet: cpuset.NewCPUSet(2, 11, 21, 22),
|
||||||
|
pod: makePod("fakePod", "fakeContainer2", "2000m", "2000m"),
|
||||||
|
topologyHint: &topologymanager.TopologyHint{NUMANodeAffinity: newNUMAAffinity(0, 2), Preferred: true},
|
||||||
|
expErr: nil,
|
||||||
|
expCPUAlloc: true,
|
||||||
|
expCSet: cpuset.NewCPUSet(2, 11),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "Align by socket: false, cpu's are taken strictly from NUMA nodes in hint",
|
||||||
|
topo: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
options: map[string]string{
|
||||||
|
AlignBySocketOption: "false",
|
||||||
|
},
|
||||||
|
numReservedCPUs: 1,
|
||||||
|
stAssignments: state.ContainerCPUAssignments{},
|
||||||
|
stDefaultCPUSet: cpuset.NewCPUSet(2, 11, 21, 22),
|
||||||
|
pod: makePod("fakePod", "fakeContainer2", "2000m", "2000m"),
|
||||||
|
topologyHint: &topologymanager.TopologyHint{NUMANodeAffinity: newNUMAAffinity(0, 2), Preferred: true},
|
||||||
|
expErr: nil,
|
||||||
|
expCPUAlloc: true,
|
||||||
|
expCSet: cpuset.NewCPUSet(2, 21),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
for _, testCase := range optionsInsensitiveTestCases {
|
for _, testCase := range optionsInsensitiveTestCases {
|
||||||
for _, options := range []map[string]string{
|
for _, options := range []map[string]string{
|
||||||
@ -514,10 +555,17 @@ func TestStaticPolicyAdd(t *testing.T) {
|
|||||||
for _, testCase := range smtalignOptionTestCases {
|
for _, testCase := range smtalignOptionTestCases {
|
||||||
runStaticPolicyTestCase(t, testCase)
|
runStaticPolicyTestCase(t, testCase)
|
||||||
}
|
}
|
||||||
|
for _, testCase := range alignBySocketOptionTestCases {
|
||||||
|
runStaticPolicyTestCaseWithFeatureGate(t, testCase)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
|
func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
|
||||||
policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), topologymanager.NewFakeManager(), testCase.options)
|
tm := topologymanager.NewFakeManager()
|
||||||
|
if testCase.topologyHint != nil {
|
||||||
|
tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint)
|
||||||
|
}
|
||||||
|
policy, _ := NewStaticPolicy(testCase.topo, testCase.numReservedCPUs, cpuset.NewCPUSet(), tm, testCase.options)
|
||||||
|
|
||||||
st := &mockState{
|
st := &mockState{
|
||||||
assignments: testCase.stAssignments,
|
assignments: testCase.stAssignments,
|
||||||
@ -558,6 +606,11 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyTest) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true)()
|
||||||
|
runStaticPolicyTestCase(t, testCase)
|
||||||
|
}
|
||||||
|
|
||||||
func TestStaticPolicyReuseCPUs(t *testing.T) {
|
func TestStaticPolicyReuseCPUs(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
staticPolicyTest
|
staticPolicyTest
|
||||||
|
@ -24,6 +24,9 @@ import (
|
|||||||
cadvisorapi "github.com/google/cadvisor/info/v1"
|
cadvisorapi "github.com/google/cadvisor/info/v1"
|
||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/types"
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
|
pkgfeatures "k8s.io/kubernetes/pkg/features"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
||||||
@ -254,6 +257,186 @@ func TestGetPodTopologyHints(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetPodTopologyHintsWithPolicyOptions(t *testing.T) {
|
||||||
|
testPod1 := makePod("fakePod", "fakeContainer", "2", "2")
|
||||||
|
testContainer1 := &testPod1.Spec.Containers[0]
|
||||||
|
|
||||||
|
testPod2 := makePod("fakePod", "fakeContainer", "41", "41")
|
||||||
|
testContainer2 := &testPod1.Spec.Containers[0]
|
||||||
|
|
||||||
|
cpu_set_across_socket, _ := cpuset.Parse("0-28,40-57")
|
||||||
|
|
||||||
|
m0001, _ := bitmask.NewBitMask(0)
|
||||||
|
m0011, _ := bitmask.NewBitMask(0, 1)
|
||||||
|
m0101, _ := bitmask.NewBitMask(0, 2)
|
||||||
|
m1001, _ := bitmask.NewBitMask(0, 3)
|
||||||
|
m0111, _ := bitmask.NewBitMask(0, 1, 2)
|
||||||
|
m1011, _ := bitmask.NewBitMask(0, 1, 3)
|
||||||
|
m1101, _ := bitmask.NewBitMask(0, 2, 3)
|
||||||
|
m1111, _ := bitmask.NewBitMask(0, 1, 2, 3)
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
description string
|
||||||
|
pod v1.Pod
|
||||||
|
container v1.Container
|
||||||
|
assignments state.ContainerCPUAssignments
|
||||||
|
defaultCPUSet cpuset.CPUSet
|
||||||
|
policyOptions map[string]string
|
||||||
|
topology *topology.CPUTopology
|
||||||
|
expectedHints []topologymanager.TopologyHint
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// CPU available on numa node[0 ,1]. CPU on numa node 0 can satisfy request of 2 CPU's
|
||||||
|
description: "AlignBySocket:false, Preferred hints does not contains socket aligned hints",
|
||||||
|
pod: *testPod1,
|
||||||
|
container: *testContainer1,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(2, 3, 11),
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
policyOptions: map[string]string{AlignBySocketOption: "false"},
|
||||||
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0001,
|
||||||
|
Preferred: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0011,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0101,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1001,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0111,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1011,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1101,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1111,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// CPU available on numa node[0 ,1]. CPU on numa node 0 can satisfy request of 2 CPU's
|
||||||
|
description: "AlignBySocket:true Preferred hints contains socket aligned hints",
|
||||||
|
pod: *testPod1,
|
||||||
|
container: *testContainer1,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(2, 3, 11),
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
policyOptions: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0001,
|
||||||
|
Preferred: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0011,
|
||||||
|
Preferred: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0101,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1001,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0111,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1011,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1101,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1111,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// CPU available on numa node[0 ,1]. CPU on numa nodes across sockets can satisfy request of 2 CPU's
|
||||||
|
description: "AlignBySocket:true Preferred hints are spread across socket since 2 sockets are required",
|
||||||
|
pod: *testPod2,
|
||||||
|
container: *testContainer2,
|
||||||
|
defaultCPUSet: cpu_set_across_socket,
|
||||||
|
topology: topoDualSocketMultiNumaPerSocketHT,
|
||||||
|
policyOptions: map[string]string{AlignBySocketOption: "true"},
|
||||||
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m0111,
|
||||||
|
Preferred: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: m1111,
|
||||||
|
Preferred: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
t.Run(testCase.description, func(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true)()
|
||||||
|
|
||||||
|
var activePods []*v1.Pod
|
||||||
|
for p := range testCase.assignments {
|
||||||
|
pod := v1.Pod{}
|
||||||
|
pod.UID = types.UID(p)
|
||||||
|
for c := range testCase.assignments[p] {
|
||||||
|
container := v1.Container{}
|
||||||
|
container.Name = c
|
||||||
|
pod.Spec.Containers = append(pod.Spec.Containers, container)
|
||||||
|
}
|
||||||
|
activePods = append(activePods, &pod)
|
||||||
|
}
|
||||||
|
policyOpt, _ := NewStaticPolicyOptions(testCase.policyOptions)
|
||||||
|
m := manager{
|
||||||
|
policy: &staticPolicy{
|
||||||
|
topology: testCase.topology,
|
||||||
|
options: policyOpt,
|
||||||
|
},
|
||||||
|
state: &mockState{
|
||||||
|
assignments: testCase.assignments,
|
||||||
|
defaultCPUSet: testCase.defaultCPUSet,
|
||||||
|
},
|
||||||
|
topology: testCase.topology,
|
||||||
|
activePods: func() []*v1.Pod { return activePods },
|
||||||
|
podStatusProvider: mockPodStatusProvider{},
|
||||||
|
sourcesReady: &sourcesReadyStub{},
|
||||||
|
}
|
||||||
|
|
||||||
|
podHints := m.GetPodTopologyHints(&testCase.pod)[string(v1.ResourceCPU)]
|
||||||
|
sort.SliceStable(podHints, func(i, j int) bool {
|
||||||
|
return podHints[i].LessThan(podHints[j])
|
||||||
|
})
|
||||||
|
sort.SliceStable(testCase.expectedHints, func(i, j int) bool {
|
||||||
|
return testCase.expectedHints[i].LessThan(testCase.expectedHints[j])
|
||||||
|
})
|
||||||
|
if !reflect.DeepEqual(testCase.expectedHints, podHints) {
|
||||||
|
t.Errorf("Expected in result to be %v , got %v", testCase.expectedHints, podHints)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func returnTestCases() []testCase {
|
func returnTestCases() []testCase {
|
||||||
testPod1 := makePod("fakePod", "fakeContainer", "2", "2")
|
testPod1 := makePod("fakePod", "fakeContainer", "2", "2")
|
||||||
testContainer1 := &testPod1.Spec.Containers[0]
|
testContainer1 := &testPod1.Spec.Containers[0]
|
||||||
|
@ -39,6 +39,10 @@ func (m *mockAffinityStore) GetAffinity(podUID string, containerName string) top
|
|||||||
return m.hint
|
return m.hint
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *mockAffinityStore) GetPolicy() topologymanager.Policy {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func makeNUMADevice(id string, numa int) pluginapi.Device {
|
func makeNUMADevice(id string, numa int) pluginapi.Device {
|
||||||
return pluginapi.Device{
|
return pluginapi.Device{
|
||||||
ID: id,
|
ID: id,
|
||||||
|
@ -25,6 +25,7 @@ import (
|
|||||||
|
|
||||||
type fakeManager struct {
|
type fakeManager struct {
|
||||||
hint *TopologyHint
|
hint *TopologyHint
|
||||||
|
policy Policy
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFakeManager returns an instance of FakeManager
|
// NewFakeManager returns an instance of FakeManager
|
||||||
@ -38,6 +39,15 @@ func NewFakeManagerWithHint(hint *TopologyHint) Manager {
|
|||||||
klog.InfoS("NewFakeManagerWithHint")
|
klog.InfoS("NewFakeManagerWithHint")
|
||||||
return &fakeManager{
|
return &fakeManager{
|
||||||
hint: hint,
|
hint: hint,
|
||||||
|
policy: NewNonePolicy(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFakeManagerWithPolicy returns an instance of fake topology manager with specified policy
|
||||||
|
func NewFakeManagerWithPolicy(policy Policy) Manager {
|
||||||
|
klog.InfoS("NewFakeManagerWithPolicy")
|
||||||
|
return &fakeManager{
|
||||||
|
policy: policy,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -50,6 +60,10 @@ func (m *fakeManager) GetAffinity(podUID string, containerName string) TopologyH
|
|||||||
return *m.hint
|
return *m.hint
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *fakeManager) GetPolicy() Policy {
|
||||||
|
return m.policy
|
||||||
|
}
|
||||||
|
|
||||||
func (m *fakeManager) AddHintProvider(h HintProvider) {
|
func (m *fakeManager) AddHintProvider(h HintProvider) {
|
||||||
klog.InfoS("AddHintProvider", "hintProvider", h)
|
klog.InfoS("AddHintProvider", "hintProvider", h)
|
||||||
}
|
}
|
||||||
|
@ -38,6 +38,7 @@ type podTopologyHints map[string]map[string]TopologyHint
|
|||||||
// Scope interface for Topology Manager
|
// Scope interface for Topology Manager
|
||||||
type Scope interface {
|
type Scope interface {
|
||||||
Name() string
|
Name() string
|
||||||
|
GetPolicy() Policy
|
||||||
Admit(pod *v1.Pod) lifecycle.PodAdmitResult
|
Admit(pod *v1.Pod) lifecycle.PodAdmitResult
|
||||||
// AddHintProvider adds a hint provider to manager to indicate the hint provider
|
// AddHintProvider adds a hint provider to manager to indicate the hint provider
|
||||||
// wants to be consoluted with when making topology hints
|
// wants to be consoluted with when making topology hints
|
||||||
@ -88,6 +89,10 @@ func (s *scope) GetAffinity(podUID string, containerName string) TopologyHint {
|
|||||||
return s.getTopologyHints(podUID, containerName)
|
return s.getTopologyHints(podUID, containerName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *scope) GetPolicy() Policy {
|
||||||
|
return s.policy
|
||||||
|
}
|
||||||
|
|
||||||
func (s *scope) AddHintProvider(h HintProvider) {
|
func (s *scope) AddHintProvider(h HintProvider) {
|
||||||
s.hintProviders = append(s.hintProviders, h)
|
s.hintProviders = append(s.hintProviders, h)
|
||||||
}
|
}
|
||||||
|
@ -95,6 +95,7 @@ type HintProvider interface {
|
|||||||
// Store interface is to allow Hint Providers to retrieve pod affinity
|
// Store interface is to allow Hint Providers to retrieve pod affinity
|
||||||
type Store interface {
|
type Store interface {
|
||||||
GetAffinity(podUID string, containerName string) TopologyHint
|
GetAffinity(podUID string, containerName string) TopologyHint
|
||||||
|
GetPolicy() Policy
|
||||||
}
|
}
|
||||||
|
|
||||||
// TopologyHint is a struct containing the NUMANodeAffinity for a Container
|
// TopologyHint is a struct containing the NUMANodeAffinity for a Container
|
||||||
@ -184,6 +185,10 @@ func (m *manager) GetAffinity(podUID string, containerName string) TopologyHint
|
|||||||
return m.scope.GetAffinity(podUID, containerName)
|
return m.scope.GetAffinity(podUID, containerName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *manager) GetPolicy() Policy {
|
||||||
|
return m.scope.GetPolicy()
|
||||||
|
}
|
||||||
|
|
||||||
func (m *manager) AddHintProvider(h HintProvider) {
|
func (m *manager) AddHintProvider(h HintProvider) {
|
||||||
m.scope.AddHintProvider(h)
|
m.scope.AddHintProvider(h)
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user