Added Single NUMA Node Policy which ensure resource are

aligned on a single NUMA node

Co-authored-by: Kevin Klues <kklues@nvidia.com>

cmd/kubelet/app/options/options.go

@@ -518,7 +518,7 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig
 	fs.StringVar(&c.CPUManagerPolicy, "cpu-manager-policy", c.CPUManagerPolicy, "CPU Manager policy to use. Possible values: 'none', 'static'. Default: 'none'")
 	fs.DurationVar(&c.CPUManagerReconcilePeriod.Duration, "cpu-manager-reconcile-period", c.CPUManagerReconcilePeriod.Duration, "<Warning: Alpha feature> CPU Manager reconciliation period. Examples: '10s', or '1m'. If not supplied, defaults to `NodeStatusUpdateFrequency`")
 	fs.Var(cliflag.NewMapStringString(&c.QOSReserved), "qos-reserved", "<Warning: Alpha feature> A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe how pod resource requests are reserved at the QoS level. Currently only memory is supported. Requires the QOSReserved feature gate to be enabled.")
-	fs.StringVar(&c.TopologyManagerPolicy, "topology-manager-policy", c.TopologyManagerPolicy, "Topology Manager policy to use. Possible values: 'none', 'best-effort', 'restricted'.")
+	fs.StringVar(&c.TopologyManagerPolicy, "topology-manager-policy", c.TopologyManagerPolicy, "Topology Manager policy to use. Possible values: 'none', 'best-effort', 'restricted', 'single-numa-node'.")
 	fs.DurationVar(&c.RuntimeRequestTimeout.Duration, "runtime-request-timeout", c.RuntimeRequestTimeout.Duration, "Timeout of all runtime requests except long running request - pull, logs, exec and attach. When timeout exceeded, kubelet will cancel the request, throw out an error and retry later.")
 	fs.StringVar(&c.HairpinMode, "hairpin-mode", c.HairpinMode, "How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are \"promiscuous-bridge\", \"hairpin-veth\" and \"none\".")
 	fs.Int32Var(&c.MaxPods, "max-pods", c.MaxPods, "Number of Pods that can run on this Kubelet.")

pkg/kubelet/apis/config/types.go

@@ -55,7 +55,7 @@ const (
 	// watches to observe changes to objects that are in its interest.
 	WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch"
 	// RestrictedTopologyManagerPolicy is a mode in which kubelet only allows
-	// pods with a single NUMA alignment of CPU and device resources.
+	// pods with optimal NUMA node alignment for requested resources
 	RestrictedTopologyManagerPolicy = "restricted"
 	// BestEffortTopologyManagerPolicy is a mode in which kubelet will favour
 	// pods with NUMA alignment of CPU and device resources.
@@ -63,6 +63,9 @@ const (
 	// NoneTopologyManager Policy is a mode in which kubelet has no knowledge
 	// of NUMA alignment of a pod's CPU and device resources.
 	NoneTopologyManagerPolicy = "none"
+	// SingleNumaNodeTopologyManager Policy iis a mode in which kubelet only allows
+	// pods with a single NUMA alignment of CPU and device resources.
+	SingleNumaNodeTopologyManager = "single-numa-node"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/kubelet/cm/topologymanager/BUILD

@@ -8,6 +8,7 @@ go_library(
         "policy_best_effort.go",
         "policy_none.go",
         "policy_restricted.go",
+        "policy_single_numa_node.go",
         "topology_manager.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager",
@@ -45,6 +46,7 @@ go_test(
         "policy_best_effort_test.go",
         "policy_none_test.go",
         "policy_restricted_test.go",
+        "policy_single_numa_node_test.go",
         "topology_manager_test.go",
     ],
     embed = [":go_default_library"],

pkg/kubelet/cm/topologymanager/policy_none_test.go

@@ -41,7 +41,7 @@ func TestName(t *testing.T) {
 func TestPolicyNoneCanAdmitPodResult(t *testing.T) {
 	tcases := []struct {
 		name     string
-		hint TopologyHint
+		hint     TopologyHint
 		expected bool
 	}{
 		{

pkg/kubelet/cm/topologymanager/policy_single_numa_node.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package topologymanager
+
+import (
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+)
+
+type singleNumaNodePolicy struct{}
+
+var _ Policy = &singleNumaNodePolicy{}
+
+// PolicySingleNumaNode policy name.
+const PolicySingleNumaNode string = "single-numa-node"
+
+// NewSingleNumaNodePolicy returns single-numa-node policy.
+func NewSingleNumaNodePolicy() Policy {
+	return &singleNumaNodePolicy{}
+}
+
+func (p *singleNumaNodePolicy) Name() string {
+	return PolicySingleNumaNode
+}
+
+func (p *singleNumaNodePolicy) CanAdmitPodResult(hint *TopologyHint) lifecycle.PodAdmitResult {
+	if !hint.Preferred || hint.NUMANodeAffinity.Count() > 1 {
+		return lifecycle.PodAdmitResult{
+			Admit:   false,
+			Reason:  "Topology Affinity Error",
+			Message: "Resources cannot be allocated with Topology Locality",
+		}
+	}
+	return lifecycle.PodAdmitResult{
+		Admit: true,
+	}
+}

pkg/kubelet/cm/topologymanager/policy_single_numa_node_test.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package topologymanager
+
+import (
+	"testing"
+)
+
+func TestPolicySingleNumaNodeCanAdmitPodResult(t *testing.T) {
+	tcases := []struct {
+		name     string
+		hint     TopologyHint
+		expected bool
+	}{
+		{
+			name:     "Preferred is set to false in topology hints",
+			hint:     TopologyHint{nil, false},
+			expected: false,
+		},
+		{
+			name:     "NUMANodeAffinity has multiple NUMA Nodes masked in topology hints",
+			hint:     TopologyHint{NewTestSocketMask(0, 1), true},
+			expected: false,
+		},
+		{
+			name:     "NUMANodeAffinity has one NUMA Node masked in topology hints",
+			hint:     TopologyHint{NewTestSocketMask(0), true},
+			expected: true,
+		},
+	}
+
+	for _, tc := range tcases {
+		policy := NewSingleNumaNodePolicy()
+		result := policy.CanAdmitPodResult(&tc.hint)
+
+		if result.Admit != tc.expected {
+			t.Errorf("Expected Admit field in result to be %t, got %t", tc.expected, result.Admit)
+		}
+
+		if tc.expected == false {
+			if len(result.Reason) == 0 {
+				t.Errorf("Expected Reason field to be not empty")
+			}
+			if len(result.Message) == 0 {
+				t.Errorf("Expected Message field to be not empty")
+			}
+		}
+	}
+}

pkg/kubelet/cm/topologymanager/topology_manager.go

@@ -103,6 +103,9 @@ func NewManager(numaNodeInfo cputopology.NUMANodeInfo, topologyPolicyName string
 	case PolicyRestricted:
 		policy = NewRestrictedPolicy()
 
+	case PolicySingleNumaNode:
+		policy = NewSingleNumaNodePolicy()
+
 	default:
 		return nil, fmt.Errorf("unknown policy: \"%s\"", topologyPolicyName)
 	}

staging/src/k8s.io/kubelet/config/v1beta1/types.go

@@ -55,7 +55,7 @@ const (
 	// watches to observe changes to objects that are in its interest.
 	WatchChangeDetectionStrategy ResourceChangeDetectionStrategy = "Watch"
 	// RestrictedTopologyManagerPolicy is a mode in which kubelet only allows
-	// pods with a single NUMA alignment of CPU and device resources.
+	// pods with optimal NUMA node alignment for requested resources
 	RestrictedTopologyManagerPolicy = "restricted"
 	// BestEffortTopologyManagerPolicy is a mode in which kubelet will favour
 	// pods with NUMA alignment of CPU and device resources.
@@ -63,6 +63,9 @@ const (
 	// NoneTopologyManager Policy is a mode in which kubelet has no knowledge
 	// of NUMA alignment of a pod's CPU and device resources.
 	NoneTopologyManagerPolicy = "none"
+	// SingleNumaNodeTopologyManager Policy iis a mode in which kubelet only allows
+	// pods with a single NUMA alignment of CPU and device resources.
+	SingleNumaNodeTopologyManager = "single-numa-node"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object