From d3d7a8f5d4b73f26f27cdf1302291fa36acf84ff Mon Sep 17 00:00:00 2001
From: Kevin Klues <kklues@nvidia.com>
Date: Wed, 14 Aug 2019 19:53:12 +0200
Subject: [PATCH] Generate TopologyHints from the devicemanager

---
 pkg/kubelet/cm/devicemanager/BUILD            |  1 +
 .../cm/devicemanager/topology_hints.go        | 98 ++++++++++++++++++-
 2 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/pkg/kubelet/cm/devicemanager/BUILD b/pkg/kubelet/cm/devicemanager/BUILD
index 3e948d61592..65878cde7c6 100644
--- a/pkg/kubelet/cm/devicemanager/BUILD
+++ b/pkg/kubelet/cm/devicemanager/BUILD
@@ -24,6 +24,7 @@ go_library(
         "//pkg/kubelet/cm/cpumanager/topology:go_default_library",
         "//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library",
         "//pkg/kubelet/cm/topologymanager:go_default_library",
+        "//pkg/kubelet/cm/topologymanager/socketmask:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/lifecycle:go_default_library",
diff --git a/pkg/kubelet/cm/devicemanager/topology_hints.go b/pkg/kubelet/cm/devicemanager/topology_hints.go
index aeed76a3e64..a7535eb01ea 100644
--- a/pkg/kubelet/cm/devicemanager/topology_hints.go
+++ b/pkg/kubelet/cm/devicemanager/topology_hints.go
@@ -18,7 +18,10 @@ package devicemanager
 
 import (
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
+	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/socketmask"
 )
 
 // GetTopologyHints implements the TopologyManager HintProvider Interface which
@@ -26,6 +29,99 @@ import (
 // container are created.
 func (m *ManagerImpl) GetTopologyHints(pod v1.Pod, container v1.Container) map[string][]topologymanager.TopologyHint {
 	deviceHints := make(map[string][]topologymanager.TopologyHint)
-	// TODO: Fill in deviceHints with proper TopologyHints
+
+	for resourceObj, requestedObj := range container.Resources.Limits {
+		resource := string(resourceObj)
+		requested := int(requestedObj.Value())
+
+		if m.isDevicePluginResource(resource) {
+			if aligned := m.deviceHasTopologyAlignment(resource); !aligned {
+				klog.Infof("[devicemanager] Resource '%v' does not have a topology preference", resource)
+				deviceHints[resource] = nil
+				continue
+			}
+
+			available := m.getAvailableDevices(resource)
+			if available.Len() < requested {
+				klog.Errorf("[devicemanager] Unable to generate topology hints: requested number of devices unavailable for '%s': requested: %d, available: %d", resource, requested, available.Len())
+				deviceHints[resource] = []topologymanager.TopologyHint{}
+				continue
+			}
+
+			deviceHints[resource] = m.generateDeviceTopologyHints(resource, available, requested)
+		}
+	}
+
 	return deviceHints
 }
+
+func (m *ManagerImpl) deviceHasTopologyAlignment(resource string) bool {
+	// If any device has Topology set, we assume they care about alignment.
+	for device := range m.allDevices[resource] {
+		if m.allDevices[resource][device].Topology != nil {
+			return true
+		}
+	}
+	return false
+}
+
+func (m *ManagerImpl) getAvailableDevices(resource string) sets.String {
+	// Gets Devices in use.
+	m.updateAllocatedDevices(m.activePods())
+	// Strip all devices in use from the list of healthy ones.
+	return m.healthyDevices[resource].Difference(m.allocatedDevices[resource])
+}
+
+func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.String, request int) []topologymanager.TopologyHint {
+	// Initialize minAffinity to a full affinity mask.
+	minAffinity, _ := socketmask.NewSocketMask(m.numaNodes...)
+
+	// Iterate through all combinations of NUMA Nodes and build hints from them.
+	hints := []topologymanager.TopologyHint{}
+	socketmask.IterateSocketMasks(m.numaNodes, func(mask socketmask.SocketMask) {
+		// Check to see if we have enough devices available on the current
+		// NUMA Node combination to satisfy the device request.
+		numMatching := 0
+		for d := range devices {
+			if m.allDevices[resource][d].Topology == nil {
+				continue
+			}
+			for _, node := range m.allDevices[resource][d].Topology.Nodes {
+				if mask.IsSet(int(node.ID)) {
+					numMatching++
+					break
+				}
+			}
+		}
+
+		// If we don't, then move onto the next combination.
+		if numMatching < request {
+			return
+		}
+
+		// Otherwise, create a new hint from the NUMA mask and add it to the
+		// list of hints.  We set all hint preferences to 'false' on the first
+		// pass through.
+		hints = append(hints, topologymanager.TopologyHint{
+			NUMANodeAffinity: mask,
+			Preferred:        false,
+		})
+
+		// Update minAffinity if relevant
+		if mask.IsNarrowerThan(minAffinity) {
+			minAffinity = mask
+		}
+	})
+
+	// Loop back through all hints and update the 'Preferred' field based on
+	// counting the number of bits sets in the affinity mask and comparing it
+	// to the minAffinity. Only those with an equal number of bits set will be
+	// considered preferred.
+	for i := range hints {
+		if hints[i].NUMANodeAffinity.Count() == minAffinity.Count() {
+			hints[i].Preferred = true
+		}
+	}
+
+	return hints
+}