Kubelet side extension to support device allocation

pkg/kubelet/cm/container_manager_linux.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	utilfile "k8s.io/kubernetes/pkg/util/file"
 	"k8s.io/kubernetes/pkg/util/mount"
@@ -117,8 +118,8 @@ type containerManagerImpl struct {
 	recorder record.EventRecorder
 	// Interface for QoS cgroup management
 	qosContainerManager QOSContainerManager
-	// Interface for device plugin management.
-	devicePluginHdler DevicePluginHandler
+	// Interface for exporting and allocating devices reported by device plugins.
+	devicePluginHandler DevicePluginHandler
 }
 
 type features struct {
@@ -181,7 +182,7 @@ func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
 // TODO(vmarmol): Add limits to the system containers.
 // Takes the absolute name of the specified containers.
 // Empty container name disables use of the specified container.
-func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig, failSwapOn bool, recorder record.EventRecorder) (ContainerManager, error) {
+func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig, failSwapOn bool, devicePluginEnabled bool, recorder record.EventRecorder) (ContainerManager, error) {
 	subsystems, err := GetCgroupSubsystems()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get mounted cgroup subsystems: %v", err)
@@ -252,7 +253,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 		return nil, err
 	}
 
-	return &containerManagerImpl{
+	cm := &containerManagerImpl{
 		cadvisorInterface:   cadvisorInterface,
 		mountUtil:           mountUtil,
 		NodeConfig:          nodeConfig,
@@ -262,7 +263,31 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 		cgroupRoot:          cgroupRoot,
 		recorder:            recorder,
 		qosContainerManager: qosContainerManager,
-	}, nil
+	}
+
+	updateDeviceCapacityFunc := func(updates v1.ResourceList) {
+		cm.Lock()
+		defer cm.Unlock()
+		for k, v := range updates {
+			if v.Value() <= 0 {
+				delete(cm.capacity, k)
+			} else {
+				cm.capacity[k] = v
+			}
+		}
+	}
+
+	glog.Infof("Creating device plugin handler: %t", devicePluginEnabled)
+	if devicePluginEnabled {
+		cm.devicePluginHandler, err = NewDevicePluginHandlerImpl(updateDeviceCapacityFunc)
+	} else {
+		cm.devicePluginHandler, err = NewDevicePluginHandlerStub()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return cm, nil
 }
 
 // NewPodContainerManager is a factory method returns a PodContainerManager object
@@ -545,6 +570,11 @@ func (cm *containerManagerImpl) Start(node *v1.Node, activePods ActivePodsFunc)
 		}
 		close(stopChan)
 	}, time.Second, stopChan)
+
+	// Starts device plugin manager.
+	if err := cm.devicePluginHandler.Start(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -562,6 +592,76 @@ func (cm *containerManagerImpl) setFsCapacity() error {
 	return nil
 }
 
+// TODO: move the GetResources logic to PodContainerManager.
+func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Container, activePods []*v1.Pod) (*kubecontainer.RunContainerOptions, error) {
+	opts := &kubecontainer.RunContainerOptions{}
+	// Gets devices, mounts, and envs from device plugin handler.
+	glog.V(3).Infof("Calling devicePluginHandler AllocateDevices")
+	// Maps to detect duplicate settings.
+	devsMap := make(map[string]string)
+	mountsMap := make(map[string]string)
+	envsMap := make(map[string]string)
+	allocResps, err := cm.devicePluginHandler.Allocate(pod, container, activePods)
+	if err != nil {
+		return opts, err
+	}
+	// Loops through AllocationResponses of all required extended resources.
+	for _, resp := range allocResps {
+		// Loops through runtime spec of all devices of the given resource.
+		for _, devRuntime := range resp.Spec {
+			// Updates RunContainerOptions.Devices.
+			for _, dev := range devRuntime.Devices {
+				if d, ok := devsMap[dev.ContainerPath]; ok {
+					glog.V(3).Infof("skip existing device %s %s", dev.ContainerPath, dev.HostPath)
+					if d != dev.HostPath {
+						glog.Errorf("Container device %s has conflicting mapping host devices: %s and %s",
+							dev.ContainerPath, d, dev.HostPath)
+					}
+					continue
+				}
+				devsMap[dev.ContainerPath] = dev.HostPath
+				opts.Devices = append(opts.Devices, kubecontainer.DeviceInfo{
+					PathOnHost:      dev.HostPath,
+					PathInContainer: dev.ContainerPath,
+					Permissions:     dev.Permissions,
+				})
+			}
+			// Updates RunContainerOptions.Mounts.
+			for _, mount := range devRuntime.Mounts {
+				if m, ok := mountsMap[mount.ContainerPath]; ok {
+					glog.V(3).Infof("skip existing mount %s %s", mount.ContainerPath, mount.HostPath)
+					if m != mount.HostPath {
+						glog.Errorf("Container mount %s has conflicting mapping host mounts: %s and %s",
+							mount.ContainerPath, m, mount.HostPath)
+					}
+					continue
+				}
+				mountsMap[mount.ContainerPath] = mount.HostPath
+				opts.Mounts = append(opts.Mounts, kubecontainer.Mount{
+					Name:           mount.ContainerPath,
+					ContainerPath:  mount.ContainerPath,
+					HostPath:       mount.HostPath,
+					ReadOnly:       mount.ReadOnly,
+					SELinuxRelabel: false,
+				})
+			}
+			// Updates RunContainerOptions.Envs.
+			for k, v := range devRuntime.Envs {
+				if e, ok := envsMap[k]; ok {
+					glog.V(3).Infof("skip existing envs %s %s", k, v)
+					if e != v {
+						glog.Errorf("Environment variable %s has conflicting setting: %s and %s", k, e, v)
+					}
+					continue
+				}
+				envsMap[k] = v
+				opts.Envs = append(opts.Envs, kubecontainer.EnvVar{Name: k, Value: v})
+			}
+		}
+	}
+	return opts, nil
+}
+
 func (cm *containerManagerImpl) SystemCgroupsLimit() v1.ResourceList {
 	cpuLimit := int64(0)
 
@@ -822,13 +922,3 @@ func (cm *containerManagerImpl) GetCapacity() v1.ResourceList {
 	defer cm.RUnlock()
 	return cm.capacity
 }
-
-// GetDevicePluginHandler returns the DevicePluginHandler
-func (m *containerManagerImpl) GetDevicePluginHandler() DevicePluginHandler {
-	return m.devicePluginHdler
-}
-
-// SetDevicePluginHandler sets the DevicePluginHandler
-func (m *containerManagerImpl) SetDevicePluginHandler(d DevicePluginHandler) {
-	m.devicePluginHdler = d
-}