github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-04-29 04:04:45 +00:00
Code Issues Projects Releases Wiki Activity

gpu: Add fwcfg helper function

Browse Source 
Added driver util function for easier handling of VFIO
devices outside of the VFIO module. At the sandbox level
we may need to set options depending if we have a VFIO/PCIe
device, like the fwCfg for confiential guests.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
This commit is contained in:
Zvonko Kaiser 2023-04-21 10:43:56 +00:00
parent 131f056a12
commit 2a830177ca
6 changed files with 56 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/runtime/cmd/kata-runtime/kata-env.go
View File

@ -307,17 +307,17 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
}
return HypervisorInfo{
Debug: config.HypervisorConfig.Debug,
MachineType: config.HypervisorConfig.HypervisorMachineType,
Version: version,
Path: hypervisorPath,
BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver,
Msize9p: config.HypervisorConfig.Msize9p,
MemorySlots: config.HypervisorConfig.MemSlots,
EntropySource: config.HypervisorConfig.EntropySource,
SharedFS: config.HypervisorConfig.SharedFS,
VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon,
Debug: config.HypervisorConfig.Debug,
MachineType: config.HypervisorConfig.HypervisorMachineType,
Version: version,
Path: hypervisorPath,
BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver,
Msize9p: config.HypervisorConfig.Msize9p,
MemorySlots: config.HypervisorConfig.MemSlots,
EntropySource: config.HypervisorConfig.EntropySource,
SharedFS: config.HypervisorConfig.SharedFS,
VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon,
ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO,
HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
PCIeRootPort: config.HypervisorConfig.PCIeRootPort,
SocketPath: socketPath,

8
src/runtime/pkg/device/drivers/utils.go
View File

@ -156,8 +156,10 @@ func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (boo
}
// GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group
// We can reuse this function at various leverls, sandbox, container.
func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIODev, error) {
// We can reuse this function at various levels, sandbox, container.
// Only the VFIO module is allowed to do bus assignments, all other modules need to
// ignore it if used as helper function to get VFIO information.
func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignment bool) ([]*config.VFIODev, error) {
vfioDevs := []*config.VFIODev{}
@ -204,7 +206,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device *config.DeviceInfo) ([]*config.VFIOD
IsPCIe: isPCIe,
Class: pciClass,
}
if isPCIe {
if isPCIe && !ignoreBusAssignment {
vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
AllPCIeDevs[deviceBDF] = true
}

66
src/runtime/pkg/device/drivers/vfio.go
View File

@ -11,7 +11,6 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/sirupsen/logrus"
@ -71,74 +70,11 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
}
}()
device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(device.DeviceInfo)
vfioGroup := filepath.Base(device.DeviceInfo.HostPath)
iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices")
deviceFiles, err := os.ReadDir(iommuDevicesPath)
device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo, false)
if err != nil {
return err
}
// Pass all devices in iommu group
for i, deviceFile := range deviceFiles {
//Get bdf of device eg 0000:00:1c.0
deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath)
if err != nil {
return err
}
id := utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize)
pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
ignorePCIDevice, err := checkIgnorePCIClass(pciClass, deviceBDF, 0x0600)
if err != nil {
return err
}
if ignorePCIDevice {
continue
}
var vfio config.VFIODev
switch vfioDeviceType {
case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
isPCIe := isPCIeDevice(deviceBDF)
// Do not directly assign to `vfio` -- need to access field still
vfioPCI := config.VFIOPCIDev{
ID: id,
Type: vfioDeviceType,
BDF: deviceBDF,
SysfsDev: deviceSysfsDev,
IsPCIe: isPCIe,
Class: pciClass,
}
if isPCIe {
vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs))
AllPCIeDevs[deviceBDF] = true
}
vfio = vfioPCI
case config.VFIOAPDeviceMediatedType:
devices, err := GetAPVFIODevices(deviceSysfsDev)
if err != nil {
return err
}
vfio = config.VFIOAPDev{
ID: id,
SysfsDev: deviceSysfsDev,
Type: config.VFIOAPDeviceMediatedType,
APDevices: devices,
}
default:
return fmt.Errorf("Failed to append device: VFIO device type unrecognized")
}
device.VfioDevs = append(device.VfioDevs, &vfio)
}
coldPlug := device.DeviceInfo.ColdPlug
deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device")

1
src/runtime/pkg/katatestutils/utils.go
View File

@ -319,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + `
hotplug_vfio_on_root_bus = ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `
cold_plug_vfio = "` + config.ColdPlugVFIO + `"
msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
guest_hook_path = "` + config.DefaultGuestHookPath + `"

44
src/runtime/virtcontainers/qemu.go
View File

@ -718,20 +718,50 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
// At the sandbox level we alreaady checked that we have a
// VFIO device, pass-through of a PCIe device needs allocated
// mmemory in the firmware otherwise BARs cannot be mapped
if len(hypervisorConfig.VFIODevices) > 0 {
fwCfg := govmmQemu.FwCfg{
Name: "opt/ovmf/X-PciMmio64Mb",
Str: "262144",
}
qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg)
// First check if we have a PCIe devices, otherwise ignore
err, fwCfg := q.appendFwCfgForConfidentialGuest(hypervisorConfig.VFIODevices)
if err != nil {
return err
}
if fwCfg != nil {
qemuConfig.FwCfg = append(qemuConfig.FwCfg, *fwCfg)
}
}
q.qemuConfig = qemuConfig
return err
}
// appendFwCfgForConfidentialGuest appends the firmware configuration for a
// VFIO and PCIe device, otherwise it will be ignored.
func (q *qemu) appendFwCfgForConfidentialGuest(vfioDevices []config.DeviceInfo) (error, *govmmQemu.FwCfg) {
var err error
for _, dev := range vfioDevices {
dev.HostPath, err = config.GetHostPath(dev, false, "")
if err != nil {
return err, nil
}
vfioDevs, err := drivers.GetAllVFIODevicesFromIOMMUGroup(dev, true)
if err != nil {
return err, nil
}
fwCfg := govmmQemu.FwCfg{}
for _, vfioDev := range vfioDevs {
switch (*vfioDev).GetType() {
case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
if (*vfioDev).(config.VFIOPCIDev).IsPCIe {
fwCfg = govmmQemu.FwCfg{
Name: "opt/ovmf/X-PciMmio64Mb",
Str: "262144",
}
return nil, &fwCfg
}
}
}
}
return nil, nil
}
func (q *qemu) checkBpfEnabled() {
if q.config.SeccompSandbox != "" {
out, err := os.ReadFile("/proc/sys/net/core/bpf_jit_enable")

2
src/runtime/virtcontainers/sandbox.go
View File

@ -623,7 +623,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
// If we have a confidential guest we need to cold-plug the PCIe VFIO devices
// until we have TDISP/IDE PCIe support.
coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO == hv.RootPort)
var devs []config.DeviceInfo
for cnt, containers := range sandboxConfig.Containers {
for dev, device := range containers.DeviceInfos {