github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-24 10:41:43 +00:00
Code Issues Projects Releases Wiki Activity

vfio: use driver_override field for device binding.

Browse Source 
The current implementation for device binding using driver bind/unbind
and new_id fails in the scenario when the physical device is not bound
to a driver before assigning it to vfio.
There exists and updated mechanism to accomplish the same that does not
have the same issue as above.
The driver_override field for a device allows us to specify the driver for a device
rather than relying on the bound driver to provide a positive match of the
device. It also has other advantages referenced here:
https://patchwork.kernel.org/project/linux-pci/patch/1396372540.476.160.camel@ul30vt.home/

So use the updated driver_override mechanism for binding/unbinding a
physical device/virtual function to vfio-pci.

Signed-off-by: liangxianlong <liang.xianlong@zte.com.cn>
Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
This commit is contained in:
Archana Shinde 2023-12-06 00:40:42 -08:00
parent 6aff5f300a
commit 2fef4bc844
2 changed files with 58 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
src/runtime/pkg/device/drivers/vfio.go
View File

@ -22,13 +22,12 @@ import (
// bind/unbind paths to aid in SRIOV VF bring-up/restore
const (
pciDriverUnbindPath = "/sys/bus/pci/devices/%s/driver/unbind"
pciDriverBindPath = "/sys/bus/pci/drivers/%s/bind"
vfioNewIDPath = "/sys/bus/pci/drivers/vfio-pci/new_id"
vfioRemoveIDPath = "/sys/bus/pci/drivers/vfio-pci/remove_id"
iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group"
vfioDevPath = "/dev/vfio/%s"
vfioAPSysfsDir = "/sys/devices/vfio_ap"
pciDriverUnbindPath = "/sys/bus/pci/devices/%s/driver/unbind"
pciDriverOverridePath = "/sys/bus/pci/devices/%s/driver_override"
driversProbePath = "/sys/bus/pci/drivers_probe"
iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group"
vfioDevPath = "/dev/vfio/%s"
vfioAPSysfsDir = "/sys/devices/vfio_ap"
)
// VFIODevice is a vfio device meant to be passed to the hypervisor
@ -269,42 +268,45 @@ func GetBDF(deviceSysStr string) string {
return tokens[1]
}
// BindDevicetoVFIO binds the device to vfio driver after unbinding from host.
// BindDevicetoVFIO binds the device to vfio driver after unbinding from host
// driver if present.
// Will be called by a network interface or a generic pcie device.
func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) {
func BindDevicetoVFIO(bdf, hostDriver string) (string, error) {
overrideDriverPath := fmt.Sprintf(pciDriverOverridePath, bdf)
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-override-path": overrideDriverPath,
}).Info("Write vfio-pci to driver_override")
// Write vfio-pci to driver_override file to allow the device to bind to vfio-pci
// Reference: https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-bus-platform
if err := utils.WriteToFile(overrideDriverPath, []byte("vfio-pci")); err != nil {
return "", err
}
// Unbind from the host driver
unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf)
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": unbindDriverPath,
}).Info("Unbinding device from driver")
if err := utils.WriteToFile(unbindDriverPath, []byte(bdf)); err != nil {
return "", err
}
// Unbind device from the host driver. In some cases, a driver may not be bound
// to the device, in which case this step may fail. Hence ignore error for this step.
utils.WriteToFile(unbindDriverPath, []byte(bdf))
// Add device id to vfio driver.
deviceLogger().WithFields(logrus.Fields{
"vendor-device-id": vendorDeviceID,
"vfio-new-id-path": vfioNewIDPath,
}).Info("Writing vendor-device-id to vfio new-id path")
"device-bdf": bdf,
"drivers-probe-path": driversProbePath,
}).Info("Writing bdf to drivers-probe-path")
if err := utils.WriteToFile(vfioNewIDPath, []byte(vendorDeviceID)); err != nil {
// Invoke drivers_probe so that the driver matching driver_override, in our case
// the vfio-pci driver will probe the device.
if err := utils.WriteToFile(driversProbePath, []byte(bdf)); err != nil {
return "", err
}
// Bind to vfio-pci driver.
bindDriverPath := fmt.Sprintf(pciDriverBindPath, "vfio-pci")
api.DeviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": bindDriverPath,
}).Info("Binding device to vfio driver")
// Device may be already bound at this time because of earlier write to new_id, ignore error
utils.WriteToFile(bindDriverPath, []byte(bdf))
// Determine the iommu group that the device belongs to.
groupPath, err := os.Readlink(fmt.Sprintf(iommuGroupPath, bdf))
if err != nil {
return "", err
@ -313,11 +315,25 @@ func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) {
return fmt.Sprintf(vfioDevPath, filepath.Base(groupPath)), nil
}
// BindDevicetoHost binds the device to the host driver after unbinding from vfio-pci.
func BindDevicetoHost(bdf, hostDriver, vendorDeviceID string) error {
// Unbind from vfio-pci driver
unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf)
// BindDevicetoHost unbinds the device from vfio-pci driver and binds it to the
// previously bound driver.
func BindDevicetoHost(bdf, hostDriver string) error {
overrideDriverPath := fmt.Sprintf(pciDriverOverridePath, bdf)
api.DeviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-override-path": overrideDriverPath,
}).Infof("Write %s to driver_override", hostDriver)
// write previously bound host driver to driver_override to allow the
// device to bind to it. This could be empty which means the device will not be
// bound to any driver later on.
if err := utils.WriteToFile(overrideDriverPath, []byte(hostDriver)); err != nil {
return err
}
// Unbind device from vfio-pci driver.
unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf)
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": unbindDriverPath,
}).Info("Unbinding device from driver")
@ -326,17 +342,12 @@ func BindDevicetoHost(bdf, hostDriver, vendorDeviceID string) error {
return err
}
// To prevent new VFs from binding to VFIO-PCI, remove_id
if err := utils.WriteToFile(vfioRemoveIDPath, []byte(vendorDeviceID)); err != nil {
return err
}
deviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"drivers-probe-path": driversProbePath,
}).Info("Writing bdf to drivers-probe-path")
// Bind back to host driver
bindDriverPath := fmt.Sprintf(pciDriverBindPath, hostDriver)
api.DeviceLogger().WithFields(logrus.Fields{
"device-bdf": bdf,
"driver-path": bindDriverPath,
}).Info("Binding back device to host driver")
return utils.WriteToFile(bindDriverPath, []byte(bdf))
// Invoke drivers_probe so that the driver matching driver_override, in this case
// the previous host driver will probe the device.
return utils.WriteToFile(driversProbePath, []byte(bdf))
}

4
src/runtime/virtcontainers/physical_endpoint.go
View File

@ -218,11 +218,11 @@ func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) {
}
func bindNICToVFIO(endpoint *PhysicalEndpoint) (string, error) {
return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver)
}
func bindNICToHost(endpoint *PhysicalEndpoint) error {
return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver)
}
func (endpoint *PhysicalEndpoint) save() persistapi.NetworkEndpoint {