github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-04-30 12:44:39 +00:00
Code Issues Projects Releases Wiki Activity

gpu: Add config to TOML

Browse Source 
Update cold-plug and hot-plug setting to include bridge, root and
switch-port

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
This commit is contained in:
Zvonko Kaiser 2023-05-08 08:04:43 +00:00
parent da42801c38
commit 55a66eb7fb
20 changed files with 199 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/runtime/config/configuration-qemu.toml.in
View File

@ -352,8 +352,15 @@ pflashes = []
# Default false # Default false
#hotplug_vfio_on_root_bus = true #hotplug_vfio_on_root_bus = true
# Enable hot-plugging of VFIO devices to a bridge-port,
# root-port or switch-port.
# The default setting is "no-port"
#hot_plug_vfio = "root-port"
# In a confidential compute environment hot-plugging can compromise # In a confidential compute environment hot-plugging can compromise
# security. Enable cold-plugging of VFIO devices to a root-port. # security.
# Enable cold-plugging of VFIO devices to a bridge-port,
# root-port or switch-port.
# The default setting is "no-port", which means disabled. # The default setting is "no-port", which means disabled.
#cold_plug_vfio = "root-port" #cold_plug_vfio = "root-port"

10
src/runtime/pkg/device/drivers/utils.go
View File

@ -47,7 +47,7 @@ func deviceLogger() *logrus.Entry {
return api.DeviceLogger() return api.DeviceLogger()
} }
// IsPCIeDevice Identify PCIe device by reading the size of the PCI config space // IsPCIeDevice Identifies PCIe device by reading the size of the PCI config space
// Plain PCI device have 256 bytes of config space where PCIe devices have 4K // Plain PCI device have 256 bytes of config space where PCIe devices have 4K
func IsPCIeDevice(bdf string) bool { func IsPCIeDevice(bdf string) bool {
if len(strings.Split(bdf, ":")) == 2 { if len(strings.Split(bdf, ":")) == 2 {
@ -157,9 +157,7 @@ func checkIgnorePCIClass(pciClass string, deviceBDF string, bitmask uint64) (boo
// GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group // GetAllVFIODevicesFromIOMMUGroup returns all the VFIO devices in the IOMMU group
// We can reuse this function at various levels, sandbox, container. // We can reuse this function at various levels, sandbox, container.
// Only the VFIO module is allowed to do bus assignments, all other modules need to func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODev, error) {
// ignore it if used as helper function to get VFIO information.
func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignment bool) ([]*config.VFIODev, error) {
vfioDevs := []*config.VFIODev{} vfioDevs := []*config.VFIODev{}
@ -207,8 +205,8 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo, ignoreBusAssignme
Class: pciClass, Class: pciClass,
Rank: -1, Rank: -1,
} }
if isPCIe && !ignoreBusAssignment { if isPCIe {
vfioPCI.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, len(AllPCIeDevs)) vfioPCI.Rank = len(AllPCIeDevs)
AllPCIeDevs[deviceBDF] = true AllPCIeDevs[deviceBDF] = true
} }
vfio = vfioPCI vfio = vfioPCI

2
src/runtime/pkg/device/drivers/vfio.go
View File

@ -73,7 +73,7 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
} }
}() }()
device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo, false) device.VfioDevs, err = GetAllVFIODevicesFromIOMMUGroup(*device.DeviceInfo)
if err != nil { if err != nil {
return err return err
} }

2
src/runtime/pkg/katautils/config-settings.go.in
View File

@ -109,7 +109,7 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
// Default config file used by stateless systems. // Default config file used by stateless systems.
var defaultRuntimeConfiguration = "@CONFIG_PATH@" var defaultRuntimeConfiguration = "@CONFIG_PATH@"
const defaultHotPlugVFIO = hv.BridgePort const defaultHotPlugVFIO = hv.NoPort
const defaultColdPlugVFIO = hv.NoPort const defaultColdPlugVFIO = hv.NoPort

25
src/runtime/pkg/katautils/config.go
View File

@ -76,6 +76,7 @@ type factory struct {
VMCacheNumber uint `toml:"vm_cache_number"` VMCacheNumber uint `toml:"vm_cache_number"`
Template bool `toml:"enable_template"` Template bool `toml:"enable_template"`
} }
type hypervisor struct { type hypervisor struct {
Path string `toml:"path"` Path string `toml:"path"`
JailerPath string `toml:"jailer_path"` JailerPath string `toml:"jailer_path"`
@ -886,6 +887,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
GuestMemoryDumpPath: h.GuestMemoryDumpPath, GuestMemoryDumpPath: h.GuestMemoryDumpPath,
GuestMemoryDumpPaging: h.GuestMemoryDumpPaging, GuestMemoryDumpPaging: h.GuestMemoryDumpPaging,
ConfidentialGuest: h.ConfidentialGuest, ConfidentialGuest: h.ConfidentialGuest,
SevSnpGuest: h.SevSnpGuest,
GuestSwap: h.GuestSwap, GuestSwap: h.GuestSwap,
Rootless: h.Rootless, Rootless: h.Rootless,
LegacySerial: h.LegacySerial, LegacySerial: h.LegacySerial,
@ -1677,10 +1679,7 @@ func checkConfig(config oci.RuntimeConfig) error {
hotPlugVFIO := config.HypervisorConfig.HotPlugVFIO hotPlugVFIO := config.HypervisorConfig.HotPlugVFIO
coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO
machineType := config.HypervisorConfig.HypervisorMachineType machineType := config.HypervisorConfig.HypervisorMachineType
if err := checkPCIeConfig(coldPlugVFIO, machineType); err != nil { if err := checkPCIeConfig(coldPlugVFIO, hotPlugVFIO, machineType); err != nil {
return err
}
if err := checkPCIeConfig(hotPlugVFIO, machineType); err != nil {
return err return err
} }
@ -1690,19 +1689,29 @@ func checkConfig(config oci.RuntimeConfig) error {
// checkPCIeConfig ensures the PCIe configuration is valid. // checkPCIeConfig ensures the PCIe configuration is valid.
// Only allow one of the following settings for cold-plug: // Only allow one of the following settings for cold-plug:
// no-port, root-port, switch-port // no-port, root-port, switch-port
func checkPCIeConfig(vfioPort hv.PCIePort, machineType string) error { func checkPCIeConfig(coldPlug hv.PCIePort, hotPlug hv.PCIePort, machineType string) error {
// Currently only QEMU q35 supports advanced PCIe topologies // Currently only QEMU q35 supports advanced PCIe topologies
// firecracker, dragonball do not have right now any PCIe support // firecracker, dragonball do not have right now any PCIe support
if machineType != "q35" { if machineType != "q35" {
return nil return nil
} }
if vfioPort == hv.NoPort || vfioPort == hv.BridgePort ||
vfioPort == hv.RootPort || vfioPort == hv.SwitchPort { if coldPlug != hv.NoPort && hotPlug != hv.NoPort {
return fmt.Errorf("invalid hot-plug=%s and cold-plug=%s settings, only one of them can be set", coldPlug, hotPlug)
}
var port hv.PCIePort
if coldPlug != hv.NoPort {
port = coldPlug
}
if hotPlug != hv.NoPort {
port = hotPlug
}
if port == hv.NoPort || port == hv.BridgePort || port == hv.RootPort || port == hv.SwitchPort {
return nil return nil
} }
return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s", return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s",
vfioPort, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort) coldPlug, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort)
} }
// checkNetNsConfig performs sanity checks on disable_new_netns config. // checkNetNsConfig performs sanity checks on disable_new_netns config.

58
src/runtime/pkg/oci/utils_test.go
View File

@ -821,22 +821,22 @@ func TestRegexpContains(t *testing.T) {
//nolint: govet //nolint: govet
type testData struct { type testData struct {
toMatch string
regexps []string regexps []string
toMatch string
expected bool expected bool
} }
data := []testData{ data := []testData{
{regexps: []string{}, toMatch: "", expected: false}, {[]string{}, "", false},
{regexps: []string{}, toMatch: "nonempty", expected: false}, {[]string{}, "nonempty", false},
{regexps: []string{"simple"}, toMatch: "simple", expected: true}, {[]string{"simple"}, "simple", true},
{regexps: []string{"simple"}, toMatch: "some_simple_text", expected: true}, {[]string{"simple"}, "some_simple_text", true},
{regexps: []string{"simple"}, toMatch: "simp", expected: false}, {[]string{"simple"}, "simp", false},
{regexps: []string{"one", "two"}, toMatch: "one", expected: true}, {[]string{"one", "two"}, "one", true},
{regexps: []string{"one", "two"}, toMatch: "two", expected: true}, {[]string{"one", "two"}, "two", true},
{regexps: []string{"o*"}, toMatch: "oooo", expected: true}, {[]string{"o*"}, "oooo", true},
{regexps: []string{"o*"}, toMatch: "oooa", expected: true}, {[]string{"o*"}, "oooa", true},
{regexps: []string{"^o*$"}, toMatch: "oooa", expected: false}, {[]string{"^o*$"}, "oooa", false},
} }
for _, d := range data { for _, d := range data {
@ -850,25 +850,25 @@ func TestCheckPathIsInGlobs(t *testing.T) {
//nolint: govet //nolint: govet
type testData struct { type testData struct {
toMatch string
globs []string globs []string
toMatch string
expected bool expected bool
} }
data := []testData{ data := []testData{
{globs: []string{}, toMatch: "", expected: false}, {[]string{}, "", false},
{globs: []string{}, toMatch: "nonempty", expected: false}, {[]string{}, "nonempty", false},
{globs: []string{"simple"}, toMatch: "simple", expected: false}, {[]string{"simple"}, "simple", false},
{globs: []string{"simple"}, toMatch: "some_simple_text", expected: false}, {[]string{"simple"}, "some_simple_text", false},
{globs: []string{"/bin/ls"}, toMatch: "/bin/ls", expected: true}, {[]string{"/bin/ls"}, "/bin/ls", true},
{globs: []string{"/bin/ls", "/bin/false"}, toMatch: "/bin/ls", expected: true}, {[]string{"/bin/ls", "/bin/false"}, "/bin/ls", true},
{globs: []string{"/bin/ls", "/bin/false"}, toMatch: "/bin/false", expected: true}, {[]string{"/bin/ls", "/bin/false"}, "/bin/false", true},
{globs: []string{"/bin/ls", "/bin/false"}, toMatch: "/bin/bar", expected: false}, {[]string{"/bin/ls", "/bin/false"}, "/bin/bar", false},
{globs: []string{"/bin/*ls*"}, toMatch: "/bin/ls", expected: true}, {[]string{"/bin/*ls*"}, "/bin/ls", true},
{globs: []string{"/bin/*ls*"}, toMatch: "/bin/false", expected: true}, {[]string{"/bin/*ls*"}, "/bin/false", true},
{globs: []string{"bin/ls"}, toMatch: "/bin/ls", expected: false}, {[]string{"bin/ls"}, "/bin/ls", false},
{globs: []string{"./bin/ls"}, toMatch: "/bin/ls", expected: false}, {[]string{"./bin/ls"}, "/bin/ls", false},
{globs: []string{"*/bin/ls"}, toMatch: "/bin/ls", expected: false}, {[]string{"*/bin/ls"}, "/bin/ls", false},
} }
for _, d := range data { for _, d := range data {
@ -923,10 +923,10 @@ func TestParseAnnotationUintConfiguration(t *testing.T) {
// nolint: govet // nolint: govet
testCases := []struct { testCases := []struct {
err error
annotations map[string]string annotations map[string]string
validFunc func(uint64) error
expected uint64 expected uint64
err error
validFunc func(uint64) error
}{ }{
{ {
annotations: map[string]string{key: ""}, annotations: map[string]string{key: ""},
@ -1007,10 +1007,10 @@ func TestParseAnnotationBoolConfiguration(t *testing.T) {
// nolint: govet // nolint: govet
testCases := []struct { testCases := []struct {
err error
annotationKey string annotationKey string
annotationValueList []string annotationValueList []string
expected bool expected bool
err error
}{ }{
{ {
annotationKey: boolKey, annotationKey: boolKey,
@ -1207,8 +1207,8 @@ func TestNewMount(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
testCases := []struct { testCases := []struct {
in specs.Mount
out vc.Mount out vc.Mount
in specs.Mount
}{ }{
{ {
in: specs.Mount{ in: specs.Mount{

2
src/runtime/virtcontainers/acrn.go
View File

@ -45,10 +45,10 @@ type AcrnState struct {
// Acrn is an Hypervisor interface implementation for the Linux acrn hypervisor. // Acrn is an Hypervisor interface implementation for the Linux acrn hypervisor.
type Acrn struct { type Acrn struct {
sandbox *Sandbox
ctx context.Context ctx context.Context
arch acrnArch arch acrnArch
store persistapi.PersistDriver store persistapi.PersistDriver
sandbox *Sandbox
id string id string
acrnConfig Config acrnConfig Config
config HypervisorConfig config HypervisorConfig

6
src/runtime/virtcontainers/clh.go
View File

@ -246,15 +246,15 @@ func (s *CloudHypervisorState) reset() {
} }
type cloudHypervisor struct { type cloudHypervisor struct {
vmconfig chclient.VmConfig
console console.Console console console.Console
virtiofsDaemon VirtiofsDaemon virtiofsDaemon VirtiofsDaemon
ctx context.Context
APIClient clhClient APIClient clhClient
ctx context.Context
id string
netDevices *[]chclient.NetConfig netDevices *[]chclient.NetConfig
devicesIds map[string]string devicesIds map[string]string
netDevicesFiles map[string][]*os.File netDevicesFiles map[string][]*os.File
id string vmconfig chclient.VmConfig
state CloudHypervisorState state CloudHypervisorState
config HypervisorConfig config HypervisorConfig
stopped int32 stopped int32

8
src/runtime/virtcontainers/clh_test.go
View File

@ -188,13 +188,13 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
} }
// nolint: govet // nolint: govet
tests := []struct { tests := []struct {
args args
name string name string
args args
wantErr bool wantErr bool
}{ }{
{name: "TapEndpoint", args: args{e: &TapEndpoint{}}, wantErr: true}, {"TapEndpoint", args{e: &TapEndpoint{}}, true},
{name: "Empty VethEndpoint", args: args{e: &VethEndpoint{}}, wantErr: true}, {"Empty VethEndpoint", args{e: &VethEndpoint{}}, true},
{name: "Valid VethEndpoint", args: args{e: validVeth}, wantErr: false}, {"Valid VethEndpoint", args{e: validVeth}, false},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {

2
src/runtime/virtcontainers/hypervisor.go
View File

@ -507,7 +507,7 @@ type HypervisorConfig struct {
// RawDevics are used to get PCIe device info early before the sandbox // RawDevics are used to get PCIe device info early before the sandbox
// is started to make better PCIe topology decisions // is started to make better PCIe topology decisions
RawDevices []config.DeviceInfo VFIODevices []config.DeviceInfo
// HotplugVFIO is used to indicate if devices need to be hotplugged on the // HotplugVFIO is used to indicate if devices need to be hotplugged on the
// root port or a switch // root port or a switch

13
src/runtime/virtcontainers/kata_agent.go
View File

@ -286,12 +286,19 @@ type KataAgentState struct {
type kataAgent struct { type kataAgent struct {
ctx context.Context ctx context.Context
vmSocket interface{} vmSocket interface{}
client *kataclient.AgentClient client *kataclient.AgentClient
reqHandlers map[string]reqFunc
state KataAgentState // lock protects the client pointer
kmodules []string
sync.Mutex sync.Mutex
state KataAgentState
reqHandlers map[string]reqFunc
kmodules []string
dialTimout uint32 dialTimout uint32
keepConn bool keepConn bool
dead bool dead bool
} }

24
src/runtime/virtcontainers/kata_agent_test.go
View File

@ -235,10 +235,10 @@ func TestHandleDeviceBlockVolume(t *testing.T) {
// nolint: govet // nolint: govet
tests := []struct { tests := []struct {
inputDev *drivers.BlockDevice
resultVol *pb.Storage
BlockDeviceDriver string BlockDeviceDriver string
inputMount Mount inputMount Mount
inputDev *drivers.BlockDevice
resultVol *pb.Storage
}{ }{
{ {
inputDev: &drivers.BlockDevice{ inputDev: &drivers.BlockDevice{
@ -1024,10 +1024,10 @@ func TestKataAgentKernelParams(t *testing.T) {
// nolint: govet // nolint: govet
type testData struct { type testData struct {
expectedParams []Param
containerPipeSize uint32
debug bool debug bool
trace bool trace bool
containerPipeSize uint32
expectedParams []Param
} }
debugParam := Param{Key: "agent.log", Value: "debug"} debugParam := Param{Key: "agent.log", Value: "debug"}
@ -1036,28 +1036,28 @@ func TestKataAgentKernelParams(t *testing.T) {
containerPipeSizeParam := Param{Key: vcAnnotations.ContainerPipeSizeKernelParam, Value: "2097152"} containerPipeSizeParam := Param{Key: vcAnnotations.ContainerPipeSizeKernelParam, Value: "2097152"}
data := []testData{ data := []testData{
{debug: false, trace: false, containerPipeSize: 0, expectedParams: []Param{}}, {false, false, 0, []Param{}},
// Debug // Debug
{debug: true, trace: false, containerPipeSize: 0, expectedParams: []Param{debugParam}}, {true, false, 0, []Param{debugParam}},
// Tracing // Tracing
{debug: false, trace: true, containerPipeSize: 0, expectedParams: []Param{traceParam}}, {false, true, 0, []Param{traceParam}},
// Debug + Tracing // Debug + Tracing
{debug: true, trace: true, containerPipeSize: 0, expectedParams: []Param{debugParam, traceParam}}, {true, true, 0, []Param{debugParam, traceParam}},
// pipesize // pipesize
{debug: false, trace: false, containerPipeSize: 2097152, expectedParams: []Param{containerPipeSizeParam}}, {false, false, 0, []Param{containerPipeSizeParam}},
// Debug + pipesize // Debug + pipesize
{debug: true, trace: false, containerPipeSize: 2097152, expectedParams: []Param{debugParam, containerPipeSizeParam}}, {true, false, 0, []Param{debugParam, containerPipeSizeParam}},
// Tracing + pipesize // Tracing + pipesize
{debug: false, trace: true, containerPipeSize: 2097152, expectedParams: []Param{traceParam, containerPipeSizeParam}}, {false, true, 0, []Param{traceParam, containerPipeSizeParam}},
// Debug + Tracing + pipesize // Debug + Tracing + pipesize
{debug: true, trace: true, containerPipeSize: 2097152, expectedParams: []Param{debugParam, traceParam, containerPipeSizeParam}}, {true, true, 0, []Param{debugParam, traceParam, containerPipeSizeParam}},
} }
for i, d := range data { for i, d := range data {

9
src/runtime/virtcontainers/monitor.go
View File

@ -22,12 +22,15 @@ var monitorLog = virtLog.WithField("subsystem", "virtcontainers/monitor")
// nolint: govet // nolint: govet
type monitor struct { type monitor struct {
sandbox *Sandbox
stopCh chan bool
watchers []chan error watchers []chan error
checkInterval time.Duration sandbox *Sandbox
wg sync.WaitGroup wg sync.WaitGroup
sync.Mutex sync.Mutex
stopCh chan bool
checkInterval time.Duration
running bool running bool
} }

65
src/runtime/virtcontainers/qemu.go
View File

@ -747,9 +747,8 @@ func (q *qemu) checkBpfEnabled() {
// There is only 64kB of IO memory each root,switch port will consume 4k hence // There is only 64kB of IO memory each root,switch port will consume 4k hence
// only 16 ports possible. // only 16 ports possible.
func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig) error { func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig) error {
// We do not need to do anything if we want to hotplug a VFIO to a // If no-port set just return no need to add PCIe Root Port or PCIe Switches
// pcie-pci-bridge, just return if hypervisorConfig.HotPlugVFIO == hv.NoPort && hypervisorConfig.ColdPlugVFIO == hv.NoPort {
if hypervisorConfig.HotPlugVFIO == hv.BridgePort {
return nil return nil
} }
// Add PCIe Root Port or PCIe Switches to the hypervisor // Add PCIe Root Port or PCIe Switches to the hypervisor
@ -772,37 +771,27 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg) qemuConfig.FwCfg = append(qemuConfig.FwCfg, fwCfg)
} }
// Get the number of hotpluggable ports needed from the provided devices // Get the number of hot(cold)-pluggable ports needed from the provided devices
var numOfHotPluggablePorts uint32 = 0 var numOfPluggablePorts uint32 = 0
for _, dev := range hypervisorConfig.RawDevices { for _, dev := range hypervisorConfig.VFIODevices {
hostPath, _ := config.GetHostPath(dev, false, "") var err error
if hostPath == "" { dev.HostPath, err = config.GetHostPath(dev, false, "")
continue
}
vfioNumberOrContainer := filepath.Base(hostPath)
// If we want to have VFIO inside of the VM we need to passthrough
// additionally /dev/vfio/vfio which is the VFIO "container".
// Ignore it and handle the remaining devices.
if strings.Compare(vfioNumberOrContainer, "vfio") == 0 {
continue
}
iommuDevicesPath := filepath.Join(config.SysIOMMUGroupPath, vfioNumberOrContainer, "devices")
deviceFiles, err := os.ReadDir(iommuDevicesPath)
if err != nil { if err != nil {
return err return fmt.Errorf("Cannot get host path for device: %v err: %v", dev, err)
} }
devicesPerIOMMUGroup, err := drivers.GetAllVFIODevicesFromIOMMUGroup(dev)
for _, deviceFile := range deviceFiles {
deviceBDF, _, _, err := drivers.GetVFIODetails(deviceFile.Name(), iommuDevicesPath)
if err != nil { if err != nil {
return err return fmt.Errorf("Cannot get all VFIO devices from IOMMU group with device: %v err: %v", dev, err)
}
if drivers.IsPCIeDevice(deviceBDF) {
numOfHotPluggablePorts = numOfHotPluggablePorts + 1
} }
for _, vfioDevice := range devicesPerIOMMUGroup {
if drivers.IsPCIeDevice(vfioDevice.BDF) {
numOfPluggablePorts = numOfPluggablePorts + 1
}
// Reset the Rank, the vfio module is going to assign the correct one
vfioDevice.Rank = -1
} }
} }
drivers.AllPCIeDevs = make(map[string]bool)
// If number of PCIe root ports > 16 then bail out otherwise we may // If number of PCIe root ports > 16 then bail out otherwise we may
// use up all slots or IO memory on the root bus and vfio-XXX-pci devices // use up all slots or IO memory on the root bus and vfio-XXX-pci devices
@ -817,20 +806,20 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
// If the user provided more root ports than we have detected // If the user provided more root ports than we have detected
// use the user provided number of PCIe root ports // use the user provided number of PCIe root ports
if numOfHotPluggablePorts < hypervisorConfig.PCIeRootPort { if numOfPluggablePorts < hypervisorConfig.PCIeRootPort {
numOfHotPluggablePorts = hypervisorConfig.PCIeRootPort numOfPluggablePorts = hypervisorConfig.PCIeRootPort
} }
// If the user provided more switch ports than we have detected // If the user provided more switch ports than we have detected
// use the user provided number of PCIe root ports // use the user provided number of PCIe root ports
if numOfHotPluggablePorts < hypervisorConfig.PCIeSwitchPort { if numOfPluggablePorts < hypervisorConfig.PCIeSwitchPort {
numOfHotPluggablePorts = hypervisorConfig.PCIeSwitchPort numOfPluggablePorts = hypervisorConfig.PCIeSwitchPort
} }
if q.state.HotPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus { if q.state.HotPlugVFIO == hv.RootPort || q.state.ColdPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus {
qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfHotPluggablePorts, memSize32bit, memSize64bit) qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit)
} }
if q.state.HotPlugVFIO == hv.SwitchPort { if q.state.HotPlugVFIO == hv.SwitchPort || q.state.ColdPlugVFIO == hv.SwitchPort {
qemuConfig.Devices = q.arch.appendPCIeSwitchPortDevice(qemuConfig.Devices, numOfHotPluggablePorts, memSize32bit, memSize64bit) qemuConfig.Devices = q.arch.appendPCIeSwitchPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit)
} }
return nil return nil
} }
@ -1847,6 +1836,7 @@ func (q *qemu) hotplugVFIODeviceSwitchPort(ctx context.Context, device *config.V
return fmt.Errorf("VFIO device is a PCIe device. Hotplug (%v) only supported on PCIe Root (%d) or PCIe Switch Ports (%v)", return fmt.Errorf("VFIO device is a PCIe device. Hotplug (%v) only supported on PCIe Root (%d) or PCIe Switch Ports (%v)",
q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort) q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort)
} }
device.Bus = fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, device.Rank) device.Bus = fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, device.Rank)
return q.executeVFIODeviceAdd(device) return q.executeVFIODeviceAdd(device)
} }
@ -1897,11 +1887,10 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op
} }
if op == AddDevice { if op == AddDevice {
buf, _ := json.Marshal(device) buf, _ := json.Marshal(device)
q.Logger().WithFields(logrus.Fields{ q.Logger().WithFields(logrus.Fields{
"machine-type": q.HypervisorConfig().HypervisorMachineType, "machine-type": q.HypervisorConfig().HypervisorMachineType,
"hotplug-vfio": q.state.HotPlugVFIO, "hot-plug-vfio": q.state.HotPlugVFIO,
"pcie-root-port": q.state.PCIeRootPort, "pcie-root-port": q.state.PCIeRootPort,
"pcie-switch-port": q.state.PCIeSwitchPort, "pcie-switch-port": q.state.PCIeSwitchPort,
"device-info": string(buf), "device-info": string(buf),

25
src/runtime/virtcontainers/sandbox.go
View File

@ -613,22 +613,26 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
if err := validateHypervisorConfig(&sandboxConfig.HypervisorConfig); err != nil { if err := validateHypervisorConfig(&sandboxConfig.HypervisorConfig); err != nil {
return nil, err return nil, err
} }
// Aggregate all the container devices and update the HV config
var devices []config.DeviceInfo
for _, ct := range sandboxConfig.Containers {
devices = append(devices, ct.DeviceInfos...)
}
sandboxConfig.HypervisorConfig.RawDevices = devices
// If we have a confidential guest we need to cold-plug the PCIe VFIO devices // If we have a confidential guest we need to cold-plug the PCIe VFIO devices
// until we have TDISP/IDE PCIe support. // until we have TDISP/IDE PCIe support.
coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort) coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
var devs []config.DeviceInfo // Aggregate all the containner devices for hot-plug and use them to dedcue
// the correct amount of ports to reserve for the hypervisor.
hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != hv.NoPort)
var vfioHotPlugDevices []config.DeviceInfo
var vfioColdPlugDevices []config.DeviceInfo
for cnt, containers := range sandboxConfig.Containers { for cnt, containers := range sandboxConfig.Containers {
for dev, device := range containers.DeviceInfos { for dev, device := range containers.DeviceInfos {
if coldPlugVFIO && deviceManager.IsVFIO(device.ContainerPath) { isVFIO := deviceManager.IsVFIO(device.ContainerPath)
if hotPlugVFIO && isVFIO {
vfioHotPlugDevices = append(vfioHotPlugDevices, device)
}
if coldPlugVFIO && isVFIO {
device.ColdPlug = true device.ColdPlug = true
devs = append(devs, device) vfioColdPlugDevices = append(vfioColdPlugDevices, device)
// We need to remove the devices marked for cold-plug // We need to remove the devices marked for cold-plug
// otherwise at the container level the kata-agent // otherwise at the container level the kata-agent
// will try to hot-plug them. // will try to hot-plug them.
@ -638,6 +642,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
} }
} }
} }
sandboxConfig.HypervisorConfig.VFIODevices = vfioHotPlugDevices
// store doesn't require hypervisor to be stored immediately // store doesn't require hypervisor to be stored immediately
if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil { if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil {
@ -652,7 +657,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
return s, nil return s, nil
} }
for _, dev := range devs { for _, dev := range vfioColdPlugDevices {
_, err := s.AddDevice(ctx, dev) _, err := s.AddDevice(ctx, dev)
if err != nil { if err != nil {
s.Logger().WithError(err).Debug("Cannot cold-plug add device") s.Logger().WithError(err).Debug("Cannot cold-plug add device")

80
src/runtime/virtcontainers/sandbox_test.go
View File

@ -650,8 +650,8 @@ func TestSandboxCreateAssets(t *testing.T) {
// nolint: govet // nolint: govet
type testData struct { type testData struct {
annotations map[string]string
assetType types.AssetType assetType types.AssetType
annotations map[string]string
} }
tmpfile, err := os.CreateTemp("", "virtcontainers-test-") tmpfile, err := os.CreateTemp("", "virtcontainers-test-")
@ -687,50 +687,50 @@ func TestSandboxCreateAssets(t *testing.T) {
data := []testData{ data := []testData{
{ {
assetType: types.FirmwareAsset, types.FirmwareAsset,
annotations: map[string]string{ map[string]string{
annotations.FirmwarePath: filename, annotations.FirmwarePath: filename,
annotations.FirmwareHash: assetContentHash, annotations.FirmwareHash: assetContentHash,
}, },
}, },
{ {
assetType: types.HypervisorAsset, types.HypervisorAsset,
annotations: map[string]string{ map[string]string{
annotations.HypervisorPath: filename, annotations.HypervisorPath: filename,
annotations.HypervisorHash: assetContentHash, annotations.HypervisorHash: assetContentHash,
}, },
}, },
{ {
assetType: types.HypervisorCtlAsset, types.HypervisorCtlAsset,
annotations: map[string]string{ map[string]string{
annotations.HypervisorCtlPath: filename, annotations.HypervisorCtlPath: filename,
annotations.HypervisorCtlHash: assetContentHash, annotations.HypervisorCtlHash: assetContentHash,
}, },
}, },
{ {
assetType: types.ImageAsset, types.ImageAsset,
annotations: map[string]string{ map[string]string{
annotations.ImagePath: filename, annotations.ImagePath: filename,
annotations.ImageHash: assetContentHash, annotations.ImageHash: assetContentHash,
}, },
}, },
{ {
assetType: types.InitrdAsset, types.InitrdAsset,
annotations: map[string]string{ map[string]string{
annotations.InitrdPath: filename, annotations.InitrdPath: filename,
annotations.InitrdHash: assetContentHash, annotations.InitrdHash: assetContentHash,
}, },
}, },
{ {
assetType: types.JailerAsset, types.JailerAsset,
annotations: map[string]string{ map[string]string{
annotations.JailerPath: filename, annotations.JailerPath: filename,
annotations.JailerHash: assetContentHash, annotations.JailerHash: assetContentHash,
}, },
}, },
{ {
assetType: types.KernelAsset, types.KernelAsset,
annotations: map[string]string{ map[string]string{
annotations.KernelPath: filename, annotations.KernelPath: filename,
annotations.KernelHash: assetContentHash, annotations.KernelHash: assetContentHash,
}, },
@ -1407,58 +1407,58 @@ func TestSandbox_Cgroups(t *testing.T) {
// nolint: govet // nolint: govet
tests := []struct { tests := []struct {
s *Sandbox
name string name string
s *Sandbox
wantErr bool wantErr bool
needRoot bool needRoot bool
}{ }{
{ {
name: "New sandbox", "New sandbox",
s: &Sandbox{}, &Sandbox{},
wantErr: true, true,
needRoot: false, false,
}, },
{ {
name: "New sandbox, new config", "New sandbox, new config",
s: &Sandbox{config: &SandboxConfig{}}, &Sandbox{config: &SandboxConfig{}},
wantErr: false, false,
needRoot: true, true,
}, },
{ {
name: "sandbox, container no sandbox type", "sandbox, container no sandbox type",
s: &Sandbox{ &Sandbox{
config: &SandboxConfig{Containers: []ContainerConfig{ config: &SandboxConfig{Containers: []ContainerConfig{
{}, {},
}}}, }}},
wantErr: false, false,
needRoot: true, true,
}, },
{ {
name: "sandbox, container sandbox type", "sandbox, container sandbox type",
s: &Sandbox{ &Sandbox{
config: &SandboxConfig{Containers: []ContainerConfig{ config: &SandboxConfig{Containers: []ContainerConfig{
sandboxContainer, sandboxContainer,
}}}, }}},
wantErr: false, false,
needRoot: true, true,
}, },
{ {
name: "sandbox, empty linux json", "sandbox, empty linux json",
s: &Sandbox{ &Sandbox{
config: &SandboxConfig{Containers: []ContainerConfig{ config: &SandboxConfig{Containers: []ContainerConfig{
emptyJSONLinux, emptyJSONLinux,
}}}, }}},
wantErr: false, false,
needRoot: true, true,
}, },
{ {
name: "sandbox, successful config", "sandbox, successful config",
s: &Sandbox{ &Sandbox{
config: &SandboxConfig{Containers: []ContainerConfig{ config: &SandboxConfig{Containers: []ContainerConfig{
successfulContainer, successfulContainer,
}}}, }}},
wantErr: false, false,
needRoot: true, true,
}, },
} }
for _, tt := range tests { for _, tt := range tests {

6
src/runtime/virtcontainers/veth_endpoint_test.go
View File

@ -85,16 +85,16 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) {
func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) { func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) {
// nolint: govet // nolint: govet
type endpointValues struct { type endpointValues struct {
ifName string
idx int idx int
ifName string
} }
assert := assert.New(t) assert := assert.New(t)
// all elements are expected to result in failure // all elements are expected to result in failure
failingValues := []endpointValues{ failingValues := []endpointValues{
{idx: -1, ifName: "bar"}, {-1, "bar"},
{idx: -1, ifName: ""}, {-1, ""},
} }
for _, d := range failingValues { for _, d := range failingValues {

14
src/runtime/virtcontainers/virtiofsd_test.go
View File

@ -17,13 +17,13 @@ import (
func TestVirtiofsdStart(t *testing.T) { func TestVirtiofsdStart(t *testing.T) {
// nolint: govet // nolint: govet
type fields struct { type fields struct {
ctx context.Context path string
socketPath string socketPath string
cache string cache string
sourcePath string
path string
extraArgs []string extraArgs []string
sourcePath string
PID int PID int
ctx context.Context
} }
sourcePath := t.TempDir() sourcePath := t.TempDir()
@ -41,13 +41,13 @@ func TestVirtiofsdStart(t *testing.T) {
// nolint: govet // nolint: govet
tests := []struct { tests := []struct {
fields fields
name string name string
fields fields
wantErr bool wantErr bool
}{ }{
{name: "empty config", fields: fields{}, wantErr: true}, {"empty config", fields{}, true},
{name: "Directory socket does not exist", fields: NoDirectorySocket, wantErr: true}, {"Directory socket does not exist", NoDirectorySocket, true},
{name: "valid config", fields: validConfig, wantErr: false}, {"valid config", validConfig, false},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {

2
tools/osbuilder/image-builder/image_builder.sh
View File

@ -271,7 +271,7 @@ calculate_required_disk_size() {
readonly image="$(mktemp)" readonly image="$(mktemp)"
readonly mount_dir="$(mktemp -d)" readonly mount_dir="$(mktemp -d)"
readonly max_tries=20 readonly max_tries=20
readonly increment=100 readonly increment=10
for i in $(seq 1 $max_tries); do for i in $(seq 1 $max_tries); do
local img_size="$((rootfs_size_mb + (i * increment)))" local img_size="$((rootfs_size_mb + (i * increment)))"

2
tools/packaging/scripts/configure-hypervisor.sh
View File

@ -14,8 +14,6 @@
# been specified. # been specified.
#--------------------------------------------------------------------- #---------------------------------------------------------------------
set -x
script_name=${0##*/} script_name=${0##*/}
arch="${3:-$(uname -m)}" arch="${3:-$(uname -m)}"