From cc80ae0afb1bd1c67b4a7f86875c4b10d4c5ed4d Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 5 Oct 2020 12:13:36 -0700 Subject: [PATCH 1/3] versions: cloud-hypervisor: Bump to version 6d30fe05 The cloud-hypervisor commit `6d30fe05` introduced a fix on its API for VFIO device hotplug (`VmAddDevice`), which is required for supporting VFIO unplug through openAPI calls in kata. Signed-off-by: Bo Chen --- .../cloud-hypervisor/client/api/openapi.yaml | 19 +++++++++++++------ .../client/docs/MemoryConfig.md | 2 +- .../client/docs/VmAddDevice.md | 2 ++ .../client/model_memory_config.go | 2 +- .../client/model_vm_add_device.go | 2 ++ .../cloud-hypervisor/cloud-hypervisor.yaml | 17 +++++++++++------ versions.yaml | 2 +- 7 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index c9a06aeb9c..e775606d88 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -828,7 +828,7 @@ components: default: false type: boolean host_numa_node: - format: uint32 + format: int32 type: integer hotplug_size: format: int64 @@ -896,7 +896,7 @@ components: default: false type: boolean balloon_size: - format: uint64 + format: int64 type: integer zones: items: @@ -1172,10 +1172,10 @@ components: destination: 3 properties: destination: - format: uint32 + format: int32 type: integer distance: - format: uint8 + format: int32 type: integer required: - destination @@ -1197,11 +1197,11 @@ components: guest_numa_id: 9 properties: guest_numa_id: - format: uint32 + format: int32 type: integer cpus: items: - format: uint8 + format: int32 type: integer type: array distances: @@ -1248,9 +1248,16 @@ components: VmAddDevice: example: path: path + iommu: false + id: id properties: path: type: string + iommu: + default: false + type: boolean + id: + type: string type: object VmRemoveDevice: example: diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md index ccafc86a73..c8dee09c48 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md @@ -12,7 +12,7 @@ Name | Type | Description | Notes **Shared** | **bool** | | [optional] [default to false] **Hugepages** | **bool** | | [optional] [default to false] **Balloon** | **bool** | | [optional] [default to false] -**BalloonSize** | **int32** | | [optional] +**BalloonSize** | **int64** | | [optional] **Zones** | [**[]MemoryZoneConfig**](MemoryZoneConfig.md) | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md index e281db1380..124ac3b5d6 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md @@ -5,6 +5,8 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Path** | **string** | | [optional] +**Iommu** | **bool** | | [optional] [default to false] +**Id** | **string** | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go index b0ccbc0348..749080979d 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go @@ -18,6 +18,6 @@ type MemoryConfig struct { Shared bool `json:"shared,omitempty"` Hugepages bool `json:"hugepages,omitempty"` Balloon bool `json:"balloon,omitempty"` - BalloonSize int32 `json:"balloon_size,omitempty"` + BalloonSize int64 `json:"balloon_size,omitempty"` Zones []MemoryZoneConfig `json:"zones,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go index ee96b9b9cd..f644fa60a2 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go @@ -11,4 +11,6 @@ package openapi // VmAddDevice struct for VmAddDevice type VmAddDevice struct { Path string `json:"path,omitempty"` + Iommu bool `json:"iommu,omitempty"` + Id string `json:"id,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 88c4b64ffd..240e02e27b 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -492,7 +492,7 @@ components: default: false host_numa_node: type: integer - format: uint32 + format: int32 hotplug_size: type: integer format: int64 @@ -532,7 +532,7 @@ components: default: false balloon_size: type: integer - format: uint64 + format: int64 zones: type: array items: @@ -754,10 +754,10 @@ components: properties: destination: type: integer - format: uint32 + format: int32 distance: type: integer - format: uint8 + format: int32 NumaConfig: required: @@ -766,12 +766,12 @@ components: properties: guest_numa_id: type: integer - format: uint32 + format: int32 cpus: type: array items: type: integer - format: uint8 + format: int32 distances: type: array items: @@ -811,6 +811,11 @@ components: properties: path: type: string + iommu: + type: boolean + default: false + id: + type: string VmRemoveDevice: type: object diff --git a/versions.yaml b/versions.yaml index 94e592a88c..5be57d05e0 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "c54452c08a467a3e35d8d72f2a91d424e9718c57" + version: "6d30fe05e4febd930d91bb36294f0219faf2254c" firecracker: description: "Firecracker micro-VMM" From 1f4dfa31667d76ef94a4027bac0890c5e3e8bfe8 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 29 Sep 2020 11:03:09 -0700 Subject: [PATCH 2/3] clh: Remove unnecessary VmmPing We can rely on the error handling of the actual HTTP API calls to catch errors, and don't need to call VmmPing explicitly in advance. Signed-off-by: Bo Chen --- src/runtime/virtcontainers/clh.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 80e5783c4b..528f920c1c 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -429,15 +429,12 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro " using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock) } + var err error + cl := clh.client() ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - _, _, err := cl.VmmPingGet(ctx) - if err != nil { - return openAPIClientError(err) - } - driveID := clhDriveIndexToID(drive.Index) //Explicitly set PCIAddr to NULL, so that VirtPath can be used @@ -466,12 +463,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - _, _, err := cl.VmmPingGet(ctx) - if err != nil { - return openAPIClientError(err) - } - - _, _, err = cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) + _, _, err := cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) if err != nil { err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } From c33ee54a216b202eb77b39ab83f431123367ab27 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 29 Sep 2020 17:03:44 -0700 Subject: [PATCH 3/3] clh: Support VFIO device unplug This patch adds the support of VFIO device unplug when using cloud-hypervisor. Fixes: #860 Signed-off-by: Bo Chen --- src/runtime/virtcontainers/clh.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 528f920c1c..d61902e929 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -463,7 +463,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - _, _, err := cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) + _, _, err := cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev, Id: device.ID}) if err != nil { err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } @@ -507,6 +507,20 @@ func (clh *cloudHypervisor) hotplugRemoveBlockDevice(drive *config.BlockDrive) e return err } +func (clh *cloudHypervisor) hotplugRemoveVfioDevice(device *config.VFIODev) error { + cl := clh.client() + ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) + defer cancel() + + _, err := cl.VmRemoveDevicePut(ctx, chclient.VmRemoveDevice{Id: device.ID}) + + if err != nil { + err = fmt.Errorf("failed to hotplug remove vfio device %+v %s", device, openAPIClientError(err)) + } + + return err +} + func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { span, _ := clh.trace("hotplugRemoveDevice") defer span.Finish() @@ -514,6 +528,8 @@ func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType dev switch devType { case blockDev: return nil, clh.hotplugRemoveBlockDevice(devInfo.(*config.BlockDrive)) + case vfioDev: + return nil, clh.hotplugRemoveVfioDevice(devInfo.(*config.VFIODev)) default: clh.Logger().WithFields(log.Fields{"devInfo": devInfo, "deviceType": devType}).Error("hotplugRemoveDevice: unsupported device")