From c5184641dccf3be39aee4f8826264d069959d53d Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Fri, 6 Mar 2020 20:54:18 +0000 Subject: [PATCH 1/4] clh: Add vfio support Add support to hotplug vfio devices. Use hypervisor API to attach devices via hotplug. Fixes: #2496 Signed-off-by: Jose Carlos Venegas Munoz --- virtcontainers/clh.go | 38 ++++++++++++++++++++++++--- virtcontainers/clh_test.go | 5 ++++ virtcontainers/device/drivers/vfio.go | 2 ++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go index af8ec3f75b..d7424544e7 100644 --- a/virtcontainers/clh.go +++ b/virtcontainers/clh.go @@ -50,8 +50,11 @@ const ( const ( // Values are mandatory by http API // Values based on: - clhTimeout = 10 - clhAPITimeout = 1 + clhTimeout = 10 + clhAPITimeout = 1 + // Timeout for hot-plug - hotplug devices can take more time, than usual API calls + // Use longer time timeout for it. + clhHotPlugAPITimeout = 5 clhStopSandboxTimeout = 3 clhSocket = "clh.sock" clhAPISocket = "clh-api.sock" @@ -76,6 +79,7 @@ type clhClient interface { VmInfoGet(ctx context.Context) (chclient.VmInfo, *http.Response, error) //nolint:golint BootVM(ctx context.Context) (*http.Response, error) VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error) + VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) } type CloudHypervisorVersion struct { @@ -372,9 +376,35 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) { return vcpuInfo, nil } +func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { + cl := clh.client() + ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) + defer cancel() + + _, _, err := cl.VmmPingGet(ctx) + if err != nil { + return openAPIClientError(err) + } + + _, err = cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) + if err != nil { + err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) + } + return err +} + func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { - clh.Logger().WithField("function", "hotplugAddDevice").Warn("hotplug add device not supported") - return nil, nil + span, _ := clh.trace("hotplugAddDevice") + defer span.Finish() + + switch devType { + case vfioDev: + device := devInfo.(*config.VFIODev) + return nil, clh.hotPlugVFIODevice(*device) + default: + return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) + } + } func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { diff --git a/virtcontainers/clh_test.go b/virtcontainers/clh_test.go index be000bd3ee..7668e1c7b6 100644 --- a/virtcontainers/clh_test.go +++ b/virtcontainers/clh_test.go @@ -86,6 +86,11 @@ func (c *clhClientMock) VmResizePut(ctx context.Context, vmResize chclient.VmRes return nil, nil } +//nolint:golint +func (c *clhClientMock) VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) { + return nil, nil +} + func TestCloudHypervisorAddVSock(t *testing.T) { assert := assert.New(t) clh := cloudHypervisor{} diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go index c8f8f75910..7bed197aad 100644 --- a/virtcontainers/device/drivers/vfio.go +++ b/virtcontainers/device/drivers/vfio.go @@ -194,6 +194,8 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS case config.VFIODeviceNormalType: // Get bdf of device eg. 0000:00:1c.0 deviceBDF = getBDF(deviceFileName) + // Get sysfs path used by cloud-hypervisor + deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName) case config.VFIODeviceMediatedType: // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) From 3251beaa239f6ad5cac366348f2ab4f2bba68197 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Fri, 6 Mar 2020 20:56:34 +0000 Subject: [PATCH 2/4] version: Update clh to master Move to master tip to get support for vfio hotplug. Changes: df79499 net: Do not check multiqueue for new interface 7d75b1f build(deps): bump quote from 1.0.2 to 1.0.3 841bf89 build(deps): bump failure from 0.1.6 to 0.1.7 86acdb9 build(deps): bump failure_derive from 0.1.6 to 0.1.7 4b32863 docs: Update api.md for VFIO hotplug e518098 scripts: Make integration tests fail if some important commands fail be6f91d tests: Refactoring vhost_user_net test cases 6341736 vhost_user_net: Provide tap option for vhost_user_net backend e0419e9 build: Don't cancel older master builds f0a3e7c build: Bump linux-loader and vm-memory dependencies 6539d4a vfio: handle case for missing iommu_group cfbebd8 build(deps): bump micro_http from `88011bd` to `02def92` 4214806 tests: Remove further use of sudo subshells 2baf5ab tests: Simplfy the shm region check 97affbe tests: Re-enable the virtio-fs tests and make them work with virtio-mmio 7b1d5c1 tests: Remove entropy check from vhost-user-block test a4cca5f tests: sha1sums --check can take a list of hashes 689415e build(deps): bump libssh2-sys from 0.2.15 to 0.2.16 09829c4 vmm: Remove IO bus strong reference from Vm 2dbb376 vmm: Remove all Weak references from DeviceManager 9e915a0 vmm: Remove all Weak references from CpuManager 49268bf pci: Remove all Weak references from PciBus ca426cf devices: Make Bus hold a list of Weak BusDevice references 7773812 vmm: Store the list of BusDevice devices from DeviceManager d0820cc vmm: Make add_vfio_device mutable 948f808 vm: Rename DeviceManager field in Vm structure aa638ea build(deps): bump backtrace from 0.3.44 to 0.3.45 1152b1a ci: Add VFIO hotplug integration test d47f733 vmm: Break the cyclic dependency between DeviceManager and IO bus c1af13e vmm: Update VmConfig when adding new device a86f436 vmm: Add VFIO PCI device hotplug support 320fea0 vmm: Factorize VFIO PCI device creation 00716f9 vmm: Store virtio-iommu device from DeviceManager 5902dfa vmm: Store VFIO KVM device from DeviceManager d9c1b43 vmm: Store MSI InterruptManager from DeviceManager 02adc40 vmm: Store PciBus from DeviceManager 3f396d8 resources: Enable ACPI PCI hotplug in the kernel config d0218e9 vmm: Trigger hotplug notification to the guest 0e58741 vmm: api: Introduce new "add-device" HTTP endpoint 0f1396a vmm: Insert PCI device hotplug operation region on IO bus 65774e8 vmm: Implement BusDevice for DeviceManager 2eb26d4 devices: acpi: Update GED to support PCI devices hotplug 8dbc843 vmm: acpi: Add PCNT method to invoke DVNT c62db97 vmm: acpi: Add _EJ0 to each PCI device slot 4dc2a39 vmm: acpi: Create PHPR container c3a0685 vmm: acpi: Add notification method for PCI device slots 5a68d5b vmm: acpi: Create PCI device slots ead86bb build(deps): bump micro_http from `9945928` to `88011bd` 22dd49d tests: Test virtio-fs with virtio-mmio 642b890 vm-virtio: mmio: Enable reporting of SHM regions via config fields 0223cf8 ci: Update ClearLinux image ed396b4 build(deps): bump vm-memory from `2099f41` to `a84a7b8` 81c2294 vhost_rs: remove unused crate 5200bf3 Cargo: switch vhost_rs to external crate 65a38e6 vm-virtio: vhost_user: Fix blk device configuration space offset value d6e6901 vmm/api: Fix vm.info response definition 8f37200 build(deps): bump micro_http from `3eb926c` to `9945928` cc2d03d build(deps): bump regex-syntax from 0.6.15 to 0.6.16 f5b37e3 build(deps): bump regex-syntax from 0.6.14 to 0.6.15 009f4d2 build(deps): bump micro_http from `8d48e73` to `3eb926c` 5ade9d4 tests: Remove unnecessary sleeps and kill on clean shutdown tests c98949b tests: Wait for VMM to exit in test_serial_file/test_console_file 2f58fb8 tests: Test rebooting works for block self spawn test e817aa6 tests: Improve VM shutdown behaviour 559b70c tests: Make output capture optional dae7608 tests: Remove duplicated network configuration 6466ad2 tests: Remove duplicated disk configuration 9f1ac24 tests: Make the GuestCommand take a reference to the guest 49e70c6 tests: Port integration tests over to GuestCommand 67a5882 tests: Introduce new GuestCommand to handle launching the guest 8142c82 vmm: Move DeviceManager into an Arc> 531f4ff vhost_user_fs: Remove an unneeded unwrap in handle_event e52129e vhost_user_fs: Process events from HIPRIO queue 0c5c470 build(deps): bump micro_http from `b85757e` to `8d48e73` 5b96dd5 ci: Don't give special capabilities to Rust vhost-user-fs backend d8d790b vhost_rs: Don't check for SLAVE_SEND_FD on SET_SLAVE_REQ_FD 1c5562b vhost_user_fs: Add support for EVENT_IDX eae4f1d vhost_user_fs: Add support for indirect descriptors ea0bc24 vhost_user_fs: Be honest about protocol supported features 42937c9 vm-virtio: Add support for indirect descriptors d7b0b98 tests: Move integration tests to their own directory 3cb4513 vhost_rs: control SlaveFsCacheReq with vhost-user-slave feature 9de3ace devices: implement Aml trait for GED device b77fdeb msi/msi-x: Prevent from losing masked interrupts 8423c08 build(deps): bump proc-macro2 from 1.0.8 to 1.0.9 6315f16 build(deps): bump syn from 1.0.15 to 1.0.16 4cf89d3 pci: handle extended configuration space properly f6b9445 pci: fix pci MMCONFIG address parsing 77ee331 resources: Enable KASLR in kernel config bba5ef3 vmm: Remove deprecated CPU syntax 374ac77 main, vmm: Remove deprecated --vhost-user-net ffd816e main, vmm: Remove deprecated --vhost-user-blk d04e0dc build(deps): bump crossbeam-utils from 0.7.0 to 0.7.2 7da5b53 build(deps): bump ssh2 from 0.7.1 to 0.8.0 109c7f7 build(deps): bump hermit-abi from 0.1.7 to 0.1.8 812a6b9 build(deps): bump syn from 1.0.14 to 1.0.15 ad30791 build(deps): bump memchr from 2.3.2 to 2.3.3 94f2fc3 release-notes: Update for v0.5.1 bug fix release f190cb0 build(deps): bump libc from 0.2.66 to 0.2.67 299eb28 build(deps): bump micro_http from `6fd1545` to `b85757e` d2f1749 vmm: config: Add poll_queue property to DiskConfig 378dd81 vmm: openapi: Add missing "direct" knob to DiskConfig 056f548 vmm: openapi: Fix "readonly" and "wce" defaults in DiskConfig 4ebf01b vhost_user_backend: Don't report out socket broken errors b5755e9 vhost_rs: vhost_user: Return error when connection broken c49e31a vmm: api: Return a resize error when resize fails ebc6391 vmm: api: Fix resize command typos 9de7553 vmm: openapi: Update DiskConfig ed1e781 vmm: Workaround double reboot triggered by the kernel 5c06b7f vhost_user_block: Implement optional static polling 0e4e27e vhost_user_block: Make use of the EVENT_IDX feature 1ef6996 vhost_user_backend: Add helpers for EVENT_IDX d17fa78 vm-virtio: Implement support for EVENT_IDX 793d4e7 vmm: Move codebase to GuestMemoryAtomic from vm-memory ddf6caf ci: Improve test_memory_mergeable_on stability af621be build(deps): bump micro_http from `57ac9df` to `6fd1545` 4970e2f vhost-user-fs: add dax tests for vhost_user_fs rust daemon 59958f0 vhost_user_fs: add the ability to set slave req fd 3f09eff vhost_user_fs: add fs cache request operations 956a84f vhost_user_fs: add necessary structs for map/unmap requests 269d660 vhost_user_fs: add SlaveFsCacheReq to handle map/unmap be78c6d vhost_rs: Fix unit test race condition f7378bc tests: Add self spawning vhost-user-block test 1f6cbad vmm: Add support for spawning vhost-user-block backend 4d60ef5 vm-virtio: vhost_user: block: On shutdown() drop the socket 7fabca3 ci: Don't run unit tests in a privileged container 2724716 build(deps): bump micro_http from `4827569` to `57ac9df` 08a68f2 build: Run unit tests on worker node f21cd31 scripts: dev_cli: Add more privileges for the integration tests a94887e build: Use dev container for integration tests 3edc2bd vmm: Prevent memory overcommitment through virtio-fs shared regions 968c90a build(deps): bump hermit-abi from 0.1.6 to 0.1.7 7485a0c Revert "build: Don't fail build on test_vfio failure" cbc0ac3 build(deps): bump micro_http from `7a23e54` to `4827569` 7fdb5ae build(deps): bump vm-memory from `eb2fc0b` to `f615b19` 0d748c5 build(deps): bump scopeguard from 1.0.0 to 1.1.0 6692fa6 build(deps): bump thiserror from 1.0.10 to 1.0.11 f03602a tests: Add self spawning vhost-user-net test bc75c1b vmm: Add support for spawning vhost-user-net backend d054ddd vm-virtio: Retry connections to vhost-user backends b04eb47 vmm: Follow the "exe" symlink from the PID directory in /proc 5038878 vm-virtio: vhost_user: net: On shutdown() drop the socket 7c9e8b1 vmm: device_manager: Shutdown all virtio devices 545ea9e vm-virtio: Add shutdown method to VirtioDevice trait ebd8369 main: Display git commit hash with the '--version' option bdb92f9 build(deps): bump micro_http from `7fb2e46` to `7a23e54` 2061f0d tests: Always create shared VFIO directory from scratch e8e4f43 tests: Use hugepages for test_vfio 296ada9 scripts: dev_cli: Fix post build permissions for the whole tree 287897d tests: Run test_vfio with PCI binary 1661444 build(deps): bump serde_json from 1.0.47 to 1.0.48 96479da build(deps): bump vm-memory from `f3d1c27` to `eb2fc0b` 88c1683 build(deps): bump memchr from 2.3.1 to 2.3.2 8d3e4f9 build(deps): bump micro_http from `c9e900c` to `7fb2e46` 53481aa docs: Update documentation related to multiqueue network 4dd16c2 vm-virtio: Detect if a tap interface supports multiqueue 8627656 net_util: Provide more accurate error messages 6e5338d build(deps): bump memchr from 2.3.0 to 2.3.1 014844d build: Don't fail build on test_vfio failure 779cbfe build(deps): bump backtrace from 0.3.43 to 0.3.44 700df9e vhost_user_net: Port to new exit event strategy c33c38b vhost_user_block: Port to new exit event strategy da7f31d bin: vhost_user_fs: Port to new exit event strategy 759a0be vhost_user_backend: Add support for handling exiting of worker thread b17bafb build(deps): bump micro_http from `1de6f32` to `c9e900c` 7ca691f vhost_user_block: Implement and use worker shutdown e619fe6 vhost_user_net: Remove "Clone" implementation 613f254 vhost_user_backend: Wait on the worker thread 97ab767 vhost_user_net: Shutdown worker thread on exit 7f032c8 bin: vhost_user_fs: Shutdown worker thread on exit 99cb8dc bin: vhost_user_fs use error! macro logging for consistency 710394b vhost_user_block: Forward the error from unexpected event 4f4c3d3 vhost_user_block: Make Error behave like net and fs versions f1e19d6 vhost_user_backend: Forward the error from main thread 80c9dc2 Revert "vhost-user-backend: Correct error handling in run" c706ca1 scripts: dev_cli: Simplify the build command exit path 0a1d6e1 scripts: dev_cli: Fix build directory permisions c8fa809 scripts: dev_cli: Run unprivileged containers as the host user 26d8cae build(deps): bump micro_http from `ae15e75` to `1de6f32` 572aaa7 build(deps): bump serde_json from 1.0.46 to 1.0.47 04cb35e scripts: Make dev_cli.sh exit on test error 9bf100c build: Run worker and master build in parallel bfbca59 scripts: Don't use interactive & terminal mode for docker 6e6eb5b build: Do cargo tests, unit tests and OpenAPI check on master a5b053f scripts: dev_cli: Use a tmpfs mount for /tmp Signed-off-by: Jose Carlos Venegas Munoz --- versions.yaml | 2 +- .../pkg/cloud-hypervisor/client/README.md | 4 +- .../cloud-hypervisor/client/api/openapi.yaml | 128 ++++++------------ .../cloud-hypervisor/client/api_default.go | 66 +++++++++ .../client/docs/DefaultApi.md | 33 +++++ .../client/docs/DiskConfig.md | 2 + .../client/docs/VhostUserBlkConfig.md | 14 -- .../{VhostUserNetConfig.md => VmAddDevice.md} | 7 +- .../cloud-hypervisor/client/docs/VmConfig.md | 2 - .../client/model_disk_config.go | 2 + .../client/model_vhost_user_blk_config.go | 17 --- .../client/model_vhost_user_net_config.go | 17 --- .../client/model_vm_add_device.go | 14 ++ .../client/model_vm_config.go | 2 - .../cloud-hypervisor/cloud-hypervisor.yaml | 73 ++++------ 15 files changed, 193 insertions(+), 190 deletions(-) delete mode 100644 virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserBlkConfig.md rename virtcontainers/pkg/cloud-hypervisor/client/docs/{VhostUserNetConfig.md => VmAddDevice.md} (57%) delete mode 100644 virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_blk_config.go delete mode 100644 virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_net_config.go create mode 100644 virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go diff --git a/versions.yaml b/versions.yaml index 671263a2d8..d5f02d608e 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "stable/v0.5.x" + version: "df794993f8abe20f829275c77fb2a52ed485f70a" firecracker: description: "Firecracker micro-VMM" diff --git a/virtcontainers/pkg/cloud-hypervisor/client/README.md b/virtcontainers/pkg/cloud-hypervisor/client/README.md index b3affe8b71..8872c9ecdb 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/README.md +++ b/virtcontainers/pkg/cloud-hypervisor/client/README.md @@ -40,6 +40,7 @@ Class | Method | HTTP request | Description *DefaultApi* | [**ResumeVM**](docs/DefaultApi.md#resumevm) | **Put** /vm.resume | Resume a previously paused VM instance. *DefaultApi* | [**ShutdownVM**](docs/DefaultApi.md#shutdownvm) | **Put** /vm.shutdown | Shut the VM instance down. *DefaultApi* | [**ShutdownVMM**](docs/DefaultApi.md#shutdownvmm) | **Put** /vmm.shutdown | Shuts the cloud-hypervisor VMM. +*DefaultApi* | [**VmAddDevicePut**](docs/DefaultApi.md#vmadddeviceput) | **Put** /vm.add-device | Add a new device to the VM *DefaultApi* | [**VmInfoGet**](docs/DefaultApi.md#vminfoget) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance. *DefaultApi* | [**VmResizePut**](docs/DefaultApi.md#vmresizeput) | **Put** /vm.resize | Resize the VM *DefaultApi* | [**VmmPingGet**](docs/DefaultApi.md#vmmpingget) | **Get** /vmm.ping | Ping the VMM to check for API server availability @@ -58,8 +59,7 @@ Class | Method | HTTP request | Description - [NetConfig](docs/NetConfig.md) - [PmemConfig](docs/PmemConfig.md) - [RngConfig](docs/RngConfig.md) - - [VhostUserBlkConfig](docs/VhostUserBlkConfig.md) - - [VhostUserNetConfig](docs/VhostUserNetConfig.md) + - [VmAddDevice](docs/VmAddDevice.md) - [VmConfig](docs/VmConfig.md) - [VmInfo](docs/VmInfo.md) - [VmResize](docs/VmResize.md) diff --git a/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index edb23b1f3d..d9da94bc9c 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -128,6 +128,21 @@ paths: "404": description: The VM instance could not be resized because it is not created. summary: Resize the VM + /vm.add-device: + put: + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/VmAddDevice' + description: The path of the new device + required: true + responses: + "204": + description: The new device was successfully added to the VM instance. + "404": + description: The new device could not be added to the VM instance. + summary: Add a new device to the VM components: schemas: VmmPingResponse: @@ -161,7 +176,9 @@ components: queue_size: 5 vhost_socket: vhost_socket vhost_user: false + direct: false wce: true + poll_queue: true - path: path num_queues: 5 readonly: false @@ -169,7 +186,9 @@ components: queue_size: 5 vhost_socket: vhost_socket vhost_user: false + direct: false wce: true + poll_queue: true cpus: boot_vcpus: 1 max_vcpus: 1 @@ -180,15 +199,6 @@ components: iommu: false kernel: path: path - vhost_user_blk: - - sock: sock - num_queues: 1 - queue_size: 1 - wce: true - - sock: sock - num_queues: 1 - queue_size: 1 - wce: true rng: iommu: false src: /dev/urandom @@ -205,15 +215,6 @@ components: cache_size: 2 dax: true tag: tag - vhost_user_net: - - sock: sock - num_queues: 7 - queue_size: 1 - mac: mac - - sock: sock - num_queues: 7 - queue_size: 1 - mac: mac vsock: - sock: sock iommu: false @@ -262,8 +263,9 @@ components: state: enum: - Created - - Booted + - Running - Shutdown + - Paused type: string required: - config @@ -288,7 +290,9 @@ components: queue_size: 5 vhost_socket: vhost_socket vhost_user: false + direct: false wce: true + poll_queue: true - path: path num_queues: 5 readonly: false @@ -296,7 +300,9 @@ components: queue_size: 5 vhost_socket: vhost_socket vhost_user: false + direct: false wce: true + poll_queue: true cpus: boot_vcpus: 1 max_vcpus: 1 @@ -307,15 +313,6 @@ components: iommu: false kernel: path: path - vhost_user_blk: - - sock: sock - num_queues: 1 - queue_size: 1 - wce: true - - sock: sock - num_queues: 1 - queue_size: 1 - wce: true rng: iommu: false src: /dev/urandom @@ -332,15 +329,6 @@ components: cache_size: 2 dax: true tag: tag - vhost_user_net: - - sock: sock - num_queues: 7 - queue_size: 1 - mac: mac - - sock: sock - num_queues: 7 - queue_size: 1 - mac: mac vsock: - sock: sock iommu: false @@ -418,14 +406,6 @@ components: items: $ref: '#/components/schemas/DeviceConfig' type: array - vhost_user_net: - items: - $ref: '#/components/schemas/VhostUserNetConfig' - type: array - vhost_user_blk: - items: - $ref: '#/components/schemas/VhostUserBlkConfig' - type: array vsock: items: $ref: '#/components/schemas/VsockConfig' @@ -498,13 +478,18 @@ components: queue_size: 5 vhost_socket: vhost_socket vhost_user: false + direct: false wce: true + poll_queue: true properties: path: type: string readonly: default: false type: boolean + direct: + default: false + type: boolean iommu: default: false type: boolean @@ -522,6 +507,9 @@ components: wce: default: true type: boolean + poll_queue: + default: true + type: boolean required: - path type: object @@ -641,7 +629,7 @@ components: - "false" - Tty - File - - None + - null type: string iommu: default: false @@ -662,47 +650,6 @@ components: required: - path type: object - VhostUserNetConfig: - example: - sock: sock - num_queues: 7 - queue_size: 1 - mac: mac - properties: - sock: - type: string - num_queues: - default: 2 - type: integer - queue_size: - default: 256 - type: integer - mac: - type: string - required: - - sock - type: object - VhostUserBlkConfig: - example: - sock: sock - num_queues: 1 - queue_size: 1 - wce: true - properties: - sock: - type: string - num_queues: - default: 1 - type: integer - queue_size: - default: 128 - type: integer - wce: - default: true - type: boolean - required: - - sock - type: object VsockConfig: example: sock: sock @@ -735,3 +682,10 @@ components: desired_ram: type: integer type: object + VmAddDevice: + example: + path: path + properties: + path: + type: string + type: object diff --git a/virtcontainers/pkg/cloud-hypervisor/client/api_default.go b/virtcontainers/pkg/cloud-hypervisor/client/api_default.go index 53e0c516be..c73202bf7e 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/api_default.go +++ b/virtcontainers/pkg/cloud-hypervisor/client/api_default.go @@ -531,6 +531,72 @@ func (a *DefaultApiService) ShutdownVMM(ctx _context.Context) (*_nethttp.Respons return localVarHTTPResponse, nil } +/* +VmAddDevicePut Add a new device to the VM + * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + * @param vmAddDevice The path of the new device +*/ +func (a *DefaultApiService) VmAddDevicePut(ctx _context.Context, vmAddDevice VmAddDevice) (*_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodPut + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + ) + + // create path and map variables + localVarPath := a.client.cfg.BasePath + "/vm.add-device" + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{"application/json"} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + // body params + localVarPostBody = &vmAddDevice + r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return nil, err + } + + localVarHTTPResponse, err := a.client.callAPI(r) + if err != nil || localVarHTTPResponse == nil { + return localVarHTTPResponse, err + } + + localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + if err != nil { + return localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + return localVarHTTPResponse, newErr + } + + return localVarHTTPResponse, nil +} + /* VmInfoGet Returns general information about the cloud-hypervisor Virtual Machine (VM) instance. * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). diff --git a/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md b/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md index 535d9c69d2..74c9e3a3f1 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md +++ b/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md @@ -12,6 +12,7 @@ Method | HTTP request | Description [**ResumeVM**](DefaultApi.md#ResumeVM) | **Put** /vm.resume | Resume a previously paused VM instance. [**ShutdownVM**](DefaultApi.md#ShutdownVM) | **Put** /vm.shutdown | Shut the VM instance down. [**ShutdownVMM**](DefaultApi.md#ShutdownVMM) | **Put** /vmm.shutdown | Shuts the cloud-hypervisor VMM. +[**VmAddDevicePut**](DefaultApi.md#VmAddDevicePut) | **Put** /vm.add-device | Add a new device to the VM [**VmInfoGet**](DefaultApi.md#VmInfoGet) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance. [**VmResizePut**](DefaultApi.md#VmResizePut) | **Put** /vm.resize | Resize the VM [**VmmPingGet**](DefaultApi.md#VmmPingGet) | **Get** /vmm.ping | Ping the VMM to check for API server availability @@ -246,6 +247,38 @@ No authorization required [[Back to README]](../README.md) +## VmAddDevicePut + +> VmAddDevicePut(ctx, vmAddDevice) + +Add a new device to the VM + +### Required Parameters + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- +**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc. +**vmAddDevice** | [**VmAddDevice**](VmAddDevice.md)| The path of the new device | + +### Return type + + (empty response body) + +### Authorization + +No authorization required + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: Not defined + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + + ## VmInfoGet > VmInfo VmInfoGet(ctx, ) diff --git a/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md b/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md index 9345f0bfc8..8df1b43153 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md +++ b/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md @@ -6,12 +6,14 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Path** | **string** | | **Readonly** | **bool** | | [optional] [default to false] +**Direct** | **bool** | | [optional] [default to false] **Iommu** | **bool** | | [optional] [default to false] **NumQueues** | **int32** | | [optional] [default to 1] **QueueSize** | **int32** | | [optional] [default to 128] **VhostUser** | **bool** | | [optional] [default to false] **VhostSocket** | **string** | | [optional] **Wce** | **bool** | | [optional] [default to true] +**PollQueue** | **bool** | | [optional] [default to true] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserBlkConfig.md b/virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserBlkConfig.md deleted file mode 100644 index 6b36c78b90..0000000000 --- a/virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserBlkConfig.md +++ /dev/null @@ -1,14 +0,0 @@ -# VhostUserBlkConfig - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**Sock** | **string** | | -**NumQueues** | **int32** | | [optional] [default to 1] -**QueueSize** | **int32** | | [optional] [default to 128] -**Wce** | **bool** | | [optional] [default to true] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserNetConfig.md b/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md similarity index 57% rename from virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserNetConfig.md rename to virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md index 0a11429d80..e281db1380 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/docs/VhostUserNetConfig.md +++ b/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md @@ -1,13 +1,10 @@ -# VhostUserNetConfig +# VmAddDevice ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**Sock** | **string** | | -**NumQueues** | **int32** | | [optional] [default to 2] -**QueueSize** | **int32** | | [optional] [default to 256] -**Mac** | **string** | | [optional] +**Path** | **string** | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md b/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md index 1c5eb6672d..ef62495a7b 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md +++ b/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md @@ -16,8 +16,6 @@ Name | Type | Description | Notes **Serial** | [**ConsoleConfig**](ConsoleConfig.md) | | [optional] **Console** | [**ConsoleConfig**](ConsoleConfig.md) | | [optional] **Devices** | [**[]DeviceConfig**](DeviceConfig.md) | | [optional] -**VhostUserNet** | [**[]VhostUserNetConfig**](VhostUserNetConfig.md) | | [optional] -**VhostUserBlk** | [**[]VhostUserBlkConfig**](VhostUserBlkConfig.md) | | [optional] **Vsock** | [**[]VsockConfig**](VsockConfig.md) | | [optional] **Iommu** | **bool** | | [optional] [default to false] diff --git a/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go b/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go index 79e56828a1..a88faba7b1 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go +++ b/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go @@ -12,10 +12,12 @@ package openapi type DiskConfig struct { Path string `json:"path"` Readonly bool `json:"readonly,omitempty"` + Direct bool `json:"direct,omitempty"` Iommu bool `json:"iommu,omitempty"` NumQueues int32 `json:"num_queues,omitempty"` QueueSize int32 `json:"queue_size,omitempty"` VhostUser bool `json:"vhost_user,omitempty"` VhostSocket string `json:"vhost_socket,omitempty"` Wce bool `json:"wce,omitempty"` + PollQueue bool `json:"poll_queue,omitempty"` } diff --git a/virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_blk_config.go b/virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_blk_config.go deleted file mode 100644 index 62e3b309cf..0000000000 --- a/virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_blk_config.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Cloud Hypervisor API - * - * Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine. - * - * API version: 0.3.0 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package openapi -// VhostUserBlkConfig struct for VhostUserBlkConfig -type VhostUserBlkConfig struct { - Sock string `json:"sock"` - NumQueues int32 `json:"num_queues,omitempty"` - QueueSize int32 `json:"queue_size,omitempty"` - Wce bool `json:"wce,omitempty"` -} diff --git a/virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_net_config.go b/virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_net_config.go deleted file mode 100644 index 3f1979c6fe..0000000000 --- a/virtcontainers/pkg/cloud-hypervisor/client/model_vhost_user_net_config.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Cloud Hypervisor API - * - * Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine. - * - * API version: 0.3.0 - * Generated by: OpenAPI Generator (https://openapi-generator.tech) - */ - -package openapi -// VhostUserNetConfig struct for VhostUserNetConfig -type VhostUserNetConfig struct { - Sock string `json:"sock"` - NumQueues int32 `json:"num_queues,omitempty"` - QueueSize int32 `json:"queue_size,omitempty"` - Mac string `json:"mac,omitempty"` -} diff --git a/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go b/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go new file mode 100644 index 0000000000..ee96b9b9cd --- /dev/null +++ b/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go @@ -0,0 +1,14 @@ +/* + * Cloud Hypervisor API + * + * Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine. + * + * API version: 0.3.0 + * Generated by: OpenAPI Generator (https://openapi-generator.tech) + */ + +package openapi +// VmAddDevice struct for VmAddDevice +type VmAddDevice struct { + Path string `json:"path,omitempty"` +} diff --git a/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go b/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go index 985ff73666..0c48392538 100644 --- a/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go +++ b/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go @@ -22,8 +22,6 @@ type VmConfig struct { Serial ConsoleConfig `json:"serial,omitempty"` Console ConsoleConfig `json:"console,omitempty"` Devices []DeviceConfig `json:"devices,omitempty"` - VhostUserNet []VhostUserNetConfig `json:"vhost_user_net,omitempty"` - VhostUserBlk []VhostUserBlkConfig `json:"vhost_user_blk,omitempty"` Vsock []VsockConfig `json:"vsock,omitempty"` Iommu bool `json:"iommu,omitempty"` } diff --git a/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 926e710a2c..fdbc088671 100644 --- a/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -139,6 +139,22 @@ paths: 404: description: The VM instance could not be resized because it is not created. + /vm.add-device: + put: + summary: Add a new device to the VM + requestBody: + description: The path of the new device + content: + application/json: + schema: + $ref: '#/components/schemas/VmAddDevice' + required: true + responses: + 204: + description: The new device was successfully added to the VM instance. + 404: + description: The new device could not be added to the VM instance. + components: schemas: @@ -161,7 +177,7 @@ components: $ref: '#/components/schemas/VmConfig' state: type: string - enum: [Created, Booted, Shutdown] + enum: [Created, Running, Shutdown, Paused] description: Virtual Machine information VmConfig: @@ -204,14 +220,6 @@ components: type: array items: $ref: '#/components/schemas/DeviceConfig' - vhost_user_net: - type: array - items: - $ref: '#/components/schemas/VhostUserNetConfig' - vhost_user_blk: - type: array - items: - $ref: '#/components/schemas/VhostUserBlkConfig' vsock: type: array items: @@ -277,6 +285,9 @@ components: readonly: type: boolean default: false + direct: + type: boolean + default: false iommu: type: boolean default: false @@ -294,6 +305,9 @@ components: wce: type: boolean default: true + poll_queue: + type: boolean + default: true NetConfig: type: object @@ -387,7 +401,7 @@ components: type: string mode: type: string - enum: [Off, Tty, File, None] + enum: [Off, Tty, File, Null] iommu: type: boolean default: false @@ -403,39 +417,6 @@ components: type: boolean default: false - VhostUserNetConfig: - required: - - sock - type: object - properties: - sock: - type: string - num_queues: - type: integer - default: 2 - queue_size: - type: integer - default: 256 - mac: - type: string - - VhostUserBlkConfig: - required: - - sock - type: object - properties: - sock: - type: string - num_queues: - type: integer - default: 1 - queue_size: - type: integer - default: 128 - wce: - type: boolean - default: true - VsockConfig: required: - cid @@ -462,3 +443,9 @@ components: type: integer desired_ram: type: integer + + VmAddDevice: + type: object + properties: + path: + type: string From 6a4e667f9c3d5d5f9c614204f68f406877afee64 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Sat, 7 Mar 2020 00:02:26 +0000 Subject: [PATCH 3/4] virtiofsd: Check if PID is valid If try to kill with an not valid PID the thread goes to panic, check to allow return a valid error from the runtime. Signed-off-by: Jose Carlos Venegas Munoz --- virtcontainers/virtiofsd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/virtcontainers/virtiofsd.go b/virtcontainers/virtiofsd.go index c004a51674..9b87c6bf9a 100644 --- a/virtcontainers/virtiofsd.go +++ b/virtcontainers/virtiofsd.go @@ -217,6 +217,10 @@ func (v *virtiofsd) kill() (err error) { span, _ := v.trace("kill") defer span.Finish() + if v.PID == 0 { + return errors.New("invalid virtiofsd PID(0)") + } + err = syscall.Kill(v.PID, syscall.SIGKILL) if err != nil { v.PID = 0 From f61eca892027811bf40b5a3f3725d6b3b236770d Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Thu, 19 Mar 2020 19:08:57 +0000 Subject: [PATCH 4/4] clh: Add comments around clh api To make easier to know what a method of API is expected to do without go to cloud-hypervisor documentation. Signed-off-by: Jose Carlos Venegas Munoz --- virtcontainers/clh.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go index d7424544e7..10a6048a0c 100644 --- a/virtcontainers/clh.go +++ b/virtcontainers/clh.go @@ -72,13 +72,20 @@ const ( // The main purpose is to hide the client in an interface to allow mock testing. // This is an interface that has to match with OpenAPI CLH client type clhClient interface { + // Check for the REST API availability VmmPingGet(ctx context.Context) (chclient.VmmPingResponse, *http.Response, error) + // Shut the VMM down ShutdownVMM(ctx context.Context) (*http.Response, error) + // Create the VM CreateVM(ctx context.Context, vmConfig chclient.VmConfig) (*http.Response, error) + // Dump the VM information // No lint: golint suggest to rename to VMInfoGet. VmInfoGet(ctx context.Context) (chclient.VmInfo, *http.Response, error) //nolint:golint + // Boot the VM BootVM(ctx context.Context) (*http.Response, error) + // Add/remove CPUs to/from the VM VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error) + // Add VFIO PCI device to the VM VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) }