Merge pull request #2556 from jcvenegas/mem-hotplug-clh-v2

clh: Enable memory hotplug

versions.yaml

@@ -75,7 +75,7 @@ assets:
       url: "https://github.com/cloud-hypervisor/cloud-hypervisor"
       uscan-url: >-
         https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz
-      version: "df794993f8abe20f829275c77fb2a52ed485f70a"
+      version: "c1e6d0022b220ecba25a10f61d0025a408d90a94"
 
     firecracker:
       description: "Firecracker micro-VMM"

virtcontainers/clh.go

@@ -214,8 +214,16 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ
 	clh.Logger().WithField("function", "createSandbox").WithError(err).Info("Sandbox not found creating ")
 
 	// Set initial memomory size of the virtual machine
-	clh.vmconfig.Memory.Size = int64(clh.config.MemorySize) << utils.MibToBytesShift
+	// Convert to int64 openApiClient only support int64
+	clh.vmconfig.Memory.Size = int64((utils.MemUnit(clh.config.MemorySize) * utils.MiB).ToBytes())
 	clh.vmconfig.Memory.File = "/dev/shm"
+	hostMemKb, err := getHostMemorySizeKb(procMemInfo)
+	if err != nil {
+		return nil
+	}
+
+	// OpenAPI only supports int64 values
+	clh.vmconfig.Memory.HotplugSize = int64((utils.MemUnit(hostMemKb) * utils.KiB).ToBytes())
 	// Set initial amount of cpu's for the virtual machine
 	clh.vmconfig.Cpus = chclient.CpusConfig{
 		// cast to int32, as openAPI has a limitation that it does not support unsigned values
@@ -424,22 +432,79 @@ func (clh *cloudHypervisor) hypervisorConfig() HypervisorConfig {
 }
 
 func (clh *cloudHypervisor) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
-	clh.Logger().WithField("function", "resizeMemory").Warn("not supported")
+
-	return 0, memoryDevice{}, nil
+	// TODO: Add support for virtio-mem
+
+	if probe {
+		return 0, memoryDevice{}, errors.New("probe memory is not supported for cloud-hypervisor")
+	}
+
+	if reqMemMB == 0 {
+		// This is a corner case if requested to resize to 0 means something went really wrong.
+		return 0, memoryDevice{}, errors.New("Can not resize memory to 0")
+	}
+
+	info, err := clh.vmInfo()
+	if err != nil {
+		return 0, memoryDevice{}, err
+	}
+
+	currentMem := utils.MemUnit(info.Config.Memory.Size) * utils.Byte
+	newMem := utils.MemUnit(reqMemMB) * utils.MiB
+
+	// Early check to verify if boot memory is the same as requested
+	if currentMem == newMem {
+		clh.Logger().WithField("memory", reqMemMB).Debugf("VM already has requested memory")
+		return uint32(currentMem.ToMiB()), memoryDevice{}, nil
+	}
+
+	if currentMem > newMem {
+		clh.Logger().Warn("Remove memory is not supported, nothing to do")
+		return uint32(currentMem.ToMiB()), memoryDevice{}, nil
+	}
+
+	blockSize := utils.MemUnit(memoryBlockSizeMB) * utils.MiB
+	hotplugSize := (newMem - currentMem).AlignMem(blockSize)
+
+	// Update memory request to increase memory aligned block
+	alignedRequest := currentMem + hotplugSize
+	if newMem != alignedRequest {
+		clh.Logger().WithFields(log.Fields{"request": newMem, "aligned-request": alignedRequest}).Debug("aligning VM memory request")
+		newMem = alignedRequest
+	}
+
+	// Check if memory is the same as requested, a second check is done
+	// to consider the memory request now that is updated to be memory aligned
+	if currentMem == newMem {
+		clh.Logger().WithFields(log.Fields{"current-memory": currentMem, "new-memory": newMem}).Debug("VM already has requested memory(after alignment)")
+		return uint32(currentMem.ToMiB()), memoryDevice{}, nil
+	}
+
+	cl := clh.client()
+	ctx, cancelResize := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
+	defer cancelResize()
+
+	// OpenApi does not support uint64, convert to int64
+	resize := chclient.VmResize{DesiredRam: int64(newMem.ToBytes())}
+	clh.Logger().WithFields(log.Fields{"current-memory": currentMem, "new-memory": newMem}).Debug("updating VM memory")
+	if _, err = cl.VmResizePut(ctx, resize); err != nil {
+		clh.Logger().WithFields(log.Fields{"current-memory": currentMem, "new-memory": newMem}).Warnf("failed to update memory %s", openAPIClientError(err))
+		err = fmt.Errorf("Failed to resize memory from %d to %d: %s", currentMem, newMem, openAPIClientError(err))
+		return uint32(currentMem.ToMiB()), memoryDevice{}, openAPIClientError(err)
+	}
+
+	return uint32(newMem.ToMiB()), memoryDevice{sizeMB: int(hotplugSize.ToMiB())}, nil
 }
 
 func (clh *cloudHypervisor) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) {
 	cl := clh.client()
 
 	// Retrieve the number of current vCPUs via HTTP API
-	ctx, cancel := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
+	info, err := clh.vmInfo()
-	info, _, err := cl.VmInfoGet(ctx)
 	if err != nil {
-		clh.Logger().WithField("function", "resizeVCPUs").WithError(openAPIClientError(err)).Info("[clh] VmInfoGet failed")
+		clh.Logger().WithField("function", "resizeVCPUs").WithError(err).Info("[clh] vmInfo failed")
 		return 0, 0, openAPIClientError(err)
 	}
-	// Reset the timer after the first HTTP API call
-	cancel()
 
 	currentVCPUs = uint32(info.Config.Cpus.BootVcpus)
 	newVCPUs = currentVCPUs
@@ -461,7 +526,7 @@ func (clh *cloudHypervisor) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, n
 	}
 
 	// Resize (hot-plug) vCPUs via HTTP API
-	ctx, cancel = context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
 	defer cancel()
 	if _, err = cl.VmResizePut(ctx, chclient.VmResize{DesiredVcpus: int32(reqVCPUs)}); err != nil {
 		return currentVCPUs, newVCPUs, errors.Wrap(err, "[clh] VmResizePut failed")
@@ -960,10 +1025,10 @@ func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
 		return openAPIClientError(err)
 	}
 
-	info, _, err := cl.VmInfoGet(ctx)
+	info, err := clh.vmInfo()
 
 	if err != nil {
-		return openAPIClientError(err)
+		return err
 	}
 
 	clh.Logger().Debugf("VM state after create: %#v", info)
@@ -979,10 +1044,10 @@ func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
 		return openAPIClientError(err)
 	}
 
-	info, _, err = cl.VmInfoGet(ctx)
+	info, err = clh.vmInfo()
 
 	if err != nil {
-		return openAPIClientError(err)
+		return err
 	}
 
 	clh.Logger().Debugf("VM state after boot: %#v", info)
@@ -1120,3 +1185,17 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error {
 
 	return nil
 }
+
+// vmInfo ask to hypervisor for current VM status
+func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
+	cl := clh.client()
+	ctx, cancelInfo := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
+	defer cancelInfo()
+
+	info, _, err := cl.VmInfoGet(ctx)
+	if err != nil {
+		clh.Logger().WithError(openAPIClientError(err)).Warn("VmInfoGet failed")
+	}
+	return info, openAPIClientError(err)
+
+}

virtcontainers/clh_test.go

@@ -10,15 +10,22 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/persist"
 	chclient "github.com/kata-containers/runtime/virtcontainers/pkg/cloud-hypervisor/client"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	FAIL = true
+	PASS = !FAIL
+)
+
 func newClhConfig() (HypervisorConfig, error) {
 
 	setupClh()
@@ -255,3 +262,57 @@ func TestClooudHypervisorStartSandbox(t *testing.T) {
 	err = clh.startSandbox(10)
 	assert.NoError(err)
 }
+
+func TestCloudHypervisorResizeMemory(t *testing.T) {
+	assert := assert.New(t)
+	clhConfig, err := newClhConfig()
+	type args struct {
+		reqMemMB          uint32
+		memoryBlockSizeMB uint32
+	}
+	tests := []struct {
+		name           string
+		args           args
+		expectedMemDev memoryDevice
+		wantErr        bool
+	}{
+		{"Resize to zero", args{0, 128}, memoryDevice{probe: false, sizeMB: 0}, FAIL},
+		{"Resize to aligned size", args{clhConfig.MemorySize + 128, 128}, memoryDevice{probe: false, sizeMB: 128}, PASS},
+		{"Resize to aligned size", args{clhConfig.MemorySize + 129, 128}, memoryDevice{probe: false, sizeMB: 256}, PASS},
+		{"Resize to NOT aligned size", args{clhConfig.MemorySize + 125, 128}, memoryDevice{probe: false, sizeMB: 128}, PASS},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.NoError(err)
+			clh := cloudHypervisor{}
+
+			mockClient := &clhClientMock{}
+			mockClient.vmInfo.Config.Memory.Size = int64(utils.MemUnit(clhConfig.MemorySize) * utils.MiB)
+			mockClient.vmInfo.Config.Memory.HotplugSize = int64(40 * utils.GiB.ToBytes())
+
+			clh.APIClient = mockClient
+			clh.config = clhConfig
+
+			newMem, memDev, err := clh.resizeMemory(tt.args.reqMemMB, tt.args.memoryBlockSizeMB, false)
+
+			if (err != nil) != tt.wantErr {
+				t.Errorf("cloudHypervisor.resizeMemory() error = %v, expected to fail = %v", err, tt.wantErr)
+				return
+			}
+
+			if err != nil {
+				return
+			}
+
+			expectedMem := clhConfig.MemorySize + uint32(tt.expectedMemDev.sizeMB)
+
+			if newMem != expectedMem {
+				t.Errorf("cloudHypervisor.resizeMemory() got = %+v, want %+v", newMem, expectedMem)
+			}
+
+			if !reflect.DeepEqual(memDev, tt.expectedMemDev) {
+				t.Errorf("cloudHypervisor.resizeMemory() got = %+v, want %+v", memDev, tt.expectedMemDev)
+			}
+		})
+	}
+}

virtcontainers/pkg/cloud-hypervisor/client/README.md

@@ -42,6 +42,7 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**ShutdownVMM**](docs/DefaultApi.md#shutdownvmm) | **Put** /vmm.shutdown | Shuts the cloud-hypervisor VMM.
 *DefaultApi* | [**VmAddDevicePut**](docs/DefaultApi.md#vmadddeviceput) | **Put** /vm.add-device | Add a new device to the VM
 *DefaultApi* | [**VmInfoGet**](docs/DefaultApi.md#vminfoget) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance.
+*DefaultApi* | [**VmRemoveDevicePut**](docs/DefaultApi.md#vmremovedeviceput) | **Put** /vm.remove-device | Remove a device from the VM
 *DefaultApi* | [**VmResizePut**](docs/DefaultApi.md#vmresizeput) | **Put** /vm.resize | Resize the VM
 *DefaultApi* | [**VmmPingGet**](docs/DefaultApi.md#vmmpingget) | **Get** /vmm.ping | Ping the VMM to check for API server availability
 
@@ -62,6 +63,7 @@ Class | Method | HTTP request | Description
  - [VmAddDevice](docs/VmAddDevice.md)
  - [VmConfig](docs/VmConfig.md)
  - [VmInfo](docs/VmInfo.md)
+ - [VmRemoveDevice](docs/VmRemoveDevice.md)
  - [VmResize](docs/VmResize.md)
  - [VmmPingResponse](docs/VmmPingResponse.md)
  - [VsockConfig](docs/VsockConfig.md)

virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml

@@ -143,6 +143,21 @@ paths:
         "404":
           description: The new device could not be added to the VM instance.
       summary: Add a new device to the VM
+  /vm.remove-device:
+    put:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VmRemoveDevice'
+        description: The identifier of the device
+        required: true
+      responses:
+        "204":
+          description: The device was successfully removed from the VM instance.
+        "404":
+          description: The device could not be removed from the VM instance.
+      summary: Remove a device from the VM
 components:
   schemas:
     VmmPingResponse:
@@ -168,12 +183,13 @@ components:
             mergeable: false
             file: file
             size: 1
+            hotplug_size: 5
           disks:
           - path: path
             num_queues: 5
             readonly: false
             iommu: false
-            queue_size: 5
+            queue_size: 2
             vhost_socket: vhost_socket
             vhost_user: false
             direct: false
@@ -183,7 +199,7 @@ components:
             num_queues: 5
             readonly: false
             iommu: false
-            queue_size: 5
+            queue_size: 2
             vhost_socket: vhost_socket
             vhost_user: false
             direct: false
@@ -195,8 +211,10 @@ components:
           devices:
           - path: path
             iommu: false
+            id: id
           - path: path
             iommu: false
+            id: id
           kernel:
             path: path
           rng:
@@ -204,15 +222,15 @@ components:
             src: /dev/urandom
           fs:
           - sock: sock
-            num_queues: 9
+            num_queues: 3
-            queue_size: 3
+            queue_size: 2
-            cache_size: 2
+            cache_size: 4
             dax: true
             tag: tag
           - sock: sock
-            num_queues: 9
+            num_queues: 3
-            queue_size: 3
+            queue_size: 2
-            cache_size: 2
+            cache_size: 4
             dax: true
             tag: tag
           vsock:
@@ -225,12 +243,14 @@ components:
           pmem:
           - mergeable: false
             file: file
-            size: 4
+            size: 7
             iommu: false
+            discard_writes: false
           - mergeable: false
             file: file
-            size: 4
+            size: 7
             iommu: false
+            discard_writes: false
           cmdline:
             args: args
           iommu: false
@@ -240,18 +260,18 @@ components:
             iommu: false
           net:
           - tap: tap
-            num_queues: 2
+            num_queues: 7
             iommu: false
-            queue_size: 7
+            queue_size: 9
             vhost_socket: vhost_socket
             vhost_user: false
             ip: 192.168.249.1
             mac: mac
             mask: 255.255.255.0
           - tap: tap
-            num_queues: 2
+            num_queues: 7
             iommu: false
-            queue_size: 7
+            queue_size: 9
             vhost_socket: vhost_socket
             vhost_user: false
             ip: 192.168.249.1
@@ -282,12 +302,13 @@ components:
           mergeable: false
           file: file
           size: 1
+          hotplug_size: 5
         disks:
         - path: path
           num_queues: 5
           readonly: false
           iommu: false
-          queue_size: 5
+          queue_size: 2
           vhost_socket: vhost_socket
           vhost_user: false
           direct: false
@@ -297,7 +318,7 @@ components:
           num_queues: 5
           readonly: false
           iommu: false
-          queue_size: 5
+          queue_size: 2
           vhost_socket: vhost_socket
           vhost_user: false
           direct: false
@@ -309,8 +330,10 @@ components:
         devices:
         - path: path
           iommu: false
+          id: id
         - path: path
           iommu: false
+          id: id
         kernel:
           path: path
         rng:
@@ -318,15 +341,15 @@ components:
           src: /dev/urandom
         fs:
         - sock: sock
-          num_queues: 9
+          num_queues: 3
-          queue_size: 3
+          queue_size: 2
-          cache_size: 2
+          cache_size: 4
           dax: true
           tag: tag
         - sock: sock
-          num_queues: 9
+          num_queues: 3
-          queue_size: 3
+          queue_size: 2
-          cache_size: 2
+          cache_size: 4
           dax: true
           tag: tag
         vsock:
@@ -339,12 +362,14 @@ components:
         pmem:
         - mergeable: false
           file: file
-          size: 4
+          size: 7
           iommu: false
+          discard_writes: false
         - mergeable: false
           file: file
-          size: 4
+          size: 7
           iommu: false
+          discard_writes: false
         cmdline:
           args: args
         iommu: false
@@ -354,18 +379,18 @@ components:
           iommu: false
         net:
         - tap: tap
-          num_queues: 2
+          num_queues: 7
           iommu: false
-          queue_size: 7
+          queue_size: 9
           vhost_socket: vhost_socket
           vhost_user: false
           ip: 192.168.249.1
           mac: mac
           mask: 255.255.255.0
         - tap: tap
-          num_queues: 2
+          num_queues: 7
           iommu: false
-          queue_size: 7
+          queue_size: 9
           vhost_socket: vhost_socket
           vhost_user: false
           ip: 192.168.249.1
@@ -439,10 +464,14 @@ components:
         mergeable: false
         file: file
         size: 1
+        hotplug_size: 5
       properties:
         size:
           format: int64
           type: integer
+        hotplug_size:
+          format: int64
+          type: integer
         file:
           type: string
         mergeable:
@@ -475,7 +504,7 @@ components:
         num_queues: 5
         readonly: false
         iommu: false
-        queue_size: 5
+        queue_size: 2
         vhost_socket: vhost_socket
         vhost_user: false
         direct: false
@@ -516,9 +545,9 @@ components:
     NetConfig:
       example:
         tap: tap
-        num_queues: 2
+        num_queues: 7
         iommu: false
-        queue_size: 7
+        queue_size: 9
         vhost_socket: vhost_socket
         vhost_user: false
         ip: 192.168.249.1
@@ -568,9 +597,9 @@ components:
     FsConfig:
       example:
         sock: sock
-        num_queues: 9
+        num_queues: 3
-        queue_size: 3
+        queue_size: 2
-        cache_size: 2
+        cache_size: 4
         dax: true
         tag: tag
       properties:
@@ -598,8 +627,9 @@ components:
       example:
         mergeable: false
         file: file
-        size: 4
+        size: 7
         iommu: false
+        discard_writes: false
       properties:
         file:
           type: string
@@ -612,6 +642,9 @@ components:
         mergeable:
           default: false
           type: boolean
+        discard_writes:
+          default: false
+          type: boolean
       required:
       - file
       - size
@@ -641,12 +674,15 @@ components:
       example:
         path: path
         iommu: false
+        id: id
       properties:
         path:
           type: string
         iommu:
           default: false
           type: boolean
+        id:
+          type: string
       required:
       - path
       type: object
@@ -680,6 +716,8 @@ components:
           minimum: 1
           type: integer
         desired_ram:
+          description: desired memory ram in bytes
+          format: int64
           type: integer
       type: object
     VmAddDevice:
@@ -689,3 +727,10 @@ components:
         path:
           type: string
       type: object
+    VmRemoveDevice:
+      example:
+        id: id
+      properties:
+        id:
+          type: string
+      type: object

virtcontainers/pkg/cloud-hypervisor/client/api_default.go

@@ -680,6 +680,72 @@ func (a *DefaultApiService) VmInfoGet(ctx _context.Context) (VmInfo, *_nethttp.R
 	return localVarReturnValue, localVarHTTPResponse, nil
 }
 
+/*
+VmRemoveDevicePut Remove a device from the VM
+ * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ * @param vmRemoveDevice The identifier of the device
+*/
+func (a *DefaultApiService) VmRemoveDevicePut(ctx _context.Context, vmRemoveDevice VmRemoveDevice) (*_nethttp.Response, error) {
+	var (
+		localVarHTTPMethod   = _nethttp.MethodPut
+		localVarPostBody     interface{}
+		localVarFormFileName string
+		localVarFileName     string
+		localVarFileBytes    []byte
+	)
+
+	// create path and map variables
+	localVarPath := a.client.cfg.BasePath + "/vm.remove-device"
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := _neturl.Values{}
+	localVarFormParams := _neturl.Values{}
+
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{"application/json"}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	// body params
+	localVarPostBody = &vmRemoveDevice
+	r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(r)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarHTTPResponse, err
+	}
+
+	localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	if err != nil {
+		return localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		return localVarHTTPResponse, newErr
+	}
+
+	return localVarHTTPResponse, nil
+}
+
 /*
 VmResizePut Resize the VM
  * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().

virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md

@@ -14,6 +14,7 @@ Method | HTTP request | Description
 [**ShutdownVMM**](DefaultApi.md#ShutdownVMM) | **Put** /vmm.shutdown | Shuts the cloud-hypervisor VMM.
 [**VmAddDevicePut**](DefaultApi.md#VmAddDevicePut) | **Put** /vm.add-device | Add a new device to the VM
 [**VmInfoGet**](DefaultApi.md#VmInfoGet) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance.
+[**VmRemoveDevicePut**](DefaultApi.md#VmRemoveDevicePut) | **Put** /vm.remove-device | Remove a device from the VM
 [**VmResizePut**](DefaultApi.md#VmResizePut) | **Put** /vm.resize | Resize the VM
 [**VmmPingGet**](DefaultApi.md#VmmPingGet) | **Get** /vmm.ping | Ping the VMM to check for API server availability
 
@@ -307,6 +308,38 @@ No authorization required
 [[Back to README]](../README.md)
 
 
+## VmRemoveDevicePut
+
+> VmRemoveDevicePut(ctx, vmRemoveDevice)
+
+Remove a device from the VM
+
+### Required Parameters
+
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc.
+**vmRemoveDevice** | [**VmRemoveDevice**](VmRemoveDevice.md)| The identifier of the device | 
+
+### Return type
+
+ (empty response body)
+
+### Authorization
+
+No authorization required
+
+### HTTP request headers
+
+- **Content-Type**: application/json
+- **Accept**: Not defined
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
 ## VmResizePut
 
 > VmResizePut(ctx, vmResize)

virtcontainers/pkg/cloud-hypervisor/client/docs/DeviceConfig.md

@@ -6,6 +6,7 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **Path** | **string** |  | 
 **Iommu** | **bool** |  | [optional] [default to false]
+**Id** | **string** |  | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md

@@ -5,6 +5,7 @@
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **Size** | **int64** |  | 
+**HotplugSize** | **int64** |  | [optional] 
 **File** | **string** |  | [optional] 
 **Mergeable** | **bool** |  | [optional] [default to false]
 

virtcontainers/pkg/cloud-hypervisor/client/docs/PmemConfig.md

@@ -8,6 +8,7 @@ Name | Type | Description | Notes
 **Size** | **int64** |  | 
 **Iommu** | **bool** |  | [optional] [default to false]
 **Mergeable** | **bool** |  | [optional] [default to false]
+**DiscardWrites** | **bool** |  | [optional] [default to false]
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

virtcontainers/pkg/cloud-hypervisor/client/docs/VmRemoveDevice.md

@@ -0,0 +1,11 @@
+# VmRemoveDevice
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Id** | **string** |  | [optional] 
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

virtcontainers/pkg/cloud-hypervisor/client/docs/VmResize.md

@@ -5,7 +5,7 @@
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **DesiredVcpus** | **int32** |  | [optional] 
-**DesiredRam** | **int32** |  | [optional] 
+**DesiredRam** | **int64** | desired memory ram in bytes | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

virtcontainers/pkg/cloud-hypervisor/client/model_device_config.go

@@ -12,4 +12,5 @@ package openapi
 type DeviceConfig struct {
 	Path string `json:"path"`
 	Iommu bool `json:"iommu,omitempty"`
+	Id string `json:"id,omitempty"`
 }

virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go

@@ -11,6 +11,7 @@ package openapi
 // MemoryConfig struct for MemoryConfig
 type MemoryConfig struct {
 	Size int64 `json:"size"`
+	HotplugSize int64 `json:"hotplug_size,omitempty"`
 	File string `json:"file,omitempty"`
 	Mergeable bool `json:"mergeable,omitempty"`
 }

virtcontainers/pkg/cloud-hypervisor/client/model_pmem_config.go

@@ -14,4 +14,5 @@ type PmemConfig struct {
 	Size int64 `json:"size"`
 	Iommu bool `json:"iommu,omitempty"`
 	Mergeable bool `json:"mergeable,omitempty"`
+	DiscardWrites bool `json:"discard_writes,omitempty"`
 }

virtcontainers/pkg/cloud-hypervisor/client/model_vm_remove_device.go

@@ -0,0 +1,14 @@
+/*
+ * Cloud Hypervisor API
+ *
+ * Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine.
+ *
+ * API version: 0.3.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+// VmRemoveDevice struct for VmRemoveDevice
+type VmRemoveDevice struct {
+	Id string `json:"id,omitempty"`
+}

virtcontainers/pkg/cloud-hypervisor/client/model_vm_resize.go

@@ -11,5 +11,6 @@ package openapi
 // VmResize struct for VmResize
 type VmResize struct {
 	DesiredVcpus int32 `json:"desired_vcpus,omitempty"`
-	DesiredRam int32 `json:"desired_ram,omitempty"`
+	// desired memory ram in bytes
+	DesiredRam int64 `json:"desired_ram,omitempty"`
 }

virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml

@@ -155,6 +155,22 @@ paths:
         404:
           description: The new device could not be added to the VM instance.
 
+  /vm.remove-device:
+    put:
+      summary: Remove a device from the VM
+      requestBody:
+        description: The identifier of the device
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VmRemoveDevice'
+        required: true
+      responses:
+        204:
+          description: The device was successfully removed from the VM instance.
+        404:
+          description: The device could not be removed from the VM instance.
+
 components:
   schemas:
 
@@ -253,6 +269,9 @@ components:
           type: integer
           format: int64
           default: 512 MB
+        hotplug_size:
+          type: integer
+          format: int64
         file:
           type: string
         mergeable:
@@ -391,6 +410,9 @@ components:
         mergeable:
           type: boolean
           default: false
+        discard_writes:
+          type: boolean
+          default: false
 
     ConsoleConfig:
       required:
@@ -416,6 +438,8 @@ components:
         iommu:
           type: boolean
           default: false
+        id:
+          type: string
 
     VsockConfig:
       required:
@@ -442,10 +466,18 @@ components:
           minimum: 1
           type: integer
         desired_ram:
+          description: desired memory ram in bytes
           type: integer
+          format: int64
 
     VmAddDevice:
       type: object
       properties:
         path:
           type: string
+
+    VmRemoveDevice:
+      type: object
+      properties:
+        id:
+          type: string

virtcontainers/utils/utils.go

@@ -240,3 +240,38 @@ func SupportsVsocks() bool {
 var StartCmd = func(c *exec.Cmd) error {
 	return c.Start()
 }
+
+// AlignMem align memory provided to a block size
+func (m MemUnit) AlignMem(blockSize MemUnit) MemUnit {
+	memSize := m
+	if m < blockSize {
+		memSize = blockSize
+
+	}
+
+	remainder := memSize % blockSize
+
+	if remainder != 0 {
+		// Align memory to memoryBlockSizeMB
+		memSize += blockSize - remainder
+
+	}
+	return memSize
+}
+
+type MemUnit uint64
+
+func (m MemUnit) ToMiB() uint64 {
+	return m.ToBytes() / (1 * MiB).ToBytes()
+}
+
+func (m MemUnit) ToBytes() uint64 {
+	return uint64(m)
+}
+
+const (
+	Byte MemUnit = 1
+	KiB          = Byte << 10
+	MiB          = KiB << 10
+	GiB          = MiB << 10
+)