From 4c164afbac3df0611a03f7dc774833ee51184178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 09:39:51 +0100 Subject: [PATCH 01/10] versions: Update Cloud Hypervisor to 5343e09e7b8db MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's bump the Cloud Hypervisor version to 5343e09e7b8db, as that brings a few fixes we're interested in, such as: * hypervisor, vmm: Handle TDX hypercalls with INVALID_OPERAND - https://github.com/cloud-hypervisor/cloud-hypervisor/pull/3723 - This is needed for the TDX support on the cloud hypervisor driver, which is part of this very same series. * openapi: Update the PciBdf types - https://github.com/cloud-hypervisor/cloud-hypervisor/pull/3748 - This is needed due to a change in a DeviceNode field, which would cause a marshalling / demarshalling error when running with a version of cloud-hypervisor that includes the TDX fixes mentioned above. * scripts: dev_cli: Don't quote $features_build * scripts: dev_cli: Add --features option - https://github.com/cloud-hypervisor/cloud-hypervisor/pull/3773 - This is needed due to changes in the scripts used to build Cloud Hypervisor, which are used as part of Kata Containers CIs and github actions. Due to this change, we're also adapting the build scripts as part of this very same commit. Signed-off-by: Fabiano Fidêncio --- .../cloud-hypervisor/client/api/openapi.yaml | 23 +++++++++--- .../client/docs/DefaultApi.md | 4 +-- .../client/docs/DeviceNode.md | 8 ++--- .../client/docs/ReceiveMigrationData.md | 9 ++--- .../client/docs/SendMigrationData.md | 9 ++--- .../client/model_device_node.go | 12 +++---- .../client/model_receive_migration_data.go | 33 +++++++---------- .../client/model_send_migration_data.go | 35 ++++++++----------- .../cloud-hypervisor/cloud-hypervisor.yaml | 19 ++++++++-- .../local-build/kata-deploy-binaries.sh | 6 +++- .../cloud-hypervisor/build-static-clh.sh | 6 ++-- versions.yaml | 2 +- 12 files changed, 88 insertions(+), 78 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index b820e01cc8..4a168e53e5 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -181,6 +181,8 @@ paths: schema: $ref: '#/components/schemas/PciDeviceInfo' description: The new device was successfully added to the VM instance. + "204": + description: The new device was successfully (cold) added to the VM instance. "404": description: The new device could not be added to the VM instance. summary: Add a new device to the VM @@ -215,6 +217,8 @@ paths: schema: $ref: '#/components/schemas/PciDeviceInfo' description: The new disk was successfully added to the VM instance. + "204": + description: The new disk was successfully (cold) added to the VM instance. "500": description: The new disk could not be added to the VM instance. summary: Add a new disk to the VM @@ -234,6 +238,8 @@ paths: schema: $ref: '#/components/schemas/PciDeviceInfo' description: The new device was successfully added to the VM instance. + "204": + description: The new device was successfully (cold) added to the VM instance. "500": description: The new device could not be added to the VM instance. summary: Add a new virtio-fs device to the VM @@ -253,6 +259,8 @@ paths: schema: $ref: '#/components/schemas/PciDeviceInfo' description: The new device was successfully added to the VM instance. + "204": + description: The new device was successfully (cold) added to the VM instance. "500": description: The new device could not be added to the VM instance. summary: Add a new pmem device to the VM @@ -272,6 +280,8 @@ paths: schema: $ref: '#/components/schemas/PciDeviceInfo' description: The new device was successfully added to the VM instance. + "204": + description: The new device was successfully (cold) added to the VM instance. "500": description: The new device could not be added to the VM instance. summary: Add a new network device to the VM @@ -291,6 +301,8 @@ paths: schema: $ref: '#/components/schemas/PciDeviceInfo' description: The new device was successfully added to the VM instance. + "204": + description: The new device was successfully (cold) added to the VM instance. "500": description: The new device could not be added to the VM instance. summary: Add a new vsock device to the VM @@ -632,7 +644,7 @@ components: children: - children - children - pci_bdf: 3 + pci_bdf: pci_bdf resources: - '{}' - '{}' @@ -663,7 +675,7 @@ components: children: - children - children - pci_bdf: 3 + pci_bdf: pci_bdf resources: - '{}' - '{}' @@ -680,8 +692,7 @@ components: type: string type: array pci_bdf: - format: int32 - type: integer + type: string type: object VmCounters: additionalProperties: @@ -1757,6 +1768,8 @@ components: properties: receiver_url: type: string + required: + - receiver_url type: object SendMigrationData: example: @@ -1767,4 +1780,6 @@ components: type: string local: type: boolean + required: + - destination_url type: object diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md index 3c2c8821d9..c5ca050b60 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md @@ -1071,7 +1071,7 @@ import ( ) func main() { - receiveMigrationData := *openapiclient.NewReceiveMigrationData() // ReceiveMigrationData | The URL for the reception of migration state + receiveMigrationData := *openapiclient.NewReceiveMigrationData("ReceiverUrl_example") // ReceiveMigrationData | The URL for the reception of migration state configuration := openapiclient.NewConfiguration() api_client := openapiclient.NewAPIClient(configuration) @@ -1381,7 +1381,7 @@ import ( ) func main() { - sendMigrationData := *openapiclient.NewSendMigrationData() // SendMigrationData | The URL for sending the migration state + sendMigrationData := *openapiclient.NewSendMigrationData("DestinationUrl_example") // SendMigrationData | The URL for sending the migration state configuration := openapiclient.NewConfiguration() api_client := openapiclient.NewAPIClient(configuration) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DeviceNode.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DeviceNode.md index 5755470551..02863e513d 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DeviceNode.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DeviceNode.md @@ -7,7 +7,7 @@ Name | Type | Description | Notes **Id** | Pointer to **string** | | [optional] **Resources** | Pointer to **[]map[string]interface{}** | | [optional] **Children** | Pointer to **[]string** | | [optional] -**PciBdf** | Pointer to **int32** | | [optional] +**PciBdf** | Pointer to **string** | | [optional] ## Methods @@ -105,20 +105,20 @@ HasChildren returns a boolean if a field has been set. ### GetPciBdf -`func (o *DeviceNode) GetPciBdf() int32` +`func (o *DeviceNode) GetPciBdf() string` GetPciBdf returns the PciBdf field if non-nil, zero value otherwise. ### GetPciBdfOk -`func (o *DeviceNode) GetPciBdfOk() (*int32, bool)` +`func (o *DeviceNode) GetPciBdfOk() (*string, bool)` GetPciBdfOk returns a tuple with the PciBdf field if it's non-nil, zero value otherwise and a boolean to check if the value has been set. ### SetPciBdf -`func (o *DeviceNode) SetPciBdf(v int32)` +`func (o *DeviceNode) SetPciBdf(v string)` SetPciBdf sets PciBdf field to given value. diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/ReceiveMigrationData.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/ReceiveMigrationData.md index 652bf96bc2..394d47cebc 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/ReceiveMigrationData.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/ReceiveMigrationData.md @@ -4,13 +4,13 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**ReceiverUrl** | Pointer to **string** | | [optional] +**ReceiverUrl** | **string** | | ## Methods ### NewReceiveMigrationData -`func NewReceiveMigrationData() *ReceiveMigrationData` +`func NewReceiveMigrationData(receiverUrl string, ) *ReceiveMigrationData` NewReceiveMigrationData instantiates a new ReceiveMigrationData object This constructor will assign default values to properties that have it defined, @@ -44,11 +44,6 @@ and a boolean to check if the value has been set. SetReceiverUrl sets ReceiverUrl field to given value. -### HasReceiverUrl - -`func (o *ReceiveMigrationData) HasReceiverUrl() bool` - -HasReceiverUrl returns a boolean if a field has been set. [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SendMigrationData.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SendMigrationData.md index 03edd489e8..d3b6a5c1bd 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SendMigrationData.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SendMigrationData.md @@ -4,14 +4,14 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**DestinationUrl** | Pointer to **string** | | [optional] +**DestinationUrl** | **string** | | **Local** | Pointer to **bool** | | [optional] ## Methods ### NewSendMigrationData -`func NewSendMigrationData() *SendMigrationData` +`func NewSendMigrationData(destinationUrl string, ) *SendMigrationData` NewSendMigrationData instantiates a new SendMigrationData object This constructor will assign default values to properties that have it defined, @@ -45,11 +45,6 @@ and a boolean to check if the value has been set. SetDestinationUrl sets DestinationUrl field to given value. -### HasDestinationUrl - -`func (o *SendMigrationData) HasDestinationUrl() bool` - -HasDestinationUrl returns a boolean if a field has been set. ### GetLocal diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_device_node.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_device_node.go index ef6aab589d..012f971967 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_device_node.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_device_node.go @@ -19,7 +19,7 @@ type DeviceNode struct { Id *string `json:"id,omitempty"` Resources *[]map[string]interface{} `json:"resources,omitempty"` Children *[]string `json:"children,omitempty"` - PciBdf *int32 `json:"pci_bdf,omitempty"` + PciBdf *string `json:"pci_bdf,omitempty"` } // NewDeviceNode instantiates a new DeviceNode object @@ -136,9 +136,9 @@ func (o *DeviceNode) SetChildren(v []string) { } // GetPciBdf returns the PciBdf field value if set, zero value otherwise. -func (o *DeviceNode) GetPciBdf() int32 { +func (o *DeviceNode) GetPciBdf() string { if o == nil || o.PciBdf == nil { - var ret int32 + var ret string return ret } return *o.PciBdf @@ -146,7 +146,7 @@ func (o *DeviceNode) GetPciBdf() int32 { // GetPciBdfOk returns a tuple with the PciBdf field value if set, nil otherwise // and a boolean to check if the value has been set. -func (o *DeviceNode) GetPciBdfOk() (*int32, bool) { +func (o *DeviceNode) GetPciBdfOk() (*string, bool) { if o == nil || o.PciBdf == nil { return nil, false } @@ -162,8 +162,8 @@ func (o *DeviceNode) HasPciBdf() bool { return false } -// SetPciBdf gets a reference to the given int32 and assigns it to the PciBdf field. -func (o *DeviceNode) SetPciBdf(v int32) { +// SetPciBdf gets a reference to the given string and assigns it to the PciBdf field. +func (o *DeviceNode) SetPciBdf(v string) { o.PciBdf = &v } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_receive_migration_data.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_receive_migration_data.go index fd7e6eff79..7549055d55 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_receive_migration_data.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_receive_migration_data.go @@ -16,15 +16,16 @@ import ( // ReceiveMigrationData struct for ReceiveMigrationData type ReceiveMigrationData struct { - ReceiverUrl *string `json:"receiver_url,omitempty"` + ReceiverUrl string `json:"receiver_url"` } // NewReceiveMigrationData instantiates a new ReceiveMigrationData object // This constructor will assign default values to properties that have it defined, // and makes sure properties required by API are set, but the set of arguments // will change when the set of required properties is changed -func NewReceiveMigrationData() *ReceiveMigrationData { +func NewReceiveMigrationData(receiverUrl string) *ReceiveMigrationData { this := ReceiveMigrationData{} + this.ReceiverUrl = receiverUrl return &this } @@ -36,41 +37,33 @@ func NewReceiveMigrationDataWithDefaults() *ReceiveMigrationData { return &this } -// GetReceiverUrl returns the ReceiverUrl field value if set, zero value otherwise. +// GetReceiverUrl returns the ReceiverUrl field value func (o *ReceiveMigrationData) GetReceiverUrl() string { - if o == nil || o.ReceiverUrl == nil { + if o == nil { var ret string return ret } - return *o.ReceiverUrl + + return o.ReceiverUrl } -// GetReceiverUrlOk returns a tuple with the ReceiverUrl field value if set, nil otherwise +// GetReceiverUrlOk returns a tuple with the ReceiverUrl field value // and a boolean to check if the value has been set. func (o *ReceiveMigrationData) GetReceiverUrlOk() (*string, bool) { - if o == nil || o.ReceiverUrl == nil { + if o == nil { return nil, false } - return o.ReceiverUrl, true + return &o.ReceiverUrl, true } -// HasReceiverUrl returns a boolean if a field has been set. -func (o *ReceiveMigrationData) HasReceiverUrl() bool { - if o != nil && o.ReceiverUrl != nil { - return true - } - - return false -} - -// SetReceiverUrl gets a reference to the given string and assigns it to the ReceiverUrl field. +// SetReceiverUrl sets field value func (o *ReceiveMigrationData) SetReceiverUrl(v string) { - o.ReceiverUrl = &v + o.ReceiverUrl = v } func (o ReceiveMigrationData) MarshalJSON() ([]byte, error) { toSerialize := map[string]interface{}{} - if o.ReceiverUrl != nil { + if true { toSerialize["receiver_url"] = o.ReceiverUrl } return json.Marshal(toSerialize) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_send_migration_data.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_send_migration_data.go index 11f913bf02..6ceeedb7e7 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_send_migration_data.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_send_migration_data.go @@ -16,16 +16,17 @@ import ( // SendMigrationData struct for SendMigrationData type SendMigrationData struct { - DestinationUrl *string `json:"destination_url,omitempty"` - Local *bool `json:"local,omitempty"` + DestinationUrl string `json:"destination_url"` + Local *bool `json:"local,omitempty"` } // NewSendMigrationData instantiates a new SendMigrationData object // This constructor will assign default values to properties that have it defined, // and makes sure properties required by API are set, but the set of arguments // will change when the set of required properties is changed -func NewSendMigrationData() *SendMigrationData { +func NewSendMigrationData(destinationUrl string) *SendMigrationData { this := SendMigrationData{} + this.DestinationUrl = destinationUrl return &this } @@ -37,36 +38,28 @@ func NewSendMigrationDataWithDefaults() *SendMigrationData { return &this } -// GetDestinationUrl returns the DestinationUrl field value if set, zero value otherwise. +// GetDestinationUrl returns the DestinationUrl field value func (o *SendMigrationData) GetDestinationUrl() string { - if o == nil || o.DestinationUrl == nil { + if o == nil { var ret string return ret } - return *o.DestinationUrl + + return o.DestinationUrl } -// GetDestinationUrlOk returns a tuple with the DestinationUrl field value if set, nil otherwise +// GetDestinationUrlOk returns a tuple with the DestinationUrl field value // and a boolean to check if the value has been set. func (o *SendMigrationData) GetDestinationUrlOk() (*string, bool) { - if o == nil || o.DestinationUrl == nil { + if o == nil { return nil, false } - return o.DestinationUrl, true + return &o.DestinationUrl, true } -// HasDestinationUrl returns a boolean if a field has been set. -func (o *SendMigrationData) HasDestinationUrl() bool { - if o != nil && o.DestinationUrl != nil { - return true - } - - return false -} - -// SetDestinationUrl gets a reference to the given string and assigns it to the DestinationUrl field. +// SetDestinationUrl sets field value func (o *SendMigrationData) SetDestinationUrl(v string) { - o.DestinationUrl = &v + o.DestinationUrl = v } // GetLocal returns the Local field value if set, zero value otherwise. @@ -103,7 +96,7 @@ func (o *SendMigrationData) SetLocal(v bool) { func (o SendMigrationData) MarshalJSON() ([]byte, error) { toSerialize := map[string]interface{}{} - if o.DestinationUrl != nil { + if true { toSerialize["destination_url"] = o.DestinationUrl } if o.Local != nil { diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index c4dcae04c2..c9ecd399e6 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -195,6 +195,8 @@ paths: application/json: schema: $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new device was successfully (cold) added to the VM instance. 404: description: The new device could not be added to the VM instance. @@ -231,6 +233,8 @@ paths: application/json: schema: $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new disk was successfully (cold) added to the VM instance. 500: description: The new disk could not be added to the VM instance. @@ -251,6 +255,8 @@ paths: application/json: schema: $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new device was successfully (cold) added to the VM instance. 500: description: The new device could not be added to the VM instance. @@ -271,6 +277,8 @@ paths: application/json: schema: $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new device was successfully (cold) added to the VM instance. 500: description: The new device could not be added to the VM instance. @@ -291,6 +299,8 @@ paths: application/json: schema: $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new device was successfully (cold) added to the VM instance. 500: description: The new device could not be added to the VM instance. @@ -311,6 +321,8 @@ paths: application/json: schema: $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new device was successfully (cold) added to the VM instance. 500: description: The new device could not be added to the VM instance. @@ -428,8 +440,7 @@ components: items: type: string pci_bdf: - type: integer - format: int32 + type: string VmCounters: type: object @@ -1055,12 +1066,16 @@ components: type: boolean ReceiveMigrationData: + required: + - receiver_url type: object properties: receiver_url: type: string SendMigrationData: + required: + - destination_url type: object properties: destination_url: diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 1864140e1e..2a215726b9 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -28,6 +28,8 @@ readonly shimv2_builder="${static_build_dir}/shim-v2/build.sh" readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_image.sh" +ARCH=$(uname -m) + workdir="${WORKDIR:-$PWD}" destdir="${workdir}/kata-static" @@ -125,7 +127,9 @@ install_firecracker() { # Install static cloud-hypervisor asset install_clh() { - export extra_build_args="--features tdx" + if [[ "${ARCH}" == "x86_64" ]]; then + export features="tdx" + fi info "build static cloud-hypervisor" "${clh_builder}" diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 61a0824ebb..1f49abcaa4 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -52,9 +52,9 @@ build_clh_from_source() { pushd "${repo_dir}" git fetch || true git checkout "${cloud_hypervisor_version}" - if [ -n "${extra_build_args}" ]; then - info "Build cloud-hypervisor with extra args: ${extra_build_args}" - ./scripts/dev_cli.sh build --release --libc musl -- ${extra_build_args} + if [ -n "${features}" ]; then + info "Build cloud-hypervisor enabling the following features: ${features}" + ./scripts/dev_cli.sh build --release --libc musl --features "${features}" else ./scripts/dev_cli.sh build --release --libc musl fi diff --git a/versions.yaml b/versions.yaml index 84463db8ef..c1b10ce893 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "55479a64d237d4c757dba19a696abefd27ec74fd" + version: "5343e09e7b8dbd5dd8ac0d90a3ad52037490dd86" firecracker: description: "Firecracker micro-VMM" From dcdc412e252ee24daca9196902a52eeef22e93d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 10:19:12 +0100 Subject: [PATCH 02/10] clh: use common kernel params from the hypervisor code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hypervisor code already defines 3 common kernel root params for the following cases: * NVDIMM * NVDIMM without DAX support * Virtio Block As parameters used for cloud-hypervisor have an overlap with the ones provided by the NVDIMM case, let's take advantage of that. Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/clh.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 79a976abff..0dc478e7eb 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -171,12 +171,9 @@ type cloudHypervisor struct { } var clhKernelParams = []Param{ - {"root", "/dev/pmem0p1"}, {"panic", "1"}, // upon kernel panic wait 1 second before reboot {"no_timer_check", ""}, // do not Check broken timer IRQ resources {"noreplace-smp", ""}, // do not replace SMP instructions - {"rootflags", "dax,data=ordered,errors=remount-ro ro"}, // mount the root filesystem as readonly - {"rootfstype", "ext4"}, } var clhDebugKernelParams = []Param{ @@ -267,7 +264,8 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs), int32(clh.config.DefaultMaxVCPUs)) // First take the default parameters defined by this driver - params := clhKernelParams + params := commonNvdimmKernelRootParams + params = append(params, clhKernelParams...) // Followed by extra debug parameters if debug enabled in configuration file if clh.config.Debug { From 9621c596914c40245ce49d95bfa421f57304132a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 11:26:57 +0100 Subject: [PATCH 03/10] clh: refactor image / initrd configuration set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a small code refactor removing a deadcode based the checks already done in the generic hypervisor abstraction. Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/clh.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 0dc478e7eb..e81e34fcdc 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -289,11 +289,6 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net return err } - initrdPath, err := clh.config.InitrdAssetPath() - if err != nil { - return err - } - if imagePath != "" { pmem := chclient.NewPmemConfig(imagePath) *pmem.DiscardWrites = true @@ -303,12 +298,15 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net } else { clh.vmconfig.Pmem = &[]chclient.PmemConfig{*pmem} } - } else if initrdPath != "" { + } else { + initrdPath, err := clh.config.InitrdAssetPath() + if err != nil { + return err + } + initrd := chclient.NewInitramfsConfig(initrdPath) clh.vmconfig.SetInitramfs(*initrd) - } else { - return errors.New("no image or initrd specified") } // Use serial port as the guest console only in debug mode, From 29ee870d20d382233001bdc97bb8e75d3f05081c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 19:44:26 +0100 Subject: [PATCH 04/10] clh: Add confidential_guest to the config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ConfidentialGuest is an option already present and exposed for QEMU, which is used for using Kata Containers together with different sorts of Guest Protections, such as TDX and SEV for x86_64, PEF for ppc64le, and SE for s390x. Right now we error out in case confidential_guest is enabled, as we will be implementing the needed blocks for this as part of this series. Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 8 ++++++++ src/runtime/pkg/katautils/config.go | 1 + src/runtime/virtcontainers/clh.go | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 07e3f31a4b..cfd63c488b 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -15,6 +15,14 @@ path = "@CLHPATH@" kernel = "@KERNELPATH_CLH@" image = "@IMAGEPATH@" +# Enable confidential guest support. +# Toggling that setting may trigger different hardware features, ranging +# from memory encryption to both memory and CPU-state encryption and integrity. +# The Kata Containers runtime dynamically detects the available feature set and +# aims at enabling the largest possible one. +# Default false +# confidential_guest = true + # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index ed67bad13d..e623559339 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -877,6 +877,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { SGXEPCSize: defaultSGXEPCSize, EnableAnnotations: h.EnableAnnotations, DisableSeccomp: h.DisableSeccomp, + ConfidentialGuest: h.ConfidentialGuest, }, nil } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index e81e34fcdc..3cf1ca3e1e 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -215,6 +215,10 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net return err } + if clh.config.ConfidentialGuest { + return errors.New("confidential guest is not yet supported with Cloud Hypervisor") + } + clh.id = id clh.state.state = clhNotReady From 28c4c044e6987b8eb40c1c3fd01400219c2db8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 20:39:15 +0100 Subject: [PATCH 05/10] hypervisors: Confidential Guests do not support VCPUs hotplug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As confidential guests do not support VCPUs hotplug, let's set the "DefaultMaxVCPUs" value to "NumVCPUs". The reason to do this is to ensure that guests will be started with the correct amount of VCPUs, without giving to the guest with all the possible VCPUs the host could provide. One clear side effect of this limitation is that workloads that would require more VCPUs on their yaml definition will not run on this scenario. Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 5 +++++ src/runtime/config/configuration-qemu.toml.in | 5 +++++ src/runtime/virtcontainers/hypervisor.go | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index cfd63c488b..0971c6c4b9 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -20,6 +20,11 @@ image = "@IMAGEPATH@" # from memory encryption to both memory and CPU-state encryption and integrity. # The Kata Containers runtime dynamically detects the available feature set and # aims at enabling the largest possible one. +# +# Known limitations: +# * Does not work by design: +# - CPU Hotplug +# # Default false # confidential_guest = true diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 59753db5d6..a603d7607a 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -21,6 +21,11 @@ machine_type = "@MACHINETYPE@" # from memory encryption to both memory and CPU-state encryption and integrity. # The Kata Containers runtime dynamically detects the available feature set and # aims at enabling the largest possible one. +# +# Known limitations: +# * Does not work by design: +# - CPU Hotplug +# # Default false # confidential_guest = true diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 713280be72..26f33f5d6b 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -564,6 +564,11 @@ func (conf *HypervisorConfig) Valid() error { conf.DefaultMaxVCPUs = defaultMaxVCPUs } + if conf.ConfidentialGuest && conf.NumVCPUs != conf.DefaultMaxVCPUs { + hvLogger.Warnf("Confidential guests do not support hotplugging of vCPUs. Setting DefaultMaxVCPUs to NumVCPUs (%d)", conf.NumVCPUs) + conf.DefaultMaxVCPUs = conf.NumVCPUs + } + if conf.Msize9p == 0 && conf.SharedFS != config.VirtioFS { conf.Msize9p = defaultMsize9p } From df8ffecde0b4190c6c8ce8ee21e871ceafe132f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 21:00:39 +0100 Subject: [PATCH 06/10] hypervisors: Confidential Guests do not support Device hotplug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similarly to VCPUs hotplug, Confidential Guests also do not support Device hotplug. Let's make it clear in the documentation and guard the code on both QEMU and Cloud Hypervisor side to ensure we don't advertise Device hotplug as being supported when running Confidential Guests. Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 1 + src/runtime/config/configuration-qemu.toml.in | 1 + src/runtime/virtcontainers/clh.go | 12 +++++++++++- src/runtime/virtcontainers/qemu_amd64.go | 5 +++-- src/runtime/virtcontainers/qemu_arch_base.go | 4 +++- src/runtime/virtcontainers/qemu_ppc64le.go | 3 ++- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 0971c6c4b9..dc7f1f9f55 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -24,6 +24,7 @@ image = "@IMAGEPATH@" # Known limitations: # * Does not work by design: # - CPU Hotplug +# - Device Hotplug # # Default false # confidential_guest = true diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index a603d7607a..1282be310b 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -25,6 +25,7 @@ machine_type = "@MACHINETYPE@" # Known limitations: # * Does not work by design: # - CPU Hotplug +# - Device Hotplug # # Default false # confidential_guest = true diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 3cf1ca3e1e..f463d17445 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -589,6 +589,10 @@ func (clh *cloudHypervisor) HotplugAddDevice(ctx context.Context, devInfo interf span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugAddDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() + if clh.config.ConfidentialGuest { + return nil, errors.New("Device hotplug addition is not supported in confidential mode") + } + switch devType { case BlockDev: drive := devInfo.(*config.BlockDrive) @@ -606,6 +610,10 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugRemoveDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() + if clh.config.ConfidentialGuest { + return nil, errors.New("Device hotplug addition is not supported in confidential mode") + } + var deviceID string switch devType { @@ -860,7 +868,9 @@ func (clh *cloudHypervisor) Capabilities(ctx context.Context) types.Capabilities clh.Logger().WithField("function", "Capabilities").Info("get Capabilities") var caps types.Capabilities caps.SetFsSharingSupport() - caps.SetBlockDeviceHotplugSupport() + if !clh.config.ConfidentialGuest { + caps.SetBlockDeviceHotplugSupport() + } return caps } diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index f3ae89b59c..067a555036 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -153,8 +153,9 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { func (q *qemuAmd64) capabilities() types.Capabilities { var caps types.Capabilities - if q.qemuMachine.Type == QemuQ35 || - q.qemuMachine.Type == QemuVirt { + if (q.qemuMachine.Type == QemuQ35 || + q.qemuMachine.Type == QemuVirt) && + q.protection == noneProtection { caps.SetBlockDeviceHotplugSupport() } diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index b65cb61e3d..6601a74d81 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -277,7 +277,9 @@ func (q *qemuArchBase) kernelParameters(debug bool) []Param { func (q *qemuArchBase) capabilities() types.Capabilities { var caps types.Capabilities - caps.SetBlockDeviceHotplugSupport() + if q.protection == noneProtection { + caps.SetBlockDeviceHotplugSupport() + } caps.SetMultiQueueSupport() caps.SetFsSharingSupport() return caps diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index f78ed24297..d6f768128b 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -96,7 +96,8 @@ func (q *qemuPPC64le) capabilities() types.Capabilities { var caps types.Capabilities // pseries machine type supports hotplugging drives - if q.qemuMachine.Type == QemuPseries { + if q.qemuMachine.Type == QemuPseries && + q.protection == noneProtection { caps.SetBlockDeviceHotplugSupport() } From f50ff9f7987ce7e0156b3ea183f6fc3678f8d07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 21:39:54 +0100 Subject: [PATCH 07/10] hypervisors: Confidential Guests do not support Memory hotplug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similarly to VCPUs and Device hotplug, Confidential Guests also do not support Memory hotplug. Let's make it clear in the documentation and guard the code on both QEMU and Cloud Hypervisor side to ensure we don't advertise Memory hotplug as being supported when running Confidential Guests. Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 1 + src/runtime/config/configuration-qemu.toml.in | 1 + src/runtime/virtcontainers/clh.go | 12 +++++++----- src/runtime/virtcontainers/qemu_amd64.go | 6 +++++- src/runtime/virtcontainers/qemu_arch_base.go | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index dc7f1f9f55..c2522cba69 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -25,6 +25,7 @@ image = "@IMAGEPATH@" # * Does not work by design: # - CPU Hotplug # - Device Hotplug +# - Memory Hotplug # # Default false # confidential_guest = true diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 1282be310b..0f21984320 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -26,6 +26,7 @@ machine_type = "@MACHINETYPE@" # * Does not work by design: # - CPU Hotplug # - Device Hotplug +# - Memory Hotplug # # Default false # confidential_guest = true diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index f463d17445..7833d4093a 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -258,12 +258,14 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net clh.vmconfig.Memory.Shared = func(b bool) *bool { return &b }(true) // Enable hugepages if needed clh.vmconfig.Memory.Hugepages = func(b bool) *bool { return &b }(clh.config.HugePages) - hostMemKb, err := GetHostMemorySizeKb(procMemInfo) - if err != nil { - return nil + if !clh.config.ConfidentialGuest { + hostMemKb, err := GetHostMemorySizeKb(procMemInfo) + if err != nil { + return nil + } + // OpenAPI only supports int64 values + clh.vmconfig.Memory.HotplugSize = func(i int64) *int64 { return &i }(int64((utils.MemUnit(hostMemKb) * utils.KiB).ToBytes())) } - // OpenAPI only supports int64 values - clh.vmconfig.Memory.HotplugSize = func(i int64) *int64 { return &i }(int64((utils.MemUnit(hostMemKb) * utils.KiB).ToBytes())) // Set initial amount of cpu's for the virtual machine clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs), int32(clh.config.DefaultMaxVCPUs)) diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 067a555036..c32c5025dc 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -189,7 +189,11 @@ func (q *qemuAmd64) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) g // Is Memory Hotplug supported by this architecture/machine type combination? func (q *qemuAmd64) supportGuestMemoryHotplug() bool { // true for all amd64 machine types except for microvm. - return q.qemuMachine.Type != govmmQemu.MachineTypeMicrovm + if q.qemuMachine.Type == govmmQemu.MachineTypeMicrovm { + return false + } + + return q.protection == noneProtection } func (q *qemuAmd64) appendImage(ctx context.Context, devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 6601a74d81..8820fd655e 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -692,7 +692,7 @@ func (q *qemuArchBase) handleImagePath(config HypervisorConfig) { } func (q *qemuArchBase) supportGuestMemoryHotplug() bool { - return true + return q.protection == noneProtection } func (q *qemuArchBase) setIgnoreSharedMemoryMigrationCaps(ctx context.Context, qmp *govmmQemu.QMP) error { From a8827e0c78938afa9d5da68d05b32cb91ca85467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 21:57:38 +0100 Subject: [PATCH 08/10] hypervisors: Confidential Guests do not support NVDIMM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NVDIMM is also not supported with Confidential Guests and Virtio Block devices should be used instead. Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 1 + src/runtime/config/configuration-qemu.toml.in | 4 ++++ src/runtime/virtcontainers/clh.go | 24 +++++++++++++++---- src/runtime/virtcontainers/qemu_amd64.go | 5 ++++ src/runtime/virtcontainers/qemu_ppc64le.go | 5 ++++ src/runtime/virtcontainers/qemu_s390x.go | 5 ++++ 6 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index c2522cba69..99acada75b 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -26,6 +26,7 @@ image = "@IMAGEPATH@" # - CPU Hotplug # - Device Hotplug # - Memory Hotplug +# - NVDIMM devices # # Default false # confidential_guest = true diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 0f21984320..804849a132 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -27,6 +27,7 @@ machine_type = "@MACHINETYPE@" # - CPU Hotplug # - Device Hotplug # - Memory Hotplug +# - NVDIMM devices # # Default false # confidential_guest = true @@ -286,6 +287,9 @@ pflashes = [] # If false and nvdimm is supported, use nvdimm device to plug guest image. # Otherwise virtio-block device is used. +# +# nvdimm is not supported when `confidential_guest = true`. +# # Default is false #disable_image_nvdimm = true diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 7833d4093a..eae13eb04f 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -271,6 +271,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net // First take the default parameters defined by this driver params := commonNvdimmKernelRootParams + if clh.config.ConfidentialGuest { + params = commonVirtioblkKernelRootParams + } params = append(params, clhKernelParams...) // Followed by extra debug parameters if debug enabled in configuration file @@ -296,13 +299,24 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net } if imagePath != "" { - pmem := chclient.NewPmemConfig(imagePath) - *pmem.DiscardWrites = true + if clh.config.ConfidentialGuest { + disk := chclient.NewDiskConfig(imagePath) + disk.SetReadonly(true) - if clh.vmconfig.Pmem != nil { - *clh.vmconfig.Pmem = append(*clh.vmconfig.Pmem, *pmem) + if clh.vmconfig.Disks != nil { + *clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk) + } else { + clh.vmconfig.Disks = &[]chclient.DiskConfig{*disk} + } } else { - clh.vmconfig.Pmem = &[]chclient.PmemConfig{*pmem} + pmem := chclient.NewPmemConfig(imagePath) + *pmem.DiscardWrites = true + + if clh.vmconfig.Pmem != nil { + *clh.vmconfig.Pmem = append(*clh.vmconfig.Pmem, *pmem) + } else { + clh.vmconfig.Pmem = &[]chclient.PmemConfig{*pmem} + } } } else { initrdPath, err := clh.config.InitrdAssetPath() diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index c32c5025dc..ef1e16adbf 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -132,6 +132,11 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { if err := q.enableProtection(); err != nil { return nil, err } + + if !q.qemuArchBase.disableNvdimm { + hvLogger.WithField("subsystem", "qemuAmd64").Warn("Nvdimm is not supported with confidential guest, disabling it.") + q.qemuArchBase.disableNvdimm = true + } } if config.SGXEPCSize != 0 { diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index d6f768128b..93c11416ce 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -83,6 +83,11 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { if err := q.enableProtection(); err != nil { return nil, err } + + if !q.qemuArchBase.disableNvdimm { + hvLogger.WithField("subsystem", "qemuPPC64le").Warn("Nvdimm is not supported with confidential guest, disabling it.") + q.qemuArchBase.disableNvdimm = true + } } q.handleImagePath(config) diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index aeddb982af..77b6f440b1 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -77,6 +77,11 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { if err := q.enableProtection(); err != nil { return nil, err } + + if !q.qemuArchBase.disableNvdimm { + hvLogger.WithField("subsystem", "qemuS390x").Warn("Nvdimm is not supported with confidential guest, disabling it.") + q.qemuArchBase.disableNvdimm = true + } } if config.ImagePath != "" { From a13b4d5ad8dc03e3ba25a5cd93c5d37919c13787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 19:55:54 +0100 Subject: [PATCH 09/10] clh: Add firmware to the config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "firmware" option was already present for a while, but it's never been exposed to the configuration file before. Let's do it now as it can be used, in combination with the newly added confidential_guest option, to boot a guest VM using the so called `td-shim`[0] with Cloud Hypervisor. [0]: https://github.com/confidential-containers/td-shim Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 99acada75b..1296d20b95 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -31,6 +31,17 @@ image = "@IMAGEPATH@" # Default false # confidential_guest = true +# Path to the firmware. +# If you want Cloud Hypervisor to use a specific firmware, set its path below. +# This is option is only used when confidential_guest is enabled. +# +# For more information about firmwared that can be used with specific TEEs, +# please, refer to: +# * TDX: +# - td-shim: https://github.com/confidential-containers/td-shim +# +# firmware = "@FIRMWAREPATH@" + # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" From 72434333aa08daaa86c5110a1ebb6839d8a8b0e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 24 Feb 2022 22:30:24 +0100 Subject: [PATCH 10/10] clh: Add TDX support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's enable TDX support for Cloud Hypervisor, using td-shim as its desired firmware. Fixes: #3632 Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/clh.go | 38 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index eae13eb04f..11c1f7e353 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -202,6 +202,34 @@ func (clh *cloudHypervisor) nydusdAPISocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.config.VMStorePath, id, nydusdAPISock) } +func (clh *cloudHypervisor) enableProtection() error { + protection, err := availableGuestProtection() + if err != nil { + return err + } + + switch protection { + case tdxProtection: + firmwarePath, err := clh.config.FirmwareAssetPath() + if err != nil { + return err + } + + if firmwarePath == "" { + return errors.New("Firmware path is not specified") + } + + clh.vmconfig.Tdx = chclient.NewTdxConfig(firmwarePath) + return nil + + case sevProtection: + return errors.New("SEV protection is not supported by Cloud Hypervisor") + + default: + return errors.New("This system doesn't support Confidentian Computing (Guest Protection)") + } +} + // For cloudHypervisor this call only sets the internal structure up. // The VM will be created and started through StartVM(). func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error { @@ -215,10 +243,6 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net return err } - if clh.config.ConfidentialGuest { - return errors.New("confidential guest is not yet supported with Cloud Hypervisor") - } - clh.id = id clh.state.state = clhNotReady @@ -252,6 +276,12 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net // Create the VM config via the constructor to ensure default values are properly assigned clh.vmconfig = *chclient.NewVmConfig(*chclient.NewKernelConfig(kernelPath)) + if clh.config.ConfidentialGuest { + if err := clh.enableProtection(); err != nil { + return err + } + } + // Create the VM memory config via the constructor to ensure default values are properly assigned clh.vmconfig.Memory = chclient.NewMemoryConfig(int64((utils.MemUnit(clh.config.MemorySize) * utils.MiB).ToBytes())) // shared memory should be enabled if using vhost-user(kata uses virtiofsd)