runtime: suppport split firmware

firmware can be split into FIRMWARE_VARS.fd (UEFI variables as
configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI
variables can be customized per each user while UEFI code is kept same.

fixes #3583

Signed-off-by: Julio Montes <julio.montes@intel.com>
This commit is contained in:
Julio Montes 2022-02-01 11:19:21 -06:00
parent 732c45de94
commit 1f29478b09
22 changed files with 140 additions and 42 deletions

View File

@ -62,6 +62,8 @@ There are several kinds of Kata configurations and they are listed below.
| `io.katacontainers.config.hypervisor.file_mem_backend` (R) | string | file based memory backend root directory |
| `io.katacontainers.config.hypervisor.firmware_hash` | string | container firmware SHA-512 hash value |
| `io.katacontainers.config.hypervisor.firmware` | string | the guest firmware that will run the container VM |
| `io.katacontainers.config.hypervisor.firmware_volume_hash` | string | container firmware volume SHA-512 hash value |
| `io.katacontainers.config.hypervisor.firmware_volume` | string | the guest firmware volume that will be passed to the container VM |
| `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
| `io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus` | `boolean` | indicate if devices need to be hotplugged on the root bus instead of a bridge|
| `io.katacontainers.config.hypervisor.hypervisor_hash` | string | container hypervisor binary SHA-512 hash value |

View File

@ -104,6 +104,7 @@ KERNELDIR := $(PKGDATADIR)
IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
FIRMWAREPATH :=
FIRMWAREVOLUMEPATH :=
# Name of default configuration file the runtime will use.
CONFIG_FILE = configuration.toml
@ -393,6 +394,7 @@ USER_VARS += KERNELPATH_CLH
USER_VARS += KERNELPATH_FC
USER_VARS += KERNELVIRTIOFSPATH
USER_VARS += FIRMWAREPATH
USER_VARS += FIRMWAREVOLUMEPATH
USER_VARS += MACHINEACCELERATORS
USER_VARS += CPUFEATURES
USER_VARS += DEFMACHINETYPE_CLH

View File

@ -56,6 +56,12 @@ kernel_params = "@KERNELPARAMS@"
# If you want that qemu uses the default firmware leave this option empty
firmware = "@FIRMWAREPATH@"
# Path to the firmware volume.
# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables
# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables
# can be customized per each user while UEFI code is kept same.
firmware_volume = "@FIRMWAREVOLUMEPATH@"
# Machine accelerators
# comma-separated list of machine accelerators to pass to the hypervisor.
# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`

View File

@ -266,6 +266,11 @@ type Object struct {
// File is the device file
File string
// FirmwareVolume is the configuration volume for the firmware
// it can be used to split the TDVF/OVMF UEFI firmware in UEFI variables
// and UEFI program image.
FirmwareVolume string
// CBitPos is the location of the C-bit in a guest page table entry
// This is only relevant for sev-guest objects
CBitPos uint32
@ -341,6 +346,9 @@ func (object Object) QemuParams(config *Config) []string {
deviceParams = append(deviceParams, string(object.Driver))
deviceParams = append(deviceParams, fmt.Sprintf("id=%s", object.DeviceID))
deviceParams = append(deviceParams, fmt.Sprintf("file=%s", object.File))
if object.FirmwareVolume != "" {
deviceParams = append(deviceParams, fmt.Sprintf("config-firmware-volume=%s", object.FirmwareVolume))
}
case SEVGuest:
objectParams = append(objectParams, string(object.Type))
objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))

View File

@ -45,6 +45,7 @@ var defaultImagePath = "/usr/share/kata-containers/kata-containers.img"
var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container"
var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
var defaultFirmwarePath = ""
var defaultFirmwareVolumePath = ""
var defaultMachineAccelerators = ""
var defaultCPUFeatures = ""
var systemdUnitName = "kata-containers.target"

View File

@ -1,4 +1,4 @@
// Copyright (c) 2018-2021 Intel Corporation
// Copyright (c) 2018-2022 Intel Corporation
// Copyright (c) 2018 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
@ -79,6 +79,7 @@ type hypervisor struct {
Initrd string `toml:"initrd"`
Image string `toml:"image"`
Firmware string `toml:"firmware"`
FirmwareVolume string `toml:"firmware_volume"`
MachineAccelerators string `toml:"machine_accelerators"`
CPUFeatures string `toml:"cpu_features"`
KernelParams string `toml:"kernel_params"`
@ -234,6 +235,19 @@ func (h hypervisor) firmware() (string, error) {
return ResolvePath(p)
}
func (h hypervisor) firmwareVolume() (string, error) {
p := h.FirmwareVolume
if p == "" {
if defaultFirmwareVolumePath == "" {
return "", nil
}
p = defaultFirmwareVolumePath
}
return ResolvePath(p)
}
func (h hypervisor) PFlash() ([]string, error) {
pflashes := h.PFlashList
@ -602,6 +616,11 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
return vc.HypervisorConfig{}, err
}
firmwareVolume, err := h.firmwareVolume()
if err != nil {
return vc.HypervisorConfig{}, err
}
machineAccelerators := h.machineAccelerators()
cpuFeatures := h.cpuFeatures()
kernelParams := h.kernelParams()
@ -644,6 +663,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
InitrdPath: initrd,
ImagePath: image,
FirmwarePath: firmware,
FirmwareVolumePath: firmwareVolume,
PFlash: pflashes,
MachineAccelerators: machineAccelerators,
CPUFeatures: cpuFeatures,
@ -998,6 +1018,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
ImagePath: defaultImagePath,
InitrdPath: defaultInitrdPath,
FirmwarePath: defaultFirmwarePath,
FirmwareVolumePath: defaultFirmwareVolumePath,
MachineAccelerators: defaultMachineAccelerators,
CPUFeatures: defaultCPUFeatures,
HypervisorMachineType: defaultMachineType,

View File

@ -1249,6 +1249,32 @@ func TestDefaultFirmware(t *testing.T) {
defaultFirmwarePath = oldDefaultFirmwarePath
}
func TestDefaultFirmwareVolume(t *testing.T) {
assert := assert.New(t)
// save default firmware path
oldDefaultFirmwareVolumePath := defaultFirmwareVolumePath
f, err := os.CreateTemp(os.TempDir(), "vol")
assert.NoError(err)
assert.NoError(f.Close())
defer os.RemoveAll(f.Name())
h := hypervisor{}
defaultFirmwareVolumePath = ""
p, err := h.firmwareVolume()
assert.NoError(err)
assert.Empty(p)
defaultFirmwareVolumePath = f.Name()
p, err = h.firmwareVolume()
assert.NoError(err)
assert.NotEmpty(p)
// restore default firmware volume path
defaultFirmwarePath = oldDefaultFirmwareVolumePath
}
func TestDefaultMachineAccelerators(t *testing.T) {
assert := assert.New(t)
machineAccelerators := "abc,123,rgb"
@ -1355,12 +1381,13 @@ func TestUpdateRuntimeConfigurationVMConfig(t *testing.T) {
tomlConf := tomlConfig{
Hypervisor: map[string]hypervisor{
qemuHypervisorTableType: {
NumVCPUs: int32(vcpus),
MemorySize: mem,
Path: "/",
Kernel: "/",
Image: "/",
Firmware: "/",
NumVCPUs: int32(vcpus),
MemorySize: mem,
Path: "/",
Kernel: "/",
Image: "/",
Firmware: "/",
FirmwareVolume: "/",
},
},
}

View File

@ -276,6 +276,9 @@ type HypervisorConfig struct {
// FirmwarePath is the bios host path
FirmwarePath string
// FirmwareVolumePath is the configuration volume path for the firmware
FirmwareVolumePath string
// MachineAccelerators are machine specific accelerators
MachineAccelerators string
@ -639,6 +642,8 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) {
return conf.JailerPath, nil
case types.FirmwareAsset:
return conf.FirmwarePath, nil
case types.FirmwareVolumeAsset:
return conf.FirmwareVolumePath, nil
default:
return "", fmt.Errorf("Unknown asset type %v", t)
}
@ -703,6 +708,11 @@ func (conf *HypervisorConfig) FirmwareAssetPath() (string, error) {
return conf.assetPath(types.FirmwareAsset)
}
// FirmwareVolumeAssetPath returns the guest firmware volume path
func (conf *HypervisorConfig) FirmwareVolumeAssetPath() (string, error) {
return conf.assetPath(types.FirmwareVolumeAsset)
}
func appendParam(params []Param, parameter string, value string) []Param {
return append(params, Param{parameter, value})
}

View File

@ -488,8 +488,9 @@ func TestAssetPath(t *testing.T) {
ImagePath: "/" + "io.katacontainers.config.hypervisor.image",
InitrdPath: "/" + "io.katacontainers.config.hypervisor.initrd",
FirmwarePath: "/" + "io.katacontainers.config.hypervisor.firmware",
JailerPath: "/" + "io.katacontainers.config.hypervisor.jailer_path",
FirmwarePath: "/" + "io.katacontainers.config.hypervisor.firmware",
FirmwareVolumePath: "/" + "io.katacontainers.config.hypervisor.firmware_volume",
JailerPath: "/" + "io.katacontainers.config.hypervisor.jailer_path",
}
for _, asset := range types.AssetTypes() {

View File

@ -55,6 +55,10 @@ const (
// FirmwarePath is a sandbox annotation for passing a per container path pointing at the guest firmware that will run the container VM.
FirmwarePath = kataAnnotHypervisorPrefix + "firmware"
// FirmwareVolumePath is a sandbox annotation for passing a per container path pointing at the guest firmware volume
// that will be passed to the container VM.
FirmwareVolumePath = kataAnnotHypervisorPrefix + "firmware_volume"
// KernelHash is a sandbox annotation for passing a container kernel image SHA-512 hash value.
KernelHash = kataAnnotHypervisorPrefix + "kernel_hash"
@ -76,6 +80,9 @@ const (
// FirmwareHash is an sandbox annotation for passing a container guest firmware SHA-512 hash value.
FirmwareHash = kataAnnotHypervisorPrefix + "firmware_hash"
// FirmwareVolumeHash is an sandbox annotation for passing a container guest firmware volume SHA-512 hash value.
FirmwareVolumeHash = kataAnnotHypervisorPrefix + "firmware_volume_hash"
// AssetHashType is the hash type used for assets verification
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"

View File

@ -575,6 +575,11 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa
return err
}
firmwareVolumePath, err := q.config.FirmwareVolumeAssetPath()
if err != nil {
return err
}
pflash, err := q.arch.getPFlash()
if err != nil {
return err
@ -610,7 +615,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa
PidFile: filepath.Join(q.config.VMStorePath, q.id, "pid"),
}
qemuConfig.Devices, qemuConfig.Bios, err = q.arch.appendProtectionDevice(qemuConfig.Devices, firmwarePath)
qemuConfig.Devices, qemuConfig.Bios, err = q.arch.appendProtectionDevice(qemuConfig.Devices, firmwarePath, firmwareVolumePath)
if err != nil {
return err
}

View File

@ -232,19 +232,20 @@ func (q *qemuAmd64) enableProtection() error {
}
// append protection device
func (q *qemuAmd64) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) {
func (q *qemuAmd64) appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error) {
switch q.protection {
case tdxProtection:
id := q.devLoadersCount
q.devLoadersCount += 1
return append(devices,
govmmQemu.Object{
Driver: govmmQemu.Loader,
Type: govmmQemu.TDXGuest,
ID: "tdx",
DeviceID: fmt.Sprintf("fd%d", id),
Debug: false,
File: firmware,
Driver: govmmQemu.Loader,
Type: govmmQemu.TDXGuest,
ID: "tdx",
DeviceID: fmt.Sprintf("fd%d", id),
Debug: false,
File: firmware,
FirmwareVolume: firmwareVolume,
}), "", nil
case sevProtection:
return append(devices,

View File

@ -252,7 +252,7 @@ func TestQemuAmd64AppendProtectionDevice(t *testing.T) {
firmware := "tdvf.fd"
var bios string
var err error
devices, bios, err = amd64.appendProtectionDevice(devices, firmware)
devices, bios, err = amd64.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
// non-protection
@ -260,20 +260,20 @@ func TestQemuAmd64AppendProtectionDevice(t *testing.T) {
// pef protection
amd64.(*qemuAmd64).protection = pefProtection
devices, bios, err = amd64.appendProtectionDevice(devices, firmware)
devices, bios, err = amd64.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
// Secure Execution protection
amd64.(*qemuAmd64).protection = seProtection
devices, bios, err = amd64.appendProtectionDevice(devices, firmware)
devices, bios, err = amd64.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
// sev protection
amd64.(*qemuAmd64).protection = sevProtection
devices, bios, err = amd64.appendProtectionDevice(devices, firmware)
devices, bios, err = amd64.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
assert.Empty(bios)
@ -293,7 +293,7 @@ func TestQemuAmd64AppendProtectionDevice(t *testing.T) {
// tdxProtection
amd64.(*qemuAmd64).protection = tdxProtection
devices, bios, err = amd64.appendProtectionDevice(devices, firmware)
devices, bios, err = amd64.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
assert.Empty(bios)

View File

@ -147,7 +147,7 @@ type qemuArch interface {
// This implementation is architecture specific, some archs may need
// a firmware, returns a string containing the path to the firmware that should
// be used with the -bios option, ommit -bios option if the path is empty.
appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error)
appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error)
}
// Kind of guest protection
@ -845,7 +845,7 @@ func (q *qemuArchBase) setPFlash(p []string) {
}
// append protection device
func (q *qemuArchBase) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) {
func (q *qemuArchBase) appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error) {
hvLogger.WithField("arch", runtime.GOARCH).Warnf("Confidential Computing has not been implemented for this architecture")
return devices, firmware, nil
}

View File

@ -169,7 +169,7 @@ func (q *qemuArm64) enableProtection() error {
return nil
}
func (q *qemuArm64) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) {
func (q *qemuArm64) appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error) {
err := q.enableProtection()
hvLogger.WithField("arch", runtime.GOARCH).Warnf("%v", err)
return devices, firmware, err

View File

@ -179,35 +179,35 @@ func TestQemuArm64AppendProtectionDevice(t *testing.T) {
var err error
// no protection
devices, bios, err = arm64.appendProtectionDevice(devices, firmware)
devices, bios, err = arm64.appendProtectionDevice(devices, firmware, "")
assert.Empty(devices)
assert.Empty(bios)
assert.NoError(err)
// PEF protection
arm64.(*qemuArm64).protection = pefProtection
devices, bios, err = arm64.appendProtectionDevice(devices, firmware)
devices, bios, err = arm64.appendProtectionDevice(devices, firmware, "")
assert.Empty(devices)
assert.Empty(bios)
assert.NoError(err)
// Secure Execution protection
arm64.(*qemuArm64).protection = seProtection
devices, bios, err = arm64.appendProtectionDevice(devices, firmware)
devices, bios, err = arm64.appendProtectionDevice(devices, firmware, "")
assert.Empty(devices)
assert.Empty(bios)
assert.NoError(err)
// SEV protection
arm64.(*qemuArm64).protection = sevProtection
devices, bios, err = arm64.appendProtectionDevice(devices, firmware)
devices, bios, err = arm64.appendProtectionDevice(devices, firmware, "")
assert.Empty(devices)
assert.Empty(bios)
assert.NoError(err)
// TDX protection
arm64.(*qemuArm64).protection = tdxProtection
devices, bios, err = arm64.appendProtectionDevice(devices, firmware)
devices, bios, err = arm64.appendProtectionDevice(devices, firmware, "")
assert.Empty(devices)
assert.Empty(bios)
assert.NoError(err)

View File

@ -154,7 +154,7 @@ func (q *qemuPPC64le) enableProtection() error {
}
// append protection device
func (q *qemuPPC64le) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) {
func (q *qemuPPC64le) appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error) {
switch q.protection {
case pefProtection:
return append(devices,

View File

@ -58,7 +58,7 @@ func TestQemuPPC64leAppendProtectionDevice(t *testing.T) {
var devices []govmmQemu.Device
var bios, firmware string
var err error
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware)
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
//no protection
@ -66,25 +66,25 @@ func TestQemuPPC64leAppendProtectionDevice(t *testing.T) {
//Secure Execution protection
ppc64le.(*qemuPPC64le).protection = seProtection
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware)
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
//SEV protection
ppc64le.(*qemuPPC64le).protection = sevProtection
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware)
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
//TDX protection
ppc64le.(*qemuPPC64le).protection = tdxProtection
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware)
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
//PEF protection
ppc64le.(*qemuPPC64le).protection = pefProtection
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware)
devices, bios, err = ppc64le.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
assert.Empty(bios)

View File

@ -333,7 +333,7 @@ func (q *qemuS390x) enableProtection() error {
// appendProtectionDevice appends a QEMU object for Secure Execution.
// Takes devices and returns updated version. Takes BIOS and returns it (no modification on s390x).
func (q *qemuS390x) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) {
func (q *qemuS390x) appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error) {
switch q.protection {
case seProtection:
return append(devices,

View File

@ -109,7 +109,7 @@ func TestQemuS390xAppendProtectionDevice(t *testing.T) {
var devices []govmmQemu.Device
var bios, firmware string
var err error
devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
devices, bios, err = s390x.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
// no protection
@ -117,26 +117,26 @@ func TestQemuS390xAppendProtectionDevice(t *testing.T) {
// PEF protection
s390x.(*qemuS390x).protection = pefProtection
devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
devices, bios, err = s390x.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
// TDX protection
s390x.(*qemuS390x).protection = tdxProtection
devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
devices, bios, err = s390x.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
// SEV protection
s390x.(*qemuS390x).protection = sevProtection
devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
devices, bios, err = s390x.appendProtectionDevice(devices, firmware, "")
assert.Error(err)
assert.Empty(bios)
// Secure Execution protection
s390x.(*qemuS390x).protection = seProtection
devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
devices, bios, err = s390x.appendProtectionDevice(devices, firmware, "")
assert.NoError(err)
assert.Empty(bios)

View File

@ -39,6 +39,8 @@ const (
// FirmwareAsset is a firmware asset.
FirmwareAsset AssetType = "firmware"
FirmwareVolumeAsset AssetType = "firmware_volume"
)
// AssetTypes returns a list of all known asset types.
@ -47,6 +49,7 @@ const (
func AssetTypes() []AssetType {
return []AssetType{
FirmwareAsset,
FirmwareVolumeAsset,
HypervisorAsset,
HypervisorCtlAsset,
ImageAsset,
@ -89,6 +92,8 @@ func (t AssetType) Annotations() (string, string, error) {
return annotations.JailerPath, annotations.JailerHash, nil
case FirmwareAsset:
return annotations.FirmwarePath, annotations.FirmwareHash, nil
case FirmwareVolumeAsset:
return annotations.FirmwareVolumePath, annotations.FirmwareVolumeHash, nil
}
return "", "", fmt.Errorf("Wrong asset type %s", t)

View File

@ -118,6 +118,7 @@ func TestAssetNew(t *testing.T) {
{annotations.HypervisorCtlPath, annotations.HypervisorCtlHash, HypervisorCtlAsset, assetContentHash, false, false},
{annotations.JailerPath, annotations.JailerHash, JailerAsset, assetContentHash, false, false},
{annotations.FirmwarePath, annotations.FirmwareHash, FirmwareAsset, assetContentHash, false, false},
{annotations.FirmwareVolumePath, annotations.FirmwareVolumeHash, FirmwareVolumeAsset, assetContentHash, false, false},
// Failure with incorrect hash
{annotations.KernelPath, annotations.KernelHash, KernelAsset, assetContentWrongHash, true, false},
@ -127,6 +128,7 @@ func TestAssetNew(t *testing.T) {
{annotations.HypervisorCtlPath, annotations.HypervisorCtlHash, HypervisorCtlAsset, assetContentWrongHash, true, false},
{annotations.JailerPath, annotations.JailerHash, JailerAsset, assetContentWrongHash, true, false},
{annotations.FirmwarePath, annotations.FirmwareHash, FirmwareAsset, assetContentWrongHash, true, false},
{annotations.FirmwareVolumePath, annotations.FirmwareVolumeHash, FirmwareVolumeAsset, assetContentWrongHash, true, false},
// Other failures
{annotations.KernelPath, annotations.KernelHash, ImageAsset, assetContentHash, false, true},