virtcontainers: Add support for Secure Execution

Secure Execution is a confidential computing technology on s390x (IBM Z
& LinuxONE). Enable the correspondent virtualization technology in QEMU
(where it is referred to as "Protected Virtualization").

- Introduce enableProtection and appendProtectionDevice functions for
  QEMU s390x.
- Introduce CheckCmdline to check for "prot_virt=1" being present on the
  kernel command line.
- Introduce CPUFacilities and avilableGuestProtection for hypervisor
  s390x to check for CPU support.

Fixes: #1771

Signed-off-by: Jakob Naucke <jakob.naucke@ibm.com>

src/runtime/virtcontainers/hypervisor.go

@@ -722,6 +722,51 @@ func getHostMemorySizeKb(memInfoPath string) (uint64, error) {
 	return 0, fmt.Errorf("unable get MemTotal from %s", memInfoPath)
 }
 
+// CheckCmdline checks whether an option or parameter is present in the kernel command line.
+// Search is case-insensitive.
+// Takes path to file that contains the kernel command line, desired option, and permitted values
+// (empty values to check for options).
+func CheckCmdline(kernelCmdlinePath, searchParam string, searchValues []string) (bool, error) {
+	f, err := os.Open(kernelCmdlinePath)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	// Create check function -- either check for verbatim option
+	// or check for parameter and permitted values
+	var check func(string, string, []string) bool
+	if len(searchValues) == 0 {
+		check = func(option, searchParam string, _ []string) bool {
+			return strings.EqualFold(option, searchParam)
+		}
+	} else {
+		check = func(param, searchParam string, searchValues []string) bool {
+			// split parameter and value
+			split := strings.SplitN(param, "=", 2)
+			if len(split) < 2 || split[0] != searchParam {
+				return false
+			}
+			for _, value := range searchValues {
+				if strings.EqualFold(value, split[1]) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		for _, field := range strings.Fields(scanner.Text()) {
+			if check(field, searchParam, searchValues) {
+				return true, nil
+			}
+		}
+	}
+	return false, err
+}
+
 func CPUFlags(cpuInfoPath string) (map[string]bool, error) {
 	flagsField := "flags"
 

src/runtime/virtcontainers/hypervisor_s390x.go

@@ -0,0 +1,99 @@
+// Copyright (c) IBM Corp. 2021
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	// This is valid in other architectures, but varcheck will complain
+	// when setting it in common code as it will be regarded unused
+	procKernelCmdline = "/proc/cmdline"
+
+	// Secure Execution
+	// https://www.kernel.org/doc/html/latest/virt/kvm/s390-pv.html
+	seCPUFacilityBit = 158
+	seCmdlineParam   = "prot_virt"
+)
+
+var seCmdlineValues = []string{
+	"1", "on", "y", "yes",
+}
+
+// CPUFacilities retrieves active CPU facilities according to "Facility Indications", Principles of Operation.
+// Takes cpuinfo path (such as /proc/cpuinfo), returns map of all active facility bits.
+func CPUFacilities(cpuInfoPath string) (map[int]bool, error) {
+	facilitiesField := "facilities"
+
+	f, err := os.Open(cpuInfoPath)
+	if err != nil {
+		return map[int]bool{}, err
+	}
+	defer f.Close()
+
+	facilities := make(map[int]bool)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		// Expected format: ["facilities", ":", ...] or ["facilities:", ...]
+		fields := strings.Fields(scanner.Text())
+		if len(fields) < 2 {
+			continue
+		}
+
+		if !strings.HasPrefix(fields[0], facilitiesField) {
+			continue
+		}
+
+		start := 1
+		if fields[1] == ":" {
+			start = 2
+		}
+		for _, field := range fields[start:] {
+			bit, err := strconv.Atoi(field)
+			if err != nil {
+				return map[int]bool{}, err
+			}
+			facilities[bit] = true
+		}
+
+		return facilities, nil
+	}
+
+	if err := scanner.Err(); err != nil {
+		return map[int]bool{}, err
+	}
+
+	return map[int]bool{}, fmt.Errorf("Couldn't find %q from %q output", facilitiesField, cpuInfoPath)
+}
+
+// availableGuestProtection returns seProtection (Secure Execution) if available.
+// Checks that Secure Execution is available (CPU facilities) and enabled (kernel command line).
+func availableGuestProtection() (guestProtection, error) {
+	facilities, err := CPUFacilities(procCPUInfo)
+	if err != nil {
+		return noneProtection, err
+	}
+	if !facilities[seCPUFacilityBit] {
+		return noneProtection, fmt.Errorf("This CPU does not support Secure Execution")
+	}
+
+	seCmdlinePresent, err := CheckCmdline(procKernelCmdline, seCmdlineParam, seCmdlineValues)
+	if err != nil {
+		return noneProtection, err
+	}
+	if !seCmdlinePresent {
+		return noneProtection, fmt.Errorf("Protected Virtualization is not enabled on kernel command line! " +
+			"Need %s=%s (or %s) to enable Secure Execution",
+			seCmdlineParam, seCmdlineValues[0], strings.Join(seCmdlineValues[1:], ", "))
+	}
+
+	return seProtection, nil
+}

src/runtime/virtcontainers/hypervisor_s390x_test.go

@@ -0,0 +1,25 @@
+// Copyright (c) IBM Corp. 2021
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCPUFacilities(t *testing.T) {
+	assert := assert.New(t)
+
+	facilities, err := CPUFacilities(procCPUInfo)
+	assert.NoError(err)
+
+	// z/Architecture facility should always be active (introduced in 2000)
+	assert.Equal(facilities[1], true)
+	// facility bits should not be as high as MaxInt
+	assert.Equal(facilities[math.MaxInt64], false)
+}

src/runtime/virtcontainers/hypervisor_test.go

@@ -401,6 +401,21 @@ func TestGetHostMemorySizeKb(t *testing.T) {
 	}
 }
 
+func TestCheckCmdline(t *testing.T) {
+	assert := assert.New(t)
+
+	cmdlineFp, err := ioutil.TempFile("", "")
+	assert.NoError(err)
+	_, err = cmdlineFp.WriteString("quiet root=/dev/sda2")
+	assert.NoError(err)
+	cmdlinePath := cmdlineFp.Name()
+	defer os.Remove(cmdlinePath)
+
+	assert.True(CheckCmdline(cmdlinePath, "quiet", []string{}))
+	assert.True(CheckCmdline(cmdlinePath, "root", []string{"/dev/sda1", "/dev/sda2"}))
+	assert.False(CheckCmdline(cmdlinePath, "ro", []string{}))
+}
+
 // nolint: unused, deadcode
 type testNestedVMMData struct {
 	content     []byte

src/runtime/virtcontainers/qemu_amd64_test.go

@@ -299,6 +299,12 @@ func TestQemuAmd64AppendProtectionDevice(t *testing.T) {
 	assert.Error(err)
 	assert.Empty(bios)
 
+	// Secure Execution protection
+	amd64.(*qemuAmd64).protection = seProtection
+	devices, bios, err = amd64.appendProtectionDevice(devices, firmware)
+	assert.Error(err)
+	assert.Empty(bios)
+
 	// sev protection
 	// TODO: update once it's supported
 	amd64.(*qemuAmd64).protection = sevProtection

src/runtime/virtcontainers/qemu_arch_base.go

@@ -168,6 +168,10 @@ const (
 	// IBM POWER 9 Protected Execution Facility
 	// https://www.kernel.org/doc/html/latest/powerpc/ultravisor.html
 	pefProtection
+
+	// IBM Secure Execution (IBM Z & LinuxONE)
+	// https://www.kernel.org/doc/html/latest/virt/kvm/s390-pv.html
+	seProtection
 )
 
 type qemuArchBase struct {

src/runtime/virtcontainers/qemu_s390x.go

@@ -14,6 +14,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
+	"github.com/sirupsen/logrus"
 )
 
 type qemuS390x struct {
@@ -27,6 +28,11 @@ const (
 	defaultQemuMachineOptions = "accel=kvm"
 	virtioSerialCCW           = "virtio-serial-ccw"
 	qmpMigrationWaitTimeout   = 5 * time.Second
+	logSubsystem              = "qemuS390x"
+
+	// Secure Execution, also known as Protected Virtualization
+	// https://qemu.readthedocs.io/en/latest/system/s390x/protvirt.html
+	secExecID = "pv0"
 )
 
 // Verify needed parameters
@@ -72,6 +78,12 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) {
 	// Set first bridge type to CCW
 	q.Bridges = append(q.Bridges, ccwbridge)
 
+	if config.ConfidentialGuest {
+		if err := q.enableProtection(); err != nil {
+			return nil, err
+		}
+	}
+
 	if config.ImagePath != "" {
 		q.kernelParams = append(q.kernelParams, commonVirtioblkKernelRootParams...)
 		q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...)
@@ -301,3 +313,42 @@ func (q *qemuS390x) addDeviceToBridge(ctx context.Context, ID string, t types.Ty
 
 	return fmt.Sprintf("%04x", addr), b, nil
 }
+
+// enableProtection enables guest protection for QEMU's machine option.
+func (q *qemuS390x) enableProtection() error {
+	protection, err := availableGuestProtection()
+	if err != nil {
+		return err
+	}
+	if protection != seProtection {
+		return fmt.Errorf("Got unexpected protection %v, only seProtection (Secure Execution) is supported", protection)
+	}
+
+	q.protection = protection
+	if q.qemuMachine.Options != "" {
+		q.qemuMachine.Options += ","
+	}
+	q.qemuMachine.Options += fmt.Sprintf("confidential-guest-support=%s", secExecID)
+	virtLog.WithFields(logrus.Fields{
+		"subsystem": logSubsystem,
+		"machine":   q.qemuMachine}).
+		Info("Enabling guest protection with Secure Execution")
+	return nil
+}
+
+// appendProtectionDevice appends a QEMU object for Secure Execution.
+// Takes devices and returns updated version. Takes BIOS and returns it (no modification on s390x).
+func (q *qemuS390x) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) {
+	switch q.protection {
+	case seProtection:
+		return append(devices,
+			govmmQemu.Object{
+				Type: govmmQemu.SecExecGuest,
+				ID:   secExecID,
+			}), firmware, nil
+	case noneProtection:
+		return devices, firmware, nil
+	default:
+		return devices, firmware, fmt.Errorf("Unsupported guest protection technology: %v", q.protection)
+	}
+}

src/runtime/virtcontainers/qemu_s390x_test.go

@@ -101,3 +101,50 @@ func TestQemuS390xAppendVhostUserDevice(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(devices, expected)
 }
+
+func TestQemuS390xAppendProtectionDevice(t *testing.T) {
+	assert := assert.New(t)
+	s390x := newTestQemu(assert, QemuCCWVirtio)
+
+	var devices []govmmQemu.Device
+	var bios, firmware string
+	var err error
+	devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
+	assert.NoError(err)
+
+	// no protection
+	assert.Empty(bios)
+
+	// PEF protection
+	s390x.(*qemuS390x).protection = pefProtection
+	devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
+	assert.Error(err)
+	assert.Empty(bios)
+
+	// TDX protection
+	s390x.(*qemuS390x).protection = tdxProtection
+	devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
+	assert.Error(err)
+	assert.Empty(bios)
+
+	// SEV protection
+	s390x.(*qemuS390x).protection = sevProtection
+	devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
+	assert.Error(err)
+	assert.Empty(bios)
+
+	// Secure Execution protection
+	s390x.(*qemuS390x).protection = seProtection
+
+	devices, bios, err = s390x.appendProtectionDevice(devices, firmware)
+	assert.NoError(err)
+	assert.Empty(bios)
+
+	expectedOut := []govmmQemu.Device{
+		govmmQemu.Object{
+			Type: govmmQemu.SecExecGuest,
+			ID:   secExecID,
+		},
+	}
+	assert.Equal(expectedOut, devices)
+}