s390x: add support for s390x

The PR adds the support for s390x.

In the case of CCW devices, the vhost-user devices are not supported.
See #659. An error message is thrown if they tried to be used.

Memory hotplug is not supported on s390 yet and an error message is thrown.

The VirtioNetPCI has been changed to VirtioNet. The generalization
allows to set the VirtioNet to the correct CCW device for s390x.

Fixes: #666

Co-authored-by: Yash D Jain ydjainopensource@gmail.com
Signed-off-by: Alice Frosi <afrosi@de.ibm.com>

arch/s390x-options.mk

@@ -0,0 +1,12 @@
+# Copyright (c) 2018 IBM
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# s390x settings
+
+MACHINETYPE := s390-ccw-virtio
+KERNELPARAMS :=
+MACHINEACCELERATORS :=
+
+QEMUCMD := qemu-system-s390x

cli/kata-check_data_s390x_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+const testCPUInfoTemplate = `
+vendor_id       : IBM/S390
+# processors    : 4
+bogomips per cpu: 20325.00
+max thread id   : 0
+features	: esan3 zarch stfle msa ldisp eimm dfp edat etf3eh highgprs te vx sie 
+cache0          : level=1 type=Data scope=Private size=128K line_size=256 associativity=8
+cache1          : level=1 type=Instruction scope=Private size=96K line_size=256 associativity=6
+cache2          : level=2 type=Data scope=Private size=2048K line_size=256 associativity=8
+cache3          : level=2 type=Instruction scope=Private size=2048K line_size=256 associativity=8
+cache4          : level=3 type=Unified scope=Shared size=65536K line_size=256 associativity=16
+cache5          : level=4 type=Unified scope=Shared size=491520K line_size=256 associativity=30
+processor 0: version = FF,  identification = FFFFFF,  machine = 2964
+`

cli/kata-check_s390x.go

@@ -0,0 +1,124 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	"github.com/sirupsen/logrus"
+	"strings"
+)
+
+const (
+	cpuFlagsTag        = genericCPUFlagsTag
+	archCPUVendorField = genericCPUVendorField
+	// On s390x the cpu model is indicated by the field machine.
+	// Example:
+	// processor 0: version = FF,  identification = 3FEC87,  machine = 2964
+	archCPUModelField = "machine"
+)
+
+// archRequiredCPUFlags maps a CPU flag value to search for and a
+// human-readable description of that value.
+var archRequiredCPUFlags = map[string]string{}
+
+// archRequiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
+// and a human-readable description of that value.
+var archRequiredCPUAttribs = map[string]string{}
+
+// archRequiredKernelModules maps a required module name to a human-readable
+// description of the modules functionality and an optional list of
+// required module parameters.
+var archRequiredKernelModules = map[string]kernelModule{
+	"kvm": {
+		desc: "Kernel-based Virtual Machine",
+	},
+}
+
+func setCPUtype() error {
+	return nil
+}
+
+// kvmIsUsable determines if it will be possible to create a full virtual machine
+// by creating a minimal VM and then deleting it.
+func kvmIsUsable() error {
+	return genericKvmIsUsable()
+}
+
+func archHostCanCreateVMContainer() error {
+	return kvmIsUsable()
+}
+
+// hostIsVMContainerCapable checks to see if the host is theoretically capable
+// of creating a VM container.
+func hostIsVMContainerCapable(details vmContainerCapableDetails) error {
+
+	_, err := getCPUInfo(details.cpuInfoFile)
+	if err != nil {
+		return err
+	}
+
+	count, err := checkKernelModules(details.requiredKernelModules, archKernelParamHandler)
+	if err != nil {
+		return err
+	}
+
+	if count == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("ERROR: %s", failMessage)
+
+}
+
+func archKernelParamHandler(onVMM bool, fields logrus.Fields, msg string) bool {
+	return genericArchKernelParamHandler(onVMM, fields, msg)
+}
+
+// getS390xCPUDetails returns the cpu information
+func getS390xCPUDetails() (vendor, model string, err error) {
+	prefixModel := "processor"
+	cpuinfo, err := getCPUInfo(procCPUInfo)
+	if err != nil {
+		return "", "", err
+	}
+
+	lines := strings.Split(cpuinfo, "\n")
+
+	for _, line := range lines {
+		if archCPUVendorField != "" {
+			if strings.HasPrefix(line, archCPUVendorField) {
+				fields := strings.Split(line, ":")
+				if len(fields) > 1 {
+					vendor = strings.TrimSpace(fields[1])
+				}
+			}
+		}
+		if archCPUModelField != "" {
+			if strings.HasPrefix(line, prefixModel) {
+				fields := strings.Split(strings.TrimSpace(line), ",")
+				cpuModel := strings.Split(fields[2], "=")
+				model = strings.TrimSpace(cpuModel[1])
+			}
+		}
+	}
+
+	if vendor == "" {
+		return "", "", fmt.Errorf("cannot find vendor field in file %v", procCPUInfo)
+	}
+
+	if model == "" {
+		return "", "", fmt.Errorf("Error in parsing cpu model from %v", procCPUInfo)
+	}
+
+	return vendor, model, nil
+}
+
+func getCPUDetails() (vendor, model string, err error) {
+	if vendor, model, err := genericGetCPUDetails(); err == nil {
+		return vendor, model, nil
+	}
+	return getS390xCPUDetails()
+}

cli/kata-check_s390x_test.go

@@ -0,0 +1,247 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+)
+
+func setupCheckHostIsVMContainerCapable(assert *assert.Assertions, cpuInfoFile string, cpuData []testCPUData, moduleData []testModuleData) {
+	createModules(assert, cpuInfoFile, moduleData)
+
+	// all the modules files have now been created, so deal with the
+	// cpuinfo data.
+	for _, d := range cpuData {
+		err := makeCPUInfoFile(cpuInfoFile, d.vendorID, d.flags)
+		assert.NoError(err)
+
+		details := vmContainerCapableDetails{
+			cpuInfoFile:           cpuInfoFile,
+			requiredCPUFlags:      archRequiredCPUFlags,
+			requiredCPUAttribs:    archRequiredCPUAttribs,
+			requiredKernelModules: archRequiredKernelModules,
+		}
+
+		err = hostIsVMContainerCapable(details)
+		if d.expectError {
+			assert.Error(err)
+		} else {
+			assert.NoError(err)
+		}
+	}
+}
+
+func TestCCCheckCLIFunction(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedSysModuleDir := sysModuleDir
+	savedProcCPUInfo := procCPUInfo
+
+	cpuInfoFile := filepath.Join(dir, "cpuinfo")
+
+	// XXX: override
+	sysModuleDir = filepath.Join(dir, "sys/module")
+	procCPUInfo = cpuInfoFile
+
+	defer func() {
+		sysModuleDir = savedSysModuleDir
+		procCPUInfo = savedProcCPUInfo
+	}()
+
+	err = os.MkdirAll(sysModuleDir, testDirMode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cpuData := []testCPUData{
+		{"", "", false},
+	}
+
+	moduleData := []testModuleData{
+		{filepath.Join(sysModuleDir, "kvm"), false, "Y"},
+	}
+
+	devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0666)
+	assert.NoError(err)
+	defer devNull.Close()
+
+	savedLogOutput := kataLog.Logger.Out
+
+	// discard normal output
+	kataLog.Logger.Out = devNull
+
+	defer func() {
+		kataLog.Logger.Out = savedLogOutput
+	}()
+
+	setupCheckHostIsVMContainerCapable(assert, cpuInfoFile, cpuData, moduleData)
+
+	ctx := createCLIContext(nil)
+	ctx.App.Name = "foo"
+
+	// create buffer to save logger output
+	buf := &bytes.Buffer{}
+
+	// capture output this time
+	kataLog.Logger.Out = buf
+
+	fn, ok := kataCheckCLICommand.Action.(func(context *cli.Context) error)
+	assert.True(ok)
+
+	err = fn(ctx)
+	assert.NoError(err)
+
+	output := buf.String()
+
+	for _, m := range moduleData {
+		name := path.Base(m.path)
+		assert.True(findAnchoredString(output, name))
+	}
+}
+
+func TestArchKernelParamHandler(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		onVMM        bool
+		fields       logrus.Fields
+		msg          string
+		expectIgnore bool
+	}
+
+	data := []testData{
+		{true, logrus.Fields{}, "", false},
+		{false, logrus.Fields{}, "", false},
+
+		{
+			false,
+			logrus.Fields{
+				// wrong type
+				"parameter": 123,
+			},
+			"foo",
+			false,
+		},
+
+		{
+			false,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+			false,
+		},
+
+		{
+			true,
+			logrus.Fields{
+				"parameter": "unrestricted_guest",
+			},
+			"",
+			true,
+		},
+
+		{
+			false,
+			logrus.Fields{
+				"parameter": "nested",
+			},
+			"",
+			true,
+		},
+	}
+
+	for i, d := range data {
+		result := archKernelParamHandler(d.onVMM, d.fields, d.msg)
+		if d.expectIgnore {
+			assert.True(result, "test %d (%+v)", i, d)
+		} else {
+			assert.False(result, "test %d (%+v)", i, d)
+		}
+	}
+}
+
+func TestKvmIsUsable(t *testing.T) {
+	assert := assert.New(t)
+
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	savedKvmDevice := kvmDevice
+	fakeKVMDevice := filepath.Join(dir, "kvm")
+	kvmDevice = fakeKVMDevice
+
+	defer func() {
+		kvmDevice = savedKvmDevice
+	}()
+
+	err = kvmIsUsable()
+	assert.Error(err)
+
+	err = createEmptyFile(fakeKVMDevice)
+	assert.NoError(err)
+
+	err = kvmIsUsable()
+	assert.Error(err)
+}
+
+type TestDataa struct {
+	contents       string
+	expectedVendor string
+	expectedModel  string
+	expectError    bool
+}
+
+func TestGetCPUDetails(t *testing.T) {
+	type testData struct {
+		contents       string
+		expectedVendor string
+		expectedModel  string
+		expectError    bool
+	}
+
+	const validVendorName = "a vendor"
+	validVendor := fmt.Sprintf(`%s  : %s`, archCPUVendorField, validVendorName)
+
+	const validModelName = "some CPU model"
+	validModel := fmt.Sprintf(`processor 0: version = 00,  identification = XXXXX, %s = %s`, archCPUModelField, validModelName)
+
+	validContents := fmt.Sprintf(`
+a       : b
+%s
+foo     : bar
+%s
+`, validVendor, validModel)
+
+	data := []TestDataa{
+		{"", "", "", true},
+		{"invalid", "", "", true},
+		{archCPUVendorField, "", "", true},
+		{validVendor, "", "", true},
+		{validModel, "", "", true},
+		{validContents, validVendorName, validModelName, false},
+	}
+	genericTestGetCPUDetails(t, validVendor, validModel, validContents, data)
+}

cli/kata-env_s390x_test.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"fmt"
+	vcUtils "github.com/kata-containers/runtime/virtcontainers/utils"
+	"path/filepath"
+	goruntime "runtime"
+)
+
+func getExpectedHostDetails(tmpdir string) (HostInfo, error) {
+	type filesToCreate struct {
+		file     string
+		contents string
+	}
+
+	const expectedKernelVersion = "99.1"
+	const expectedArch = goruntime.GOARCH
+
+	expectedDistro := DistroInfo{
+		Name:    "Foo",
+		Version: "42",
+	}
+
+	expectedCPU := CPUInfo{
+		Vendor: "moi",
+		Model:  "awesome XI",
+	}
+
+	expectedHostDetails := HostInfo{
+		Kernel:             expectedKernelVersion,
+		Architecture:       expectedArch,
+		Distro:             expectedDistro,
+		CPU:                expectedCPU,
+		VMContainerCapable: true,
+		SupportVSocks:      vcUtils.SupportsVsocks(),
+	}
+
+	testProcCPUInfo := filepath.Join(tmpdir, "cpuinfo")
+	testOSRelease := filepath.Join(tmpdir, "os-release")
+
+	// XXX: This file is *NOT* created by this function on purpose
+	// (to ensure the only file checked by the tests is
+	// testOSRelease). osReleaseClr handling is tested in
+	// utils_test.go.
+	testOSReleaseClr := filepath.Join(tmpdir, "os-release-clr")
+
+	testProcVersion := filepath.Join(tmpdir, "proc-version")
+
+	// override
+	procVersion = testProcVersion
+	osRelease = testOSRelease
+	osReleaseClr = testOSReleaseClr
+	procCPUInfo = testProcCPUInfo
+
+	procVersionContents := fmt.Sprintf("Linux version %s a b c",
+		expectedKernelVersion)
+
+	osReleaseContents := fmt.Sprintf(`
+NAME="%s"
+VERSION_ID="%s"
+`, expectedDistro.Name, expectedDistro.Version)
+
+	procCPUInfoContents := fmt.Sprintf(`
+%s	: %s
+processor 0: version = 00,  identification = 3929E7,  %s = %s
+`,
+		archCPUVendorField,
+		expectedCPU.Vendor,
+		archCPUModelField,
+		expectedCPU.Model)
+
+	data := []filesToCreate{
+		{procVersion, procVersionContents},
+		{osRelease, osReleaseContents},
+		{procCPUInfo, procCPUInfoContents},
+	}
+
+	for _, d := range data {
+		err := createFile(d.file, d.contents)
+		if err != nil {
+			return HostInfo{}, err
+		}
+	}
+
+	return expectedHostDetails, nil
+}

cli/utils_s390x.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+func archConvertStatFs(cgroupFsType int) uint32 {
+	return uint32(cgroupFsType)
+}

versions.yaml

@@ -83,6 +83,9 @@ assets:
       ppc64le:
         name: "centos"
         version: "latest"
+      s390x:
+        name: "ubuntu"
+        version: "latest"
       x86_64:
         name: &default-image-name "clearlinux"
         version: "20640"
@@ -101,6 +104,9 @@ assets:
       ppc64le:
         name: *default-initrd-name
         version: *default-initrd-version
+      s390x:
+        name: *default-initrd-name
+        version: *default-initrd-version
       x86_64:
         name: *default-initrd-name
         version: *default-initrd-version

virtcontainers/hypervisor.go

@@ -514,7 +514,7 @@ func getHostMemorySizeKb(memInfoPath string) (uint64, error) {
 
 // RunningOnVMM checks if the system is running inside a VM.
 func RunningOnVMM(cpuInfoPath string) (bool, error) {
-	if runtime.GOARCH == "arm64" || runtime.GOARCH == "ppc64le" {
+	if runtime.GOARCH == "arm64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" {
 		virtLog.Info("Unable to know if the system is running inside a VM")
 		return false, nil
 	}

virtcontainers/qemu.go

@@ -867,18 +867,20 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error {
 		return err
 	}
 	var tap TapInterface
+	devID := "virtio-" + tap.ID
+
+	switch endpoint.Type() {
+	case VethEndpointType:
+		drive := endpoint.(*VethEndpoint)
+		tap = drive.NetPair.TapInterface
+	case TapEndpointType:
+		drive := endpoint.(*TapEndpoint)
+		tap = drive.TapInterface
+	default:
+		return fmt.Errorf("this endpoint is not supported")
+	}
 
 	if op == addDevice {
-		switch endpoint.Type() {
-		case VethEndpointType:
-			drive := endpoint.(*VethEndpoint)
-			tap = drive.NetPair.TapInterface
-		case TapEndpointType:
-			drive := endpoint.(*TapEndpoint)
-			tap = drive.TapInterface
-		default:
-			return fmt.Errorf("this endpoint is not supported")
-		}
 
 		if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
 			return err
@@ -891,34 +893,28 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error {
 		pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr)
 		endpoint.SetPciAddr(pciAddr)
 
-		devID := "virtio-" + tap.ID
-		if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), q.arch.runNested()); err != nil {
+		var machine govmmQemu.Machine
+		machine, err = q.getQemuMachine()
+		if err != nil {
 			return err
 		}
-	} else {
-		switch endpoint.Type() {
-		case VethEndpointType:
-			drive := endpoint.(*VethEndpoint)
-			tap = drive.NetPair.TapInterface
-		case TapEndpointType:
-			drive := endpoint.(*TapEndpoint)
-			tap = drive.TapInterface
-		default:
-			return fmt.Errorf("this endpoint is not supported")
-		}
-
-		if err := q.removeDeviceFromBridge(tap.ID); err != nil {
-			return err
-		}
-
-		devID := "virtio-" + tap.ID
-		if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
-			return err
-		}
-		if err := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, tap.Name); err != nil {
-			return err
+		if machine.Type == QemuCCWVirtio {
+			return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, int(q.config.NumVCPUs))
 		}
+		return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), q.arch.runNested())
 	}
+
+	if err := q.removeDeviceFromBridge(tap.ID); err != nil {
+		return err
+	}
+
+	if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
+		return err
+	}
+	if err := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, tap.Name); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -1144,6 +1140,7 @@ func (q *qemu) resumeSandbox() error {
 
 // addDevice will add extra devices to Qemu command line.
 func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
+	var err error
 	span, _ := q.trace("addDevice")
 	defer span.Finish()
 
@@ -1160,14 +1157,14 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
 	case config.BlockDrive:
 		q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v)
 	case config.VhostUserDeviceAttrs:
-		q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, v)
+		q.qemuConfig.Devices, err = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, v)
 	case config.VFIODev:
 		q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v)
 	default:
 		break
 	}
 
-	return nil
+	return err
 }
 
 // getSandboxConsole builds the path of the console where we can read
@@ -1288,6 +1285,8 @@ func genericBridges(number uint32, machineType string) []Bridge {
 		bt = pcieBridge
 	case QemuPseries:
 		bt = pciBridge
+	case QemuCCWVirtio:
+		bt = pciBridge
 	default:
 		return nil
 	}

virtcontainers/qemu_arch_base.go

@@ -86,7 +86,7 @@ type qemuArch interface {
 	appendBlockDevice(devices []govmmQemu.Device, drive config.BlockDrive) []govmmQemu.Device
 
 	// appendVhostUserDevice appends a vhost user device to devices
-	appendVhostUserDevice(devices []govmmQemu.Device, drive config.VhostUserDeviceAttrs) []govmmQemu.Device
+	appendVhostUserDevice(devices []govmmQemu.Device, drive config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error)
 
 	// appendVFIODevice appends a VFIO device to devices
 	appendVFIODevice(devices []govmmQemu.Device, vfioDevice config.VFIODev) []govmmQemu.Device
@@ -151,6 +151,9 @@ const (
 
 	// QemuPseries is a QEMU virt machine type for ppc64le
 	QemuPseries = "pseries"
+
+	// QemuCCWVirtio is a QEMU virt machine type for for s390x
+	QemuCCWVirtio = "s390-ccw-virtio"
 )
 
 // kernelParamsNonDebug is a list of the default kernel
@@ -456,7 +459,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi
 		devices = append(devices,
 			govmmQemu.NetDevice{
 				Type:          networkModelToQemuType(netPair.NetInterworkingModel),
-				Driver:        govmmQemu.VirtioNetPCI,
+				Driver:        govmmQemu.VirtioNet,
 				ID:            fmt.Sprintf("network-%d", q.networkIndex),
 				IFName:        netPair.TAPIface.Name,
 				MACAddress:    netPair.TAPIface.HardAddr,
@@ -473,7 +476,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi
 		devices = append(devices,
 			govmmQemu.NetDevice{
 				Type:          govmmQemu.MACVTAP,
-				Driver:        govmmQemu.VirtioNetPCI,
+				Driver:        govmmQemu.VirtioNet,
 				ID:            fmt.Sprintf("network-%d", q.networkIndex),
 				IFName:        ep.Name(),
 				MACAddress:    ep.HardwareAddr(),
@@ -516,7 +519,7 @@ func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive confi
 	return devices
 }
 
-func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) []govmmQemu.Device {
+func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) {
 	qemuVhostUserDevice := govmmQemu.VhostUserDevice{}
 
 	switch attr.Type {
@@ -534,7 +537,7 @@ func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, attr co
 
 	devices = append(devices, qemuVhostUserDevice)
 
-	return devices
+	return devices, nil
 }
 
 func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDev config.VFIODev) []govmmQemu.Device {

virtcontainers/qemu_arch_base_test.go

@@ -199,6 +199,7 @@ func TestQemuArchBaseMemoryTopology(t *testing.T) {
 
 func testQemuArchBaseAppend(t *testing.T, structure interface{}, expected []govmmQemu.Device) {
 	var devices []govmmQemu.Device
+	var err error
 	assert := assert.New(t)
 	qemuArchBase := newQemuArchBase()
 
@@ -212,9 +213,10 @@ func testQemuArchBaseAppend(t *testing.T, structure interface{}, expected []govm
 	case config.VFIODev:
 		devices = qemuArchBase.appendVFIODevice(devices, s)
 	case config.VhostUserDeviceAttrs:
-		devices = qemuArchBase.appendVhostUserDevice(devices, s)
+		devices, err = qemuArchBase.appendVhostUserDevice(devices, s)
 	}
 
+	assert.NoError(err)
 	assert.Equal(devices, expected)
 }
 
@@ -471,7 +473,7 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) {
 	expectedOut := []govmmQemu.Device{
 		govmmQemu.NetDevice{
 			Type:       networkModelToQemuType(macvlanEp.NetPair.NetInterworkingModel),
-			Driver:     govmmQemu.VirtioNetPCI,
+			Driver:     govmmQemu.VirtioNet,
 			ID:         fmt.Sprintf("network-%d", 0),
 			IFName:     macvlanEp.NetPair.TAPIface.Name,
 			MACAddress: macvlanEp.NetPair.TAPIface.HardAddr,
@@ -482,7 +484,7 @@ func TestQemuArchBaseAppendNetwork(t *testing.T) {
 		},
 		govmmQemu.NetDevice{
 			Type:       govmmQemu.MACVTAP,
-			Driver:     govmmQemu.VirtioNetPCI,
+			Driver:     govmmQemu.VirtioNet,
 			ID:         fmt.Sprintf("network-%d", 1),
 			IFName:     macvtapEp.Name(),
 			MACAddress: macvtapEp.HardwareAddr(),

virtcontainers/qemu_s390x.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"fmt"
+	govmmQemu "github.com/intel/govmm/qemu"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
+)
+
+type qemuS390x struct {
+	// inherit from qemuArchBase, overwrite methods if needed
+	qemuArchBase
+}
+
+const defaultQemuPath = "/usr/bin/qemu-system-s390x"
+
+const defaultQemuMachineType = QemuCCWVirtio
+
+const defaultQemuMachineOptions = "accel=kvm"
+
+const defaultPCBridgeBus = "pci.0"
+
+const VirtioSerialCCW = "virtio-serial-ccw"
+
+var qemuPaths = map[string]string{
+	QemuCCWVirtio: defaultQemuPath,
+}
+
+var kernelRootParams = []Param{}
+
+// Verify needed parameters
+var kernelParams = []Param{
+	{"console", "ttysclp0"},
+}
+
+var supportedQemuMachines = []govmmQemu.Machine{
+	{
+		Type:    QemuCCWVirtio,
+		Options: defaultQemuMachineOptions,
+	},
+}
+
+// MaxQemuVCPUs returns the maximum number of vCPUs supported
+func MaxQemuVCPUs() uint32 {
+	// Max number of virtual Cpu defined in qemu. See
+	// https://github.com/qemu/qemu/blob/80422b00196a7af4c6efb628fae0ad8b644e98af/target/s390x/cpu.h#L55
+	// #define S390_MAX_CPUS 248
+	return uint32(248)
+}
+
+func newQemuArch(config HypervisorConfig) qemuArch {
+	machineType := config.HypervisorMachineType
+	if machineType == "" {
+		machineType = defaultQemuMachineType
+	}
+
+	q := &qemuS390x{
+		qemuArchBase{
+			machineType:           machineType,
+			qemuPaths:             qemuPaths,
+			supportedQemuMachines: supportedQemuMachines,
+			kernelParamsNonDebug:  kernelParamsNonDebug,
+			kernelParamsDebug:     kernelParamsDebug,
+			kernelParams:          kernelParams,
+		},
+	}
+
+	if config.ImagePath != "" {
+		q.kernelParams = append(q.kernelParams, kernelRootParams...)
+		q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...)
+		q.kernelParamsDebug = append(q.kernelParamsDebug, kernelParamsSystemdDebug...)
+	}
+
+	return q
+}
+
+// appendBridges appends to devices the given bridges
+func (q *qemuS390x) appendBridges(devices []govmmQemu.Device, bridges []Bridge) []govmmQemu.Device {
+	return genericAppendBridges(devices, bridges, q.machineType)
+}
+
+// appendConsole appends a console to devices.
+// The function has been overwriten to correctly set the driver to the CCW device
+func (q *qemuS390x) appendConsole(devices []govmmQemu.Device, path string) []govmmQemu.Device {
+	serial := govmmQemu.SerialDevice{
+		Driver:        VirtioSerialCCW,
+		ID:            "serial0",
+		DisableModern: q.nestedRun,
+	}
+
+	devices = append(devices, serial)
+
+	var console govmmQemu.CharDevice
+
+	console = govmmQemu.CharDevice{
+		Driver:   govmmQemu.Console,
+		Backend:  govmmQemu.Socket,
+		DeviceID: "console0",
+		ID:       "charconsole0",
+		Path:     path,
+	}
+
+	devices = append(devices, console)
+
+	return devices
+}
+
+// appendVhostUserDevice throws an error if vhost devices are tried to be used.
+// See issue https://github.com/kata-containers/runtime/issues/659
+func (q *qemuS390x) appendVhostUserDevice(devices []govmmQemu.Device, attr config.VhostUserDeviceAttrs) ([]govmmQemu.Device, error) {
+	return nil, fmt.Errorf("No vhost-user devices supported on s390x")
+}
+
+// supportGuestMemoryHotplug return false for s390x architecture. The pc-dimm backend device for s390x
+// is not support. PC-DIMM is not listed in the devices supported by qemu-system-s390x -device help
+func (q *qemuS390x) supportGuestMemoryHotplug() bool {
+	return false
+}

virtcontainers/qemu_s390x_test.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2018 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"fmt"
+	"testing"
+
+	govmmQemu "github.com/intel/govmm/qemu"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func newTestQemu(machineType string) qemuArch {
+	config := HypervisorConfig{
+		HypervisorMachineType: machineType,
+	}
+	return newQemuArch(config)
+}
+
+func TestQemuS390xCPUModel(t *testing.T) {
+	assert := assert.New(t)
+	s390x := newTestQemu(QemuCCWVirtio)
+
+	expectedOut := defaultCPUModel
+	model := s390x.cpuModel()
+	assert.Equal(expectedOut, model)
+
+	s390x.enableNestingChecks()
+	expectedOut = defaultCPUModel + ",pmu=off"
+	model = s390x.cpuModel()
+	assert.Equal(expectedOut, model)
+}
+
+func TestQemuS390xMemoryTopology(t *testing.T) {
+	assert := assert.New(t)
+	s390x := newTestQemu(QemuCCWVirtio)
+	memoryOffset := 1024
+
+	hostMem := uint64(1024)
+	mem := uint64(120)
+	slots := uint8(10)
+	expectedMemory := govmmQemu.Memory{
+		Size:   fmt.Sprintf("%dM", mem),
+		Slots:  slots,
+		MaxMem: fmt.Sprintf("%dM", hostMem+uint64(memoryOffset)),
+	}
+
+	m := s390x.memoryTopology(mem, hostMem, slots)
+	assert.Equal(expectedMemory, m)
+}
+
+func TestQemuS390xAppendVhostUserDevice(t *testing.T) {
+	macAddress := "00:11:22:33:44:55:66"
+	qemu := qemuS390x{}
+	assert := assert.New(t)
+
+	vhostUserDevice := config.VhostUserDeviceAttrs{
+		Type:       config.VhostUserNet,
+		MacAddress: macAddress,
+	}
+
+	_, err := qemu.appendVhostUserDevice(nil, vhostUserDevice)
+	assert.Error(err)
+}