Merge pull request #362 from bergwolf/runtime-port-4

Runtime port  -- wave 3

src/runtime/Makefile

@@ -176,8 +176,13 @@ DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs
 DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd
 # Default DAX mapping cache size in MiB
 DEFVIRTIOFSCACHESIZE := 1024
-DEFVIRTIOFSCACHE := always
+DEFVIRTIOFSCACHE ?= auto
-DEFVIRTIOFSEXTRAARGS := []
+# Format example:
+#   [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"]
+#
+# see `virtiofsd -h` for possible options.
+# Make sure you quote args.
+DEFVIRTIOFSEXTRAARGS ?= []
 DEFENABLEIOTHREADS := false
 DEFENABLEMEMPREALLOC := false
 DEFENABLEHUGEPAGES := false
@@ -398,6 +403,7 @@ USER_VARS += KERNELTYPE_ACRN
 USER_VARS += KERNELTYPE_CLH
 USER_VARS += FIRMWAREPATH
 USER_VARS += MACHINEACCELERATORS
+USER_VARS += CPUFEATURES
 USER_VARS += DEFMACHINETYPE_CLH
 USER_VARS += KERNELPARAMS
 USER_VARS += LIBEXECDIR
@@ -606,6 +612,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 		-e "s|@INITRDPATH@|$(INITRDPATH)|g" \
 		-e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \
 		-e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \
+		-e "s|@CPUFEATURES@|$(CPUFEATURES)|g" \
 		-e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \
 		-e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \
 		-e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \

src/runtime/arch/amd64-options.mk

@@ -8,6 +8,7 @@
 MACHINETYPE := pc
 KERNELPARAMS :=
 MACHINEACCELERATORS :=
+CPUFEATURES := pmu=off
 
 QEMUCMD := qemu-system-x86_64
 

src/runtime/arch/arm64-options.mk

@@ -8,6 +8,7 @@
 MACHINETYPE := virt
 KERNELPARAMS :=
 MACHINEACCELERATORS :=
+CPUFEATURES := pmu=off
 
 QEMUCMD := qemu-system-aarch64
 

src/runtime/arch/ppc64le-options.mk

@@ -8,5 +8,6 @@
 MACHINETYPE := pseries
 KERNELPARAMS :=
 MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off"
+CPUFEATURES :=
 KERNELTYPE := uncompressed #This architecture must use an uncompressed kernel.
 QEMUCMD := qemu-system-ppc64

src/runtime/arch/s390x-options.mk

@@ -8,5 +8,6 @@
 MACHINETYPE := s390-ccw-virtio
 KERNELPARAMS :=
 MACHINEACCELERATORS :=
+CPUFEATURES :=
 
 QEMUCMD := qemu-system-s390x

src/runtime/cli/config/configuration-clh.toml.in

@@ -65,8 +65,33 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
 # Default size of DAX cache in MiB
 virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
 
-# cloud-hypervisor prefers virtiofs caching (dax) for performance reasons
+# Extra args for virtiofsd daemon
-virtio_fs_cache = "always"
+#
+# Format example:
+#   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+#
+# see `virtiofsd -h` for possible options.
+virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
+# Block storage driver to be used for the hypervisor in case the container
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
+# or nvdimm.
+block_device_driver = "virtio-blk"
 
 # This option changes the default hypervisor and kernel parameters
 # to enable debug output where available. This extra output is added

src/runtime/cli/config/configuration-qemu-virtiofs.toml.in

@@ -37,6 +37,11 @@ firmware = "@FIRMWAREPATH@"
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
 machine_accelerators="@MACHINEACCELERATORS@"
 
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
 # Default number of vCPUs per SB/VM:
 # unspecified or 0                --> will be set to @DEFVCPUS@
 # < 0                             --> will be set to the actual number of physical cores

src/runtime/cli/config/configuration-qemu.toml.in

@@ -38,6 +38,11 @@ firmware = "@FIRMWAREPATH@"
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
 machine_accelerators="@MACHINEACCELERATORS@"
 
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
 # Default number of vCPUs per SB/VM:
 # unspecified or 0                --> will be set to @DEFVCPUS@
 # < 0                             --> will be set to the actual number of physical cores

src/runtime/cli/kata-env.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"errors"
 	"os"
-	runtim "runtime"
 	"strings"
 
 	"github.com/BurntSushi/toml"
@@ -223,9 +222,6 @@ func getHostInfo() (HostInfo, error) {
 	}
 
 	hostVMContainerCapable := true
-	if runtim.GOARCH == "ppc64le" {
-		hostVMContainerCapable = false
-	}
 
 	details := vmContainerCapableDetails{
 		cpuInfoFile:           procCPUInfo,

src/runtime/cli/kata-env_ppc64le_test.go

@@ -10,7 +10,7 @@ import "testing"
 func getExpectedHostDetails(tmpdir string) (HostInfo, error) {
 	expectedVendor := ""
 	expectedModel := "POWER8"
-	expectedVMContainerCapable := false
+	expectedVMContainerCapable := true
 	return genericGetExpectedHostDetails(tmpdir, expectedVendor, expectedModel, expectedVMContainerCapable)
 }
 

src/runtime/pkg/katautils/config-settings.go.in

@@ -16,6 +16,7 @@ var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container"
 var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
 var defaultFirmwarePath = ""
 var defaultMachineAccelerators = ""
+var defaultCPUFeatures = ""
 var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
 var systemdUnitName = "kata-containers.target"
 

src/runtime/pkg/katautils/config.go

@@ -93,6 +93,7 @@ type hypervisor struct {
 	Image                   string   `toml:"image"`
 	Firmware                string   `toml:"firmware"`
 	MachineAccelerators     string   `toml:"machine_accelerators"`
+	CPUFeatures             string   `toml:"cpu_features"`
 	KernelParams            string   `toml:"kernel_params"`
 	MachineType             string   `toml:"machine_type"`
 	BlockDeviceDriver       string   `toml:"block_device_driver"`
@@ -244,11 +245,9 @@ func (h hypervisor) firmware() (string, error) {
 
 func (h hypervisor) machineAccelerators() string {
 	var machineAccelerators string
-	accelerators := strings.Split(h.MachineAccelerators, ",")
+	for _, accelerator := range strings.Split(h.MachineAccelerators, ",") {
-	acceleratorsLen := len(accelerators)
+		if accelerator != "" {
-	for i := 0; i < acceleratorsLen; i++ {
+			machineAccelerators += strings.TrimSpace(accelerator) + ","
-		if accelerators[i] != "" {
-			machineAccelerators += strings.Trim(accelerators[i], "\r\t\n ") + ","
 		}
 	}
 
@@ -257,6 +256,19 @@ func (h hypervisor) machineAccelerators() string {
 	return machineAccelerators
 }
 
+func (h hypervisor) cpuFeatures() string {
+	var cpuFeatures string
+	for _, feature := range strings.Split(h.CPUFeatures, ",") {
+		if feature != "" {
+			cpuFeatures += strings.TrimSpace(feature) + ","
+		}
+	}
+
+	cpuFeatures = strings.Trim(cpuFeatures, ",")
+
+	return cpuFeatures
+}
+
 func (h hypervisor) kernelParams() string {
 	if h.KernelParams == "" {
 		return defaultKernelParams
@@ -624,6 +636,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}
 
 	machineAccelerators := h.machineAccelerators()
+	cpuFeatures := h.cpuFeatures()
 	kernelParams := h.kernelParams()
 	machineType := h.machineType()
 
@@ -677,6 +690,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		ImagePath:               image,
 		FirmwarePath:            firmware,
 		MachineAccelerators:     machineAccelerators,
+		CPUFeatures:             cpuFeatures,
 		KernelParams:            vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType:   machineType,
 		NumVCPUs:                h.defaultVCPUs(),
@@ -865,6 +879,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         true,
 		UseVSock:                true,
+		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 	}, nil
 }
 
@@ -1129,6 +1144,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		InitrdPath:              defaultInitrdPath,
 		FirmwarePath:            defaultFirmwarePath,
 		MachineAccelerators:     defaultMachineAccelerators,
+		CPUFeatures:             defaultCPUFeatures,
 		HypervisorMachineType:   defaultMachineType,
 		NumVCPUs:                defaultVCPUCount,
 		DefaultMaxVCPUs:         defaultMaxVCPUCount,

src/runtime/pkg/katautils/config_test.go

@@ -1604,6 +1604,53 @@ func TestDefaultMachineAccelerators(t *testing.T) {
 	assert.Equal(machineAccelerators, h.machineAccelerators())
 }
 
+func TestDefaultCPUFeatures(t *testing.T) {
+	assert := assert.New(t)
+	cpuFeatures := "abc,123,rgb"
+	h := hypervisor{CPUFeatures: cpuFeatures}
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = ""
+	h.CPUFeatures = cpuFeatures
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = cpuFeatures
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = "abc,,123"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = ",,abc,,123,,,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = "abc,,123,,,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = ",,abc,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = ", , abc , ,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = " abc "
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = ", abc , 123 ,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = ",, abc ,,, 123 ,,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+}
+
 func TestUpdateRuntimeConfiguration(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/virtcontainers/clh.go

@@ -6,10 +6,11 @@
 package virtcontainers
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -59,9 +60,7 @@ const (
 	clhStopSandboxTimeout = 3
 	clhSocket             = "clh.sock"
 	clhAPISocket          = "clh-api.sock"
-	clhLogFile            = "clh.log"
 	virtioFsSocket        = "virtiofsd.sock"
-	clhSerial             = "serial-tty.log"
 	supportedMajorVersion = 0
 	supportedMinorVersion = 5
 	defaultClhPath        = "/usr/local/bin/cloud-hypervisor"
@@ -88,6 +87,8 @@ type clhClient interface {
 	VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error)
 	// Add VFIO PCI device to the VM
 	VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error)
+	// Add a new disk device to the VM
+	VmAddDiskPut(ctx context.Context, diskConfig chclient.DiskConfig) (*http.Response, error)
 }
 
 type CloudHypervisorVersion struct {
@@ -120,7 +121,6 @@ type cloudHypervisor struct {
 	APIClient clhClient
 	version   CloudHypervisorVersion
 	vmconfig  chclient.VmConfig
-	cmdOutput bytes.Buffer
 	virtiofsd Virtiofsd
 	store     persistapi.PersistDriver
 }
@@ -131,7 +131,6 @@ var clhKernelParams = []Param{
 	{"panic", "1"},         // upon kernel panic wait 1 second before reboot
 	{"no_timer_check", ""}, // do not check broken timer IRQ resources
 	{"noreplace-smp", ""},  // do not replace SMP instructions
-	{"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, // tell the agent where to send the logs
 	{"rootflags", "data=ordered,errors=remount-ro ro"}, // mount the root filesystem as readonly
 	{"rootfstype", "ext4"},
 }
@@ -139,9 +138,7 @@ var clhKernelParams = []Param{
 var clhDebugKernelParams = []Param{
 
 	{"console", "ttyS0,115200n8"},     // enable serial console
-	{"systemd.log_level", "debug"},    // enable systemd debug output
 	{"systemd.log_target", "console"}, // send loggng to the console
-	{"initcall_debug", "1"},           // print init call timing information to the console
 }
 
 //###########################################################
@@ -286,13 +283,8 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ
 
 	// set the serial console to the cloud hypervisor
 	if clh.config.Debug {
-		serialPath, err := clh.serialPath(clh.id)
-		if err != nil {
-			return err
-		}
 		clh.vmconfig.Serial = chclient.ConsoleConfig{
-			Mode: cctFILE,
+			Mode: cctTTY,
-			File: serialPath,
 		}
 
 	} else {
@@ -368,17 +360,12 @@ func (clh *cloudHypervisor) startSandbox(timeout int) error {
 	var strErr string
 	strErr, pid, err := clh.LaunchClh()
 	if err != nil {
-		return fmt.Errorf("failed to launch cloud-hypervisor: %s, error messages from log: %s", err, strErr)
-	}
-	clh.state.PID = pid
-
-	if err := clh.waitVMM(clhTimeout); err != nil {
-		clh.Logger().WithField("error", err).WithField("output", clh.cmdOutput.String()).Warn("cloud-hypervisor init failed")
 		if shutdownErr := clh.virtiofsd.Stop(); shutdownErr != nil {
 			clh.Logger().WithField("error", shutdownErr).Warn("error shutting down Virtiofsd")
 		}
-		return err
+		return fmt.Errorf("failed to launch cloud-hypervisor: %q, hypervisor output:\n%s", err, strErr)
 	}
+	clh.state.PID = pid
 
 	if err := clh.bootVM(ctx); err != nil {
 		return err
@@ -410,6 +397,41 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) {
 	return vcpuInfo, nil
 }
 
+func (clh *cloudHypervisor) hotplugBlockDevice(drive *config.BlockDrive) error {
+	if clh.config.BlockDeviceDriver != config.VirtioBlock {
+		return fmt.Errorf("incorrect hypervisor configuration on 'block_device_driver':"+
+			" using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock)
+	}
+
+	cl := clh.client()
+	ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second)
+	defer cancel()
+
+	_, _, err := cl.VmmPingGet(ctx)
+	if err != nil {
+		return openAPIClientError(err)
+	}
+
+	//Explicitly set PCIAddr to NULL, so that VirtPath can be used
+	drive.PCIAddr = ""
+
+	if drive.Pmem {
+		err = fmt.Errorf("pmem device hotplug not supported")
+	} else {
+		blkDevice := chclient.DiskConfig{
+			Path:      drive.File,
+			Readonly:  drive.ReadOnly,
+			VhostUser: false,
+		}
+		_, err = cl.VmAddDiskPut(ctx, blkDevice)
+	}
+
+	if err != nil {
+		err = fmt.Errorf("failed to hotplug block device %+v %s", drive, openAPIClientError(err))
+	}
+	return err
+}
+
 func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error {
 	cl := clh.client()
 	ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second)
@@ -432,6 +454,9 @@ func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType device
 	defer span.Finish()
 
 	switch devType {
+	case blockDev:
+		drive := devInfo.(*config.BlockDrive)
+		return nil, clh.hotplugBlockDevice(drive)
 	case vfioDev:
 		device := devInfo.(*config.VFIODev)
 		return nil, clh.hotPlugVFIODevice(*device)
@@ -663,6 +688,7 @@ func (clh *cloudHypervisor) capabilities() types.Capabilities {
 	clh.Logger().WithField("function", "capabilities").Info("get Capabilities")
 	var caps types.Capabilities
 	caps.SetFsSharingSupport()
+	caps.SetBlockDeviceHotplugSupport()
 	return caps
 }
 
@@ -771,18 +797,10 @@ func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) {
 	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSocket)
 }
 
-func (clh *cloudHypervisor) serialPath(id string) (string, error) {
-	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSerial)
-}
-
 func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) {
 	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhAPISocket)
 }
 
-func (clh *cloudHypervisor) logFilePath(id string) (string, error) {
-	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhLogFile)
-}
-
 func (clh *cloudHypervisor) waitVMM(timeout uint) error {
 	clhRunning, err := clh.isClhRunning(timeout)
 
@@ -873,8 +891,6 @@ func (clh *cloudHypervisor) getAvailableVersion() error {
 
 func (clh *cloudHypervisor) LaunchClh() (string, int, error) {
 
-	errStr := ""
-
 	clhPath, err := clh.clhPath()
 	if err != nil {
 		return "", -1, err
@@ -882,36 +898,71 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) {
 
 	args := []string{cscAPIsocket, clh.state.apiSocket}
 	if clh.config.Debug {
-
+		// Cloud hypervisor log levels
-		logfile, err := clh.logFilePath(clh.id)
+		// 'v' occurrences increase the level
-		if err != nil {
+		//0 =>  Error
-			return "", -1, err
+		//1 =>  Warn
-		}
+		//2 =>  Info
-		args = append(args, cscLogFile)
+		//3 =>  Debug
-		args = append(args, logfile)
+		//4+ => Trace
+		// Use Info, the CI runs with debug enabled
+		// a high level of logging increases the boot time
+		// and in a nested environment this could increase
+		// the chances to fail because agent is not
+		// ready on time.
+		args = append(args, "-vv")
 	}
 
 	clh.Logger().WithField("path", clhPath).Info()
 	clh.Logger().WithField("args", strings.Join(args, " ")).Info()
 
-	cmd := exec.Command(clhPath, args...)
+	cmdHypervisor := exec.Command(clhPath, args...)
-	cmd.Stdout = &clh.cmdOutput
+	var hypervisorOutput io.ReadCloser
-	cmd.Stderr = &clh.cmdOutput
+	if clh.config.Debug {
+		cmdHypervisor.Env = os.Environ()
+		cmdHypervisor.Env = append(cmdHypervisor.Env, "RUST_BACKTRACE=full")
+		// Get  StdoutPipe only for debug, without debug golang will redirect to /dev/null
+		hypervisorOutput, err = cmdHypervisor.StdoutPipe()
+		if err != nil {
+			return "", -1, err
+		}
+	}
+
+	cmdHypervisor.Stderr = cmdHypervisor.Stdout
+
+	err = utils.StartCmd(cmdHypervisor)
+	if err != nil {
+		return "", -1, err
+	}
+
+	if err := clh.waitVMM(clhTimeout); err != nil {
+		clh.Logger().WithField("error", err).Warn("cloud-hypervisor init failed")
+		var output string
+
+		if hypervisorOutput != nil {
+			b, errRead := ioutil.ReadAll(hypervisorOutput)
+			if errRead != nil {
+				output = "failed to read hypervisor output to get error information"
+			} else {
+				output = string(b)
+			}
+		} else {
+			output = "Please enable hypervisor logging to get stdout information"
+		}
+
+		return output, -1, err
+	}
 
 	if clh.config.Debug {
-		cmd.Env = os.Environ()
+		cmdLogger := utils.NewProgramLogger("kata-hypervisor")
-		cmd.Env = append(cmd.Env, "RUST_BACKTRACE=full")
+		clh.Logger().Debugf("Starting process logger(%s) for hypervisor", cmdLogger)
+		if err := cmdLogger.StartLogger(hypervisorOutput); err != nil {
+			// Not critical to run a container, but output wont be logged
+			clh.Logger().Warnf("Failed start process logger(%s) %s", cmdLogger, err)
+		}
 	}
 
-	if err := utils.StartCmd(cmd); err != nil {
+	return "", cmdHypervisor.Process.Pid, nil
-		fmt.Println("Error starting cloudHypervisor", err)
-		if cmd.Process != nil {
-			cmd.Process.Kill()
-		}
-		return errStr, 0, err
-	}
-
-	return errStr, cmd.Process.Pid, nil
 }
 
 //###########################################################################
@@ -922,13 +973,12 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) {
 
 const (
 	cctOFF  string = "Off"
-	cctFILE string = "File"
 	cctNULL string = "Null"
+	cctTTY  string = "Tty"
 )
 
 const (
 	cscAPIsocket string = "--api-socket"
-	cscLogFile   string = "--log-file"
 )
 
 //****************************************
@@ -1083,7 +1133,7 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) {
 		"cid":  cid,
 	}).Info("Adding HybridVSock")
 
-	clh.vmconfig.Vsock = chclient.VsockConfig{Cid: cid, Sock: path}
+	clh.vmconfig.Vsock = chclient.VsockConfig{Cid: cid, Socket: path}
 }
 
 func (clh *cloudHypervisor) addNet(e Endpoint) error {
@@ -1127,14 +1177,14 @@ func (clh *cloudHypervisor) addVolume(volume types.Volume) error {
 			{
 				Tag:       volume.MountTag,
 				CacheSize: int64(clh.config.VirtioFSCacheSize << 20),
-				Sock:      vfsdSockPath,
+				Socket:    vfsdSockPath,
 			},
 		}
 	} else {
 		clh.vmconfig.Fs = []chclient.FsConfig{
 			{
 				Tag:    volume.MountTag,
-				Sock: vfsdSockPath,
+				Socket: vfsdSockPath,
 			},
 		}
 

src/runtime/virtcontainers/clh_test.go

@@ -99,13 +99,18 @@ func (c *clhClientMock) VmAddDevicePut(ctx context.Context, vmAddDevice chclient
 	return nil, nil
 }
 
+//nolint:golint
+func (c *clhClientMock) VmAddDiskPut(ctx context.Context, diskConfig chclient.DiskConfig) (*http.Response, error) {
+	return nil, nil
+}
+
 func TestCloudHypervisorAddVSock(t *testing.T) {
 	assert := assert.New(t)
 	clh := cloudHypervisor{}
 
 	clh.addVSock(1, "path")
 	assert.Equal(clh.vmconfig.Vsock.Cid, int64(1))
-	assert.Equal(clh.vmconfig.Vsock.Sock, "path")
+	assert.Equal(clh.vmconfig.Vsock.Socket, "path")
 }
 
 // Check addNet appends to the network config list new configurations.
@@ -357,3 +362,25 @@ func TestCheckVersion(t *testing.T) {
 		}
 	}
 }
+
+func TestCloudHypervisorHotplugBlockDevice(t *testing.T) {
+	assert := assert.New(t)
+
+	clhConfig, err := newClhConfig()
+	assert.NoError(err)
+
+	clh := &cloudHypervisor{}
+	clh.config = clhConfig
+	clh.APIClient = &clhClientMock{}
+
+	clh.config.BlockDeviceDriver = config.VirtioBlock
+	err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: false})
+	assert.NoError(err, "Hotplug disk block device expected no error")
+
+	err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: true})
+	assert.Error(err, "Hotplug pmem block device expected error")
+
+	clh.config.BlockDeviceDriver = config.VirtioSCSI
+	err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: false})
+	assert.Error(err, "Hotplug block device not using 'virtio-blk' expected error")
+}

src/runtime/virtcontainers/documentation/api/1.0/api.md

@@ -132,6 +132,9 @@ type HypervisorConfig struct {
 	// MachineAccelerators are machine specific accelerators
 	MachineAccelerators string
 
+	// CPUFeatures are cpu specific features
+	CPUFeatures string
+
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 

src/runtime/virtcontainers/hypervisor.go

@@ -275,6 +275,9 @@ type HypervisorConfig struct {
 	// MachineAccelerators are machine specific accelerators
 	MachineAccelerators string
 
+	// CPUFeatures are cpu specific features
+	CPUFeatures string
+
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 

src/runtime/virtcontainers/kata_agent.go

@@ -110,6 +110,7 @@ const (
 	grpcUpdateInterfaceRequest   = "grpc.UpdateInterfaceRequest"
 	grpcListInterfacesRequest    = "grpc.ListInterfacesRequest"
 	grpcListRoutesRequest        = "grpc.ListRoutesRequest"
+	grpcAddARPNeighborsRequest   = "grpc.AddARPNeighborsRequest"
 	grpcOnlineCPUMemRequest      = "grpc.OnlineCPUMemRequest"
 	grpcListProcessesRequest     = "grpc.ListProcessesRequest"
 	grpcUpdateContainerRequest   = "grpc.UpdateContainerRequest"
@@ -638,6 +639,30 @@ func (k *kataAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, err
 	return nil, nil
 }
 
+func (k *kataAgent) addARPNeighbors(neighs []*vcTypes.ARPNeighbor) error {
+	if neighs != nil {
+		neighsReq := &grpc.AddARPNeighborsRequest{
+			Neighbors: &grpc.ARPNeighbors{
+				ARPNeighbors: k.convertToKataAgentNeighbors(neighs),
+			},
+		}
+		_, err := k.sendReq(neighsReq)
+		if err != nil {
+			if grpcStatus.Convert(err).Code() == codes.Unimplemented {
+				k.Logger().WithFields(logrus.Fields{
+					"arpneighbors-requested": fmt.Sprintf("%+v", neighs),
+				}).Warn("add ARP neighbors request failed due to old agent, please upgrade Kata Containers image version")
+				return nil
+			}
+			k.Logger().WithFields(logrus.Fields{
+				"arpneighbors-requested": fmt.Sprintf("%+v", neighs),
+			}).WithError(err).Error("add ARP neighbors request failed")
+		}
+		return err
+	}
+	return nil
+}
+
 func (k *kataAgent) listInterfaces() ([]*vcTypes.Interface, error) {
 	req := &grpc.ListInterfacesRequest{}
 	resultingInterfaces, err := k.sendReq(req)
@@ -843,7 +868,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	//
 	// Setup network interfaces and routes
 	//
-	interfaces, routes, err := generateInterfacesAndRoutes(sandbox.networkNS)
+	interfaces, routes, neighs, err := generateVCNetworkStructures(sandbox.networkNS)
 	if err != nil {
 		return err
 	}
@@ -853,6 +878,9 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	if _, err = k.updateRoutes(routes); err != nil {
 		return err
 	}
+	if err = k.addARPNeighbors(neighs); err != nil {
+		return err
+	}
 
 	storages := setupStorages(sandbox)
 
@@ -1193,6 +1221,7 @@ func (k *kataAgent) appendBlockDevice(dev ContainerDevice, c *Container) *grpc.D
 	case config.VirtioBlock:
 		kataDevice.Type = kataBlkDevType
 		kataDevice.Id = d.PCIAddr
+		kataDevice.VmPath = d.VirtPath
 	case config.VirtioSCSI:
 		kataDevice.Type = kataSCSIDevType
 		kataDevice.Id = d.SCSIAddr
@@ -1559,7 +1588,11 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*g
 		vol.Source = blockDrive.DevNo
 	case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock:
 		vol.Driver = kataBlkDevType
+		if blockDrive.PCIAddr == "" {
+			vol.Source = blockDrive.VirtPath
+		} else {
 			vol.Source = blockDrive.PCIAddr
+		}
 	case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio:
 		vol.Driver = kataMmioBlkDevType
 		vol.Source = blockDrive.VirtPath
@@ -1999,6 +2032,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers[grpcListRoutesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.ListRoutes(ctx, req.(*grpc.ListRoutesRequest))
 	}
+	k.reqHandlers[grpcAddARPNeighborsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.AddARPNeighbors(ctx, req.(*grpc.AddARPNeighborsRequest))
+	}
 	k.reqHandlers[grpcOnlineCPUMemRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest))
 	}
@@ -2175,18 +2211,27 @@ func (k *kataAgent) convertToIPFamily(ipFamily aTypes.IPFamily) int {
 	return netlink.FAMILY_V4
 }
 
+func (k *kataAgent) convertToKataAgentIPAddress(ipAddr *vcTypes.IPAddress) (aIPAddr *aTypes.IPAddress) {
+	if ipAddr == nil {
+		return nil
+	}
+
+	aIPAddr = &aTypes.IPAddress{
+		Family:  k.convertToKataAgentIPFamily(ipAddr.Family),
+		Address: ipAddr.Address,
+		Mask:    ipAddr.Mask,
+	}
+
+	return aIPAddr
+}
+
 func (k *kataAgent) convertToKataAgentIPAddresses(ipAddrs []*vcTypes.IPAddress) (aIPAddrs []*aTypes.IPAddress) {
 	for _, ipAddr := range ipAddrs {
 		if ipAddr == nil {
 			continue
 		}
 
-		aIPAddr := &aTypes.IPAddress{
+		aIPAddr := k.convertToKataAgentIPAddress(ipAddr)
-			Family:  k.convertToKataAgentIPFamily(ipAddr.Family),
-			Address: ipAddr.Address,
-			Mask:    ipAddr.Mask,
-		}
-
 		aIPAddrs = append(aIPAddrs, aIPAddr)
 	}
 
@@ -2268,6 +2313,25 @@ func (k *kataAgent) convertToKataAgentRoutes(routes []*vcTypes.Route) (aRoutes [
 	return aRoutes
 }
 
+func (k *kataAgent) convertToKataAgentNeighbors(neighs []*vcTypes.ARPNeighbor) (aNeighs []*aTypes.ARPNeighbor) {
+	for _, neigh := range neighs {
+		if neigh == nil {
+			continue
+		}
+
+		aNeigh := &aTypes.ARPNeighbor{
+			ToIPAddress: k.convertToKataAgentIPAddress(neigh.ToIPAddress),
+			Device:      neigh.Device,
+			State:       int32(neigh.State),
+			Lladdr:      neigh.LLAddr,
+		}
+
+		aNeighs = append(aNeighs, aNeigh)
+	}
+
+	return aNeighs
+}
+
 func (k *kataAgent) convertToRoutes(aRoutes []*aTypes.Route) (routes []*vcTypes.Route) {
 	for _, aRoute := range aRoutes {
 		if aRoute == nil {

src/runtime/virtcontainers/kata_agent_test.go

@@ -39,8 +39,13 @@ import (
 
 var (
 	testKataProxyURLTempl  = "unix://%s/kata-proxy-test.sock"
+	testBlkDriveFormat     = "testBlkDriveFormat"
 	testBlockDeviceCtrPath = "testBlockDeviceCtrPath"
+	testDevNo              = "testDevNo"
+	testNvdimmID           = "testNvdimmID"
 	testPCIAddr            = "04/02"
+	testSCSIAddr           = "testSCSIAddr"
+	testVirtPath           = "testVirtPath"
 )
 
 func testGenerateKataProxySockDir() (string, error) {
@@ -398,6 +403,110 @@ func TestHandleLocalStorage(t *testing.T) {
 	assert.Equal(t, localMountPoint, expected)
 }
 
+func TestHandleDeviceBlockVolume(t *testing.T) {
+	k := kataAgent{}
+
+	tests := []struct {
+		BlockDeviceDriver string
+		inputDev          *drivers.BlockDevice
+		resultVol         *pb.Storage
+	}{
+		{
+			inputDev: &drivers.BlockDevice{
+				BlockDrive: &config.BlockDrive{
+					Pmem:     true,
+					NvdimmID: testNvdimmID,
+					Format:   testBlkDriveFormat,
+				},
+			},
+			resultVol: &pb.Storage{
+				Driver:  kataNvdimmDevType,
+				Source:  fmt.Sprintf("/dev/pmem%s", testNvdimmID),
+				Fstype:  testBlkDriveFormat,
+				Options: []string{"dax"},
+			},
+		},
+		{
+			BlockDeviceDriver: config.VirtioBlockCCW,
+			inputDev: &drivers.BlockDevice{
+				BlockDrive: &config.BlockDrive{
+					DevNo: testDevNo,
+				},
+			},
+			resultVol: &pb.Storage{
+				Driver: kataBlkCCWDevType,
+				Source: testDevNo,
+			},
+		},
+		{
+			BlockDeviceDriver: config.VirtioBlock,
+			inputDev: &drivers.BlockDevice{
+				BlockDrive: &config.BlockDrive{
+					PCIAddr:  testPCIAddr,
+					VirtPath: testVirtPath,
+				},
+			},
+			resultVol: &pb.Storage{
+				Driver: kataBlkDevType,
+				Source: testPCIAddr,
+			},
+		},
+		{
+			BlockDeviceDriver: config.VirtioBlock,
+			inputDev: &drivers.BlockDevice{
+				BlockDrive: &config.BlockDrive{
+					VirtPath: testVirtPath,
+				},
+			},
+			resultVol: &pb.Storage{
+				Driver: kataBlkDevType,
+				Source: testVirtPath,
+			},
+		},
+		{
+			BlockDeviceDriver: config.VirtioMmio,
+			inputDev: &drivers.BlockDevice{
+				BlockDrive: &config.BlockDrive{
+					VirtPath: testVirtPath,
+				},
+			},
+			resultVol: &pb.Storage{
+				Driver: kataMmioBlkDevType,
+				Source: testVirtPath,
+			},
+		},
+		{
+			BlockDeviceDriver: config.VirtioSCSI,
+			inputDev: &drivers.BlockDevice{
+				BlockDrive: &config.BlockDrive{
+					SCSIAddr: testSCSIAddr,
+				},
+			},
+			resultVol: &pb.Storage{
+				Driver: kataSCSIDevType,
+				Source: testSCSIAddr,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		c := &Container{
+			sandbox: &Sandbox{
+				config: &SandboxConfig{
+					HypervisorConfig: HypervisorConfig{
+						BlockDeviceDriver: test.BlockDeviceDriver,
+					},
+				},
+			},
+		}
+
+		vol, _ := k.handleDeviceBlockVolume(c, test.inputDev)
+		assert.True(t, reflect.DeepEqual(vol, test.resultVol),
+			"Volume didn't match: got %+v, expecting %+v",
+			vol, test.resultVol)
+	}
+}
+
 func TestHandleBlockVolume(t *testing.T) {
 	k := kataAgent{}
 

src/runtime/virtcontainers/network.go

@@ -122,6 +122,7 @@ type NetworkInfo struct {
 	Addrs     []netlink.Addr
 	Routes    []netlink.Route
 	DNS       DNSInfo
+	Neighbors []netlink.Neigh
 }
 
 // NetworkInterface defines a network interface.
@@ -942,14 +943,15 @@ func deleteNetNS(netNSPath string) error {
 	return nil
 }
 
-func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) {
+func generateVCNetworkStructures(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, []*vcTypes.ARPNeighbor, error) {
 
 	if networkNS.NetNsPath == "" {
-		return nil, nil, nil
+		return nil, nil, nil, nil
 	}
 
 	var routes []*vcTypes.Route
 	var ifaces []*vcTypes.Interface
+	var neighs []*vcTypes.ARPNeighbor
 
 	for _, endpoint := range networkNS.Endpoints {
 
@@ -1008,10 +1010,36 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa
 			r.Device = endpoint.Name()
 			r.Scope = uint32(route.Scope)
 			routes = append(routes, &r)
+		}
 
+		for _, neigh := range endpoint.Properties().Neighbors {
+			var n vcTypes.ARPNeighbor
+
+			// We add only static ARP entries
+			if neigh.State != netlink.NUD_PERMANENT {
+				continue
+			}
+
+			n.Device = endpoint.Name()
+			n.State = neigh.State
+			n.Flags = neigh.Flags
+
+			if neigh.HardwareAddr != nil {
+				n.LLAddr = neigh.HardwareAddr.String()
+			}
+
+			n.ToIPAddress = &vcTypes.IPAddress{
+				Family:  netlink.FAMILY_V4,
+				Address: neigh.IP.String(),
+			}
+			if neigh.IP.To4() == nil {
+				n.ToIPAddress.Family = netlink.FAMILY_V6
+			}
+
+			neighs = append(neighs, &n)
 		}
 	}
-	return ifaces, routes, nil
+	return ifaces, routes, neighs, nil
 }
 
 func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) {
@@ -1071,6 +1099,11 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo
 		return NetworkInfo{}, err
 	}
 
+	neighbors, err := handle.NeighList(link.Attrs().Index, netlink.FAMILY_ALL)
+	if err != nil {
+		return NetworkInfo{}, err
+	}
+
 	return NetworkInfo{
 		Iface: NetlinkIface{
 			LinkAttrs: *(link.Attrs()),
@@ -1078,6 +1111,7 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo
 		},
 		Addrs:     addrs,
 		Routes:    routes,
+		Neighbors: neighbors,
 	}, nil
 }
 

src/runtime/virtcontainers/network_test.go

@@ -65,6 +65,12 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) {
 		{LinkIndex: 329, Dst: nil, Src: nil, Gw: gatewayV6},
 	}
 
+	arpMAC, _ := net.ParseMAC("6a:92:3a:59:70:aa")
+
+	neighs := []netlink.Neigh{
+		{LinkIndex: 329, IP: net.IPv4(192, 168, 0, 101), State: netlink.NUD_PERMANENT, HardwareAddr: arpMAC},
+	}
+
 	networkInfo := NetworkInfo{
 		Iface: NetlinkIface{
 			LinkAttrs: netlink.LinkAttrs{MTU: 1500},
@@ -72,6 +78,7 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) {
 		},
 		Addrs:     addrs,
 		Routes:    routes,
+		Neighbors: neighs,
 	}
 
 	ep0 := &PhysicalEndpoint{
@@ -84,7 +91,7 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) {
 
 	nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints}
 
-	resInterfaces, resRoutes, err := generateInterfacesAndRoutes(nns)
+	resInterfaces, resRoutes, resNeighs, err := generateVCNetworkStructures(nns)
 
 	//
 	// Build expected results:
@@ -106,6 +113,15 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) {
 		{Dest: "", Gateway: "2001:db8:1::1", Device: "eth0", Source: ""},
 	}
 
+	expectedNeighs := []*vcTypes.ARPNeighbor{
+		{
+			Device:      "eth0",
+			State:       netlink.NUD_PERMANENT,
+			LLAddr:      "6a:92:3a:59:70:aa",
+			ToIPAddress: &vcTypes.IPAddress{Address: "192.168.0.101", Family: netlink.FAMILY_V4},
+		},
+	}
+
 	for _, r := range resRoutes {
 		fmt.Printf("resRoute: %+v\n", r)
 	}
@@ -115,7 +131,8 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) {
 		"Interfaces returned didn't match: got %+v, expecting %+v", resInterfaces, expectedInterfaces)
 	assert.True(t, reflect.DeepEqual(resRoutes, expectedRoutes),
 		"Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes)
-
+	assert.True(t, reflect.DeepEqual(resNeighs, expectedNeighs),
+		"ARP Neighbors returned didn't match: got %+v, expecting %+v", resNeighs, expectedNeighs)
 }
 
 func TestNetInterworkingModelIsValid(t *testing.T) {

src/runtime/virtcontainers/persist.go

@@ -221,6 +221,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		InitrdPath:              sconfig.HypervisorConfig.InitrdPath,
 		FirmwarePath:            sconfig.HypervisorConfig.FirmwarePath,
 		MachineAccelerators:     sconfig.HypervisorConfig.MachineAccelerators,
+		CPUFeatures:             sconfig.HypervisorConfig.CPUFeatures,
 		HypervisorPath:          sconfig.HypervisorConfig.HypervisorPath,
 		HypervisorCtlPath:       sconfig.HypervisorConfig.HypervisorCtlPath,
 		JailerPath:              sconfig.HypervisorConfig.JailerPath,
@@ -512,6 +513,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		InitrdPath:              hconf.InitrdPath,
 		FirmwarePath:            hconf.FirmwarePath,
 		MachineAccelerators:     hconf.MachineAccelerators,
+		CPUFeatures:             hconf.CPUFeatures,
 		HypervisorPath:          hconf.HypervisorPath,
 		HypervisorCtlPath:       hconf.HypervisorCtlPath,
 		JailerPath:              hconf.JailerPath,

src/runtime/virtcontainers/persist/api/config.go

@@ -54,6 +54,9 @@ type HypervisorConfig struct {
 	// MachineAccelerators are machine specific accelerators
 	MachineAccelerators string
 
+	// CPUFeatures are cpu specific features
+	CPUFeatures string
+
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -81,6 +81,9 @@ const (
 	// MachineAccelerators is a sandbox annotation to specify machine specific accelerators for the hypervisor.
 	MachineAccelerators = kataAnnotHypervisorPrefix + "machine_accelerators"
 
+	// CPUFeatures is a sandbox annotation to specify cpu specific features.
+	CPUFeatures = kataAnnotHypervisorPrefix + "cpu_features"
+
 	// DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host.
 	DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net"
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml

@@ -306,7 +306,6 @@ components:
             vhost_socket: vhost_socket
             vhost_user: false
             direct: false
-            wce: true
             poll_queue: true
             id: id
           - path: path
@@ -317,7 +316,6 @@ components:
             vhost_socket: vhost_socket
             vhost_user: false
             direct: false
-            wce: true
             poll_queue: true
             id: id
           cpus:
@@ -336,23 +334,23 @@ components:
             iommu: false
             src: /dev/urandom
           fs:
-          - sock: sock
+          - num_queues: 3
-            num_queues: 3
             queue_size: 2
             cache_size: 4
             dax: true
             tag: tag
+            socket: socket
             id: id
-          - sock: sock
+          - num_queues: 3
-            num_queues: 3
             queue_size: 2
             cache_size: 4
             dax: true
             tag: tag
+            socket: socket
             id: id
           vsock:
-            sock: sock
             iommu: false
+            socket: socket
             id: id
             cid: 3
           pmem:
@@ -436,7 +434,6 @@ components:
           vhost_socket: vhost_socket
           vhost_user: false
           direct: false
-          wce: true
           poll_queue: true
           id: id
         - path: path
@@ -447,7 +444,6 @@ components:
           vhost_socket: vhost_socket
           vhost_user: false
           direct: false
-          wce: true
           poll_queue: true
           id: id
         cpus:
@@ -466,23 +462,23 @@ components:
           iommu: false
           src: /dev/urandom
         fs:
-        - sock: sock
+        - num_queues: 3
-          num_queues: 3
           queue_size: 2
           cache_size: 4
           dax: true
           tag: tag
+          socket: socket
           id: id
-        - sock: sock
+        - num_queues: 3
-          num_queues: 3
           queue_size: 2
           cache_size: 4
           dax: true
           tag: tag
+          socket: socket
           id: id
         vsock:
-          sock: sock
           iommu: false
+          socket: socket
           id: id
           cid: 3
         pmem:
@@ -662,7 +658,6 @@ components:
         vhost_socket: vhost_socket
         vhost_user: false
         direct: false
-        wce: true
         poll_queue: true
         id: id
       properties:
@@ -688,9 +683,6 @@ components:
           type: boolean
         vhost_socket:
           type: string
-        wce:
-          default: true
-          type: boolean
         poll_queue:
           default: true
           type: boolean
@@ -756,17 +748,17 @@ components:
       type: object
     FsConfig:
       example:
-        sock: sock
         num_queues: 3
         queue_size: 2
         cache_size: 4
         dax: true
         tag: tag
+        socket: socket
         id: id
       properties:
         tag:
           type: string
-        sock:
+        socket:
           type: string
         num_queues:
           default: 1
@@ -783,7 +775,7 @@ components:
         id:
           type: string
       required:
-      - sock
+      - socket
       - tag
       type: object
     PmemConfig:
@@ -853,8 +845,8 @@ components:
       type: object
     VsockConfig:
       example:
-        sock: sock
         iommu: false
+        socket: socket
         id: id
         cid: 3
       properties:
@@ -863,7 +855,7 @@ components:
           format: int64
           minimum: 3
           type: integer
-        sock:
+        socket:
           description: Path to UNIX domain socket, used to proxy vsock connections.
           type: string
         iommu:
@@ -873,7 +865,7 @@ components:
           type: string
       required:
       - cid
-      - sock
+      - socket
       type: object
     VmResize:
       example:

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md

@@ -12,7 +12,6 @@ Name | Type | Description | Notes
 **QueueSize** | **int32** |  | [optional] [default to 128]
 **VhostUser** | **bool** |  | [optional] [default to false]
 **VhostSocket** | **string** |  | [optional] 
-**Wce** | **bool** |  | [optional] [default to true]
 **PollQueue** | **bool** |  | [optional] [default to true]
 **Id** | **string** |  | [optional]
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md

@@ -5,7 +5,7 @@
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **Tag** | **string** |  | 
-**Sock** | **string** |  | 
+**Socket** | **string** |  |
 **NumQueues** | **int32** |  | [optional] [default to 1]
 **QueueSize** | **int32** |  | [optional] [default to 1024]
 **Dax** | **bool** |  | [optional] [default to true]

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md

@@ -5,7 +5,7 @@
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **Cid** | **int64** | Guest Vsock CID | 
-**Sock** | **string** | Path to UNIX domain socket, used to proxy vsock connections. | 
+**Socket** | **string** | Path to UNIX domain socket, used to proxy vsock connections. |
 **Iommu** | **bool** |  | [optional] [default to false]
 **Id** | **string** |  | [optional]
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go

@@ -18,7 +18,6 @@ type DiskConfig struct {
 	QueueSize int32 `json:"queue_size,omitempty"`
 	VhostUser bool `json:"vhost_user,omitempty"`
 	VhostSocket string `json:"vhost_socket,omitempty"`
-	Wce bool `json:"wce,omitempty"`
 	PollQueue bool `json:"poll_queue,omitempty"`
 	Id string `json:"id,omitempty"`
 }

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go

@@ -11,7 +11,7 @@ package openapi
 // FsConfig struct for FsConfig
 type FsConfig struct {
 	Tag string `json:"tag"`
-	Sock string `json:"sock"`
+	Socket string `json:"socket"`
 	NumQueues int32 `json:"num_queues,omitempty"`
 	QueueSize int32 `json:"queue_size,omitempty"`
 	Dax bool `json:"dax,omitempty"`

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go

@@ -13,7 +13,7 @@ type VsockConfig struct {
 	// Guest Vsock CID
 	Cid int64 `json:"cid"`
 	// Path to UNIX domain socket, used to proxy vsock connections.
-	Sock string `json:"sock"`
+	Socket string `json:"socket"`
 	Iommu bool `json:"iommu,omitempty"`
 	Id string `json:"id,omitempty"`
 }

src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml

@@ -454,9 +454,6 @@ components:
           default: false
         vhost_socket:
           type: string
-        wce:
-          type: boolean
-          default: true
         poll_queue:
           type: boolean
           default: true
@@ -509,12 +506,12 @@ components:
     FsConfig:
       required:
       - tag
-      - sock
+      - socket
       type: object
       properties:
         tag:
           type: string
-        sock:
+        socket:
           type: string
         num_queues:
           type: integer
@@ -584,7 +581,7 @@ components:
     VsockConfig:
       required:
       - cid
-      - sock
+      - socket
       type: object
       properties:
         cid:
@@ -592,7 +589,7 @@ components:
           format: int64
           minimum: 3
           description: Guest Vsock CID
-        sock:
+        socket:
           type: string
           description: Path to UNIX domain socket, used to proxy vsock connections.
         iommu:

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -700,6 +700,12 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon
 }
 
 func addHypervisporNetworkOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
+	if value, ok := ocispec.Annotations[vcAnnotations.CPUFeatures]; ok {
+		if value != "" {
+			sbConfig.HypervisorConfig.CPUFeatures = value
+		}
+	}
+
 	if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok {
 		disableVhostNet, err := strconv.ParseBool(value)
 		if err != nil {

src/runtime/virtcontainers/pkg/oci/utils_test.go

@@ -784,6 +784,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.Msize9p] = "512"
 	ocispec.Annotations[vcAnnotations.MachineType] = "q35"
 	ocispec.Annotations[vcAnnotations.MachineAccelerators] = "nofw"
+	ocispec.Annotations[vcAnnotations.CPUFeatures] = "pmu=off"
 	ocispec.Annotations[vcAnnotations.DisableVhostNet] = "true"
 	ocispec.Annotations[vcAnnotations.GuestHookPath] = "/usr/bin/"
 	ocispec.Annotations[vcAnnotations.UseVSock] = "true"
@@ -819,6 +820,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(config.HypervisorConfig.Msize9p, uint32(512))
 	assert.Equal(config.HypervisorConfig.HypervisorMachineType, "q35")
 	assert.Equal(config.HypervisorConfig.MachineAccelerators, "nofw")
+	assert.Equal(config.HypervisorConfig.CPUFeatures, "pmu=off")
 	assert.Equal(config.HypervisorConfig.DisableVhostNet, true)
 	assert.Equal(config.HypervisorConfig.GuestHookPath, "/usr/bin/")
 	assert.Equal(config.HypervisorConfig.UseVSock, true)

src/runtime/virtcontainers/pkg/types/types.go

@@ -39,3 +39,11 @@ type Route struct {
 	Source  string
 	Scope   uint32
 }
+
+type ARPNeighbor struct {
+	ToIPAddress *IPAddress
+	Device      string
+	LLAddr      string
+	State       int
+	Flags       int
+}

src/runtime/virtcontainers/qemu.go

@@ -537,8 +537,9 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 	}
 
 	rtc := govmmQemu.RTC{
-		Base:     "utc",
+		Base:     govmmQemu.UTC,
-		DriftFix: "slew",
+		Clock:    govmmQemu.Host,
+		DriftFix: govmmQemu.Slew,
 	}
 
 	if q.state.UUID == "" {
@@ -556,6 +557,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 	}
 
 	cpuModel := q.arch.cpuModel()
+	cpuModel += "," + q.config.CPUFeatures
 
 	firmwarePath, err := q.config.FirmwareAssetPath()
 	if err != nil {

src/runtime/virtcontainers/qemu_amd64.go

@@ -158,9 +158,6 @@ func (q *qemuAmd64) bridges(number uint32) {
 
 func (q *qemuAmd64) cpuModel() string {
 	cpuModel := defaultCPUModel
-	if q.nestedRun {
-		cpuModel += ",pmu=off"
-	}
 
 	// VMX is not migratable yet.
 	// issue: https://github.com/kata-containers/kata-containers/src/runtime/issues/1750

src/runtime/virtcontainers/qemu_amd64_test.go

@@ -95,11 +95,6 @@ func TestQemuAmd64CPUModel(t *testing.T) {
 	model := amd64.cpuModel()
 	assert.Equal(expectedOut, model)
 
-	amd64.enableNestingChecks()
-	expectedOut = defaultCPUModel + ",pmu=off"
-	model = amd64.cpuModel()
-	assert.Equal(expectedOut, model)
-
 	amd64.disableNestingChecks()
 	base, ok := amd64.(*qemuAmd64)
 	assert.True(ok)

src/runtime/virtcontainers/utils/utils.go

@@ -9,6 +9,7 @@ import (
 	"crypto/rand"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -297,3 +298,23 @@ const (
 	MiB          = KiB << 10
 	GiB          = MiB << 10
 )
+
+// Binary to use to log program output
+const LoggerBinaryName = "systemd-cat"
+
+type ProgramLogger struct {
+	cmd *exec.Cmd
+}
+
+func NewProgramLogger(loggerLabel string) ProgramLogger {
+	return ProgramLogger{cmd: exec.Command(LoggerBinaryName, "-t", loggerLabel)}
+}
+
+func (p *ProgramLogger) StartLogger(output io.ReadCloser) error {
+	p.cmd.Stdin = output
+	return StartCmd(p.cmd)
+}
+
+func (p ProgramLogger) String() string {
+	return p.cmd.Path
+}

versions.yaml

@@ -75,7 +75,7 @@ assets:
       url: "https://github.com/cloud-hypervisor/cloud-hypervisor"
       uscan-url: >-
         https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz
-      version: "v0.7.0"
+      version: "v0.8.0"
 
     firecracker:
       description: "Firecracker micro-VMM"