diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 819833e03..fbfa922dc 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -176,8 +176,13 @@ DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd # Default DAX mapping cache size in MiB DEFVIRTIOFSCACHESIZE := 1024 -DEFVIRTIOFSCACHE := always -DEFVIRTIOFSEXTRAARGS := [] +DEFVIRTIOFSCACHE ?= auto +# Format example: +# [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"] +# +# see `virtiofsd -h` for possible options. +# Make sure you quote args. +DEFVIRTIOFSEXTRAARGS ?= [] DEFENABLEIOTHREADS := false DEFENABLEMEMPREALLOC := false DEFENABLEHUGEPAGES := false @@ -398,6 +403,7 @@ USER_VARS += KERNELTYPE_ACRN USER_VARS += KERNELTYPE_CLH USER_VARS += FIRMWAREPATH USER_VARS += MACHINEACCELERATORS +USER_VARS += CPUFEATURES USER_VARS += DEFMACHINETYPE_CLH USER_VARS += KERNELPARAMS USER_VARS += LIBEXECDIR @@ -606,6 +612,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ -e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \ -e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \ + -e "s|@CPUFEATURES@|$(CPUFEATURES)|g" \ -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk index 7bfc5ff97..b04acb8f7 100644 --- a/src/runtime/arch/amd64-options.mk +++ b/src/runtime/arch/amd64-options.mk @@ -8,6 +8,7 @@ MACHINETYPE := pc KERNELPARAMS := MACHINEACCELERATORS := +CPUFEATURES := pmu=off QEMUCMD := qemu-system-x86_64 diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk index f8c62c371..02227d2e8 100644 --- a/src/runtime/arch/arm64-options.mk +++ b/src/runtime/arch/arm64-options.mk @@ -8,6 +8,7 @@ MACHINETYPE := virt KERNELPARAMS := MACHINEACCELERATORS := +CPUFEATURES := pmu=off QEMUCMD := qemu-system-aarch64 diff --git a/src/runtime/arch/ppc64le-options.mk b/src/runtime/arch/ppc64le-options.mk index 9ce0a253c..f5b63f85d 100644 --- a/src/runtime/arch/ppc64le-options.mk +++ b/src/runtime/arch/ppc64le-options.mk @@ -8,5 +8,6 @@ MACHINETYPE := pseries KERNELPARAMS := MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" +CPUFEATURES := KERNELTYPE := uncompressed #This architecture must use an uncompressed kernel. QEMUCMD := qemu-system-ppc64 diff --git a/src/runtime/arch/s390x-options.mk b/src/runtime/arch/s390x-options.mk index 3256e4628..f54c06457 100644 --- a/src/runtime/arch/s390x-options.mk +++ b/src/runtime/arch/s390x-options.mk @@ -8,5 +8,6 @@ MACHINETYPE := s390-ccw-virtio KERNELPARAMS := MACHINEACCELERATORS := +CPUFEATURES := QEMUCMD := qemu-system-s390x diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 00185ea2e..6718f4a0d 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -65,8 +65,33 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ -# cloud-hypervisor prefers virtiofs caching (dax) for performance reasons -virtio_fs_cache = "always" +# Extra args for virtiofsd daemon +# +# Format example: +# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# +# see `virtiofsd -h` for possible options. +virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ + +# Cache mode: +# +# - none +# Metadata, data, and pathname lookup are not cached in guest. They are +# always fetched from host and any changes are immediately pushed to host. +# +# - auto +# Metadata and pathname lookup cache expires after a configured amount of +# time (default is 1 second). Data is cached while the file is open (close +# to open consistency). +# +# - always +# Metadata, data, and pathname lookup are cached in guest and never expire. +virtio_fs_cache = "@DEFVIRTIOFSCACHE@" + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. This is virtio-scsi, virtio-blk +# or nvdimm. +block_device_driver = "virtio-blk" # This option changes the default hypervisor and kernel parameters # to enable debug output where available. This extra output is added diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 8a220feaa..579d26434 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -37,6 +37,11 @@ firmware = "@FIRMWAREPATH@" # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` machine_accelerators="@MACHINEACCELERATORS@" +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="@CPUFEATURES@" + # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to @DEFVCPUS@ # < 0 --> will be set to the actual number of physical cores diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 7ac6d9098..4bf1d6914 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -38,6 +38,11 @@ firmware = "@FIRMWAREPATH@" # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` machine_accelerators="@MACHINEACCELERATORS@" +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="@CPUFEATURES@" + # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to @DEFVCPUS@ # < 0 --> will be set to the actual number of physical cores diff --git a/src/runtime/cli/kata-env.go b/src/runtime/cli/kata-env.go index 12afdd649..193368f4b 100644 --- a/src/runtime/cli/kata-env.go +++ b/src/runtime/cli/kata-env.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "os" - runtim "runtime" "strings" "github.com/BurntSushi/toml" @@ -223,9 +222,6 @@ func getHostInfo() (HostInfo, error) { } hostVMContainerCapable := true - if runtim.GOARCH == "ppc64le" { - hostVMContainerCapable = false - } details := vmContainerCapableDetails{ cpuInfoFile: procCPUInfo, diff --git a/src/runtime/cli/kata-env_ppc64le_test.go b/src/runtime/cli/kata-env_ppc64le_test.go index 81666f716..a794ba338 100644 --- a/src/runtime/cli/kata-env_ppc64le_test.go +++ b/src/runtime/cli/kata-env_ppc64le_test.go @@ -10,7 +10,7 @@ import "testing" func getExpectedHostDetails(tmpdir string) (HostInfo, error) { expectedVendor := "" expectedModel := "POWER8" - expectedVMContainerCapable := false + expectedVMContainerCapable := true return genericGetExpectedHostDetails(tmpdir, expectedVendor, expectedModel, expectedVMContainerCapable) } diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 18e2c074a..21bf1b60e 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -16,6 +16,7 @@ var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container" var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" var defaultFirmwarePath = "" var defaultMachineAccelerators = "" +var defaultCPUFeatures = "" var defaultShimPath = "/usr/libexec/kata-containers/kata-shim" var systemdUnitName = "kata-containers.target" diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 405bfc6d0..474476431 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -93,6 +93,7 @@ type hypervisor struct { Image string `toml:"image"` Firmware string `toml:"firmware"` MachineAccelerators string `toml:"machine_accelerators"` + CPUFeatures string `toml:"cpu_features"` KernelParams string `toml:"kernel_params"` MachineType string `toml:"machine_type"` BlockDeviceDriver string `toml:"block_device_driver"` @@ -244,11 +245,9 @@ func (h hypervisor) firmware() (string, error) { func (h hypervisor) machineAccelerators() string { var machineAccelerators string - accelerators := strings.Split(h.MachineAccelerators, ",") - acceleratorsLen := len(accelerators) - for i := 0; i < acceleratorsLen; i++ { - if accelerators[i] != "" { - machineAccelerators += strings.Trim(accelerators[i], "\r\t\n ") + "," + for _, accelerator := range strings.Split(h.MachineAccelerators, ",") { + if accelerator != "" { + machineAccelerators += strings.TrimSpace(accelerator) + "," } } @@ -257,6 +256,19 @@ func (h hypervisor) machineAccelerators() string { return machineAccelerators } +func (h hypervisor) cpuFeatures() string { + var cpuFeatures string + for _, feature := range strings.Split(h.CPUFeatures, ",") { + if feature != "" { + cpuFeatures += strings.TrimSpace(feature) + "," + } + } + + cpuFeatures = strings.Trim(cpuFeatures, ",") + + return cpuFeatures +} + func (h hypervisor) kernelParams() string { if h.KernelParams == "" { return defaultKernelParams @@ -624,6 +636,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } machineAccelerators := h.machineAccelerators() + cpuFeatures := h.cpuFeatures() kernelParams := h.kernelParams() machineType := h.machineType() @@ -677,6 +690,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { ImagePath: image, FirmwarePath: firmware, MachineAccelerators: machineAccelerators, + CPUFeatures: cpuFeatures, KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: h.defaultVCPUs(), @@ -865,6 +879,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { PCIeRootPort: h.PCIeRootPort, DisableVhostNet: true, UseVSock: true, + VirtioFSExtraArgs: h.VirtioFSExtraArgs, }, nil } @@ -1129,6 +1144,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { InitrdPath: defaultInitrdPath, FirmwarePath: defaultFirmwarePath, MachineAccelerators: defaultMachineAccelerators, + CPUFeatures: defaultCPUFeatures, HypervisorMachineType: defaultMachineType, NumVCPUs: defaultVCPUCount, DefaultMaxVCPUs: defaultMaxVCPUCount, diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 7ffd68436..a2bb662c7 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -1604,6 +1604,53 @@ func TestDefaultMachineAccelerators(t *testing.T) { assert.Equal(machineAccelerators, h.machineAccelerators()) } +func TestDefaultCPUFeatures(t *testing.T) { + assert := assert.New(t) + cpuFeatures := "abc,123,rgb" + h := hypervisor{CPUFeatures: cpuFeatures} + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "" + h.CPUFeatures = cpuFeatures + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = cpuFeatures + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = "abc,,123" + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = ",,abc,,123,,," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = "abc,,123,,," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = ",,abc," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = ", , abc , ," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = " abc " + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = ", abc , 123 ," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = ",, abc ,,, 123 ,," + assert.Equal(cpuFeatures, h.cpuFeatures()) +} + func TestUpdateRuntimeConfiguration(t *testing.T) { assert := assert.New(t) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index c77e2b2b8..979dad2b0 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -6,10 +6,11 @@ package virtcontainers import ( - "bytes" "context" "encoding/json" "fmt" + "io" + "io/ioutil" "net" "net/http" "os" @@ -59,9 +60,7 @@ const ( clhStopSandboxTimeout = 3 clhSocket = "clh.sock" clhAPISocket = "clh-api.sock" - clhLogFile = "clh.log" virtioFsSocket = "virtiofsd.sock" - clhSerial = "serial-tty.log" supportedMajorVersion = 0 supportedMinorVersion = 5 defaultClhPath = "/usr/local/bin/cloud-hypervisor" @@ -88,6 +87,8 @@ type clhClient interface { VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error) // Add VFIO PCI device to the VM VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) + // Add a new disk device to the VM + VmAddDiskPut(ctx context.Context, diskConfig chclient.DiskConfig) (*http.Response, error) } type CloudHypervisorVersion struct { @@ -120,7 +121,6 @@ type cloudHypervisor struct { APIClient clhClient version CloudHypervisorVersion vmconfig chclient.VmConfig - cmdOutput bytes.Buffer virtiofsd Virtiofsd store persistapi.PersistDriver } @@ -131,17 +131,14 @@ var clhKernelParams = []Param{ {"panic", "1"}, // upon kernel panic wait 1 second before reboot {"no_timer_check", ""}, // do not check broken timer IRQ resources {"noreplace-smp", ""}, // do not replace SMP instructions - {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, // tell the agent where to send the logs - {"rootflags", "data=ordered,errors=remount-ro ro"}, // mount the root filesystem as readonly + {"rootflags", "data=ordered,errors=remount-ro ro"}, // mount the root filesystem as readonly {"rootfstype", "ext4"}, } var clhDebugKernelParams = []Param{ {"console", "ttyS0,115200n8"}, // enable serial console - {"systemd.log_level", "debug"}, // enable systemd debug output {"systemd.log_target", "console"}, // send loggng to the console - {"initcall_debug", "1"}, // print init call timing information to the console } //########################################################### @@ -286,13 +283,8 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ // set the serial console to the cloud hypervisor if clh.config.Debug { - serialPath, err := clh.serialPath(clh.id) - if err != nil { - return err - } clh.vmconfig.Serial = chclient.ConsoleConfig{ - Mode: cctFILE, - File: serialPath, + Mode: cctTTY, } } else { @@ -368,17 +360,12 @@ func (clh *cloudHypervisor) startSandbox(timeout int) error { var strErr string strErr, pid, err := clh.LaunchClh() if err != nil { - return fmt.Errorf("failed to launch cloud-hypervisor: %s, error messages from log: %s", err, strErr) - } - clh.state.PID = pid - - if err := clh.waitVMM(clhTimeout); err != nil { - clh.Logger().WithField("error", err).WithField("output", clh.cmdOutput.String()).Warn("cloud-hypervisor init failed") if shutdownErr := clh.virtiofsd.Stop(); shutdownErr != nil { clh.Logger().WithField("error", shutdownErr).Warn("error shutting down Virtiofsd") } - return err + return fmt.Errorf("failed to launch cloud-hypervisor: %q, hypervisor output:\n%s", err, strErr) } + clh.state.PID = pid if err := clh.bootVM(ctx); err != nil { return err @@ -410,6 +397,41 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) { return vcpuInfo, nil } +func (clh *cloudHypervisor) hotplugBlockDevice(drive *config.BlockDrive) error { + if clh.config.BlockDeviceDriver != config.VirtioBlock { + return fmt.Errorf("incorrect hypervisor configuration on 'block_device_driver':"+ + " using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock) + } + + cl := clh.client() + ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) + defer cancel() + + _, _, err := cl.VmmPingGet(ctx) + if err != nil { + return openAPIClientError(err) + } + + //Explicitly set PCIAddr to NULL, so that VirtPath can be used + drive.PCIAddr = "" + + if drive.Pmem { + err = fmt.Errorf("pmem device hotplug not supported") + } else { + blkDevice := chclient.DiskConfig{ + Path: drive.File, + Readonly: drive.ReadOnly, + VhostUser: false, + } + _, err = cl.VmAddDiskPut(ctx, blkDevice) + } + + if err != nil { + err = fmt.Errorf("failed to hotplug block device %+v %s", drive, openAPIClientError(err)) + } + return err +} + func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { cl := clh.client() ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) @@ -432,6 +454,9 @@ func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType device defer span.Finish() switch devType { + case blockDev: + drive := devInfo.(*config.BlockDrive) + return nil, clh.hotplugBlockDevice(drive) case vfioDev: device := devInfo.(*config.VFIODev) return nil, clh.hotPlugVFIODevice(*device) @@ -663,6 +688,7 @@ func (clh *cloudHypervisor) capabilities() types.Capabilities { clh.Logger().WithField("function", "capabilities").Info("get Capabilities") var caps types.Capabilities caps.SetFsSharingSupport() + caps.SetBlockDeviceHotplugSupport() return caps } @@ -771,18 +797,10 @@ func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSocket) } -func (clh *cloudHypervisor) serialPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSerial) -} - func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhAPISocket) } -func (clh *cloudHypervisor) logFilePath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhLogFile) -} - func (clh *cloudHypervisor) waitVMM(timeout uint) error { clhRunning, err := clh.isClhRunning(timeout) @@ -873,8 +891,6 @@ func (clh *cloudHypervisor) getAvailableVersion() error { func (clh *cloudHypervisor) LaunchClh() (string, int, error) { - errStr := "" - clhPath, err := clh.clhPath() if err != nil { return "", -1, err @@ -882,36 +898,71 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) { args := []string{cscAPIsocket, clh.state.apiSocket} if clh.config.Debug { - - logfile, err := clh.logFilePath(clh.id) - if err != nil { - return "", -1, err - } - args = append(args, cscLogFile) - args = append(args, logfile) + // Cloud hypervisor log levels + // 'v' occurrences increase the level + //0 => Error + //1 => Warn + //2 => Info + //3 => Debug + //4+ => Trace + // Use Info, the CI runs with debug enabled + // a high level of logging increases the boot time + // and in a nested environment this could increase + // the chances to fail because agent is not + // ready on time. + args = append(args, "-vv") } clh.Logger().WithField("path", clhPath).Info() clh.Logger().WithField("args", strings.Join(args, " ")).Info() - cmd := exec.Command(clhPath, args...) - cmd.Stdout = &clh.cmdOutput - cmd.Stderr = &clh.cmdOutput + cmdHypervisor := exec.Command(clhPath, args...) + var hypervisorOutput io.ReadCloser + if clh.config.Debug { + cmdHypervisor.Env = os.Environ() + cmdHypervisor.Env = append(cmdHypervisor.Env, "RUST_BACKTRACE=full") + // Get StdoutPipe only for debug, without debug golang will redirect to /dev/null + hypervisorOutput, err = cmdHypervisor.StdoutPipe() + if err != nil { + return "", -1, err + } + } + + cmdHypervisor.Stderr = cmdHypervisor.Stdout + + err = utils.StartCmd(cmdHypervisor) + if err != nil { + return "", -1, err + } + + if err := clh.waitVMM(clhTimeout); err != nil { + clh.Logger().WithField("error", err).Warn("cloud-hypervisor init failed") + var output string + + if hypervisorOutput != nil { + b, errRead := ioutil.ReadAll(hypervisorOutput) + if errRead != nil { + output = "failed to read hypervisor output to get error information" + } else { + output = string(b) + } + } else { + output = "Please enable hypervisor logging to get stdout information" + } + + return output, -1, err + } if clh.config.Debug { - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, "RUST_BACKTRACE=full") - } - - if err := utils.StartCmd(cmd); err != nil { - fmt.Println("Error starting cloudHypervisor", err) - if cmd.Process != nil { - cmd.Process.Kill() + cmdLogger := utils.NewProgramLogger("kata-hypervisor") + clh.Logger().Debugf("Starting process logger(%s) for hypervisor", cmdLogger) + if err := cmdLogger.StartLogger(hypervisorOutput); err != nil { + // Not critical to run a container, but output wont be logged + clh.Logger().Warnf("Failed start process logger(%s) %s", cmdLogger, err) } - return errStr, 0, err } - return errStr, cmd.Process.Pid, nil + return "", cmdHypervisor.Process.Pid, nil } //########################################################################### @@ -922,13 +973,12 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) { const ( cctOFF string = "Off" - cctFILE string = "File" cctNULL string = "Null" + cctTTY string = "Tty" ) const ( cscAPIsocket string = "--api-socket" - cscLogFile string = "--log-file" ) //**************************************** @@ -1083,7 +1133,7 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) { "cid": cid, }).Info("Adding HybridVSock") - clh.vmconfig.Vsock = chclient.VsockConfig{Cid: cid, Sock: path} + clh.vmconfig.Vsock = chclient.VsockConfig{Cid: cid, Socket: path} } func (clh *cloudHypervisor) addNet(e Endpoint) error { @@ -1127,14 +1177,14 @@ func (clh *cloudHypervisor) addVolume(volume types.Volume) error { { Tag: volume.MountTag, CacheSize: int64(clh.config.VirtioFSCacheSize << 20), - Sock: vfsdSockPath, + Socket: vfsdSockPath, }, } } else { clh.vmconfig.Fs = []chclient.FsConfig{ { - Tag: volume.MountTag, - Sock: vfsdSockPath, + Tag: volume.MountTag, + Socket: vfsdSockPath, }, } diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 594762875..260e80882 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -99,13 +99,18 @@ func (c *clhClientMock) VmAddDevicePut(ctx context.Context, vmAddDevice chclient return nil, nil } +//nolint:golint +func (c *clhClientMock) VmAddDiskPut(ctx context.Context, diskConfig chclient.DiskConfig) (*http.Response, error) { + return nil, nil +} + func TestCloudHypervisorAddVSock(t *testing.T) { assert := assert.New(t) clh := cloudHypervisor{} clh.addVSock(1, "path") assert.Equal(clh.vmconfig.Vsock.Cid, int64(1)) - assert.Equal(clh.vmconfig.Vsock.Sock, "path") + assert.Equal(clh.vmconfig.Vsock.Socket, "path") } // Check addNet appends to the network config list new configurations. @@ -357,3 +362,25 @@ func TestCheckVersion(t *testing.T) { } } } + +func TestCloudHypervisorHotplugBlockDevice(t *testing.T) { + assert := assert.New(t) + + clhConfig, err := newClhConfig() + assert.NoError(err) + + clh := &cloudHypervisor{} + clh.config = clhConfig + clh.APIClient = &clhClientMock{} + + clh.config.BlockDeviceDriver = config.VirtioBlock + err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: false}) + assert.NoError(err, "Hotplug disk block device expected no error") + + err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: true}) + assert.Error(err, "Hotplug pmem block device expected error") + + clh.config.BlockDeviceDriver = config.VirtioSCSI + err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: false}) + assert.Error(err, "Hotplug block device not using 'virtio-blk' expected error") +} diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index f5d3a7347..b690f193b 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -132,6 +132,9 @@ type HypervisorConfig struct { // MachineAccelerators are machine specific accelerators MachineAccelerators string + // CPUFeatures are cpu specific features + CPUFeatures string + // HypervisorPath is the hypervisor executable host path. HypervisorPath string diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index f640a8a70..eb68a6afa 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -275,6 +275,9 @@ type HypervisorConfig struct { // MachineAccelerators are machine specific accelerators MachineAccelerators string + // CPUFeatures are cpu specific features + CPUFeatures string + // HypervisorPath is the hypervisor executable host path. HypervisorPath string diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 1726bba0d..92098af25 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -110,6 +110,7 @@ const ( grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" grpcListInterfacesRequest = "grpc.ListInterfacesRequest" grpcListRoutesRequest = "grpc.ListRoutesRequest" + grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" grpcListProcessesRequest = "grpc.ListProcessesRequest" grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" @@ -638,6 +639,30 @@ func (k *kataAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, err return nil, nil } +func (k *kataAgent) addARPNeighbors(neighs []*vcTypes.ARPNeighbor) error { + if neighs != nil { + neighsReq := &grpc.AddARPNeighborsRequest{ + Neighbors: &grpc.ARPNeighbors{ + ARPNeighbors: k.convertToKataAgentNeighbors(neighs), + }, + } + _, err := k.sendReq(neighsReq) + if err != nil { + if grpcStatus.Convert(err).Code() == codes.Unimplemented { + k.Logger().WithFields(logrus.Fields{ + "arpneighbors-requested": fmt.Sprintf("%+v", neighs), + }).Warn("add ARP neighbors request failed due to old agent, please upgrade Kata Containers image version") + return nil + } + k.Logger().WithFields(logrus.Fields{ + "arpneighbors-requested": fmt.Sprintf("%+v", neighs), + }).WithError(err).Error("add ARP neighbors request failed") + } + return err + } + return nil +} + func (k *kataAgent) listInterfaces() ([]*vcTypes.Interface, error) { req := &grpc.ListInterfacesRequest{} resultingInterfaces, err := k.sendReq(req) @@ -843,7 +868,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { // // Setup network interfaces and routes // - interfaces, routes, err := generateInterfacesAndRoutes(sandbox.networkNS) + interfaces, routes, neighs, err := generateVCNetworkStructures(sandbox.networkNS) if err != nil { return err } @@ -853,6 +878,9 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { if _, err = k.updateRoutes(routes); err != nil { return err } + if err = k.addARPNeighbors(neighs); err != nil { + return err + } storages := setupStorages(sandbox) @@ -1193,6 +1221,7 @@ func (k *kataAgent) appendBlockDevice(dev ContainerDevice, c *Container) *grpc.D case config.VirtioBlock: kataDevice.Type = kataBlkDevType kataDevice.Id = d.PCIAddr + kataDevice.VmPath = d.VirtPath case config.VirtioSCSI: kataDevice.Type = kataSCSIDevType kataDevice.Id = d.SCSIAddr @@ -1559,7 +1588,11 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*g vol.Source = blockDrive.DevNo case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: vol.Driver = kataBlkDevType - vol.Source = blockDrive.PCIAddr + if blockDrive.PCIAddr == "" { + vol.Source = blockDrive.VirtPath + } else { + vol.Source = blockDrive.PCIAddr + } case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: vol.Driver = kataMmioBlkDevType vol.Source = blockDrive.VirtPath @@ -1999,6 +2032,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcListRoutesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.ListRoutes(ctx, req.(*grpc.ListRoutesRequest)) } + k.reqHandlers[grpcAddARPNeighborsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.AddARPNeighbors(ctx, req.(*grpc.AddARPNeighborsRequest)) + } k.reqHandlers[grpcOnlineCPUMemRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest)) } @@ -2175,18 +2211,27 @@ func (k *kataAgent) convertToIPFamily(ipFamily aTypes.IPFamily) int { return netlink.FAMILY_V4 } +func (k *kataAgent) convertToKataAgentIPAddress(ipAddr *vcTypes.IPAddress) (aIPAddr *aTypes.IPAddress) { + if ipAddr == nil { + return nil + } + + aIPAddr = &aTypes.IPAddress{ + Family: k.convertToKataAgentIPFamily(ipAddr.Family), + Address: ipAddr.Address, + Mask: ipAddr.Mask, + } + + return aIPAddr +} + func (k *kataAgent) convertToKataAgentIPAddresses(ipAddrs []*vcTypes.IPAddress) (aIPAddrs []*aTypes.IPAddress) { for _, ipAddr := range ipAddrs { if ipAddr == nil { continue } - aIPAddr := &aTypes.IPAddress{ - Family: k.convertToKataAgentIPFamily(ipAddr.Family), - Address: ipAddr.Address, - Mask: ipAddr.Mask, - } - + aIPAddr := k.convertToKataAgentIPAddress(ipAddr) aIPAddrs = append(aIPAddrs, aIPAddr) } @@ -2268,6 +2313,25 @@ func (k *kataAgent) convertToKataAgentRoutes(routes []*vcTypes.Route) (aRoutes [ return aRoutes } +func (k *kataAgent) convertToKataAgentNeighbors(neighs []*vcTypes.ARPNeighbor) (aNeighs []*aTypes.ARPNeighbor) { + for _, neigh := range neighs { + if neigh == nil { + continue + } + + aNeigh := &aTypes.ARPNeighbor{ + ToIPAddress: k.convertToKataAgentIPAddress(neigh.ToIPAddress), + Device: neigh.Device, + State: int32(neigh.State), + Lladdr: neigh.LLAddr, + } + + aNeighs = append(aNeighs, aNeigh) + } + + return aNeighs +} + func (k *kataAgent) convertToRoutes(aRoutes []*aTypes.Route) (routes []*vcTypes.Route) { for _, aRoute := range aRoutes { if aRoute == nil { diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 082bf157d..3848a12d9 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -39,8 +39,13 @@ import ( var ( testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock" + testBlkDriveFormat = "testBlkDriveFormat" testBlockDeviceCtrPath = "testBlockDeviceCtrPath" + testDevNo = "testDevNo" + testNvdimmID = "testNvdimmID" testPCIAddr = "04/02" + testSCSIAddr = "testSCSIAddr" + testVirtPath = "testVirtPath" ) func testGenerateKataProxySockDir() (string, error) { @@ -398,6 +403,110 @@ func TestHandleLocalStorage(t *testing.T) { assert.Equal(t, localMountPoint, expected) } +func TestHandleDeviceBlockVolume(t *testing.T) { + k := kataAgent{} + + tests := []struct { + BlockDeviceDriver string + inputDev *drivers.BlockDevice + resultVol *pb.Storage + }{ + { + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + Pmem: true, + NvdimmID: testNvdimmID, + Format: testBlkDriveFormat, + }, + }, + resultVol: &pb.Storage{ + Driver: kataNvdimmDevType, + Source: fmt.Sprintf("/dev/pmem%s", testNvdimmID), + Fstype: testBlkDriveFormat, + Options: []string{"dax"}, + }, + }, + { + BlockDeviceDriver: config.VirtioBlockCCW, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + DevNo: testDevNo, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkCCWDevType, + Source: testDevNo, + }, + }, + { + BlockDeviceDriver: config.VirtioBlock, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + PCIAddr: testPCIAddr, + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkDevType, + Source: testPCIAddr, + }, + }, + { + BlockDeviceDriver: config.VirtioBlock, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkDevType, + Source: testVirtPath, + }, + }, + { + BlockDeviceDriver: config.VirtioMmio, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataMmioBlkDevType, + Source: testVirtPath, + }, + }, + { + BlockDeviceDriver: config.VirtioSCSI, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + SCSIAddr: testSCSIAddr, + }, + }, + resultVol: &pb.Storage{ + Driver: kataSCSIDevType, + Source: testSCSIAddr, + }, + }, + } + + for _, test := range tests { + c := &Container{ + sandbox: &Sandbox{ + config: &SandboxConfig{ + HypervisorConfig: HypervisorConfig{ + BlockDeviceDriver: test.BlockDeviceDriver, + }, + }, + }, + } + + vol, _ := k.handleDeviceBlockVolume(c, test.inputDev) + assert.True(t, reflect.DeepEqual(vol, test.resultVol), + "Volume didn't match: got %+v, expecting %+v", + vol, test.resultVol) + } +} + func TestHandleBlockVolume(t *testing.T) { k := kataAgent{} diff --git a/src/runtime/virtcontainers/network.go b/src/runtime/virtcontainers/network.go index 3114e3366..baeee9bb9 100644 --- a/src/runtime/virtcontainers/network.go +++ b/src/runtime/virtcontainers/network.go @@ -118,10 +118,11 @@ type NetlinkIface struct { // NetworkInfo gathers all information related to a network interface. // It can be used to store the description of the underlying network. type NetworkInfo struct { - Iface NetlinkIface - Addrs []netlink.Addr - Routes []netlink.Route - DNS DNSInfo + Iface NetlinkIface + Addrs []netlink.Addr + Routes []netlink.Route + DNS DNSInfo + Neighbors []netlink.Neigh } // NetworkInterface defines a network interface. @@ -942,14 +943,15 @@ func deleteNetNS(netNSPath string) error { return nil } -func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { +func generateVCNetworkStructures(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, []*vcTypes.ARPNeighbor, error) { if networkNS.NetNsPath == "" { - return nil, nil, nil + return nil, nil, nil, nil } var routes []*vcTypes.Route var ifaces []*vcTypes.Interface + var neighs []*vcTypes.ARPNeighbor for _, endpoint := range networkNS.Endpoints { @@ -1008,10 +1010,36 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa r.Device = endpoint.Name() r.Scope = uint32(route.Scope) routes = append(routes, &r) + } + for _, neigh := range endpoint.Properties().Neighbors { + var n vcTypes.ARPNeighbor + + // We add only static ARP entries + if neigh.State != netlink.NUD_PERMANENT { + continue + } + + n.Device = endpoint.Name() + n.State = neigh.State + n.Flags = neigh.Flags + + if neigh.HardwareAddr != nil { + n.LLAddr = neigh.HardwareAddr.String() + } + + n.ToIPAddress = &vcTypes.IPAddress{ + Family: netlink.FAMILY_V4, + Address: neigh.IP.String(), + } + if neigh.IP.To4() == nil { + n.ToIPAddress.Family = netlink.FAMILY_V6 + } + + neighs = append(neighs, &n) } } - return ifaces, routes, nil + return ifaces, routes, neighs, nil } func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { @@ -1071,13 +1099,19 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo return NetworkInfo{}, err } + neighbors, err := handle.NeighList(link.Attrs().Index, netlink.FAMILY_ALL) + if err != nil { + return NetworkInfo{}, err + } + return NetworkInfo{ Iface: NetlinkIface{ LinkAttrs: *(link.Attrs()), Type: link.Type(), }, - Addrs: addrs, - Routes: routes, + Addrs: addrs, + Routes: routes, + Neighbors: neighbors, }, nil } diff --git a/src/runtime/virtcontainers/network_test.go b/src/runtime/virtcontainers/network_test.go index cbb303274..fdf0c402f 100644 --- a/src/runtime/virtcontainers/network_test.go +++ b/src/runtime/virtcontainers/network_test.go @@ -65,13 +65,20 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {LinkIndex: 329, Dst: nil, Src: nil, Gw: gatewayV6}, } + arpMAC, _ := net.ParseMAC("6a:92:3a:59:70:aa") + + neighs := []netlink.Neigh{ + {LinkIndex: 329, IP: net.IPv4(192, 168, 0, 101), State: netlink.NUD_PERMANENT, HardwareAddr: arpMAC}, + } + networkInfo := NetworkInfo{ Iface: NetlinkIface{ LinkAttrs: netlink.LinkAttrs{MTU: 1500}, Type: "", }, - Addrs: addrs, - Routes: routes, + Addrs: addrs, + Routes: routes, + Neighbors: neighs, } ep0 := &PhysicalEndpoint{ @@ -84,7 +91,7 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints} - resInterfaces, resRoutes, err := generateInterfacesAndRoutes(nns) + resInterfaces, resRoutes, resNeighs, err := generateVCNetworkStructures(nns) // // Build expected results: @@ -106,6 +113,15 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {Dest: "", Gateway: "2001:db8:1::1", Device: "eth0", Source: ""}, } + expectedNeighs := []*vcTypes.ARPNeighbor{ + { + Device: "eth0", + State: netlink.NUD_PERMANENT, + LLAddr: "6a:92:3a:59:70:aa", + ToIPAddress: &vcTypes.IPAddress{Address: "192.168.0.101", Family: netlink.FAMILY_V4}, + }, + } + for _, r := range resRoutes { fmt.Printf("resRoute: %+v\n", r) } @@ -115,7 +131,8 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Interfaces returned didn't match: got %+v, expecting %+v", resInterfaces, expectedInterfaces) assert.True(t, reflect.DeepEqual(resRoutes, expectedRoutes), "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) - + assert.True(t, reflect.DeepEqual(resNeighs, expectedNeighs), + "ARP Neighbors returned didn't match: got %+v, expecting %+v", resNeighs, expectedNeighs) } func TestNetInterworkingModelIsValid(t *testing.T) { diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index f545eeb62..595a8c470 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -221,6 +221,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { InitrdPath: sconfig.HypervisorConfig.InitrdPath, FirmwarePath: sconfig.HypervisorConfig.FirmwarePath, MachineAccelerators: sconfig.HypervisorConfig.MachineAccelerators, + CPUFeatures: sconfig.HypervisorConfig.CPUFeatures, HypervisorPath: sconfig.HypervisorConfig.HypervisorPath, HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath, JailerPath: sconfig.HypervisorConfig.JailerPath, @@ -512,6 +513,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { InitrdPath: hconf.InitrdPath, FirmwarePath: hconf.FirmwarePath, MachineAccelerators: hconf.MachineAccelerators, + CPUFeatures: hconf.CPUFeatures, HypervisorPath: hconf.HypervisorPath, HypervisorCtlPath: hconf.HypervisorCtlPath, JailerPath: hconf.JailerPath, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 2292dcae0..84705ff0a 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -54,6 +54,9 @@ type HypervisorConfig struct { // MachineAccelerators are machine specific accelerators MachineAccelerators string + // CPUFeatures are cpu specific features + CPUFeatures string + // HypervisorPath is the hypervisor executable host path. HypervisorPath string diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index e1ab73bb1..f32f90694 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -81,6 +81,9 @@ const ( // MachineAccelerators is a sandbox annotation to specify machine specific accelerators for the hypervisor. MachineAccelerators = kataAnnotHypervisorPrefix + "machine_accelerators" + // CPUFeatures is a sandbox annotation to specify cpu specific features. + CPUFeatures = kataAnnotHypervisorPrefix + "cpu_features" + // DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host. DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net" diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index cc446a439..ef2cfcc49 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -306,7 +306,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id - path: path @@ -317,7 +316,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id cpus: @@ -336,23 +334,23 @@ components: iommu: false src: /dev/urandom fs: - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id vsock: - sock: sock iommu: false + socket: socket id: id cid: 3 pmem: @@ -436,7 +434,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id - path: path @@ -447,7 +444,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id cpus: @@ -466,23 +462,23 @@ components: iommu: false src: /dev/urandom fs: - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id vsock: - sock: sock iommu: false + socket: socket id: id cid: 3 pmem: @@ -662,7 +658,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id properties: @@ -688,9 +683,6 @@ components: type: boolean vhost_socket: type: string - wce: - default: true - type: boolean poll_queue: default: true type: boolean @@ -756,17 +748,17 @@ components: type: object FsConfig: example: - sock: sock num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id properties: tag: type: string - sock: + socket: type: string num_queues: default: 1 @@ -783,7 +775,7 @@ components: id: type: string required: - - sock + - socket - tag type: object PmemConfig: @@ -853,8 +845,8 @@ components: type: object VsockConfig: example: - sock: sock iommu: false + socket: socket id: id cid: 3 properties: @@ -863,7 +855,7 @@ components: format: int64 minimum: 3 type: integer - sock: + socket: description: Path to UNIX domain socket, used to proxy vsock connections. type: string iommu: @@ -873,7 +865,7 @@ components: type: string required: - cid - - sock + - socket type: object VmResize: example: diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md index 24698161d..09f8e7b5c 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md @@ -12,7 +12,6 @@ Name | Type | Description | Notes **QueueSize** | **int32** | | [optional] [default to 128] **VhostUser** | **bool** | | [optional] [default to false] **VhostSocket** | **string** | | [optional] -**Wce** | **bool** | | [optional] [default to true] **PollQueue** | **bool** | | [optional] [default to true] **Id** | **string** | | [optional] diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md index df645302b..cc1ad674b 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md @@ -5,7 +5,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Tag** | **string** | | -**Sock** | **string** | | +**Socket** | **string** | | **NumQueues** | **int32** | | [optional] [default to 1] **QueueSize** | **int32** | | [optional] [default to 1024] **Dax** | **bool** | | [optional] [default to true] diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md index 81ac68e3b..c8715276d 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md @@ -5,7 +5,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Cid** | **int64** | Guest Vsock CID | -**Sock** | **string** | Path to UNIX domain socket, used to proxy vsock connections. | +**Socket** | **string** | Path to UNIX domain socket, used to proxy vsock connections. | **Iommu** | **bool** | | [optional] [default to false] **Id** | **string** | | [optional] diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go index 603b697fc..24bf8c11e 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go @@ -18,7 +18,6 @@ type DiskConfig struct { QueueSize int32 `json:"queue_size,omitempty"` VhostUser bool `json:"vhost_user,omitempty"` VhostSocket string `json:"vhost_socket,omitempty"` - Wce bool `json:"wce,omitempty"` PollQueue bool `json:"poll_queue,omitempty"` Id string `json:"id,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go index 0744e0583..568915feb 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go @@ -11,7 +11,7 @@ package openapi // FsConfig struct for FsConfig type FsConfig struct { Tag string `json:"tag"` - Sock string `json:"sock"` + Socket string `json:"socket"` NumQueues int32 `json:"num_queues,omitempty"` QueueSize int32 `json:"queue_size,omitempty"` Dax bool `json:"dax,omitempty"` diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go index 34cf6119e..2fed17461 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go @@ -13,7 +13,7 @@ type VsockConfig struct { // Guest Vsock CID Cid int64 `json:"cid"` // Path to UNIX domain socket, used to proxy vsock connections. - Sock string `json:"sock"` + Socket string `json:"socket"` Iommu bool `json:"iommu,omitempty"` Id string `json:"id,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 3c677d927..5c5edb283 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -454,9 +454,6 @@ components: default: false vhost_socket: type: string - wce: - type: boolean - default: true poll_queue: type: boolean default: true @@ -509,12 +506,12 @@ components: FsConfig: required: - tag - - sock + - socket type: object properties: tag: type: string - sock: + socket: type: string num_queues: type: integer @@ -584,7 +581,7 @@ components: VsockConfig: required: - cid - - sock + - socket type: object properties: cid: @@ -592,7 +589,7 @@ components: format: int64 minimum: 3 description: Guest Vsock CID - sock: + socket: type: string description: Path to UNIX domain socket, used to proxy vsock connections. iommu: diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 2c1537ba7..27310247f 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -700,6 +700,12 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon } func addHypervisporNetworkOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.CPUFeatures]; ok { + if value != "" { + sbConfig.HypervisorConfig.CPUFeatures = value + } + } + if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok { disableVhostNet, err := strconv.ParseBool(value) if err != nil { diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index d97dbf468..94dec4402 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -784,6 +784,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.Msize9p] = "512" ocispec.Annotations[vcAnnotations.MachineType] = "q35" ocispec.Annotations[vcAnnotations.MachineAccelerators] = "nofw" + ocispec.Annotations[vcAnnotations.CPUFeatures] = "pmu=off" ocispec.Annotations[vcAnnotations.DisableVhostNet] = "true" ocispec.Annotations[vcAnnotations.GuestHookPath] = "/usr/bin/" ocispec.Annotations[vcAnnotations.UseVSock] = "true" @@ -819,6 +820,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Equal(config.HypervisorConfig.Msize9p, uint32(512)) assert.Equal(config.HypervisorConfig.HypervisorMachineType, "q35") assert.Equal(config.HypervisorConfig.MachineAccelerators, "nofw") + assert.Equal(config.HypervisorConfig.CPUFeatures, "pmu=off") assert.Equal(config.HypervisorConfig.DisableVhostNet, true) assert.Equal(config.HypervisorConfig.GuestHookPath, "/usr/bin/") assert.Equal(config.HypervisorConfig.UseVSock, true) diff --git a/src/runtime/virtcontainers/pkg/types/types.go b/src/runtime/virtcontainers/pkg/types/types.go index 0d4a9cfa1..5abb4922c 100644 --- a/src/runtime/virtcontainers/pkg/types/types.go +++ b/src/runtime/virtcontainers/pkg/types/types.go @@ -39,3 +39,11 @@ type Route struct { Source string Scope uint32 } + +type ARPNeighbor struct { + ToIPAddress *IPAddress + Device string + LLAddr string + State int + Flags int +} diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index df3208354..b8e466bc5 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -537,8 +537,9 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa } rtc := govmmQemu.RTC{ - Base: "utc", - DriftFix: "slew", + Base: govmmQemu.UTC, + Clock: govmmQemu.Host, + DriftFix: govmmQemu.Slew, } if q.state.UUID == "" { @@ -556,6 +557,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa } cpuModel := q.arch.cpuModel() + cpuModel += "," + q.config.CPUFeatures firmwarePath, err := q.config.FirmwareAssetPath() if err != nil { diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 1f559d9e9..773e93c24 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -158,9 +158,6 @@ func (q *qemuAmd64) bridges(number uint32) { func (q *qemuAmd64) cpuModel() string { cpuModel := defaultCPUModel - if q.nestedRun { - cpuModel += ",pmu=off" - } // VMX is not migratable yet. // issue: https://github.com/kata-containers/kata-containers/src/runtime/issues/1750 diff --git a/src/runtime/virtcontainers/qemu_amd64_test.go b/src/runtime/virtcontainers/qemu_amd64_test.go index f5ab24e7f..ba810299f 100644 --- a/src/runtime/virtcontainers/qemu_amd64_test.go +++ b/src/runtime/virtcontainers/qemu_amd64_test.go @@ -95,11 +95,6 @@ func TestQemuAmd64CPUModel(t *testing.T) { model := amd64.cpuModel() assert.Equal(expectedOut, model) - amd64.enableNestingChecks() - expectedOut = defaultCPUModel + ",pmu=off" - model = amd64.cpuModel() - assert.Equal(expectedOut, model) - amd64.disableNestingChecks() base, ok := amd64.(*qemuAmd64) assert.True(ok) diff --git a/src/runtime/virtcontainers/utils/utils.go b/src/runtime/virtcontainers/utils/utils.go index 72dff4380..a83466833 100644 --- a/src/runtime/virtcontainers/utils/utils.go +++ b/src/runtime/virtcontainers/utils/utils.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -297,3 +298,23 @@ const ( MiB = KiB << 10 GiB = MiB << 10 ) + +// Binary to use to log program output +const LoggerBinaryName = "systemd-cat" + +type ProgramLogger struct { + cmd *exec.Cmd +} + +func NewProgramLogger(loggerLabel string) ProgramLogger { + return ProgramLogger{cmd: exec.Command(LoggerBinaryName, "-t", loggerLabel)} +} + +func (p *ProgramLogger) StartLogger(output io.ReadCloser) error { + p.cmd.Stdin = output + return StartCmd(p.cmd) +} + +func (p ProgramLogger) String() string { + return p.cmd.Path +} diff --git a/versions.yaml b/versions.yaml index df43811e9..15c964088 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "v0.7.0" + version: "v0.8.0" firecracker: description: "Firecracker micro-VMM"