From 8b5eed70f744a7a6cf9e51fd5639c2f55b50d8a0 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 19:58:00 -0700 Subject: [PATCH 01/21] clh: Enable disk block device hotplug support [ port from runtime commit c5f97b24d7a1eaac216f144b2c5429feb3451553 ] With this patch, the container image can be shared from host with guest as a block device when the 'devicemapper' is used as the storage driver for docker. Note: The 'block_device_driver="virtio-blk"' entry is required in the hypervisor config file to work properly. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/virtcontainers/clh.go | 33 ++++++++++++++++++++++++++ src/runtime/virtcontainers/clh_test.go | 27 +++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index c77e2b2b8..4ef64f982 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -88,6 +88,8 @@ type clhClient interface { VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error) // Add VFIO PCI device to the VM VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) + // Add a new disk device to the VM + VmAddDiskPut(ctx context.Context, diskConfig chclient.DiskConfig) (*http.Response, error) } type CloudHypervisorVersion struct { @@ -410,6 +412,33 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) { return vcpuInfo, nil } +func (clh *cloudHypervisor) hotplugBlockDevice(drive *config.BlockDrive) error { + cl := clh.client() + ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) + defer cancel() + + _, _, err := cl.VmmPingGet(ctx) + if err != nil { + return openAPIClientError(err) + } + + if drive.Pmem { + err = fmt.Errorf("pmem device hotplug not supported") + } else { + blkDevice := chclient.DiskConfig{ + Path: drive.File, + Readonly: drive.ReadOnly, + VhostUser: false, + } + _, err = cl.VmAddDiskPut(ctx, blkDevice) + } + + if err != nil { + err = fmt.Errorf("failed to hotplug block device %+v %s", drive, openAPIClientError(err)) + } + return err +} + func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { cl := clh.client() ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) @@ -432,6 +461,9 @@ func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType device defer span.Finish() switch devType { + case blockDev: + drive := devInfo.(*config.BlockDrive) + return nil, clh.hotplugBlockDevice(drive) case vfioDev: device := devInfo.(*config.VFIODev) return nil, clh.hotPlugVFIODevice(*device) @@ -663,6 +695,7 @@ func (clh *cloudHypervisor) capabilities() types.Capabilities { clh.Logger().WithField("function", "capabilities").Info("get Capabilities") var caps types.Capabilities caps.SetFsSharingSupport() + caps.SetBlockDeviceHotplugSupport() return caps } diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 594762875..d72728aea 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -99,6 +99,11 @@ func (c *clhClientMock) VmAddDevicePut(ctx context.Context, vmAddDevice chclient return nil, nil } +//nolint:golint +func (c *clhClientMock) VmAddDiskPut(ctx context.Context, diskConfig chclient.DiskConfig) (*http.Response, error) { + return nil, nil +} + func TestCloudHypervisorAddVSock(t *testing.T) { assert := assert.New(t) clh := cloudHypervisor{} @@ -357,3 +362,25 @@ func TestCheckVersion(t *testing.T) { } } } + +func TestCloudHypervisorHotplugBlockDevice(t *testing.T) { + assert := assert.New(t) + + clhConfig, err := newClhConfig() + assert.NoError(err) + + clh := &cloudHypervisor{} + clh.config = clhConfig + clh.APIClient = &clhClientMock{} + + clh.config.BlockDeviceDriver = config.VirtioBlock + err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: false}) + assert.NoError(err, "Hotplug disk block device expected no error") + + err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: true}) + assert.Error(err, "Hotplug pmem block device expected error") + + clh.config.BlockDeviceDriver = config.VirtioSCSI + err = clh.hotplugBlockDevice(&config.BlockDrive{Pmem: false}) + assert.Error(err, "Hotplug block device not using 'virtio-blk' expected error") +} From 21f83348e91ce4b5a51efbcd8aaa754da77a3b83 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 19:59:02 -0700 Subject: [PATCH 02/21] clh: Set 'virtio-blk' as the default block device driver [ port from runtime commit 5e5527204c03036f1d1a6b3122c1e0c3e1d1ba94 ] The block device driver defaults to 'virtio-scsi' when it is not set in the hypervisor configuration file, while cloud-hypervisor supports only 'virtio-blk' for its block devices. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/cli/config/configuration-clh.toml.in | 5 +++++ src/runtime/virtcontainers/clh.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 00185ea2e..c9d0f5f08 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -68,6 +68,11 @@ virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ # cloud-hypervisor prefers virtiofs caching (dax) for performance reasons virtio_fs_cache = "always" +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. This is virtio-scsi, virtio-blk +# or nvdimm. +block_device_driver = "virtio-blk" + # This option changes the default hypervisor and kernel parameters # to enable debug output where available. This extra output is added # to the proxy logs, but only when proxy debug is also enabled. diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 4ef64f982..598cba19c 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -413,6 +413,11 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) { } func (clh *cloudHypervisor) hotplugBlockDevice(drive *config.BlockDrive) error { + if clh.config.BlockDeviceDriver != config.VirtioBlock { + return fmt.Errorf("incorrect hypervisor configuration on 'block_device_driver':"+ + " using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock) + } + cl := clh.client() ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() From fd5d139470fd956b867476c532c575e8ebcc54e1 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Mon, 29 Jun 2020 20:00:34 -0700 Subject: [PATCH 03/21] clh: Improve hypervisor logging [ port from runtime commit e1ee00d16ed621594a92ce0456eb048362962ff0 ] Use systemd-cat to collect hypervisor output. The `systemd-cat` program will open a journal fd and call `cat(1)` to redirect all the output to the fd. This requires an extra binary to read from hypervisor stdout (that has combined stdin, stderr and serial terminal). But because it is cat the overhead is minimal and only is started on Kata debug mode. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Peng Tao --- src/runtime/virtcontainers/clh.go | 110 ++++++++++++---------- src/runtime/virtcontainers/utils/utils.go | 21 +++++ 2 files changed, 82 insertions(+), 49 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 598cba19c..4a493c134 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -6,10 +6,11 @@ package virtcontainers import ( - "bytes" "context" "encoding/json" "fmt" + "io" + "io/ioutil" "net" "net/http" "os" @@ -59,9 +60,7 @@ const ( clhStopSandboxTimeout = 3 clhSocket = "clh.sock" clhAPISocket = "clh-api.sock" - clhLogFile = "clh.log" virtioFsSocket = "virtiofsd.sock" - clhSerial = "serial-tty.log" supportedMajorVersion = 0 supportedMinorVersion = 5 defaultClhPath = "/usr/local/bin/cloud-hypervisor" @@ -122,7 +121,6 @@ type cloudHypervisor struct { APIClient clhClient version CloudHypervisorVersion vmconfig chclient.VmConfig - cmdOutput bytes.Buffer virtiofsd Virtiofsd store persistapi.PersistDriver } @@ -288,13 +286,8 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ // set the serial console to the cloud hypervisor if clh.config.Debug { - serialPath, err := clh.serialPath(clh.id) - if err != nil { - return err - } clh.vmconfig.Serial = chclient.ConsoleConfig{ - Mode: cctFILE, - File: serialPath, + Mode: cctTTY, } } else { @@ -370,17 +363,12 @@ func (clh *cloudHypervisor) startSandbox(timeout int) error { var strErr string strErr, pid, err := clh.LaunchClh() if err != nil { - return fmt.Errorf("failed to launch cloud-hypervisor: %s, error messages from log: %s", err, strErr) - } - clh.state.PID = pid - - if err := clh.waitVMM(clhTimeout); err != nil { - clh.Logger().WithField("error", err).WithField("output", clh.cmdOutput.String()).Warn("cloud-hypervisor init failed") if shutdownErr := clh.virtiofsd.Stop(); shutdownErr != nil { clh.Logger().WithField("error", shutdownErr).Warn("error shutting down Virtiofsd") } - return err + return fmt.Errorf("failed to launch cloud-hypervisor: %q, hypervisor output:\n%s", err, strErr) } + clh.state.PID = pid if err := clh.bootVM(ctx); err != nil { return err @@ -809,18 +797,10 @@ func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSocket) } -func (clh *cloudHypervisor) serialPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSerial) -} - func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhAPISocket) } -func (clh *cloudHypervisor) logFilePath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhLogFile) -} - func (clh *cloudHypervisor) waitVMM(timeout uint) error { clhRunning, err := clh.isClhRunning(timeout) @@ -911,8 +891,6 @@ func (clh *cloudHypervisor) getAvailableVersion() error { func (clh *cloudHypervisor) LaunchClh() (string, int, error) { - errStr := "" - clhPath, err := clh.clhPath() if err != nil { return "", -1, err @@ -920,36 +898,71 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) { args := []string{cscAPIsocket, clh.state.apiSocket} if clh.config.Debug { - - logfile, err := clh.logFilePath(clh.id) - if err != nil { - return "", -1, err - } - args = append(args, cscLogFile) - args = append(args, logfile) + // Cloud hypervisor log levels + // 'v' occurrences increase the level + //0 => Error + //1 => Warn + //2 => Info + //3 => Debug + //4+ => Trace + // Use Info, the CI runs with debug enabled + // a high level of logging increases the boot time + // and in a nested environment this could increase + // the chances to fail because agent is not + // ready on time. + args = append(args, "-vv") } clh.Logger().WithField("path", clhPath).Info() clh.Logger().WithField("args", strings.Join(args, " ")).Info() - cmd := exec.Command(clhPath, args...) - cmd.Stdout = &clh.cmdOutput - cmd.Stderr = &clh.cmdOutput + cmdHypervisor := exec.Command(clhPath, args...) + var hypervisorOutput io.ReadCloser + if clh.config.Debug { + cmdHypervisor.Env = os.Environ() + cmdHypervisor.Env = append(cmdHypervisor.Env, "RUST_BACKTRACE=full") + // Get StdoutPipe only for debug, without debug golang will redirect to /dev/null + hypervisorOutput, err = cmdHypervisor.StdoutPipe() + if err != nil { + return "", -1, err + } + } + + cmdHypervisor.Stderr = cmdHypervisor.Stdout + + err = utils.StartCmd(cmdHypervisor) + if err != nil { + return "", -1, err + } + + if err := clh.waitVMM(clhTimeout); err != nil { + clh.Logger().WithField("error", err).Warn("cloud-hypervisor init failed") + var output string + + if hypervisorOutput != nil { + b, errRead := ioutil.ReadAll(hypervisorOutput) + if errRead != nil { + output = "failed to read hypervisor output to get error information" + } else { + output = string(b) + } + } else { + output = "Please enable hypervisor logging to get stdout information" + } + + return output, -1, err + } if clh.config.Debug { - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, "RUST_BACKTRACE=full") - } - - if err := utils.StartCmd(cmd); err != nil { - fmt.Println("Error starting cloudHypervisor", err) - if cmd.Process != nil { - cmd.Process.Kill() + cmdLogger := utils.NewProgramLogger("kata-hypervisor") + clh.Logger().Debugf("Starting process logger(%s) for hypervisor", cmdLogger) + if err := cmdLogger.StartLogger(hypervisorOutput); err != nil { + // Not critical to run a container, but output wont be logged + clh.Logger().Warnf("Failed start process logger(%s) %s", cmdLogger, err) } - return errStr, 0, err } - return errStr, cmd.Process.Pid, nil + return "", cmdHypervisor.Process.Pid, nil } //########################################################################### @@ -960,13 +973,12 @@ func (clh *cloudHypervisor) LaunchClh() (string, int, error) { const ( cctOFF string = "Off" - cctFILE string = "File" cctNULL string = "Null" + cctTTY string = "Tty" ) const ( cscAPIsocket string = "--api-socket" - cscLogFile string = "--log-file" ) //**************************************** diff --git a/src/runtime/virtcontainers/utils/utils.go b/src/runtime/virtcontainers/utils/utils.go index 72dff4380..a83466833 100644 --- a/src/runtime/virtcontainers/utils/utils.go +++ b/src/runtime/virtcontainers/utils/utils.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -297,3 +298,23 @@ const ( MiB = KiB << 10 GiB = MiB << 10 ) + +// Binary to use to log program output +const LoggerBinaryName = "systemd-cat" + +type ProgramLogger struct { + cmd *exec.Cmd +} + +func NewProgramLogger(loggerLabel string) ProgramLogger { + return ProgramLogger{cmd: exec.Command(LoggerBinaryName, "-t", loggerLabel)} +} + +func (p *ProgramLogger) StartLogger(output io.ReadCloser) error { + p.cmd.Stdin = output + return StartCmd(p.cmd) +} + +func (p ProgramLogger) String() string { + return p.cmd.Path +} From 70137962d619d88efba3329011d52e864fe52c92 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Mon, 29 Jun 2020 20:02:30 -0700 Subject: [PATCH 04/21] clh: Remove vsock log port in kernel cmdline [ port from runtime commit 160e3a7c98043a52032b15cc8f6e32a91b032258 ] Cloud hypervisor logs console via stdout. Using console logs help to get not only agent logs but early boot kernel logs. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Peng Tao --- src/runtime/virtcontainers/clh.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 4a493c134..0196b2f9d 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -131,8 +131,7 @@ var clhKernelParams = []Param{ {"panic", "1"}, // upon kernel panic wait 1 second before reboot {"no_timer_check", ""}, // do not check broken timer IRQ resources {"noreplace-smp", ""}, // do not replace SMP instructions - {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, // tell the agent where to send the logs - {"rootflags", "data=ordered,errors=remount-ro ro"}, // mount the root filesystem as readonly + {"rootflags", "data=ordered,errors=remount-ro ro"}, // mount the root filesystem as readonly {"rootfstype", "ext4"}, } From 117ce4ac410a32b609359228f64db7dd41b746ad Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Mon, 29 Jun 2020 20:03:16 -0700 Subject: [PATCH 05/21] clh: remove slow boot debug flags from kernel cmdline [ port from runtime commit 6c517548429da06d33172c8e135dc9b9a297175d ] The systemd debug and kernel init call debug flags make slow the boot. The flags are not really related with the hypervisor and can be added if needed using extra kernel command line options. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Peng Tao --- src/runtime/virtcontainers/clh.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 0196b2f9d..58324a956 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -138,9 +138,7 @@ var clhKernelParams = []Param{ var clhDebugKernelParams = []Param{ {"console", "ttyS0,115200n8"}, // enable serial console - {"systemd.log_level", "debug"}, // enable systemd debug output {"systemd.log_target", "console"}, // send loggng to the console - {"initcall_debug", "1"}, // print init call timing information to the console } //########################################################### From 520295b938fed6b75b5a0ee0628735d973c474c5 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Mon, 29 Jun 2020 20:06:15 -0700 Subject: [PATCH 06/21] network: Detect and add static ARP entries [ port from runtime commit 67d3e2c5c5d11738c0c0ff46b1228909a6c81ab0 ] Some network plugins add static arp entries in the network namespace. Scan namespace for static entries and pass these on to the agent to be added within the guest. If the grpc api is not implemented by the agent due to a older running agent, check for this and do not error out to maintain backward compatibility. Signed-off-by: Archana Shinde Signed-off-by: Peng Tao --- src/runtime/virtcontainers/kata_agent.go | 73 +++++++++++++++++-- src/runtime/virtcontainers/network.go | 52 ++++++++++--- src/runtime/virtcontainers/network_test.go | 25 ++++++- src/runtime/virtcontainers/pkg/types/types.go | 8 ++ 4 files changed, 138 insertions(+), 20 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 1726bba0d..6ce30c391 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -110,6 +110,7 @@ const ( grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" grpcListInterfacesRequest = "grpc.ListInterfacesRequest" grpcListRoutesRequest = "grpc.ListRoutesRequest" + grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" grpcListProcessesRequest = "grpc.ListProcessesRequest" grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" @@ -638,6 +639,30 @@ func (k *kataAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, err return nil, nil } +func (k *kataAgent) addARPNeighbors(neighs []*vcTypes.ARPNeighbor) error { + if neighs != nil { + neighsReq := &grpc.AddARPNeighborsRequest{ + Neighbors: &grpc.ARPNeighbors{ + ARPNeighbors: k.convertToKataAgentNeighbors(neighs), + }, + } + _, err := k.sendReq(neighsReq) + if err != nil { + if grpcStatus.Convert(err).Code() == codes.Unimplemented { + k.Logger().WithFields(logrus.Fields{ + "arpneighbors-requested": fmt.Sprintf("%+v", neighs), + }).Warn("add ARP neighbors request failed due to old agent, please upgrade Kata Containers image version") + return nil + } + k.Logger().WithFields(logrus.Fields{ + "arpneighbors-requested": fmt.Sprintf("%+v", neighs), + }).WithError(err).Error("add ARP neighbors request failed") + } + return err + } + return nil +} + func (k *kataAgent) listInterfaces() ([]*vcTypes.Interface, error) { req := &grpc.ListInterfacesRequest{} resultingInterfaces, err := k.sendReq(req) @@ -843,7 +868,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { // // Setup network interfaces and routes // - interfaces, routes, err := generateInterfacesAndRoutes(sandbox.networkNS) + interfaces, routes, neighs, err := generateVCNetworkStructures(sandbox.networkNS) if err != nil { return err } @@ -853,6 +878,9 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { if _, err = k.updateRoutes(routes); err != nil { return err } + if err = k.addARPNeighbors(neighs); err != nil { + return err + } storages := setupStorages(sandbox) @@ -1999,6 +2027,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcListRoutesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.ListRoutes(ctx, req.(*grpc.ListRoutesRequest)) } + k.reqHandlers[grpcAddARPNeighborsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.AddARPNeighbors(ctx, req.(*grpc.AddARPNeighborsRequest)) + } k.reqHandlers[grpcOnlineCPUMemRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest)) } @@ -2175,18 +2206,27 @@ func (k *kataAgent) convertToIPFamily(ipFamily aTypes.IPFamily) int { return netlink.FAMILY_V4 } +func (k *kataAgent) convertToKataAgentIPAddress(ipAddr *vcTypes.IPAddress) (aIPAddr *aTypes.IPAddress) { + if ipAddr == nil { + return nil + } + + aIPAddr = &aTypes.IPAddress{ + Family: k.convertToKataAgentIPFamily(ipAddr.Family), + Address: ipAddr.Address, + Mask: ipAddr.Mask, + } + + return aIPAddr +} + func (k *kataAgent) convertToKataAgentIPAddresses(ipAddrs []*vcTypes.IPAddress) (aIPAddrs []*aTypes.IPAddress) { for _, ipAddr := range ipAddrs { if ipAddr == nil { continue } - aIPAddr := &aTypes.IPAddress{ - Family: k.convertToKataAgentIPFamily(ipAddr.Family), - Address: ipAddr.Address, - Mask: ipAddr.Mask, - } - + aIPAddr := k.convertToKataAgentIPAddress(ipAddr) aIPAddrs = append(aIPAddrs, aIPAddr) } @@ -2268,6 +2308,25 @@ func (k *kataAgent) convertToKataAgentRoutes(routes []*vcTypes.Route) (aRoutes [ return aRoutes } +func (k *kataAgent) convertToKataAgentNeighbors(neighs []*vcTypes.ARPNeighbor) (aNeighs []*aTypes.ARPNeighbor) { + for _, neigh := range neighs { + if neigh == nil { + continue + } + + aNeigh := &aTypes.ARPNeighbor{ + ToIPAddress: k.convertToKataAgentIPAddress(neigh.ToIPAddress), + Device: neigh.Device, + State: int32(neigh.State), + Lladdr: neigh.LLAddr, + } + + aNeighs = append(aNeighs, aNeigh) + } + + return aNeighs +} + func (k *kataAgent) convertToRoutes(aRoutes []*aTypes.Route) (routes []*vcTypes.Route) { for _, aRoute := range aRoutes { if aRoute == nil { diff --git a/src/runtime/virtcontainers/network.go b/src/runtime/virtcontainers/network.go index 3114e3366..baeee9bb9 100644 --- a/src/runtime/virtcontainers/network.go +++ b/src/runtime/virtcontainers/network.go @@ -118,10 +118,11 @@ type NetlinkIface struct { // NetworkInfo gathers all information related to a network interface. // It can be used to store the description of the underlying network. type NetworkInfo struct { - Iface NetlinkIface - Addrs []netlink.Addr - Routes []netlink.Route - DNS DNSInfo + Iface NetlinkIface + Addrs []netlink.Addr + Routes []netlink.Route + DNS DNSInfo + Neighbors []netlink.Neigh } // NetworkInterface defines a network interface. @@ -942,14 +943,15 @@ func deleteNetNS(netNSPath string) error { return nil } -func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { +func generateVCNetworkStructures(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, []*vcTypes.ARPNeighbor, error) { if networkNS.NetNsPath == "" { - return nil, nil, nil + return nil, nil, nil, nil } var routes []*vcTypes.Route var ifaces []*vcTypes.Interface + var neighs []*vcTypes.ARPNeighbor for _, endpoint := range networkNS.Endpoints { @@ -1008,10 +1010,36 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa r.Device = endpoint.Name() r.Scope = uint32(route.Scope) routes = append(routes, &r) + } + for _, neigh := range endpoint.Properties().Neighbors { + var n vcTypes.ARPNeighbor + + // We add only static ARP entries + if neigh.State != netlink.NUD_PERMANENT { + continue + } + + n.Device = endpoint.Name() + n.State = neigh.State + n.Flags = neigh.Flags + + if neigh.HardwareAddr != nil { + n.LLAddr = neigh.HardwareAddr.String() + } + + n.ToIPAddress = &vcTypes.IPAddress{ + Family: netlink.FAMILY_V4, + Address: neigh.IP.String(), + } + if neigh.IP.To4() == nil { + n.ToIPAddress.Family = netlink.FAMILY_V6 + } + + neighs = append(neighs, &n) } } - return ifaces, routes, nil + return ifaces, routes, neighs, nil } func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { @@ -1071,13 +1099,19 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo return NetworkInfo{}, err } + neighbors, err := handle.NeighList(link.Attrs().Index, netlink.FAMILY_ALL) + if err != nil { + return NetworkInfo{}, err + } + return NetworkInfo{ Iface: NetlinkIface{ LinkAttrs: *(link.Attrs()), Type: link.Type(), }, - Addrs: addrs, - Routes: routes, + Addrs: addrs, + Routes: routes, + Neighbors: neighbors, }, nil } diff --git a/src/runtime/virtcontainers/network_test.go b/src/runtime/virtcontainers/network_test.go index cbb303274..fdf0c402f 100644 --- a/src/runtime/virtcontainers/network_test.go +++ b/src/runtime/virtcontainers/network_test.go @@ -65,13 +65,20 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {LinkIndex: 329, Dst: nil, Src: nil, Gw: gatewayV6}, } + arpMAC, _ := net.ParseMAC("6a:92:3a:59:70:aa") + + neighs := []netlink.Neigh{ + {LinkIndex: 329, IP: net.IPv4(192, 168, 0, 101), State: netlink.NUD_PERMANENT, HardwareAddr: arpMAC}, + } + networkInfo := NetworkInfo{ Iface: NetlinkIface{ LinkAttrs: netlink.LinkAttrs{MTU: 1500}, Type: "", }, - Addrs: addrs, - Routes: routes, + Addrs: addrs, + Routes: routes, + Neighbors: neighs, } ep0 := &PhysicalEndpoint{ @@ -84,7 +91,7 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints} - resInterfaces, resRoutes, err := generateInterfacesAndRoutes(nns) + resInterfaces, resRoutes, resNeighs, err := generateVCNetworkStructures(nns) // // Build expected results: @@ -106,6 +113,15 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {Dest: "", Gateway: "2001:db8:1::1", Device: "eth0", Source: ""}, } + expectedNeighs := []*vcTypes.ARPNeighbor{ + { + Device: "eth0", + State: netlink.NUD_PERMANENT, + LLAddr: "6a:92:3a:59:70:aa", + ToIPAddress: &vcTypes.IPAddress{Address: "192.168.0.101", Family: netlink.FAMILY_V4}, + }, + } + for _, r := range resRoutes { fmt.Printf("resRoute: %+v\n", r) } @@ -115,7 +131,8 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Interfaces returned didn't match: got %+v, expecting %+v", resInterfaces, expectedInterfaces) assert.True(t, reflect.DeepEqual(resRoutes, expectedRoutes), "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) - + assert.True(t, reflect.DeepEqual(resNeighs, expectedNeighs), + "ARP Neighbors returned didn't match: got %+v, expecting %+v", resNeighs, expectedNeighs) } func TestNetInterworkingModelIsValid(t *testing.T) { diff --git a/src/runtime/virtcontainers/pkg/types/types.go b/src/runtime/virtcontainers/pkg/types/types.go index 0d4a9cfa1..5abb4922c 100644 --- a/src/runtime/virtcontainers/pkg/types/types.go +++ b/src/runtime/virtcontainers/pkg/types/types.go @@ -39,3 +39,11 @@ type Route struct { Source string Scope uint32 } + +type ARPNeighbor struct { + ToIPAddress *IPAddress + Device string + LLAddr string + State int + Flags int +} From fa9d619e8a9a8b9dcf96d1af19fb64a0743f8a35 Mon Sep 17 00:00:00 2001 From: Jia He Date: Mon, 29 Jun 2020 20:16:11 -0700 Subject: [PATCH 07/21] qemu: add cpu_features option [ port from runtime commit 0100af18a2afdd6dfcc95129ec6237ba4915b3e5 ] To control whether guest can enable/disable some CPU features. E.g. pmu=off, vmx=off. As discussed in the thread [1], the best approach is to let users specify them. How about adding a new option in the configuration file. Currently this patch only supports this option in qemu,no other vmm. [1] https://github.com/kata-containers/runtime/pull/2559#issuecomment-603998256 Signed-off-by: Jia He Signed-off-by: Peng Tao --- .../configuration-qemu-virtiofs.toml.in | 5 ++ .../cli/config/configuration-qemu.toml.in | 5 ++ .../pkg/katautils/config-settings.go.in | 1 + src/runtime/pkg/katautils/config.go | 25 ++++++++-- src/runtime/pkg/katautils/config_test.go | 47 +++++++++++++++++++ .../documentation/api/1.0/api.md | 3 ++ src/runtime/virtcontainers/hypervisor.go | 3 ++ src/runtime/virtcontainers/persist.go | 2 + .../virtcontainers/persist/api/config.go | 3 ++ src/runtime/virtcontainers/qemu.go | 1 + 10 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 8a220feaa..579d26434 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -37,6 +37,11 @@ firmware = "@FIRMWAREPATH@" # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` machine_accelerators="@MACHINEACCELERATORS@" +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="@CPUFEATURES@" + # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to @DEFVCPUS@ # < 0 --> will be set to the actual number of physical cores diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 7ac6d9098..4bf1d6914 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -38,6 +38,11 @@ firmware = "@FIRMWAREPATH@" # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` machine_accelerators="@MACHINEACCELERATORS@" +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="@CPUFEATURES@" + # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to @DEFVCPUS@ # < 0 --> will be set to the actual number of physical cores diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 18e2c074a..21bf1b60e 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -16,6 +16,7 @@ var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container" var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" var defaultFirmwarePath = "" var defaultMachineAccelerators = "" +var defaultCPUFeatures = "" var defaultShimPath = "/usr/libexec/kata-containers/kata-shim" var systemdUnitName = "kata-containers.target" diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 405bfc6d0..4e0a7cc03 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -93,6 +93,7 @@ type hypervisor struct { Image string `toml:"image"` Firmware string `toml:"firmware"` MachineAccelerators string `toml:"machine_accelerators"` + CPUFeatures string `toml:"cpu_features"` KernelParams string `toml:"kernel_params"` MachineType string `toml:"machine_type"` BlockDeviceDriver string `toml:"block_device_driver"` @@ -244,11 +245,9 @@ func (h hypervisor) firmware() (string, error) { func (h hypervisor) machineAccelerators() string { var machineAccelerators string - accelerators := strings.Split(h.MachineAccelerators, ",") - acceleratorsLen := len(accelerators) - for i := 0; i < acceleratorsLen; i++ { - if accelerators[i] != "" { - machineAccelerators += strings.Trim(accelerators[i], "\r\t\n ") + "," + for _, accelerator := range strings.Split(h.MachineAccelerators, ",") { + if accelerator != "" { + machineAccelerators += strings.TrimSpace(accelerator) + "," } } @@ -257,6 +256,19 @@ func (h hypervisor) machineAccelerators() string { return machineAccelerators } +func (h hypervisor) cpuFeatures() string { + var cpuFeatures string + for _, feature := range strings.Split(h.CPUFeatures, ",") { + if feature != "" { + cpuFeatures += strings.TrimSpace(feature) + "," + } + } + + cpuFeatures = strings.Trim(cpuFeatures, ",") + + return cpuFeatures +} + func (h hypervisor) kernelParams() string { if h.KernelParams == "" { return defaultKernelParams @@ -624,6 +636,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } machineAccelerators := h.machineAccelerators() + cpuFeatures := h.cpuFeatures() kernelParams := h.kernelParams() machineType := h.machineType() @@ -677,6 +690,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { ImagePath: image, FirmwarePath: firmware, MachineAccelerators: machineAccelerators, + CPUFeatures: cpuFeatures, KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: h.defaultVCPUs(), @@ -1129,6 +1143,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { InitrdPath: defaultInitrdPath, FirmwarePath: defaultFirmwarePath, MachineAccelerators: defaultMachineAccelerators, + CPUFeatures: defaultCPUFeatures, HypervisorMachineType: defaultMachineType, NumVCPUs: defaultVCPUCount, DefaultMaxVCPUs: defaultMaxVCPUCount, diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 7ffd68436..a2bb662c7 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -1604,6 +1604,53 @@ func TestDefaultMachineAccelerators(t *testing.T) { assert.Equal(machineAccelerators, h.machineAccelerators()) } +func TestDefaultCPUFeatures(t *testing.T) { + assert := assert.New(t) + cpuFeatures := "abc,123,rgb" + h := hypervisor{CPUFeatures: cpuFeatures} + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "" + h.CPUFeatures = cpuFeatures + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = cpuFeatures + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = "abc,,123" + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = ",,abc,,123,,," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = "abc,,123,,," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = ",,abc," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = ", , abc , ," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc" + h.CPUFeatures = " abc " + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = ", abc , 123 ," + assert.Equal(cpuFeatures, h.cpuFeatures()) + + cpuFeatures = "abc,123" + h.CPUFeatures = ",, abc ,,, 123 ,," + assert.Equal(cpuFeatures, h.cpuFeatures()) +} + func TestUpdateRuntimeConfiguration(t *testing.T) { assert := assert.New(t) diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index f5d3a7347..b690f193b 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -132,6 +132,9 @@ type HypervisorConfig struct { // MachineAccelerators are machine specific accelerators MachineAccelerators string + // CPUFeatures are cpu specific features + CPUFeatures string + // HypervisorPath is the hypervisor executable host path. HypervisorPath string diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index f640a8a70..eb68a6afa 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -275,6 +275,9 @@ type HypervisorConfig struct { // MachineAccelerators are machine specific accelerators MachineAccelerators string + // CPUFeatures are cpu specific features + CPUFeatures string + // HypervisorPath is the hypervisor executable host path. HypervisorPath string diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index f545eeb62..595a8c470 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -221,6 +221,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { InitrdPath: sconfig.HypervisorConfig.InitrdPath, FirmwarePath: sconfig.HypervisorConfig.FirmwarePath, MachineAccelerators: sconfig.HypervisorConfig.MachineAccelerators, + CPUFeatures: sconfig.HypervisorConfig.CPUFeatures, HypervisorPath: sconfig.HypervisorConfig.HypervisorPath, HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath, JailerPath: sconfig.HypervisorConfig.JailerPath, @@ -512,6 +513,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { InitrdPath: hconf.InitrdPath, FirmwarePath: hconf.FirmwarePath, MachineAccelerators: hconf.MachineAccelerators, + CPUFeatures: hconf.CPUFeatures, HypervisorPath: hconf.HypervisorPath, HypervisorCtlPath: hconf.HypervisorCtlPath, JailerPath: hconf.JailerPath, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 2292dcae0..84705ff0a 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -54,6 +54,9 @@ type HypervisorConfig struct { // MachineAccelerators are machine specific accelerators MachineAccelerators string + // CPUFeatures are cpu specific features + CPUFeatures string + // HypervisorPath is the hypervisor executable host path. HypervisorPath string diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index df3208354..a0b5b75d1 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -556,6 +556,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa } cpuModel := q.arch.cpuModel() + cpuModel += "," + q.config.CPUFeatures firmwarePath, err := q.config.FirmwareAssetPath() if err != nil { From 115dfa19cd472e7854b40685d0c190b4b779a78c Mon Sep 17 00:00:00 2001 From: Jia He Date: Mon, 29 Jun 2020 20:19:21 -0700 Subject: [PATCH 08/21] annotations: add cpu_features [ port from runtime commit f03c17d107999fd68da87d98ab3e242ac7843051 ] So that users can use annotations to set it. Signed-off-by: Jia He Signed-off-by: Peng Tao --- src/runtime/virtcontainers/pkg/annotations/annotations.go | 3 +++ src/runtime/virtcontainers/pkg/oci/utils.go | 6 ++++++ src/runtime/virtcontainers/pkg/oci/utils_test.go | 2 ++ 3 files changed, 11 insertions(+) diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index e1ab73bb1..f32f90694 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -81,6 +81,9 @@ const ( // MachineAccelerators is a sandbox annotation to specify machine specific accelerators for the hypervisor. MachineAccelerators = kataAnnotHypervisorPrefix + "machine_accelerators" + // CPUFeatures is a sandbox annotation to specify cpu specific features. + CPUFeatures = kataAnnotHypervisorPrefix + "cpu_features" + // DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host. DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net" diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 2c1537ba7..27310247f 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -700,6 +700,12 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon } func addHypervisporNetworkOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.CPUFeatures]; ok { + if value != "" { + sbConfig.HypervisorConfig.CPUFeatures = value + } + } + if value, ok := ocispec.Annotations[vcAnnotations.DisableVhostNet]; ok { disableVhostNet, err := strconv.ParseBool(value) if err != nil { diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index d97dbf468..94dec4402 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -784,6 +784,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.Msize9p] = "512" ocispec.Annotations[vcAnnotations.MachineType] = "q35" ocispec.Annotations[vcAnnotations.MachineAccelerators] = "nofw" + ocispec.Annotations[vcAnnotations.CPUFeatures] = "pmu=off" ocispec.Annotations[vcAnnotations.DisableVhostNet] = "true" ocispec.Annotations[vcAnnotations.GuestHookPath] = "/usr/bin/" ocispec.Annotations[vcAnnotations.UseVSock] = "true" @@ -819,6 +820,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Equal(config.HypervisorConfig.Msize9p, uint32(512)) assert.Equal(config.HypervisorConfig.HypervisorMachineType, "q35") assert.Equal(config.HypervisorConfig.MachineAccelerators, "nofw") + assert.Equal(config.HypervisorConfig.CPUFeatures, "pmu=off") assert.Equal(config.HypervisorConfig.DisableVhostNet, true) assert.Equal(config.HypervisorConfig.GuestHookPath, "/usr/bin/") assert.Equal(config.HypervisorConfig.UseVSock, true) From 06571f03778ebd9197558832e67c29382aaa437f Mon Sep 17 00:00:00 2001 From: Jia He Date: Mon, 29 Jun 2020 20:20:58 -0700 Subject: [PATCH 09/21] build: Add "pmu=off" to default cpu_features option [ port from runtime commit 41a06d4961f51af4ec4799aaee202c744584f31e ] The user sometimes doesn't care about pmu usage(e.g. perf tool profiling). But pmu will cost significant overhead on boot time and virtualization context switch. E.g. on arm64, if guest pmu is enabled, kvm should save and restore all PMU registers when guest/host switching. for dmesg comparision: Before: [ 0.007620] bus: 'platform': driver_probe_device: matched device pmu with driver armv8-pmu [ 0.007622] bus: 'platform': really_probe: probing driver armv8-pmu with device pmu [ 0.036282] hw perfevents: enabled with armv8_pmuv3 PMU driver, 7 counters available [ 0.036285] driver: 'armv8-pmu': driver_bound: bound to device 'pmu' [ 0.036295] bus: 'platform': really_probe: bound device pmu to driver armv8-pmu After: [ 0.007935] bus: 'platform': driver_probe_device: matched device alarmtimer with driver alarmtimer [ 0.007937] bus: 'platform': really_probe: probing driver alarmtimer with device alarmtimer [ 0.007940] driver: 'alarmtimer': driver_bound: bound to device 'alarmtimer' [ 0.007944] bus: 'platform': really_probe: bound device alarmtimer to driver alarmtimer Because s390 doest support "pmu=off", keep the default CPUFEATURES to be "" instead of "pmu=off". Signed-off-by: Jia He Signed-off-by: Peng Tao --- src/runtime/Makefile | 2 ++ src/runtime/arch/amd64-options.mk | 1 + src/runtime/arch/arm64-options.mk | 1 + src/runtime/arch/ppc64le-options.mk | 1 + src/runtime/arch/s390x-options.mk | 1 + 5 files changed, 6 insertions(+) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 819833e03..5442f026a 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -398,6 +398,7 @@ USER_VARS += KERNELTYPE_ACRN USER_VARS += KERNELTYPE_CLH USER_VARS += FIRMWAREPATH USER_VARS += MACHINEACCELERATORS +USER_VARS += CPUFEATURES USER_VARS += DEFMACHINETYPE_CLH USER_VARS += KERNELPARAMS USER_VARS += LIBEXECDIR @@ -606,6 +607,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ -e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \ -e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \ + -e "s|@CPUFEATURES@|$(CPUFEATURES)|g" \ -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk index 7bfc5ff97..b04acb8f7 100644 --- a/src/runtime/arch/amd64-options.mk +++ b/src/runtime/arch/amd64-options.mk @@ -8,6 +8,7 @@ MACHINETYPE := pc KERNELPARAMS := MACHINEACCELERATORS := +CPUFEATURES := pmu=off QEMUCMD := qemu-system-x86_64 diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk index f8c62c371..02227d2e8 100644 --- a/src/runtime/arch/arm64-options.mk +++ b/src/runtime/arch/arm64-options.mk @@ -8,6 +8,7 @@ MACHINETYPE := virt KERNELPARAMS := MACHINEACCELERATORS := +CPUFEATURES := pmu=off QEMUCMD := qemu-system-aarch64 diff --git a/src/runtime/arch/ppc64le-options.mk b/src/runtime/arch/ppc64le-options.mk index 9ce0a253c..2e1cffc81 100644 --- a/src/runtime/arch/ppc64le-options.mk +++ b/src/runtime/arch/ppc64le-options.mk @@ -8,5 +8,6 @@ MACHINETYPE := pseries KERNELPARAMS := MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" +CPUFEATURES := pmu=off KERNELTYPE := uncompressed #This architecture must use an uncompressed kernel. QEMUCMD := qemu-system-ppc64 diff --git a/src/runtime/arch/s390x-options.mk b/src/runtime/arch/s390x-options.mk index 3256e4628..f54c06457 100644 --- a/src/runtime/arch/s390x-options.mk +++ b/src/runtime/arch/s390x-options.mk @@ -8,5 +8,6 @@ MACHINETYPE := s390-ccw-virtio KERNELPARAMS := MACHINEACCELERATORS := +CPUFEATURES := QEMUCMD := qemu-system-s390x From 3a17e7aafefb2ef00239187bd03f90223e9e455c Mon Sep 17 00:00:00 2001 From: Jia He Date: Mon, 29 Jun 2020 20:33:46 -0700 Subject: [PATCH 10/21] qemu: Remove pmu limitation in nested virtualization of amd/ppc64le [ port from runtime commit 18662e16687453185ff4cf99b495a34e3ea9935f ] It's up to the user enable/disable pmu. After previous commit, the default pmu option has been set to off. This patch removes the hard limitation and unit test codes. Signed-off-by: Jia He Signed-off-by: Peng Tao --- src/runtime/virtcontainers/qemu_amd64.go | 3 --- src/runtime/virtcontainers/qemu_amd64_test.go | 5 ----- 2 files changed, 8 deletions(-) diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 1f559d9e9..773e93c24 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -158,9 +158,6 @@ func (q *qemuAmd64) bridges(number uint32) { func (q *qemuAmd64) cpuModel() string { cpuModel := defaultCPUModel - if q.nestedRun { - cpuModel += ",pmu=off" - } // VMX is not migratable yet. // issue: https://github.com/kata-containers/kata-containers/src/runtime/issues/1750 diff --git a/src/runtime/virtcontainers/qemu_amd64_test.go b/src/runtime/virtcontainers/qemu_amd64_test.go index f5ab24e7f..ba810299f 100644 --- a/src/runtime/virtcontainers/qemu_amd64_test.go +++ b/src/runtime/virtcontainers/qemu_amd64_test.go @@ -95,11 +95,6 @@ func TestQemuAmd64CPUModel(t *testing.T) { model := amd64.cpuModel() assert.Equal(expectedOut, model) - amd64.enableNestingChecks() - expectedOut = defaultCPUModel + ",pmu=off" - model = amd64.cpuModel() - assert.Equal(expectedOut, model) - amd64.disableNestingChecks() base, ok := amd64.(*qemuAmd64) assert.True(ok) From 51a6d60ab4b42f4a002d0edeec1fcfeed9f1f63c Mon Sep 17 00:00:00 2001 From: "Pradipta Kr. Banerjee" Date: Mon, 29 Jun 2020 20:49:07 -0700 Subject: [PATCH 11/21] qemu: Remove PMU feature for Power (ppc64le) platform [ port from runtime commit 6b32472c2138536ea7e859360498f175601d9ec9 ] The bug got introduced in 06571f0 Signed-off-by: bpradipt@in.ibm.com Signed-off-by: Peng Tao --- src/runtime/arch/ppc64le-options.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/arch/ppc64le-options.mk b/src/runtime/arch/ppc64le-options.mk index 2e1cffc81..f5b63f85d 100644 --- a/src/runtime/arch/ppc64le-options.mk +++ b/src/runtime/arch/ppc64le-options.mk @@ -8,6 +8,6 @@ MACHINETYPE := pseries KERNELPARAMS := MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" -CPUFEATURES := pmu=off +CPUFEATURES := KERNELTYPE := uncompressed #This architecture must use an uncompressed kernel. QEMUCMD := qemu-system-ppc64 From bdd386ba14abd3d1e46d277013177d3cc6fe4ae9 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Mon, 29 Jun 2020 20:51:15 -0700 Subject: [PATCH 12/21] qemu: Fix rtc parameter is not set to qemu [ port from runtime commit 379f19f7ccd71ebe938d9d6fe3cfe5f05f4f02bf ] Add default value for Clock, otherwise rtc parameter will be dropped by Valid function. "host" is the default value in qemu for rtc clock. Signed-off-by: Shuicheng Lin Signed-off-by: Peng Tao --- src/runtime/virtcontainers/qemu.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index a0b5b75d1..b8e466bc5 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -537,8 +537,9 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa } rtc := govmmQemu.RTC{ - Base: "utc", - DriftFix: "slew", + Base: govmmQemu.UTC, + Clock: govmmQemu.Host, + DriftFix: govmmQemu.Slew, } if q.state.UUID == "" { From 56ae20990da1bced06fe00d218e199a1376d868d Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 20:54:39 -0700 Subject: [PATCH 13/21] kata_agent: Allow to use "VirtPath" as volume source for blk devices [ port from runtime commit aea29b64b66f75049cb045f9e41dff2becdbebdc ] When the "PCIAddr" of block device is not available (e.g. cloud-hypervisor), we allow to use the "VirtPath" as the volume source for creating containers. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/virtcontainers/kata_agent.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 6ce30c391..0001fce4b 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1587,7 +1587,11 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*g vol.Source = blockDrive.DevNo case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock: vol.Driver = kataBlkDevType - vol.Source = blockDrive.PCIAddr + if blockDrive.PCIAddr == "" { + vol.Source = blockDrive.VirtPath + } else { + vol.Source = blockDrive.PCIAddr + } case c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio: vol.Driver = kataMmioBlkDevType vol.Source = blockDrive.VirtPath From 2d6c0731eb1bd46440871d6d0e817e2a30efd81f Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 20:55:43 -0700 Subject: [PATCH 14/21] kata_agent: Pass "VirtPath" with "PCIAddr" of blk devices to agent [ port from runtime commit 50c1dce137bb3d608daa931c01e4941ed5fdb6cc ] In case the "PCIAddr" of block devices is not available (e.g. cloud-hypervisor), we also pass the "VirtPath" to the agent for adding block devices to the container. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/virtcontainers/kata_agent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 0001fce4b..92098af25 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1221,6 +1221,7 @@ func (k *kataAgent) appendBlockDevice(dev ContainerDevice, c *Container) *grpc.D case config.VirtioBlock: kataDevice.Type = kataBlkDevType kataDevice.Id = d.PCIAddr + kataDevice.VmPath = d.VirtPath case config.VirtioSCSI: kataDevice.Type = kataSCSIDevType kataDevice.Id = d.SCSIAddr From 6da49a04188305656d02472714841462cee7aec0 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 20:57:04 -0700 Subject: [PATCH 15/21] clh: Clear the "PCIAddr" field while blk device hotplug [ port from runtime commit 5b96e01f1ba3b0458539c1c920d0c1aab7d5968e ] We explicitly set "PCIAddr" to NULL, so that the "VirtPath" field can be used by the agent to create the container. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/virtcontainers/clh.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 58324a956..9f2925bee 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -412,6 +412,9 @@ func (clh *cloudHypervisor) hotplugBlockDevice(drive *config.BlockDrive) error { return openAPIClientError(err) } + //Explicitly set PCIAddr to NULL, so that VirtPath can be used + drive.PCIAddr = "" + if drive.Pmem { err = fmt.Errorf("pmem device hotplug not supported") } else { From 653df674d381b574d7869eb1c4c341e2a5ebbb6e Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 21:01:05 -0700 Subject: [PATCH 16/21] kata_agent: Add unit tests [ port from runtime commit 6be76fcd07a3d74ca5521af2feaf966dd6f2c344 ] This patch adds the unit test for 'handleDeviceBlockVolume()'. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/virtcontainers/kata_agent_test.go | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 082bf157d..3848a12d9 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -39,8 +39,13 @@ import ( var ( testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock" + testBlkDriveFormat = "testBlkDriveFormat" testBlockDeviceCtrPath = "testBlockDeviceCtrPath" + testDevNo = "testDevNo" + testNvdimmID = "testNvdimmID" testPCIAddr = "04/02" + testSCSIAddr = "testSCSIAddr" + testVirtPath = "testVirtPath" ) func testGenerateKataProxySockDir() (string, error) { @@ -398,6 +403,110 @@ func TestHandleLocalStorage(t *testing.T) { assert.Equal(t, localMountPoint, expected) } +func TestHandleDeviceBlockVolume(t *testing.T) { + k := kataAgent{} + + tests := []struct { + BlockDeviceDriver string + inputDev *drivers.BlockDevice + resultVol *pb.Storage + }{ + { + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + Pmem: true, + NvdimmID: testNvdimmID, + Format: testBlkDriveFormat, + }, + }, + resultVol: &pb.Storage{ + Driver: kataNvdimmDevType, + Source: fmt.Sprintf("/dev/pmem%s", testNvdimmID), + Fstype: testBlkDriveFormat, + Options: []string{"dax"}, + }, + }, + { + BlockDeviceDriver: config.VirtioBlockCCW, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + DevNo: testDevNo, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkCCWDevType, + Source: testDevNo, + }, + }, + { + BlockDeviceDriver: config.VirtioBlock, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + PCIAddr: testPCIAddr, + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkDevType, + Source: testPCIAddr, + }, + }, + { + BlockDeviceDriver: config.VirtioBlock, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkDevType, + Source: testVirtPath, + }, + }, + { + BlockDeviceDriver: config.VirtioMmio, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataMmioBlkDevType, + Source: testVirtPath, + }, + }, + { + BlockDeviceDriver: config.VirtioSCSI, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + SCSIAddr: testSCSIAddr, + }, + }, + resultVol: &pb.Storage{ + Driver: kataSCSIDevType, + Source: testSCSIAddr, + }, + }, + } + + for _, test := range tests { + c := &Container{ + sandbox: &Sandbox{ + config: &SandboxConfig{ + HypervisorConfig: HypervisorConfig{ + BlockDeviceDriver: test.BlockDeviceDriver, + }, + }, + }, + } + + vol, _ := k.handleDeviceBlockVolume(c, test.inputDev) + assert.True(t, reflect.DeepEqual(vol, test.resultVol), + "Volume didn't match: got %+v, expecting %+v", + vol, test.resultVol) + } +} + func TestHandleBlockVolume(t *testing.T) { k := kataAgent{} From 94fdec4e4bf34693f3b9698bc858a5eda6267779 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Mon, 29 Jun 2020 21:03:26 -0700 Subject: [PATCH 17/21] clh: Allow add virtiofs args and cache options from config [ port from runtime commit e5a3211c74e20e9878fd0f5d1c80a3c4354eabd1 ] Today some options for virtiofsd could improve compatibility for example xattrs for dnf or cache=auto for file consistency for changes in the host. Allow users can enabled as requiered. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Peng Tao --- src/runtime/Makefile | 9 +++++-- .../cli/config/configuration-clh.toml.in | 24 +++++++++++++++++-- src/runtime/pkg/katautils/config.go | 1 + 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 5442f026a..ecec0dc29 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -176,8 +176,13 @@ DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd # Default DAX mapping cache size in MiB DEFVIRTIOFSCACHESIZE := 1024 -DEFVIRTIOFSCACHE := always -DEFVIRTIOFSEXTRAARGS := [] +DEFVIRTIOFSCACHE ?= always +# Format example: +# [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"] +# +# see `virtiofsd -h` for possible options. +# Make sure you quote args. +DEFVIRTIOFSEXTRAARGS ?= [] DEFENABLEIOTHREADS := false DEFENABLEMEMPREALLOC := false DEFENABLEHUGEPAGES := false diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index c9d0f5f08..6718f4a0d 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -65,8 +65,28 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ -# cloud-hypervisor prefers virtiofs caching (dax) for performance reasons -virtio_fs_cache = "always" +# Extra args for virtiofsd daemon +# +# Format example: +# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +# +# see `virtiofsd -h` for possible options. +virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ + +# Cache mode: +# +# - none +# Metadata, data, and pathname lookup are not cached in guest. They are +# always fetched from host and any changes are immediately pushed to host. +# +# - auto +# Metadata and pathname lookup cache expires after a configured amount of +# time (default is 1 second). Data is cached while the file is open (close +# to open consistency). +# +# - always +# Metadata, data, and pathname lookup are cached in guest and never expire. +virtio_fs_cache = "@DEFVIRTIOFSCACHE@" # Block storage driver to be used for the hypervisor in case the container # rootfs is backed by a block device. This is virtio-scsi, virtio-blk diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 4e0a7cc03..474476431 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -879,6 +879,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { PCIeRootPort: h.PCIeRootPort, DisableVhostNet: true, UseVSock: true, + VirtioFSExtraArgs: h.VirtioFSExtraArgs, }, nil } From d66f21921b5a34f655b1bfb303ffe119baff8a14 Mon Sep 17 00:00:00 2001 From: "Pradipta Kr. Banerjee" Date: Mon, 29 Jun 2020 21:06:28 -0700 Subject: [PATCH 18/21] cli: Fix kata-env output on Power [ port from runtime commit 9ac39116b08148de8e66abfca2e5407bc153af87 ] kata-env output always shows "VMContainerCapable=false" on Power. This patch fixes the same. Signed-off-by: bpradipt@in.ibm.com Signed-off-by: Peng Tao --- src/runtime/cli/kata-env.go | 4 ---- src/runtime/cli/kata-env_ppc64le_test.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/runtime/cli/kata-env.go b/src/runtime/cli/kata-env.go index 12afdd649..193368f4b 100644 --- a/src/runtime/cli/kata-env.go +++ b/src/runtime/cli/kata-env.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "os" - runtim "runtime" "strings" "github.com/BurntSushi/toml" @@ -223,9 +222,6 @@ func getHostInfo() (HostInfo, error) { } hostVMContainerCapable := true - if runtim.GOARCH == "ppc64le" { - hostVMContainerCapable = false - } details := vmContainerCapableDetails{ cpuInfoFile: procCPUInfo, diff --git a/src/runtime/cli/kata-env_ppc64le_test.go b/src/runtime/cli/kata-env_ppc64le_test.go index 81666f716..a794ba338 100644 --- a/src/runtime/cli/kata-env_ppc64le_test.go +++ b/src/runtime/cli/kata-env_ppc64le_test.go @@ -10,7 +10,7 @@ import "testing" func getExpectedHostDetails(tmpdir string) (HostInfo, error) { expectedVendor := "" expectedModel := "POWER8" - expectedVMContainerCapable := false + expectedVMContainerCapable := true return genericGetExpectedHostDetails(tmpdir, expectedVendor, expectedModel, expectedVMContainerCapable) } From 9177d3a3b7e8b8b30b4d7083b86332695c315d15 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Mon, 29 Jun 2020 21:11:08 -0700 Subject: [PATCH 19/21] virtiofsd: Use cache=auto [ port from runtime commit 4645d3e6ef2e99dae1f2b3a7bfded6fc304d3023 ] Today for virtiofsd kata sets by default `cache=always`. This option is useful for performance but if the shared files are modified from the host changes are not updated in the guest as virtiofsd uses cached value all time. This patch changes to `cache=auto` to fix consistency issues. The option can still be set to always if it is wanted by the user. Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Peng Tao --- src/runtime/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index ecec0dc29..fbfa922dc 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -176,7 +176,7 @@ DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd # Default DAX mapping cache size in MiB DEFVIRTIOFSCACHESIZE := 1024 -DEFVIRTIOFSCACHE ?= always +DEFVIRTIOFSCACHE ?= auto # Format example: # [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"] # From d8a333b935ed877734fb80a39e8533b6cae5bb99 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 23:26:01 -0700 Subject: [PATCH 20/21] versions: Move to cloud-hypervisor v0.8.0 [ port from runtime commit 17d265af6fc1f0913545bfa64e3e1a497f3e44c0 ] Major new functionalities added in clh v0.8.0 include Experimental Snapshot and Restore Support, Experimental ARM64 Support, 5-level guest paging support, etc. Also, there are quite some bug fixings and CLI/API changes for cleanup. More details can be found in the release note: https://github.com/cloud-hypervisor/cloud-hypervisor/releases/tag/v0.8.0. Changes: 52b83969 build, release-notes: Document 0.8.0 release 776f8fc5 build: Update Cargo.lock 3f18f93f docs: Add a guide for testing on AArch64 97a1e5e1 vmm: Exit VMM event loop after guest shutdown for AArch64 5cd1730b vmm: Configure VM on AArch64 917219fa vmm: Enable VCPU for AArch64 b5f1c912 vmm: Enable memory manager for AArch64 eeeb45bb vmm: Enable device manager for AArch64 e9488846 vm-allocator: Enable vm-allocator for AArch64 5343b0ac net_util: Fix usage of deprecated mac_address method bf37ebdc arch: x86_64: Add 5th level of paging when needed abd6204d source: Fix file permissions 02ac1820 scripts: Ensure musl-gcc is used by musl build cc85d896 tests: Extend test_*_reboot with checks on fd leaking 2ae547cf build(deps): bump vmm-sys-util from 0.6.0 to 0.6.1 f3556279 build(deps): bump serde_json from 1.0.54 to 1.0.55 dc034eb3 scripts: Only use musl for the Rust components 176d6716 build: Run musl builds in parallel to glibc builds 083189e5 build(deps): bump vcpkg from 0.2.9 to 0.2.10 2334b521 build(deps): bump syn from 1.0.30 to 1.0.31 99c99c24 build(deps): bump serde_json from 1.0.53 to 1.0.54 96a5e22b resources: kernel: Enable 5 levels of page table 653087d7 vmm: Reduce MMIO address space by 4KiB 5f0b6201 arch: x86_64: Enable CR4 LA57 feature 09fd3259 build: Use fork of vm-memory with less performance impact 5f9e079a device: Add AArch64 RTC PL031 implementation 625bab69 vmm: api: Allow to delete non-booted VMs 313883f6 remove duplicated structure InitrdConfig afe60808 build(deps): bump synstructure from 0.12.3 to 0.12.4 aa79a92c tests: Add integration test for unprivileged network 9b71ba20 vmm, vm-virtio: Stop always autogenerating a host MAC address 1f8b6fa9 net_util: Allow retrieving the MAC address from the TAP device 929d70bc net_util: Only try and enable the TAP device if it not already enabled eda9bfc7 vhost_user_fs: Replace the '--sock' parameter with '--socket' a8cdf2f0 tests,vm-virtio,vmm: Use 'socket' for all CLI/API parameters 90e7accf ch-remote: Show response body from error e436bbf3 build: Install libfdt in github cross-build workflow 2d13751d aarch64: Porting fdt related files from Firecracker 5a18dd36 aarch64: Porting AArch64 register implementation from Firecracker d605fda3 aarch64: Porting GIC source files from Firecracker ce624a6d aarch64: Add memory layout for AArch64 c7d44b88 build(deps): bump quote from 1.0.6 to 1.0.7 7c91dfae build(deps): bump proc-macro-nested from 0.1.4 to 0.1.5 17c16e5c build(deps): bump pin-project from 0.4.19 to 0.4.20 a2398742 build(deps): bump arc-swap from 0.4.6 to 0.4.7 b31fe72e build(deps): bump openssl-sys from 0.9.57 to 0.9.58 96497004 build(deps): bump dirs-sys from 0.3.4 to 0.3.5 eabf43fb Revert "tests: Extend test_*_reboot with checks on fd leaking" 7dc4e913 tests: Extend test_*_reboot with checks on fd leaking 601d898f build(deps): bump pin-project from 0.4.17 to 0.4.19 6ff107af vm-device: Switch to use get_host_address_range in vfio-ioctls 3336e801 vfio: Switch to the vfio-ioctls crate ch branch d24aa72d vfio: Rename to vfio-ioctls 53ce5298 vfio: Move the PCI implementation to the PCI crate 8f7dc735 vmm: Move Vcpu::configure() to arch crate 969e5e0b vmm: Split configure_system() from load_kernel() for x86_64 20cf21cd vmm: Change booting process to cover AArch64 requirements 61aa4615 vhost_user_net: Implement VIRTIO_RING_F_EVENT_IDX a4d377a0 vm-virtio: net: Implement VIRTIO_RING_F_EVENT_IDX f0697073 vm-virtio: net: Handle lost interrupts on restore a5596020 vm-virtio: Add some info! level debugging interrupt generation cc51fdb8 vhost_user_net: Use NetQueuePair from vm-virtio fcc62efc vm-virtio: net: Prepare NetQueuePair for use in vhost-user-net 2dbd1186 vm-virtio: net: Split network handling 237cb184 vm-virtio: net: Add further missing error reporting 36d072e6 vm-virtio: Add error propagation for TAP listener (un)registration 3151b5d8 vm-virtio: net: Refactor to support code reuse 22be88d3 build(deps): bump vfio-bindings from `887b3cf` to `f08cbcb` 6121f462 build(deps): bump vfio-bindings from `46ef9d4` to `887b3cf` b731e63a build(deps): bump ryu from 1.0.4 to 1.0.5 d2d5ccb1 build(deps): bump proc-macro2 from 1.0.17 to 1.0.18 a1b9131b build(deps): bump syn from 1.0.29 to 1.0.30 2571b279 build(deps): bump vcpkg from 0.2.8 to 0.2.9 57f477ef build(deps): bump syn from 1.0.28 to 1.0.29 8a08ea46 build(deps): bump serde_derive from 1.0.110 to 1.0.111 b8ae30d4 build(deps): bump serde from 1.0.110 to 1.0.111 0a0fb246 build(deps): bump syn from 1.0.27 to 1.0.28 bc2921b2 build(deps): bump regex from 1.3.8 to 1.3.9 917ad530 build(deps): bump regex from 1.3.7 to 1.3.8 aac87196 build(deps): bump vm-memory from 0.2.0 to 0.2.1 4c2e6054 build: Update to latest version of container c471ae94 Dockerfile: Update to latest Rust toolchain: 1.43.0 c31ad72e build: Address issues found by 1.43.0 clippy fbd1a6c5 vmm: api: Return complete error responses in handle_http_request() 0728bece vmm: seccomp: Ensure that umask() can be reprogrammed 3497eeff main: Set the umask to 0077 c1d15de7 build(deps): bump syn from 1.0.25 to 1.0.27 a4bb96d4 build(deps): bump libc from 0.2.70 to 0.2.71 bfd52ad8 build(deps): bump linux-loader from `bd01b6d` to `1af92d2` 8f1f9d9e devices: Implement InterruptController on AArch64 b32d3025 devices: Refactor IOAPIC to cover other architectures d5884180 build(deps): bump syn from 1.0.24 to 1.0.25 83c18de5 build(deps): bump proc-macro-hack from 0.5.15 to 0.5.16 7708b95e build(deps): bump syn from 1.0.23 to 1.0.24 749f2f03 build(deps): bump proc-macro2 from 1.0.15 to 1.0.17 c98d6fd0 build(deps): bump openssl-sys from 0.9.56 to 0.9.57 a9ca493b build(deps): bump proc-macro2 from 1.0.14 to 1.0.15 974c7138 build(deps): bump thiserror from 1.0.18 to 1.0.19 321c479b build(deps): bump proc-macro2 from 1.0.13 to 1.0.14 4f5c8be3 build: Added a workflow to cross-build targetting AArch64 1befae87 build: Fixed build errors and warnings on AArch64 0090ec2d build: Updated development utilities for AArch64 af8292b6 vmm, config, vhost_user_blk: remove "wce" parameter 9101bdd7 vm-virtio: block: Ensure backing file consistency dc66eee8 vhost_user_block: Ensure backing file consistency 10db2131 vm-virtio: block: Add "writeback" control to Request b94d9a30 vhost_user_backend: Allow backends to know features that can be used 9d88ba7a vhost_user_block: Use VirtioBlockConfig from vm-virtio 1fac2632 vm-virtio: Use config name as per spec 077a5c36 build(deps): bump syn from 1.0.22 to 1.0.23 a813b57f vm-virtio, vhost_user_{fs,block,backend}: Move EVENT_IDX handling 8ae7a38d build: Use same virtio-bindings version 3947809c vm-virtio: block: Ensure that VIRTIO_BLK_T_FLUSH requests actually sync ca6edafb build(deps): bump cc from 1.0.53 to 1.0.54 a7f236b8 ci: Extend snapshot/restore to validate virtio-vsock f442c62b vm-virtio: Implement Snapshottable trait for Vsock f9759988 ci: Extend snapshot/restore test with virtio-iommu 646d33fe vm-virtio: Set queue fields explicitely during restore 02cbea54 vm-virtio: Implement Snapshottable trait for Iommu 4f89cb05 build(deps): bump linux-loader from `43d1c51` to `bd01b6d` 14db7b0a build(deps): bump addr2line from 0.12.0 to 0.12.1 9f2eddd9 ci: Fix test_serial_off 7c3e19c6 vhost_user_backend, vmm: Close leaked file descriptors 35782bd9 vm-virtio: Close file descriptors created by epoll::create() 039accc1 vhost_user_net, vm-virtio: Interrupt guest when TX queue is updated c8a081e4 build(deps): bump pin-project from 0.4.16 to 0.4.17 b80a7d01 build(deps): bump vmm-sys-util from 0.5.0 to 0.6.0 e6fd6d63 vhost_user_block: Implement VIRTIO_BLK_F_FLUSH 95e3edda build(deps): bump quote from 1.0.5 to 1.0.6 d760010c build(deps): bump ppv-lite86 from 0.2.6 to 0.2.8 0cde08a7 build(deps): bump hermit-abi from 0.1.12 to 0.1.13 3adfe3fb build(deps): bump syn from 1.0.21 to 1.0.22 85aadd15 build(deps): bump proc-macro2 from 1.0.12 to 1.0.13 c764c212 build(deps): bump thiserror from 1.0.17 to 1.0.18 4366dd92 vm-virtio: block: Add support for VIRTIO_RING_F_EVENT_IDX 5a55fc07 vhost_user_fs: Fix seccomp filter for musl 391508f0 tests: Add tests checking for host MAC address setting 1b8b5ac1 vhost-user_net, vm-virtio, vmm: Permit host MAC address setting 11049401 vmm: seccomp: Add ioctl() commands interface hardware address 59e1361f net_util: tap: Add support for setting tap MAC address 68fc4329 vmm: Update seccomp filters with clock_nanosleep badf8261 build(deps): bump anyhow from 1.0.30 to 1.0.31 7b10f732 build(deps): bump cc from 1.0.52 to 1.0.53 4120a7de vhost_user_fs: Add seccomp 6aa29bdb vmm: api: Use a common handler for data actions too 0fe223f0 vmm: api: Extend VmAction to reduce code duplication 6ec605a7 vmm: api: Refactor generic action handler c652625b vmm: api: Add a default implementation for simple PUT requests a3e8bea0 vmm: api: Move HttpError enum to http module 6aab0a54 vhost_user_fs: Implement support for optional sandboxing c4bf383f vhost_user_*: Create a vhost::Listener in advance fa844865 vhost_user_fs: Allow callers to provide a fd for /proc/self/fd 831cff3f vhost_user_fs: Use a fd for /proc/self/fd instead of /proc ba4ec7fc ci: Extend snapshot_restore_test with hotplug 9e165c2c ci: Enable snapshot/restore integration test c566f1f0 build(deps): bump once_cell from 1.3.1 to 1.4.0 7ffde295 build(deps): bump backtrace from 0.3.47 to 0.3.48 e9c2dbc8 build(deps): bump anyhow from 1.0.29 to 1.0.30 9ccc7daa build, vmm: Update to latest kvm-ioctls 80aa0a75 tests: Test unplugging virtio-fs 88ec93d0 vmm: config: Add missing "id" from FsConfig parsing 0f89f5ec build(deps): bump anyhow from 1.0.28 to 1.0.29 ab3d374a build(deps): bump syn from 1.0.20 to 1.0.21 35b8992e build(deps): bump thiserror from 1.0.16 to 1.0.17 3415b11d build(deps): bump quote from 1.0.4 to 1.0.5 6989bf05 build(deps): bump backtrace from 0.3.46 to 0.3.47 2991fd2a build(deps): bump libc from 0.2.69 to 0.2.70 c37da600 vmm: Update DeviceTree upon PCI BAR reprogramming d0ae9d7c vmm: Share the DeviceTree across threads 5e9d2545 vmm: Store and restore virtio-pci BAR resources 02bd50f6 vm-virtio: Add helper to set the configuration BAR value 8a826ae2 vmm: Store and restore virtio-pci device on right PCI slot 98dac352 vmm: Add optional PCI b/d/f to each DeviceNode 1e0ebb76 pci: Allow specific PCI b/d/f to be reserved e577b64a build(deps): bump syn from 1.0.19 to 1.0.20 36bffff2 tests: Expand the test_large_memory() test to cover lots of vCPUs b9ba81c3 arch, vmm: Don't build mptable when using ACPI 16ac24d8 tests: Only test "noacpi" test when we don't build with ACPI bb8d19bb arch: Check RSDP address does not go past memory 1c44e917 build(deps): bump clap from 2.33.0 to 2.33.1 4cd2eccf build(deps): bump signal-hook from 0.1.14 to 0.1.15 308b790c vm-virtio: Implement Snapshottable trait for VirtioPciDevice 6d594286 vm-virtio: Implement Snapshottable trait for VirtioPciCommonConfig e1701f11 pci: Implement Snapshottable trait for PciConfiguration 376db311 pci: Implement Snapshottable trait for MsixConfig 52ac3779 tests: Remove network interface from test_memory_overhead b57eeb96 vhost_user_block: Add "queue_size" to --block-backend 5016fcf8 vhost_user_block: Use config::OptionParser to simplify block backend parsing 592de97f vhost_user_net: Use config::OptionParser to simplify net backend parsing f3f398eb vhost_user_block: Consolidate the vhost-user-block backend syntax 3220292d vhost_user_net: Consolidate the vhost-user-net backend syntax 0d2be3b6 build(deps): bump serde from 1.0.107 to 1.0.110 9d8754c6 build(deps): bump pin-project from 0.4.13 to 0.4.16 9bac13de build(deps): bump serde_json from 1.0.52 to 1.0.53 e8d4a13e build(deps): bump serde_derive from 1.0.107 to 1.0.110 d8f181c5 build(deps): bump futures from 0.3.4 to 0.3.5 1e44ac51 build(deps): bump serde_derive from 1.0.106 to 1.0.107 c197bd6f build(deps): bump serde from 1.0.106 to 1.0.107 475040b2 vm-virtio: Correctly reset the virtqueues d809f2fe vm-virtio: Add virtio reset() support to MmioDevice 0d720cc3 bin: ch-remote: Ensure ch-remote supports syntax it advertises 74d88c4c build(deps): bump openssl-sys from 0.9.55 to 0.9.56 9adc32a0 tests: Print out details for smaps in test_memory_overhead 250f825f tests: Check that requesting tap name for virtio-net succeeds 006da040 tests: Check tap name provided is used for vhost_user_net tests 54b3329f tests: Add tests that use (non-existing) named tap 6fde2d18 build: Strip the binaries before using/releasing them a4d23c3c build(deps): bump syn from 1.0.18 to 1.0.19 12e00c0f vmm: cpu: Retry sending signals if necessary 31bde4f5 vmm: Unpark the DeviceManager threads in shutdown 801e72ac vmm: cpu: Unpause vCPU threads 91a4a258 vmm: cpu: When coming out of the pause event check for a kill signal cd60de8f Revert "vmm: vm: Unpark the threads before shutdown when the current state is paused" 797cd13d build(deps): bump vec_map from 0.8.1 to 0.8.2 f6a71bec vmm: Add unit tests for DeviceTree 64e01684 vmm: Create new module device_tree 3b77be90 vmm: Add device_node!() macro to improve code readability 83ec716e vmm: Create breadth-first search iterator for the DeviceTree b91ab1e3 vmm: Remove the list of migratable devices 1be70372 vmm: Don't use migratable_devices for restore bc608439 vmm: Add migratable field to the DeviceNode 7fec020f vmm: Create a dedicated DeviceTree structure 14b379de vmm: Add an identifier field to DeviceNode structure 0805d458 vmm: Add support for multiple children per DeviceNode daaeba51 vmm: Change Node into DeviceNode 5c7df03e vmm: Store and restore virtio-pmem resources 2e6895d9 vmm: Store and restore virtio-fs resources 987f8215 vmm: Store and restore virtio-mmio resources 9cb1e1cc vmm: Perform MMIO allocation from virtio-mmio device creation adf29706 vmm: Create devices in different path if restoring the VM d39f91de vmm: Reorganize DeviceManager creation 89c2a586 vmm: Restore devices following the device tree 52c80cfc vmm: Snapshot and restore DeviceManager state 5b408eec vmm: Create a device tree a6fde0bb vm-device: Define a Resource b8841d7a tests: Validate vsock functionality works across a reboot fec97e05 vm-virtio, vmm: Delete unix socket on shutdown 5109f914 vmm: config: Reject attempts to use VFIO or IOMMU without PCI cb220ae1 tests: Add some debugging to test_memory_overhead eb3d9d15 build(deps): bump ssh2 from 0.8.0 to 0.8.1 59b73034 build(deps): bump failure from 0.1.7 to 0.1.8 dd0791d7 build(deps): bump pnet from 0.25.0 to 0.26.0 7660a104 build(deps): bump failure_derive from 0.1.7 to 0.1.8 327d67fa virtio-mem: Return reize error in MemEpollHandler.run bc318b64 build(deps): bump proc-macro2 from 1.0.10 to 1.0.12 5571c6af build(deps): bump signal-hook from 0.1.13 to 0.1.14 af3d0802 build(deps): bump pnet_macros from 0.25.0 to 0.26.0 678855e8 build(deps): bump term_size from 0.3.1 to 0.3.2 2a16ce7e build(deps): bump quote from 1.0.3 to 1.0.4 99e3a150 build(deps): bump backtrace-sys from 0.1.36 to 0.1.37 Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- .../cloud-hypervisor/client/api/openapi.yaml | 40 ++++++++----------- .../client/docs/DiskConfig.md | 1 - .../cloud-hypervisor/client/docs/FsConfig.md | 2 +- .../client/docs/VsockConfig.md | 2 +- .../client/model_disk_config.go | 1 - .../client/model_fs_config.go | 2 +- .../client/model_vsock_config.go | 2 +- .../cloud-hypervisor/cloud-hypervisor.yaml | 11 ++--- versions.yaml | 2 +- 9 files changed, 25 insertions(+), 38 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index cc446a439..ef2cfcc49 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -306,7 +306,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id - path: path @@ -317,7 +316,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id cpus: @@ -336,23 +334,23 @@ components: iommu: false src: /dev/urandom fs: - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id vsock: - sock: sock iommu: false + socket: socket id: id cid: 3 pmem: @@ -436,7 +434,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id - path: path @@ -447,7 +444,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id cpus: @@ -466,23 +462,23 @@ components: iommu: false src: /dev/urandom fs: - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id - - sock: sock - num_queues: 3 + - num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id vsock: - sock: sock iommu: false + socket: socket id: id cid: 3 pmem: @@ -662,7 +658,6 @@ components: vhost_socket: vhost_socket vhost_user: false direct: false - wce: true poll_queue: true id: id properties: @@ -688,9 +683,6 @@ components: type: boolean vhost_socket: type: string - wce: - default: true - type: boolean poll_queue: default: true type: boolean @@ -756,17 +748,17 @@ components: type: object FsConfig: example: - sock: sock num_queues: 3 queue_size: 2 cache_size: 4 dax: true tag: tag + socket: socket id: id properties: tag: type: string - sock: + socket: type: string num_queues: default: 1 @@ -783,7 +775,7 @@ components: id: type: string required: - - sock + - socket - tag type: object PmemConfig: @@ -853,8 +845,8 @@ components: type: object VsockConfig: example: - sock: sock iommu: false + socket: socket id: id cid: 3 properties: @@ -863,7 +855,7 @@ components: format: int64 minimum: 3 type: integer - sock: + socket: description: Path to UNIX domain socket, used to proxy vsock connections. type: string iommu: @@ -873,7 +865,7 @@ components: type: string required: - cid - - sock + - socket type: object VmResize: example: diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md index 24698161d..09f8e7b5c 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md @@ -12,7 +12,6 @@ Name | Type | Description | Notes **QueueSize** | **int32** | | [optional] [default to 128] **VhostUser** | **bool** | | [optional] [default to false] **VhostSocket** | **string** | | [optional] -**Wce** | **bool** | | [optional] [default to true] **PollQueue** | **bool** | | [optional] [default to true] **Id** | **string** | | [optional] diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md index df645302b..cc1ad674b 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/FsConfig.md @@ -5,7 +5,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Tag** | **string** | | -**Sock** | **string** | | +**Socket** | **string** | | **NumQueues** | **int32** | | [optional] [default to 1] **QueueSize** | **int32** | | [optional] [default to 1024] **Dax** | **bool** | | [optional] [default to true] diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md index 81ac68e3b..c8715276d 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VsockConfig.md @@ -5,7 +5,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Cid** | **int64** | Guest Vsock CID | -**Sock** | **string** | Path to UNIX domain socket, used to proxy vsock connections. | +**Socket** | **string** | Path to UNIX domain socket, used to proxy vsock connections. | **Iommu** | **bool** | | [optional] [default to false] **Id** | **string** | | [optional] diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go index 603b697fc..24bf8c11e 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go @@ -18,7 +18,6 @@ type DiskConfig struct { QueueSize int32 `json:"queue_size,omitempty"` VhostUser bool `json:"vhost_user,omitempty"` VhostSocket string `json:"vhost_socket,omitempty"` - Wce bool `json:"wce,omitempty"` PollQueue bool `json:"poll_queue,omitempty"` Id string `json:"id,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go index 0744e0583..568915feb 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_fs_config.go @@ -11,7 +11,7 @@ package openapi // FsConfig struct for FsConfig type FsConfig struct { Tag string `json:"tag"` - Sock string `json:"sock"` + Socket string `json:"socket"` NumQueues int32 `json:"num_queues,omitempty"` QueueSize int32 `json:"queue_size,omitempty"` Dax bool `json:"dax,omitempty"` diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go index 34cf6119e..2fed17461 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vsock_config.go @@ -13,7 +13,7 @@ type VsockConfig struct { // Guest Vsock CID Cid int64 `json:"cid"` // Path to UNIX domain socket, used to proxy vsock connections. - Sock string `json:"sock"` + Socket string `json:"socket"` Iommu bool `json:"iommu,omitempty"` Id string `json:"id,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 3c677d927..5c5edb283 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -454,9 +454,6 @@ components: default: false vhost_socket: type: string - wce: - type: boolean - default: true poll_queue: type: boolean default: true @@ -509,12 +506,12 @@ components: FsConfig: required: - tag - - sock + - socket type: object properties: tag: type: string - sock: + socket: type: string num_queues: type: integer @@ -584,7 +581,7 @@ components: VsockConfig: required: - cid - - sock + - socket type: object properties: cid: @@ -592,7 +589,7 @@ components: format: int64 minimum: 3 description: Guest Vsock CID - sock: + socket: type: string description: Path to UNIX domain socket, used to proxy vsock connections. iommu: diff --git a/versions.yaml b/versions.yaml index df43811e9..15c964088 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "v0.7.0" + version: "v0.8.0" firecracker: description: "Firecracker micro-VMM" From 7ae4376bf498a218183d65f4fb986d080bb5a87c Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 29 Jun 2020 23:27:08 -0700 Subject: [PATCH 21/21] clh: vsock: Use the updated VsockConfig [ port runtime commit 364435a6a18bfbb1277512431040bf085554ffdf ] The new release of clh v0.8.0 updated the 'VsockConfig' of its HTTP API, which requires changes on our clh driver. Signed-off-by: Bo Chen Signed-off-by: Peng Tao --- src/runtime/virtcontainers/clh.go | 8 ++++---- src/runtime/virtcontainers/clh_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 9f2925bee..979dad2b0 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -1133,7 +1133,7 @@ func (clh *cloudHypervisor) addVSock(cid int64, path string) { "cid": cid, }).Info("Adding HybridVSock") - clh.vmconfig.Vsock = chclient.VsockConfig{Cid: cid, Sock: path} + clh.vmconfig.Vsock = chclient.VsockConfig{Cid: cid, Socket: path} } func (clh *cloudHypervisor) addNet(e Endpoint) error { @@ -1177,14 +1177,14 @@ func (clh *cloudHypervisor) addVolume(volume types.Volume) error { { Tag: volume.MountTag, CacheSize: int64(clh.config.VirtioFSCacheSize << 20), - Sock: vfsdSockPath, + Socket: vfsdSockPath, }, } } else { clh.vmconfig.Fs = []chclient.FsConfig{ { - Tag: volume.MountTag, - Sock: vfsdSockPath, + Tag: volume.MountTag, + Socket: vfsdSockPath, }, } diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index d72728aea..260e80882 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -110,7 +110,7 @@ func TestCloudHypervisorAddVSock(t *testing.T) { clh.addVSock(1, "path") assert.Equal(clh.vmconfig.Vsock.Cid, int64(1)) - assert.Equal(clh.vmconfig.Vsock.Sock, "path") + assert.Equal(clh.vmconfig.Vsock.Socket, "path") } // Check addNet appends to the network config list new configurations.