From ce11a6f6320ca79ab4aa3d7401115ba5d8c89c58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fidencio@redhat.com>
Date: Tue, 13 Jul 2021 10:23:45 +0200
Subject: [PATCH 01/16] versions: Update containerd to v1.3.10
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bump containerd version to the latest v1.3.x, so we don't have to do
intrusive changes in our codebase (as done for in the main branch).

Fixes: #2350

Signed-off-by: Fabiano Fidêncio <fidencio@redhat.com>
---
 versions.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/versions.yaml b/versions.yaml
index f09a238f5e..4ef917d116 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -184,11 +184,11 @@ externals:
   cri-containerd:
     description: |
       Containerd Plugin for Kubernetes Container Runtime Interface.
-    url: "github.com/containerd/cri"
-    tarball_url: "https://storage.googleapis.com/cri-containerd-release"
+    url: "github.com/containerd/containerd"
+    tarball_url: "https://github.com/containerd/containerd/releases/download"
     # Next commit from 1.3 branch contains fix to be able to run
     # tests using go 1.13
-    version: "v1.3.7"
+    version: "v1.3.10"
 
   critools:
     description: "CLI tool for Container Runtime Interface (CRI)"

From 8dbf4f3ade005023bf10b8e239db97b94fb8e93a Mon Sep 17 00:00:00 2001
From: Tim Zhang <tim@hyper.sh>
Date: Tue, 15 Jun 2021 11:21:57 +0800
Subject: [PATCH 02/16] agent: Upgrade mio to v0.7.13 to fix epoll_fd leak
 problem

Fixes: #2035
Fixes: tokio-rs/tokio/#3809

Signed-off-by: Tim Zhang <tim@hyper.sh>
---
 src/agent/Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock
index a86bb27a5a..8d9c02404d 100644
--- a/src/agent/Cargo.lock
+++ b/src/agent/Cargo.lock
@@ -579,9 +579,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.7.6"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b"
+checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
 dependencies = [
  "libc",
  "log",

From d1de06c9ea2f78f5676b1e9f62591ebfdee3f098 Mon Sep 17 00:00:00 2001
From: Jakob Naucke <jakob.naucke@ibm.com>
Date: Wed, 16 Jun 2021 11:55:13 +0200
Subject: [PATCH 03/16] virtcontainers: Don't fail memory hotplug

Architectures that do not support memory hotplugging will fail when
memory limits are set because that amount is hotplugged. Issue a warning
instead. The long-term solution is virtio-mem.

Fixes: #1412
Signed-off-by: Jakob Naucke <jakob.naucke@ibm.com>
---
 src/runtime/virtcontainers/qemu.go    | 4 +++-
 src/runtime/virtcontainers/sandbox.go | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go
index c5b26facfc..9e1c4af5a1 100644
--- a/src/runtime/virtcontainers/qemu.go
+++ b/src/runtime/virtcontainers/qemu.go
@@ -126,6 +126,8 @@ const (
 	qemuStopSandboxTimeoutSecs = 15
 )
 
+var noGuestMemHotplugErr error = errors.New("guest memory hotplug not supported")
+
 // agnostic list of kernel parameters
 var defaultKernelParameters = []Param{
 	{"panic", "1"},
@@ -1785,7 +1787,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
 func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) {
 
 	if !q.arch.supportGuestMemoryHotplug() {
-		return 0, fmt.Errorf("guest memory hotplug not supported")
+		return 0, noGuestMemHotplugErr
 	}
 	if memDev.sizeMB < 0 {
 		return 0, fmt.Errorf("cannot hotplug negative size (%d) memory", memDev.sizeMB)
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index 5f2bd56d57..95f68bfeb8 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -1836,7 +1836,11 @@ func (s *Sandbox) updateResources(ctx context.Context) error {
 	s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
 	newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(ctx, uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
 	if err != nil {
-		return err
+		if err == noGuestMemHotplugErr {
+			s.Logger().Warnf("%s, memory specifications cannot be guaranteed", err)
+		} else {
+			return err
+		}
 	}
 	s.Logger().Debugf("Sandbox memory size: %d MB", newMemory)
 	if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 {

From 5454c3d710c3063713c5e912f7cf85a4435e16da Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Wed, 16 Jun 2021 12:43:00 -0400
Subject: [PATCH 04/16] ci: snap: Fetch history to all branches and tags

The snap/snapcraft.yaml set AGENT_VERSION to the current VERSION. The osbuilder script
will try to checkout the AGENT_VERSION tag. Let's ensure that all tags and branches
are fetched by the github's checkout action so the tag checkout does not fail.

Fixes #2052
Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 .github/workflows/snap-release.yaml | 2 ++
 .github/workflows/snap.yaml         | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/snap-release.yaml b/.github/workflows/snap-release.yaml
index 9fca4ec370..bae353cad2 100644
--- a/.github/workflows/snap-release.yaml
+++ b/.github/workflows/snap-release.yaml
@@ -9,6 +9,8 @@ jobs:
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Install Snapcraft
         uses: samuelmeuli/action-snapcraft@v1
diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml
index 40abbde98a..165a3a3bef 100644
--- a/.github/workflows/snap.yaml
+++ b/.github/workflows/snap.yaml
@@ -6,6 +6,8 @@ jobs:
     steps:
       - name: Check out
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Install Snapcraft
         uses: samuelmeuli/action-snapcraft@v1

From 8138a16b8b7a6cab74bcc0a2c6f5fde1fb16820d Mon Sep 17 00:00:00 2001
From: Julio Montes <julio.montes@intel.com>
Date: Fri, 18 Jun 2021 09:02:03 -0500
Subject: [PATCH 05/16] runtime: do not hot-remove PMEM devices

PMEM devices cannot be hot-removed from a running VM.

fixes #2018

Signed-off-by: Julio Montes <julio.montes@intel.com>
---
 src/runtime/virtcontainers/sandbox.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index 95f68bfeb8..dc4d3bf731 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -1711,6 +1711,11 @@ func (s *Sandbox) HotplugRemoveDevice(ctx context.Context, device api.Device, de
 		if !ok {
 			return fmt.Errorf("device type mismatch, expect device type to be %s", devType)
 		}
+		// PMEM devices cannot be hot removed
+		if blockDrive.Pmem {
+			s.Logger().WithField("path", blockDrive.File).Infof("Skip device: cannot hot remove PMEM devices")
+			return nil
+		}
 		_, err := s.hypervisor.hotplugRemoveDevice(ctx, blockDrive, blockDev)
 		return err
 	case config.VhostUserBlk:

From 754e73ca9933a07b175c1686f84c83bbc10c4234 Mon Sep 17 00:00:00 2001
From: "fupan.lfp" <fupan.lfp@antgroup.com>
Date: Wed, 23 Jun 2021 19:20:28 +0800
Subject: [PATCH 06/16] shimv2: fix the issue of leaking wait goroutines

After create an container/exec successfully, containerd
would wait it immediately, and if start it failed, there
is no chance to send value to exitCh, thus the wait goroutine
would blocked for ever and had no chance to exit.

Fixes: #2087

Signed-off-by: fupan.lfp <fupan.lfp@antgroup.com>
---
 src/runtime/containerd-shim-v2/start.go | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/runtime/containerd-shim-v2/start.go b/src/runtime/containerd-shim-v2/start.go
index 72420e411e..60306d428e 100644
--- a/src/runtime/containerd-shim-v2/start.go
+++ b/src/runtime/containerd-shim-v2/start.go
@@ -13,8 +13,14 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 )
 
-func startContainer(ctx context.Context, s *service, c *container) error {
-	//start a container
+func startContainer(ctx context.Context, s *service, c *container) (retErr error) {
+	defer func() {
+		if retErr != nil {
+			// notify the wait goroutine to continue
+			c.exitCh <- exitCode255
+		}
+	}()
+	// start a container
 	if c.cType == "" {
 		err := fmt.Errorf("Bug, the container %s type is empty", c.id)
 		return err
@@ -87,8 +93,8 @@ func startContainer(ctx context.Context, s *service, c *container) error {
 	return nil
 }
 
-func startExec(ctx context.Context, s *service, containerID, execID string) (*exec, error) {
-	//start an exec
+func startExec(ctx context.Context, s *service, containerID, execID string) (e *exec, retErr error) {
+	// start an exec
 	c, err := s.getContainer(containerID)
 	if err != nil {
 		return nil, err
@@ -99,6 +105,13 @@ func startExec(ctx context.Context, s *service, containerID, execID string) (*ex
 		return nil, err
 	}
 
+	defer func() {
+		if retErr != nil {
+			// notify the wait goroutine to continue
+			execs.exitCh <- exitCode255
+		}
+	}()
+
 	_, proc, err := s.sandbox.EnterContainer(ctx, containerID, *execs.cmds)
 	if err != nil {
 		err := fmt.Errorf("cannot enter container %s, with err %s", containerID, err)

From 0fd747c75207aca747f9d12fe21edae1ea963a8c Mon Sep 17 00:00:00 2001
From: bin <bin@hyper.sh>
Date: Wed, 30 Jun 2021 17:08:27 +0800
Subject: [PATCH 07/16] agent: update netlink libraries

Update rtnetlink to use crate.io to make cargo vendor work.
Add vendor/ to .gitignore.

Fixes: #2111

Signed-off-by: bin <bin@hyper.sh>
---
 src/agent/.gitignore |  1 +
 src/agent/Cargo.lock | 58 +++++++++++++++-----------------------------
 src/agent/Cargo.toml | 10 +++-----
 3 files changed, 25 insertions(+), 44 deletions(-)

diff --git a/src/agent/.gitignore b/src/agent/.gitignore
index 581de56347..ac1ee857f1 100644
--- a/src/agent/.gitignore
+++ b/src/agent/.gitignore
@@ -1 +1,2 @@
 tarpaulin-report.html
+vendor/
diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock
index 8d9c02404d..530fce6e09 100644
--- a/src/agent/Cargo.lock
+++ b/src/agent/Cargo.lock
@@ -472,8 +472,8 @@ dependencies = [
  "libc",
  "log",
  "logging",
- "netlink-packet-utils 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "netlink-sys 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "netlink-packet-utils",
+ "netlink-sys",
  "nix 0.17.0",
  "oci",
  "prctl",
@@ -609,43 +609,34 @@ checksum = "2eb04b9f127583ed176e163fb9ec6f3e793b87e21deedd5734a69386a18a0151"
 [[package]]
 name = "netlink-packet-core"
 version = "0.2.4"
-source = "git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326#a9367bc4700496ddebc088110c28f40962923326"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac48279d5062bdf175bdbcb6b58ff1d6b0ecd54b951f7a0ff4bc0550fe903ccb"
 dependencies = [
  "anyhow",
  "byteorder",
  "libc",
- "netlink-packet-utils 0.4.0 (git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326)",
+ "netlink-packet-utils",
 ]
 
 [[package]]
 name = "netlink-packet-route"
-version = "0.7.0"
-source = "git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326#a9367bc4700496ddebc088110c28f40962923326"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c92a86a6528fe6d0a811c48d28213ca896a2b8bf2f6cadf2ab5bb12d32ec0f1"
 dependencies = [
  "anyhow",
  "bitflags",
  "byteorder",
  "libc",
  "netlink-packet-core",
- "netlink-packet-utils 0.4.0 (git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326)",
+ "netlink-packet-utils",
 ]
 
 [[package]]
 name = "netlink-packet-utils"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c2afb159d0e3ac700e85f0df25b8438b99d43ed0c0b685242fcdf1b5673e54d"
-dependencies = [
- "anyhow",
- "byteorder",
- "paste",
- "thiserror",
-]
-
-[[package]]
-name = "netlink-packet-utils"
-version = "0.4.0"
-source = "git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326#a9367bc4700496ddebc088110c28f40962923326"
+checksum = "5fcfb6f758b66e964b2339596d94078218d96aad5b32003e8e2a1d23c27a6784"
 dependencies = [
  "anyhow",
  "byteorder",
@@ -655,34 +646,24 @@ dependencies = [
 
 [[package]]
 name = "netlink-proto"
-version = "0.6.0"
-source = "git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326#a9367bc4700496ddebc088110c28f40962923326"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd06e90449ae973fe3888c1ff85949604ef5189b4ac9a2ae39518da1e00762d"
 dependencies = [
  "bytes 1.0.1",
  "futures",
  "log",
  "netlink-packet-core",
- "netlink-sys 0.6.0 (git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326)",
+ "netlink-sys",
  "tokio",
  "tokio-util",
 ]
 
 [[package]]
 name = "netlink-sys"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d61c5374735aa0cd07cb7fd820b656062b187b5588d79517f72956b57c6de9ef"
-dependencies = [
- "futures",
- "libc",
- "log",
- "tokio",
-]
-
-[[package]]
-name = "netlink-sys"
-version = "0.6.0"
-source = "git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326#a9367bc4700496ddebc088110c28f40962923326"
+checksum = "f48ea34ea0678719815c3753155067212f853ad2d8ef4a49167bae7f7c254188"
 dependencies = [
  "futures",
  "libc",
@@ -1144,8 +1125,9 @@ dependencies = [
 
 [[package]]
 name = "rtnetlink"
-version = "0.7.0"
-source = "git+https://github.com/little-dude/netlink?rev=a9367bc4700496ddebc088110c28f40962923326#a9367bc4700496ddebc088110c28f40962923326"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "279f7e9a312496b3e726e776cbd1f3102bd5ffe66503c3f44d642f7327995919"
 dependencies = [
  "byteordered",
  "futures",
diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml
index 03fca4ed25..88017791be 100644
--- a/src/agent/Cargo.toml
+++ b/src/agent/Cargo.toml
@@ -23,13 +23,11 @@ regex = "1"
 async-trait = "0.1.42"
 tokio = { version = "1.2.0", features = ["rt", "rt-multi-thread", "sync", "macros", "io-util", "time", "signal", "io-std", "process", "fs"] }
 futures = "0.3.12"
-netlink-sys = { version = "0.6.0", features = ["tokio_socket",]}
 tokio-vsock = "0.3.1"
-# Because the author has no time to maintain the crate, we switch the dependency to github,
-# Once the new version released on crates.io, we switch it back.
-# https://github.com/little-dude/netlink/issues/161
-rtnetlink = { git = "https://github.com/little-dude/netlink", rev = "a9367bc4700496ddebc088110c28f40962923326" }
-netlink-packet-utils = "0.4.0"
+
+netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
+rtnetlink = "0.8.0"
+netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
 # slog:

From aa4a3053a03ea754e95de98fa7471bc25c0e1d02 Mon Sep 17 00:00:00 2001
From: Tim Zhang <tim@hyper.sh>
Date: Fri, 2 Jul 2021 11:39:58 +0800
Subject: [PATCH 08/16] agent: enhance tests of execute_hook

Use which to find the full path of exe before run execute_hook
to avoid error: 'No such file or directory'

Fixes: #2172

Signed-off-by: Tim Zhang <tim@hyper.sh>
---
 src/agent/rustjail/src/container.rs | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
index 5113a4826b..8c9c2f21a6 100644
--- a/src/agent/rustjail/src/container.rs
+++ b/src/agent/rustjail/src/container.rs
@@ -1557,6 +1557,7 @@ mod tests {
     use std::os::unix::fs::MetadataExt;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
+    use tokio::process::Command;
 
     macro_rules! sl {
         () => {
@@ -1564,12 +1565,27 @@ mod tests {
         };
     }
 
+    async fn which(cmd: &str) -> String {
+        let output: std::process::Output = Command::new("which")
+            .arg(cmd)
+            .output()
+            .await
+            .expect("which command failed to run");
+
+        match String::from_utf8(output.stdout) {
+            Ok(v) => v.trim_end_matches('\n').to_string(),
+            Err(e) => panic!("Invalid UTF-8 sequence: {}", e),
+        }
+    }
+
     #[tokio::test]
     async fn test_execute_hook() {
+        let xargs = which("xargs").await;
+
         execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: "/usr/bin/xargs".to_string(),
+                path: xargs,
                 args: vec![],
                 env: vec![],
                 timeout: None,
@@ -1589,10 +1605,12 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute_hook_with_timeout() {
+        let sleep = which("sleep").await;
+
         let res = execute_hook(
             &slog_scope::logger(),
             &Hook {
-                path: "/usr/bin/sleep".to_string(),
+                path: sleep,
                 args: vec!["2".to_string()],
                 env: vec![],
                 timeout: Some(1),

From 2e045fc10518acd75e1fb7816da5c0b9f5b53bd0 Mon Sep 17 00:00:00 2001
From: David Gibson <david@gibson.dropbear.id.au>
Date: Tue, 6 Jul 2021 12:11:29 +1000
Subject: [PATCH 09/16] agent: Fix to parsing of /proc/self/mountinfo

get_mounts() parses /proc/self/mountinfo in order to get the mountpoints
for various cgroup filesystems.  One of the entries in mountinfo is the
"device" for each filesystem, but for virtual filesystems like /proc, /sys
and cgroups, the device entry is arbitrary.  Depending on the exact rootfs
setup, it can end up being "-".

This breaks get_mounts() because it uses " - " as a separator.  There
really is a " - " separator in mountinfo, but in this case the device entry
shows up as a second one.  Fix this, by changing a split to a splitn, which
will effectively only consider the first " - " in the line.

While we're there, make the warning message more useful, by having it
actually show which line it wasn't able to parse.

fixes #2182

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 src/agent/rustjail/src/cgroups/fs/mod.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs
index 04aa575379..b18bfc6ace 100644
--- a/src/agent/rustjail/src/cgroups/fs/mod.rs
+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs
@@ -923,12 +923,12 @@ pub fn get_mounts() -> Result<HashMap<String, String>> {
     let paths = get_paths()?;
 
     for l in fs::read_to_string(MOUNTS)?.lines() {
-        let p: Vec<&str> = l.split(" - ").collect();
+        let p: Vec<&str> = l.splitn(2, " - ").collect();
         let pre: Vec<&str> = p[0].split(' ').collect();
         let post: Vec<&str> = p[1].split(' ').collect();
 
         if post.len() != 3 {
-            warn!(sl!(), "mountinfo corrupted!");
+            warn!(sl!(), "can't parse {} line {:?}", MOUNTS, l);
             continue;
         }
 

From f3cf46a621864fd15bee3019ba956872e022957b Mon Sep 17 00:00:00 2001
From: "fupan.lfp" <fupan.lfp@antgroup.com>
Date: Thu, 8 Jul 2021 15:39:12 +0800
Subject: [PATCH 10/16] shimv2: fix the issue of leaking the hypervisor
 processes

Since we only send an shutdown qmp command to qemu when do
stopSandbox, and didn't wait until qemu process's exit, thus
we'd better to make sure it had exited when shimv2 terminated.
Thus here to do the last cleanup of the hypervisor.

Fixes: #2198

Signed-off-by: fupan.lfp <fupan.lfp@antgroup.com>
---
 src/runtime/containerd-shim-v2/service.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/runtime/containerd-shim-v2/service.go b/src/runtime/containerd-shim-v2/service.go
index efb5d60ff8..6a57e7e90a 100644
--- a/src/runtime/containerd-shim-v2/service.go
+++ b/src/runtime/containerd-shim-v2/service.go
@@ -905,6 +905,11 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 
 	s.cancel()
 
+	// Since we only send an shutdown qmp command to qemu when do stopSandbox, and
+	// didn't wait until qemu process's exit, thus we'd better to make sure it had
+	// exited when shimv2 terminated. Thus here to do the last cleanup of the hypervisor.
+	syscall.Kill(int(s.hpid), syscall.SIGKILL)
+
 	os.Exit(0)
 
 	// This will never be called, but this is only there to make sure the

From 0f7a54adf3c83b47046aba1ab058abd3eb9d0e01 Mon Sep 17 00:00:00 2001
From: "fupan.lfp" <fupan.lfp@antgroup.com>
Date: Sat, 10 Jul 2021 11:21:54 +0800
Subject: [PATCH 11/16] qemu: stop the virtiofsd specifically

We'd better stop the virtiofsd specifically after stop qemu,
instead of depending on the qemu's termination to notify virtiofsd
to exit.

Fixes: #2211

Signed-off-by: fupan.lfp <fupan.lfp@antgroup.com>
---
 src/runtime/virtcontainers/qemu.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go
index 9e1c4af5a1..d69f4dbb1c 100644
--- a/src/runtime/virtcontainers/qemu.go
+++ b/src/runtime/virtcontainers/qemu.go
@@ -716,7 +716,8 @@ func (q *qemu) stopVirtiofsd(ctx context.Context) (err error) {
 
 	// kill virtiofsd
 	if q.state.VirtiofsdPid == 0 {
-		return errors.New("invalid virtiofsd PID(0)")
+		q.Logger().Warn("The virtiofsd had stopped")
+		return nil
 	}
 
 	err = syscall.Kill(q.state.VirtiofsdPid, syscall.SIGKILL)
@@ -1018,6 +1019,10 @@ func (q *qemu) stopSandbox(ctx context.Context, waitOnly bool) error {
 		}
 	}
 
+	if err := q.stopVirtiofsd(ctx); err != nil {
+		return err
+	}
+
 	return nil
 }
 

From e7322e235a975880d382ddcea657cee5e994cec5 Mon Sep 17 00:00:00 2001
From: Snir Sheriber <ssheribe@redhat.com>
Date: Mon, 12 Jul 2021 14:34:44 +0300
Subject: [PATCH 12/16] ci: static checks: use defined target_branch

Fixes: #2214
Signed-off-by: Snir Sheriber <ssheribe@redhat.com>
---
 .github/workflows/static-checks.yaml | 2 +-
 ci/lib.sh                            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml
index 9b8af9c53b..882682bdee 100644
--- a/.github/workflows/static-checks.yaml
+++ b/.github/workflows/static-checks.yaml
@@ -13,7 +13,7 @@ jobs:
       TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
       TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
       RUST_BACKTRACE: "1"
-      target_branch: ${TRAVIS_BRANCH}
+      target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
diff --git a/ci/lib.sh b/ci/lib.sh
index de816a4a5c..1d9aa4b167 100644
--- a/ci/lib.sh
+++ b/ci/lib.sh
@@ -5,7 +5,7 @@
 
 export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
-export branch="${branch:-main}"
+export branch="${target_branch:-main}"
 
 # Clones the tests repository and checkout to the branch pointed out by
 # the global $branch variable.

From c1dcc5f7f159eb11c14a7e3eef849dc97d268377 Mon Sep 17 00:00:00 2001
From: bin <bin@hyper.sh>
Date: Thu, 1 Jul 2021 17:50:22 +0800
Subject: [PATCH 13/16] osbuilder: Skip installing golang for building rootfs

Building rootfs does not depend on golang, delete intalling
golang may save build time.

And there is only rust agent now, the code for golang agent should
be deleted too.

Fixes: #2170

Signed-off-by: bin <bin@hyper.sh>
---
 .../rootfs-builder/centos/Dockerfile.in       |  3 +-
 .../rootfs-builder/clearlinux/Dockerfile.in   |  3 +-
 .../debian/Dockerfile-aarch64.in              |  3 +-
 .../rootfs-builder/debian/Dockerfile.in       |  3 +-
 .../rootfs-builder/fedora/Dockerfile.in       |  3 +-
 .../rootfs-builder/gentoo/Dockerfile.in       |  3 +-
 tools/osbuilder/rootfs-builder/rootfs.sh      | 42 +++------------
 .../rootfs-builder/suse/Dockerfile.in         |  3 +-
 .../template/Dockerfile.template              |  5 +-
 .../ubuntu/Dockerfile-aarch64.in              |  3 +-
 .../rootfs-builder/ubuntu/Dockerfile.in       |  4 +-
 tools/osbuilder/scripts/lib.sh                | 51 ++-----------------
 12 files changed, 24 insertions(+), 102 deletions(-)

diff --git a/tools/osbuilder/rootfs-builder/centos/Dockerfile.in b/tools/osbuilder/rootfs-builder/centos/Dockerfile.in
index f5b004daf0..529bd7ba97 100644
--- a/tools/osbuilder/rootfs-builder/centos/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/centos/Dockerfile.in
@@ -34,7 +34,6 @@ RUN yum -y update && yum install -y \
     vim \
     which
 
+# This will install the proper packages to build Kata components
 @INSTALL_MUSL@
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in b/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in
index 0146b168c7..abbc413474 100644
--- a/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in
@@ -37,7 +37,6 @@ RUN dnf -y update && dnf install -y \
     vim \
     which
 
-# This will install the proper golang to build Kata components
+# This will install the proper packages to build Kata components
 @INSTALL_MUSL@
-@INSTALL_GO@
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in b/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in
index 537fc61f15..4395913f9a 100644
--- a/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in
+++ b/tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in
@@ -29,7 +29,6 @@ RUN apt-get update && apt-get install -y \
     systemd \
     tar \
     vim
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
+# This will install the proper packages to build Kata components
 @INSTALL_MUSL@
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/debian/Dockerfile.in b/tools/osbuilder/rootfs-builder/debian/Dockerfile.in
index 27d7957bd9..489940d447 100644
--- a/tools/osbuilder/rootfs-builder/debian/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/debian/Dockerfile.in
@@ -36,6 +36,5 @@ RUN apt-get update && apt-get --no-install-recommends install -y \
     vim \
     wget
 
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
+# This will install the proper packages to build Kata components
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/fedora/Dockerfile.in b/tools/osbuilder/rootfs-builder/fedora/Dockerfile.in
index 7b914a901f..dac32f5050 100644
--- a/tools/osbuilder/rootfs-builder/fedora/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/fedora/Dockerfile.in
@@ -37,7 +37,6 @@ RUN dnf -y update && dnf install -y \
     vim \
     which
 
-# This will install the proper golang to build Kata components
+# This will install the proper packages to build Kata components
 @INSTALL_MUSL@
-@INSTALL_GO@
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/gentoo/Dockerfile.in b/tools/osbuilder/rootfs-builder/gentoo/Dockerfile.in
index 509d89703f..8a06ff921f 100644
--- a/tools/osbuilder/rootfs-builder/gentoo/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/gentoo/Dockerfile.in
@@ -9,6 +9,5 @@ FROM ${IMAGE_REGISTRY}/gentoo/stage3-amd64:latest
 # This dockerfile needs to provide all the componets need to build a rootfs
 # Install any package need to create a rootfs (package manager, extra tools)
 
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
+# This will install the proper rust to build Kata components
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh
index 9fbe49bad1..2b970d8f81 100755
--- a/tools/osbuilder/rootfs-builder/rootfs.sh
+++ b/tools/osbuilder/rootfs-builder/rootfs.sh
@@ -13,9 +13,6 @@ set -o errtrace
 script_name="${0##*/}"
 script_dir="$(dirname $(readlink -f $0))"
 AGENT_VERSION=${AGENT_VERSION:-}
-GO_AGENT_PKG=${GO_AGENT_PKG:-github.com/kata-containers/agent}
-RUST_AGENT_PKG=${RUST_AGENT_PKG:-github.com/kata-containers/kata-containers}
-RUST_AGENT=${RUST_AGENT:-yes}
 RUST_VERSION="null"
 MUSL_VERSION=${MUSL_VERSION:-"null"}
 AGENT_BIN=${AGENT_BIN:-kata-agent}
@@ -23,7 +20,7 @@ AGENT_INIT=${AGENT_INIT:-no}
 KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
 OSBUILDER_VERSION="unknown"
 DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc}
-GO_VERSION="null"
+# this GOPATH is for installing yq from install_yq.sh
 export GOPATH=${GOPATH:-${HOME}/go}
 LIBC=${LIBC:-musl}
 
@@ -113,9 +110,6 @@ DISTRO_REPO         Use host repositories to install guest packages.
 DOCKER_RUNTIME      Docker runtime to use when USE_DOCKER is set.
                     Default value: runc
 
-GO_AGENT_PKG        URL of the Git repository hosting the agent package.
-                    Default value: ${GO_AGENT_PKG}
-
 GRACEFUL_EXIT       If set, and if the DISTRO configuration specifies a
                     non-empty BUILD_CAN_FAIL variable, do not return with an
                     error code in case any of the build step fails.
@@ -134,12 +128,6 @@ KERNEL_MODULES_DIR  Path to a directory containing kernel modules to include in
 ROOTFS_DIR          Path to the directory that is populated with the rootfs.
                     Default value: <${script_name} path>/rootfs-<DISTRO-name>
 
-RUST_AGENT          When set to "no", build kata-agent from go agent instead of kata-rust-agent
-                    Default value: "yes"
-
-RUST_AGENT_PKG      URL of the Git repository hosting the agent package.
-                    Default value: ${RUST_AGENT_PKG}
-
 USE_DOCKER          If set, build the rootfs inside a container (requires
                     Docker).
                     Default value: <not set>
@@ -291,17 +279,11 @@ compare_versions()
 
 check_env_variables()
 {
-	# Fetch the first element from GOPATH as working directory
-	# as go get only works against the first item in the GOPATH
-	[ -z "$GOPATH" ] && die "GOPATH not set"
+	# this will be mounted to container for using yq on the host side.
 	GOPATH_LOCAL="${GOPATH%%:*}"
 
 	[ "$AGENT_INIT" == "yes" -o "$AGENT_INIT" == "no" ] || die "AGENT_INIT($AGENT_INIT) is invalid (must be yes or no)"
 
-	if [ -z "${AGENT_SOURCE_BIN}" ]; then
-		[ "$RUST_AGENT" == "yes" -o "$RUST_AGENT" == "no" ] || die "RUST_AGENT($RUST_AGENT) is invalid (must be yes or no)"
-	fi
-
 	[ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
 
 	[ -n "${OSBUILDER_VERSION}" ] || die "need osbuilder version"
@@ -346,24 +328,18 @@ build_rootfs_distro()
 
 	mkdir -p ${ROOTFS_DIR}
 
-	detect_go_version ||
-		die "Could not detect the required Go version for AGENT_VERSION='${AGENT_VERSION:-master}'."
-
-	echo "Required Go version: $GO_VERSION"
-
 	# need to detect rustc's version too?
 	detect_rust_version ||
-		die "Could not detect the required rust version for AGENT_VERSION='${AGENT_VERSION:-master}'."
+		die "Could not detect the required rust version for AGENT_VERSION='${AGENT_VERSION:-main}'."
 
 	echo "Required rust version: $RUST_VERSION"
 
 	detect_musl_version ||
-		die "Could not detect the required musl version for AGENT_VERSION='${AGENT_VERSION:-master}'."
+		die "Could not detect the required musl version for AGENT_VERSION='${AGENT_VERSION:-main}'."
 
 	echo "Required musl version: $MUSL_VERSION"
 
 	if [ -z "${USE_DOCKER}" ] && [ -z "${USE_PODMAN}" ]; then
-		#Generate an error if the local Go version is too old
 		info "build directly"
 		build_rootfs ${ROOTFS_DIR}
 	else
@@ -380,7 +356,7 @@ build_rootfs_distro()
 			REGISTRY_ARG="--build-arg IMAGE_REGISTRY=${IMAGE_REGISTRY}"
 		fi
 
-		# setup to install go or rust here
+		# setup to install rust here
 		generate_dockerfile "${distro_config_dir}"
 		"$container_engine" build  \
 			${REGISTRY_ARG} \
@@ -396,12 +372,7 @@ build_rootfs_distro()
 		docker_run_args+=" --runtime ${DOCKER_RUNTIME}"
 
 		if [ -z "${AGENT_SOURCE_BIN}" ] ; then
-			if [ "$RUST_AGENT" == "no" ]; then
-				docker_run_args+=" --env GO_AGENT_PKG=${GO_AGENT_PKG}"
-			else
-				docker_run_args+=" --env RUST_AGENT_PKG=${RUST_AGENT_PKG}"
-			fi
-			docker_run_args+=" --env RUST_AGENT=${RUST_AGENT} -v ${GOPATH_LOCAL}:${GOPATH_LOCAL} --env GOPATH=${GOPATH_LOCAL}"
+			docker_run_args+=" -v ${GOPATH_LOCAL}:${GOPATH_LOCAL} --env GOPATH=${GOPATH_LOCAL}"
 		else
 			docker_run_args+=" --env AGENT_SOURCE_BIN=${AGENT_SOURCE_BIN}"
 			docker_run_args+=" -v ${AGENT_SOURCE_BIN}:${AGENT_SOURCE_BIN}"
@@ -577,7 +548,6 @@ EOT
 		test -r "${HOME}/.cargo/env" && source "${HOME}/.cargo/env"
 		[ "$ARCH" == "aarch64" ] && OLD_PATH=$PATH && export PATH=$PATH:/usr/local/musl/bin
 
-		agent_pkg="${RUST_AGENT_PKG}"
 		agent_dir="${script_dir}/../../../src/agent/"
 		# For now, rust-agent doesn't support seccomp yet.
 		SECCOMP="no"
diff --git a/tools/osbuilder/rootfs-builder/suse/Dockerfile.in b/tools/osbuilder/rootfs-builder/suse/Dockerfile.in
index c64fb1fa44..70948a4b13 100644
--- a/tools/osbuilder/rootfs-builder/suse/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/suse/Dockerfile.in
@@ -15,7 +15,6 @@ COPY install-packages.sh config.sh /
 # RUN commands
 RUN chmod +x /install-packages.sh; /install-packages.sh
 
-# This will install the proper golang to build Kata components
+# This will install the proper packages to build Kata components
 @INSTALL_MUSL@
-@INSTALL_GO@
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/template/Dockerfile.template b/tools/osbuilder/rootfs-builder/template/Dockerfile.template
index d8ff345ab1..b881dac439 100644
--- a/tools/osbuilder/rootfs-builder/template/Dockerfile.template
+++ b/tools/osbuilder/rootfs-builder/template/Dockerfile.template
@@ -13,5 +13,6 @@ FROM ${IMAGE_REGISTRY}/@distro@:@OS_VERSION@
 
 # RUN commands
 
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
+# This will install the proper packages to build Kata components
+@INSTALL_MUSL@
+@INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in
index 9ebbec084f..50673b230c 100644
--- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in
+++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in
@@ -33,7 +33,6 @@ RUN apt-get update && apt-get install -y \
     systemd \
     tar \
     vim
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
+# This will install the proper packages to build Kata components
 @INSTALL_MUSL@
 @INSTALL_RUST@
diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in
index 13c98dc0d2..07bf30ce89 100644
--- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in
+++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in
@@ -40,6 +40,6 @@ RUN apt-get update && apt-get --no-install-recommends install -y \
     tar \
     vim \
     wget
-# This will install the proper golang to build Kata components
-@INSTALL_GO@
+
+# This will install the proper packages to build Kata components
 @INSTALL_RUST@
diff --git a/tools/osbuilder/scripts/lib.sh b/tools/osbuilder/scripts/lib.sh
index 5a7e41544d..05930a88e5 100644
--- a/tools/osbuilder/scripts/lib.sh
+++ b/tools/osbuilder/scripts/lib.sh
@@ -150,6 +150,8 @@ build_rootfs()
 	else
 		DNF="${DNF} --releasever=${OS_VERSION}"
 	fi
+
+	info "install packages for rootfs"
 	$DNF install ${EXTRA_PKGS} ${PACKAGES}
 }
 
@@ -190,14 +192,8 @@ create_summary_file()
 	local agent="${AGENT_DEST}"
 	[ "$AGENT_INIT" = yes ] && agent="${init}"
 
-	local agent_version
-	if [ "${RUST_AGENT}" == "no" ]; then
-		agent_version=$("$agent" --version|awk '{print $NF}')
-	else
-		local -r agentdir="${script_dir}/../../../"
-		agent_version=$(cat ${agentdir}/VERSION)
-	fi
-
+	local -r agentdir="${script_dir}/../../../"
+	local -r agent_version=$(cat ${agentdir}/VERSION)
 
 	cat >"$file"<<-EOT
 	---
@@ -241,37 +237,20 @@ generate_dockerfile()
 	local libc=musl
 	case "$(uname -m)" in
 		"ppc64le")
-			goarch=ppc64le
 			rustarch=powerpc64le
 			muslarch=powerpc64
 			libc=gnu
 			;;
-
-		"aarch64")
-			goarch=arm64
-			;;
 		"s390x")
-			goarch=s390x
 			libc=gnu
 			;;
 
 		*)
-			goarch=amd64
 			;;
 	esac
 
 	[ -n "${http_proxy:-}" ] && readonly set_proxy="RUN sed -i '$ a proxy="${http_proxy:-}"' /etc/dnf/dnf.conf /etc/yum.conf; true"
 
-	curlOptions=("-OL")
-	[ -n "${http_proxy:-}" ] && curlOptions+=("-x ${http_proxy:-}")
-
-	readonly install_go="
-RUN cd /tmp ; curl ${curlOptions[@]} https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${goarch}.tar.gz
-RUN tar -C /usr/ -xzf /tmp/go${GO_VERSION}.linux-${goarch}.tar.gz
-ENV GOROOT=/usr/go
-ENV PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin
-"
-
 	# Rust agent
 	# rust installer should set path apropiately, just in case
 	# install musl for compiling rust-agent
@@ -327,8 +306,6 @@ RUN . /root/.cargo/env; \
 	rustup target install ${rustarch}-unknown-linux-${libc}
 RUN ln -sf /usr/bin/g++ /bin/musl-g++
 "
-	# rust agent still need go to build
-	# because grpc-sys need go to build
 	pushd "${dir}"
 	dockerfile_template="Dockerfile.in"
 	dockerfile_arch_template="Dockerfile-${architecture}.in"
@@ -344,10 +321,8 @@ RUN ln -sf /usr/bin/g++ /bin/musl-g++
 	# also long double representation problem when building musl-libc
 	if [ "${architecture}" == "ppc64le" ]; then
 		sed \
-			-e "s|@GO_VERSION@|${GO_VERSION}|g" \
 			-e "s|@OS_VERSION@|${OS_VERSION:-}|g" \
 			-e "s|@INSTALL_MUSL@||g" \
-			-e "s|@INSTALL_GO@|${install_go//$'\n'/\\n}|g" \
 			-e "s|@INSTALL_RUST@|${install_rust//$'\n'/\\n}|g" \
 			-e "s|@SET_PROXY@|${set_proxy:-}|g" \
 			"${dockerfile_template}" > Dockerfile
@@ -364,10 +339,8 @@ RUN ln -sf /usr/bin/g++ /bin/musl-g++
 			"${dockerfile_template}" > Dockerfile
 	else
 		sed \
-			-e "s|@GO_VERSION@|${GO_VERSION}|g" \
 			-e "s|@OS_VERSION@|${OS_VERSION:-}|g" \
 			-e "s|@INSTALL_MUSL@|${install_musl//$'\n'/\\n}|g" \
-			-e "s|@INSTALL_GO@|${install_go//$'\n'/\\n}|g" \
 			-e "s|@INSTALL_RUST@|${install_rust//$'\n'/\\n}|g" \
 			-e "s|@SET_PROXY@|${set_proxy:-}|g" \
 			"${dockerfile_template}" > Dockerfile
@@ -375,20 +348,6 @@ RUN ln -sf /usr/bin/g++ /bin/musl-g++
 	popd
 }
 
-detect_go_version()
-{
-	info "Detecting go version"
-	typeset yq=$(command -v yq || command -v ${GOPATH}/bin/yq || echo "${GOPATH}/bin/yq")
-	if [ ! -f "$yq" ]; then
-		source "$yq_file"
-	fi
-
-	info "Get Go version from ${kata_versions_file}"
-	GO_VERSION="$(cat "${kata_versions_file}"  | $yq r -X - "languages.golang.meta.newest-version")"
-
-	[ "$?" == "0" ] && [ "$GO_VERSION" != "null" ]
-}
-
 detect_rust_version()
 {
 	info "Detecting agent rust version"
@@ -398,7 +357,7 @@ detect_rust_version()
 	fi
 
 	info "Get rust version from ${kata_versions_file}"
-	RUST_VERSION="$(cat "${kata_versions_file}"  | $yq r -X - "languages.rust.meta.newest-version")"
+	RUST_VERSION="$(cat "${kata_versions_file}" | $yq r -X - "languages.rust.meta.newest-version")"
 
 	[ "$?" == "0" ] && [ "$RUST_VERSION" != "null" ]
 }

From 6456cb282b23250802c6195d64663ff60d8c258f Mon Sep 17 00:00:00 2001
From: Francesco Giudici <fgiudici@redhat.com>
Date: Thu, 22 Jul 2021 14:28:32 +0200
Subject: [PATCH 14/16] osbuilder/scripts: add support to yq version 4 and
 above

yq changed syntax in an incompatible way starting from version 4 and
above. Deal with that.

Fixes: #2297

Signed-off-by: Francesco Giudici <fgiudici@redhat.com>
---
 tools/osbuilder/scripts/lib.sh | 42 +++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/tools/osbuilder/scripts/lib.sh b/tools/osbuilder/scripts/lib.sh
index 05930a88e5..c0d114862b 100644
--- a/tools/osbuilder/scripts/lib.sh
+++ b/tools/osbuilder/scripts/lib.sh
@@ -348,32 +348,52 @@ RUN ln -sf /usr/bin/g++ /bin/musl-g++
 	popd
 }
 
-detect_rust_version()
+get_package_version_from_kata_yaml()
 {
-	info "Detecting agent rust version"
+    local yq_path="$1"
+    local yq_version
+    local yq_args
+
 	typeset -r yq=$(command -v yq || command -v "${GOPATH}/bin/yq" || echo "${GOPATH}/bin/yq")
 	if [ ! -f "$yq" ]; then
 		source "$yq_file"
 	fi
 
-	info "Get rust version from ${kata_versions_file}"
-	RUST_VERSION="$(cat "${kata_versions_file}" | $yq r -X - "languages.rust.meta.newest-version")"
+    yq_version=$($yq -V)
+    case $yq_version in
+    *"version "[1-3]*)
+        yq_args="r -X - ${yq_path}"
+        ;;
+    *)
+        yq_args="e .${yq_path} -"
+        ;;
+    esac
 
-	[ "$?" == "0" ] && [ "$RUST_VERSION" != "null" ]
+	PKG_VERSION="$(cat "${kata_versions_file}" | $yq ${yq_args})"
+
+	[ "$?" == "0" ] && [ "$PKG_VERSION" != "null" ] && echo "$PKG_VERSION" || echo ""
+}
+
+detect_rust_version()
+{
+	info "Detecting agent rust version"
+    local yq_path="languages.rust.meta.newest-version"
+
+	info "Get rust version from ${kata_versions_file}"
+	RUST_VERSION="$(get_package_version_from_kata_yaml "$yq_path")"
+
+	[ -n "$RUST_VERSION" ]
 }
 
 detect_musl_version()
 {
 	info "Detecting musl version"
-	typeset -r yq=$(command -v yq || command -v "${GOPATH}/bin/yq" || echo "${GOPATH}/bin/yq")
-	if [ ! -f "$yq" ]; then
-		source "$yq_file"
-	fi
+    local yq_path="externals.musl.version"
 
 	info "Get musl version from ${kata_versions_file}"
-	MUSL_VERSION="$(cat "${kata_versions_file}"  | $yq r -X - "externals.musl.version")"
+	MUSL_VERSION="$(get_package_version_from_kata_yaml "$yq_path")"
 
-	[ "$?" == "0" ] && [ "$MUSL_VERSION" != "null" ]
+	[ -n "$MUSL_VERSION" ]
 }
 
 before_starting_container() {

From d3bab50496c726711b3cb902a99a500b15888adf Mon Sep 17 00:00:00 2001
From: Julio Montes <julio.montes@intel.com>
Date: Thu, 22 Jul 2021 09:57:23 -0500
Subject: [PATCH 15/16] runtime: virtcontainers: make rootfs image read-only

Improve security by making rootfs image read-only, nobody
will be able to modify it from the guest.

fixes #1916

Signed-off-by: Julio Montes <julio.montes@intel.com>
---
 src/runtime/virtcontainers/qemu_amd64_test.go | 1 +
 src/runtime/virtcontainers/qemu_arch_base.go  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/runtime/virtcontainers/qemu_amd64_test.go b/src/runtime/virtcontainers/qemu_amd64_test.go
index 1d321e9353..ccee1ac216 100644
--- a/src/runtime/virtcontainers/qemu_amd64_test.go
+++ b/src/runtime/virtcontainers/qemu_amd64_test.go
@@ -153,6 +153,7 @@ func TestQemuAmd64AppendImage(t *testing.T) {
 			ID:       "mem0",
 			MemPath:  f.Name(),
 			Size:     (uint64)(imageStat.Size()),
+			ReadOnly: true,
 		},
 	}
 
diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go
index d2ffac4a1d..43684d2c41 100644
--- a/src/runtime/virtcontainers/qemu_arch_base.go
+++ b/src/runtime/virtcontainers/qemu_arch_base.go
@@ -378,6 +378,7 @@ func (q *qemuArchBase) appendNvdimmImage(devices []govmmQemu.Device, path string
 		ID:       "mem0",
 		MemPath:  path,
 		Size:     (uint64)(imageStat.Size()),
+		ReadOnly: true,
 	}
 
 	devices = append(devices, object)

From b54ca3de6a78425b355210d33664eec56af8921a Mon Sep 17 00:00:00 2001
From: Julio Montes <julio.montes@intel.com>
Date: Thu, 22 Jul 2021 08:46:02 -0500
Subject: [PATCH 16/16] vendor: update govmm

Bring read-only nvdimm support

Shortlog:
335fa81 qemu: fix golangci-lint errors
61b6378 .github/workflows: reimplement github actions CI
9d6e797 go: support go modules
0d21263 qemu: support read-only nvdimm
ff34d28 qemu: Consistent parameter building

Signed-off-by: Julio Montes <julio.montes@intel.com>
(Based-on: 070590fb533afee77bc8e0a0ae376928bbbfb34b)
---
 src/runtime/go.mod                            |   2 +-
 src/runtime/go.sum                            |   4 +-
 .../kata-containers/govmm/qemu/qemu.go        | 357 +++++++++---------
 src/runtime/vendor/modules.txt                |   2 +-
 4 files changed, 188 insertions(+), 177 deletions(-)

diff --git a/src/runtime/go.mod b/src/runtime/go.mod
index fe7c6d9076..e16c6b8b41 100644
--- a/src/runtime/go.mod
+++ b/src/runtime/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.1
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/kata-containers/govmm v0.0.0-20210428163604-f0e9a35308ee
+	github.com/kata-containers/govmm v0.0.0-20210722115311-0173713ea912
 	github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c
diff --git a/src/runtime/go.sum b/src/runtime/go.sum
index 2914977c52..e2bab652cf 100644
--- a/src/runtime/go.sum
+++ b/src/runtime/go.sum
@@ -268,8 +268,8 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kata-containers/govmm v0.0.0-20210428163604-f0e9a35308ee h1:M4N7AdSHgWz/ubV5AZQdeqmK+9Ztpea6oqeXgk8GCHk=
-github.com/kata-containers/govmm v0.0.0-20210428163604-f0e9a35308ee/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk=
+github.com/kata-containers/govmm v0.0.0-20210722115311-0173713ea912 h1:Ejn4L5OXYbdgnh1ghmlKvTPEC1b415PQDsNhzA2kWLQ=
+github.com/kata-containers/govmm v0.0.0-20210722115311-0173713ea912/go.mod h1:A6QaNB6N6PRQ9mTRpFtUxiF5T5CJpzLALjxBrUQPlFI=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go
index 61ab2cc20b..b39ebbb4ba 100644
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go
@@ -258,6 +258,9 @@ type Object struct {
 
 	// File is the device file
 	File string
+
+	// ReadOnly specifies whether `MemPath` is opened read-only or read/write (default)
+	ReadOnly bool
 }
 
 // Valid returns true if the Object structure is valid and complete.
@@ -287,30 +290,39 @@ func (object Object) QemuParams(config *Config) []string {
 	var qemuParams []string
 
 	deviceParams = append(deviceParams, string(object.Driver))
-	deviceParams = append(deviceParams, fmt.Sprintf(",id=%s", object.DeviceID))
+	deviceParams = append(deviceParams, fmt.Sprintf("id=%s", object.DeviceID))
 
 	switch object.Type {
 	case MemoryBackendFile:
 		objectParams = append(objectParams, string(object.Type))
-		objectParams = append(objectParams, fmt.Sprintf(",id=%s", object.ID))
-		objectParams = append(objectParams, fmt.Sprintf(",mem-path=%s", object.MemPath))
-		objectParams = append(objectParams, fmt.Sprintf(",size=%d", object.Size))
+		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
+		objectParams = append(objectParams, fmt.Sprintf("mem-path=%s", object.MemPath))
+		objectParams = append(objectParams, fmt.Sprintf("size=%d", object.Size))
 
-		deviceParams = append(deviceParams, fmt.Sprintf(",memdev=%s", object.ID))
+		deviceParams = append(deviceParams, fmt.Sprintf("memdev=%s", object.ID))
+		if object.ReadOnly {
+			objectParams = append(objectParams, "readonly=on")
+			deviceParams = append(deviceParams, "unarmed=on")
+		}
 	case TDXGuest:
 		objectParams = append(objectParams, string(object.Type))
-		objectParams = append(objectParams, fmt.Sprintf(",id=%s", object.ID))
+		objectParams = append(objectParams, fmt.Sprintf("id=%s", object.ID))
 		if object.Debug {
-			objectParams = append(objectParams, ",debug=on")
+			objectParams = append(objectParams, "debug=on")
 		}
-		deviceParams = append(deviceParams, fmt.Sprintf(",file=%s", object.File))
+		deviceParams = append(deviceParams, fmt.Sprintf("file=%s", object.File))
+
 	}
 
-	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	if len(deviceParams) > 0 {
+		qemuParams = append(qemuParams, "-device")
+		qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
+	}
 
-	qemuParams = append(qemuParams, "-object")
-	qemuParams = append(qemuParams, strings.Join(objectParams, ""))
+	if len(objectParams) > 0 {
+		qemuParams = append(qemuParams, "-object")
+		qemuParams = append(qemuParams, strings.Join(objectParams, ","))
+	}
 
 	return qemuParams
 }
@@ -425,34 +437,34 @@ func (fsdev FSDevice) QemuParams(config *Config) []string {
 
 	deviceParams = append(deviceParams, fsdev.deviceName(config))
 	if s := fsdev.Transport.disableModern(config, fsdev.DisableModern); s != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",%s", s))
+		deviceParams = append(deviceParams, s)
 	}
-	deviceParams = append(deviceParams, fmt.Sprintf(",fsdev=%s", fsdev.ID))
-	deviceParams = append(deviceParams, fmt.Sprintf(",mount_tag=%s", fsdev.MountTag))
+	deviceParams = append(deviceParams, fmt.Sprintf("fsdev=%s", fsdev.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("mount_tag=%s", fsdev.MountTag))
 	if fsdev.Transport.isVirtioPCI(config) && fsdev.ROMFile != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", fsdev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", fsdev.ROMFile))
 	}
 	if fsdev.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
 			deviceParams = append(deviceParams, ",iommu_platform=on")
 		}
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", fsdev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", fsdev.DevNo))
 	}
 
 	fsParams = append(fsParams, string(fsdev.FSDriver))
-	fsParams = append(fsParams, fmt.Sprintf(",id=%s", fsdev.ID))
-	fsParams = append(fsParams, fmt.Sprintf(",path=%s", fsdev.Path))
-	fsParams = append(fsParams, fmt.Sprintf(",security_model=%s", fsdev.SecurityModel))
+	fsParams = append(fsParams, fmt.Sprintf("id=%s", fsdev.ID))
+	fsParams = append(fsParams, fmt.Sprintf("path=%s", fsdev.Path))
+	fsParams = append(fsParams, fmt.Sprintf("security_model=%s", fsdev.SecurityModel))
 
 	if fsdev.Multidev != "" {
-		fsParams = append(fsParams, fmt.Sprintf(",multidevs=%s", fsdev.Multidev))
+		fsParams = append(fsParams, fmt.Sprintf("multidevs=%s", fsdev.Multidev))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	qemuParams = append(qemuParams, "-fsdev")
-	qemuParams = append(qemuParams, strings.Join(fsParams, ""))
+	qemuParams = append(qemuParams, strings.Join(fsParams, ","))
 
 	return qemuParams
 }
@@ -551,41 +563,41 @@ func (cdev CharDevice) QemuParams(config *Config) []string {
 	deviceParams = append(deviceParams, cdev.deviceName(config))
 	if cdev.Driver == VirtioSerial {
 		if s := cdev.Transport.disableModern(config, cdev.DisableModern); s != "" {
-			deviceParams = append(deviceParams, fmt.Sprintf(",%s", s))
+			deviceParams = append(deviceParams, s)
 		}
 	}
 	if cdev.Bus != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",bus=%s", cdev.Bus))
+		deviceParams = append(deviceParams, fmt.Sprintf("bus=%s", cdev.Bus))
 	}
-	deviceParams = append(deviceParams, fmt.Sprintf(",chardev=%s", cdev.ID))
-	deviceParams = append(deviceParams, fmt.Sprintf(",id=%s", cdev.DeviceID))
+	deviceParams = append(deviceParams, fmt.Sprintf("chardev=%s", cdev.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("id=%s", cdev.DeviceID))
 	if cdev.Name != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",name=%s", cdev.Name))
+		deviceParams = append(deviceParams, fmt.Sprintf("name=%s", cdev.Name))
 	}
 	if cdev.Driver == VirtioSerial && cdev.Transport.isVirtioPCI(config) && cdev.ROMFile != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", cdev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", cdev.ROMFile))
 	}
 
 	if cdev.Driver == VirtioSerial && cdev.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
-			deviceParams = append(deviceParams, ",iommu_platform=on")
+			deviceParams = append(deviceParams, "iommu_platform=on")
 		}
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", cdev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", cdev.DevNo))
 	}
 
 	cdevParams = append(cdevParams, string(cdev.Backend))
-	cdevParams = append(cdevParams, fmt.Sprintf(",id=%s", cdev.ID))
+	cdevParams = append(cdevParams, fmt.Sprintf("id=%s", cdev.ID))
 	if cdev.Backend == Socket {
-		cdevParams = append(cdevParams, fmt.Sprintf(",path=%s,server=on,wait=off", cdev.Path))
+		cdevParams = append(cdevParams, fmt.Sprintf("path=%s,server=on,wait=off", cdev.Path))
 	} else {
-		cdevParams = append(cdevParams, fmt.Sprintf(",path=%s", cdev.Path))
+		cdevParams = append(cdevParams, fmt.Sprintf("path=%s", cdev.Path))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	qemuParams = append(qemuParams, "-chardev")
-	qemuParams = append(qemuParams, strings.Join(cdevParams, ""))
+	qemuParams = append(qemuParams, strings.Join(cdevParams, ","))
 
 	return qemuParams
 }
@@ -781,7 +793,7 @@ func (netdev NetDevice) Valid() bool {
 // multi-queue option mq needs to be activated. See comment in libvirt code at
 // https://github.com/libvirt/libvirt/blob/6e7e965dcd3d885739129b1454ce19e819b54c25/src/qemu/qemu_command.c#L3633
 func (netdev NetDevice) mqParameter(config *Config) string {
-	p := []string{",mq=on"}
+	p := []string{"mq=on"}
 
 	if netdev.Transport.isVirtioPCI(config) {
 		// https://www.linux-kvm.org/page/Multiqueue
@@ -794,10 +806,10 @@ func (netdev NetDevice) mqParameter(config *Config) string {
 		// The agent implementation should do this to ensure that it is
 		// always set
 		vectors := len(netdev.FDs)*2 + 2
-		p = append(p, fmt.Sprintf(",vectors=%d", vectors))
+		p = append(p, fmt.Sprintf("vectors=%d", vectors))
 	}
 
-	return strings.Join(p, "")
+	return strings.Join(p, ",")
 }
 
 // QemuDeviceParams returns the -device parameters for this network device
@@ -810,21 +822,21 @@ func (netdev NetDevice) QemuDeviceParams(config *Config) []string {
 	}
 
 	deviceParams = append(deviceParams, fmt.Sprintf("driver=%s", driver))
-	deviceParams = append(deviceParams, fmt.Sprintf(",netdev=%s", netdev.ID))
-	deviceParams = append(deviceParams, fmt.Sprintf(",mac=%s", netdev.MACAddress))
+	deviceParams = append(deviceParams, fmt.Sprintf("netdev=%s", netdev.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("mac=%s", netdev.MACAddress))
 
 	if netdev.Bus != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",bus=%s", netdev.Bus))
+		deviceParams = append(deviceParams, fmt.Sprintf("bus=%s", netdev.Bus))
 	}
 
 	if netdev.Addr != "" {
 		addr, err := strconv.Atoi(netdev.Addr)
 		if err == nil && addr >= 0 {
-			deviceParams = append(deviceParams, fmt.Sprintf(",addr=%x", addr))
+			deviceParams = append(deviceParams, fmt.Sprintf("addr=%x", addr))
 		}
 	}
 	if s := netdev.Transport.disableModern(config, netdev.DisableModern); s != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",%s", s))
+		deviceParams = append(deviceParams, s)
 	}
 
 	if len(netdev.FDs) > 0 {
@@ -833,14 +845,14 @@ func (netdev NetDevice) QemuDeviceParams(config *Config) []string {
 	}
 
 	if netdev.Transport.isVirtioPCI(config) && netdev.ROMFile != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", netdev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", netdev.ROMFile))
 	}
 
 	if netdev.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
-			deviceParams = append(deviceParams, ",iommu_platform=on")
+			deviceParams = append(deviceParams, "iommu_platform=on")
 		}
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", netdev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", netdev.DevNo))
 	}
 
 	return deviceParams
@@ -856,17 +868,17 @@ func (netdev NetDevice) QemuNetdevParams(config *Config) []string {
 	}
 
 	netdevParams = append(netdevParams, netdevType)
-	netdevParams = append(netdevParams, fmt.Sprintf(",id=%s", netdev.ID))
+	netdevParams = append(netdevParams, fmt.Sprintf("id=%s", netdev.ID))
 
 	if netdev.VHost {
-		netdevParams = append(netdevParams, ",vhost=on")
+		netdevParams = append(netdevParams, "vhost=on")
 		if len(netdev.VhostFDs) > 0 {
 			var fdParams []string
 			qemuFDs := config.appendFDs(netdev.VhostFDs)
 			for _, fd := range qemuFDs {
 				fdParams = append(fdParams, fmt.Sprintf("%d", fd))
 			}
-			netdevParams = append(netdevParams, fmt.Sprintf(",vhostfds=%s", strings.Join(fdParams, ":")))
+			netdevParams = append(netdevParams, fmt.Sprintf("vhostfds=%s", strings.Join(fdParams, ":")))
 		}
 	}
 
@@ -878,15 +890,15 @@ func (netdev NetDevice) QemuNetdevParams(config *Config) []string {
 			fdParams = append(fdParams, fmt.Sprintf("%d", fd))
 		}
 
-		netdevParams = append(netdevParams, fmt.Sprintf(",fds=%s", strings.Join(fdParams, ":")))
+		netdevParams = append(netdevParams, fmt.Sprintf("fds=%s", strings.Join(fdParams, ":")))
 
 	} else {
-		netdevParams = append(netdevParams, fmt.Sprintf(",ifname=%s", netdev.IFName))
+		netdevParams = append(netdevParams, fmt.Sprintf("ifname=%s", netdev.IFName))
 		if netdev.DownScript != "" {
-			netdevParams = append(netdevParams, fmt.Sprintf(",downscript=%s", netdev.DownScript))
+			netdevParams = append(netdevParams, fmt.Sprintf("downscript=%s", netdev.DownScript))
 		}
 		if netdev.Script != "" {
-			netdevParams = append(netdevParams, fmt.Sprintf(",script=%s", netdev.Script))
+			netdevParams = append(netdevParams, fmt.Sprintf("script=%s", netdev.Script))
 		}
 	}
 	return netdevParams
@@ -907,7 +919,7 @@ func (netdev NetDevice) QemuParams(config *Config) []string {
 		netdevParams = netdev.QemuNetdevParams(config)
 		if netdevParams != nil {
 			qemuParams = append(qemuParams, "-netdev")
-			qemuParams = append(qemuParams, strings.Join(netdevParams, ""))
+			qemuParams = append(qemuParams, strings.Join(netdevParams, ","))
 		}
 	}
 
@@ -915,7 +927,7 @@ func (netdev NetDevice) QemuParams(config *Config) []string {
 		deviceParams = netdev.QemuDeviceParams(config)
 		if deviceParams != nil {
 			qemuParams = append(qemuParams, "-device")
-			qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+			qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 		}
 	}
 
@@ -962,25 +974,25 @@ func (dev SerialDevice) QemuParams(config *Config) []string {
 
 	deviceParams = append(deviceParams, dev.deviceName(config))
 	if s := dev.Transport.disableModern(config, dev.DisableModern); s != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",%s", s))
+		deviceParams = append(deviceParams, s)
 	}
-	deviceParams = append(deviceParams, fmt.Sprintf(",id=%s", dev.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("id=%s", dev.ID))
 	if dev.Transport.isVirtioPCI(config) && dev.ROMFile != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", dev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", dev.ROMFile))
 		if dev.Driver == VirtioSerial && dev.MaxPorts != 0 {
-			deviceParams = append(deviceParams, fmt.Sprintf(",max_ports=%d", dev.MaxPorts))
+			deviceParams = append(deviceParams, fmt.Sprintf("max_ports=%d", dev.MaxPorts))
 		}
 	}
 
 	if dev.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
-			deviceParams = append(deviceParams, ",iommu_platform=on")
+			deviceParams = append(deviceParams, "iommu_platform=on")
 		}
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", dev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", dev.DevNo))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1085,46 +1097,46 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string {
 
 	deviceParams = append(deviceParams, blkdev.deviceName(config))
 	if s := blkdev.Transport.disableModern(config, blkdev.DisableModern); s != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",%s", s))
+		deviceParams = append(deviceParams, s)
 	}
-	deviceParams = append(deviceParams, fmt.Sprintf(",drive=%s", blkdev.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("drive=%s", blkdev.ID))
 	if !blkdev.SCSI {
-		deviceParams = append(deviceParams, ",scsi=off")
+		deviceParams = append(deviceParams, "scsi=off")
 	}
 
 	if !blkdev.WCE {
-		deviceParams = append(deviceParams, ",config-wce=off")
+		deviceParams = append(deviceParams, "config-wce=off")
 	}
 
 	if blkdev.Transport.isVirtioPCI(config) && blkdev.ROMFile != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", blkdev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", blkdev.ROMFile))
 	}
 
 	if blkdev.Transport.isVirtioCCW(config) {
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", blkdev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", blkdev.DevNo))
 	}
 
 	if blkdev.ShareRW {
-		deviceParams = append(deviceParams, fmt.Sprintf(",share-rw=on"))
+		deviceParams = append(deviceParams, "share-rw=on")
 	}
 
-	deviceParams = append(deviceParams, fmt.Sprintf(",serial=%s", blkdev.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("serial=%s", blkdev.ID))
 
 	blkParams = append(blkParams, fmt.Sprintf("id=%s", blkdev.ID))
-	blkParams = append(blkParams, fmt.Sprintf(",file=%s", blkdev.File))
-	blkParams = append(blkParams, fmt.Sprintf(",aio=%s", blkdev.AIO))
-	blkParams = append(blkParams, fmt.Sprintf(",format=%s", blkdev.Format))
-	blkParams = append(blkParams, fmt.Sprintf(",if=%s", blkdev.Interface))
+	blkParams = append(blkParams, fmt.Sprintf("file=%s", blkdev.File))
+	blkParams = append(blkParams, fmt.Sprintf("aio=%s", blkdev.AIO))
+	blkParams = append(blkParams, fmt.Sprintf("format=%s", blkdev.Format))
+	blkParams = append(blkParams, fmt.Sprintf("if=%s", blkdev.Interface))
 
 	if blkdev.ReadOnly {
-		blkParams = append(blkParams, ",readonly")
+		blkParams = append(blkParams, "readonly")
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	qemuParams = append(qemuParams, "-drive")
-	qemuParams = append(qemuParams, strings.Join(blkParams, ""))
+	qemuParams = append(qemuParams, strings.Join(blkParams, ","))
 
 	return qemuParams
 }
@@ -1184,14 +1196,14 @@ func (dev LoaderDevice) Valid() bool {
 // QemuParams returns the qemu parameters built out of this loader device.
 func (dev LoaderDevice) QemuParams(config *Config) []string {
 	var qemuParams []string
-	var devParams []string
+	var deviceParams []string
 
-	devParams = append(devParams, "loader")
-	devParams = append(devParams, fmt.Sprintf("file=%s", dev.File))
-	devParams = append(devParams, fmt.Sprintf("id=%s", dev.ID))
+	deviceParams = append(deviceParams, "loader")
+	deviceParams = append(deviceParams, fmt.Sprintf("file=%s", dev.File))
+	deviceParams = append(deviceParams, fmt.Sprintf("id=%s", dev.ID))
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(devParams, ","))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1282,7 +1294,7 @@ func (vhostuserDev VhostUserDevice) Valid() bool {
 func (vhostuserDev VhostUserDevice) QemuNetParams(config *Config) []string {
 	var qemuParams []string
 	var netParams []string
-	var devParams []string
+	var deviceParams []string
 
 	driver := vhostuserDev.deviceName(config)
 	if driver == "" {
@@ -1294,18 +1306,18 @@ func (vhostuserDev VhostUserDevice) QemuNetParams(config *Config) []string {
 	netParams = append(netParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
 	netParams = append(netParams, "vhostforce")
 
-	devParams = append(devParams, driver)
-	devParams = append(devParams, fmt.Sprintf("netdev=%s", vhostuserDev.TypeDevID))
-	devParams = append(devParams, fmt.Sprintf("mac=%s", vhostuserDev.Address))
+	deviceParams = append(deviceParams, driver)
+	deviceParams = append(deviceParams, fmt.Sprintf("netdev=%s", vhostuserDev.TypeDevID))
+	deviceParams = append(deviceParams, fmt.Sprintf("mac=%s", vhostuserDev.Address))
 
 	if vhostuserDev.Transport.isVirtioPCI(config) && vhostuserDev.ROMFile != "" {
-		devParams = append(devParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
 	}
 
 	qemuParams = append(qemuParams, "-netdev")
 	qemuParams = append(qemuParams, strings.Join(netParams, ","))
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(devParams, ","))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1313,23 +1325,23 @@ func (vhostuserDev VhostUserDevice) QemuNetParams(config *Config) []string {
 // QemuSCSIParams builds QEMU device parameters for a VhostUserSCSI device
 func (vhostuserDev VhostUserDevice) QemuSCSIParams(config *Config) []string {
 	var qemuParams []string
-	var devParams []string
+	var deviceParams []string
 
 	driver := vhostuserDev.deviceName(config)
 	if driver == "" {
 		return nil
 	}
 
-	devParams = append(devParams, driver)
-	devParams = append(devParams, fmt.Sprintf("id=%s", vhostuserDev.TypeDevID))
-	devParams = append(devParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
+	deviceParams = append(deviceParams, driver)
+	deviceParams = append(deviceParams, fmt.Sprintf("id=%s", vhostuserDev.TypeDevID))
+	deviceParams = append(deviceParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
 
 	if vhostuserDev.Transport.isVirtioPCI(config) && vhostuserDev.ROMFile != "" {
-		devParams = append(devParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(devParams, ","))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1337,24 +1349,24 @@ func (vhostuserDev VhostUserDevice) QemuSCSIParams(config *Config) []string {
 // QemuBlkParams builds QEMU device parameters for a VhostUserBlk device
 func (vhostuserDev VhostUserDevice) QemuBlkParams(config *Config) []string {
 	var qemuParams []string
-	var devParams []string
+	var deviceParams []string
 
 	driver := vhostuserDev.deviceName(config)
 	if driver == "" {
 		return nil
 	}
 
-	devParams = append(devParams, driver)
-	devParams = append(devParams, "logical_block_size=4096")
-	devParams = append(devParams, "size=512M")
-	devParams = append(devParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
+	deviceParams = append(deviceParams, driver)
+	deviceParams = append(deviceParams, "logical_block_size=4096")
+	deviceParams = append(deviceParams, "size=512M")
+	deviceParams = append(deviceParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
 
 	if vhostuserDev.Transport.isVirtioPCI(config) && vhostuserDev.ROMFile != "" {
-		devParams = append(devParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(devParams, ","))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1362,31 +1374,31 @@ func (vhostuserDev VhostUserDevice) QemuBlkParams(config *Config) []string {
 // QemuFSParams builds QEMU device parameters for a VhostUserFS device
 func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string {
 	var qemuParams []string
-	var devParams []string
+	var deviceParams []string
 
 	driver := vhostuserDev.deviceName(config)
 	if driver == "" {
 		return nil
 	}
 
-	devParams = append(devParams, driver)
-	devParams = append(devParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
-	devParams = append(devParams, fmt.Sprintf("tag=%s", vhostuserDev.Tag))
+	deviceParams = append(deviceParams, driver)
+	deviceParams = append(deviceParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
+	deviceParams = append(deviceParams, fmt.Sprintf("tag=%s", vhostuserDev.Tag))
 	if vhostuserDev.CacheSize != 0 {
-		devParams = append(devParams, fmt.Sprintf("cache-size=%dM", vhostuserDev.CacheSize))
+		deviceParams = append(deviceParams, fmt.Sprintf("cache-size=%dM", vhostuserDev.CacheSize))
 	}
 	if vhostuserDev.SharedVersions {
-		devParams = append(devParams, "versiontable=/dev/shm/fuse_shared_versions")
+		deviceParams = append(deviceParams, "versiontable=/dev/shm/fuse_shared_versions")
 	}
 	if vhostuserDev.Transport.isVirtioCCW(config) {
-		devParams = append(devParams, fmt.Sprintf("devno=%s", vhostuserDev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", vhostuserDev.DevNo))
 	}
 	if vhostuserDev.Transport.isVirtioPCI(config) && vhostuserDev.ROMFile != "" {
-		devParams = append(devParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vhostuserDev.ROMFile))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(devParams, ","))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1395,7 +1407,7 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string {
 func (vhostuserDev VhostUserDevice) QemuParams(config *Config) []string {
 	var qemuParams []string
 	var charParams []string
-	var devParams []string
+	var deviceParams []string
 
 	charParams = append(charParams, "socket")
 	charParams = append(charParams, fmt.Sprintf("id=%s", vhostuserDev.CharDevID))
@@ -1406,19 +1418,19 @@ func (vhostuserDev VhostUserDevice) QemuParams(config *Config) []string {
 
 	switch vhostuserDev.VhostUserType {
 	case VhostUserNet:
-		devParams = vhostuserDev.QemuNetParams(config)
+		deviceParams = vhostuserDev.QemuNetParams(config)
 	case VhostUserSCSI:
-		devParams = vhostuserDev.QemuSCSIParams(config)
+		deviceParams = vhostuserDev.QemuSCSIParams(config)
 	case VhostUserBlk:
-		devParams = vhostuserDev.QemuBlkParams(config)
+		deviceParams = vhostuserDev.QemuBlkParams(config)
 	case VhostUserFS:
-		devParams = vhostuserDev.QemuFSParams(config)
+		deviceParams = vhostuserDev.QemuFSParams(config)
 	default:
 		return nil
 	}
 
-	if devParams != nil {
-		return append(qemuParams, devParams...)
+	if deviceParams != nil {
+		return append(qemuParams, deviceParams...)
 	}
 
 	return nil
@@ -1591,26 +1603,26 @@ func (vfioDev VFIODevice) QemuParams(config *Config) []string {
 	deviceParams = append(deviceParams, fmt.Sprintf("%s,host=%s", driver, vfioDev.BDF))
 	if vfioDev.Transport.isVirtioPCI(config) {
 		if vfioDev.VendorID != "" {
-			deviceParams = append(deviceParams, fmt.Sprintf(",x-pci-vendor-id=%s", vfioDev.VendorID))
+			deviceParams = append(deviceParams, fmt.Sprintf("x-pci-vendor-id=%s", vfioDev.VendorID))
 		}
 		if vfioDev.DeviceID != "" {
-			deviceParams = append(deviceParams, fmt.Sprintf(",x-pci-device-id=%s", vfioDev.DeviceID))
+			deviceParams = append(deviceParams, fmt.Sprintf("x-pci-device-id=%s", vfioDev.DeviceID))
 		}
 		if vfioDev.ROMFile != "" {
-			deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", vfioDev.ROMFile))
+			deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vfioDev.ROMFile))
 		}
 	}
 
 	if vfioDev.Bus != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",bus=%s", vfioDev.Bus))
+		deviceParams = append(deviceParams, fmt.Sprintf("bus=%s", vfioDev.Bus))
 	}
 
 	if vfioDev.Transport.isVirtioCCW(config) {
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", vfioDev.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", vfioDev.DevNo))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1667,35 +1679,35 @@ func (scsiCon SCSIController) Valid() bool {
 // QemuParams returns the qemu parameters built out of this SCSIController device.
 func (scsiCon SCSIController) QemuParams(config *Config) []string {
 	var qemuParams []string
-	var devParams []string
+	var deviceParams []string
 
 	driver := scsiCon.deviceName(config)
-	devParams = append(devParams, fmt.Sprintf("%s,id=%s", driver, scsiCon.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, scsiCon.ID))
 	if scsiCon.Bus != "" {
-		devParams = append(devParams, fmt.Sprintf("bus=%s", scsiCon.Bus))
+		deviceParams = append(deviceParams, fmt.Sprintf("bus=%s", scsiCon.Bus))
 	}
 	if scsiCon.Addr != "" {
-		devParams = append(devParams, fmt.Sprintf("addr=%s", scsiCon.Addr))
+		deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", scsiCon.Addr))
 	}
 	if s := scsiCon.Transport.disableModern(config, scsiCon.DisableModern); s != "" {
-		devParams = append(devParams, s)
+		deviceParams = append(deviceParams, s)
 	}
 	if scsiCon.IOThread != "" {
-		devParams = append(devParams, fmt.Sprintf("iothread=%s", scsiCon.IOThread))
+		deviceParams = append(deviceParams, fmt.Sprintf("iothread=%s", scsiCon.IOThread))
 	}
 	if scsiCon.Transport.isVirtioPCI(config) && scsiCon.ROMFile != "" {
-		devParams = append(devParams, fmt.Sprintf("romfile=%s", scsiCon.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", scsiCon.ROMFile))
 	}
 
 	if scsiCon.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
-			devParams = append(devParams, ",iommu_platform=on")
+			deviceParams = append(deviceParams, "iommu_platform=on")
 		}
-		devParams = append(devParams, fmt.Sprintf("devno=%s", scsiCon.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", scsiCon.DevNo))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(devParams, ","))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1765,36 +1777,36 @@ func (bridgeDev BridgeDevice) Valid() bool {
 // QemuParams returns the qemu parameters built out of this bridge device.
 func (bridgeDev BridgeDevice) QemuParams(config *Config) []string {
 	var qemuParams []string
-	var deviceParam []string
+	var deviceParams []string
 	var driver DeviceDriver
 
 	switch bridgeDev.Type {
 	case PCIEBridge:
 		driver = PCIePCIBridgeDriver
-		deviceParam = append(deviceParam, fmt.Sprintf("%s,bus=%s,id=%s", driver, bridgeDev.Bus, bridgeDev.ID))
+		deviceParams = append(deviceParams, fmt.Sprintf("%s,bus=%s,id=%s", driver, bridgeDev.Bus, bridgeDev.ID))
 	default:
 		driver = PCIBridgeDriver
 		shpc := "off"
 		if bridgeDev.SHPC {
 			shpc = "on"
 		}
-		deviceParam = append(deviceParam, fmt.Sprintf("%s,bus=%s,id=%s,chassis_nr=%d,shpc=%s", driver, bridgeDev.Bus, bridgeDev.ID, bridgeDev.Chassis, shpc))
+		deviceParams = append(deviceParams, fmt.Sprintf("%s,bus=%s,id=%s,chassis_nr=%d,shpc=%s", driver, bridgeDev.Bus, bridgeDev.ID, bridgeDev.Chassis, shpc))
 	}
 
 	if bridgeDev.Addr != "" {
 		addr, err := strconv.Atoi(bridgeDev.Addr)
 		if err == nil && addr >= 0 {
-			deviceParam = append(deviceParam, fmt.Sprintf(",addr=%x", addr))
+			deviceParams = append(deviceParams, fmt.Sprintf("addr=%x", addr))
 		}
 	}
 
 	var transport VirtioTransport
 	if transport.isVirtioPCI(config) && bridgeDev.ROMFile != "" {
-		deviceParam = append(deviceParam, fmt.Sprintf(",romfile=%s", bridgeDev.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", bridgeDev.ROMFile))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParam, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1857,30 +1869,30 @@ func (vsock VSOCKDevice) QemuParams(config *Config) []string {
 	var qemuParams []string
 
 	driver := vsock.deviceName(config)
-	deviceParams = append(deviceParams, string(driver))
+	deviceParams = append(deviceParams, driver)
 	if s := vsock.Transport.disableModern(config, vsock.DisableModern); s != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",%s", s))
+		deviceParams = append(deviceParams, s)
 	}
 	if vsock.VHostFD != nil {
 		qemuFDs := config.appendFDs([]*os.File{vsock.VHostFD})
-		deviceParams = append(deviceParams, fmt.Sprintf(",vhostfd=%d", qemuFDs[0]))
+		deviceParams = append(deviceParams, fmt.Sprintf("vhostfd=%d", qemuFDs[0]))
 	}
-	deviceParams = append(deviceParams, fmt.Sprintf(",id=%s", vsock.ID))
-	deviceParams = append(deviceParams, fmt.Sprintf(",%s=%d", VSOCKGuestCID, vsock.ContextID))
+	deviceParams = append(deviceParams, fmt.Sprintf("id=%s", vsock.ID))
+	deviceParams = append(deviceParams, fmt.Sprintf("%s=%d", VSOCKGuestCID, vsock.ContextID))
 
 	if vsock.Transport.isVirtioPCI(config) && vsock.ROMFile != "" {
-		deviceParams = append(deviceParams, fmt.Sprintf(",romfile=%s", vsock.ROMFile))
+		deviceParams = append(deviceParams, fmt.Sprintf("romfile=%s", vsock.ROMFile))
 	}
 
 	if vsock.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
-			deviceParams = append(deviceParams, ",iommu_platform=on")
+			deviceParams = append(deviceParams, "iommu_platform=on")
 		}
-		deviceParams = append(deviceParams, fmt.Sprintf(",devno=%s", vsock.DevNo))
+		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", vsock.DevNo))
 	}
 
 	qemuParams = append(qemuParams, "-device")
-	qemuParams = append(qemuParams, strings.Join(deviceParams, ""))
+	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
 
 	return qemuParams
 }
@@ -1947,7 +1959,7 @@ func (v RngDevice) QemuParams(config *Config) []string {
 
 	if v.Transport.isVirtioCCW(config) {
 		if config.Knobs.IOMMUPlatform {
-			deviceParams = append(deviceParams, ",iommu_platform=on")
+			deviceParams = append(deviceParams, "iommu_platform=on")
 		}
 		deviceParams = append(deviceParams, fmt.Sprintf("devno=%s", v.DevNo))
 	}
@@ -2032,7 +2044,7 @@ func (b BalloonDevice) QemuParams(config *Config) []string {
 		deviceParams = append(deviceParams, "deflate-on-oom=off")
 	}
 	if s := b.Transport.disableModern(config, b.DisableModern); s != "" {
-		deviceParams = append(deviceParams, string(s))
+		deviceParams = append(deviceParams, s)
 	}
 	qemuParams = append(qemuParams, "-device")
 	qemuParams = append(qemuParams, strings.Join(deviceParams, ","))
@@ -2283,16 +2295,16 @@ func (fwcfg FwCfg) QemuParams(config *Config) []string {
 			fwcfgParams = append(fwcfgParams, fmt.Sprintf("name=%s", f.Name))
 
 			if f.File != "" {
-				fwcfgParams = append(fwcfgParams, fmt.Sprintf(",file=%s", f.File))
+				fwcfgParams = append(fwcfgParams, fmt.Sprintf("file=%s", f.File))
 			}
 
 			if f.Str != "" {
-				fwcfgParams = append(fwcfgParams, fmt.Sprintf(",string=%s", f.Str))
+				fwcfgParams = append(fwcfgParams, fmt.Sprintf("string=%s", f.Str))
 			}
 		}
 
 		qemuParams = append(qemuParams, "-fw_cfg")
-		qemuParams = append(qemuParams, strings.Join(fwcfgParams, ""))
+		qemuParams = append(qemuParams, strings.Join(fwcfgParams, ","))
 	}
 
 	return qemuParams
@@ -2482,15 +2494,15 @@ func (config *Config) appendMachine() {
 		machineParams = append(machineParams, config.Machine.Type)
 
 		if config.Machine.Acceleration != "" {
-			machineParams = append(machineParams, fmt.Sprintf(",accel=%s", config.Machine.Acceleration))
+			machineParams = append(machineParams, fmt.Sprintf("accel=%s", config.Machine.Acceleration))
 		}
 
 		if config.Machine.Options != "" {
-			machineParams = append(machineParams, fmt.Sprintf(",%s", config.Machine.Options))
+			machineParams = append(machineParams, config.Machine.Options)
 		}
 
 		config.qemuParams = append(config.qemuParams, "-machine")
-		config.qemuParams = append(config.qemuParams, strings.Join(machineParams, ""))
+		config.qemuParams = append(config.qemuParams, strings.Join(machineParams, ","))
 	}
 }
 
@@ -2507,17 +2519,16 @@ func (config *Config) appendQMPSockets() {
 			continue
 		}
 
-		qmpParams := append([]string{}, fmt.Sprintf("%s:", q.Type))
-		qmpParams = append(qmpParams, q.Name)
+		qmpParams := append([]string{}, fmt.Sprintf("%s:%s", q.Type, q.Name))
 		if q.Server {
-			qmpParams = append(qmpParams, ",server=on")
+			qmpParams = append(qmpParams, "server=on")
 			if q.NoWait {
-				qmpParams = append(qmpParams, ",wait=off")
+				qmpParams = append(qmpParams, "wait=off")
 			}
 		}
 
 		config.qemuParams = append(config.qemuParams, "-qmp")
-		config.qemuParams = append(config.qemuParams, strings.Join(qmpParams, ""))
+		config.qemuParams = append(config.qemuParams, strings.Join(qmpParams, ","))
 	}
 }
 
@@ -2545,15 +2556,15 @@ func (config *Config) appendMemory() {
 		memoryParams = append(memoryParams, config.Memory.Size)
 
 		if config.Memory.Slots > 0 {
-			memoryParams = append(memoryParams, fmt.Sprintf(",slots=%d", config.Memory.Slots))
+			memoryParams = append(memoryParams, fmt.Sprintf("slots=%d", config.Memory.Slots))
 		}
 
 		if config.Memory.MaxMem != "" {
-			memoryParams = append(memoryParams, fmt.Sprintf(",maxmem=%s", config.Memory.MaxMem))
+			memoryParams = append(memoryParams, fmt.Sprintf("maxmem=%s", config.Memory.MaxMem))
 		}
 
 		config.qemuParams = append(config.qemuParams, "-m")
-		config.qemuParams = append(config.qemuParams, strings.Join(memoryParams, ""))
+		config.qemuParams = append(config.qemuParams, strings.Join(memoryParams, ","))
 	}
 }
 
@@ -2564,15 +2575,15 @@ func (config *Config) appendCPUs() error {
 		SMPParams = append(SMPParams, fmt.Sprintf("%d", config.SMP.CPUs))
 
 		if config.SMP.Cores > 0 {
-			SMPParams = append(SMPParams, fmt.Sprintf(",cores=%d", config.SMP.Cores))
+			SMPParams = append(SMPParams, fmt.Sprintf("cores=%d", config.SMP.Cores))
 		}
 
 		if config.SMP.Threads > 0 {
-			SMPParams = append(SMPParams, fmt.Sprintf(",threads=%d", config.SMP.Threads))
+			SMPParams = append(SMPParams, fmt.Sprintf("threads=%d", config.SMP.Threads))
 		}
 
 		if config.SMP.Sockets > 0 {
-			SMPParams = append(SMPParams, fmt.Sprintf(",sockets=%d", config.SMP.Sockets))
+			SMPParams = append(SMPParams, fmt.Sprintf("sockets=%d", config.SMP.Sockets))
 		}
 
 		if config.SMP.MaxCPUs > 0 {
@@ -2580,11 +2591,11 @@ func (config *Config) appendCPUs() error {
 				return fmt.Errorf("MaxCPUs %d must be equal to or greater than CPUs %d",
 					config.SMP.MaxCPUs, config.SMP.CPUs)
 			}
-			SMPParams = append(SMPParams, fmt.Sprintf(",maxcpus=%d", config.SMP.MaxCPUs))
+			SMPParams = append(SMPParams, fmt.Sprintf("maxcpus=%d", config.SMP.MaxCPUs))
 		}
 
 		config.qemuParams = append(config.qemuParams, "-smp")
-		config.qemuParams = append(config.qemuParams, strings.Join(SMPParams, ""))
+		config.qemuParams = append(config.qemuParams, strings.Join(SMPParams, ","))
 	}
 
 	return nil
@@ -2600,15 +2611,15 @@ func (config *Config) appendRTC() {
 	RTCParams = append(RTCParams, fmt.Sprintf("base=%s", string(config.RTC.Base)))
 
 	if config.RTC.DriftFix != "" {
-		RTCParams = append(RTCParams, fmt.Sprintf(",driftfix=%s", config.RTC.DriftFix))
+		RTCParams = append(RTCParams, fmt.Sprintf("driftfix=%s", config.RTC.DriftFix))
 	}
 
 	if config.RTC.Clock != "" {
-		RTCParams = append(RTCParams, fmt.Sprintf(",clock=%s", config.RTC.Clock))
+		RTCParams = append(RTCParams, fmt.Sprintf("clock=%s", config.RTC.Clock))
 	}
 
 	config.qemuParams = append(config.qemuParams, "-rtc")
-	config.qemuParams = append(config.qemuParams, strings.Join(RTCParams, ""))
+	config.qemuParams = append(config.qemuParams, strings.Join(RTCParams, ","))
 }
 
 func (config *Config) appendGlobalParam() {
diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt
index 52cbc54f8c..bf345a685a 100644
--- a/src/runtime/vendor/modules.txt
+++ b/src/runtime/vendor/modules.txt
@@ -221,7 +221,7 @@ github.com/hashicorp/errwrap
 # github.com/hashicorp/go-multierror v1.0.0
 ## explicit
 github.com/hashicorp/go-multierror
-# github.com/kata-containers/govmm v0.0.0-20210428163604-f0e9a35308ee
+# github.com/kata-containers/govmm v0.0.0-20210722115311-0173713ea912
 ## explicit
 github.com/kata-containers/govmm/qemu
 # github.com/konsorten/go-windows-terminal-sequences v1.0.1