Merge pull request #1675 from fidencio/2.1.0-alpha2-branch-bump

# Kata Containers 2.1.0-alpha2

.github/workflows/kata-deploy-test.yaml

@@ -1,9 +1,12 @@
-on: issue_comment
+on:
+  issue_comment:
+    types: [created, edited]
+
 name: test-kata-deploy
+
 jobs:
   check_comments:
     if: ${{ github.event.issue.pull_request }}
-    types: [created, edited]
     runs-on: ubuntu-latest
     steps:
       - name: Check for Command
@@ -11,7 +14,7 @@ jobs:
         uses: kata-containers/slash-command-action@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          command: "test-kata-deploy"
+          command: "test_kata_deploy"
           reaction: "true"
           reaction-type: "eyes"
           allow-edits: "false"
@@ -19,6 +22,7 @@ jobs:
       - name: verify command arg is kata-deploy
         run: |
            echo "The command was '${{ steps.command.outputs.command-name }}' with arguments '${{ steps.command.outputs.command-arguments }}'"
+
   create-and-test-container:
     needs: check_comments
     runs-on: ubuntu-latest
@@ -29,22 +33,26 @@ jobs:
             ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
             echo "reference for PR: " ${ref}
             echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2-beta
+
+      - name: check out
+        uses: actions/checkout@v2
         with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
+
       - name: build-container-image
         id: build-container-image
         run: |
             PR_SHA=$(git log --format=format:%H -n1)
-            VERSION=$(curl https://raw.githubusercontent.com/kata-containers/kata-containers/main/VERSION)
+            VERSION="2.0.0"
             ARTIFACT_URL="https://github.com/kata-containers/kata-containers/releases/download/${VERSION}/kata-static-${VERSION}-x86_64.tar.xz"
-            wget "${ARTIFACT_URL}" -O ./kata-deploy/kata-static.tar.xz
-            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./kata-deploy
+            wget "${ARTIFACT_URL}" -O tools/packaging/kata-deploy/kata-static.tar.xz
+            docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:${PR_SHA} ./tools/packaging/kata-deploy
             docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
             docker push katadocker/kata-deploy-ci:$PR_SHA
             echo "##[set-output name=pr-sha;]${PR_SHA}"
+
       - name: test-kata-deploy-ci-in-aks
-        uses: ./kata-deploy/action
+        uses: ./tools/packaging/kata-deploy/action
         with:
           packaging-sha: ${{ steps.build-container-image.outputs.pr-sha }}
         env:

VERSION

@@ -1 +1 @@
-2.1-alpha1
+2.1.0-alpha2

ci/lib.sh

@@ -7,16 +7,25 @@ export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
 export tests_repo_dir="$GOPATH/src/$tests_repo"
 export branch="${branch:-main}"
 
+# Clones the tests repository and checkout to the branch pointed out by
+# the global $branch variable.
+# If the clone exists and `CI` is exported then it does nothing. Otherwise
+# it will clone the repository or `git pull` the latest code.
+#
 clone_tests_repo()
 {
-	if [ -d "$tests_repo_dir" -a -n "$CI" ]
-	then
-		return
+	if [ -d "$tests_repo_dir" ]; then
+		[ -n "$CI" ] && return
+		pushd "${tests_repo_dir}"
+		git checkout "${branch}"
+		git pull
+		popd
+	else
+		git clone -q "https://${tests_repo}" "$tests_repo_dir"
+		pushd "${tests_repo_dir}"
+		git checkout "${branch}"
+		popd
 	fi
-
-	go get -d -u "$tests_repo" || true
-
-	pushd "${tests_repo_dir}" && git checkout "${branch}" && popd
 }
 
 run_static_checks()

ci/openshift-ci/images/Dockerfile.buildroot

@@ -0,0 +1,9 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This is the build root image for Kata Containers on OpenShift CI.
+#
+FROM centos:8
+
+RUN yum -y update && yum -y install git sudo wget

docs/Developer-Guide.md

@@ -1,54 +1,54 @@
-* [Warning](#warning)
-* [Assumptions](#assumptions)
-* [Initial setup](#initial-setup)
-* [Requirements to build individual components](#requirements-to-build-individual-components)
-* [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
-* [Check hardware requirements](#check-hardware-requirements)
-    * [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
-    * [Enable full debug](#enable-full-debug)
-        * [debug logs and shimv2](#debug-logs-and-shimv2)
-            * [Enabling full `containerd` debug](#enabling-full-containerd-debug)
-            * [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
-            * [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
-        * [journald rate limiting](#journald-rate-limiting)
-            * [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
-            * [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
-* [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
-    * [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
-    * [Get the osbuilder](#get-the-osbuilder)
-    * [Create a rootfs image](#create-a-rootfs-image)
-        * [Create a local rootfs](#create-a-local-rootfs)
-        * [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
-        * [Build a rootfs image](#build-a-rootfs-image)
-        * [Install the rootfs image](#install-the-rootfs-image)
-    * [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
-        * [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
-        * [Build an initrd image](#build-an-initrd-image)
-        * [Install the initrd image](#install-the-initrd-image)
-* [Install guest kernel images](#install-guest-kernel-images)
-* [Install a hypervisor](#install-a-hypervisor)
-    * [Build a custom QEMU](#build-a-custom-qemu)
-        * [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
-* [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
-* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
-* [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
-* [Appendices](#appendices)
-    * [Checking Docker default runtime](#checking-docker-default-runtime)
-    * [Set up a debug console](#set-up-a-debug-console)
-      * [Simple debug console setup](#simple-debug-console-setup)
-          * [Enable agent debug console](#enable-agent-debug-console)
-          * [Connect to debug console](#connect-to-debug-console)
-      * [Traditional debug console setup](#traditional-debug-console-setup)
-          * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
-          * [Build the debug image](#build-the-debug-image)
-          * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
-          * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-              * [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
-              * [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
-          * [Create a container](#create-a-container)
-          * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
-    * [Obtain details of the image](#obtain-details-of-the-image)
-    * [Capturing kernel boot logs](#capturing-kernel-boot-logs)
+- [Warning](#warning)
+- [Assumptions](#assumptions)
+- [Initial setup](#initial-setup)
+- [Requirements to build individual components](#requirements-to-build-individual-components)
+- [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime)
+- [Check hardware requirements](#check-hardware-requirements)
+  - [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image)
+  - [Enable full debug](#enable-full-debug)
+    - [debug logs and shimv2](#debug-logs-and-shimv2)
+      - [Enabling full `containerd` debug](#enabling-full-containerd-debug)
+      - [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug)
+      - [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug)
+    - [journald rate limiting](#journald-rate-limiting)
+      - [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages)
+      - [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting)
+- [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image)
+  - [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional)
+  - [Get the osbuilder](#get-the-osbuilder)
+  - [Create a rootfs image](#create-a-rootfs-image)
+    - [Create a local rootfs](#create-a-local-rootfs)
+    - [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional)
+    - [Build a rootfs image](#build-a-rootfs-image)
+    - [Install the rootfs image](#install-the-rootfs-image)
+  - [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional)
+    - [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image)
+    - [Build an initrd image](#build-an-initrd-image)
+    - [Install the initrd image](#install-the-initrd-image)
+- [Install guest kernel images](#install-guest-kernel-images)
+- [Install a hypervisor](#install-a-hypervisor)
+  - [Build a custom QEMU](#build-a-custom-qemu)
+    - [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required)
+- [Run Kata Containers with Containerd](#run-kata-containers-with-containerd)
+- [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes)
+- [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
+- [Appendices](#appendices)
+  - [Checking Docker default runtime](#checking-docker-default-runtime)
+  - [Set up a debug console](#set-up-a-debug-console)
+    - [Simple debug console setup](#simple-debug-console-setup)
+      - [Enable agent debug console](#enable-agent-debug-console)
+      - [Connect to debug console](#connect-to-debug-console)
+    - [Traditional debug console setup](#traditional-debug-console-setup)
+      - [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
+      - [Build the debug image](#build-the-debug-image)
+      - [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image)
+      - [Create a container](#create-a-container)
+      - [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console)
+        - [Enabling debug console for QEMU](#enabling-debug-console-for-qemu)
+        - [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker)
+        - [Connecting to the debug console](#connecting-to-the-debug-console)
+  - [Obtain details of the image](#obtain-details-of-the-image)
+  - [Capturing kernel boot logs](#capturing-kernel-boot-logs)
 
 # Warning
 
@@ -384,22 +384,19 @@ You can build and install the guest kernel image as shown [here](../tools/packag
 
 # Install a hypervisor
 
-When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the `qemu-lite` hypervisor is installed automatically. For other installation methods, you will need to manually install a suitable hypervisor.
+When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the
+`QEMU` VMM is installed automatically. Cloud-Hypervisor and Firecracker VMMs are available from the [release tarballs](https://github.com/kata-containers/kata-containers/releases), as well as through [`kata-deploy`](../tools/packaging/kata-deploy/README.md).
+You may choose to manually build your VMM/hypervisor.
 
 ## Build a custom QEMU
 
-Your QEMU directory need to be prepared with source code. Alternatively, you can use the [Kata containers QEMU](https://github.com/kata-containers/qemu/tree/master) and checkout the recommended branch:
+Kata Containers makes use of upstream QEMU branch. The exact version
+and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
-```
-$ go get -d github.com/kata-containers/qemu
-$ qemu_branch=$(grep qemu-lite- ${GOPATH}/src/github.com/kata-containers/kata-containers/versions.yaml | cut -d '"' -f2)
-$ cd ${GOPATH}/src/github.com/kata-containers/qemu
-$ git checkout -b $qemu_branch remotes/origin/$qemu_branch
-$ your_qemu_directory=${GOPATH}/src/github.com/kata-containers/qemu
-```
-
-To build a version of QEMU using the same options as the default `qemu-lite` version , you could use the `configure-hypervisor.sh` script:
+Kata often utilizes patches for not-yet-upstream fixes for components,
+including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches)
 
+To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
 ```
 $ go get -d github.com/kata-containers/kata-containers/tools/packaging
 $ cd $your_qemu_directory
@@ -409,6 +406,8 @@ $ make -j $(nproc)
 $ sudo -E make install
 ```
 
+See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
+
 ### Build a custom QEMU for aarch64/arm64 - REQUIRED
 > **Note:**
 >
@@ -613,8 +612,11 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons
 > **Note** Ports 1024 and 1025 are reserved for communication with the agent
 > and gathering of agent logs respectively. 
 
-Next, connect to the debug console. The VSOCKS paths vary slightly between
-cloud-hypervisor and firecracker.
+##### Connecting to the debug console
+
+Next, connect to the debug console. The VSOCKS paths vary slightly between each
+VMM solution.
+
 In case of cloud-hypervisor, connect to the `vsock` as shown:
 ```
 $ sudo su -c 'cd /var/run/vc/vm/{sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
@@ -631,6 +633,12 @@ CONNECT 1026
 
 **Note**: You need to press the `RETURN` key to see the shell prompt.
 
+
+For QEMU, connect to the `vsock` as shown:
+```
+$ sudo su -c 'cd /var/run/vc/vm/{sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"
+```
+
 To disconnect from the virtual machine, type `CONTROL+q` (hold down the
 `CONTROL` key and press `q`).
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -95,6 +95,8 @@ There are several kinds of Kata configurations and they are listed below.
 
 In case of CRI-O, all annotations specified in the pod spec are passed down to Kata.
 
+# containerd Configuration
+
 For containerd, annotations specified in the pod spec are passed down to Kata
 starting with version `1.3.0` of containerd. Additionally, extra configuration is
 needed for containerd, by providing a `pod_annotations` field in the containerd config
@@ -107,11 +109,9 @@ for passing annotations to Kata from containerd:
 $ cat /etc/containerd/config
 ....
 
-[plugins.cri.containerd.runtimes.kata]
-           runtime_type = "io.containerd.runc.v1"
+         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
+           runtime_type = "io.containerd.kata.v2"
            pod_annotations = ["io.katacontainers.*"]
-           [plugins.cri.containerd.runtimes.kata.options]
-             BinaryName = "/usr/bin/kata-runtime"
 ....
 
 ```

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -7,9 +7,10 @@
     * [Configure Kubelet to use containerd](#configure-kubelet-to-use-containerd)
     * [Configure HTTP proxy - OPTIONAL](#configure-http-proxy---optional)
 * [Start Kubernetes](#start-kubernetes)
-* [Install a Pod Network](#install-a-pod-network)
+* [Configure Pod Network](#configure-pod-network)
 * [Allow pods to run in the master node](#allow-pods-to-run-in-the-master-node)
-* [Create an untrusted pod using Kata Containers](#create-an-untrusted-pod-using-kata-containers)
+* [Create runtime class for Kata Containers](#create-runtime-class-for-kata-containers)
+* [Run pod in Kata Containers](#run-pod-in-kata-containers)
 * [Delete created pod](#delete-created-pod)
 
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
@@ -18,9 +19,6 @@ The Kubernetes cluster will use the
 [CRI containerd plugin](https://github.com/containerd/cri) and
 [Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
-For Kata Containers 1.5.0-rc2 and above, we will use `containerd-shim-kata-v2` (short as `shimv2` in this documentation)
-to launch Kata Containers. For the previous version of Kata Containers, the Pods are launched with `kata-runtime`.
-
 ## Requirements
 
 - Kubernetes, Kubelet, `kubeadm`
@@ -125,43 +123,33 @@ $ sudo systemctl daemon-reload
   $ sudo -E kubectl get pods
   ```
 
-## Install a Pod Network
+## Configure Pod Network
 
 A pod network plugin is needed to allow pods to communicate with each other.
+You can find more about CNI plugins from the [Creating a cluster with `kubeadm`](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions) guide.
 
-- Install the `flannel` plugin by following the
-  [Using `kubeadm` to Create a Cluster](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#instructions)
-  guide, starting from the **Installing a pod network** section.
-
-- Create a pod network using flannel
-
-  > **Note:** There is no known way to determine programmatically the best version (commit) to use.
-  > See https://github.com/coreos/flannel/issues/995.
+By default the CNI plugin binaries is installed under `/opt/cni/bin` (in package `kubernetes-cni`), you only need to create a configuration file for CNI plugin.
 
   ```bash
-  $ sudo -E kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
-  ```
+  $ sudo -E mkdir -p /etc/cni/net.d
 
-- Wait for the pod network to become available
-
-  ```bash
-  # number of seconds to wait for pod network to become available
-  $ timeout_dns=420
-
-  $ while [ "$timeout_dns" -gt 0 ]; do
-      if sudo -E kubectl get pods --all-namespaces | grep dns | grep Running; then
-          break
-      fi
-
-      sleep 1s
-      ((timeout_dns--))
-   done
-  ```
-
-- Check the pod network is running
-
-  ```bash
-  $ sudo -E kubectl get pods --all-namespaces | grep dns | grep Running && echo "OK" || ( echo "FAIL" && false )
+  $ sudo -E cat > /etc/cni/net.d/10-mynet.conf <<EOF
+  {
+    "cniVersion": "0.2.0",
+    "name": "mynet",
+    "type": "bridge",
+    "bridge": "cni0",
+    "isGateway": true,
+    "ipMasq": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "172.19.0.0/24",
+      "routes": [
+        { "dst": "0.0.0.0/0" }
+      ]
+    }
+  }
+  EOF
   ```
 
 ## Allow pods to run in the master node
@@ -172,24 +160,38 @@ By default, the cluster will not schedule pods in the master node. To enable mas
 $ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
 ```
 
-## Create an untrusted pod using Kata Containers
+## Create runtime class for Kata Containers
 
 By default, all pods are created with the default runtime configured in CRI containerd plugin.
+From Kubernetes v1.12, users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
 
-If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true"`, the CRI plugin runs the pod with the
+```bash
+$ cat > runtime.yaml <<EOF
+apiVersion: node.k8s.io/v1beta1
+kind: RuntimeClass
+metadata:
+  name: kata
+handler: kata
+EOF
+
+$ sudo -E kubectl apply -f runtime.yaml
+```
+
+## Run pod in Kata Containers
+
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
 [Kata Containers runtime](../../src/runtime/README.md).
 
-- Create an untrusted pod configuration
+- Create an pod configuration that using Kata Containers runtime
 
   ```bash
-  $ cat << EOT | tee nginx-untrusted.yaml
+  $ cat << EOT | tee nginx-kata.yaml
   apiVersion: v1
   kind: Pod
   metadata:
-    name: nginx-untrusted
-    annotations:
-      io.kubernetes.cri.untrusted-workload: "true"
+    name: nginx-kata
   spec:
+    runtimeClassName: kata
     containers:
     - name: nginx
       image: nginx
@@ -197,9 +199,9 @@ If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true
   EOT
   ```
 
-- Create an untrusted pod
+- Create the pod
   ```bash
-  $ sudo -E kubectl apply -f nginx-untrusted.yaml
+  $ sudo -E kubectl apply -f nginx-kata.yaml
   ```
 
 - Check pod is running
@@ -216,5 +218,5 @@ If a pod has the `io.kubernetes.cri.untrusted-workload` annotation set to `"true
 ## Delete created pod
 
 ```bash
-$ sudo -E kubectl delete -f nginx-untrusted.yaml
+$ sudo -E kubectl delete -f nginx-kata.yaml
 ```

docs/install/README.md

@@ -50,7 +50,7 @@ Kata packages are provided by official distribution repositories for:
 | Distribution (link to installation guide)                | Minimum versions                                                               |
 |----------------------------------------------------------|--------------------------------------------------------------------------------|
 | [CentOS](centos-installation-guide.md)                   | 8                                                                              |
-| [Fedora](fedora-installation-guide.md)                   | 32, Rawhide                                                                    |
+| [Fedora](fedora-installation-guide.md)                   | 34                                                                             |
 
 > **Note::**
 >

docs/install/centos-installation-guide.md

@@ -3,15 +3,9 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
+   $ sudo -E dnf install -y centos-release-advanced-virtualization
+   $ sudo -E dnf module disable -y virt:rhel
    $ source /etc/os-release
-   $ cat <<EOF | sudo -E tee /etc/yum.repos.d/advanced-virt.repo
-     [advanced-virt]
-     name=Advanced Virtualization
-     baseurl=http://mirror.centos.org/\$contentdir/\$releasever/virt/\$basearch/advanced-virtualization
-     enabled=1
-     gpgcheck=1
-     skip_if_unavailable=1
-     EOF
    $ cat <<EOF | sudo -E tee /etc/yum.repos.d/kata-containers.repo
      [kata-containers]
      name=Kata Containers
@@ -20,8 +14,7 @@
      gpgcheck=1
      skip_if_unavailable=1
      EOF
-   $ sudo -E dnf module disable -y virt:rhel
-   $ sudo -E dnf install -y kata-runtime
+   $ sudo -E dnf install -y kata-containers
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:

docs/install/fedora-installation-guide.md

@@ -3,7 +3,7 @@
 1. Install the Kata Containers components with the following commands:
 
    ```bash
-   $ sudo -E dnf -y install kata-runtime
+   $ sudo -E dnf -y install kata-containers
    ```
 
 2. Decide which container manager to use and select the corresponding link that follows:

docs/install/snap-installation-guide.md

@@ -2,9 +2,6 @@
 
 * [Install Kata Containers](#install-kata-containers)
 * [Configure Kata Containers](#configure-kata-containers)
-* [Integration with non-compatible shim v2 Container Engines](#integration-with-non-compatible-shim-v2-container-engines)
-    * [Integration with Docker](#integration-with-docker)
-    * [Integration with Podman](#integration-with-podman)
 * [Integration with shim v2 Container Engines](#integration-with-shim-v2-container-engines)
 * [Remove Kata Containers snap package](#remove-kata-containers-snap-package)
 
@@ -14,20 +11,7 @@
 Kata Containers can be installed in any Linux distribution that supports
 [snapd](https://docs.snapcraft.io/installing-snapd).
 
-> NOTE: From Kata Containers 2.x, only the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)
-> is supported, note that some container engines (`docker`, `podman`, etc) may not
-> be able to run Kata Containers 2.x.
-
-Kata Containers 1.x is released through the *stable* channel while Kata Containers
-2.x is available in the *candidate* channel.
-
-Run the following command to install **Kata Containers 1.x**:
-
-```sh
-$ sudo snap install kata-containers --classic
-```
-
-Run the following command to install **Kata Containers 2.x**:
+Run the following command to install **Kata Containers**:
 
 ```sh
 $ sudo snap install kata-containers --candidate --classic
@@ -46,55 +30,6 @@ $ sudo cp /snap/kata-containers/current/usr/share/defaults/kata-containers/confi
 $ $EDITOR /etc/kata-containers/configuration.toml
 ```
 
-## Integration with non-compatible shim v2 Container Engines
-
-At the time of writing this document, `docker` and `podman` **do not support Kata
-Containers 2.x, therefore Kata Containers 1.x must be used instead.**
-
-The path to the runtime provided by the Kata Containers 1.x snap package is
-`/snap/bin/kata-containers.runtime`, it should be used to run Kata Containers 1.x.
-
-### Integration with Docker
-
-`/etc/docker/daemon.json` is the configuration file for `docker`, use the
-following configuration to add a new runtime (`kata`) to `docker`.
-
-```json
-{
-  "runtimes": {
-    "kata": {
-      "path": "/snap/bin/kata-containers.runtime"
-    }
-  }
-}
-```
-
-Once the above configuration has been applied, use the
-following commands to restart `docker` and run Kata Containers 1.x.
-
-```sh
-$ sudo systemctl restart docker
-$ docker run -ti --runtime kata busybox sh
-```
-
-### Integration with Podman
-
-`/usr/share/containers/containers.conf` is the configuration file for `podman`,
-add the following configuration in the `[engine.runtimes]` section.
-
-```toml
-kata = [
-   "/snap/bin/kata-containers.runtime"
-]
-```
-
-Once the above configuration has been applied, use the following command to run
-Kata Containers 1.x with `podman`
-
-```sh
-$ sudo podman run -ti --runtime kata docker.io/library/busybox sh
-```
-
 ## Integration with shim v2 Container Engines
 
 The Container engine daemon (`cri-o`, `containerd`, etc) needs to be able to find the

docs/install/ubuntu-installation-guide.md

@@ -1,15 +0,0 @@
-# Install Kata Containers on Ubuntu
-
-1. Install the Kata Containers components with the following commands:
-
-   ```bash
-   $ ARCH=$(arch)
-   $ BRANCH="${BRANCH:-master}"
-   $ sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/ /' > /etc/apt/sources.list.d/kata-containers.list"
-   $ curl -sL  http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/Release.key | sudo apt-key add -
-   $ sudo -E apt-get update
-   $ sudo -E apt-get -y install kata-runtime kata-proxy kata-shim
-   ```
-
-2. Decide which container manager to use and select the corresponding link that follows:
-   - [Kubernetes](../Developer-Guide.md#run-kata-containers-with-kubernetes)

docs/use-cases/zun_kata.md

@@ -10,9 +10,6 @@ Currently, the instructions are based on the following links:
 
 - https://docs.openstack.org/zun/latest/admin/clear-containers.html
 
-- ../install/ubuntu-installation-guide.md
-
-
 ## Install Git to use with DevStack
 
 ```sh
@@ -54,7 +51,7 @@ $ zun delete test
 
 ## Install Kata Containers
 
-Follow [these instructions](../install/ubuntu-installation-guide.md)
+Follow [these instructions](../install/README.md)
 to install the Kata Containers components.
 
 ## Update Docker with new Kata Containers runtime

src/agent/Cargo.lock

@@ -147,14 +147,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.2.2"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52d1133681d746cc4807ad3b8005019af9299a086f61d5ed1de0e3a2000184f2"
+checksum = "d4cec688ee0fcd143ffd7893ce2c9857bfc656eb1f2a27202244b72f08f5f8ed"
 dependencies = [
  "libc",
  "log",
- "nix 0.18.0",
- "procinfo",
+ "nix 0.20.0",
  "regex",
 ]
 
@@ -511,9 +510,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.81"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
+checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7"
 
 [[package]]
 name = "libflate"
@@ -704,18 +703,6 @@ dependencies = [
  "void",
 ]
 
-[[package]]
-name = "nix"
-version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83450fe6a6142ddd95fb064b746083fc4ef1705fe81f64a64e1d4b39f54a1055"
-dependencies = [
- "bitflags",
- "cc",
- "cfg-if 0.1.10",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.19.1"
@@ -729,10 +716,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "nom"
-version = "2.2.1"
+name = "nix"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff"
+checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if 1.0.0",
+ "libc",
+]
 
 [[package]]
 name = "ntapi"
@@ -932,18 +925,6 @@ dependencies = [
  "libflate",
 ]
 
-[[package]]
-name = "procinfo"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c"
-dependencies = [
- "byteorder",
- "libc",
- "nom",
- "rustc_version",
-]
-
 [[package]]
 name = "prometheus"
 version = "0.9.0"
@@ -1176,15 +1157,6 @@ version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
 
-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver",
-]
-
 [[package]]
 name = "rustjail"
 version = "0.1.0"
@@ -1238,21 +1210,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser",
-]
-
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
 [[package]]
 name = "serde"
 version = "1.0.118"

src/agent/Cargo.toml

@@ -45,7 +45,7 @@ tempfile = "3.1.0"
 prometheus = { version = "0.9.0", features = ["process"] }
 procfs = "0.7.9"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.2" }
+cgroups = { package = "cgroups-rs", version = "0.2.5" }
 
 [workspace]
 members = [

src/agent/VERSION

@@ -1 +0,0 @@
-2.0.0

src/agent/VERSION

@@ -0,0 +1 @@
+../../VERSION

src/agent/oci/src/lib.rs

@@ -8,7 +8,7 @@ extern crate serde;
 extern crate serde_derive;
 extern crate serde_json;
 
-use libc::mode_t;
+use libc::{self, mode_t};
 use std::collections::HashMap;
 
 mod serialize;
@@ -27,6 +27,10 @@ where
     *d == T::default()
 }
 
+fn default_seccomp_errno() -> u32 {
+    libc::EPERM as u32
+}
+
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)]
 pub struct Spec {
     #[serde(
@@ -710,6 +714,8 @@ pub struct LinuxSeccomp {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub architectures: Vec<Arch>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub flags: Vec<LinuxSeccompFlag>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub syscalls: Vec<LinuxSyscall>,
 }
 
@@ -733,14 +739,20 @@ pub const ARCHS390: &str = "SCMP_ARCH_S390";
 pub const ARCHS390X: &str = "SCMP_ARCH_S390X";
 pub const ARCHPARISC: &str = "SCMP_ARCH_PARISC";
 pub const ARCHPARISC64: &str = "SCMP_ARCH_PARISC64";
+pub const ARCHRISCV64: &str = "SCMP_ARCH_RISCV64";
+
+pub type LinuxSeccompFlag = String;
 
 pub type LinuxSeccompAction = String;
 
 pub const ACTKILL: &str = "SCMP_ACT_KILL";
+pub const ACTKILLPROCESS: &str = "SCMP_ACT_KILL_PROCESS";
+pub const ACTKILLTHREAD: &str = "SCMP_ACT_KILL_THREAD";
 pub const ACTTRAP: &str = "SCMP_ACT_TRAP";
 pub const ACTERRNO: &str = "SCMP_ACT_ERRNO";
 pub const ACTTRACE: &str = "SCMP_ACT_TRACE";
 pub const ACTALLOW: &str = "SCMP_ACT_ALLOW";
+pub const ACTLOG: &str = "SCMP_ACT_LOG";
 
 pub type LinuxSeccompOperator = String;
 
@@ -770,6 +782,8 @@ pub struct LinuxSyscall {
     pub names: Vec<String>,
     #[serde(default, skip_serializing_if = "String::is_empty")]
     pub action: LinuxSeccompAction,
+    #[serde(default = "default_seccomp_errno", rename = "errnoRet")]
+    pub errno_ret: u32,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub args: Vec<LinuxSeccompArg>,
 }
@@ -1565,9 +1579,11 @@ mod tests {
                 seccomp: Some(crate::LinuxSeccomp {
                     default_action: "SCMP_ACT_ALLOW".to_string(),
                     architectures: vec!["SCMP_ARCH_X86".to_string(), "SCMP_ARCH_X32".to_string()],
+                    flags: vec![],
                     syscalls: vec![crate::LinuxSyscall {
                         names: vec!["getcwd".to_string(), "chmod".to_string()],
                         action: "SCMP_ACT_ERRNO".to_string(),
+                        errno_ret: crate::default_seccomp_errno(),
                         args: vec![],
                     }],
                 }),

src/agent/protocols/protos/oci.proto

@@ -441,7 +441,8 @@ message LinuxInterfacePriority {
 message LinuxSeccomp {
 	string DefaultAction = 1;
 	repeated string Architectures = 2;
-	repeated LinuxSyscall Syscalls = 3  [(gogoproto.nullable) = false];
+	repeated string Flags = 3;
+	repeated LinuxSyscall Syscalls = 4  [(gogoproto.nullable) = false];
 }
 
 message LinuxSeccompArg {
@@ -454,7 +455,8 @@ message LinuxSeccompArg {
 message LinuxSyscall {
 	repeated string Names = 1;
 	string Action = 2;
-	repeated LinuxSeccompArg Args = 3  [(gogoproto.nullable) = false];
+	uint32 ErrnoRet = 3;
+	repeated LinuxSeccompArg Args = 4  [(gogoproto.nullable) = false];
 }
 
 message LinuxIntelRdt {

src/agent/rustjail/Cargo.toml

@@ -23,7 +23,7 @@ scan_fmt = "0.2"
 regex = "1.1"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.1" }
+cgroups = { package = "cgroups-rs", version = "0.2.5" }
 tempfile = "3.1.0"
 rlimit = "0.5.3"
 

src/agent/rustjail/src/container.rs

@@ -634,12 +634,12 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         env::set_var(v[0], v[1]);
     }
 
-    // set the "HOME" env getting from "/etc/passwd"
+    // set the "HOME" env getting from "/etc/passwd", if
+    // there's no uid entry in /etc/passwd, set "/" as the
+    // home env.
     if env::var_os(HOME_ENV_KEY).is_none() {
-        match utils::home_dir(guser.uid) {
-            Ok(home_dir) => env::set_var(HOME_ENV_KEY, home_dir),
-            Err(e) => log_child!(cfd_log, "failed to get home dir: {:?}", e),
-        }
+        let home_dir = utils::home_dir(guser.uid).unwrap_or_else(|_| String::from("/"));
+        env::set_var(HOME_ENV_KEY, home_dir);
     }
 
     let exec_file = Path::new(&args[0]);

src/agent/rustjail/src/lib.rs

@@ -412,6 +412,7 @@ fn seccomp_grpc_to_oci(sec: &grpcLinuxSeccomp) -> ociLinuxSeccomp {
             r.push(ociLinuxSyscall {
                 names: sys.Names.clone().into_vec(),
                 action: sys.Action.clone(),
+                errno_ret: sys.ErrnoRet,
                 args,
             });
         }
@@ -421,6 +422,7 @@ fn seccomp_grpc_to_oci(sec: &grpcLinuxSeccomp) -> ociLinuxSeccomp {
     ociLinuxSeccomp {
         default_action: sec.DefaultAction.clone(),
         architectures: sec.Architectures.clone().into_vec(),
+        flags: sec.Flags.clone().into_vec(),
         syscalls,
     }
 }

src/agent/rustjail/src/mount.rs

@@ -52,10 +52,12 @@ const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
 const PROC_PATH: &str = "/proc";
 
 // since libc didn't defined this const for musl, thus redefined it here.
-#[cfg(all(target_os = "linux", target_env = "gnu"))]
+#[cfg(all(target_os = "linux", target_env = "gnu", not(target_arch = "s390x")))]
 const PROC_SUPER_MAGIC: libc::c_long = 0x00009fa0;
 #[cfg(all(target_os = "linux", target_env = "musl"))]
 const PROC_SUPER_MAGIC: libc::c_ulong = 0x00009fa0;
+#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "s390x"))]
+const PROC_SUPER_MAGIC: libc::c_uint = 0x00009fa0;
 
 lazy_static! {
     static ref PROPAGATION: HashMap<&'static str, MsFlags> = {
@@ -1365,7 +1367,7 @@ mod tests {
             let msg = format!("{}, result: {:?}", msg, result);
 
             // Perform the checks
-            assert!(result == t.result, msg);
+            assert!(result == t.result, "{}", msg);
         }
     }
 }

src/agent/rustjail/src/pipestream.rs

@@ -77,10 +77,6 @@ impl PipeStream {
         Ok(Self(AsyncFd::new(StreamFd(fd))?))
     }
 
-    pub fn shutdown(&mut self) -> io::Result<()> {
-        self.0.get_mut().close()
-    }
-
     pub fn from_fd(fd: RawFd) -> Self {
         unsafe { Self::from_raw_fd(fd) }
     }
@@ -164,7 +160,44 @@ impl AsyncWrite for PipeStream {
     }
 
     fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        self.get_mut().shutdown()?;
+        // Do nothing in shutdown is very important
+        // The only right way to shutdown pipe is drop it
+        // Otherwise PipeStream will conflict with its twins
+        // Because they both have same fd, and both registered.
         Poll::Ready(Ok(()))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use nix::fcntl::OFlag;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    #[tokio::test]
+    // Shutdown should never close the inner fd.
+    async fn test_pipestream_shutdown() {
+        let (_, wfd1) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
+        let mut writer1 = PipeStream::new(wfd1).unwrap();
+
+        // if close fd in shutdown, the fd will be reused
+        // and the test will failed
+        let _ = writer1.shutdown().await.unwrap();
+
+        // let _ = unistd::close(wfd1);
+
+        let (rfd2, wfd2) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap(); // reuse fd number, rfd2 == wfd1
+
+        let mut reader2 = PipeStream::new(rfd2).unwrap();
+        let mut writer2 = PipeStream::new(wfd2).unwrap();
+
+        // deregister writer1, then reader2 which has the same fd will be deregistered from epoll
+        drop(writer1);
+
+        let _ = writer2.write(b"1").await;
+
+        let mut content = vec![0u8; 1];
+        // Will Block here if shutdown close the fd.
+        let _ = reader2.read(&mut content).await;
+    }
+}

src/agent/rustjail/src/sync_with_async.rs

@@ -117,28 +117,20 @@ pub async fn write_async(pipe_w: &mut PipeStream, msg_type: i32, data_str: &str)
     }
 
     match msg_type {
-        SYNC_FAILED => match write_count(pipe_w, data_str.as_bytes(), data_str.len()).await {
-            Ok(_) => pipe_w.shutdown()?,
-            Err(e) => {
-                pipe_w.shutdown()?;
+        SYNC_FAILED => {
+            if let Err(e) = write_count(pipe_w, data_str.as_bytes(), data_str.len()).await {
                 return Err(anyhow!(e).context("error in send message to process"));
             }
-        },
+        }
         SYNC_DATA => {
             let length: i32 = data_str.len() as i32;
             write_count(pipe_w, &length.to_be_bytes(), MSG_SIZE)
                 .await
-                .or_else(|e| {
-                    pipe_w.shutdown()?;
-                    Err(anyhow!(e).context("error in send message to process"))
-                })?;
+                .map_err(|e| anyhow!(e).context("error in send message to process"))?;
 
             write_count(pipe_w, data_str.as_bytes(), data_str.len())
                 .await
-                .or_else(|e| {
-                    pipe_w.shutdown()?;
-                    Err(anyhow!(e).context("error in send message to process"))
-                })?;
+                .map_err(|e| anyhow!(e).context("error in send message to process"))?;
         }
 
         _ => (),

src/agent/src/config.rs

@@ -334,7 +334,7 @@ mod tests {
             if $expected_result.is_ok() {
                 let expected_level = $expected_result.as_ref().unwrap();
                 let actual_level = $actual_result.unwrap();
-                assert!(*expected_level == actual_level, $msg);
+                assert!(*expected_level == actual_level, "{}", $msg);
             } else {
                 let expected_error = $expected_result.as_ref().unwrap_err();
                 let expected_error_msg = format!("{:?}", expected_error);
@@ -342,9 +342,9 @@ mod tests {
                 if let Err(actual_error) = $actual_result {
                     let actual_error_msg = format!("{:?}", actual_error);
 
-                    assert!(expected_error_msg == actual_error_msg, $msg);
+                    assert!(expected_error_msg == actual_error_msg, "{}", $msg);
                 } else {
-                    assert!(expected_error_msg == "expected error, got OK", $msg);
+                    assert!(expected_error_msg == "expected error, got OK", "{}", $msg);
                 }
             }
         };

src/agent/src/console.rs

@@ -0,0 +1,294 @@
+// Copyright (c) 2021 Ant Group
+// Copyright (c) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::util;
+use anyhow::{anyhow, Result};
+use nix::fcntl::{self, FcntlArg, FdFlag, OFlag};
+use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO};
+use nix::pty::{openpty, OpenptyResult};
+use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
+use nix::sys::stat::Mode;
+use nix::sys::wait;
+use nix::unistd::{self, close, dup2, fork, setsid, ForkResult, Pid};
+use rustjail::pipestream::PipeStream;
+use slog::Logger;
+use std::ffi::CString;
+use std::os::unix::io::{FromRawFd, RawFd};
+use std::path::PathBuf;
+use std::process::Stdio;
+use std::sync::Arc;
+use std::sync::Mutex as SyncMutex;
+
+use futures::StreamExt;
+use tokio::io::{AsyncRead, AsyncWrite};
+use tokio::select;
+use tokio::sync::watch::Receiver;
+
+const CONSOLE_PATH: &str = "/dev/console";
+
+lazy_static! {
+    static ref SHELLS: Arc<SyncMutex<Vec<String>>> = {
+        let mut v = Vec::new();
+
+        if !cfg!(test) {
+            v.push("/bin/bash".to_string());
+            v.push("/bin/sh".to_string());
+        }
+
+        Arc::new(SyncMutex::new(v))
+    };
+}
+
+pub fn initialize() {
+    lazy_static::initialize(&SHELLS);
+}
+
+pub async fn debug_console_handler(
+    logger: Logger,
+    port: u32,
+    mut shutdown: Receiver<bool>,
+) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "debug-console"));
+
+    let shells = SHELLS.lock().unwrap().to_vec();
+
+    let shell = shells
+        .into_iter()
+        .find(|sh| PathBuf::from(sh).exists())
+        .ok_or_else(|| anyhow!("no shell found to launch debug console"))?;
+
+    if port > 0 {
+        let listenfd = socket::socket(
+            AddressFamily::Vsock,
+            SockType::Stream,
+            SockFlag::SOCK_CLOEXEC,
+            None,
+        )?;
+        let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port);
+        socket::bind(listenfd, &addr)?;
+        socket::listen(listenfd, 1)?;
+
+        let mut incoming = util::get_vsock_incoming(listenfd);
+
+        loop {
+            select! {
+                _ = shutdown.changed() => {
+                    info!(logger, "debug console got shutdown request");
+                    break;
+                }
+
+                conn = incoming.next() => {
+                    if let Some(conn) = conn {
+                        // Accept a new connection
+                        match conn {
+                            Ok(stream) => {
+                                let logger = logger.clone();
+                                let shell = shell.clone();
+                                // Do not block(await) here, or we'll never receive the shutdown signal
+                                tokio::spawn(async move {
+                                    let _ = run_debug_console_vsock(logger, shell, stream).await;
+                                });
+                            }
+                            Err(e) => {
+                                error!(logger, "{:?}", e);
+                            }
+                        }
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
+    } else {
+        let mut flags = OFlag::empty();
+        flags.insert(OFlag::O_RDWR);
+        flags.insert(OFlag::O_CLOEXEC);
+
+        let fd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?;
+
+        select! {
+            _ = shutdown.changed() => {
+                info!(logger, "debug console got shutdown request");
+            }
+
+            result = run_debug_console_serial(shell.clone(), fd) => {
+               match result {
+                   Ok(_) => {
+                       info!(logger, "run_debug_console_shell session finished");
+                   }
+                   Err(err) => {
+                       error!(logger, "run_debug_console_shell failed: {:?}", err);
+                   }
+               }
+            }
+        }
+    };
+
+    Ok(())
+}
+
+fn run_in_child(slave_fd: libc::c_int, shell: String) -> Result<()> {
+    // create new session with child as session leader
+    setsid()?;
+
+    // dup stdin, stdout, stderr to let child act as a terminal
+    dup2(slave_fd, STDIN_FILENO)?;
+    dup2(slave_fd, STDOUT_FILENO)?;
+    dup2(slave_fd, STDERR_FILENO)?;
+
+    // set tty
+    unsafe {
+        libc::ioctl(0, libc::TIOCSCTTY);
+    }
+
+    let cmd = CString::new(shell).unwrap();
+
+    // run shell
+    let _ = unistd::execvp(cmd.as_c_str(), &[]).map_err(|e| match e {
+        nix::Error::Sys(errno) => {
+            std::process::exit(errno as i32);
+        }
+        _ => std::process::exit(-2),
+    });
+
+    Ok(())
+}
+
+async fn run_in_parent<T: AsyncRead + AsyncWrite>(
+    logger: Logger,
+    stream: T,
+    pseudo: OpenptyResult,
+    child_pid: Pid,
+) -> Result<()> {
+    info!(logger, "get debug shell pid {:?}", child_pid);
+
+    let master_fd = pseudo.master;
+    let _ = close(pseudo.slave);
+
+    let (mut socket_reader, mut socket_writer) = tokio::io::split(stream);
+    let (mut master_reader, mut master_writer) = tokio::io::split(PipeStream::from_fd(master_fd));
+
+    select! {
+        res = tokio::io::copy(&mut master_reader, &mut socket_writer) => {
+            debug!(
+                logger,
+                "master closed: {:?}", res
+            );
+        }
+        res = tokio::io::copy(&mut socket_reader, &mut master_writer) => {
+            info!(
+                logger,
+                "socket closed: {:?}", res
+            );
+        }
+    }
+
+    let wait_status = wait::waitpid(child_pid, None);
+    info!(logger, "debug console process exit code: {:?}", wait_status);
+
+    Ok(())
+}
+
+async fn run_debug_console_vsock<T: AsyncRead + AsyncWrite>(
+    logger: Logger,
+    shell: String,
+    stream: T,
+) -> Result<()> {
+    let logger = logger.new(o!("subsystem" => "debug-console-shell"));
+
+    let pseudo = openpty(None, None)?;
+    let _ = fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
+    let _ = fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
+
+    let slave_fd = pseudo.slave;
+
+    match fork() {
+        Ok(ForkResult::Child) => run_in_child(slave_fd, shell),
+        Ok(ForkResult::Parent { child: child_pid }) => {
+            run_in_parent(logger.clone(), stream, pseudo, child_pid).await
+        }
+        Err(err) => Err(anyhow!("fork error: {:?}", err)),
+    }
+}
+
+async fn run_debug_console_serial(shell: String, fd: RawFd) -> Result<()> {
+    let mut child = match tokio::process::Command::new(shell)
+        .arg("-i")
+        .kill_on_drop(true)
+        .stdin(unsafe { Stdio::from_raw_fd(fd) })
+        .stdout(unsafe { Stdio::from_raw_fd(fd) })
+        .stderr(unsafe { Stdio::from_raw_fd(fd) })
+        .spawn()
+    {
+        Ok(c) => c,
+        Err(_) => return Err(anyhow!("failed to spawn shell")),
+    };
+
+    child.wait().await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+    use tokio::sync::watch;
+
+    #[tokio::test]
+    async fn test_setup_debug_console_no_shells() {
+        {
+            // Guarantee no shells have been added
+            // (required to avoid racing with
+            // test_setup_debug_console_invalid_shell()).
+            let shells_ref = SHELLS.clone();
+            let mut shells = shells_ref.lock().unwrap();
+            shells.clear();
+        }
+
+        let logger = slog_scope::logger();
+
+        let (_, rx) = watch::channel(true);
+        let result = debug_console_handler(logger, 0, rx).await;
+
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "no shell found to launch debug console"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_setup_debug_console_invalid_shell() {
+        {
+            let shells_ref = SHELLS.clone();
+            let mut shells = shells_ref.lock().unwrap();
+
+            let dir = tempdir().expect("failed to create tmpdir");
+
+            // Add an invalid shell
+            let shell = dir
+                .path()
+                .join("enoent")
+                .to_str()
+                .expect("failed to construct shell path")
+                .to_string();
+
+            shells.push(shell);
+        }
+
+        let logger = slog_scope::logger();
+
+        let (_, rx) = watch::channel(true);
+        let result = debug_console_handler(logger, 0, rx).await;
+
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "no shell found to launch debug console"
+        );
+    }
+}

src/agent/src/device.rs

@@ -5,6 +5,7 @@
 
 use libc::{c_uint, major, minor};
 use nix::sys::stat;
+use regex::Regex;
 use std::collections::HashMap;
 use std::fs;
 use std::os::unix::fs::MetadataExt;
@@ -17,7 +18,7 @@ use crate::linux_abi::*;
 use crate::mount::{DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE};
 use crate::pci;
 use crate::sandbox::Sandbox;
-use crate::{AGENT_CONFIG, GLOBAL_DEVICE_WATCHER};
+use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
 use anyhow::{anyhow, Result};
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
@@ -87,75 +88,116 @@ fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<String>
     Ok(relpath)
 }
 
-async fn get_device_name(sandbox: &Arc<Mutex<Sandbox>>, dev_addr: &str) -> Result<String> {
-    // Keep the same lock order as uevent::handle_block_add_event(), otherwise it may cause deadlock.
-    let mut w = GLOBAL_DEVICE_WATCHER.lock().await;
-    let sb = sandbox.lock().await;
-    for (key, value) in sb.pci_device_map.iter() {
-        if key.contains(dev_addr) {
-            info!(sl!(), "Device {} found in pci device map", dev_addr);
-            return Ok(format!("{}/{}", SYSTEM_DEV_PATH, value));
-        }
+// FIXME: This matcher is only correct if the guest has at most one
+// SCSI host.
+#[derive(Debug)]
+struct ScsiBlockMatcher {
+    search: String,
+}
+
+impl ScsiBlockMatcher {
+    fn new(scsi_addr: &str) -> Result<ScsiBlockMatcher> {
+        let search = format!(r"/0:0:{}/block/", scsi_addr);
+
+        Ok(ScsiBlockMatcher { search })
     }
-    drop(sb);
+}
 
-    // If device is not found in the device map, hotplug event has not
-    // been received yet, create and add channel to the watchers map.
-    // The key of the watchers map is the device we are interested in.
-    // Note this is done inside the lock, not to miss any events from the
-    // global udev listener.
-    let (tx, rx) = tokio::sync::oneshot::channel::<String>();
-    w.insert(dev_addr.to_string(), Some(tx));
-    drop(w);
-
-    info!(sl!(), "Waiting on channel for device notification\n");
-    let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
-
-    let dev_name = match tokio::time::timeout(hotplug_timeout, rx).await {
-        Ok(v) => v?,
-        Err(_) => {
-            let watcher = GLOBAL_DEVICE_WATCHER.clone();
-            let mut w = watcher.lock().await;
-            w.remove_entry(dev_addr);
-
-            return Err(anyhow!(
-                "Timeout reached after {:?} waiting for device {}",
-                hotplug_timeout,
-                dev_addr
-            ));
-        }
-    };
-
-    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &dev_name))
+impl UeventMatcher for ScsiBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == "block" && uev.devpath.contains(&self.search) && !uev.devname.is_empty()
+    }
 }
 
 pub async fn get_scsi_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     scsi_addr: &str,
 ) -> Result<String> {
-    let dev_sub_path = format!("{}{}/{}", SCSI_HOST_CHANNEL, scsi_addr, SCSI_BLOCK_SUFFIX);
+    let matcher = ScsiBlockMatcher::new(scsi_addr)?;
 
     scan_scsi_bus(scsi_addr)?;
-    get_device_name(sandbox, &dev_sub_path).await
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
-pub async fn get_pci_device_name(
+#[derive(Debug)]
+struct VirtioBlkPciMatcher {
+    rex: Regex,
+}
+
+impl VirtioBlkPciMatcher {
+    fn new(relpath: &str) -> Result<VirtioBlkPciMatcher> {
+        let root_bus = create_pci_root_bus_path();
+        let re = format!(r"^{}{}/virtio[0-9]+/block/", root_bus, relpath);
+        Ok(VirtioBlkPciMatcher {
+            rex: Regex::new(&re).unwrap(),
+        })
+    }
+}
+
+impl UeventMatcher for VirtioBlkPciMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == "block" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty()
+    }
+}
+
+pub async fn get_virtio_blk_pci_device_name(
     sandbox: &Arc<Mutex<Sandbox>>,
     pcipath: &pci::Path,
 ) -> Result<String> {
     let root_bus_sysfs = format!("{}{}", SYSFS_DIR, create_pci_root_bus_path());
     let sysfs_rel_path = pcipath_to_sysfs(&root_bus_sysfs, pcipath)?;
+    let matcher = VirtioBlkPciMatcher::new(&sysfs_rel_path)?;
 
     rescan_pci_bus()?;
-    get_device_name(sandbox, &sysfs_rel_path).await
+
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname))
 }
 
-pub async fn get_pmem_device_name(
-    sandbox: &Arc<Mutex<Sandbox>>,
-    pmem_devname: &str,
-) -> Result<String> {
-    let dev_sub_path = format!("/{}/{}", SCSI_BLOCK_SUFFIX, pmem_devname);
-    get_device_name(sandbox, &dev_sub_path).await
+#[derive(Debug)]
+struct PmemBlockMatcher {
+    suffix: String,
+}
+
+impl PmemBlockMatcher {
+    fn new(devname: &str) -> Result<PmemBlockMatcher> {
+        let suffix = format!(r"/block/{}", devname);
+
+        Ok(PmemBlockMatcher { suffix })
+    }
+}
+
+impl UeventMatcher for PmemBlockMatcher {
+    fn is_match(&self, uev: &Uevent) -> bool {
+        uev.subsystem == "block"
+            && uev.devpath.starts_with(ACPI_DEV_PATH)
+            && uev.devpath.ends_with(&self.suffix)
+            && !uev.devname.is_empty()
+    }
+}
+
+pub async fn wait_for_pmem_device(sandbox: &Arc<Mutex<Sandbox>>, devpath: &str) -> Result<()> {
+    let devname = match devpath.strip_prefix("/dev/") {
+        Some(dev) => dev,
+        None => {
+            return Err(anyhow!(
+                "Storage source '{}' must start with /dev/",
+                devpath
+            ))
+        }
+    };
+
+    let matcher = PmemBlockMatcher::new(devname)?;
+    let uev = wait_for_uevent(sandbox, matcher).await?;
+    if uev.devname != devname {
+        return Err(anyhow!(
+            "Unexpected device name {} for pmem device (expected {})",
+            uev.devname,
+            devname
+        ));
+    }
+    Ok(())
 }
 
 /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN)
@@ -296,7 +338,7 @@ async fn virtio_blk_device_handler(
     // path for cloud-hypervisor when BDF information is not available
     if !device.id.is_empty() {
         let pcipath = pci::Path::from_str(&device.id)?;
-        dev.vm_path = get_pci_device_name(sandbox, &pcipath).await?;
+        dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
     }
 
     update_spec_device_list(&dev, spec, devidx)
@@ -430,6 +472,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::uevent::spawn_test_watcher;
     use oci::Linux;
     use tempfile::tempdir;
 
@@ -776,4 +819,107 @@ mod tests {
         let relpath = pcipath_to_sysfs(rootbuspath, &path234);
         assert_eq!(relpath.unwrap(), "/0000:00:02.0/0000:01:03.0/0000:02:04.0");
     }
+
+    // We use device specific variants of this for real cases, but
+    // they have some complications that make them troublesome to unit
+    // test
+    async fn example_get_device_name(
+        sandbox: &Arc<Mutex<Sandbox>>,
+        relpath: &str,
+    ) -> Result<String> {
+        let matcher = VirtioBlkPciMatcher::new(relpath)?;
+
+        let uev = wait_for_uevent(sandbox, matcher).await?;
+
+        Ok(uev.devname)
+    }
+
+    #[tokio::test]
+    async fn test_get_device_name() {
+        let devname = "vda";
+        let root_bus = create_pci_root_bus_path();
+        let relpath = "/0000:00:0a.0/0000:03:0b.0";
+        let devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath, devname);
+
+        let mut uev = crate::uevent::Uevent::default();
+        uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = "block".to_string();
+        uev.devpath = devpath.clone();
+        uev.devname = devname.to_string();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let sandbox = Arc::new(Mutex::new(Sandbox::new(&logger).unwrap()));
+
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.insert(devpath.clone(), uev);
+        drop(sb); // unlock
+
+        let name = example_get_device_name(&sandbox, relpath).await;
+        assert!(name.is_ok(), "{}", name.unwrap_err());
+        assert_eq!(name.unwrap(), devname);
+
+        let mut sb = sandbox.lock().await;
+        let uev = sb.uevent_map.remove(&devpath).unwrap();
+        drop(sb); // unlock
+
+        spawn_test_watcher(sandbox.clone(), uev);
+
+        let name = example_get_device_name(&sandbox, relpath).await;
+        assert!(name.is_ok(), "{}", name.unwrap_err());
+        assert_eq!(name.unwrap(), devname);
+    }
+
+    #[tokio::test]
+    async fn test_virtio_blk_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let devname = "vda";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        let relpath_a = "/0000:00:0a.0";
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = "block".to_string();
+        uev_a.devname = devname.to_string();
+        uev_a.devpath = format!("{}{}/virtio4/block/{}", root_bus, relpath_a, devname);
+        let matcher_a = VirtioBlkPciMatcher::new(&relpath_a).unwrap();
+
+        let mut uev_b = uev_a.clone();
+        let relpath_b = "/0000:00:0a.0/0000:00:0b.0";
+        uev_b.devpath = format!("{}{}/virtio0/block/{}", root_bus, relpath_b, devname);
+        let matcher_b = VirtioBlkPciMatcher::new(&relpath_b).unwrap();
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
+
+    #[tokio::test]
+    async fn test_scsi_block_matcher() {
+        let root_bus = create_pci_root_bus_path();
+        let devname = "sda";
+
+        let mut uev_a = crate::uevent::Uevent::default();
+        let addr_a = "0:0";
+        uev_a.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev_a.subsystem = "block".to_string();
+        uev_a.devname = devname.to_string();
+        uev_a.devpath = format!(
+            "{}/0000:00:00.0/virtio0/host0/target0:0:0/0:0:{}/block/sda",
+            root_bus, addr_a
+        );
+        let matcher_a = ScsiBlockMatcher::new(&addr_a).unwrap();
+
+        let mut uev_b = uev_a.clone();
+        let addr_b = "2:0";
+        uev_b.devpath = format!(
+            "{}/0000:00:00.0/virtio0/host0/target0:0:2/0:0:{}/block/sdb",
+            root_bus, addr_b
+        );
+        let matcher_b = ScsiBlockMatcher::new(&addr_b).unwrap();
+
+        assert!(matcher_a.is_match(&uev_a));
+        assert!(matcher_b.is_match(&uev_b));
+        assert!(!matcher_b.is_match(&uev_a));
+        assert!(!matcher_a.is_match(&uev_b));
+    }
 }

src/agent/src/linux_abi.rs

@@ -78,11 +78,6 @@ pub const SYSFS_MEMORY_BLOCK_SIZE_PATH: &str = "/sys/devices/system/memory/block
 pub const SYSFS_MEMORY_HOTPLUG_PROBE_PATH: &str = "/sys/devices/system/memory/probe";
 pub const SYSFS_MEMORY_ONLINE_PATH: &str = "/sys/devices/system/memory";
 
-// Here in "0:0", the first number is the SCSI host number because
-// only one SCSI controller has been plugged, while the second number
-// is always 0.
-pub const SCSI_HOST_CHANNEL: &str = "0:0:";
-pub const SCSI_BLOCK_SUFFIX: &str = "block";
 pub const SYSFS_SCSI_HOST_PATH: &str = "/sys/class/scsi_host";
 
 pub const SYSFS_CGROUPPATH: &str = "/sys/fs/cgroup";

src/agent/src/main.rs

@@ -20,27 +20,21 @@ extern crate scopeguard;
 extern crate slog;
 
 use anyhow::{anyhow, Context, Result};
-use nix::fcntl::{self, OFlag};
-use nix::fcntl::{FcntlArg, FdFlag};
-use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO};
-use nix::pty;
-use nix::sys::select::{select, FdSet};
+use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
-use nix::sys::wait;
-use nix::unistd::{self, close, dup, dup2, fork, setsid, ForkResult};
-use std::collections::HashMap;
+use nix::unistd::{self, dup, Pid};
 use std::env;
-use std::ffi::{CStr, CString, OsStr};
+use std::ffi::OsStr;
 use std::fs::{self, File};
-use std::io::{Read, Write};
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs as unixfs;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
+use std::process::exit;
 use std::sync::Arc;
-use unistd::Pid;
 
 mod config;
+mod console;
 mod device;
 mod linux_abi;
 mod metrics;
@@ -64,33 +58,23 @@ use signal::setup_signal_handler;
 use slog::Logger;
 use uevent::watch_uevents;
 
-use std::sync::Mutex as SyncMutex;
-
 use futures::future::join_all;
-use futures::StreamExt as _;
 use rustjail::pipestream::PipeStream;
 use tokio::{
     io::AsyncWrite,
     sync::{
-        oneshot::Sender,
         watch::{channel, Receiver},
         Mutex, RwLock,
     },
     task::JoinHandle,
 };
-use tokio_vsock::{Incoming, VsockListener, VsockStream};
 
 mod rpc;
 
 const NAME: &str = "kata-agent";
 const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline";
-const CONSOLE_PATH: &str = "/dev/console";
-
-const DEFAULT_BUF_SIZE: usize = 8 * 1024;
 
 lazy_static! {
-    static ref GLOBAL_DEVICE_WATCHER: Arc<Mutex<HashMap<String, Option<Sender<String>>>>> =
-        Arc::new(Mutex::new(HashMap::new()));
     static ref AGENT_CONFIG: Arc<RwLock<AgentConfig>> =
         Arc::new(RwLock::new(config::AgentConfig::new()));
 }
@@ -108,27 +92,6 @@ fn announce(logger: &Logger, config: &AgentConfig) {
     );
 }
 
-fn set_fd_close_exec(fd: RawFd) -> Result<RawFd> {
-    if let Err(e) = fcntl::fcntl(fd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) {
-        return Err(anyhow!("failed to set fd: {} as close-on-exec: {}", fd, e));
-    }
-    Ok(fd)
-}
-
-fn get_vsock_incoming(fd: RawFd) -> Incoming {
-    let incoming;
-    unsafe {
-        incoming = VsockListener::from_raw_fd(fd).incoming();
-    }
-    incoming
-}
-
-async fn get_vsock_stream(fd: RawFd) -> Result<VsockStream> {
-    let stream = get_vsock_incoming(fd).next().await.unwrap().unwrap();
-    set_fd_close_exec(stream.as_raw_fd())?;
-    Ok(stream)
-}
-
 // Create a thread to handle reading from the logger pipe. The thread will
 // output to the vsock port specified, or stdout.
 async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool>) -> Result<()> {
@@ -147,7 +110,7 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool
         socket::bind(listenfd, &addr).unwrap();
         socket::listen(listenfd, 1).unwrap();
 
-        writer = Box::new(get_vsock_stream(listenfd).await.unwrap());
+        writer = Box::new(util::get_vsock_stream(listenfd).await.unwrap());
     } else {
         writer = Box::new(tokio::io::stdout());
     }
@@ -163,7 +126,7 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     // List of tasks that need to be stopped for a clean shutdown
     let mut tasks: Vec<JoinHandle<Result<()>>> = vec![];
 
-    lazy_static::initialize(&SHELLS);
+    console::initialize();
 
     lazy_static::initialize(&AGENT_CONFIG);
 
@@ -303,26 +266,17 @@ async fn start_sandbox(
     tasks: &mut Vec<JoinHandle<Result<()>>>,
     shutdown: Receiver<bool>,
 ) -> Result<()> {
-    let shells = SHELLS.clone();
     let debug_console_vport = config.debug_console_vport as u32;
 
-    let shell_handle = if config.debug_console {
-        let thread_logger = logger.clone();
-        let shells = shells.lock().unwrap().to_vec();
+    if config.debug_console {
+        let debug_console_task = tokio::task::spawn(console::debug_console_handler(
+            logger.clone(),
+            debug_console_vport,
+            shutdown.clone(),
+        ));
 
-        let handle = tokio::task::spawn_blocking(move || {
-            let result = setup_debug_console(&thread_logger, shells, debug_console_vport);
-            if result.is_err() {
-                // Report error, but don't fail
-                warn!(thread_logger, "failed to setup debug console";
-                    "error" => format!("{}", result.unwrap_err()));
-            }
-        });
-
-        Some(handle)
-    } else {
-        None
-    };
+        tasks.push(debug_console_task);
+    }
 
     // Initialize unique sandbox structure.
     let s = Sandbox::new(&logger).context("Failed to create sandbox")?;
@@ -354,10 +308,6 @@ async fn start_sandbox(
     let _ = rx.await?;
     server.shutdown().await?;
 
-    if let Some(handle) = shell_handle {
-        handle.await.map_err(|e| anyhow!("{:?}", e))?;
-    }
-
     Ok(())
 }
 
@@ -408,284 +358,5 @@ fn sethostname(hostname: &OsStr) -> Result<()> {
     }
 }
 
-lazy_static! {
-    static ref SHELLS: Arc<SyncMutex<Vec<String>>> = {
-        let mut v = Vec::new();
-
-        if !cfg!(test) {
-            v.push("/bin/bash".to_string());
-            v.push("/bin/sh".to_string());
-        }
-
-        Arc::new(SyncMutex::new(v))
-    };
-}
-
 use crate::config::AgentConfig;
-use nix::sys::stat::Mode;
 use std::os::unix::io::{FromRawFd, RawFd};
-use std::path::PathBuf;
-use std::process::exit;
-
-fn setup_debug_console(logger: &Logger, shells: Vec<String>, port: u32) -> Result<()> {
-    let shell = shells
-        .iter()
-        .find(|sh| PathBuf::from(sh).exists())
-        .ok_or_else(|| anyhow!("no shell found to launch debug console"))?;
-
-    if port > 0 {
-        let listenfd = socket::socket(
-            AddressFamily::Vsock,
-            SockType::Stream,
-            SockFlag::SOCK_CLOEXEC,
-            None,
-        )?;
-        let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port);
-        socket::bind(listenfd, &addr)?;
-        socket::listen(listenfd, 1)?;
-        loop {
-            let f: RawFd = socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)?;
-            match run_debug_console_shell(logger, shell, f) {
-                Ok(_) => {
-                    info!(logger, "run_debug_console_shell session finished");
-                }
-                Err(err) => {
-                    error!(logger, "run_debug_console_shell failed: {:?}", err);
-                }
-            }
-        }
-    } else {
-        let mut flags = OFlag::empty();
-        flags.insert(OFlag::O_RDWR);
-        flags.insert(OFlag::O_CLOEXEC);
-        loop {
-            let f: RawFd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?;
-            match run_debug_console_shell(logger, shell, f) {
-                Ok(_) => {
-                    info!(logger, "run_debug_console_shell session finished");
-                }
-                Err(err) => {
-                    error!(logger, "run_debug_console_shell failed: {:?}", err);
-                }
-            }
-        }
-    };
-}
-
-fn io_copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> std::io::Result<u64>
-where
-    R: Read,
-    W: Write,
-{
-    let mut buf = [0; DEFAULT_BUF_SIZE];
-    let buf_len;
-
-    match reader.read(&mut buf) {
-        Ok(0) => return Ok(0),
-        Ok(len) => buf_len = len,
-        Err(err) => return Err(err),
-    };
-
-    // write and return
-    match writer.write_all(&buf[..buf_len]) {
-        Ok(_) => Ok(buf_len as u64),
-        Err(err) => Err(err),
-    }
-}
-
-fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Result<()> {
-    let pseduo = pty::openpty(None, None)?;
-    let _ = fcntl::fcntl(pseduo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
-    let _ = fcntl::fcntl(pseduo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
-
-    let slave_fd = pseduo.slave;
-
-    match fork() {
-        Ok(ForkResult::Child) => {
-            // create new session with child as session leader
-            setsid()?;
-
-            // dup stdin, stdout, stderr to let child act as a terminal
-            dup2(slave_fd, STDIN_FILENO)?;
-            dup2(slave_fd, STDOUT_FILENO)?;
-            dup2(slave_fd, STDERR_FILENO)?;
-
-            // set tty
-            unsafe {
-                libc::ioctl(0, libc::TIOCSCTTY);
-            }
-
-            let cmd = CString::new(shell).unwrap();
-            let args: Vec<&CStr> = vec![];
-
-            // run shell
-            let _ = unistd::execvp(cmd.as_c_str(), args.as_slice()).map_err(|e| match e {
-                nix::Error::Sys(errno) => {
-                    std::process::exit(errno as i32);
-                }
-                _ => std::process::exit(-2),
-            });
-        }
-
-        Ok(ForkResult::Parent { child: child_pid }) => {
-            info!(logger, "get debug shell pid {:?}", child_pid);
-
-            let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?;
-            let master_fd = pseduo.master;
-            let debug_shell_logger = logger.clone();
-
-            // channel that used to sync between thread and main process
-            let (tx, rx) = std::sync::mpsc::channel::<i32>();
-
-            // start a thread to do IO copy between socket and pseduo.master
-            std::thread::spawn(move || {
-                let mut master_reader = unsafe { File::from_raw_fd(master_fd) };
-                let mut master_writer = unsafe { File::from_raw_fd(master_fd) };
-                let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) };
-                let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) };
-
-                loop {
-                    let mut fd_set = FdSet::new();
-                    fd_set.insert(rfd);
-                    fd_set.insert(master_fd);
-                    fd_set.insert(socket_fd);
-
-                    match select(
-                        Some(fd_set.highest().unwrap() + 1),
-                        &mut fd_set,
-                        None,
-                        None,
-                        None,
-                    ) {
-                        Ok(_) => (),
-                        Err(e) => {
-                            if e == nix::Error::from(nix::errno::Errno::EINTR) {
-                                continue;
-                            } else {
-                                error!(debug_shell_logger, "select error {:?}", e);
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                        }
-                    }
-
-                    if fd_set.contains(rfd) {
-                        info!(
-                            debug_shell_logger,
-                            "debug shell process {} exited", child_pid
-                        );
-                        tx.send(1).unwrap();
-                        break;
-                    }
-
-                    if fd_set.contains(master_fd) {
-                        match io_copy(&mut master_reader, &mut socket_writer) {
-                            Ok(0) => {
-                                debug!(debug_shell_logger, "master fd closed");
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                            Ok(_) => {}
-                            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
-                            Err(e) => {
-                                error!(debug_shell_logger, "read master fd error {:?}", e);
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                        }
-                    }
-
-                    if fd_set.contains(socket_fd) {
-                        match io_copy(&mut socket_reader, &mut master_writer) {
-                            Ok(0) => {
-                                debug!(debug_shell_logger, "socket fd closed");
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                            Ok(_) => {}
-                            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
-                            Err(e) => {
-                                error!(debug_shell_logger, "read socket fd error {:?}", e);
-                                tx.send(1).unwrap();
-                                break;
-                            }
-                        }
-                    }
-                }
-            });
-
-            let wait_status = wait::waitpid(child_pid, None);
-            info!(logger, "debug console process exit code: {:?}", wait_status);
-
-            info!(logger, "notify debug monitor thread to exit");
-            // close pipe to exit select loop
-            let _ = close(wfd);
-
-            // wait for thread exit.
-            let _ = rx.recv().unwrap();
-            info!(logger, "debug monitor thread has exited");
-
-            // close files
-            let _ = close(rfd);
-            let _ = close(master_fd);
-            let _ = close(slave_fd);
-        }
-        Err(err) => {
-            return Err(anyhow!("fork error: {:?}", err));
-        }
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tempfile::tempdir;
-
-    #[test]
-    fn test_setup_debug_console_no_shells() {
-        // Guarantee no shells have been added
-        // (required to avoid racing with
-        // test_setup_debug_console_invalid_shell()).
-        let shells_ref = SHELLS.clone();
-        let mut shells = shells_ref.lock().unwrap();
-        shells.clear();
-        let logger = slog_scope::logger();
-
-        let result = setup_debug_console(&logger, shells.to_vec(), 0);
-
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "no shell found to launch debug console"
-        );
-    }
-
-    #[test]
-    fn test_setup_debug_console_invalid_shell() {
-        let shells_ref = SHELLS.clone();
-        let mut shells = shells_ref.lock().unwrap();
-
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        // Add an invalid shell
-        let shell = dir
-            .path()
-            .join("enoent")
-            .to_str()
-            .expect("failed to construct shell path")
-            .to_string();
-
-        shells.push(shell);
-        let logger = slog_scope::logger();
-
-        let result = setup_debug_console(&logger, shells.to_vec(), 0);
-
-        assert!(result.is_err());
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "no shell found to launch debug console"
-        );
-    }
-}

src/agent/src/mount.rs

@@ -17,13 +17,14 @@ use tokio::sync::Mutex;
 
 use libc::{c_void, mount};
 use nix::mount::{self, MsFlags};
+use nix::unistd::Gid;
 
 use regex::Regex;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
 use crate::device::{
-    get_pci_device_name, get_pmem_device_name, get_scsi_device_name, online_device,
+    get_scsi_device_name, get_virtio_blk_pci_device_name, online_device, wait_for_pmem_device,
 };
 use crate::linux_abi::*;
 use crate::pci;
@@ -45,6 +46,9 @@ pub const TYPE_ROOTFS: &str = "rootfs";
 
 pub const MOUNT_GUEST_TAG: &str = "kataShared";
 
+// Allocating an FSGroup that owns the pod's volumes
+const FS_GID: &str = "fsgid";
+
 #[rustfmt::skip]
 lazy_static! {
     pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = {
@@ -266,11 +270,24 @@ async fn local_storage_handler(
     let opts_vec: Vec<String> = storage.options.to_vec();
 
     let opts = parse_options(opts_vec);
-    let mode = opts.get("mode");
-    if let Some(mode) = mode {
+
+    let mut need_set_fsgid = false;
+    if let Some(fsgid) = opts.get(FS_GID) {
+        let gid = fsgid.parse::<u32>()?;
+
+        nix::unistd::chown(storage.mount_point.as_str(), None, Some(Gid::from_raw(gid)))?;
+        need_set_fsgid = true;
+    }
+
+    if let Some(mode) = opts.get("mode") {
         let mut permission = fs::metadata(&storage.mount_point)?.permissions();
 
-        let o_mode = u32::from_str_radix(mode, 8)?;
+        let mut o_mode = u32::from_str_radix(mode, 8)?;
+
+        if need_set_fsgid {
+            // set SetGid mode mask.
+            o_mode |= 0o2000;
+        }
         permission.set_mode(o_mode);
 
         fs::set_permissions(&storage.mount_point, permission)?;
@@ -325,7 +342,7 @@ async fn virtio_blk_storage_handler(
         }
     } else {
         let pcipath = pci::Path::from_str(&storage.source)?;
-        let dev_path = get_pci_device_name(&sandbox, &pcipath).await?;
+        let dev_path = get_virtio_blk_pci_device_name(&sandbox, &pcipath).await?;
         storage.source = dev_path;
     }
 
@@ -360,22 +377,10 @@ async fn nvdimm_storage_handler(
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
 ) -> Result<String> {
-    let mut storage = storage.clone();
-    // If hot-plugged, get the device node path based on the PCI address else
-    // use the virt path provided in Storage Source
-    let pmem_devname = match storage.source.strip_prefix("/dev/") {
-        Some(dev) => dev,
-        None => {
-            return Err(anyhow!(
-                "Storage source '{}' must start with /dev/",
-                storage.source
-            ))
-        }
-    };
+    let storage = storage.clone();
 
     // Retrieve the device path from NVDIMM address.
-    let dev_path = get_pmem_device_name(&sandbox, pmem_devname).await?;
-    storage.source = dev_path;
+    wait_for_pmem_device(&sandbox, &storage.source).await?;
 
     common_storage_handler(logger, &storage)
 }
@@ -388,7 +393,10 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
     // If so, skip doing the mount. This facilitates mounting the sharedfs automatically
     // in the guest before the agent service starts.
     if storage.source == MOUNT_GUEST_TAG && is_mounted(&storage.mount_point)? {
-        warn!(logger, "kataShared already mounted, ignoring...");
+        warn!(
+            logger,
+            "{} already mounted on {}, ignoring...", MOUNT_GUEST_TAG, &storage.mount_point
+        );
         return Ok(());
     }
 
@@ -900,7 +908,7 @@ mod tests {
             let msg = format!("{}: result: {:?}", msg, result);
 
             if d.error_contains.is_empty() {
-                assert!(result.is_ok(), msg);
+                assert!(result.is_ok(), "{}", msg);
 
                 // Cleanup
                 unsafe {
@@ -912,7 +920,7 @@ mod tests {
 
                     let msg = format!("{}: umount result: {:?}", msg, result);
 
-                    assert!(ret == 0, msg);
+                    assert!(ret == 0, "{}", msg);
                 };
 
                 continue;
@@ -920,7 +928,7 @@ mod tests {
 
             let err = result.unwrap_err();
             let error_msg = format!("{}", err);
-            assert!(error_msg.contains(d.error_contains), msg);
+            assert!(error_msg.contains(d.error_contains), "{}", msg);
         }
     }
 
@@ -1026,13 +1034,13 @@ mod tests {
             let msg = format!("{}: result: {:?}", msg, result);
 
             if d.error_contains.is_empty() {
-                assert!(result.is_ok(), msg);
+                assert!(result.is_ok(), "{}", msg);
                 continue;
             }
 
             let error_msg = format!("{:#}", result.unwrap_err());
 
-            assert!(error_msg.contains(d.error_contains), msg);
+            assert!(error_msg.contains(d.error_contains), "{}", msg);
         }
     }
 
@@ -1108,6 +1116,7 @@ mod tests {
 
             assert!(
                 format!("{}", err).contains("No such file or directory"),
+                "{}",
                 msg
             );
         }
@@ -1136,13 +1145,13 @@ mod tests {
             if d.error_contains.is_empty() {
                 let fs_type = result.unwrap();
 
-                assert!(d.fs_type == fs_type, msg);
+                assert!(d.fs_type == fs_type, "{}", msg);
 
                 continue;
             }
 
             let error_msg = format!("{}", result.unwrap_err());
-            assert!(error_msg.contains(d.error_contains), msg);
+            assert!(error_msg.contains(d.error_contains), "{}", msg);
         }
     }
 
@@ -1291,34 +1300,34 @@ mod tests {
             let msg = format!("{}: result: {:?}", msg, result);
 
             if !d.error_contains.is_empty() {
-                assert!(result.is_err(), msg);
+                assert!(result.is_err(), "{}", msg);
 
                 let error_msg = format!("{}", result.unwrap_err());
-                assert!(error_msg.contains(d.error_contains), msg);
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
                 continue;
             }
 
-            assert!(result.is_ok(), msg);
+            assert!(result.is_ok(), "{}", msg);
 
             let mounts = result.unwrap();
             let count = mounts.len();
 
             if !d.devices_cgroup {
-                assert!(count == 0, msg);
+                assert!(count == 0, "{}", msg);
                 continue;
             }
 
             // get_cgroup_mounts() adds the device cgroup plus two other mounts.
-            assert!(count == (1 + 2), msg);
+            assert!(count == (1 + 2), "{}", msg);
 
             // First mount
-            assert!(mounts[0].eq(&first_mount), msg);
+            assert!(mounts[0].eq(&first_mount), "{}", msg);
 
             // Last mount
-            assert!(mounts[2].eq(&last_mount), msg);
+            assert!(mounts[2].eq(&last_mount), "{}", msg);
 
             // Devices cgroup
-            assert!(mounts[1].eq(&cg_devices_mount), msg);
+            assert!(mounts[1].eq(&cg_devices_mount), "{}", msg);
         }
     }
 }

src/agent/src/rpc.rs

@@ -2018,9 +2018,9 @@ mod tests {
             let msg = format!("{}, result: {:?}", msg, result);
 
             if result.is_ok() {
-                assert!(!d.expect_error, msg);
+                assert!(!d.expect_error, "{}", msg);
             } else {
-                assert!(d.expect_error, msg);
+                assert!(d.expect_error, "{}", msg);
             }
         }
     }

src/agent/src/sandbox.rs

@@ -8,6 +8,7 @@ use crate::mount::{get_mount_fs_type, remove_mounts, TYPE_ROOTFS};
 use crate::namespace::Namespace;
 use crate::netlink::Handle;
 use crate::network::Network;
+use crate::uevent::{Uevent, UeventMatcher};
 use anyhow::{anyhow, Context, Result};
 use libc::pid_t;
 use oci::{Hook, Hooks};
@@ -25,8 +26,11 @@ use std::path::Path;
 use std::sync::Arc;
 use std::{thread, time};
 use tokio::sync::mpsc::{channel, Receiver, Sender};
+use tokio::sync::oneshot;
 use tokio::sync::Mutex;
 
+type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
+
 #[derive(Debug)]
 pub struct Sandbox {
     pub logger: Logger,
@@ -36,7 +40,8 @@ pub struct Sandbox {
     pub network: Network,
     pub mounts: Vec<String>,
     pub container_mounts: HashMap<String, Vec<String>>,
-    pub pci_device_map: HashMap<String, String>,
+    pub uevent_map: HashMap<String, Uevent>,
+    pub uevent_watchers: Vec<Option<UeventWatcher>>,
     pub shared_utsns: Namespace,
     pub shared_ipcns: Namespace,
     pub sandbox_pidns: Option<Namespace>,
@@ -65,7 +70,8 @@ impl Sandbox {
             containers: HashMap::new(),
             mounts: Vec::new(),
             container_mounts: HashMap::new(),
-            pci_device_map: HashMap::new(),
+            uevent_map: HashMap::new(),
+            uevent_watchers: Vec::new(),
             shared_utsns: Namespace::new(&logger),
             shared_ipcns: Namespace::new(&logger),
             sandbox_pidns: None,

src/agent/src/uevent.rs

@@ -6,26 +6,38 @@
 use crate::device::online_device;
 use crate::linux_abi::*;
 use crate::sandbox::Sandbox;
-use crate::GLOBAL_DEVICE_WATCHER;
+use crate::AGENT_CONFIG;
 use slog::Logger;
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use netlink_sys::{protocols, SocketAddr, TokioSocket};
 use nix::errno::Errno;
+use std::fmt::Debug;
 use std::os::unix::io::FromRawFd;
 use std::sync::Arc;
 use tokio::select;
 use tokio::sync::watch::Receiver;
 use tokio::sync::Mutex;
 
-#[derive(Debug, Default)]
-struct Uevent {
-    action: String,
-    devpath: String,
-    devname: String,
-    subsystem: String,
+// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger().new(o!("subsystem" => "uevent"))
+    };
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct Uevent {
+    pub action: String,
+    pub devpath: String,
+    pub devname: String,
+    pub subsystem: String,
     seqnum: String,
-    interface: String,
+    pub interface: String,
+}
+
+pub trait UeventMatcher: Sync + Send + Debug + 'static {
+    fn is_match(&self, uev: &Uevent) -> bool;
 }
 
 impl Uevent {
@@ -52,89 +64,87 @@ impl Uevent {
         event
     }
 
-    // Check whether this is a block device hot-add event.
-    fn is_block_add_event(&self) -> bool {
-        let pci_root_bus_path = create_pci_root_bus_path();
-        self.action == U_EVENT_ACTION_ADD
-            && self.subsystem == "block"
-            && {
-                self.devpath.starts_with(pci_root_bus_path.as_str())
-                    || self.devpath.starts_with(ACPI_DEV_PATH) // NVDIMM/PMEM devices
-            }
-            && !self.devname.is_empty()
-    }
+    async fn process_add(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
+        // Special case for memory hot-adds first
+        let online_path = format!("{}/{}/online", SYSFS_DIR, &self.devpath);
+        if online_path.starts_with(SYSFS_MEMORY_ONLINE_PATH) {
+            let _ = online_device(online_path.as_ref()).map_err(|e| {
+                error!(
+                    *logger,
+                    "failed to online device";
+                    "device" => &self.devpath,
+                    "error" => format!("{}", e),
+                )
+            });
+            return;
+        }
 
-    async fn handle_block_add_event(&self, sandbox: &Arc<Mutex<Sandbox>>) {
-        let pci_root_bus_path = create_pci_root_bus_path();
-
-        // Keep the same lock order as device::get_device_name(), otherwise it may cause deadlock.
-        let watcher = GLOBAL_DEVICE_WATCHER.clone();
-        let mut w = watcher.lock().await;
         let mut sb = sandbox.lock().await;
 
-        // Add the device node name to the pci device map.
-        sb.pci_device_map
-            .insert(self.devpath.clone(), self.devname.clone());
+        // Record the event by sysfs path
+        sb.uevent_map.insert(self.devpath.clone(), self.clone());
 
         // Notify watchers that are interested in the udev event.
-        // Close the channel after watcher has been notified.
-        let devpath = self.devpath.clone();
-        let empties: Vec<_> = w
-            .iter_mut()
-            .filter(|(dev_addr, _)| {
-                let pci_p = format!("{}/{}", pci_root_bus_path, *dev_addr);
-
-                // blk block device
-                devpath.starts_with(pci_p.as_str()) ||
-                // scsi block device
-                {
-                    (*dev_addr).ends_with(SCSI_BLOCK_SUFFIX) &&
-                        devpath.contains(*dev_addr)
-                } ||
-                // nvdimm/pmem device
-                {
-                    let pmem_suffix = format!("/{}/{}", SCSI_BLOCK_SUFFIX, self.devname);
-                    devpath.starts_with(ACPI_DEV_PATH) &&
-                        devpath.ends_with(pmem_suffix.as_str()) &&
-                        dev_addr.ends_with(pmem_suffix.as_str())
+        for watch in &mut sb.uevent_watchers {
+            if let Some((matcher, _)) = watch {
+                if matcher.is_match(&self) {
+                    let (_, sender) = watch.take().unwrap();
+                    let _ = sender.send(self.clone());
                 }
-            })
-            .map(|(k, sender)| {
-                let devname = self.devname.clone();
-                let sender = sender.take().unwrap();
-                let _ = sender.send(devname);
-                k.clone()
-            })
-            .collect();
-
-        // Remove notified nodes from the watcher map.
-        for empty in empties {
-            w.remove(&empty);
+            }
         }
     }
 
     async fn process(&self, logger: &Logger, sandbox: &Arc<Mutex<Sandbox>>) {
-        if self.is_block_add_event() {
-            return self.handle_block_add_event(sandbox).await;
-        } else if self.action == U_EVENT_ACTION_ADD {
-            let online_path = format!("{}/{}/online", SYSFS_DIR, &self.devpath);
-            // It's a memory hot-add event.
-            if online_path.starts_with(SYSFS_MEMORY_ONLINE_PATH) {
-                let _ = online_device(online_path.as_ref()).map_err(|e| {
-                    error!(
-                        *logger,
-                        "failed to online device";
-                        "device" => &self.devpath,
-                        "error" => format!("{}", e),
-                    )
-                });
-                return;
-            }
+        if self.action == U_EVENT_ACTION_ADD {
+            return self.process_add(logger, sandbox).await;
         }
         debug!(*logger, "ignoring event"; "uevent" => format!("{:?}", self));
     }
 }
 
+pub async fn wait_for_uevent(
+    sandbox: &Arc<Mutex<Sandbox>>,
+    matcher: impl UeventMatcher,
+) -> Result<Uevent> {
+    let mut sb = sandbox.lock().await;
+    for uev in sb.uevent_map.values() {
+        if matcher.is_match(uev) {
+            info!(sl!(), "Device {:?} found in pci device map", uev);
+            return Ok(uev.clone());
+        }
+    }
+
+    // If device is not found in the device map, hotplug event has not
+    // been received yet, create and add channel to the watchers map.
+    // The key of the watchers map is the device we are interested in.
+    // Note this is done inside the lock, not to miss any events from the
+    // global udev listener.
+    let (tx, rx) = tokio::sync::oneshot::channel::<Uevent>();
+    let idx = sb.uevent_watchers.len();
+    sb.uevent_watchers.push(Some((Box::new(matcher), tx)));
+    drop(sb); // unlock
+
+    info!(sl!(), "Waiting on channel for uevent notification\n");
+    let hotplug_timeout = AGENT_CONFIG.read().await.hotplug_timeout;
+
+    let uev = match tokio::time::timeout(hotplug_timeout, rx).await {
+        Ok(v) => v?,
+        Err(_) => {
+            let mut sb = sandbox.lock().await;
+            let matcher = sb.uevent_watchers[idx].take().unwrap().0;
+
+            return Err(anyhow!(
+                "Timeout after {:?} waiting for uevent {:?}",
+                hotplug_timeout,
+                &matcher
+            ));
+        }
+    };
+
+    Ok(uev)
+}
+
 pub async fn watch_uevents(
     sandbox: Arc<Mutex<Sandbox>>,
     mut shutdown: Receiver<bool>,
@@ -199,3 +209,69 @@ pub async fn watch_uevents(
 
     Ok(())
 }
+
+// Used in the device module unit tests
+#[cfg(test)]
+pub(crate) fn spawn_test_watcher(sandbox: Arc<Mutex<Sandbox>>, uev: Uevent) {
+    tokio::spawn(async move {
+        loop {
+            let mut sb = sandbox.lock().await;
+            for w in &mut sb.uevent_watchers {
+                if let Some((matcher, _)) = w {
+                    if matcher.is_match(&uev) {
+                        let (_, sender) = w.take().unwrap();
+                        let _ = sender.send(uev);
+                        return;
+                    }
+                }
+            }
+            drop(sb); // unlock
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Debug, Clone, Copy)]
+    struct AlwaysMatch();
+
+    impl UeventMatcher for AlwaysMatch {
+        fn is_match(&self, _: &Uevent) -> bool {
+            true
+        }
+    }
+
+    #[tokio::test]
+    async fn test_wait_for_uevent() {
+        let mut uev = crate::uevent::Uevent::default();
+        uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string();
+        uev.subsystem = "test".to_string();
+        uev.devpath = "/test/sysfs/path".to_string();
+        uev.devname = "testdevname".to_string();
+
+        let matcher = AlwaysMatch();
+
+        let logger = slog::Logger::root(slog::Discard, o!());
+        let sandbox = Arc::new(Mutex::new(Sandbox::new(&logger).unwrap()));
+
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.insert(uev.devpath.clone(), uev.clone());
+        drop(sb); // unlock
+
+        let uev2 = wait_for_uevent(&sandbox, matcher).await;
+        assert!(uev2.is_ok());
+        assert_eq!(uev2.unwrap(), uev);
+
+        let mut sb = sandbox.lock().await;
+        sb.uevent_map.remove(&uev.devpath).unwrap();
+        drop(sb); // unlock
+
+        spawn_test_watcher(sandbox.clone(), uev.clone());
+
+        let uev2 = wait_for_uevent(&sandbox, matcher).await;
+        assert!(uev2.is_ok());
+        assert_eq!(uev2.unwrap(), uev);
+    }
+}

src/agent/src/util.rs

@@ -3,10 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use anyhow::Result;
+use futures::StreamExt;
 use std::io;
 use std::io::ErrorKind;
+use std::os::unix::io::{FromRawFd, RawFd};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::sync::watch::Receiver;
+use tokio_vsock::{Incoming, VsockListener, VsockStream};
 
 // Size of I/O read buffer
 const BUF_SIZE: usize = 8192;
@@ -52,6 +56,15 @@ where
     Ok(total_bytes)
 }
 
+pub fn get_vsock_incoming(fd: RawFd) -> Incoming {
+    unsafe { VsockListener::from_raw_fd(fd).incoming() }
+}
+
+pub async fn get_vsock_stream(fd: RawFd) -> Result<VsockStream> {
+    let stream = get_vsock_incoming(fd).next().await.unwrap()?;
+    Ok(stream)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/runtime/VERSION

@@ -1 +0,0 @@
-2.0.0

src/runtime/VERSION

@@ -0,0 +1 @@
+../../VERSION

src/runtime/containerd-shim-v2/create.go

@@ -23,6 +23,7 @@ import (
 
 	// only register the proto type
 	_ "github.com/containerd/containerd/runtime/linux/runctypes"
+	_ "github.com/containerd/containerd/runtime/v2/runc/options"
 	crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"

src/runtime/pkg/katautils/tracing.go

@@ -30,7 +30,7 @@ var _ export.SpanExporter = (*kataSpanExporter)(nil)
 // ExportSpans exports SpanData to Jaeger.
 func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
 	for _, span := range spans {
-		kataUtilsLogger.Infof("Reporting span %+v", span)
+		kataUtilsLogger.Tracef("Reporting span %+v", span)
 	}
 	return nil
 }

src/runtime/virtcontainers/agent.go

@@ -163,10 +163,10 @@ type agent interface {
 	resumeContainer(ctx context.Context, sandbox *Sandbox, c Container) error
 
 	// configure will update agent settings based on provided arguments
-	configure(ctx context.Context, h hypervisor, id, sharePath string, config interface{}) error
+	configure(ctx context.Context, h hypervisor, id, sharePath string, config KataAgentConfig) error
 
 	// configureFromGrpc will update agent settings based on provided arguments which from Grpc
-	configureFromGrpc(h hypervisor, id string, config interface{}) error
+	configureFromGrpc(h hypervisor, id string, config KataAgentConfig) error
 
 	// reseedRNG will reseed the guest random number generator
 	reseedRNG(ctx context.Context, data []byte) error

src/runtime/virtcontainers/api_test.go

@@ -144,7 +144,7 @@ func TestCreateSandboxNoopAgentSuccessful(t *testing.T) {
 	assert.True(ok)
 	assert.NotNil(s)
 
-	sandboxDir := filepath.Join(s.newStore.RunStoragePath(), p.ID())
+	sandboxDir := filepath.Join(s.store.RunStoragePath(), p.ID())
 	_, err = os.Stat(sandboxDir)
 	assert.NoError(err)
 }
@@ -175,7 +175,7 @@ func TestCreateSandboxKataAgentSuccessful(t *testing.T) {
 
 	s, ok := p.(*Sandbox)
 	assert.True(ok)
-	sandboxDir := filepath.Join(s.newStore.RunStoragePath(), p.ID())
+	sandboxDir := filepath.Join(s.store.RunStoragePath(), p.ID())
 	_, err = os.Stat(sandboxDir)
 	assert.NoError(err)
 }
@@ -224,7 +224,7 @@ func createAndStartSandbox(ctx context.Context, config SandboxConfig) (sandbox V
 	if !ok {
 		return nil, "", fmt.Errorf("Could not get Sandbox")
 	}
-	sandboxDir = filepath.Join(s.newStore.RunStoragePath(), sandbox.ID())
+	sandboxDir = filepath.Join(s.store.RunStoragePath(), sandbox.ID())
 	_, err = os.Stat(sandboxDir)
 	if err != nil {
 		return nil, "", err
@@ -285,7 +285,7 @@ func TestCleanupContainer(t *testing.T) {
 
 	s, ok := p.(*Sandbox)
 	assert.True(ok)
-	sandboxDir := filepath.Join(s.newStore.RunStoragePath(), p.ID())
+	sandboxDir := filepath.Join(s.store.RunStoragePath(), p.ID())
 
 	_, err = os.Stat(sandboxDir)
 	if err == nil {

src/runtime/virtcontainers/container_test.go

@@ -316,11 +316,11 @@ func TestContainerAddDriveDir(t *testing.T) {
 		},
 	}
 
-	sandbox.newStore, err = persist.GetDriver()
+	sandbox.store, err = persist.GetDriver()
 	assert.NoError(err)
-	assert.NotNil(sandbox.newStore)
+	assert.NotNil(sandbox.store)
 
-	defer sandbox.newStore.Destroy(sandbox.id)
+	defer sandbox.store.Destroy(sandbox.id)
 
 	contID := "100"
 	container := Container{

src/runtime/virtcontainers/fc.go

@@ -273,20 +273,8 @@ func (fc *firecracker) vmRunning(ctx context.Context) bool {
 		fc.Logger().WithError(err).Error("getting vm status failed")
 		return false
 	}
-
-	// Be explicit
-	switch *resp.Payload.State {
-	case models.InstanceInfoStateStarting:
-		// Unsure what we should do here
-		fc.Logger().WithField("unexpected-state", models.InstanceInfoStateStarting).Debug("vmRunning")
-		return false
-	case models.InstanceInfoStateRunning:
-		return true
-	case models.InstanceInfoStateUninitialized:
-		return false
-	default:
-		return false
-	}
+	// The current state of the Firecracker instance (swagger:model InstanceInfo)
+	return resp.Payload.Started
 }
 
 func (fc *firecracker) getVersionNumber() (string, error) {
@@ -302,7 +290,7 @@ func (fc *firecracker) getVersionNumber() (string, error) {
 	fields := strings.Split(string(data), " ")
 	if len(fields) > 1 {
 		// The output format of `Firecracker --verion` is as follows
-		// Firecracker v0.21.1
+		// Firecracker v0.23.1
 		version = strings.TrimPrefix(strings.TrimSpace(fields[1]), "v")
 		return version, nil
 	}
@@ -612,16 +600,26 @@ func (fc *firecracker) fcSetLogger(ctx context.Context) error {
 		return fmt.Errorf("Failed setting log: %s", err)
 	}
 
+	fc.fcConfig.Logger = &models.Logger{
+		Level:   &fcLogLevel,
+		LogPath: &jailedLogFifo,
+	}
+
+	return err
+}
+
+func (fc *firecracker) fcSetMetrics(ctx context.Context) error {
+	span, _ := fc.trace(ctx, "fcSetMetrics")
+	defer span.End()
+
 	// listen to metrics file and transfer error info
 	jailedMetricsFifo, err := fc.fcListenToFifo(fcMetricsFifo, fc.updateMetrics)
 	if err != nil {
 		return fmt.Errorf("Failed setting log: %s", err)
 	}
 
-	fc.fcConfig.Logger = &models.Logger{
-		Level:       &fcLogLevel,
-		LogFifo:     &jailedLogFifo,
-		MetricsFifo: &jailedMetricsFifo,
+	fc.fcConfig.Metrics = &models.Metrics{
+		MetricsPath: &jailedMetricsFifo,
 	}
 
 	return err
@@ -745,6 +743,10 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error {
 		return err
 	}
 
+	if err := fc.fcSetMetrics(ctx); err != nil {
+		return err
+	}
+
 	fc.state.set(cfReady)
 	for _, d := range fc.pendingDevices {
 		if err := fc.addDevice(ctx, d.dev, d.devType); err != nil {

src/runtime/virtcontainers/kata_agent.go

@@ -51,6 +51,9 @@ const (
 	// containers.
 	KataLocalDevType = "local"
 
+	// Allocating an FSGroup that owns the pod's volumes
+	fsGid = "fsgid"
+
 	// path to vfio devices
 	vfioPath = "/dev/vfio/"
 
@@ -368,19 +371,12 @@ func (k *kataAgent) capabilities() types.Capabilities {
 	return caps
 }
 
-func (k *kataAgent) internalConfigure(h hypervisor, id string, config interface{}) error {
+func (k *kataAgent) internalConfigure(h hypervisor, id string, config KataAgentConfig) error {
 	var err error
-	if config != nil {
-		switch c := config.(type) {
-		case KataAgentConfig:
-			if k.vmSocket, err = h.generateSocket(id); err != nil {
-				return err
-			}
-			k.keepConn = c.LongLiveConn
-		default:
-			return vcTypes.ErrInvalidConfigType
-		}
+	if k.vmSocket, err = h.generateSocket(id); err != nil {
+		return err
 	}
+	k.keepConn = config.LongLiveConn
 
 	return nil
 }
@@ -429,7 +425,7 @@ func cleanupSandboxBindMounts(sandbox *Sandbox) error {
 	return nil
 }
 
-func (k *kataAgent) configure(ctx context.Context, h hypervisor, id, sharePath string, config interface{}) error {
+func (k *kataAgent) configure(ctx context.Context, h hypervisor, id, sharePath string, config KataAgentConfig) error {
 	err := k.internalConfigure(h, id, config)
 	if err != nil {
 		return err
@@ -471,7 +467,7 @@ func (k *kataAgent) configure(ctx context.Context, h hypervisor, id, sharePath s
 	return h.addDevice(ctx, sharedVolume, fsDev)
 }
 
-func (k *kataAgent) configureFromGrpc(h hypervisor, id string, config interface{}) error {
+func (k *kataAgent) configureFromGrpc(h hypervisor, id string, config KataAgentConfig) error {
 	return k.internalConfigure(h, id, config)
 }
 
@@ -1327,7 +1323,11 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
 	epheStorages := k.handleEphemeralStorage(ociSpec.Mounts)
 	ctrStorages = append(ctrStorages, epheStorages...)
 
-	localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
+	localStorages, err := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
+	if err != nil {
+		return nil, err
+	}
+
 	ctrStorages = append(ctrStorages, localStorages...)
 
 	// We replace all OCI mount sources that match our container mount
@@ -1426,10 +1426,27 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage
 
 // handleLocalStorage handles local storage within the VM
 // by creating a directory in the VM from the source of the mount point.
-func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) []*grpc.Storage {
+func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, rootfsSuffix string) ([]*grpc.Storage, error) {
 	var localStorages []*grpc.Storage
 	for idx, mnt := range mounts {
 		if mnt.Type == KataLocalDevType {
+			origin_src := mounts[idx].Source
+			stat := syscall.Stat_t{}
+			err := syscall.Stat(origin_src, &stat)
+			if err != nil {
+				k.Logger().WithError(err).Errorf("failed to stat %s", origin_src)
+				return nil, err
+			}
+
+			dir_options := localDirOptions
+
+			// if volume's gid isn't root group(default group), this means there's
+			// an specific fsGroup is set on this local volume, then it should pass
+			// to guest.
+			if stat.Gid != 0 {
+				dir_options = append(dir_options, fmt.Sprintf("%s=%d", fsGid, stat.Gid))
+			}
+
 			// Set the mount source path to a the desired directory point in the VM.
 			// In this case it is located in the sandbox directory.
 			// We rely on the fact that the first container in the VM has the same ID as the sandbox ID.
@@ -1444,12 +1461,12 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r
 				Source:     KataLocalDevType,
 				Fstype:     KataLocalDevType,
 				MountPoint: mounts[idx].Source,
-				Options:    localDirOptions,
+				Options:    dir_options,
 			}
 			localStorages = append(localStorages, localStorage)
 		}
 	}
-	return localStorages
+	return localStorages, nil
 }
 
 // handleDeviceBlockVolume handles volume that is block device file
@@ -2015,7 +2032,7 @@ func (k *kataAgent) sendReq(spanCtx context.Context, request interface{}) (inter
 	if cancel != nil {
 		defer cancel()
 	}
-	k.Logger().WithField("name", msgName).WithField("req", message.String()).Debug("sending request")
+	k.Logger().WithField("name", msgName).WithField("req", message.String()).Trace("sending request")
 
 	defer func() {
 		agentRPCDurationsHistogram.WithLabelValues(msgName).Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond)))

src/runtime/virtcontainers/kata_agent_test.go

@@ -205,7 +205,8 @@ func TestHandleEphemeralStorage(t *testing.T) {
 func TestHandleLocalStorage(t *testing.T) {
 	k := kataAgent{}
 	var ociMounts []specs.Mount
-	mountSource := "mountPoint"
+	mountSource := "/tmp/mountPoint"
+	os.Mkdir(mountSource, 0755)
 
 	mount := specs.Mount{
 		Type:   KataLocalDevType,
@@ -216,7 +217,7 @@ func TestHandleLocalStorage(t *testing.T) {
 	rootfsSuffix := "rootfs"
 
 	ociMounts = append(ociMounts, mount)
-	localStorages := k.handleLocalStorage(ociMounts, sandboxID, rootfsSuffix)
+	localStorages, _ := k.handleLocalStorage(ociMounts, sandboxID, rootfsSuffix)
 
 	assert.NotNil(t, localStorages)
 	assert.Equal(t, len(localStorages), 1)
@@ -830,10 +831,10 @@ func TestAgentCreateContainer(t *testing.T) {
 		hypervisor: &mockHypervisor{},
 	}
 
-	newStore, err := persist.GetDriver()
+	store, err := persist.GetDriver()
 	assert.NoError(err)
-	assert.NotNil(newStore)
-	sandbox.newStore = newStore
+	assert.NotNil(store)
+	sandbox.store = store
 
 	container := &Container{
 		ctx:       sandbox.ctx,

src/runtime/virtcontainers/mock_agent.go

@@ -176,12 +176,12 @@ func (n *mockAgent) resumeContainer(ctx context.Context, sandbox *Sandbox, c Con
 	return nil
 }
 
-// configHypervisor is the Noop agent hypervisor configuration implementation. It does nothing.
-func (n *mockAgent) configure(ctx context.Context, h hypervisor, id, sharePath string, config interface{}) error {
+// configure is the Noop agent configuration implementation. It does nothing.
+func (n *mockAgent) configure(ctx context.Context, h hypervisor, id, sharePath string, config KataAgentConfig) error {
 	return nil
 }
 
-func (n *mockAgent) configureFromGrpc(h hypervisor, id string, config interface{}) error {
+func (n *mockAgent) configureFromGrpc(h hypervisor, id string, config KataAgentConfig) error {
 	return nil
 }
 

src/runtime/virtcontainers/persist.go

@@ -287,7 +287,7 @@ func (s *Sandbox) Save() error {
 	s.dumpNetwork(&ss)
 	s.dumpConfig(&ss)
 
-	if err := s.newStore.ToDisk(ss, cs); err != nil {
+	if err := s.store.ToDisk(ss, cs); err != nil {
 		return err
 	}
 
@@ -397,7 +397,7 @@ func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) {
 
 // Restore will restore sandbox data from persist file on disk
 func (s *Sandbox) Restore() error {
-	ss, _, err := s.newStore.FromDisk(s.id)
+	ss, _, err := s.store.FromDisk(s.id)
 	if err != nil {
 		return err
 	}
@@ -412,7 +412,7 @@ func (s *Sandbox) Restore() error {
 
 // Restore will restore container data from persist file on disk
 func (c *Container) Restore() error {
-	_, css, err := c.sandbox.newStore.FromDisk(c.sandbox.id)
+	_, css, err := c.sandbox.store.FromDisk(c.sandbox.id)
 	if err != nil {
 		return err
 	}

src/runtime/virtcontainers/persist_test.go

@@ -38,9 +38,9 @@ func TestSandboxRestore(t *testing.T) {
 		state:      types.SandboxState{BlockIndexMap: make(map[int]struct{})},
 	}
 
-	sandbox.newStore, err = persist.GetDriver()
+	sandbox.store, err = persist.GetDriver()
 	assert.NoError(err)
-	assert.NotNil(sandbox.newStore)
+	assert.NotNil(sandbox.store)
 
 	// if we don't call Save(), we can get nothing from disk
 	err = sandbox.Restore()
@@ -67,7 +67,7 @@ func TestSandboxRestore(t *testing.T) {
 
 	// empty the sandbox
 	sandbox.state = types.SandboxState{}
-	if sandbox.newStore, err = persist.GetDriver(); err != nil || sandbox.newStore == nil {
+	if sandbox.store, err = persist.GetDriver(); err != nil || sandbox.store == nil {
 		t.Fatal("failed to get persist driver")
 	}
 

src/runtime/virtcontainers/pkg/agent/protocols/client/client.go

@@ -33,7 +33,7 @@ const (
 	MockHybridVSockScheme = "mock"
 )
 
-var defaultDialTimeout = 15 * time.Second
+var defaultDialTimeout = 30 * time.Second
 
 var hybridVSockPort uint32
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-all: | update-yaml generate-client-code
+all: | update-yaml generate-client-code go-fmt
 MK_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 YQ_INSTALLER := "$(MK_DIR)/../../../../../ci/install_yq.sh"
 VERSIONS_FILE := "$(MK_DIR)/../../../../../versions.yaml"
@@ -17,8 +17,9 @@ generate-client-code: clean-generated-code
 		-i /local/cloud-hypervisor.yaml \
 		-g go \
 		-o /local/client
-
-
+go-fmt:
+	rm client/go.mod; \
+	go fmt ./...
 
 update-yaml:
 ifndef YQ

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md

@@ -36,6 +36,7 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**CreateVM**](docs/DefaultApi.md#createvm) | **Put** /vm.create | Create the cloud-hypervisor Virtual Machine (VM) instance. The instance is not booted, only created.
 *DefaultApi* | [**DeleteVM**](docs/DefaultApi.md#deletevm) | **Put** /vm.delete | Delete the cloud-hypervisor Virtual Machine (VM) instance.
 *DefaultApi* | [**PauseVM**](docs/DefaultApi.md#pausevm) | **Put** /vm.pause | Pause a previously booted VM instance.
+*DefaultApi* | [**PowerButtonVM**](docs/DefaultApi.md#powerbuttonvm) | **Put** /vm.power-button | Trigger a power button in the VM
 *DefaultApi* | [**RebootVM**](docs/DefaultApi.md#rebootvm) | **Put** /vm.reboot | Reboot the VM instance.
 *DefaultApi* | [**ResumeVM**](docs/DefaultApi.md#resumevm) | **Put** /vm.resume | Resume a previously paused VM instance.
 *DefaultApi* | [**ShutdownVM**](docs/DefaultApi.md#shutdownvm) | **Put** /vm.shutdown | Shut the VM instance down.
@@ -76,9 +77,11 @@ Class | Method | HTTP request | Description
  - [NumaDistance](docs/NumaDistance.md)
  - [PciDeviceInfo](docs/PciDeviceInfo.md)
  - [PmemConfig](docs/PmemConfig.md)
+ - [RateLimiterConfig](docs/RateLimiterConfig.md)
  - [RestoreConfig](docs/RestoreConfig.md)
  - [RngConfig](docs/RngConfig.md)
  - [SgxEpcConfig](docs/SgxEpcConfig.md)
+ - [TokenBucket](docs/TokenBucket.md)
  - [VmAddDevice](docs/VmAddDevice.md)
  - [VmConfig](docs/VmConfig.md)
  - [VmInfo](docs/VmInfo.md)

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml

@@ -123,6 +123,18 @@ paths:
         "405":
           description: The VM instance could not reboot because it is not booted.
       summary: Reboot the VM instance.
+  /vm.power-button:
+    put:
+      operationId: power-buttonVM
+      responses:
+        "204":
+          description: Power button successfully triggered in the VM
+        "404":
+          description: The button could not be triggered because it is not created
+            yet
+        "405":
+          description: The button could not be triggered because it is not booted.
+      summary: Trigger a power button in the VM
   /vm.resize:
     put:
       requestBody:
@@ -332,7 +344,7 @@ components:
     VmInfo:
       description: Virtual Machine information
       example:
-        memory_actual_size: 2
+        memory_actual_size: 3
         state: Created
         config:
           console:
@@ -340,54 +352,75 @@ components:
             file: file
             iommu: false
           balloon:
-            size: 4
+            size: 9
           memory:
             hugepages: false
             shared: false
+            hugepage_size: 4
             mergeable: false
             size: 9
             hotplugged_size: 2
             zones:
             - hugepages: false
               shared: false
+              hugepage_size: 1
               mergeable: false
               file: file
-              size: 4
-              hotplugged_size: 1
-              host_numa_node: 7
+              size: 7
+              hotplugged_size: 6
+              host_numa_node: 1
               id: id
               hotplug_size: 1
             - hugepages: false
               shared: false
+              hugepage_size: 1
               mergeable: false
               file: file
-              size: 4
-              hotplugged_size: 1
-              host_numa_node: 7
+              size: 7
+              hotplugged_size: 6
+              host_numa_node: 1
               id: id
               hotplug_size: 1
             hotplug_size: 3
             hotplug_method: acpi
           disks:
           - path: path
-            num_queues: 1
+            num_queues: 7
             readonly: false
             iommu: false
-            queue_size: 6
+            queue_size: 1
             vhost_socket: vhost_socket
             vhost_user: false
             direct: false
             poll_queue: true
+            rate_limiter_config:
+              ops:
+                size: 0
+                one_time_burst: 0
+                refill_time: 0
+              bandwidth:
+                size: 0
+                one_time_burst: 0
+                refill_time: 0
             id: id
           - path: path
-            num_queues: 1
+            num_queues: 7
             readonly: false
             iommu: false
-            queue_size: 6
+            queue_size: 1
             vhost_socket: vhost_socket
             vhost_user: false
             direct: false
             poll_queue: true
+            rate_limiter_config:
+              ops:
+                size: 0
+                one_time_burst: 0
+                refill_time: 0
+              bandwidth:
+                size: 0
+                one_time_burst: 0
+                refill_time: 0
             id: id
           cpus:
             topology:
@@ -409,25 +442,25 @@ components:
             path: path
           numa:
           - distances:
-            - distance: 1
+            - distance: 3
               destination: 6
-            - distance: 1
+            - distance: 3
               destination: 6
             cpus:
-            - 3
-            - 3
+            - 5
+            - 5
             memory_zones:
             - memory_zones
             - memory_zones
             guest_numa_id: 6
           - distances:
-            - distance: 1
+            - distance: 3
               destination: 6
-            - distance: 1
+            - distance: 3
               destination: 6
             cpus:
-            - 3
-            - 3
+            - 5
+            - 5
             memory_zones:
             - memory_zones
             - memory_zones
@@ -437,20 +470,20 @@ components:
             src: /dev/urandom
           sgx_epc:
           - prefault: false
-            size: 9
+            size: 6
           - prefault: false
-            size: 9
+            size: 6
           fs:
-          - num_queues: 5
-            queue_size: 9
-            cache_size: 9
+          - num_queues: 6
+            queue_size: 3
+            cache_size: 6
             dax: true
             tag: tag
             socket: socket
             id: id
-          - num_queues: 5
-            queue_size: 9
-            cache_size: 9
+          - num_queues: 6
+            queue_size: 3
+            cache_size: 6
             dax: true
             tag: tag
             socket: socket
@@ -463,13 +496,13 @@ components:
           pmem:
           - mergeable: false
             file: file
-            size: 6
+            size: 1
             iommu: false
             id: id
             discard_writes: false
           - mergeable: false
             file: file
-            size: 6
+            size: 1
             iommu: false
             id: id
             discard_writes: false
@@ -485,31 +518,37 @@ components:
             path: path
           net:
           - tap: tap
-            num_queues: 7
+            num_queues: 9
             iommu: false
-            queue_size: 1
+            queue_size: 6
             vhost_socket: vhost_socket
             vhost_user: false
             ip: 192.168.249.1
             id: id
             mac: mac
+            fd:
+            - 8
+            - 8
             mask: 255.255.255.0
           - tap: tap
-            num_queues: 7
+            num_queues: 9
             iommu: false
-            queue_size: 1
+            queue_size: 6
             vhost_socket: vhost_socket
             vhost_user: false
             ip: 192.168.249.1
             id: id
             mac: mac
+            fd:
+            - 8
+            - 8
             mask: 255.255.255.0
         device_tree:
           key:
             children:
             - children
             - children
-            pci_bdf: 6
+            pci_bdf: 7
             resources:
             - '{}'
             - '{}'
@@ -540,7 +579,7 @@ components:
         children:
         - children
         - children
-        pci_bdf: 6
+        pci_bdf: 7
         resources:
         - '{}'
         - '{}'
@@ -589,54 +628,75 @@ components:
           file: file
           iommu: false
         balloon:
-          size: 4
+          size: 9
         memory:
           hugepages: false
           shared: false
+          hugepage_size: 4
           mergeable: false
           size: 9
           hotplugged_size: 2
           zones:
           - hugepages: false
             shared: false
+            hugepage_size: 1
             mergeable: false
             file: file
-            size: 4
-            hotplugged_size: 1
-            host_numa_node: 7
+            size: 7
+            hotplugged_size: 6
+            host_numa_node: 1
             id: id
             hotplug_size: 1
           - hugepages: false
             shared: false
+            hugepage_size: 1
             mergeable: false
             file: file
-            size: 4
-            hotplugged_size: 1
-            host_numa_node: 7
+            size: 7
+            hotplugged_size: 6
+            host_numa_node: 1
             id: id
             hotplug_size: 1
           hotplug_size: 3
           hotplug_method: acpi
         disks:
         - path: path
-          num_queues: 1
+          num_queues: 7
           readonly: false
           iommu: false
-          queue_size: 6
+          queue_size: 1
           vhost_socket: vhost_socket
           vhost_user: false
           direct: false
           poll_queue: true
+          rate_limiter_config:
+            ops:
+              size: 0
+              one_time_burst: 0
+              refill_time: 0
+            bandwidth:
+              size: 0
+              one_time_burst: 0
+              refill_time: 0
           id: id
         - path: path
-          num_queues: 1
+          num_queues: 7
           readonly: false
           iommu: false
-          queue_size: 6
+          queue_size: 1
           vhost_socket: vhost_socket
           vhost_user: false
           direct: false
           poll_queue: true
+          rate_limiter_config:
+            ops:
+              size: 0
+              one_time_burst: 0
+              refill_time: 0
+            bandwidth:
+              size: 0
+              one_time_burst: 0
+              refill_time: 0
           id: id
         cpus:
           topology:
@@ -658,25 +718,25 @@ components:
           path: path
         numa:
         - distances:
-          - distance: 1
+          - distance: 3
             destination: 6
-          - distance: 1
+          - distance: 3
             destination: 6
           cpus:
-          - 3
-          - 3
+          - 5
+          - 5
           memory_zones:
           - memory_zones
           - memory_zones
           guest_numa_id: 6
         - distances:
-          - distance: 1
+          - distance: 3
             destination: 6
-          - distance: 1
+          - distance: 3
             destination: 6
           cpus:
-          - 3
-          - 3
+          - 5
+          - 5
           memory_zones:
           - memory_zones
           - memory_zones
@@ -686,20 +746,20 @@ components:
           src: /dev/urandom
         sgx_epc:
         - prefault: false
-          size: 9
+          size: 6
         - prefault: false
-          size: 9
+          size: 6
         fs:
-        - num_queues: 5
-          queue_size: 9
-          cache_size: 9
+        - num_queues: 6
+          queue_size: 3
+          cache_size: 6
           dax: true
           tag: tag
           socket: socket
           id: id
-        - num_queues: 5
-          queue_size: 9
-          cache_size: 9
+        - num_queues: 6
+          queue_size: 3
+          cache_size: 6
           dax: true
           tag: tag
           socket: socket
@@ -712,13 +772,13 @@ components:
         pmem:
         - mergeable: false
           file: file
-          size: 6
+          size: 1
           iommu: false
           id: id
           discard_writes: false
         - mergeable: false
           file: file
-          size: 6
+          size: 1
           iommu: false
           id: id
           discard_writes: false
@@ -734,24 +794,30 @@ components:
           path: path
         net:
         - tap: tap
-          num_queues: 7
+          num_queues: 9
           iommu: false
-          queue_size: 1
+          queue_size: 6
           vhost_socket: vhost_socket
           vhost_user: false
           ip: 192.168.249.1
           id: id
           mac: mac
+          fd:
+          - 8
+          - 8
           mask: 255.255.255.0
         - tap: tap
-          num_queues: 7
+          num_queues: 9
           iommu: false
-          queue_size: 1
+          queue_size: 6
           vhost_socket: vhost_socket
           vhost_user: false
           ip: 192.168.249.1
           id: id
           mac: mac
+          fd:
+          - 8
+          - 8
           mask: 255.255.255.0
       properties:
         cpus:
@@ -858,11 +924,12 @@ components:
       example:
         hugepages: false
         shared: false
+        hugepage_size: 1
         mergeable: false
         file: file
-        size: 4
-        hotplugged_size: 1
-        host_numa_node: 7
+        size: 7
+        hotplugged_size: 6
+        host_numa_node: 1
         id: id
         hotplug_size: 1
       properties:
@@ -882,6 +949,9 @@ components:
         hugepages:
           default: false
           type: boolean
+        hugepage_size:
+          format: int64
+          type: integer
         host_numa_node:
           format: int32
           type: integer
@@ -899,26 +969,29 @@ components:
       example:
         hugepages: false
         shared: false
+        hugepage_size: 4
         mergeable: false
         size: 9
         hotplugged_size: 2
         zones:
         - hugepages: false
           shared: false
+          hugepage_size: 1
           mergeable: false
           file: file
-          size: 4
-          hotplugged_size: 1
-          host_numa_node: 7
+          size: 7
+          hotplugged_size: 6
+          host_numa_node: 1
           id: id
           hotplug_size: 1
         - hugepages: false
           shared: false
+          hugepage_size: 1
           mergeable: false
           file: file
-          size: 4
-          hotplugged_size: 1
-          host_numa_node: 7
+          size: 7
+          hotplugged_size: 6
+          host_numa_node: 1
           id: id
           hotplug_size: 1
         hotplug_size: 3
@@ -945,6 +1018,9 @@ components:
         hugepages:
           default: false
           type: boolean
+        hugepage_size:
+          format: int64
+          type: integer
         zones:
           items:
             $ref: '#/components/schemas/MemoryZoneConfig'
@@ -980,17 +1056,78 @@ components:
       required:
       - args
       type: object
+    TokenBucket:
+      description: Defines a token bucket with a maximum capacity (_size_), an initial
+        burst size (_one_time_burst_) and an interval for refilling purposes (_refill_time_).
+        The refill-rate is derived from _size_ and _refill_time_, and it is the constant
+        rate at which the tokens replenish. The refill process only starts happening
+        after the initial burst budget is consumed. Consumption from the token bucket
+        is unbounded in speed which allows for bursts bound in size by the amount
+        of tokens available. Once the token bucket is empty, consumption speed is
+        bound by the refill-rate.
+      example:
+        size: 0
+        one_time_burst: 0
+        refill_time: 0
+      properties:
+        size:
+          description: The total number of tokens this bucket can hold.
+          format: int64
+          minimum: 0
+          type: integer
+        one_time_burst:
+          description: The initial size of a token bucket.
+          format: int64
+          minimum: 0
+          type: integer
+        refill_time:
+          description: The amount of milliseconds it takes for the bucket to refill.
+          format: int64
+          minimum: 0
+          type: integer
+      required:
+      - refill_time
+      - size
+      type: object
+    RateLimiterConfig:
+      description: Defines an IO rate limiter with independent bytes/s and ops/s limits.
+        Limits are defined by configuring each of the _bandwidth_ and _ops_ token
+        buckets.
+      example:
+        ops:
+          size: 0
+          one_time_burst: 0
+          refill_time: 0
+        bandwidth:
+          size: 0
+          one_time_burst: 0
+          refill_time: 0
+      properties:
+        bandwidth:
+          $ref: '#/components/schemas/TokenBucket'
+        ops:
+          $ref: '#/components/schemas/TokenBucket'
+      type: object
     DiskConfig:
       example:
         path: path
-        num_queues: 1
+        num_queues: 7
         readonly: false
         iommu: false
-        queue_size: 6
+        queue_size: 1
         vhost_socket: vhost_socket
         vhost_user: false
         direct: false
         poll_queue: true
+        rate_limiter_config:
+          ops:
+            size: 0
+            one_time_burst: 0
+            refill_time: 0
+          bandwidth:
+            size: 0
+            one_time_burst: 0
+            refill_time: 0
         id: id
       properties:
         path:
@@ -1018,6 +1155,8 @@ components:
         poll_queue:
           default: true
           type: boolean
+        rate_limiter_config:
+          $ref: '#/components/schemas/RateLimiterConfig'
         id:
           type: string
       required:
@@ -1026,14 +1165,17 @@ components:
     NetConfig:
       example:
         tap: tap
-        num_queues: 7
+        num_queues: 9
         iommu: false
-        queue_size: 1
+        queue_size: 6
         vhost_socket: vhost_socket
         vhost_user: false
         ip: 192.168.249.1
         id: id
         mac: mac
+        fd:
+        - 8
+        - 8
         mask: 255.255.255.0
       properties:
         tap:
@@ -1063,6 +1205,11 @@ components:
           type: string
         id:
           type: string
+        fd:
+          items:
+            format: int32
+            type: integer
+          type: array
       type: object
     RngConfig:
       example:
@@ -1080,7 +1227,7 @@ components:
       type: object
     BalloonConfig:
       example:
-        size: 4
+        size: 9
       properties:
         size:
           format: int64
@@ -1090,9 +1237,9 @@ components:
       type: object
     FsConfig:
       example:
-        num_queues: 5
-        queue_size: 9
-        cache_size: 9
+        num_queues: 6
+        queue_size: 3
+        cache_size: 6
         dax: true
         tag: tag
         socket: socket
@@ -1128,7 +1275,7 @@ components:
       example:
         mergeable: false
         file: file
-        size: 6
+        size: 1
         iommu: false
         id: id
         discard_writes: false
@@ -1163,6 +1310,7 @@ components:
         mode:
           enum:
           - "false"
+          - Pty
           - Tty
           - File
           - null
@@ -1216,7 +1364,7 @@ components:
     SgxEpcConfig:
       example:
         prefault: false
-        size: 9
+        size: 6
       properties:
         size:
           format: int64
@@ -1229,7 +1377,7 @@ components:
       type: object
     NumaDistance:
       example:
-        distance: 1
+        distance: 3
         destination: 6
       properties:
         destination:
@@ -1245,13 +1393,13 @@ components:
     NumaConfig:
       example:
         distances:
-        - distance: 1
+        - distance: 3
           destination: 6
-        - distance: 1
+        - distance: 3
           destination: 6
         cpus:
-        - 3
-        - 3
+        - 5
+        - 5
         memory_zones:
         - memory_zones
         - memory_zones

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go

@@ -279,6 +279,69 @@ func (a *DefaultApiService) PauseVM(ctx _context.Context) (*_nethttp.Response, e
 	return localVarHTTPResponse, nil
 }
 
+/*
+PowerButtonVM Trigger a power button in the VM
+ * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+*/
+func (a *DefaultApiService) PowerButtonVM(ctx _context.Context) (*_nethttp.Response, error) {
+	var (
+		localVarHTTPMethod   = _nethttp.MethodPut
+		localVarPostBody     interface{}
+		localVarFormFileName string
+		localVarFileName     string
+		localVarFileBytes    []byte
+	)
+
+	// create path and map variables
+	localVarPath := a.client.cfg.BasePath + "/vm.power-button"
+	localVarHeaderParams := make(map[string]string)
+	localVarQueryParams := _neturl.Values{}
+	localVarFormParams := _neturl.Values{}
+
+	// to determine the Content-Type header
+	localVarHTTPContentTypes := []string{}
+
+	// set Content-Type header
+	localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+	if localVarHTTPContentType != "" {
+		localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+	}
+
+	// to determine the Accept header
+	localVarHTTPHeaderAccepts := []string{}
+
+	// set Accept header
+	localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+	if localVarHTTPHeaderAccept != "" {
+		localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+	}
+	r, err := a.client.prepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	localVarHTTPResponse, err := a.client.callAPI(r)
+	if err != nil || localVarHTTPResponse == nil {
+		return localVarHTTPResponse, err
+	}
+
+	localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body)
+	localVarHTTPResponse.Body.Close()
+	if err != nil {
+		return localVarHTTPResponse, err
+	}
+
+	if localVarHTTPResponse.StatusCode >= 300 {
+		newErr := GenericOpenAPIError{
+			body:  localVarBody,
+			error: localVarHTTPResponse.Status,
+		}
+		return localVarHTTPResponse, newErr
+	}
+
+	return localVarHTTPResponse, nil
+}
+
 /*
 RebootVM Reboot the VM instance.
  * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md

@@ -8,6 +8,7 @@ Method | HTTP request | Description
 [**CreateVM**](DefaultApi.md#CreateVM) | **Put** /vm.create | Create the cloud-hypervisor Virtual Machine (VM) instance. The instance is not booted, only created.
 [**DeleteVM**](DefaultApi.md#DeleteVM) | **Put** /vm.delete | Delete the cloud-hypervisor Virtual Machine (VM) instance.
 [**PauseVM**](DefaultApi.md#PauseVM) | **Put** /vm.pause | Pause a previously booted VM instance.
+[**PowerButtonVM**](DefaultApi.md#PowerButtonVM) | **Put** /vm.power-button | Trigger a power button in the VM
 [**RebootVM**](DefaultApi.md#RebootVM) | **Put** /vm.reboot | Reboot the VM instance.
 [**ResumeVM**](DefaultApi.md#ResumeVM) | **Put** /vm.resume | Resume a previously paused VM instance.
 [**ShutdownVM**](DefaultApi.md#ShutdownVM) | **Put** /vm.shutdown | Shut the VM instance down.
@@ -145,6 +146,34 @@ No authorization required
 [[Back to README]](../README.md)
 
 
+## PowerButtonVM
+
+> PowerButtonVM(ctx, )
+
+Trigger a power button in the VM
+
+### Required Parameters
+
+This endpoint does not need any parameter.
+
+### Return type
+
+ (empty response body)
+
+### Authorization
+
+No authorization required
+
+### HTTP request headers
+
+- **Content-Type**: Not defined
+- **Accept**: Not defined
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
 ## RebootVM
 
 > RebootVM(ctx, )

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DiskConfig.md

@@ -13,6 +13,7 @@ Name | Type | Description | Notes
 **VhostUser** | **bool** |  | [optional] [default to false]
 **VhostSocket** | **string** |  | [optional] 
 **PollQueue** | **bool** |  | [optional] [default to true]
+**RateLimiterConfig** | [**RateLimiterConfig**](RateLimiterConfig.md) |  | [optional] 
 **Id** | **string** |  | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md

@@ -11,6 +11,7 @@ Name | Type | Description | Notes
 **HotplugMethod** | **string** |  | [optional] [default to acpi]
 **Shared** | **bool** |  | [optional] [default to false]
 **Hugepages** | **bool** |  | [optional] [default to false]
+**HugepageSize** | **int64** |  | [optional] 
 **Zones** | [**[]MemoryZoneConfig**](MemoryZoneConfig.md) |  | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryZoneConfig.md

@@ -10,6 +10,7 @@ Name | Type | Description | Notes
 **Mergeable** | **bool** |  | [optional] [default to false]
 **Shared** | **bool** |  | [optional] [default to false]
 **Hugepages** | **bool** |  | [optional] [default to false]
+**HugepageSize** | **int64** |  | [optional] 
 **HostNumaNode** | **int32** |  | [optional] 
 **HotplugSize** | **int64** |  | [optional] 
 **HotpluggedSize** | **int64** |  | [optional] 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/NetConfig.md

@@ -14,6 +14,7 @@ Name | Type | Description | Notes
 **VhostUser** | **bool** |  | [optional] [default to false]
 **VhostSocket** | **string** |  | [optional] 
 **Id** | **string** |  | [optional] 
+**Fd** | **[]int32** |  | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/RateLimiterConfig.md

@@ -0,0 +1,12 @@
+# RateLimiterConfig
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Bandwidth** | [**TokenBucket**](TokenBucket.md) |  | [optional] 
+**Ops** | [**TokenBucket**](TokenBucket.md) |  | [optional] 
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/TokenBucket.md

@@ -0,0 +1,13 @@
+# TokenBucket
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Size** | **int64** | The total number of tokens this bucket can hold. | 
+**OneTimeBurst** | **int64** | The initial size of a token bucket. | [optional] 
+**RefillTime** | **int64** | The amount of milliseconds it takes for the bucket to refill. | 
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_disk_config.go

@@ -11,14 +11,15 @@ package openapi
 
 // DiskConfig struct for DiskConfig
 type DiskConfig struct {
-	Path        string `json:"path"`
-	Readonly    bool   `json:"readonly,omitempty"`
-	Direct      bool   `json:"direct,omitempty"`
-	Iommu       bool   `json:"iommu,omitempty"`
-	NumQueues   int32  `json:"num_queues,omitempty"`
-	QueueSize   int32  `json:"queue_size,omitempty"`
-	VhostUser   bool   `json:"vhost_user,omitempty"`
-	VhostSocket string `json:"vhost_socket,omitempty"`
-	PollQueue   bool   `json:"poll_queue,omitempty"`
-	Id          string `json:"id,omitempty"`
+	Path              string            `json:"path"`
+	Readonly          bool              `json:"readonly,omitempty"`
+	Direct            bool              `json:"direct,omitempty"`
+	Iommu             bool              `json:"iommu,omitempty"`
+	NumQueues         int32             `json:"num_queues,omitempty"`
+	QueueSize         int32             `json:"queue_size,omitempty"`
+	VhostUser         bool              `json:"vhost_user,omitempty"`
+	VhostSocket       string            `json:"vhost_socket,omitempty"`
+	PollQueue         bool              `json:"poll_queue,omitempty"`
+	RateLimiterConfig RateLimiterConfig `json:"rate_limiter_config,omitempty"`
+	Id                string            `json:"id,omitempty"`
 }

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go

@@ -18,5 +18,6 @@ type MemoryConfig struct {
 	HotplugMethod  string             `json:"hotplug_method,omitempty"`
 	Shared         bool               `json:"shared,omitempty"`
 	Hugepages      bool               `json:"hugepages,omitempty"`
+	HugepageSize   int64              `json:"hugepage_size,omitempty"`
 	Zones          []MemoryZoneConfig `json:"zones,omitempty"`
 }

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_zone_config.go

@@ -17,6 +17,7 @@ type MemoryZoneConfig struct {
 	Mergeable      bool   `json:"mergeable,omitempty"`
 	Shared         bool   `json:"shared,omitempty"`
 	Hugepages      bool   `json:"hugepages,omitempty"`
+	HugepageSize   int64  `json:"hugepage_size,omitempty"`
 	HostNumaNode   int32  `json:"host_numa_node,omitempty"`
 	HotplugSize    int64  `json:"hotplug_size,omitempty"`
 	HotpluggedSize int64  `json:"hotplugged_size,omitempty"`

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_net_config.go

@@ -11,14 +11,15 @@ package openapi
 
 // NetConfig struct for NetConfig
 type NetConfig struct {
-	Tap         string `json:"tap,omitempty"`
-	Ip          string `json:"ip,omitempty"`
-	Mask        string `json:"mask,omitempty"`
-	Mac         string `json:"mac,omitempty"`
-	Iommu       bool   `json:"iommu,omitempty"`
-	NumQueues   int32  `json:"num_queues,omitempty"`
-	QueueSize   int32  `json:"queue_size,omitempty"`
-	VhostUser   bool   `json:"vhost_user,omitempty"`
-	VhostSocket string `json:"vhost_socket,omitempty"`
-	Id          string `json:"id,omitempty"`
+	Tap         string  `json:"tap,omitempty"`
+	Ip          string  `json:"ip,omitempty"`
+	Mask        string  `json:"mask,omitempty"`
+	Mac         string  `json:"mac,omitempty"`
+	Iommu       bool    `json:"iommu,omitempty"`
+	NumQueues   int32   `json:"num_queues,omitempty"`
+	QueueSize   int32   `json:"queue_size,omitempty"`
+	VhostUser   bool    `json:"vhost_user,omitempty"`
+	VhostSocket string  `json:"vhost_socket,omitempty"`
+	Id          string  `json:"id,omitempty"`
+	Fd          []int32 `json:"fd,omitempty"`
 }

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_rate_limiter_config.go

@@ -0,0 +1,16 @@
+/*
+ * Cloud Hypervisor API
+ *
+ * Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine.
+ *
+ * API version: 0.3.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+// RateLimiterConfig Defines an IO rate limiter with independent bytes/s and ops/s limits. Limits are defined by configuring each of the _bandwidth_ and _ops_ token buckets.
+type RateLimiterConfig struct {
+	Bandwidth TokenBucket `json:"bandwidth,omitempty"`
+	Ops       TokenBucket `json:"ops,omitempty"`
+}

src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_token_bucket.go

@@ -0,0 +1,20 @@
+/*
+ * Cloud Hypervisor API
+ *
+ * Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine.
+ *
+ * API version: 0.3.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+// TokenBucket Defines a token bucket with a maximum capacity (_size_), an initial burst size (_one_time_burst_) and an interval for refilling purposes (_refill_time_). The refill-rate is derived from _size_ and _refill_time_, and it is the constant rate at which the tokens replenish. The refill process only starts happening after the initial burst budget is consumed. Consumption from the token bucket is unbounded in speed which allows for bursts bound in size by the amount of tokens available. Once the token bucket is empty, consumption speed is bound by the refill-rate.
+type TokenBucket struct {
+	// The total number of tokens this bucket can hold.
+	Size int64 `json:"size"`
+	// The initial size of a token bucket.
+	OneTimeBurst int64 `json:"one_time_burst,omitempty"`
+	// The amount of milliseconds it takes for the bucket to refill.
+	RefillTime int64 `json:"refill_time"`
+}

src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml

@@ -134,6 +134,18 @@ paths:
         405:
           description: The VM instance could not reboot because it is not booted.
 
+  /vm.power-button:
+    put:
+      summary: Trigger a power button in the VM
+      operationId: power-buttonVM
+      responses:
+        204:
+          description: Power button successfully triggered in the VM
+        404:
+          description: The button could not be triggered because it is not created yet
+        405:
+          description: The button could not be triggered because it is not booted.
+
   /vm.resize:
     put:
       summary: Resize the VM
@@ -386,7 +398,7 @@ components:
         pci_bdf:
           type: integer
           format: int32
-        
+
     VmCounters:
       type: object
       additionalProperties:
@@ -522,6 +534,9 @@ components:
         hugepages:
           type: boolean
           default: false
+        hugepage_size:
+          type: integer
+          format: int64
         host_numa_node:
           type: integer
           format: int32
@@ -559,6 +574,9 @@ components:
         hugepages:
           type: boolean
           default: false
+        hugepage_size:
+          type: integer
+          format: int64
         zones:
           type: array
           items:
@@ -589,6 +607,48 @@ components:
         args:
           type: string
 
+    TokenBucket:
+      required:
+      - size
+      - refill_time
+      type: object
+      properties:
+        size:
+          type: integer
+          format: int64
+          minimum: 0
+          description: The total number of tokens this bucket can hold.
+        one_time_burst:
+          type: integer
+          format: int64
+          minimum: 0
+          description: The initial size of a token bucket.
+        refill_time:
+          type: integer
+          format: int64
+          minimum: 0
+          description: The amount of milliseconds it takes for the bucket to refill.
+      description:
+        Defines a token bucket with a maximum capacity (_size_), an initial burst size
+        (_one_time_burst_) and an interval for refilling purposes (_refill_time_).
+        The refill-rate is derived from _size_ and _refill_time_, and it is the constant
+        rate at which the tokens replenish. The refill process only starts happening after
+        the initial burst budget is consumed.
+        Consumption from the token bucket is unbounded in speed which allows for bursts
+        bound in size by the amount of tokens available.
+        Once the token bucket is empty, consumption speed is bound by the refill-rate.
+
+    RateLimiterConfig:
+      type: object
+      properties:
+        bandwidth:
+          $ref: '#/components/schemas/TokenBucket'
+        ops:
+          $ref: '#/components/schemas/TokenBucket'
+      description:
+        Defines an IO rate limiter with independent bytes/s and ops/s limits.
+        Limits are defined by configuring each of the _bandwidth_ and _ops_ token buckets.
+
     DiskConfig:
       required:
       - path
@@ -619,6 +679,8 @@ components:
         poll_queue:
           type: boolean
           default: true
+        rate_limiter_config:
+            $ref: '#/components/schemas/RateLimiterConfig'
         id:
           type: string
 
@@ -652,6 +714,11 @@ components:
           type: string
         id:
           type: string
+        fd:
+          type: array
+          items:
+            type: integer
+            format: int32
 
     RngConfig:
       required:
@@ -735,7 +802,7 @@ components:
           type: string
         mode:
           type: string
-          enum: [Off, Tty, File, Null]
+          enum: [Off, Pty, Tty, File, Null]
         iommu:
           type: boolean
           default: false

src/runtime/virtcontainers/pkg/firecracker/client/models/boot_source.go

@@ -20,6 +20,9 @@ type BootSource struct {
 	// Kernel boot arguments
 	BootArgs string `json:"boot_args,omitempty"`
 
+	// Host level path to the initrd image used to boot the guest
+	InitrdPath string `json:"initrd_path,omitempty"`
+
 	// Host level path to the kernel image used to boot the guest
 	// Required: true
 	KernelImagePath *string `json:"kernel_image_path"`

src/runtime/virtcontainers/pkg/firecracker/client/models/instance_info.go

@@ -6,8 +6,6 @@ package models
 // Editing this file might prove futile when you re-run the swagger generate command
 
 import (
-	"encoding/json"
-
 	strfmt "github.com/go-openapi/strfmt"
 
 	"github.com/go-openapi/errors"
@@ -19,14 +17,15 @@ import (
 // swagger:model InstanceInfo
 type InstanceInfo struct {
 
+	// Application name.
+	AppName string `json:"app_name,omitempty"`
+
 	// MicroVM / instance ID.
 	// Required: true
 	ID *string `json:"id"`
 
 	// The current detailed state of the Firecracker instance. This value is read-only for the control-plane.
-	// Required: true
-	// Enum: [Uninitialized Starting Running]
-	State *string `json:"state"`
+	Started bool `json:"started,omitempty"`
 
 	// MicroVM hypervisor build version.
 	// Required: true
@@ -41,10 +40,6 @@ func (m *InstanceInfo) Validate(formats strfmt.Registry) error {
 		res = append(res, err)
 	}
 
-	if err := m.validateState(formats); err != nil {
-		res = append(res, err)
-	}
-
 	if err := m.validateVmmVersion(formats); err != nil {
 		res = append(res, err)
 	}
@@ -64,52 +59,6 @@ func (m *InstanceInfo) validateID(formats strfmt.Registry) error {
 	return nil
 }
 
-var instanceInfoTypeStatePropEnum []interface{}
-
-func init() {
-	var res []string
-	if err := json.Unmarshal([]byte(`["Uninitialized","Starting","Running"]`), &res); err != nil {
-		panic(err)
-	}
-	for _, v := range res {
-		instanceInfoTypeStatePropEnum = append(instanceInfoTypeStatePropEnum, v)
-	}
-}
-
-const (
-
-	// InstanceInfoStateUninitialized captures enum value "Uninitialized"
-	InstanceInfoStateUninitialized string = "Uninitialized"
-
-	// InstanceInfoStateStarting captures enum value "Starting"
-	InstanceInfoStateStarting string = "Starting"
-
-	// InstanceInfoStateRunning captures enum value "Running"
-	InstanceInfoStateRunning string = "Running"
-)
-
-// prop value enum
-func (m *InstanceInfo) validateStateEnum(path, location string, value string) error {
-	if err := validate.Enum(path, location, value, instanceInfoTypeStatePropEnum); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (m *InstanceInfo) validateState(formats strfmt.Registry) error {
-
-	if err := validate.Required("state", "body", m.State); err != nil {
-		return err
-	}
-
-	// value enum
-	if err := m.validateStateEnum("state", "body", *m.State); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (m *InstanceInfo) validateVmmVersion(formats strfmt.Registry) error {
 
 	if err := validate.Required("vmm_version", "body", m.VmmVersion); err != nil {

src/runtime/virtcontainers/pkg/firecracker/client/models/logger.go

@@ -19,17 +19,13 @@ import (
 // swagger:model Logger
 type Logger struct {
 
-	// Set the level.
+	// Set the level. The possible values are case-insensitive.
 	// Enum: [Error Warning Info Debug]
 	Level *string `json:"level,omitempty"`
 
-	// The named pipe for the human readable log output.
+	// Path to the named pipe or file for the human readable log output.
 	// Required: true
-	LogFifo *string `json:"log_fifo"`
-
-	// The named pipe where the JSON-formatted metrics will be flushed.
-	// Required: true
-	MetricsFifo *string `json:"metrics_fifo"`
+	LogPath *string `json:"log_path"`
 
 	// Whether or not to output the level in the logs.
 	ShowLevel *bool `json:"show_level,omitempty"`
@@ -46,11 +42,7 @@ func (m *Logger) Validate(formats strfmt.Registry) error {
 		res = append(res, err)
 	}
 
-	if err := m.validateLogFifo(formats); err != nil {
-		res = append(res, err)
-	}
-
-	if err := m.validateMetricsFifo(formats); err != nil {
+	if err := m.validateLogPath(formats); err != nil {
 		res = append(res, err)
 	}
 
@@ -109,18 +101,9 @@ func (m *Logger) validateLevel(formats strfmt.Registry) error {
 	return nil
 }
 
-func (m *Logger) validateLogFifo(formats strfmt.Registry) error {
+func (m *Logger) validateLogPath(formats strfmt.Registry) error {
 
-	if err := validate.Required("log_fifo", "body", m.LogFifo); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *Logger) validateMetricsFifo(formats strfmt.Registry) error {
-
-	if err := validate.Required("metrics_fifo", "body", m.MetricsFifo); err != nil {
+	if err := validate.Required("log_path", "body", m.LogPath); err != nil {
 		return err
 	}
 

src/runtime/virtcontainers/pkg/firecracker/client/models/machine_configuration.go

@@ -28,6 +28,9 @@ type MachineConfiguration struct {
 	// Required: true
 	MemSizeMib *int64 `json:"mem_size_mib"`
 
+	// Enable dirty page tracking. If this is enabled, then incremental guest memory snapshots can be created. These belong to diff snapshots, which contain, besides the microVM state, only the memory dirtied since a previous snapshot. Full snapshots each contain a full copy of the guest memory.
+	TrackDirtyPages bool `json:"track_dirty_pages,omitempty"`
+
 	// Number of vCPUs (either 1 or an even number)
 	// Required: true
 	// Maximum: 32

src/runtime/virtcontainers/pkg/firecracker/client/models/metrics.go

@@ -0,0 +1,64 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/swag"
+	"github.com/go-openapi/validate"
+)
+
+// Metrics Describes the configuration option for the metrics capability.
+// swagger:model Metrics
+type Metrics struct {
+
+	// Path to the named pipe or file where the JSON-formatted metrics are flushed.
+	// Required: true
+	MetricsPath *string `json:"metrics_path"`
+}
+
+// Validate validates this metrics
+func (m *Metrics) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateMetricsPath(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *Metrics) validateMetricsPath(formats strfmt.Registry) error {
+
+	if err := validate.Required("metrics_path", "body", m.MetricsPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *Metrics) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *Metrics) UnmarshalBinary(b []byte) error {
+	var res Metrics
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/models/mmds_config.go

@@ -0,0 +1,43 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/swag"
+)
+
+// MmdsConfig Defines the MMDS configuration.
+// swagger:model MmdsConfig
+type MmdsConfig struct {
+
+	// A valid IPv4 link-local address.
+	IPV4Address *string `json:"ipv4_address,omitempty"`
+}
+
+// Validate validates this mmds config
+func (m *MmdsConfig) Validate(formats strfmt.Registry) error {
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *MmdsConfig) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *MmdsConfig) UnmarshalBinary(b []byte) error {
+	var res MmdsConfig
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/models/snapshot_create_params.go

@@ -0,0 +1,134 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/swag"
+	"github.com/go-openapi/validate"
+)
+
+// SnapshotCreateParams snapshot create params
+// swagger:model SnapshotCreateParams
+type SnapshotCreateParams struct {
+
+	// Path to the file that will contain the guest memory.
+	// Required: true
+	MemFilePath *string `json:"mem_file_path"`
+
+	// Path to the file that will contain the microVM state.
+	// Required: true
+	SnapshotPath *string `json:"snapshot_path"`
+
+	// Type of snapshot to create. It is optional and by default, a full snapshot is created.
+	// Enum: [Full]
+	SnapshotType string `json:"snapshot_type,omitempty"`
+
+	// The microVM version for which we want to create the snapshot. It is optional and it defaults to the current version.
+	Version string `json:"version,omitempty"`
+}
+
+// Validate validates this snapshot create params
+func (m *SnapshotCreateParams) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateMemFilePath(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if err := m.validateSnapshotPath(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if err := m.validateSnapshotType(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *SnapshotCreateParams) validateMemFilePath(formats strfmt.Registry) error {
+
+	if err := validate.Required("mem_file_path", "body", m.MemFilePath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *SnapshotCreateParams) validateSnapshotPath(formats strfmt.Registry) error {
+
+	if err := validate.Required("snapshot_path", "body", m.SnapshotPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+var snapshotCreateParamsTypeSnapshotTypePropEnum []interface{}
+
+func init() {
+	var res []string
+	if err := json.Unmarshal([]byte(`["Full"]`), &res); err != nil {
+		panic(err)
+	}
+	for _, v := range res {
+		snapshotCreateParamsTypeSnapshotTypePropEnum = append(snapshotCreateParamsTypeSnapshotTypePropEnum, v)
+	}
+}
+
+const (
+
+	// SnapshotCreateParamsSnapshotTypeFull captures enum value "Full"
+	SnapshotCreateParamsSnapshotTypeFull string = "Full"
+)
+
+// prop value enum
+func (m *SnapshotCreateParams) validateSnapshotTypeEnum(path, location string, value string) error {
+	if err := validate.Enum(path, location, value, snapshotCreateParamsTypeSnapshotTypePropEnum); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *SnapshotCreateParams) validateSnapshotType(formats strfmt.Registry) error {
+
+	if swag.IsZero(m.SnapshotType) { // not required
+		return nil
+	}
+
+	// value enum
+	if err := m.validateSnapshotTypeEnum("snapshot_type", "body", m.SnapshotType); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *SnapshotCreateParams) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *SnapshotCreateParams) UnmarshalBinary(b []byte) error {
+	var res SnapshotCreateParams
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/models/snapshot_load_params.go

@@ -0,0 +1,84 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/swag"
+	"github.com/go-openapi/validate"
+)
+
+// SnapshotLoadParams snapshot load params
+// swagger:model SnapshotLoadParams
+type SnapshotLoadParams struct {
+
+	// Enable support for incremental (diff) snapshots by tracking dirty guest pages.
+	EnableDiffSnapshots bool `json:"enable_diff_snapshots,omitempty"`
+
+	// Path to the file that contains the guest memory to be loaded.
+	// Required: true
+	MemFilePath *string `json:"mem_file_path"`
+
+	// Path to the file that contains the microVM state to be loaded.
+	// Required: true
+	SnapshotPath *string `json:"snapshot_path"`
+}
+
+// Validate validates this snapshot load params
+func (m *SnapshotLoadParams) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateMemFilePath(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if err := m.validateSnapshotPath(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *SnapshotLoadParams) validateMemFilePath(formats strfmt.Registry) error {
+
+	if err := validate.Required("mem_file_path", "body", m.MemFilePath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *SnapshotLoadParams) validateSnapshotPath(formats strfmt.Registry) error {
+
+	if err := validate.Required("snapshot_path", "body", m.SnapshotPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *SnapshotLoadParams) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *SnapshotLoadParams) UnmarshalBinary(b []byte) error {
+	var res SnapshotLoadParams
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/models/vm.go

@@ -0,0 +1,101 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/swag"
+	"github.com/go-openapi/validate"
+)
+
+// VM Defines the microVM running state. It is especially useful in the snapshotting context.
+// swagger:model Vm
+type VM struct {
+
+	// state
+	// Required: true
+	// Enum: [Paused Resumed]
+	State *string `json:"state"`
+}
+
+// Validate validates this Vm
+func (m *VM) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateState(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+var vmTypeStatePropEnum []interface{}
+
+func init() {
+	var res []string
+	if err := json.Unmarshal([]byte(`["Paused","Resumed"]`), &res); err != nil {
+		panic(err)
+	}
+	for _, v := range res {
+		vmTypeStatePropEnum = append(vmTypeStatePropEnum, v)
+	}
+}
+
+const (
+
+	// VMStatePaused captures enum value "Paused"
+	VMStatePaused string = "Paused"
+
+	// VMStateResumed captures enum value "Resumed"
+	VMStateResumed string = "Resumed"
+)
+
+// prop value enum
+func (m *VM) validateStateEnum(path, location string, value string) error {
+	if err := validate.Enum(path, location, value, vmTypeStatePropEnum); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *VM) validateState(formats strfmt.Registry) error {
+
+	if err := validate.Required("state", "body", m.State); err != nil {
+		return err
+	}
+
+	// value enum
+	if err := m.validateStateEnum("state", "body", *m.State); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *VM) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *VM) UnmarshalBinary(b []byte) error {
+	var res VM
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/create_snapshot_parameters.go

@@ -0,0 +1,139 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// NewCreateSnapshotParams creates a new CreateSnapshotParams object
+// with the default values initialized.
+func NewCreateSnapshotParams() *CreateSnapshotParams {
+	var ()
+	return &CreateSnapshotParams{
+
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewCreateSnapshotParamsWithTimeout creates a new CreateSnapshotParams object
+// with the default values initialized, and the ability to set a timeout on a request
+func NewCreateSnapshotParamsWithTimeout(timeout time.Duration) *CreateSnapshotParams {
+	var ()
+	return &CreateSnapshotParams{
+
+		timeout: timeout,
+	}
+}
+
+// NewCreateSnapshotParamsWithContext creates a new CreateSnapshotParams object
+// with the default values initialized, and the ability to set a context for a request
+func NewCreateSnapshotParamsWithContext(ctx context.Context) *CreateSnapshotParams {
+	var ()
+	return &CreateSnapshotParams{
+
+		Context: ctx,
+	}
+}
+
+// NewCreateSnapshotParamsWithHTTPClient creates a new CreateSnapshotParams object
+// with the default values initialized, and the ability to set a custom HTTPClient for a request
+func NewCreateSnapshotParamsWithHTTPClient(client *http.Client) *CreateSnapshotParams {
+	var ()
+	return &CreateSnapshotParams{
+		HTTPClient: client,
+	}
+}
+
+/*CreateSnapshotParams contains all the parameters to send to the API endpoint
+for the create snapshot operation typically these are written to a http.Request
+*/
+type CreateSnapshotParams struct {
+
+	/*Body
+	  The configuration used for creating a snaphot.
+
+	*/
+	Body *models.SnapshotCreateParams
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithTimeout adds the timeout to the create snapshot params
+func (o *CreateSnapshotParams) WithTimeout(timeout time.Duration) *CreateSnapshotParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the create snapshot params
+func (o *CreateSnapshotParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the create snapshot params
+func (o *CreateSnapshotParams) WithContext(ctx context.Context) *CreateSnapshotParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the create snapshot params
+func (o *CreateSnapshotParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the create snapshot params
+func (o *CreateSnapshotParams) WithHTTPClient(client *http.Client) *CreateSnapshotParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the create snapshot params
+func (o *CreateSnapshotParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithBody adds the body to the create snapshot params
+func (o *CreateSnapshotParams) WithBody(body *models.SnapshotCreateParams) *CreateSnapshotParams {
+	o.SetBody(body)
+	return o
+}
+
+// SetBody adds the body to the create snapshot params
+func (o *CreateSnapshotParams) SetBody(body *models.SnapshotCreateParams) {
+	o.Body = body
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *CreateSnapshotParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+
+	if o.Body != nil {
+		if err := r.SetBodyParam(o.Body); err != nil {
+			return err
+		}
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/create_snapshot_responses.go

@@ -0,0 +1,140 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// CreateSnapshotReader is a Reader for the CreateSnapshot structure.
+type CreateSnapshotReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *CreateSnapshotReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	switch response.Code() {
+
+	case 204:
+		result := NewCreateSnapshotNoContent()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return result, nil
+
+	case 400:
+		result := NewCreateSnapshotBadRequest()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return nil, result
+
+	default:
+		result := NewCreateSnapshotDefault(response.Code())
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		if response.Code()/100 == 2 {
+			return result, nil
+		}
+		return nil, result
+	}
+}
+
+// NewCreateSnapshotNoContent creates a CreateSnapshotNoContent with default headers values
+func NewCreateSnapshotNoContent() *CreateSnapshotNoContent {
+	return &CreateSnapshotNoContent{}
+}
+
+/*CreateSnapshotNoContent handles this case with default header values.
+
+Snapshot created
+*/
+type CreateSnapshotNoContent struct {
+}
+
+func (o *CreateSnapshotNoContent) Error() string {
+	return fmt.Sprintf("[PUT /snapshot/create][%d] createSnapshotNoContent ", 204)
+}
+
+func (o *CreateSnapshotNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	return nil
+}
+
+// NewCreateSnapshotBadRequest creates a CreateSnapshotBadRequest with default headers values
+func NewCreateSnapshotBadRequest() *CreateSnapshotBadRequest {
+	return &CreateSnapshotBadRequest{}
+}
+
+/*CreateSnapshotBadRequest handles this case with default header values.
+
+Snapshot cannot be created due to bad input
+*/
+type CreateSnapshotBadRequest struct {
+	Payload *models.Error
+}
+
+func (o *CreateSnapshotBadRequest) Error() string {
+	return fmt.Sprintf("[PUT /snapshot/create][%d] createSnapshotBadRequest  %+v", 400, o.Payload)
+}
+
+func (o *CreateSnapshotBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+// NewCreateSnapshotDefault creates a CreateSnapshotDefault with default headers values
+func NewCreateSnapshotDefault(code int) *CreateSnapshotDefault {
+	return &CreateSnapshotDefault{
+		_statusCode: code,
+	}
+}
+
+/*CreateSnapshotDefault handles this case with default header values.
+
+Internal server error
+*/
+type CreateSnapshotDefault struct {
+	_statusCode int
+
+	Payload *models.Error
+}
+
+// Code gets the status code for the create snapshot default response
+func (o *CreateSnapshotDefault) Code() int {
+	return o._statusCode
+}
+
+func (o *CreateSnapshotDefault) Error() string {
+	return fmt.Sprintf("[PUT /snapshot/create][%d] createSnapshot default  %+v", o._statusCode, o.Payload)
+}
+
+func (o *CreateSnapshotDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/get_mmds_responses.go

@@ -32,8 +32,8 @@ func (o *GetMmdsReader) ReadResponse(response runtime.ClientResponse, consumer r
 		}
 		return result, nil
 
-	case 400:
-		result := NewGetMmdsBadRequest()
+	case 404:
+		result := NewGetMmdsNotFound()
 		if err := result.readResponse(response, consumer, o.formats); err != nil {
 			return nil, err
 		}
@@ -78,24 +78,24 @@ func (o *GetMmdsOK) readResponse(response runtime.ClientResponse, consumer runti
 	return nil
 }
 
-// NewGetMmdsBadRequest creates a GetMmdsBadRequest with default headers values
-func NewGetMmdsBadRequest() *GetMmdsBadRequest {
-	return &GetMmdsBadRequest{}
+// NewGetMmdsNotFound creates a GetMmdsNotFound with default headers values
+func NewGetMmdsNotFound() *GetMmdsNotFound {
+	return &GetMmdsNotFound{}
 }
 
-/*GetMmdsBadRequest handles this case with default header values.
+/*GetMmdsNotFound handles this case with default header values.
 
-Cannot get the MMDS data store due to bad input.
+The MMDS data store content can not be found.
 */
-type GetMmdsBadRequest struct {
+type GetMmdsNotFound struct {
 	Payload *models.Error
 }
 
-func (o *GetMmdsBadRequest) Error() string {
-	return fmt.Sprintf("[GET /mmds][%d] getMmdsBadRequest  %+v", 400, o.Payload)
+func (o *GetMmdsNotFound) Error() string {
+	return fmt.Sprintf("[GET /mmds][%d] getMmdsNotFound  %+v", 404, o.Payload)
 }
 
-func (o *GetMmdsBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+func (o *GetMmdsNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
 
 	o.Payload = new(models.Error)
 

src/runtime/virtcontainers/pkg/firecracker/client/operations/load_snapshot_parameters.go

@@ -0,0 +1,139 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// NewLoadSnapshotParams creates a new LoadSnapshotParams object
+// with the default values initialized.
+func NewLoadSnapshotParams() *LoadSnapshotParams {
+	var ()
+	return &LoadSnapshotParams{
+
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewLoadSnapshotParamsWithTimeout creates a new LoadSnapshotParams object
+// with the default values initialized, and the ability to set a timeout on a request
+func NewLoadSnapshotParamsWithTimeout(timeout time.Duration) *LoadSnapshotParams {
+	var ()
+	return &LoadSnapshotParams{
+
+		timeout: timeout,
+	}
+}
+
+// NewLoadSnapshotParamsWithContext creates a new LoadSnapshotParams object
+// with the default values initialized, and the ability to set a context for a request
+func NewLoadSnapshotParamsWithContext(ctx context.Context) *LoadSnapshotParams {
+	var ()
+	return &LoadSnapshotParams{
+
+		Context: ctx,
+	}
+}
+
+// NewLoadSnapshotParamsWithHTTPClient creates a new LoadSnapshotParams object
+// with the default values initialized, and the ability to set a custom HTTPClient for a request
+func NewLoadSnapshotParamsWithHTTPClient(client *http.Client) *LoadSnapshotParams {
+	var ()
+	return &LoadSnapshotParams{
+		HTTPClient: client,
+	}
+}
+
+/*LoadSnapshotParams contains all the parameters to send to the API endpoint
+for the load snapshot operation typically these are written to a http.Request
+*/
+type LoadSnapshotParams struct {
+
+	/*Body
+	  The configuration used for loading a snaphot.
+
+	*/
+	Body *models.SnapshotLoadParams
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithTimeout adds the timeout to the load snapshot params
+func (o *LoadSnapshotParams) WithTimeout(timeout time.Duration) *LoadSnapshotParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the load snapshot params
+func (o *LoadSnapshotParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the load snapshot params
+func (o *LoadSnapshotParams) WithContext(ctx context.Context) *LoadSnapshotParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the load snapshot params
+func (o *LoadSnapshotParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the load snapshot params
+func (o *LoadSnapshotParams) WithHTTPClient(client *http.Client) *LoadSnapshotParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the load snapshot params
+func (o *LoadSnapshotParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithBody adds the body to the load snapshot params
+func (o *LoadSnapshotParams) WithBody(body *models.SnapshotLoadParams) *LoadSnapshotParams {
+	o.SetBody(body)
+	return o
+}
+
+// SetBody adds the body to the load snapshot params
+func (o *LoadSnapshotParams) SetBody(body *models.SnapshotLoadParams) {
+	o.Body = body
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *LoadSnapshotParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+
+	if o.Body != nil {
+		if err := r.SetBodyParam(o.Body); err != nil {
+			return err
+		}
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/load_snapshot_responses.go

@@ -0,0 +1,140 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// LoadSnapshotReader is a Reader for the LoadSnapshot structure.
+type LoadSnapshotReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *LoadSnapshotReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	switch response.Code() {
+
+	case 204:
+		result := NewLoadSnapshotNoContent()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return result, nil
+
+	case 400:
+		result := NewLoadSnapshotBadRequest()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return nil, result
+
+	default:
+		result := NewLoadSnapshotDefault(response.Code())
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		if response.Code()/100 == 2 {
+			return result, nil
+		}
+		return nil, result
+	}
+}
+
+// NewLoadSnapshotNoContent creates a LoadSnapshotNoContent with default headers values
+func NewLoadSnapshotNoContent() *LoadSnapshotNoContent {
+	return &LoadSnapshotNoContent{}
+}
+
+/*LoadSnapshotNoContent handles this case with default header values.
+
+Snapshot loaded
+*/
+type LoadSnapshotNoContent struct {
+}
+
+func (o *LoadSnapshotNoContent) Error() string {
+	return fmt.Sprintf("[PUT /snapshot/load][%d] loadSnapshotNoContent ", 204)
+}
+
+func (o *LoadSnapshotNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	return nil
+}
+
+// NewLoadSnapshotBadRequest creates a LoadSnapshotBadRequest with default headers values
+func NewLoadSnapshotBadRequest() *LoadSnapshotBadRequest {
+	return &LoadSnapshotBadRequest{}
+}
+
+/*LoadSnapshotBadRequest handles this case with default header values.
+
+Snapshot cannot be loaded due to bad input
+*/
+type LoadSnapshotBadRequest struct {
+	Payload *models.Error
+}
+
+func (o *LoadSnapshotBadRequest) Error() string {
+	return fmt.Sprintf("[PUT /snapshot/load][%d] loadSnapshotBadRequest  %+v", 400, o.Payload)
+}
+
+func (o *LoadSnapshotBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+// NewLoadSnapshotDefault creates a LoadSnapshotDefault with default headers values
+func NewLoadSnapshotDefault(code int) *LoadSnapshotDefault {
+	return &LoadSnapshotDefault{
+		_statusCode: code,
+	}
+}
+
+/*LoadSnapshotDefault handles this case with default header values.
+
+Internal server error
+*/
+type LoadSnapshotDefault struct {
+	_statusCode int
+
+	Payload *models.Error
+}
+
+// Code gets the status code for the load snapshot default response
+func (o *LoadSnapshotDefault) Code() int {
+	return o._statusCode
+}
+
+func (o *LoadSnapshotDefault) Error() string {
+	return fmt.Sprintf("[PUT /snapshot/load][%d] loadSnapshot default  %+v", o._statusCode, o.Payload)
+}
+
+func (o *LoadSnapshotDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/operations_client.go

@@ -108,6 +108,66 @@ func (a *Client) PutMmds(params *PutMmdsParams) (*PutMmdsNoContent, error) {
 
 }
 
+/*
+PutMmdsConfig sets m m d s configuration pre boot only
+
+Creates MMDS configuration to be used by the MMDS network stack.
+*/
+func (a *Client) PutMmdsConfig(params *PutMmdsConfigParams) (*PutMmdsConfigNoContent, error) {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewPutMmdsConfigParams()
+	}
+
+	result, err := a.transport.Submit(&runtime.ClientOperation{
+		ID:                 "PutMmdsConfig",
+		Method:             "PUT",
+		PathPattern:        "/mmds/config",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &PutMmdsConfigReader{formats: a.formats},
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(*PutMmdsConfigNoContent), nil
+
+}
+
+/*
+CreateSnapshot creates a full snapshot post boot only
+
+Creates a snapshot of the microVM state. The microVM should be in the `Paused` state.
+*/
+func (a *Client) CreateSnapshot(params *CreateSnapshotParams) (*CreateSnapshotNoContent, error) {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewCreateSnapshotParams()
+	}
+
+	result, err := a.transport.Submit(&runtime.ClientOperation{
+		ID:                 "createSnapshot",
+		Method:             "PUT",
+		PathPattern:        "/snapshot/create",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &CreateSnapshotReader{formats: a.formats},
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(*CreateSnapshotNoContent), nil
+
+}
+
 /*
 CreateSyncAction creates a synchronous action
 */
@@ -195,7 +255,37 @@ func (a *Client) GetMachineConfiguration(params *GetMachineConfigurationParams)
 }
 
 /*
-PatchGuestDriveByID updates the properties of a drive
+LoadSnapshot loads a snapshot pre boot only
+
+Loads the microVM state from a snapshot. Only accepted on a fresh Firecracker process (before configuring any resource other than the Logger and Metrics).
+*/
+func (a *Client) LoadSnapshot(params *LoadSnapshotParams) (*LoadSnapshotNoContent, error) {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewLoadSnapshotParams()
+	}
+
+	result, err := a.transport.Submit(&runtime.ClientOperation{
+		ID:                 "loadSnapshot",
+		Method:             "PUT",
+		PathPattern:        "/snapshot/load",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &LoadSnapshotReader{formats: a.formats},
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(*LoadSnapshotNoContent), nil
+
+}
+
+/*
+PatchGuestDriveByID updates the properties of a drive post boot only
 
 Updates the properties of the drive with the ID specified by drive_id path parameter. Will fail if update is not possible.
 */
@@ -225,7 +315,7 @@ func (a *Client) PatchGuestDriveByID(params *PatchGuestDriveByIDParams) (*PatchG
 }
 
 /*
-PatchGuestNetworkInterfaceByID updates the rate limiters applied to a network interface
+PatchGuestNetworkInterfaceByID updates the rate limiters applied to a network interface post boot only
 
 Updates the rate limiters applied to a network interface.
 */
@@ -255,7 +345,7 @@ func (a *Client) PatchGuestNetworkInterfaceByID(params *PatchGuestNetworkInterfa
 }
 
 /*
-PatchMachineConfiguration partiallies updates the machine configuration of the VM
+PatchMachineConfiguration partiallies updates the machine configuration of the VM pre boot only
 
 Partially updates the Virtual Machine Configuration with the specified input. If any of the parameters has an incorrect value, the whole update fails.
 */
@@ -285,9 +375,39 @@ func (a *Client) PatchMachineConfiguration(params *PatchMachineConfigurationPara
 }
 
 /*
-PutGuestBootSource creates or updates the boot source
+PatchVM updates the micro VM state
 
-Creates new boot source if one does not already exist, otherwise updates it. Will fail if update is not possible. Note that the only currently supported boot source is LocalImage.
+Sets the desired state (Paused or Resumed) for the microVM.
+*/
+func (a *Client) PatchVM(params *PatchVMParams) (*PatchVMNoContent, error) {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewPatchVMParams()
+	}
+
+	result, err := a.transport.Submit(&runtime.ClientOperation{
+		ID:                 "patchVm",
+		Method:             "PATCH",
+		PathPattern:        "/vm",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &PatchVMReader{formats: a.formats},
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(*PatchVMNoContent), nil
+
+}
+
+/*
+PutGuestBootSource creates or updates the boot source pre boot only
+
+Creates new boot source if one does not already exist, otherwise updates it. Will fail if update is not possible.
 */
 func (a *Client) PutGuestBootSource(params *PutGuestBootSourceParams) (*PutGuestBootSourceNoContent, error) {
 	// TODO: Validate the params before sending
@@ -315,7 +435,7 @@ func (a *Client) PutGuestBootSource(params *PutGuestBootSourceParams) (*PutGuest
 }
 
 /*
-PutGuestDriveByID creates or updates a drive
+PutGuestDriveByID creates or updates a drive pre boot only
 
 Creates new drive with ID specified by drive_id path parameter. If a drive with the specified ID already exists, updates its state based on new input. Will fail if update is not possible.
 */
@@ -345,7 +465,7 @@ func (a *Client) PutGuestDriveByID(params *PutGuestDriveByIDParams) (*PutGuestDr
 }
 
 /*
-PutGuestNetworkInterfaceByID creates a network interface
+PutGuestNetworkInterfaceByID creates a network interface pre boot only
 
 Creates new network interface with ID specified by iface_id path parameter.
 */
@@ -375,7 +495,7 @@ func (a *Client) PutGuestNetworkInterfaceByID(params *PutGuestNetworkInterfaceBy
 }
 
 /*
-PutGuestVsock creates updates a vsock device
+PutGuestVsock creates updates a vsock device pre boot only
 
 The first call creates the device with the configuration specified in body. Subsequent calls will update the device configuration. May fail if update is not possible.
 */
@@ -405,7 +525,7 @@ func (a *Client) PutGuestVsock(params *PutGuestVsockParams) (*PutGuestVsockNoCon
 }
 
 /*
-PutLogger initializes the logger by specifying two named pipes i e for the logs and metrics output
+PutLogger initializes the logger by specifying a named pipe or a file for the logs output
 */
 func (a *Client) PutLogger(params *PutLoggerParams) (*PutLoggerNoContent, error) {
 	// TODO: Validate the params before sending
@@ -433,7 +553,7 @@ func (a *Client) PutLogger(params *PutLoggerParams) (*PutLoggerNoContent, error)
 }
 
 /*
-PutMachineConfiguration updates the machine configuration of the VM
+PutMachineConfiguration updates the machine configuration of the VM pre boot only
 
 Updates the Virtual Machine Configuration with the specified input. Firecracker starts with default values for vCPU count (=1) and memory size (=128 MiB). With Hyperthreading enabled, the vCPU count is restricted to be 1 or an even number, otherwise there are no restrictions regarding the vCPU count. If any of the parameters has an incorrect value, the whole update fails.
 */
@@ -462,6 +582,34 @@ func (a *Client) PutMachineConfiguration(params *PutMachineConfigurationParams)
 
 }
 
+/*
+PutMetrics initializes the metrics system by specifying a named pipe or a file for the metrics output
+*/
+func (a *Client) PutMetrics(params *PutMetricsParams) (*PutMetricsNoContent, error) {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewPutMetricsParams()
+	}
+
+	result, err := a.transport.Submit(&runtime.ClientOperation{
+		ID:                 "putMetrics",
+		Method:             "PUT",
+		PathPattern:        "/metrics",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &PutMetricsReader{formats: a.formats},
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(*PutMetricsNoContent), nil
+
+}
+
 // SetTransport changes the transport on the client
 func (a *Client) SetTransport(transport runtime.ClientTransport) {
 	a.transport = transport

src/runtime/virtcontainers/pkg/firecracker/client/operations/patch_vm_parameters.go

@@ -0,0 +1,139 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// NewPatchVMParams creates a new PatchVMParams object
+// with the default values initialized.
+func NewPatchVMParams() *PatchVMParams {
+	var ()
+	return &PatchVMParams{
+
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewPatchVMParamsWithTimeout creates a new PatchVMParams object
+// with the default values initialized, and the ability to set a timeout on a request
+func NewPatchVMParamsWithTimeout(timeout time.Duration) *PatchVMParams {
+	var ()
+	return &PatchVMParams{
+
+		timeout: timeout,
+	}
+}
+
+// NewPatchVMParamsWithContext creates a new PatchVMParams object
+// with the default values initialized, and the ability to set a context for a request
+func NewPatchVMParamsWithContext(ctx context.Context) *PatchVMParams {
+	var ()
+	return &PatchVMParams{
+
+		Context: ctx,
+	}
+}
+
+// NewPatchVMParamsWithHTTPClient creates a new PatchVMParams object
+// with the default values initialized, and the ability to set a custom HTTPClient for a request
+func NewPatchVMParamsWithHTTPClient(client *http.Client) *PatchVMParams {
+	var ()
+	return &PatchVMParams{
+		HTTPClient: client,
+	}
+}
+
+/*PatchVMParams contains all the parameters to send to the API endpoint
+for the patch Vm operation typically these are written to a http.Request
+*/
+type PatchVMParams struct {
+
+	/*Body
+	  The microVM state
+
+	*/
+	Body *models.VM
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithTimeout adds the timeout to the patch Vm params
+func (o *PatchVMParams) WithTimeout(timeout time.Duration) *PatchVMParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the patch Vm params
+func (o *PatchVMParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the patch Vm params
+func (o *PatchVMParams) WithContext(ctx context.Context) *PatchVMParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the patch Vm params
+func (o *PatchVMParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the patch Vm params
+func (o *PatchVMParams) WithHTTPClient(client *http.Client) *PatchVMParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the patch Vm params
+func (o *PatchVMParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithBody adds the body to the patch Vm params
+func (o *PatchVMParams) WithBody(body *models.VM) *PatchVMParams {
+	o.SetBody(body)
+	return o
+}
+
+// SetBody adds the body to the patch Vm params
+func (o *PatchVMParams) SetBody(body *models.VM) {
+	o.Body = body
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *PatchVMParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+
+	if o.Body != nil {
+		if err := r.SetBodyParam(o.Body); err != nil {
+			return err
+		}
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/patch_vm_responses.go

@@ -0,0 +1,140 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// PatchVMReader is a Reader for the PatchVM structure.
+type PatchVMReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *PatchVMReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	switch response.Code() {
+
+	case 204:
+		result := NewPatchVMNoContent()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return result, nil
+
+	case 400:
+		result := NewPatchVMBadRequest()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return nil, result
+
+	default:
+		result := NewPatchVMDefault(response.Code())
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		if response.Code()/100 == 2 {
+			return result, nil
+		}
+		return nil, result
+	}
+}
+
+// NewPatchVMNoContent creates a PatchVMNoContent with default headers values
+func NewPatchVMNoContent() *PatchVMNoContent {
+	return &PatchVMNoContent{}
+}
+
+/*PatchVMNoContent handles this case with default header values.
+
+Vm state updated
+*/
+type PatchVMNoContent struct {
+}
+
+func (o *PatchVMNoContent) Error() string {
+	return fmt.Sprintf("[PATCH /vm][%d] patchVmNoContent ", 204)
+}
+
+func (o *PatchVMNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	return nil
+}
+
+// NewPatchVMBadRequest creates a PatchVMBadRequest with default headers values
+func NewPatchVMBadRequest() *PatchVMBadRequest {
+	return &PatchVMBadRequest{}
+}
+
+/*PatchVMBadRequest handles this case with default header values.
+
+Vm state cannot be updated due to bad input
+*/
+type PatchVMBadRequest struct {
+	Payload *models.Error
+}
+
+func (o *PatchVMBadRequest) Error() string {
+	return fmt.Sprintf("[PATCH /vm][%d] patchVmBadRequest  %+v", 400, o.Payload)
+}
+
+func (o *PatchVMBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+// NewPatchVMDefault creates a PatchVMDefault with default headers values
+func NewPatchVMDefault(code int) *PatchVMDefault {
+	return &PatchVMDefault{
+		_statusCode: code,
+	}
+}
+
+/*PatchVMDefault handles this case with default header values.
+
+Internal server error
+*/
+type PatchVMDefault struct {
+	_statusCode int
+
+	Payload *models.Error
+}
+
+// Code gets the status code for the patch Vm default response
+func (o *PatchVMDefault) Code() int {
+	return o._statusCode
+}
+
+func (o *PatchVMDefault) Error() string {
+	return fmt.Sprintf("[PATCH /vm][%d] patchVm default  %+v", o._statusCode, o.Payload)
+}
+
+func (o *PatchVMDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/put_metrics_parameters.go

@@ -0,0 +1,139 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// NewPutMetricsParams creates a new PutMetricsParams object
+// with the default values initialized.
+func NewPutMetricsParams() *PutMetricsParams {
+	var ()
+	return &PutMetricsParams{
+
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewPutMetricsParamsWithTimeout creates a new PutMetricsParams object
+// with the default values initialized, and the ability to set a timeout on a request
+func NewPutMetricsParamsWithTimeout(timeout time.Duration) *PutMetricsParams {
+	var ()
+	return &PutMetricsParams{
+
+		timeout: timeout,
+	}
+}
+
+// NewPutMetricsParamsWithContext creates a new PutMetricsParams object
+// with the default values initialized, and the ability to set a context for a request
+func NewPutMetricsParamsWithContext(ctx context.Context) *PutMetricsParams {
+	var ()
+	return &PutMetricsParams{
+
+		Context: ctx,
+	}
+}
+
+// NewPutMetricsParamsWithHTTPClient creates a new PutMetricsParams object
+// with the default values initialized, and the ability to set a custom HTTPClient for a request
+func NewPutMetricsParamsWithHTTPClient(client *http.Client) *PutMetricsParams {
+	var ()
+	return &PutMetricsParams{
+		HTTPClient: client,
+	}
+}
+
+/*PutMetricsParams contains all the parameters to send to the API endpoint
+for the put metrics operation typically these are written to a http.Request
+*/
+type PutMetricsParams struct {
+
+	/*Body
+	  Metrics system description
+
+	*/
+	Body *models.Metrics
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithTimeout adds the timeout to the put metrics params
+func (o *PutMetricsParams) WithTimeout(timeout time.Duration) *PutMetricsParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the put metrics params
+func (o *PutMetricsParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the put metrics params
+func (o *PutMetricsParams) WithContext(ctx context.Context) *PutMetricsParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the put metrics params
+func (o *PutMetricsParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the put metrics params
+func (o *PutMetricsParams) WithHTTPClient(client *http.Client) *PutMetricsParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the put metrics params
+func (o *PutMetricsParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithBody adds the body to the put metrics params
+func (o *PutMetricsParams) WithBody(body *models.Metrics) *PutMetricsParams {
+	o.SetBody(body)
+	return o
+}
+
+// SetBody adds the body to the put metrics params
+func (o *PutMetricsParams) SetBody(body *models.Metrics) {
+	o.Body = body
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *PutMetricsParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+
+	if o.Body != nil {
+		if err := r.SetBodyParam(o.Body); err != nil {
+			return err
+		}
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/put_metrics_responses.go

@@ -0,0 +1,140 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// PutMetricsReader is a Reader for the PutMetrics structure.
+type PutMetricsReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *PutMetricsReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	switch response.Code() {
+
+	case 204:
+		result := NewPutMetricsNoContent()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return result, nil
+
+	case 400:
+		result := NewPutMetricsBadRequest()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return nil, result
+
+	default:
+		result := NewPutMetricsDefault(response.Code())
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		if response.Code()/100 == 2 {
+			return result, nil
+		}
+		return nil, result
+	}
+}
+
+// NewPutMetricsNoContent creates a PutMetricsNoContent with default headers values
+func NewPutMetricsNoContent() *PutMetricsNoContent {
+	return &PutMetricsNoContent{}
+}
+
+/*PutMetricsNoContent handles this case with default header values.
+
+Metrics system created.
+*/
+type PutMetricsNoContent struct {
+}
+
+func (o *PutMetricsNoContent) Error() string {
+	return fmt.Sprintf("[PUT /metrics][%d] putMetricsNoContent ", 204)
+}
+
+func (o *PutMetricsNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	return nil
+}
+
+// NewPutMetricsBadRequest creates a PutMetricsBadRequest with default headers values
+func NewPutMetricsBadRequest() *PutMetricsBadRequest {
+	return &PutMetricsBadRequest{}
+}
+
+/*PutMetricsBadRequest handles this case with default header values.
+
+Metrics system cannot be initialized due to bad input.
+*/
+type PutMetricsBadRequest struct {
+	Payload *models.Error
+}
+
+func (o *PutMetricsBadRequest) Error() string {
+	return fmt.Sprintf("[PUT /metrics][%d] putMetricsBadRequest  %+v", 400, o.Payload)
+}
+
+func (o *PutMetricsBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+// NewPutMetricsDefault creates a PutMetricsDefault with default headers values
+func NewPutMetricsDefault(code int) *PutMetricsDefault {
+	return &PutMetricsDefault{
+		_statusCode: code,
+	}
+}
+
+/*PutMetricsDefault handles this case with default header values.
+
+Internal server error.
+*/
+type PutMetricsDefault struct {
+	_statusCode int
+
+	Payload *models.Error
+}
+
+// Code gets the status code for the put metrics default response
+func (o *PutMetricsDefault) Code() int {
+	return o._statusCode
+}
+
+func (o *PutMetricsDefault) Error() string {
+	return fmt.Sprintf("[PUT /metrics][%d] putMetrics default  %+v", o._statusCode, o.Payload)
+}
+
+func (o *PutMetricsDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/put_mmds_config_parameters.go

@@ -0,0 +1,139 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// NewPutMmdsConfigParams creates a new PutMmdsConfigParams object
+// with the default values initialized.
+func NewPutMmdsConfigParams() *PutMmdsConfigParams {
+	var ()
+	return &PutMmdsConfigParams{
+
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewPutMmdsConfigParamsWithTimeout creates a new PutMmdsConfigParams object
+// with the default values initialized, and the ability to set a timeout on a request
+func NewPutMmdsConfigParamsWithTimeout(timeout time.Duration) *PutMmdsConfigParams {
+	var ()
+	return &PutMmdsConfigParams{
+
+		timeout: timeout,
+	}
+}
+
+// NewPutMmdsConfigParamsWithContext creates a new PutMmdsConfigParams object
+// with the default values initialized, and the ability to set a context for a request
+func NewPutMmdsConfigParamsWithContext(ctx context.Context) *PutMmdsConfigParams {
+	var ()
+	return &PutMmdsConfigParams{
+
+		Context: ctx,
+	}
+}
+
+// NewPutMmdsConfigParamsWithHTTPClient creates a new PutMmdsConfigParams object
+// with the default values initialized, and the ability to set a custom HTTPClient for a request
+func NewPutMmdsConfigParamsWithHTTPClient(client *http.Client) *PutMmdsConfigParams {
+	var ()
+	return &PutMmdsConfigParams{
+		HTTPClient: client,
+	}
+}
+
+/*PutMmdsConfigParams contains all the parameters to send to the API endpoint
+for the put mmds config operation typically these are written to a http.Request
+*/
+type PutMmdsConfigParams struct {
+
+	/*Body
+	  The MMDS configuration as JSON.
+
+	*/
+	Body *models.MmdsConfig
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithTimeout adds the timeout to the put mmds config params
+func (o *PutMmdsConfigParams) WithTimeout(timeout time.Duration) *PutMmdsConfigParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the put mmds config params
+func (o *PutMmdsConfigParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the put mmds config params
+func (o *PutMmdsConfigParams) WithContext(ctx context.Context) *PutMmdsConfigParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the put mmds config params
+func (o *PutMmdsConfigParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the put mmds config params
+func (o *PutMmdsConfigParams) WithHTTPClient(client *http.Client) *PutMmdsConfigParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the put mmds config params
+func (o *PutMmdsConfigParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithBody adds the body to the put mmds config params
+func (o *PutMmdsConfigParams) WithBody(body *models.MmdsConfig) *PutMmdsConfigParams {
+	o.SetBody(body)
+	return o
+}
+
+// SetBody adds the body to the put mmds config params
+func (o *PutMmdsConfigParams) SetBody(body *models.MmdsConfig) {
+	o.Body = body
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *PutMmdsConfigParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+
+	if o.Body != nil {
+		if err := r.SetBodyParam(o.Body); err != nil {
+			return err
+		}
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/client/operations/put_mmds_config_responses.go

@@ -0,0 +1,140 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package operations
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
+)
+
+// PutMmdsConfigReader is a Reader for the PutMmdsConfig structure.
+type PutMmdsConfigReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *PutMmdsConfigReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	switch response.Code() {
+
+	case 204:
+		result := NewPutMmdsConfigNoContent()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return result, nil
+
+	case 400:
+		result := NewPutMmdsConfigBadRequest()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return nil, result
+
+	default:
+		result := NewPutMmdsConfigDefault(response.Code())
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		if response.Code()/100 == 2 {
+			return result, nil
+		}
+		return nil, result
+	}
+}
+
+// NewPutMmdsConfigNoContent creates a PutMmdsConfigNoContent with default headers values
+func NewPutMmdsConfigNoContent() *PutMmdsConfigNoContent {
+	return &PutMmdsConfigNoContent{}
+}
+
+/*PutMmdsConfigNoContent handles this case with default header values.
+
+MMDS configuration was created/updated.
+*/
+type PutMmdsConfigNoContent struct {
+}
+
+func (o *PutMmdsConfigNoContent) Error() string {
+	return fmt.Sprintf("[PUT /mmds/config][%d] putMmdsConfigNoContent ", 204)
+}
+
+func (o *PutMmdsConfigNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	return nil
+}
+
+// NewPutMmdsConfigBadRequest creates a PutMmdsConfigBadRequest with default headers values
+func NewPutMmdsConfigBadRequest() *PutMmdsConfigBadRequest {
+	return &PutMmdsConfigBadRequest{}
+}
+
+/*PutMmdsConfigBadRequest handles this case with default header values.
+
+MMDS configuration cannot be updated due to bad input.
+*/
+type PutMmdsConfigBadRequest struct {
+	Payload *models.Error
+}
+
+func (o *PutMmdsConfigBadRequest) Error() string {
+	return fmt.Sprintf("[PUT /mmds/config][%d] putMmdsConfigBadRequest  %+v", 400, o.Payload)
+}
+
+func (o *PutMmdsConfigBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+// NewPutMmdsConfigDefault creates a PutMmdsConfigDefault with default headers values
+func NewPutMmdsConfigDefault(code int) *PutMmdsConfigDefault {
+	return &PutMmdsConfigDefault{
+		_statusCode: code,
+	}
+}
+
+/*PutMmdsConfigDefault handles this case with default header values.
+
+Internal server error
+*/
+type PutMmdsConfigDefault struct {
+	_statusCode int
+
+	Payload *models.Error
+}
+
+// Code gets the status code for the put mmds config default response
+func (o *PutMmdsConfigDefault) Code() int {
+	return o._statusCode
+}
+
+func (o *PutMmdsConfigDefault) Error() string {
+	return fmt.Sprintf("[PUT /mmds/config][%d] PutMmdsConfig default  %+v", o._statusCode, o.Payload)
+}
+
+func (o *PutMmdsConfigDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	o.Payload = new(models.Error)
+
+	// response payload
+	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

src/runtime/virtcontainers/pkg/firecracker/firecracker.yaml

@@ -2,10 +2,10 @@ swagger: "2.0"
 info:
   title: Firecracker API
   description: RESTful public-facing API.
-               The API is accessible through HTTP calls on specific URLs
-               carrying JSON modeled data.
-               The transport medium is a Unix Domain Socket.
-  version: 0.21.1
+    The API is accessible through HTTP calls on specific URLs
+    carrying JSON modeled data.
+    The transport medium is a Unix Domain Socket.
+  version: 0.23.1
   termsOfService: ""
   contact:
     email: "compute-capsule@amazon.com"
@@ -43,11 +43,11 @@ paths:
       summary: Creates a synchronous action.
       operationId: createSyncAction
       parameters:
-      - name: info
-        in: body
-        required: true
-        schema:
-          $ref: "#/definitions/InstanceActionInfo"
+        - name: info
+          in: body
+          required: true
+          schema:
+            $ref: "#/definitions/InstanceActionInfo"
       responses:
         204:
           description: The update was successful
@@ -62,19 +62,18 @@ paths:
 
   /boot-source:
     put:
-      summary: Creates or updates the boot source.
+      summary: Creates or updates the boot source. Pre-boot only.
       description:
         Creates new boot source if one does not already exist, otherwise updates it.
         Will fail if update is not possible.
-        Note that the only currently supported boot source is LocalImage.
       operationId: putGuestBootSource
       parameters:
-      - name: body
-        in: body
-        description: Guest boot source properties
-        required: true
-        schema:
-          $ref: "#/definitions/BootSource"
+        - name: body
+          in: body
+          description: Guest boot source properties
+          required: true
+          schema:
+            $ref: "#/definitions/BootSource"
       responses:
         204:
           description: Boot source created/updated
@@ -89,24 +88,24 @@ paths:
 
   /drives/{drive_id}:
     put:
-      summary: Creates or updates a drive.
+      summary: Creates or updates a drive. Pre-boot only.
       description:
         Creates new drive with ID specified by drive_id path parameter.
         If a drive with the specified ID already exists, updates its state based on new input.
         Will fail if update is not possible.
       operationId: putGuestDriveByID
       parameters:
-      - name: drive_id
-        in: path
-        description: The id of the guest drive
-        required: true
-        type: string
-      - name: body
-        in: body
-        description: Guest drive properties
-        required: true
-        schema:
-          $ref: "#/definitions/Drive"
+        - name: drive_id
+          in: path
+          description: The id of the guest drive
+          required: true
+          type: string
+        - name: body
+          in: body
+          description: Guest drive properties
+          required: true
+          schema:
+            $ref: "#/definitions/Drive"
       responses:
         204:
           description: Drive created/updated
@@ -119,23 +118,23 @@ paths:
           schema:
             $ref: "#/definitions/Error"
     patch:
-      summary: Updates the properties of a drive.
+      summary: Updates the properties of a drive. Post-boot only.
       description:
         Updates the properties of the drive with the ID specified by drive_id path parameter.
         Will fail if update is not possible.
       operationId: patchGuestDriveByID
       parameters:
-      - name: drive_id
-        in: path
-        description: The id of the guest drive
-        required: true
-        type: string
-      - name: body
-        in: body
-        description: Guest drive properties
-        required: true
-        schema:
-          $ref: "#/definitions/PartialDrive"
+        - name: drive_id
+          in: path
+          description: The id of the guest drive
+          required: true
+          type: string
+        - name: body
+          in: body
+          description: Guest drive properties
+          required: true
+          schema:
+            $ref: "#/definitions/PartialDrive"
       responses:
         204:
           description: Drive updated
@@ -149,27 +148,27 @@ paths:
             $ref: "#/definitions/Error"
 
   /logger:
-      put:
-        summary: Initializes the logger by specifying two named pipes (i.e. for the logs and metrics output).
-        operationId: putLogger
-        parameters:
+    put:
+      summary: Initializes the logger by specifying a named pipe or a file for the logs output.
+      operationId: putLogger
+      parameters:
         - name: body
           in: body
           description: Logging system description
           required: true
           schema:
             $ref: "#/definitions/Logger"
-        responses:
-          204:
-            description: Logger created.
-          400:
-            description: Logger cannot be initialized due to bad input.
-            schema:
-              $ref: "#/definitions/Error"
-          default:
-            description: Internal server error.
-            schema:
-              $ref: "#/definitions/Error"
+      responses:
+        204:
+          description: Logger created.
+        400:
+          description: Logger cannot be initialized due to bad input.
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error.
+          schema:
+            $ref: "#/definitions/Error"
 
   /machine-config:
     get:
@@ -190,7 +189,7 @@ paths:
             $ref: "#/definitions/Error"
 
     put:
-      summary: Updates the Machine Configuration of the VM.
+      summary: Updates the Machine Configuration of the VM. Pre-boot only.
       description:
         Updates the Virtual Machine Configuration with the specified input.
         Firecracker starts with default values for vCPU count (=1) and memory size (=128 MiB).
@@ -199,11 +198,11 @@ paths:
         If any of the parameters has an incorrect value, the whole update fails.
       operationId: putMachineConfiguration
       parameters:
-      - name: body
-        in: body
-        description: Machine Configuration Parameters
-        schema:
-          $ref: "#/definitions/MachineConfiguration"
+        - name: body
+          in: body
+          description: Machine Configuration Parameters
+          schema:
+            $ref: "#/definitions/MachineConfiguration"
       responses:
         204:
           description: Machine Configuration created/updated
@@ -217,17 +216,17 @@ paths:
             $ref: "#/definitions/Error"
 
     patch:
-      summary: Partially updates the Machine Configuration of the VM.
+      summary: Partially updates the Machine Configuration of the VM. Pre-boot only.
       description:
         Partially updates the Virtual Machine Configuration with the specified input.
         If any of the parameters has an incorrect value, the whole update fails.
       operationId: patchMachineConfiguration
       parameters:
-      - name: body
-        in: body
-        description: A subset of Machine Configuration Parameters
-        schema:
-          $ref: "#/definitions/MachineConfiguration"
+        - name: body
+          in: body
+          description: A subset of Machine Configuration Parameters
+          schema:
+            $ref: "#/definitions/MachineConfiguration"
       responses:
         204:
           description: Machine Configuration created/updated
@@ -240,6 +239,29 @@ paths:
           schema:
             $ref: "#/definitions/Error"
 
+  /metrics:
+    put:
+      summary: Initializes the metrics system by specifying a named pipe or a file for the metrics output.
+      operationId: putMetrics
+      parameters:
+        - name: body
+          in: body
+          description: Metrics system description
+          required: true
+          schema:
+            $ref: "#/definitions/Metrics"
+      responses:
+        204:
+          description: Metrics system created.
+        400:
+          description: Metrics system cannot be initialized due to bad input.
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error.
+          schema:
+            $ref: "#/definitions/Error"
+
   /mmds:
     put:
       summary: Creates a MMDS (Microvm Metadata Service) data store.
@@ -286,8 +308,32 @@ paths:
           description: The MMDS data store JSON.
           schema:
             type: object
+        404:
+          description: The MMDS data store content can not be found.
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
+
+  /mmds/config:
+    put:
+      summary: Set MMDS configuration. Pre-boot only.
+      description:
+        Creates MMDS configuration to be used by the MMDS network stack.
+      parameters:
+        - name: body
+          in: body
+          description: The MMDS configuration as JSON.
+          required: true
+          schema:
+            $ref: "#/definitions/MmdsConfig"
+      responses:
+        204:
+          description: MMDS configuration was created/updated.
         400:
-          description: Cannot get the MMDS data store due to bad input.
+          description: MMDS configuration cannot be updated due to bad input.
           schema:
             $ref: "#/definitions/Error"
         default:
@@ -297,22 +343,22 @@ paths:
 
   /network-interfaces/{iface_id}:
     put:
-      summary: Creates a network interface.
+      summary: Creates a network interface. Pre-boot only.
       description:
         Creates new network interface with ID specified by iface_id path parameter.
       operationId: putGuestNetworkInterfaceByID
       parameters:
-      - name: iface_id
-        in: path
-        description: The id of the guest network interface
-        required: true
-        type: string
-      - name: body
-        in: body
-        description: Guest network interface properties
-        required: true
-        schema:
-          $ref: "#/definitions/NetworkInterface"
+        - name: iface_id
+          in: path
+          description: The id of the guest network interface
+          required: true
+          type: string
+        - name: body
+          in: body
+          description: Guest network interface properties
+          required: true
+          schema:
+            $ref: "#/definitions/NetworkInterface"
       responses:
         204:
           description: Network interface created/updated
@@ -325,7 +371,7 @@ paths:
           schema:
             $ref: "#/definitions/Error"
     patch:
-      summary: Updates the rate limiters applied to a network interface.
+      summary: Updates the rate limiters applied to a network interface. Post-boot only.
       description:
         Updates the rate limiters applied to a network interface.
       operationId: patchGuestNetworkInterfaceByID
@@ -353,21 +399,99 @@ paths:
           schema:
             $ref: "#/definitions/Error"
 
+  /snapshot/create:
+    put:
+      summary: Creates a full snapshot. Post-boot only.
+      description:
+        Creates a snapshot of the microVM state. The microVM should be
+        in the `Paused` state.
+      operationId: createSnapshot
+      parameters:
+        - name: body
+          in: body
+          description: The configuration used for creating a snaphot.
+          required: true
+          schema:
+            $ref: "#/definitions/SnapshotCreateParams"
+      responses:
+        204:
+          description: Snapshot created
+        400:
+          description: Snapshot cannot be created due to bad input
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
+
+  /snapshot/load:
+    put:
+      summary: Loads a snapshot. Pre-boot only.
+      description:
+        Loads the microVM state from a snapshot.
+        Only accepted on a fresh Firecracker process (before configuring
+        any resource other than the Logger and Metrics).
+      operationId: loadSnapshot
+      parameters:
+        - name: body
+          in: body
+          description: The configuration used for loading a snaphot.
+          required: true
+          schema:
+            $ref: "#/definitions/SnapshotLoadParams"
+      responses:
+        204:
+          description: Snapshot loaded
+        400:
+          description: Snapshot cannot be loaded due to bad input
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
+
+  /vm:
+    patch:
+      summary: Updates the microVM state.
+      description:
+        Sets the desired state (Paused or Resumed) for the microVM.
+      operationId: patchVm
+      parameters:
+        - name: body
+          in: body
+          description: The microVM state
+          required: true
+          schema:
+            $ref: "#/definitions/Vm"
+      responses:
+        204:
+          description: Vm state updated
+        400:
+          description: Vm state cannot be updated due to bad input
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
+
   /vsock:
     put:
-      summary: Creates/updates a vsock device.
+      summary: Creates/updates a vsock device. Pre-boot only.
       description:
-        The first call creates the device with the configuration specified 
-        in body. Subsequent calls will update the device configuration. 
+        The first call creates the device with the configuration specified
+        in body. Subsequent calls will update the device configuration.
         May fail if update is not possible.
       operationId: putGuestVsock
       parameters:
-      - name: body
-        in: body
-        description: Guest vsock properties
-        required: true
-        schema:
-          $ref: "#/definitions/Vsock"
+        - name: body
+          in: body
+          description: Guest vsock properties
+          required: true
+          schema:
+            $ref: "#/definitions/Vsock"
       responses:
         204:
           description: Vsock created/updated
@@ -388,12 +512,15 @@ definitions:
     description:
       Boot source descriptor.
     properties:
-      kernel_image_path:
-        type: string
-        description: Host level path to the kernel image used to boot the guest
       boot_args:
         type: string
         description: Kernel boot arguments
+      initrd_path:
+        type: string
+        description: Host level path to the initrd image used to boot the guest
+      kernel_image_path:
+        type: string
+        description: Host level path to the kernel image used to boot the guest
 
   CpuTemplate:
     type: string
@@ -408,15 +535,14 @@ definitions:
     type: object
     required:
       - drive_id
-      - path_on_host
-      - is_root_device
       - is_read_only
+      - is_root_device
+      - path_on_host
     properties:
       drive_id:
         type: string
-      path_on_host:
-        type: string
-        description: Host level path for the guest drive
+      is_read_only:
+        type: boolean
       is_root_device:
         type: boolean
       partuuid:
@@ -425,8 +551,9 @@ definitions:
           Represents the unique id of the boot partition of this device. It is
           optional and it will be taken into account only if the is_root_device
           field is true.
-      is_read_only:
-        type: boolean
+      path_on_host:
+        type: string
+        description: Host level path for the guest drive
       rate_limiter:
         $ref: "#/definitions/RateLimiter"
 
@@ -449,9 +576,9 @@ definitions:
         description: Enumeration indicating what type of action is contained in the payload
         type: string
         enum:
-        - FlushMetrics
-        - InstanceStart
-        - SendCtrlAltDel
+          - FlushMetrics
+          - InstanceStart
+          - SendCtrlAltDel
 
   InstanceInfo:
     type: object
@@ -459,21 +586,19 @@ definitions:
       Describes MicroVM instance information.
     required:
       - id
-      - state
       - vmm_version
     properties:
+      app_name:
+        description: Application name.
+        type: string
       id:
         description: MicroVM / instance ID.
         type: string
-      state:
+      started:
         description:
           The current detailed state of the Firecracker instance.
           This value is read-only for the control-plane.
-        type: string
-        enum:
-          - Uninitialized
-          - Starting
-          - Running
+        type: boolean
       vmm_version:
         description: MicroVM hypervisor build version.
         type: string
@@ -483,20 +608,16 @@ definitions:
     description:
       Describes the configuration option for the logging capability.
     required:
-      - log_fifo
-      - metrics_fifo
+      - log_path
     properties:
-      log_fifo:
-        type: string
-        description: The named pipe for the human readable log output.
-      metrics_fifo:
-        type: string
-        description: The named pipe where the JSON-formatted metrics will be flushed.
       level:
         type: string
-        description: Set the level.
+        description: Set the level. The possible values are case-insensitive.
         enum: [Error, Warning, Info, Debug]
         default: Warning
+      log_path:
+        type: string
+        description: Path to the named pipe or file for the human readable log output.
       show_level:
         type: boolean
         description: Whether or not to output the level in the logs.
@@ -512,39 +633,61 @@ definitions:
       Describes the number of vCPUs, memory size, Hyperthreading capabilities and
       the CPU template.
     required:
-      - vcpu_count
-      - mem_size_mib
       - ht_enabled
+      - mem_size_mib
+      - vcpu_count
     properties:
+      cpu_template:
+        $ref: "#/definitions/CpuTemplate"
+      ht_enabled:
+        type: boolean
+        description: Flag for enabling/disabling Hyperthreading
+      mem_size_mib:
+        type: integer
+        description: Memory size of VM
+      track_dirty_pages:
+        type: boolean
+        description:
+          Enable dirty page tracking. If this is enabled, then incremental guest memory
+          snapshots can be created. These belong to diff snapshots, which contain, besides
+          the microVM state, only the memory dirtied since a previous snapshot. Full snapshots
+          each contain a full copy of the guest memory.
       vcpu_count:
         type: integer
         minimum: 1
         maximum: 32
         description: Number of vCPUs (either 1 or an even number)
-      mem_size_mib:
-        type: integer
-        description: Memory size of VM
-      ht_enabled:
-        type: boolean
-        description: Flag for enabling/disabling Hyperthreading
-      cpu_template:
-        $ref: "#/definitions/CpuTemplate"
+
+  Metrics:
+    type: object
+    description:
+      Describes the configuration option for the metrics capability.
+    required:
+      - metrics_path
+    properties:
+      metrics_path:
+        type: string
+        description: Path to the named pipe or file where the JSON-formatted metrics are flushed.
+
+  MmdsConfig:
+    type: object
+    description:
+      Defines the MMDS configuration.
+    properties:
+      ipv4_address:
+        type: string
+        format: "169.254.([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-4]).([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])"
+        default: "169.254.169.254"
+        description: A valid IPv4 link-local address.
 
   NetworkInterface:
     type: object
     description:
       Defines a network interface.
     required:
-      - iface_id
       - host_dev_name
+      - iface_id
     properties:
-      iface_id:
-        type: string
-      guest_mac:
-        type: string
-      host_dev_name:
-        type: string
-        description: Host level path for the guest network interface
       allow_mmds_requests:
         type: boolean
         description:
@@ -553,6 +696,13 @@ definitions:
           both ARP requests for 169.254.169.254 and TCP segments heading to the
           same address are intercepted by the device model, and do not reach
           the associated TAP device.
+      guest_mac:
+        type: string
+      host_dev_name:
+        type: string
+        description: Host level path for the guest network interface
+      iface_id:
+        type: string
       rx_rate_limiter:
         $ref: "#/definitions/RateLimiter"
       tx_rate_limiter:
@@ -598,6 +748,48 @@ definitions:
         $ref: "#/definitions/TokenBucket"
         description: Token bucket with operations as tokens
 
+  SnapshotCreateParams:
+    type: object
+    required:
+      - mem_file_path
+      - snapshot_path
+    properties:
+      mem_file_path:
+        type: string
+        description: Path to the file that will contain the guest memory.
+      snapshot_path:
+        type: string
+        description: Path to the file that will contain the microVM state.
+      snapshot_type:
+        type: string
+        enum:
+          - Full
+        description:
+          Type of snapshot to create. It is optional and by default, a full
+          snapshot is created.
+      version:
+        type: string
+        description:
+          The microVM version for which we want to create the snapshot.
+          It is optional and it defaults to the current version.
+
+  SnapshotLoadParams:
+    type: object
+    required:
+      - mem_file_path
+      - snapshot_path
+    properties:
+      enable_diff_snapshots:
+        type: boolean
+        description:
+          Enable support for incremental (diff) snapshots by tracking dirty guest pages.
+      mem_file_path:
+        type: string
+        description: Path to the file that contains the guest memory to be loaded.
+      snapshot_path:
+        type: string
+        description: Path to the file that contains the microVM state to be loaded.
+
   TokenBucket:
     type: object
     description:
@@ -610,14 +802,9 @@ definitions:
       bound in size by the amount of tokens available.
       Once the token bucket is empty, consumption speed is bound by the refill_rate.
     required:
-      - size
       - refill_time
+      - size
     properties:
-      size:
-        type: integer
-        format: uint64
-        description: The total number of tokens this bucket can hold.
-        minimum: 0
       one_time_burst:
         type: integer
         format: uint64
@@ -628,6 +815,24 @@ definitions:
         format: uint64
         description: The amount of milliseconds it takes for the bucket to refill.
         minimum: 0
+      size:
+        type: integer
+        format: uint64
+        description: The total number of tokens this bucket can hold.
+        minimum: 0
+
+  Vm:
+    type: object
+    description:
+      Defines the microVM running state. It is especially useful in the snapshotting context.
+    required:
+      - state
+    properties:
+      state:
+        type: string
+        enum:
+          - Paused
+          - Resumed
 
   Vsock:
     type: object
@@ -642,12 +847,10 @@ definitions:
       bound and listening on Unix sockets at `uds_path_<PORT>`.
       E.g. "/path/to/host_vsock.sock_52" for port number 52.
     required:
-      - vsock_id
       - guest_cid
       - uds_path
+      - vsock_id
     properties:
-      vsock_id:
-        type: string
       guest_cid:
         type: integer
         minimum: 3
@@ -655,3 +858,5 @@ definitions:
       uds_path:
         type: string
         description: Path to UNIX domain socket, used to proxy vsock connections.
+      vsock_id:
+        type: string

src/runtime/virtcontainers/qemu_test.go

@@ -194,7 +194,7 @@ func TestQemuKnobs(t *testing.T) {
 	assert.NoError(err)
 
 	q := &qemu{
-		store: sandbox.newStore,
+		store: sandbox.store,
 	}
 	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig)
 	assert.NoError(err)
@@ -462,7 +462,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	assert.NoError(err)
 
 	q := &qemu{
-		store: sandbox.newStore,
+		store: sandbox.store,
 	}
 	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
 	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig)
@@ -477,7 +477,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	assert.NoError(err)
 
 	q = &qemu{
-		store: sandbox.newStore,
+		store: sandbox.store,
 	}
 	sandbox.config.HypervisorConfig.BootToBeTemplate = true
 	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
@@ -493,7 +493,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	assert.NoError(err)
 
 	q = &qemu{
-		store: sandbox.newStore,
+		store: sandbox.store,
 	}
 	sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc"
 	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig)
@@ -507,7 +507,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	assert.NoError(err)
 
 	q = &qemu{
-		store: sandbox.newStore,
+		store: sandbox.store,
 	}
 	sandbox.config.HypervisorConfig.EnableVhostUserStore = true
 	sandbox.config.HypervisorConfig.HugePages = true
@@ -520,7 +520,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	assert.NoError(err)
 
 	q = &qemu{
-		store: sandbox.newStore,
+		store: sandbox.store,
 	}
 	sandbox.config.HypervisorConfig.EnableVhostUserStore = true
 	sandbox.config.HypervisorConfig.HugePages = false
@@ -541,11 +541,11 @@ func createQemuSandboxConfig() (*Sandbox, error) {
 		},
 	}
 
-	newStore, err := persist.GetDriver()
+	store, err := persist.GetDriver()
 	if err != nil {
 		return &Sandbox{}, err
 	}
-	sandbox.newStore = newStore
+	sandbox.store = store
 
 	return &sandbox, nil
 }

src/runtime/virtcontainers/sandbox.go

@@ -168,7 +168,7 @@ type Sandbox struct {
 	factory    Factory
 	hypervisor hypervisor
 	agent      agent
-	newStore   persistapi.PersistDriver
+	store      persistapi.PersistDriver
 
 	network Network
 	monitor *monitor
@@ -282,7 +282,7 @@ func (s *Sandbox) GetContainer(containerID string) VCContainer {
 	return nil
 }
 
-// Release closes the agent connection and removes sandbox from internal list.
+// Release closes the agent connection.
 func (s *Sandbox) Release(ctx context.Context) error {
 	s.Logger().Info("release sandbox")
 	if s.monitor != nil {
@@ -293,7 +293,6 @@ func (s *Sandbox) Release(ctx context.Context) error {
 }
 
 // Status gets the status of the sandbox
-// TODO: update container status properly, see kata-containers/runtime#253
 func (s *Sandbox) Status() SandboxStatus {
 	var contStatusList []ContainerStatus
 	for _, c := range s.containers {
@@ -491,8 +490,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 	}
 
 	// create agent instance
-	newAagentFunc := getNewAgentFunc(ctx)
-	agent := newAagentFunc()
+	agent := getNewAgentFunc(ctx)()
 
 	hypervisor, err := newHypervisor(sandboxConfig.HypervisorType)
 	if err != nil {
@@ -518,14 +516,14 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 
 	hypervisor.setSandbox(s)
 
-	if s.newStore, err = persist.GetDriver(); err != nil || s.newStore == nil {
+	if s.store, err = persist.GetDriver(); err != nil || s.store == nil {
 		return nil, fmt.Errorf("failed to get fs persist driver: %v", err)
 	}
 
 	defer func() {
 		if retErr != nil {
-			s.Logger().WithError(retErr).WithField("sandboxid", s.id).Error("Create new sandbox failed")
-			s.newStore.Destroy(s.id)
+			s.Logger().WithError(retErr).Error("Create new sandbox failed")
+			s.store.Destroy(s.id)
 		}
 	}()
 
@@ -543,7 +541,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		s.Logger().WithError(err).Debug("restore sandbox failed")
 	}
 
-	// new store doesn't require hypervisor to be stored immediately
+	// store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig); err != nil {
 		return nil, err
 	}
@@ -708,7 +706,7 @@ func (s *Sandbox) Delete(ctx context.Context) error {
 
 	s.agent.cleanup(ctx, s)
 
-	return s.newStore.Destroy(s.id)
+	return s.store.Destroy(s.id)
 }
 
 func (s *Sandbox) startNetworkMonitor(ctx context.Context) error {
@@ -2283,7 +2281,7 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
 	// load sandbox config fromld store.
 	c, err := loadSandboxConfig(sandboxID)
 	if err != nil {
-		virtLog.WithError(err).Warning("failed to get sandbox config from new store")
+		virtLog.WithError(err).Warning("failed to get sandbox config from store")
 		return nil, err
 	}
 

src/runtime/virtcontainers/sandbox_test.go

@@ -1009,7 +1009,7 @@ func TestDeleteStoreWhenNewContainerFail(t *testing.T) {
 	}
 	_, err = newContainer(context.Background(), p, &contConfig)
 	assert.NotNil(t, err, "New container with invalid device info should fail")
-	storePath := filepath.Join(p.newStore.RunStoragePath(), testSandboxID, contID)
+	storePath := filepath.Join(p.store.RunStoragePath(), testSandboxID, contID)
 	_, err = os.Stat(storePath)
 	assert.NotNil(t, err, "Should delete configuration root after failed to create a container")
 }

src/runtime/virtcontainers/types/fcConfig.go

@@ -21,4 +21,6 @@ type FcConfig struct {
 	NetworkInterfaces []*models.NetworkInterface `json:"network-interfaces,omitempty"`
 
 	Logger *models.Logger `json:"logger,omitempty"`
+
+	Metrics *models.Metrics `json:"metrics,omitempty"`
 }

tools/agent-ctl/Cargo.lock

@@ -146,14 +146,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.2.1"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "348eb6d8e20a9f5247209686b7d0ffc2f4df40ddcb95f9940de55a94a655b3f5"
+checksum = "d4cec688ee0fcd143ffd7893ce2c9857bfc656eb1f2a27202244b72f08f5f8ed"
 dependencies = [
  "libc",
  "log",
- "nix 0.18.0",
- "procinfo",
+ "nix 0.20.0",
  "regex",
 ]
 
@@ -484,9 +483,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.79"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
+checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
 
 [[package]]
 name = "log"
@@ -585,18 +584,6 @@ dependencies = [
  "void",
 ]
 
-[[package]]
-name = "nix"
-version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83450fe6a6142ddd95fb064b746083fc4ef1705fe81f64a64e1d4b39f54a1055"
-dependencies = [
- "bitflags",
- "cc",
- "cfg-if 0.1.10",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.19.1"
@@ -610,10 +597,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "nom"
-version = "2.2.1"
+name = "nix"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff"
+checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if 1.0.0",
+ "libc",
+]
 
 [[package]]
 name = "ntapi"
@@ -738,7 +731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "059a34f111a9dee2ce1ac2826a68b24601c4298cfeb1a587c3cb493d5ab46f52"
 dependencies = [
  "libc",
- "nix 0.18.0",
+ "nix 0.19.1",
 ]
 
 [[package]]
@@ -771,18 +764,6 @@ dependencies = [
  "unicode-xid 0.2.1",
 ]
 
-[[package]]
-name = "procinfo"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c"
-dependencies = [
- "byteorder",
- "libc",
- "nom",
- "rustc_version",
-]
-
 [[package]]
 name = "prost"
 version = "0.5.0"
@@ -977,15 +958,6 @@ version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
 
-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver",
-]
-
 [[package]]
 name = "rustjail"
 version = "0.1.0"
@@ -1038,21 +1010,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser",
-]
-
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
 [[package]]
 name = "serde"
 version = "1.0.114"

tools/osbuilder/VERSION

@@ -1 +0,0 @@
-2.0.0

tools/osbuilder/VERSION

@@ -0,0 +1 @@
+../../VERSION

tools/osbuilder/dockerfiles/QAT/Dockerfile

@@ -9,9 +9,9 @@ FROM fedora:latest
 # images being pulled from the registry.
 # Set AGENT_VERSION as an env variable to specify a specific version of Kata Agent to install
 
-LABEL DOCKERFILE_VERSION="2.0"
+LABEL DOCKERFILE_VERSION="2.1"
 
-ENV QAT_DRIVER_VER "qat1.7.l.4.12.0-00011.tar.gz"
+ENV QAT_DRIVER_VER "QAT1.7.L.4.13.0-00009.tar.gz"
 ENV QAT_DRIVER_URL "https://downloadmirror.intel.com/30178/eng/${QAT_DRIVER_VER}"
 ENV QAT_CONFIGURE_OPTIONS "--enable-icp-sriov=guest"
 ENV KATA_REPO_VERSION "main"
@@ -22,7 +22,6 @@ ENV OUTPUT_DIR "/output"
 RUN dnf install -y \
     bc \
     bison \
-    curl \
     debootstrap \
     diffutils \
     e2fsprogs \

tools/osbuilder/dockerfiles/QAT/run.sh

@@ -4,7 +4,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
 set -u
 
 # NOTE: Some env variables are set in the Dockerfile - those that are
@@ -20,12 +19,27 @@ kata_repo_path=${GOPATH}/src/${kata_repo}
 tests_repo=github.com/kata-containers/tests
 tests_repo_path=${GOPATH}/src/${tests_repo}
 
+grab_qat_drivers()
+{
+    /bin/echo -e "\n\e[1;42mDownload and extract the drivers\e[0m" 
+    mkdir -p $QAT_SRC
+    cd $QAT_SRC
+    wget $QAT_DRIVER_URL
+    if [ ! -f ${QAT_SRC}/${QAT_DRIVER_VER} ];then
+         /bin/echo -e "\e[1;41mQAT Driver ${QAT_DRIVER_VER} doesn't exist\e[0m"
+         echo "Check https://01.org/intel-quickassist-technology to find the latest"
+         echo "QAT driver version, update the Dockerfile, and try again."
+         exit 1
+    fi
+    tar xzf ${QAT_DRIVER_VER}
+}
+
 grab_kata_repos()
 {
     # Check out all the repos we will use now, so we can try and ensure they use the specified branch
     # Only check out the branch needed, and make it shallow and thus space/bandwidth efficient
     # Use a green prompt with white text for easy viewing
-    bin/echo -e "\n\e[1;42mClone and checkout Kata repos\e[0m" 
+    /bin/echo -e "\n\e[1;42mClone and checkout Kata repos\e[0m" 
     git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${kata_repo} ${kata_repo_path}
     git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${tests_repo} ${tests_repo_path}
 }
@@ -64,14 +78,6 @@ build_rootfs()
     sudo -E SECCOMP=no EXTRA_PKGS='kmod' ${kata_repo_path}/tools/osbuilder/rootfs-builder/rootfs.sh $ROOTFS_OS 
 }
 
-grab_qat_drivers()
-{
-    /bin/echo -e "\n\e[1;42mDownload and extract the drivers\e[0m" 
-    mkdir -p $QAT_SRC
-    cd $QAT_SRC
-    curl -L $QAT_DRIVER_URL | tar zx
-}
-
 build_qat_drivers()
 {
     /bin/echo -e "\n\e[1;42mCompile driver modules\e[0m"
@@ -157,11 +163,11 @@ main()
 	done
 	shift $((OPTIND-1))
 
+	grab_qat_drivers
 	grab_kata_repos
 	configure_kernel
 	build_kernel
 	build_rootfs
-	grab_qat_drivers
 	build_qat_drivers
 	add_qat_to_rootfs
 	copy_outputs

tools/osbuilder/image-builder/Dockerfile

@@ -2,8 +2,8 @@
 # Copyright (c) 2018 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
-
-From registry.fedoraproject.org/fedora:latest
+ARG IMAGE_REGISTRY=registry.fedoraproject.org
+FROM ${IMAGE_REGISTRY}/fedora:latest
 
 RUN [ -n "$http_proxy" ] && sed -i '$ a proxy='$http_proxy /etc/dnf/dnf.conf ; true
 

tools/osbuilder/rootfs-builder/alpine/Dockerfile.in

@@ -3,7 +3,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-From docker.io/alpine:3.11.6
+ARG IMAGE_REGISTRY=docker.io
+FROM ${IMAGE_REGISTRY}/alpine:3.11.6
 
 RUN apk update && apk add \
 		  bash \