Merge pull request #8292 from fidencio/topic/release-ensure-gh-is-used-from-a-git-repo

release: Always use actions/checkout to ensure we're in a git repo

.github/workflows/add-backport-label.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install hub extension script
         run: |

.github/workflows/add-issues-to-project.yaml

@@ -39,7 +39,7 @@ jobs:
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Add issue to issue backlog
         env:

.github/workflows/add-pr-sizing-label.yaml

@@ -21,7 +21,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install PR sizing label script
         run: |

.github/workflows/basic-ci-amd64.yaml

@@ -0,0 +1,200 @@
+name: CI | Basic amd64 tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+  run-containerd-stability:
+    strategy:
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/stability/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/stability/gha-run.sh install-kata kata-artifacts
+
+      - name: Run containerd-stability tests
+        run: bash tests/stability/gha-run.sh run
+
+  run-nydus:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu', 'dragonball']
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nydus/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nydus tests
+        run: bash tests/integration/nydus/gha-run.sh run
+
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run
+
+  run-vfio:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+    runs-on: garm-ubuntu-2304
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/vfio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Run vfio tests
+        timeout-minutes: 15
+        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -16,6 +16,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
@@ -23,9 +27,13 @@ jobs:
     strategy:
       matrix:
         asset:
+          - agent
+          - agent-opa
+          - agent-ctl
           - cloud-hypervisor
           - cloud-hypervisor-glibc
           - firecracker
+          - kata-ctl
           - kernel
           - kernel-sev
           - kernel-dragonball-experimental
@@ -33,6 +41,7 @@ jobs:
           - kernel-nvidia-gpu
           - kernel-nvidia-gpu-snp
           - kernel-nvidia-gpu-tdx-experimental
+          - log-parser-rs
           - nydus
           - ovmf
           - ovmf-sev
@@ -44,12 +53,18 @@ jobs:
           - rootfs-initrd
           - rootfs-initrd-mariner
           - rootfs-initrd-sev
+          - runk
           - shim-v2
           - tdvf
+          - trace-forwarder
           - virtiofsd
         stage:
           - ${{ inputs.stage }}
         exclude:
+          - asset: agent
+            stage: release
+          - asset: agent-opa
+            stage: release
           - asset: cloud-hypervisor-glibc
             stage: release
     steps:
@@ -61,11 +76,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
 
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -76,6 +97,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -89,9 +114,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-asset
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -16,10 +16,14 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
-    runs-on: arm64
+    runs-on: arm64-builder
     strategy:
       matrix:
         asset:
@@ -48,10 +52,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -62,6 +73,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -72,16 +87,22 @@ jobs:
           if-no-files-found: error
 
   create-kata-tarball:
-    runs-on: arm64
+    runs-on: arm64-builder
     needs: build-asset
     steps:
       - name: Adjust a permission for repo
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -16,6 +16,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-asset:
@@ -44,10 +48,17 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
           fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"
@@ -59,6 +70,10 @@ jobs:
           KATA_ASSET: ${{ matrix.asset }}
           TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
           PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: store-artifact ${{ matrix.asset }}
         uses: actions/upload-artifact@v3
@@ -76,9 +91,15 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
       - name: get-artifacts
         uses: actions/download-artifact@v3
         with:

.github/workflows/cargo-deny-runner.yaml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout Code
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Generate Action
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: bash cargo-deny-generator.sh

.github/workflows/cc-payload-after-push-amd64.yaml

@@ -1,170 +0,0 @@
-name: CI | Publish CC runtime payload for amd64
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        measured_rootfs:
-          - no
-        asset:
-          - cloud-hypervisor
-          - qemu
-          - virtiofsd
-          - kernel-sev
-          - ovmf-sev
-          - ovmf
-          - qemu-snp-experimental
-          - qemu-tdx-experimental
-          - rootfs-initrd-sev
-          - cc-tdx-td-shim
-          - tdvf
-        include:
-          - measured_rootfs: yes
-            asset: kernel
-          - measured_rootfs: yes
-            asset: kernel-tdx-experimental
-          - measured_rootfs: yes
-            asset: cc-rootfs-image
-          - measured_rootfs: yes
-            asset: rootfs-image-tdx
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-      - name: Build ${{ matrix.asset }}
-        run: |
-          USE_CACHE="no" make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_tdx.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/root_hash_tdx.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_tdx.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          USE_CACHE="no" make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          PUSH_TO_REGISTRY: yes
-          MEASURED_ROOTFS: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push-s390x.yaml

@@ -1,207 +0,0 @@
-name: CI | Publish CC runtime payload for s390x
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        measured_rootfs:
-          - no
-        asset:
-          - qemu
-          - cc-rootfs-initrd
-          - virtiofsd
-        include:
-          - measured_rootfs: yes
-            asset: kernel
-          - measured_rootfs: yes
-            asset: cc-rootfs-image
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          USE_CACHE="no" make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          USE_CACHE="no" make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          PUSH_TO_REGISTRY: yes
-          MEASURED_ROOTFS: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  build-asset-cc-se-image:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-
-      - name: Place a host key document
-        run: |
-          mkdir -p "host-key-document"
-          cp "${CI_HKD_PATH}" "host-key-document"
-        env:
-          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
-
-      - name: Build cc-se-image
-        run: |
-          base_dir=tools/packaging/kata-deploy/local-build/
-          cp -r kata-artifacts ${base_dir}/build
-          # Skip building dependant artifacts of cc-se-image-tarball
-          # because we already have them from the previous build
-          sed -i 's/\(^cc-se-image-tarball:\).*/\1/g' ${base_dir}/Makefile
-          USE_CACHE="no" make cc-se-image-tarball
-          build_dir=$(readlink -f build)
-          sudo cp -r "${build_dir}" "kata-build"
-          sudo chown -R $(id -u):$(id -g) "kata-build"
-        env:
-          HKD_PATH: "host-key-document"
-
-      - name: store-artifact cc-se-image
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-se-image.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: [build-asset, build-asset-cc-shim-v2, build-asset-cc-se-image]
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload-after-push.yaml

@@ -1,47 +0,0 @@
-name: CI | Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    branches:
-      - CCv0
-  workflow_dispatch:
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-after-push-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-after-push-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push commit multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA} \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-${GITHUB_SHA}
-
-      - name: Push latest multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload-ci:kata-containers-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload-ci:kata-containers-latest

.github/workflows/cc-payload-amd64.yaml

@@ -1,155 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers (amd64)
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        measured_rootfs:
-          - no
-        asset:
-          - cloud-hypervisor
-          - qemu
-          - virtiofsd
-          - kernel-sev
-          - kernel-snp-experimental
-          - ovmf-sev
-          - ovmf
-          - qemu-snp-experimental
-          - qemu-tdx-experimental
-          - rootfs-initrd-sev
-          - cc-tdx-td-shim
-          - tdvf
-        include:
-          - measured_rootfs: yes
-            asset: kernel
-          - measured_rootfs: yes
-            asset: kernel-tdx-experimental
-          - measured_rootfs: yes
-            asset: cc-rootfs-image
-          - measured_rootfs: yes
-            asset: rootfs-image-tdx
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_tdx.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/root_hash_tdx.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_tdx.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_tdx.txt
-          path: tools/osbuilder/
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          MEASURED_ROOTFS: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          "quay.io/confidential-containers/runtime-payload" \
-          "kata-containers-${{ inputs.target-arch }}"
-

.github/workflows/cc-payload-s390x.yaml

@@ -1,142 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers (s390x)
-on:
-  workflow_call:
-    inputs:
-      target-arch:
-        required: true
-        type: string
-
-jobs:
-  build-asset:
-    runs-on: s390x
-    strategy:
-      matrix:
-        measured_rootfs:
-          - no
-        asset:
-          - qemu
-          - virtiofsd
-        include:
-          - measured_rootfs: yes
-            asset: kernel
-          - measured_rootfs: yes
-            asset: cc-rootfs-image
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          MEASURED_ROOTFS: ${{ matrix.measured_rootfs }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-      - name: store-artifact root_hash_vanilla.txt 
-        uses: actions/upload-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/root_hash_vanilla.txt
-          retention-days: 1
-          if-no-files-found: ignore
-
-  build-asset-cc-shim-v2:
-    runs-on: s390x
-    needs: build-asset
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-
-      - name: Get root_hash_vanilla.txt
-        uses: actions/download-artifact@v3
-        with:
-          name: root_hash_vanilla.txt-s390x
-          path: tools/osbuilder/
-
-      - name: Build cc-shim-v2
-        run: |
-          make cc-shim-v2-tarball
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          MEASURED_ROOTFS: yes
-
-      - name: store-artifact cc-shim-v2
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-build/kata-static-cc-shim-v2.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: s390x
-    needs: [build-asset, build-asset-cc-shim-v2]
-    steps:
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts-s390x
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: s390x
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Adjust a permission for repo
-        run: |
-          sudo chown -R $USER:$USER $GITHUB_WORKSPACE
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-s390x
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
-          $(pwd)/kata-static.tar.xz \
-          "quay.io/confidential-containers/runtime-payload" \
-          "kata-containers-${{ inputs.target-arch }}"

.github/workflows/cc-payload.yaml

@@ -1,46 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    tags:
-      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
-
-jobs:
-  build-assets-amd64:
-    uses: ./.github/workflows/cc-payload-amd64.yaml
-    with:
-      target-arch: amd64
-    secrets: inherit
-
-  build-assets-s390x:
-    uses: ./.github/workflows/cc-payload-s390x.yaml
-    with:
-      target-arch: s390x
-    secrets: inherit
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build-assets-amd64, build-assets-s390x]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - name: Push commit multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA} \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-${GITHUB_SHA}
-
-      - name: Push latest multi-arch manifest
-        run: |
-          docker manifest create quay.io/confidential-containers/runtime-payload:kata-containers-latest \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-amd64 \
-          --amend quay.io/confidential-containers/runtime-payload:kata-containers-s390x
-          docker manifest push quay.io/confidential-containers/runtime-payload:kata-containers-latest

.github/workflows/ci-nightly.yaml

@@ -15,4 +15,5 @@ jobs:
       commit-hash: ${{ github.sha }}
       pr-number: "nightly"
       tag: ${{ github.sha }}-nightly
+      target-branch: ${{ github.ref_name }}
     secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -28,4 +28,5 @@ jobs:
       commit-hash: ${{ github.event.pull_request.head.sha }}
       pr-number: ${{ github.event.pull_request.number }}
       tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
     secrets: inherit

.github/workflows/ci.yaml

@@ -11,6 +11,10 @@ on:
       tag:
         required: true
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   build-kata-static-tarball-amd64:
@@ -18,6 +22,7 @@ jobs:
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
   publish-kata-deploy-payload-amd64:
     needs: build-kata-static-tarball-amd64
@@ -28,15 +33,23 @@ jobs:
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
   build-and-publish-tee-confidential-unencrypted-image:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -60,6 +73,54 @@ jobs:
           platforms: linux/amd64, linux/s390x
           file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
 
+  run-docker-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-docker-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-nerdctl-tests-on-garm:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-nerdctl-tests-on-garm.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-kata-deploy-tests-on-aks:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-aks.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-deploy-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-monitor-tests:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-kata-monitor-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
   run-k8s-tests-on-aks:
     needs: publish-kata-deploy-payload-amd64
     uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
@@ -69,37 +130,43 @@ jobs:
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
-  run-k8s-tests-on-sev:
-    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
-    uses: ./.github/workflows/run-k8s-tests-on-sev.yaml
+  run-k8s-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-garm.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
 
-  run-k8s-tests-on-snp:
-    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
-    uses: ./.github/workflows/run-k8s-tests-on-snp.yaml
+  run-k8s-tests-with-crio-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-with-crio-on-garm.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
 
-  run-k8s-tests-on-tdx:
+  run-kata-coco-tests:
     needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
-    uses: ./.github/workflows/run-k8s-tests-on-tdx.yaml
+    uses: ./.github/workflows/run-kata-coco-tests.yaml
     with:
       registry: ghcr.io
       repo: ${{ github.repository_owner }}/kata-deploy-ci
       tag: ${{ inputs.tag }}-amd64
       commit-hash: ${{ inputs.commit-hash }}
       pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
 
   run-metrics-tests:
     needs: build-kata-static-tarball-amd64
@@ -107,24 +174,12 @@ jobs:
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
 
-  run-cri-containerd-tests:
+  run-basic-amd64-tests:
     needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-cri-containerd-tests.yaml
-    with:
-      tarball-suffix: -${{ inputs.tag }}
-      commit-hash: ${{ inputs.commit-hash }}
-
-  run-nydus-tests:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-nydus-tests.yaml
-    with:
-      tarball-suffix: -${{ inputs.tag }}
-      commit-hash: ${{ inputs.commit-hash }}
-
-  run-vfio-tests:
-    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-vfio-tests.yaml
+    uses: ./.github/workflows/basic-ci-amd64.yaml
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}

.github/workflows/commit-message-check.yaml

@@ -51,7 +51,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^.{0,75}(\n.*)*$'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -102,6 +102,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/darwin-tests.yaml

@@ -21,6 +21,6 @@ jobs:
       with:
         go-version: 1.19.3
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/deploy-ccv0-demo.yaml

@@ -1,124 +0,0 @@
-on:
-  issue_comment:
-    types: [created, edited]
-
-name: deploy-ccv0-demo
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
-    steps:
-      - name: Check membership
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare confidential container rootfs
-        if: ${{ matrix.asset == 'rootfs-initrd' }}
-        run: |
-          pushd include_rootfs/etc
-          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
-          mkdir kata-containers
-          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
-          popd
-        env:
-          AA_KBC_PARAMS: offline_fs_kbc::null
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          AA_KBC: offline_fs_kbc
-          INCLUDE_ROOTFS: include_rootfs
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/docs-url-alive-check.yaml

@@ -22,7 +22,7 @@ jobs:
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}

.github/workflows/kata-runtime-classes-sync.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Ensure the split out runtime classes match the all-in-one file
       run: |
         pushd tools/packaging/kata-deploy/runtimeclasses/

.github/workflows/move-issues-to-in-progress.yaml

@@ -38,7 +38,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Move issue to "In progress"
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}

.github/workflows/payload-after-push.yaml

@@ -4,6 +4,7 @@ on:
     branches:
       - main
       - stable-*
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -15,6 +16,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   build-assets-arm64:
@@ -22,6 +24,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   build-assets-s390x:
@@ -29,6 +32,7 @@ jobs:
     with:
       commit-hash: ${{ github.sha }}
       push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-amd64:
@@ -39,6 +43,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-amd64
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-arm64:
@@ -49,6 +54,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-arm64
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-kata-deploy-payload-s390x:
@@ -59,6 +65,7 @@ jobs:
       registry: quay.io
       repo: kata-containers/kata-deploy-ci
       tag: kata-containers-s390x
+      target-branch: ${{ github.ref_name }}
     secrets: inherit
 
   publish-manifest:
@@ -66,7 +73,7 @@ jobs:
     needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Kata Containers quay.io
         uses: docker/login-action@v2

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -17,14 +17,25 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -17,18 +17,29 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
-    runs-on: arm64
+    runs-on: arm64-builder
     steps:
       - name: Adjust a permission for repo
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -17,6 +17,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   kata-payload:
@@ -26,9 +30,16 @@ jobs:
         run: |
           sudo chown -R $USER:$USER $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3

.github/workflows/release-amd64.yaml

@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release-arm64.yaml

@@ -14,7 +14,7 @@ jobs:
 
   kata-deploy:
     needs: build-kata-static-tarball-arm64
-    runs-on: arm64
+    runs-on: arm64-builder
     steps:
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release-s390x.yaml

@@ -29,7 +29,7 @@ jobs:
           username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
           password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
         with:

.github/workflows/release.yaml

@@ -32,7 +32,7 @@ jobs:
     needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Kata Containers docker.io
         uses: docker/login-action@v2
@@ -73,11 +73,7 @@ jobs:
     needs: publish-multi-arch-images
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: install hub
-        run: |
-          wget -q -O- https://github.com/mislav/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
+      - uses: actions/checkout@v4
 
       - name: download-artifacts-amd64
         uses: actions/download-artifact@v3
@@ -90,7 +86,7 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
       - name: download-artifacts-arm64
@@ -104,7 +100,7 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
       - name: download-artifacts-s390x
@@ -118,13 +114,13 @@ jobs:
           mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
           pushd $GITHUB_WORKSPACE
           echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-versions-yaml:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: upload versions.yaml
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -133,28 +129,28 @@ jobs:
           pushd $GITHUB_WORKSPACE
           versions_file="kata-containers-$tag-versions.yaml"
           cp versions.yaml ${versions_file}
-          hub release edit -m "" -a "${versions_file}" "${tag}"
+          gh release upload "${tag}" "${versions_file}"
           popd
 
   upload-cargo-vendored-tarball:
     needs: upload-multi-arch-static-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: generate-and-upload-tarball
         run: |
           tag=$(echo $GITHUB_REF | cut -d/ -f3-)
           tarball="kata-containers-$tag-vendor.tar.gz"
           pushd $GITHUB_WORKSPACE
           bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
+          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} gh release upload "${tag}" "${tarball}"
           popd
 
   upload-libseccomp-tarball:
     needs: upload-cargo-vendored-tarball
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: download-and-upload-tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
@@ -174,6 +170,6 @@ jobs:
           # "-m" option should be empty to re-use the existing release title
           # without opening a text editor.
           # For the details, check https://hub.github.com/hub-release.1.html.
-          hub release edit -m "" -a "${tarball}" "${tag}"
-          hub release edit -m "" -a "${asc}" "${tag}"
+          gh release upload "${tag}" "${tarball}"
+          gh release upload "${tag}" "${asc}"
           popd

.github/workflows/require-pr-porting-labels.yaml

@@ -36,7 +36,17 @@ jobs:
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install porting checker script
         run: |

.github/workflows/run-cri-containerd-tests.yaml

@@ -1,42 +0,0 @@
-name: CI | Run cri-containerd tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-cri-containerd:
-    strategy:
-      fail-fast: true
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'qemu']
-    runs-on: garm-ubuntu-2204
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
-
-      - name: Run cri-containerd tests
-        run: bash tests/integration/cri-containerd/gha-run.sh run

.github/workflows/run-docker-tests-on-garm.yaml

@@ -0,0 +1,56 @@
+name: CI | Run docker integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-docker-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/docker/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/docker/gha-run.sh install-kata kata-artifacts
+
+      - name: Run docker smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/docker/gha-run.sh run

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -17,6 +17,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   run-k8s-tests:
@@ -29,6 +33,9 @@ jobs:
           - clh
           - dragonball
           - qemu
+        instance-type:
+          - small
+          - normal
         include:
           - host_os: cbl-mariner
             vmm: clh
@@ -40,11 +47,20 @@ jobs:
       GH_PR_NUMBER: ${{ inputs.pr-number }}
       KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
       USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: Download Azure CLI
         run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli

.github/workflows/run-k8s-tests-on-garm.yaml

@@ -0,0 +1,88 @@
+name: CI | Run kubernetes tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh #cloud-hypervisor
+          - fc #firecracker
+          - qemu
+        snapshotter:
+          - devmapper
+        k8s:
+          - k3s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -1,52 +0,0 @@
-name: CI | Run kubernetes tests on SEV
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      pr-number:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-sev
-    runs-on: sev
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBECONFIG: /home/kata/.kube/config
-      USING_NFD: "false"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -1,52 +0,0 @@
-name: CI | Run kubernetes tests on SEV-SNP
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      pr-number:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-snp
-    runs-on: sev-snp
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      KUBECONFIG: /home/kata/.kube/config
-      USING_NFD: "false"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-      
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -1,51 +0,0 @@
-name: CI | Run kubernetes tests on TDX
-on:
-  workflow_call:
-    inputs:
-      registry:
-        required: true
-        type: string
-      repo:
-        required: true
-        type: string
-      tag:
-        required: true
-        type: string
-      pr-number:
-        required: true
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-k8s-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm:
-          - qemu-tdx
-    runs-on: tdx
-    env:
-      DOCKER_REGISTRY: ${{ inputs.registry }}
-      DOCKER_REPO: ${{ inputs.repo }}
-      DOCKER_TAG: ${{ inputs.tag }}
-      PR_NUMBER: ${{ inputs.pr-number }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-      USING_NFD: "true"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Deploy Kata
-        timeout-minutes: 10
-        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
-  
-      - name: Run tests
-        timeout-minutes: 30
-        run: bash tests/integration/kubernetes/gha-run.sh run-tests
-  
-      - name: Delete kata-deploy
-        if: always()
-        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx

.github/workflows/run-k8s-tests-with-crio-on-garm.yaml

@@ -0,0 +1,86 @@
+name: CI | Run kubernetes tests, using CRI-O, on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        k8s:
+          - k0s
+        instance:
+          - garm-ubuntu-2004
+          - garm-ubuntu-2004-smaller
+        include:
+          - instance: garm-ubuntu-2004
+            instance-type: normal
+          - instance: garm-ubuntu-2004-smaller
+            instance-type: small
+          - k8s: k0s
+            k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"'
+    runs-on: ${{ matrix.instance }}
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Configure CRI-O
+        run: bash tests/integration/kubernetes/gha-run.sh setup-crio
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+  
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+  
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm

.github/workflows/run-kata-coco-tests.yaml

@@ -0,0 +1,176 @@
+name: CI | Run kata coco tests
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+
+  run-k8s-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      USING_NFD: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
+
+  run-k8s-tests-on-sev:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-sev
+    runs-on: sev
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-sev
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-sev
+
+  run-k8s-tests-sev-snp:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: "baremetal"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -0,0 +1,89 @@
+name: CI | Run kata-deploy tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Download Azure CLI
+        run: bash tests/functional/kata-deploy/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/functional/kata-deploy/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+
+      - name: Create AKS cluster
+        timeout-minutes: 10
+        run: bash tests/functional/kata-deploy/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/functional/kata-deploy/gha-run.sh get-cluster-credentials
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+      
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -0,0 +1,65 @@
+name: CI | Run kata-deploy tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+        k8s:
+          - k0s
+          - k3s
+          - rke2
+    runs-on: garm-ubuntu-2004-smaller
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests

.github/workflows/run-kata-monitor-tests.yaml

@@ -0,0 +1,59 @@
+name: CI | Run kata-monitor tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-monitor:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        container_engine:
+          - crio
+          - containerd
+        include:
+          - container_engine: containerd
+            containerd_version: lts
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINER_ENGINE: ${{ matrix.container_engine }}
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/kata-monitor/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/kata-monitor/gha-run.sh install-kata kata-artifacts
+
+      - name: Run kata-monitor tests
+        run: bash tests/functional/kata-monitor/gha-run.sh run

.github/workflows/run-metrics.yaml

@@ -8,6 +8,10 @@ on:
       commit-hash:
         required: false
         type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
 
 jobs:
   setup-kata:
@@ -16,9 +20,16 @@ jobs:
     env:
       GOPATH: ${{ github.workspace }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
 
       - name: get-kata-tarball
         uses: actions/download-artifact@v3
@@ -32,7 +43,10 @@ jobs:
   run-metrics:
     needs: setup-kata
     strategy:
-      fail-fast: true
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
       matrix:
         vmm: ['clh', 'qemu']
       max-parallel: 1
@@ -65,6 +79,9 @@ jobs:
       - name: run iperf test
         run:  bash tests/metrics/gha-run.sh run-test-iperf
 
+      - name: run latency test
+        run:  bash tests/metrics/gha-run.sh run-test-latency
+
       - name: make metrics tarball ${{ matrix.vmm }}
         run: bash tests/metrics/gha-run.sh make-tarball-results
 

.github/workflows/run-nerdctl-tests-on-garm.yaml

@@ -0,0 +1,57 @@
+name: CI | Run nerdctl integration tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-nerdctl-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+    runs-on: garm-ubuntu-2304-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nerdctl/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nerdctl/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nerdctl smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/nerdctl/gha-run.sh run

.github/workflows/run-nydus-tests.yaml

@@ -1,42 +0,0 @@
-name: CI | Run nydus tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-nydus:
-    strategy:
-      fail-fast: true
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'qemu', 'dragonball']
-    runs-on: garm-ubuntu-2204
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/integration/nydus/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
-
-      - name: Run nydus tests
-        run: bash tests/integration/nydus/gha-run.sh run

.github/workflows/run-runk-tests.yaml

@@ -0,0 +1,46 @@
+name: CI | Run runk tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-runk:
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        run: bash tests/integration/runk/gha-run.sh run

.github/workflows/run-vfio-tests.yaml

@@ -1,37 +0,0 @@
-name: CI | Run vfio tests
-on:
-  workflow_call:
-    inputs:
-      tarball-suffix:
-        required: false
-        type: string
-      commit-hash:
-        required: false
-        type: string
-
-jobs:
-  run-vfio:
-    strategy:
-      fail-fast: false
-      matrix:
-        vmm: ['clh', 'qemu']
-    runs-on: garm-ubuntu-2204
-    env:
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ inputs.commit-hash }}
-
-      - name: Install dependencies
-        run: bash tests/functional/vfio/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Run vfio tests
-        run: bash tests/functional/vfio/gha-run.sh run

.github/workflows/static-checks-dragonball.yaml

@@ -1,37 +0,0 @@
-on:
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-name: Static checks dragonball
-jobs:
-  test-dragonball:
-    runs-on: dragonball
-    env:
-      RUST_BACKTRACE: "1"
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set env
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-      - name: Install Rust
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          ./ci/install_rust.sh
-          echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
-      - name: Run Unit Test
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          cd src/dragonball
-          cargo version
-          rustc --version
-          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.github/workflows/static-checks.yaml

@@ -12,84 +12,183 @@ concurrency:
 
 name: Static checks
 jobs:
-  static-checks:
-    runs-on: garm-ubuntu-2004
+  check-kernel-config-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Ensure the kernel config version has been updated
+        run: |
+          kernel_dir="tools/packaging/kernel/"
+          kernel_version_file="${kernel_dir}kata_config_version"
+          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
+          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+            echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+            if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+              echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+            else
+              echo "Readme file changed, no need for kernel config version update."
+            fi
+            echo "Check passed"
+          fi
+
+  build-checks:
+    runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
-        cmd:
+        component:
+          - agent
+          - dragonball
+          - runtime
+          - runtime-rs
+          - agent-ctl
+          - kata-ctl
+          - log-parser-rs
+          - runk
+          - trace-forwarder
+        command:
           - "make vendor"
-          - "make static-checks"
           - "make check"
           - "make test"
           - "sudo -E PATH=\"$PATH\" make test"
+        include:
+          - component: agent
+            component-path: src/agent
+          - component: dragonball
+            component-path: src/dragonball
+          - component: runtime
+            component-path: src/runtime
+          - component: runtime-rs
+            component-path: src/runtime-rs
+          - component: agent-ctl
+            component-path: src/tools/agent-ctl
+          - component: kata-ctl
+            component-path: src/tools/kata-ctl
+          - component: log-parser-rs
+            component-path: src/tools/log-parser-rs
+          - component: runk
+            component-path: src/tools/runk
+          - component: trace-forwarder
+            component-path: src/tools/trace-forwarder
+          - install-libseccomp: no
+          - component: agent
+            install-libseccomp: yes
+          - component: runk
+            install-libseccomp: yes
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install yq
+        run: |
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        if: ${{ matrix.component == 'runtime' }}
+        run: |
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install rust
+        if: ${{ matrix.component != 'runtime' }}
+        run: |
+          ./tests/install_rust.sh
+          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+      - name: Install musl-tools
+        if: ${{ matrix.component != 'runtime' }}
+        run: sudo apt-get -y install musl-tools
+      - name: Install libseccomp
+        if: ${{ matrix.command != 'make vendor'  &&  matrix.command != 'make check' &&  matrix.install-libseccomp == 'yes' }}
+        run: |
+          libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+          gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+          ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+          echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+          echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
+      - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
+        if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
+        run: |
+          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  build-checks-depending-on-kvm:
+    runs-on: garm-ubuntu-2004-smaller
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - runtime-rs
+        include:
+          - component: runtime-rs
+            command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test"
+          - component: runtime-rs
+            component-path: src/dragonball
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install system deps
+        run: |
+          sudo apt-get install -y build-essential musl-tools
+      - name: Install yq
+        run: |
+          sudo -E ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install rust
+        run: |
+          export PATH="$PATH:/usr/local/bin"
+          ./tests/install_rust.sh
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          export PATH="$PATH:${HOME}/.cargo/bin"
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  static-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        cmd:
+          - "make static-checks"
     env:
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
       GOPATH: ${{ github.workspace }}
     steps:
-    - name: Free disk space
-      run: |
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-    - name: Checkout code
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y --no-install-recommends \
-          build-essential \
-          haveged \
-          libdevmapper-dev \
-          clang
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.3
-    - name: Check kernel config version
-      run: |
-        cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
-        kernel_dir="tools/packaging/kernel/"
-        kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/CCv0..HEAD)
-        if git diff --name-only origin/CCv0..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
-          echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
-          if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
-            echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
-          else
-            echo "Readme file changed, no need for kernel config version update."
-          fi
-          echo "Check passed"
-        fi
-    - name: Set PATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        export PATH=$PATH:"$HOME/.cargo/bin"
-        export XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: ./src/github.com/${{ github.repository }}
+      - name: Install yq
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install system dependencies
+        run: |
+          sudo apt-get -y install moreutils hunspell pandoc
+      - name: Run check
+        run: |
+          export PATH=${PATH}:${GOPATH}/bin
+          cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

VERSION

@@ -1 +1 @@
-3.2.0-rc0
+3.3.0-alpha0

ci/install_libseccomp.sh

@@ -7,12 +7,10 @@
 
 set -o errexit
 
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+script_name="$(basename "${BASH_SOURCE[0]}")"
 
-clone_tests_repo
-
-source "${tests_repo_dir}/.ci/lib.sh"
+source "${script_dir}/../tests/common.bash"
 
 # The following variables if set on the environment will change the behavior
 # of gperf and libseccomp configure scripts, that may lead this script to
@@ -25,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_version "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_version "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -38,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_version "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps "externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_version "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps "externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"
@@ -72,8 +70,7 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # gperf is a build time dependency of libseccomp and not to be used in the target.
-    # Unset $CC since that might point to a cross compiler.
+    # Unset $CC for configure, we will always use native for gperf
     CC= ./configure --prefix="${gperf_install_dir}"
     make
     make install

ci/lib.sh

@@ -64,86 +64,3 @@ run_get_pr_changed_file_details()
 	source "$tests_repo_dir/.ci/lib.sh"
 	get_pr_changed_file_details
 }
-
-# Check if the 1st argument version is greater than and equal to 2nd one
-# Version format: [0-9]+ separated by period (e.g. 2.4.6, 1.11.3 and etc.)
-#
-# Parameters:
-#	$1	- a version to be tested
-#	$2	- a target version
-#
-# Return:
-# 	0 if $1 is greater than and equal to $2
-#	1 otherwise
-version_greater_than_equal() {
-	local current_version=$1
-	local target_version=$2
-	smaller_version=$(echo -e "$current_version\n$target_version" | sort -V | head -1)
-	if [ "${smaller_version}" = "${target_version}" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
-# Build a IBM zSystem secure execution (SE) image
-#
-# Parameters:
-#	$1	- kernel_parameters
-#	$2	- a source directory where kernel and initrd are located
-#	$3	- a destination directory where a SE image is built
-#
-# Return:
-# 	0 if the image is successfully built
-#	1 otherwise
-build_secure_image() {
-	kernel_params="${1:-}"
-	install_src_dir="${2:-}"
-	install_dest_dir="${3:-}"
-
-	if [ ! -f "${install_src_dir}/vmlinuz.container" ] ||
-		[ ! -f "${install_src_dir}/kata-containers-initrd.img" ]; then
-		cat << EOF >&2
-Either kernel or initrd does not exist or is mistakenly named
-A file name for kernel must be vmlinuz.container (raw binary)
-A file name for initrd must be kata-containers-initrd.img
-EOF
-		return 1
-	fi
-
-	cmdline="${kernel_params} panic=1 scsi_mod.scan=none swiotlb=262144"
-	parmfile="$(mktemp --suffix=-cmdline)"
-	echo "${cmdline}" > "${parmfile}"
-	chmod 600 "${parmfile}"
-
-	[ -n "${HKD_PATH:-}" ] || (echo >&2 "No host key document specified." && return 1)
-	cert_list=($(ls -1 $HKD_PATH))
-	declare hkd_options
-	eval "for cert in ${cert_list[*]}; do
-		hkd_options+=\"--host-key-document=\\\"\$HKD_PATH/\$cert\\\" \"
-	done"
-
-	command -v genprotimg > /dev/null 2>&1 || { apt update; apt install -y s390-tools; }
-	extra_arguments=""
-	genprotimg_version=$(genprotimg --version | grep -Po '(?<=version )[^-]+')
-	if ! version_greater_than_equal "${genprotimg_version}" "2.17.0"; then
-		extra_arguments="--x-pcf '0xe0'"
-	fi
-
-	eval genprotimg \
-		"${extra_arguments}" \
-		"${hkd_options}" \
-		--output="${install_dest_dir}/kata-containers-secure.img" \
-		--image="${install_src_dir}/vmlinuz.container" \
-		--ramdisk="${install_src_dir}/kata-containers-initrd.img" \
-		--parmfile="${parmfile}" \
-		--no-verify # no verification for CI testing purposes
-
-	build_result=$?
-	rm -f "${parmfile}"
-	if [ $build_result -eq 0 ]; then
-		return 0
-	else
-		return 1
-	fi
-}

docs/design/kata-vra.md

@@ -43,7 +43,7 @@ and perform DMA transactions _anywhere_.
 
 The second feature is ACS (Access Control Services), which controls which
 devices are allowed to communicate with one another and thus avoids improper
-routing of packets irrespectively of whether IOMMU is enabled or not.
+routing of packets `irrespectively` of whether IOMMU is enabled or not.
 
 When IOMMU is enabled, ACS is normally configured to force all PCI Express DMA
 to go through the root complex so IOMMU can translate it, impacting performance
@@ -126,7 +126,7 @@ efficient P2P communication.
 ## PCI Express Virtual P2P Approval Capability
 
 Most of the time, the PCI Express topology is flattened and obfuscated to ensure
-easy migration of the VM image between different physical hardware topologies.
+easy migration of the VM image between different physical hardware `topologies`.
 In Kata, we can configure the hypervisor to use PCI Express root ports to
 hotplug the VFIO  devices one is passing through. A user can select how many PCI
 Express root ports to allocate depending on how many devices are passed through.
@@ -220,7 +220,7 @@ containers that he wants to run with Kata. The goal is to make such things as
 transparent as possible, so we also introduced
 [CDI](https://github.com/container-orchestrated-devices/container-device-interface)
 (Container Device Interface) to Kata. CDI is a[
-specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/master/SPEC.md)
+specification](https://github.com/container-orchestrated-devices/container-device-interface/blob/main/SPEC.md)
 for container runtimes to support third-party devices.
 
 As written before, we can provide a clique ID for the devices that belong
@@ -300,7 +300,7 @@ pcie_switch_port = 8
 ```
 
 Each device that is passed through is attached to a PCI Express downstream port
-as illustrated below. We can even replicate the host’s two DPUs topologies with
+as illustrated below. We can even replicate the host’s two DPUs `topologies` with
 added metadata through the CDI. Most of the time, a container only needs one
 pair of GPU and NIC for GPUDirect RDMA. This is more of a showcase of what we
 can do with the power of Kata and CDI. One could even think of adding groups of
@@ -328,7 +328,7 @@ $ lspci -tv
 ```
 
 The configuration of using either the root port or switch port can be applied on
-a per Container or Pod basis, meaning we can switch PCI Express topologies on
+a per Container or Pod basis, meaning we can switch PCI Express `topologies` on
 each run of an application.
 
 ## Hypervisor Resource Limits

docs/how-to/README.md

@@ -45,8 +45,4 @@
 - [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
 - [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
 - [How to use EROFS to build rootfs in Kata Containers](how-to-use-erofs-build-rootfs.md)
-- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)
-
-## Confidential Containers
-- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
-- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)
+- [How to run Kata Containers with kinds of Block Volumes](how-to-run-kata-containers-with-kinds-of-Block-Volumes.md)

docs/how-to/ccv0.sh

@@ -1,635 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright (c) 2021, 2023 IBM Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-# Disclaimer: This script is work in progress for supporting the CCv0 prototype
-# It shouldn't be considered supported by the Kata Containers community, or anyone else
-
-# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
-# but with elements of the tests/.ci scripts used
-
-readonly script_name="$(basename "${BASH_SOURCE[0]}")"
-
-# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
-export GO111MODULE="auto"
-
-# Setup kata containers environments if not set - we default to use containerd
-export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
-export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
-export CRIO=${CRIO:-"no"}
-export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
-export KUBERNETES=${KUBERNETES:-"no"}
-export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
-export AA_KBC="${AA_KBC:-offline_fs_kbc}"
-export KATA_BUILD_CC=${KATA_BUILD_CC:-"yes"}
-export TEE_TYPE=${TEE_TYPE:-}
-export PREFIX="${PREFIX:-/opt/confidential-containers}"
-export RUNTIME_CONFIG_PATH="${RUNTIME_CONFIG_PATH:-${PREFIX}/share/defaults/kata-containers/configuration.toml}"
-
-# Allow the user to overwrite the default repo and branch names if they want to build from a fork
-export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
-export katacontainers_branch="${katacontainers_branch:-CCv0}" 
-export kata_default_branch=${katacontainers_branch}
-export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
-export tests_branch="${tests_branch:-CCv0}"
-export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
-
-# if .bash_profile exists then use it, otherwise fall back to .profile
-export PROFILE="${HOME}/.profile"
-if [ -r "${HOME}/.bash_profile" ]; then
-    export PROFILE="${HOME}/.bash_profile"
-fi
-# Stop PS1: unbound variable error happening
-export PS1=${PS1:-}
-
-# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
-. ${PROFILE}
-if [ -z ${GOPATH} ]; then
-    export GOPATH=${HOME}/go
-fi
-export tests_repo_dir="${GOPATH}/src/${tests_repo}"
-export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
-export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
-export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
-export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
-source /etc/os-release || source /usr/lib/os-release
-grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
-
-
-# If we've already checked out the test repo then source the confidential scripts
-if [ "${KUBERNETES}" == "yes" ]; then
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-else
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-fi
-
-[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-
-usage() {
-    exit_code="$1"
-    cat <<EOF
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
-Usage:
-    ${script_name} [options] <command>
-Commands:
-- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
-- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
-- all:                              Build and install everything, test kata with containerd and capture the logs
-- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
-- build_and_install_all:            Build and install everything
-- build_and_install_rootfs:         Builds and installs the rootfs image
-- build_kata_runtime:               Build and install the kata runtime
-- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                       Checkout, patch, build and install QEMU
-- configure:                        Configure Kata to use rootfs and enable debug
-- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
-- copy_signature_files_to_guest     Copies signature verification files to guest
-- create_rootfs:                    Create a local rootfs
-- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
-- crictl_create_cc_pod              Use crictl to create a new kata cc pod
-- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
-- help:                             Display this help
-- init_kubernetes:                  initialize a Kubernetes cluster on this system
-- initialize:                       Install dependencies and check out kata-containers source
-- install_guest_kernel:             Setup, build and install the guest kernel
-- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
-- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
-- open_kata_shell:                  Open a shell into the kata runtime
-- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
-- shim_pull_image:                  Run PullImage command against the shim with ctr
-- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
-- test:                             Test using kata with containerd
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-EOF
-    # if script sourced don't exit as this will exit the main shell, just return instead
-    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
-}
-
-build_and_install_all() {
-    initialize
-    build_and_install_kata_runtime
-    configure
-    create_a_local_rootfs
-    build_and_install_rootfs
-    install_guest_kernel_image
-    case "$KATA_HYPERVISOR" in
-        "qemu") 
-            build_qemu
-            ;;
-        "cloud-hypervisor") 
-            build_cloud_hypervisor
-            ;;
-        *)
-            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
-            ;;
-    esac
-
-    check_kata_runtime
-    if [ "${KUBERNETES}" == "yes" ]; then
-        init_kubernetes
-    fi
-}
-
-rebuild_and_install_kata() {
-    checkout_tests_repo
-    checkout_kata_containers_repo
-    build_and_install_kata_runtime
-    build_and_add_agent_to_rootfs
-    build_and_install_rootfs
-    check_kata_runtime
-}
-
-# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
-initialize() {
-    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
-    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
-    
-    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
-    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
-    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
-    
-    # Load the new go and PATH parameters from the profile
-    . ${PROFILE}
-    mkdir -p "${GOPATH}"
-
-    checkout_tests_repo
-
-    pushd "${tests_repo_dir}"
-    local ci_dir_name=".ci"
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
-    # Need to change ownership of rustup so later process can create temp files there
-    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
-
-    checkout_kata_containers_repo
-
-    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
-    export INSTALL_KATA="no"
-    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
-    # Reload the profile to pick up installed dependencies
-    . ${PROFILE}
-    popd
-}
-
-checkout_tests_repo() {
-    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
-    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
-    # to be owned by root as it is re-checked out in rootfs.sh
-    mkdir -p $(dirname "${tests_repo_dir}")
-    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
-    sudo -E chown -R root:root "${tests_repo_dir}"
-    pushd "${tests_repo_dir}"
-    sudo -E git fetch
-    if [ -n "${tests_branch}" ]; then
-        sudo -E git checkout ${tests_branch}
-    fi
-    sudo -E git reset --hard origin/${tests_branch}
-    popd
-
-    source "${BATS_TEST_DIRNAME}/lib.sh"
-    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-}
-
-# Note: clone_katacontainers_repo using go, so that needs to be installed first
-checkout_kata_containers_repo() {
-    source "${tests_repo_dir}/.ci/lib.sh"
-    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
-    clone_katacontainers_repo
-    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
-}
-
-build_and_install_kata_runtime() {
-    export DEFAULT_HYPERVISOR=${KATA_HYPERVISOR}
-    ${tests_repo_dir}/.ci/install_runtime.sh
-}
-
-configure() {
-    # configure kata to use rootfs, not initrd
-    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
-
-    enable_full_debug
-    enable_agent_console
-
-    # Switch image offload to true in kata config
-    switch_image_service_offload "on"
-
-    configure_cc_containerd
-    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
-    sudo crictl config --set timeout=10
-
-    # Verity checks aren't working locally, as we aren't re-genning the hash maybe? so remove it from the kernel parameters
-    remove_kernel_param "cc_rootfs_verity.scheme"
-}
-
-build_and_add_agent_to_rootfs() {
-    build_a_custom_kata_agent
-    add_custom_agent_to_rootfs
-}
-
-build_a_custom_kata_agent() {
-    # Install libseccomp for static linking
-    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
-    export LIBSECCOMP_LINK_TYPE=static
-    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
-
-    . "$HOME/.cargo/env"
-    pushd ${katacontainers_repo_dir}/src/agent
-    sudo -E PATH=$PATH make
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
-    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
-    popd
-}
-
-create_a_local_rootfs() {
-    sudo rm -rf "${ROOTFS_DIR}"
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-    export distro="ubuntu"
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
-
-     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
-    pushd "${tests_repo_dir}"
-    sudo -E git checkout ${tests_branch}
-    popd
-    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
-    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
-
-    popd
-}
-
-add_custom_agent_to_rootfs() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
-    popd
-}
-
-build_and_install_rootfs() {
-    build_rootfs_image
-    install_rootfs_image
-}
-
-build_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
-    popd
-}
-
-install_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    local commit=$(git log --format=%h -1 HEAD)
-    local date=$(date +%Y-%m-%d-%T.%N%z)
-    local image="kata-containers-${date}-${commit}"
-    sudo install -o root -g root -m 0640 -D kata-containers.img "${PREFIX}/share/kata-containers/${image}"
-    (cd ${PREFIX}/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
-    echo "Built Rootfs from ${ROOTFS_DIR} to ${PREFIX}/share/kata-containers/${image}"
-    ls -al ${PREFIX}/share/kata-containers
-    popd
-}
-
-install_guest_kernel_image() {
-   ${tests_repo_dir}/.ci/install_kata_kernel.sh
-}
-
-build_qemu() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_qemu.sh
-}
-
-build_cloud_hypervisor() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
-}
-
-check_kata_runtime() {
-    sudo kata-runtime check
-}
-
-k8s_pod_file="${HOME}/busybox-cc.yaml"
-init_kubernetes() {
-    # Check that kubeadm was installed and install it otherwise
-    if ! [ -x "$(command -v kubeadm)" ]; then
-        pushd "${tests_repo_dir}/.ci"
-        sudo -E PATH=$PATH -s install_kubernetes.sh
-        if [ "${CRI_CONTAINERD}" == "yes" ]; then
-            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
-        fi
-        popd
-    fi
-
-    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
-    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
-    if [ -n "${cid}" ]; then
-        sudo docker stop ${cid} && sudo docker rm ${cid}
-    fi
-    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
-    if [ -n "${k8s_nodes}" ]; then
-        sudo kubeadm reset -f
-    fi
-
-    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
-    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
-    cat << EOF > ${k8s_pod_file}
-apiVersion: v1
-kind: Pod
-metadata:
-  name: busybox-cc
-spec:
-  runtimeClassName: kata
-  containers:
-  - name: nginx
-    image: quay.io/kata-containers/confidential-containers:signed
-    imagePullPolicy: Always  
-EOF
-}
-
-call_kubernetes_create_cc_pod() {
-    kubernetes_create_cc_pod ${k8s_pod_file}
-}
-
-call_kubernetes_delete_cc_pod() {
-    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_cc_pod $pod_name
-}
-
-call_kubernetes_create_ssh_demo_pod() {
-    setup_decryption_files_in_guest
-    kubernetes_create_ssh_demo_pod
-}
-
-call_connect_to_ssh_demo_pod() {
-    connect_to_ssh_demo_pod
-}
-
-call_kubernetes_delete_ssh_demo_pod() {
-    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_ssh_demo_pod $pod
-}
-
-crictl_sandbox_name=kata-cc-busybox-sandbox
-call_crictl_create_cc_pod() {
-    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
-    sudo iptables -P FORWARD ACCEPT
-    
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
-    crictl_create_cc_pod "${pod_config}"
-    sudo crictl pods
-}
-
-call_crictl_create_cc_container() {
-    # Create container configuration yaml based on our test copy of busybox
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
-    local pod_name=${crictl_sandbox_name}
-    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
-    sudo crictl ps -a
-}
-
-crictl_delete_cc() {
-    crictl_delete_cc_pod ${crictl_sandbox_name}
-}
-
-test_kata_runtime() {
-    echo "Running ctr with the kata runtime..."
-    local test_image="quay.io/kata-containers/confidential-containers:signed"
-    if [ -z $(sudo ctr images ls -q name=="${test_image}") ]; then
-        sudo ctr image pull "${test_image}"
-    fi
-    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
-}
-
-run_kata_and_capture_logs() {
-    echo "Clearing systemd journal..."
-    sudo systemctl stop systemd-journald
-    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
-    sudo systemctl start systemd-journald
-    test_kata_runtime
-    echo "Collecting logs..."
-    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
-    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
-    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
-}
-
-get_ids() {
-    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
-    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-}
-
-open_kata_shell() {
-    get_ids
-    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
-}
-
-build_bundle_dir_if_necessary() {
-    bundle_dir="/tmp/bundle"
-    if [ ! -d "${bundle_dir}" ]; then
-        rootfs_dir="$bundle_dir/rootfs"
-        image="quay.io/kata-containers/confidential-containers:signed"
-        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
-        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
-    fi
-    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
-    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
-}
-
-build_agent_ctl() {
-    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
-    if [ -e "${HOME}/.cargo/registry" ]; then
-        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
-    fi
-    sudo -E PATH=$PATH -s  make
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
-}
-
-run_agent_ctl_command() {
-    get_ids
-    build_bundle_dir_if_necessary
-    command=$1
-    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
-    if [ ! -x kata-agent-ctl ]; then
-        build_agent_ctl
-    fi 
-     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
-}
-
-agent_pull_image() {
-    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
-}
-
-agent_create_container() {
-    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
-}
-
-shim_pull_image() {
-    get_ids
-    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
-    echo "Issuing command '${ctr_shim_command}'"
-    ${ctr_shim_command}
-}
-
-call_copy_signature_files_to_guest() {
-    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
-    export DEBUG_CONSOLE="true"
-    
-    if [ "${SKOPEO:-}" = "yes" ]; then
-        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
-        setup_skopeo_signature_files_in_guest
-    else
-        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
-        setup_offline_fs_kbc_signature_files_in_guest
-    fi
-}
-
-main() {
-    while getopts "dh" opt; do
-        case "$opt" in
-            d) 
-                export DEBUG="-d"
-                set -x
-                ;;
-            h) 
-                usage 0
-                ;;
-            \?)
-                echo "Invalid option: -$OPTARG" >&2
-                usage 1
-                ;;
-        esac
-    done
-
-    shift $((OPTIND - 1))
-
-    subcmd="${1:-}"
-
-    [ -z "${subcmd}" ] && usage 1
-
-    case "${subcmd}" in
-        all)
-            build_and_install_all
-            run_kata_and_capture_logs
-            ;;
-        build_and_install_all)
-            build_and_install_all
-            ;;
-        rebuild_and_install_kata)
-            rebuild_and_install_kata
-            ;;
-        initialize)
-            initialize
-            ;;
-        build_kata_runtime)
-            build_and_install_kata_runtime
-            ;;
-        configure)
-            configure
-            ;;
-        create_rootfs)
-            create_a_local_rootfs
-            ;;
-        build_and_add_agent_to_rootfs)
-            build_and_add_agent_to_rootfs
-            ;;
-        build_and_install_rootfs)
-            build_and_install_rootfs
-            ;;
-        install_guest_kernel)
-            install_guest_kernel_image
-            ;;
-        build_cloud_hypervisor)
-            build_cloud_hypervisor
-            ;;
-        build_qemu)
-            build_qemu
-            ;;
-        init_kubernetes)
-            init_kubernetes
-            ;;
-        crictl_create_cc_pod)
-            call_crictl_create_cc_pod
-            ;;
-        crictl_create_cc_container)
-            call_crictl_create_cc_container
-            ;;
-        crictl_delete_cc)
-            crictl_delete_cc
-            ;;
-        kubernetes_create_cc_pod)
-            call_kubernetes_create_cc_pod
-            ;;
-        kubernetes_delete_cc_pod)
-            call_kubernetes_delete_cc_pod
-            ;;
-        kubernetes_create_ssh_demo_pod)
-            call_kubernetes_create_ssh_demo_pod
-            ;;
-        connect_to_ssh_demo_pod)
-            call_connect_to_ssh_demo_pod
-            ;;
-        kubernetes_delete_ssh_demo_pod)
-            call_kubernetes_delete_ssh_demo_pod
-            ;;
-        test)
-            test_kata_runtime
-            ;;
-        test_capture_logs)
-            run_kata_and_capture_logs
-            ;;
-        open_kata_console)
-            open_kata_console
-            ;;
-        open_kata_shell)
-            open_kata_shell
-            ;;
-        agent_pull_image)
-            agent_pull_image
-            ;;
-        shim_pull_image)
-            shim_pull_image
-            ;;
-        agent_create_container)
-            agent_create_container
-            ;;
-        copy_signature_files_to_guest)
-            call_copy_signature_files_to_guest
-            ;;
-        *)
-            usage 1
-            ;;
-    esac
-}
-
-main $@

docs/how-to/data/confidential-agent-config.toml.in

@@ -1,45 +0,0 @@
-# Copyright (c) 2021 IBM Corp.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-aa_kbc_params = "$AA_KBC_PARAMS"
-https_proxy = "$HTTPS_PROXY"
-[endpoints]
-allowed = [
-"AddARPNeighborsRequest",
-"AddSwapRequest",
-"CloseStdinRequest",
-"CopyFileRequest",
-"CreateContainerRequest",
-"CreateSandboxRequest",
-"DestroySandboxRequest",
-#"ExecProcessRequest",
-"GetMetricsRequest",
-"GetOOMEventRequest",
-"GuestDetailsRequest",
-"ListInterfacesRequest",
-"ListRoutesRequest",
-"MemHotplugByProbeRequest",
-"OnlineCPUMemRequest",
-"PauseContainerRequest",
-"PullImageRequest",
-"ReadStreamRequest",
-"RemoveContainerRequest",
-#"ReseedRandomDevRequest",
-"ResizeVolumeRequest",
-"ResumeContainerRequest",
-"SetGuestDateTimeRequest",
-"SignalProcessRequest",
-"StartContainerRequest",
-"StartTracingRequest",
-"StatsContainerRequest",
-"StopTracingRequest",
-"TtyWinResizeRequest",
-"UpdateContainerRequest",
-"UpdateInterfaceRequest",
-"UpdateRoutesRequest",
-"VolumeStatsRequest",
-"WaitProcessRequest",
-"WriteStreamRequest"
-]

docs/how-to/how-to-build-and-test-ccv0.md

@@ -1,475 +0,0 @@
-# How to build, run and test Kata CCv0
-
-## Introduction and Background
-
-In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
-shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
-[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
-different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
-Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
-scripts in order to have less duplicated code.
-As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
-we have working.
-
-*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
-*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
-*fully supported process by the Kata Containers community.*
-
-### Basic script set-up and optional environment variables
-
-In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
-- Provision a new VM
-    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
-    *packages, so these will need re-working to be compatible with other platforms.*
-- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
-```bash
-$ chmod u+x ccv0.sh
-```
-- Optionally set up some environment variables
-    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
-      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
-      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
-      and `tests_branch` variables e.g.
-      ```bash
-      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
-      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
-      $ export tests_repo=github.com/stevenhorsman/tests
-      $ export tests_branch=stevenh/add-ccv0-changes-to-build
-      ```
-      before running the script.
-    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
-      set:
-      ```
-      $ export KATA_HYPERVISOR="cloud-hypervisor"
-      ```
-      before running the build.
-
-- At this point you can provision a Kata confidential containers pod and container with either
-  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
-  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-  and then test and use it. 
-
-### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
-  ```bash
-  $ export KUBERNETES="no"
-  $ export KATA_HYPERVISOR="qemu"
-  $ ~/ccv0.sh -d build_and_install_all
-  ```
-    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
-    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
-      *of the process without having to reload the session.*
-    - The steps that `build_and_install_all` takes is:
-        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
-        (default to `CCv0` branches if they are not supplied)
-        - Use the `tests/.ci` scripts to install the build dependencies
-        - Build and install the Kata runtime
-        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
-          enabling console access to the Kata guest shell, which should only be done in development)
-        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
-        20.04.
-        - Build the Kata guest kernel
-        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
-        variable can be used to select between `qemu` or `cloud-hypervisor`)
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull ubuntu
-  >  ```
-  >  then re-run the command.
-    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
-      installed and they can be further cut down by not running all the above steps 
-      [see "Additional script usage" below](#additional-script-usage)
-   
-- Create a new Kata sandbox pod using `crictl` with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_pod
-  ```
-    - This creates a pod configuration file, creates the pod from this using 
-    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
-- Create a new Kata confidential container with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
-      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
-
-As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the `crictl` pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh crictl_delete_cc
-  ```
-### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM
-  as a single node cluster: 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a Kata confidential containers pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_cc_pod
-  ```
-- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
-  ```bash
-  $ kubectl get pods
-  ```
-  which should show something like:
-  ```text
-  NAME         READY   STATUS    RESTARTS   AGE
-  busybox-cc   1/1     Running   0          54s
-  ```
-
-- As this point you should have a Kubernetes pod and container running, that is using the Kata 
-confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the Kubernetes pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_cc_pod
-  ```
-
-### Validate that the container image was pulled on the guest
-
-There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
-the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
-there.
-- To check the guest's file system:
-    - Open a shell into the Kata guest with:
-      ```bash
-      $ ~/ccv0.sh open_kata_shell
-      ```
-    - List the files in the directory that the container image bundle should have been unpacked to with:
-      ```bash
-      $ ls -ltr /run/kata-containers/confidential-containers_signed/
-      ```
-    - This should give something like
-        ```
-        total 72
-        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
-        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
-        ```
-        which shows how the image has been pulled and then unbundled on the guest.
-    - Leave the Kata guest shell by running:
-      ```bash
-      $ exit
-      ```
-- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
-  should only see a single bundle for the pause container as the `busybox` based container image should have been
-  pulled on the guest:
-    - Find all the `rootfs` directories under in the pod's shared directory with:
-      ```bash
-      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
-      ```
-      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
-    - Looking that `rootfs` directory with
-      ```bash
-      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
-      ```
-      shows something similar to
-      ```
-      total 668
-      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
-      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
-      ```
-      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
-
-### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
-
-Once you have a kata pod sandbox created as described above, either using 
-[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-, you can use this to test specific components of the Kata confidential
-containers architecture. This can be useful for development and debugging to isolate and test features
-that aren't broadly supported end-to-end. Here are some examples:
-
-- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
-This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
-interact with the shim directly. The command takes the form
-`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
-the `ccv0.sh` script to automatically fill in the variables:
-  - Optionally, set up some environment variables to set the image and credentials used:
-    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
-     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
-      image, set the `PULL_IMAGE` environment variable e.g. 
-      ```bash
-      $ export PULL_IMAGE="docker.io/library/busybox:latest"
-      ```
-      Currently the containerd shim pull image
-      code doesn't support using a container registry that requires authentication, so if this is required, see the 
-      below steps to run the pull image command against the agent directly.
-  - Run the pull image agent endpoint with:
-    ```bash
-    $ ~/ccv0.sh shim_pull_image
-    ```
-    which we print the `ctr shim` command for reference
-- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
-  credentials in order to pull from an authenticated registry:
-    - Optionally set up some environment variables to set the image and credentials used:
-        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
-          if a specific container image is required.
-        - If the container registry for the image requires authentication then this can be set with an environment
-          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
-          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
-            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
-              proof of concept to allow more images to be pulled and tested. Once we have support for getting
-              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
-              credentials to be looked up using that mechanism.
-    - Run the pull image agent endpoint with
-      ```bash
-      $ ~/ccv0.sh agent_pull_image
-      ```
-      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
-      that the `PullImage` request was successful e.g.
-      ```
-      Finished release [optimized] target(s) in 0.21s
-      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
-      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
-      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
-      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
-      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
-      ```
-      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
-      which may take a few minutes.
-- To validate that the image pull was successful, you can open a shell into the Kata guest with:
-  ```bash
-  $ ~/ccv0.sh open_kata_shell
-  ```
-- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
-  named either `01234556789` (for the container id), or the container image name, e.g.
-  ```bash
-  $ ls -ltr /run/kata-containers/confidential-containers_signed/
-  ```
-  which should show something like
-  ```
-  total 72
-  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
-  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
-  ```
-- Leave the Kata shell by running:
-  ```bash
-  $ exit
-  ```
-
-## Verifying signed images
-
-For this sample demo, we use local attestation to pass through the required
-configuration to do container image signature verification. Due to this, the ability to verify images is limited
-to a pre-created selection of test images in our test
-repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
-For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
-of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
-or remote attestation will be available in future.
-
-In our test repository there are three tagged images:
-
-| Test Image | Base Image used | Signature status | GPG key status |
-| --- | --- | --- | --- |
-| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/public.gpg) embedded in kata rootfs |
-| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
-| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/x86_64/signatures.tar) embedded in kata rootfs | GPG key not kept |
-
-Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
-test a few scenarios.
-
-In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
-configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
-into the guest image which specifies that any container image from `quay.io/kata-containers`
-must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
-With this policy set a few tests of image verification can be done to test different scenarios by attempting
-to create containers from these images using `crictl`:
-
-- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
-[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-up to the `~/ccv0.sh crictl_create_cc_pod` command.
-
-- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
-needed by running 
-`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
-your pod - adding in the new files.
- 
-- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
-image by running:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes showing the running container e.g.
-  ```text
-  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
-  ```
-- To test that an unsigned image from our *protected* test container registry is rejected we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This correctly results in an error message from `crictl`:
-  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
-- To test that the signed image our *protected* test container registry is accepted we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
-  ```text
-  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
-  ...
-  ```
-- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
-want to protect with the attestation agent in future) fails we can run: 
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - Again this results in an error message from `crictl`:
-  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
-
-### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
-
-The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
-demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
-[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
-To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
-hardware as well.
-
-If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
-steps:
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ export AA_KBC=offline_fs_kbc
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
-      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
-      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
-      and an agent configuration file
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a sample Kata confidential containers ssh pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
-  ```
-- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
-the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
-in the rootfs.
-During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
-it ever being available to the host.
-
-- To validate that the container is working you, can connect to the image via SSH by running:
-  ```bash
-  $ ~/ccv0.sh connect_to_ssh_demo_pod
-  ```
-  - During this connection the host key fingerprint is shown and should match:
-    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
-  - After you are finished connecting then run:
-    ```bash
-    $ exit
-    ```
-  
-- To delete the sample SSH demo pod run:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
-  ```
-
-## Additional script usage
-
-As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
-re-build bits of it by running the script with different parameters. For example after the first build you will often
-not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
-runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
-*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
-`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
-There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
-`~/ccv0.sh help`:
-```
-$ ~/ccv0.sh help
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
-Usage:
-    ccv0.sh [options] <command>
-Commands:
-- help:                         Display this help
-- all:                          Build and install everything, test kata with containerd and capture the logs
-- build_and_install_all:        Build and install everything
-- initialize:                   Install dependencies and check out kata-containers source
-- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
-- build_kata_runtime:           Build and install the kata runtime
-- configure:                    Configure Kata to use rootfs and enable debug
-- create_rootfs:                Create a local rootfs
-- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
-- build_and_install_rootfs:     Builds and installs the rootfs image
-- install_guest_kernel:         Setup, build and install the guest kernel
-- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                   Checkout, patch, build and install QEMU
-- init_kubernetes:              initialize a Kubernetes cluster on this system
-- crictl_create_cc_pod          Use crictl to create a new kata cc pod
-- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
-- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
-- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
-- open_kata_shell:              Open a shell into the kata runtime
-- agent_pull_image:             Run PullImage command against the agent with agent-ctl
-- shim_pull_image:              Run PullImage command against the shim with ctr
-- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
-- test:                         Test using kata with containerd
-- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -1,44 +0,0 @@
-# Generating a Kata Containers payload for the Confidential Containers Operator
-
-[Confidential Containers
-Operator](https://github.com/confidential-containers/operator) consumes a Kata
-Containers payload, generated from the `CCv0` branch, and here one can find all
-the necessary info on how to build such a payload.
-
-## Requirements
-
-* `make` installed in the machine
-* Docker installed in the machine
-* `sudo` access to the machine
-
-## Process
-
-* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
-  ```sh
-  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
-  ```
-  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
-    ```sh
-    git checkout CCv0
-    ```
-  * Ensure your tree is clean and in sync with upstream `CCv0`
-    ```sh
-    git clean -xfd
-    git reset --hard <upstream>/CCv0
-    ```
-* Make sure you're authenticated to `quay.io`
-    ```sh
-    sudo docker login quay.io
-    ```
-* From the top repo directory, run:
-  ```sh
-  sudo make cc-payload
-  ```
-* Make sure the image was upload to the [Confidential Containers
-  runtime-payload
-registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
-
-## Notes
-
-Make sure to run it on a machine that's not the one you're hacking on, prepare a
-cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -28,10 +28,10 @@ __Steps from the Developer Guide:__
 __SNP-specific steps:__
 - Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
 ```bash
-$ pushd kata-containers/tools/packaging/kernel/
-$ ./build-kernel.sh -a x86_64 -x snp setup
-$ ./build-kernel.sh -a x86_64 -x snp build
-$ sudo -E PATH="${PATH}" ./build-kernel.sh -x snp install
+$ pushd kata-containers/tools/packaging/
+$ ./kernel/build-kernel.sh -a x86_64 -x snp setup
+$ ./kernel/build-kernel.sh -a x86_64 -x snp build
+$ sudo -E PATH="${PATH}" ./kernel/build-kernel.sh -x snp install
 $ popd
 ```
 - Build a current OVMF capable of SEV-SNP:

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -27,8 +27,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.runtime.internetworking_model` | string| determines how the VM should be connected to the container network interface. Valid values are `macvtap`, `tcfilter` and `none` |
 | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
 | `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
-| `io.katacontainers.config.runtime.image_request_timeout` | `uint64` | the timeout for pulling an image within the guest in `seconds`, default is `60` |
-| `io.katacontainers.config.runtime.sealed_secret_enabled` | `boolean` | enables the sealed secret feature, default is `false` |
 
 ## Agent Options
 | Key | Value Type | Comments |
@@ -96,16 +94,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
 | `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
-## Confidential Computing Options
-| Key | Value Type | Comments |
-|-------| ----- | ----- |
-| `io.katacontainers.config.pre_attestation.enabled"` | `bool` |
-determines if SEV/-ES attestation is enabled |
-| `io.katacontainers.config.pre_attestation.uri"` | `string` |
-specify the location of the attestation server |
-| `io.katacontainers.config.sev.policy"` | `uint32` |
-specify the SEV guest policy |
-
 ## Container Options
 | Key | Value Type | Comments |
 |-------| ----- | ----- |

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -29,7 +29,7 @@ Then you can build and install the guest kernel image as shown [here](../../tool
 
 ## Run a Kata Container utilizing `virtio-mem`
 
-Use following command to enable memory overcommitment of a Linux kernel.  Because QEMU `virtio-mem` device need to allocate a lot of memory.
+Use following command to enable memory over-commitment of a Linux kernel.  Because QEMU `virtio-mem` device need to allocate a lot of memory.
 ```
 $ echo 1 | sudo tee /proc/sys/vm/overcommit_memory
 ```

src/agent/Cargo.toml

@@ -8,7 +8,7 @@ license = "Apache-2.0"
 [dependencies]
 oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
-protocols = { path = "../libs/protocols", features = ["async"] }
+protocols = { path = "../libs/protocols", features = ["async", "with-serde"] }
 lazy_static = "1.3.0"
 ttrpc = { version = "0.7.1", features = ["async"], default-features = false }
 protobuf = "3.2.0"
@@ -23,20 +23,17 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
-const_format = "0.2.30"
-url = "2.2.2"
-derivative = "2.2.0"
 
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.28"
+futures = "0.3.17"
 
 # Async runtime
 tokio = { version = "1.28.1", features = ["full"] }
 tokio-vsock = "0.3.1"
 
-netlink-sys = { version = "0.7.0", features = ["tokio_socket"] }
+netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
 rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
@@ -62,7 +59,7 @@ cgroups = { package = "cgroups-rs", version = "0.3.2" }
 tracing = "0.1.26"
 tracing-subscriber = "0.2.18"
 tracing-opentelemetry = "0.13.0"
-opentelemetry = { version = "0.14.0", features = ["rt-tokio-current-thread"] }
+opentelemetry = { version = "0.14.0", features = ["rt-tokio-current-thread"]}
 vsock-exporter = { path = "vsock-exporter" }
 
 # Configuration
@@ -70,18 +67,11 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
-# "vendored" feature for openssl is required by musl build
-openssl = { version = "0.10.38", features = ["vendored"] }
-
-# Image pull/decrypt
-image-rs = { git = "https://github.com/confidential-containers/guest-components", tag = "v0.8.0", default-features = false, features = [
-    "kata-cc-native-tls",
-    "verity",
-    "signature-simple-xrss",
-] }
-
-[patch.crates-io]
-oci-distribution = { git = "https://github.com/krustlet/oci-distribution.git", rev = "f44124c" }
+# Communication with the OPA service
+http = { version = "0.2.8", optional = true }
+reqwest = { version = "0.11.14", optional = true }
+# The "vendored" feature for openssl is required for musl build
+openssl = { version = "0.10.54", features = ["vendored"], optional = true }
 
 [dev-dependencies]
 tempfile = "3.1.0"
@@ -89,17 +79,17 @@ test-utils = { path = "../libs/test-utils" }
 which = "4.3.0"
 
 [workspace]
-resolver = "2"
-members = ["rustjail"]
+members = [
+    "rustjail",
+]
 
 [profile.release]
 lto = true
 
 [features]
-confidential-data-hub = []
 seccomp = ["rustjail/seccomp"]
-sealed-secret = ["protocols/sealed-secret", "confidential-data-hub"]
 standard-oci-runtime = ["rustjail/standard-oci-runtime"]
+agent-policy = ["http", "openssl", "reqwest"]
 
 [[bin]]
 name = "kata-agent"

src/agent/Makefile

@@ -33,11 +33,12 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
-SEALED_SECRET ?= no
+##VAR AGENT_POLICY=yes|no define if agent enables the policy feature
+AGENT_POLICY ?= no
 
-# Enable sealed-secret feature of rust build
-ifeq ($(SEALED_SECRET),yes)
-    override EXTRA_RUSTFEATURES += sealed-secret
+# Enable the policy feature of rust build
+ifeq ($(AGENT_POLICY),yes)
+    override EXTRA_RUSTFEATURES += agent-policy
 endif
 
 include ../../utils.mk
@@ -61,7 +62,7 @@ endif
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
-DESTDIR :=
+DESTDIR ?=
 ##VAR BINDIR=<path> is a directory for installing executable programs
 BINDIR := /usr/bin
 
@@ -147,7 +148,7 @@ vendor:
 
 
 #TARGET test: run cargo tests
-test:
+test: $(GENERATED_FILES)
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -541,11 +541,8 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match &d.r#type {
-        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
-            Some(t_c) => t_c,
-            None => return None,
-        },
+    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
+        Some(t) => t,
         None => return None,
     };
 
@@ -602,7 +599,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -611,7 +608,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("b".to_string()),
+                r#type: "b".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -620,7 +617,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -629,7 +626,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -638,7 +635,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -647,7 +644,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),

src/agent/rustjail/src/container.rs

@@ -80,6 +80,7 @@ const CLOG_FD: &str = "CLOG_FD";
 const FIFO_FD: &str = "FIFO_FD";
 const HOME_ENV_KEY: &str = "HOME";
 const PIDNS_FD: &str = "PIDNS_FD";
+const PIDNS_ENABLED: &str = "PIDNS_ENABLED";
 const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD";
 
 #[derive(Debug)]
@@ -280,6 +281,17 @@ pub struct SyncPc {
     pid: pid_t,
 }
 
+#[derive(Debug, Clone)]
+pub struct PidNs {
+    enabled: bool,
+    fd: Option<i32>,
+}
+impl PidNs {
+    pub fn new(enabled: bool, fd: Option<i32>) -> Self {
+        Self { enabled, fd }
+    }
+}
+
 pub trait Container: BaseContainer {
     fn pause(&mut self) -> Result<()>;
     fn resume(&mut self) -> Result<()>;
@@ -339,16 +351,20 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let crfd = std::env::var(CRFD_FD)?.parse::<i32>().unwrap();
     let cfd_log = std::env::var(CLOG_FD)?.parse::<i32>().unwrap();
 
-    // get the pidns fd from parent, if parent had passed the pidns fd,
-    // then get it and join in this pidns; otherwise, create a new pidns
-    // by unshare from the parent pidns.
-    match std::env::var(PIDNS_FD) {
-        Ok(fd) => {
-            let pidns_fd = fd.parse::<i32>().context("get parent pidns fd")?;
-            sched::setns(pidns_fd, CloneFlags::CLONE_NEWPID).context("failed to join pidns")?;
-            let _ = unistd::close(pidns_fd);
+    if std::env::var(PIDNS_ENABLED)?.eq(format!("{}", true).as_str()) {
+        // get the pidns fd from parent, if parent had passed the pidns fd,
+        // then get it and join in this pidns; otherwise, create a new pidns
+        // by unshare from the parent pidns.
+        match std::env::var(PIDNS_FD) {
+            Ok(fd) => {
+                let pidns_fd = fd.parse::<i32>().context("get parent pidns fd")?;
+                sched::setns(pidns_fd, CloneFlags::CLONE_NEWPID).context("failed to join pidns")?;
+                let _ = unistd::close(pidns_fd);
+            }
+            Err(_e) => {
+                sched::unshare(CloneFlags::CLONE_NEWPID)?;
+            }
         }
-        Err(_e) => sched::unshare(CloneFlags::CLONE_NEWPID)?,
     }
 
     match unsafe { fork() } {
@@ -983,9 +999,13 @@ impl BaseContainer for LinuxContainer {
         }
 
         let pidns = get_pid_namespace(&self.logger, linux)?;
+        #[cfg(not(feature = "standard-oci-runtime"))]
+        if !pidns.enabled {
+            return Err(anyhow!("cannot find the pid ns"));
+        }
 
-        defer!(if let Some(pid) = pidns {
-            let _ = unistd::close(pid);
+        defer!(if let Some(fd) = pidns.fd {
+            let _ = unistd::close(fd);
         });
 
         let exec_path = std::env::current_exe()?;
@@ -1008,14 +1028,15 @@ impl BaseContainer for LinuxContainer {
             .env(CRFD_FD, format!("{}", crfd))
             .env(CWFD_FD, format!("{}", cwfd))
             .env(CLOG_FD, format!("{}", cfd_log))
-            .env(CONSOLE_SOCKET_FD, console_name);
+            .env(CONSOLE_SOCKET_FD, console_name)
+            .env(PIDNS_ENABLED, format!("{}", pidns.enabled));
 
         if p.init {
             child = child.env(FIFO_FD, format!("{}", fifofd));
         }
 
-        if pidns.is_some() {
-            child = child.env(PIDNS_FD, format!("{}", pidns.unwrap()));
+        if pidns.fd.is_some() {
+            child = child.env(PIDNS_FD, format!("{}", pidns.fd.unwrap()));
         }
 
         child.spawn()?;
@@ -1249,11 +1270,11 @@ pub fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> R
     Ok(())
 }
 
-fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<Option<RawFd>> {
+fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<PidNs> {
     for ns in &linux.namespaces {
         if ns.r#type == "pid" {
             if ns.path.is_empty() {
-                return Ok(None);
+                return Ok(PidNs::new(true, None));
             }
 
             let fd =
@@ -1269,11 +1290,11 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result<Option<RawFd>> {
                     e
                 })?;
 
-            return Ok(Some(fd));
+            return Ok(PidNs::new(true, Some(fd)));
         }
     }
 
-    Err(anyhow!("cannot find the pid ns"))
+    Ok(PidNs::new(false, None))
 }
 
 fn is_userns_enabled(linux: &Linux) -> bool {

src/agent/rustjail/src/lib.rs

@@ -241,12 +241,6 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
-            let dev_type = if dev.Type.is_empty() {
-                None
-            } else {
-                Some(dev.Type.clone())
-            };
-
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -260,7 +254,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev_type,
+                r#type: dev.Type.clone(),
                 major,
                 minor,
                 access: dev.Access.clone(),
@@ -429,12 +423,18 @@ fn linux_grpc_to_oci(l: &grpc::Linux) -> oci::Linux {
         let mut r = Vec::new();
 
         for d in l.Devices.iter() {
+            // if the filemode for the device is 0 (unset), use a default value as runc does
+            let filemode = if d.FileMode != 0 {
+                Some(d.FileMode)
+            } else {
+                Some(0o666)
+            };
             r.push(oci::LinuxDevice {
                 path: d.Path.clone(),
                 r#type: d.Type.clone(),
                 major: d.Major,
                 minor: d.Minor,
-                file_mode: Some(d.FileMode),
+                file_mode: filemode,
                 uid: Some(d.UID),
                 gid: Some(d.GID),
             });

src/agent/src/cdh.rs

@@ -1,289 +0,0 @@
-// Copyright (c) 2023 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-// Confidential Data Hub client wrapper.
-// Confidential Data Hub is a service running inside guest to provide resource related APIs.
-// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
-
-use anyhow::{anyhow, Result};
-use oci::{Mount, Spec};
-use protocols::{
-    sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient,
-};
-use std::fs;
-use std::os::unix::fs::symlink;
-use std::path::Path;
-const CDH_ADDR: &str = "unix:///run/confidential-containers/cdh.sock";
-const SECRETS_DIR: &str = "/run/secrets/";
-const SEALED_SECRET_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000;
-
-// Convenience function to obtain the scope logger.
-fn sl() -> slog::Logger {
-    slog_scope::logger()
-}
-
-#[derive(Clone)]
-pub struct CDHClient {
-    sealed_secret_client: Option<SealedSecretServiceClient>,
-}
-
-impl CDHClient {
-    pub fn new() -> Result<Self> {
-        let c = ttrpc::asynchronous::Client::connect(CDH_ADDR);
-        match c {
-            Ok(v) => {
-                let ssclient = sealed_secret_ttrpc_async::SealedSecretServiceClient::new(v);
-                Ok(CDHClient {
-                    sealed_secret_client: Some(ssclient),
-                })
-            }
-            Err(_) => Ok(CDHClient {
-                sealed_secret_client: None,
-            }),
-        }
-    }
-
-    pub async fn unseal_secret_async(
-        &self,
-        sealed: &str,
-    ) -> Result<sealed_secret::UnsealSecretOutput> {
-        let secret = sealed
-            .strip_prefix("sealed.")
-            .ok_or(anyhow!("strip_prefix \"sealed.\" failed"))?;
-        let mut input = sealed_secret::UnsealSecretInput::new();
-        input.set_secret(secret.into());
-        let unseal = self
-            .sealed_secret_client
-            .as_ref()
-            .ok_or(anyhow!("unwrap sealed_secret_client failed"))?
-            .unseal_secret(ttrpc::context::with_timeout(SEALED_SECRET_TIMEOUT), &input)
-            .await?;
-        Ok(unseal)
-    }
-
-    pub async fn unseal_env(&self, env: &str) -> Result<String> {
-        let (key, value) = env.split_once('=').unwrap_or(("", ""));
-        if value.starts_with("sealed.") {
-            let unsealed_value = self.unseal_secret_async(value).await;
-            match unsealed_value {
-                Ok(v) => {
-                    let plain_env = format!("{}={}", key, std::str::from_utf8(&v.plaintext)?);
-                    return Ok(plain_env);
-                }
-                Err(e) => {
-                    return Err(e);
-                }
-            };
-        }
-        Ok((*env.to_owned()).to_string())
-    }
-
-    pub async fn unseal_file(&self, sealed_source_path: &String) -> Result<()> {
-        if !Path::new(sealed_source_path).exists() {
-            info!(
-                sl(),
-                "sealed source path {:?} does not exist", sealed_source_path
-            );
-            return Ok(());
-        }
-
-        for entry in fs::read_dir(sealed_source_path)? {
-            let entry = entry?;
-
-            if !entry.file_type()?.is_symlink()
-                && !fs::metadata(entry.path())?.file_type().is_file()
-            {
-                info!(
-                    sl(),
-                    "skipping sealed source entry {:?} because its file type is {:?}",
-                    entry,
-                    entry.file_type()?
-                );
-                continue;
-            }
-
-            let target_path = fs::canonicalize(&entry.path())?;
-            info!(sl(), "sealed source entry target path: {:?}", target_path);
-            if !target_path.is_file() {
-                info!(sl(), "sealed source is not a file: {:?}", target_path);
-                continue;
-            }
-
-            let secret_name = entry.file_name();
-            let contents = fs::read_to_string(&target_path)?;
-            if contents.starts_with("sealed.") {
-                info!(sl(), "sealed source entry found: {:?}", target_path);
-                let unsealed_filename = SECRETS_DIR.to_string()
-                    + secret_name
-                        .as_os_str()
-                        .to_str()
-                        .ok_or(anyhow!("create unsealed_filename failed"))?;
-                let unsealed_value = self.unseal_secret_async(&contents).await?;
-                fs::write(&unsealed_filename, unsealed_value.plaintext)?;
-                fs::remove_file(&entry.path())?;
-                symlink(unsealed_filename, &entry.path())?;
-            }
-        }
-        Ok(())
-    }
-
-    pub fn create_sealed_secret_mounts(&self, spec: &mut Spec) -> Result<Vec<String>> {
-        let mut sealed_source_path: Vec<String> = vec![];
-        for m in spec.mounts.iter_mut() {
-            if let Some(unsealed_mount_point) = m.destination.strip_prefix("/sealed") {
-                info!(
-                    sl(),
-                    "sealed mount destination: {:?} source: {:?}", m.destination, m.source
-                );
-                sealed_source_path.push(m.source.clone());
-                m.destination = unsealed_mount_point.to_string();
-            }
-        }
-
-        if !sealed_source_path.is_empty() {
-            let sealed_mounts = Mount {
-                destination: SECRETS_DIR.to_string(),
-                r#type: "bind".to_string(),
-                source: SECRETS_DIR.to_string(),
-                options: vec!["bind".to_string()],
-            };
-            spec.mounts.push(sealed_mounts);
-        }
-        fs::create_dir_all(SECRETS_DIR)?;
-        Ok(sealed_source_path)
-    }
-} /* end of impl CDHClient */
-
-#[cfg(test)]
-#[cfg(feature = "sealed-secret")]
-mod tests {
-    use crate::cdh::CDHClient;
-    use crate::cdh::CDH_ADDR;
-    use crate::cdh::SECRETS_DIR;
-    use anyhow::anyhow;
-    use async_trait::async_trait;
-    use protocols::{sealed_secret, sealed_secret_ttrpc_async};
-    use std::fs;
-    use std::fs::File;
-    use std::io::{Read, Write};
-    use std::path::Path;
-    use std::sync::Arc;
-    use tokio::signal::unix::{signal, SignalKind};
-
-    struct TestService;
-
-    #[async_trait]
-    impl sealed_secret_ttrpc_async::SealedSecretService for TestService {
-        async fn unseal_secret(
-            &self,
-            _ctx: &::ttrpc::asynchronous::TtrpcContext,
-            _req: sealed_secret::UnsealSecretInput,
-        ) -> ttrpc::error::Result<sealed_secret::UnsealSecretOutput> {
-            let mut output = sealed_secret::UnsealSecretOutput::new();
-            output.set_plaintext("unsealed".into());
-            Ok(output)
-        }
-    }
-
-    fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> {
-        let path = sock_addr
-            .strip_prefix("unix://")
-            .expect("socket address does not have the expected format.");
-
-        if std::path::Path::new(path).exists() {
-            std::fs::remove_file(path)?;
-        }
-
-        Ok(())
-    }
-
-    fn start_ttrpc_server() {
-        tokio::spawn(async move {
-            let ss = Box::new(TestService {})
-                as Box<dyn sealed_secret_ttrpc_async::SealedSecretService + Send + Sync>;
-            let ss = Arc::new(ss);
-            let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss);
-
-            remove_if_sock_exist(CDH_ADDR).unwrap();
-
-            let mut server = ttrpc::asynchronous::Server::new()
-                .bind(CDH_ADDR)
-                .unwrap()
-                .register_service(ss_service);
-
-            server.start().await.unwrap();
-
-            let mut interrupt = signal(SignalKind::interrupt()).unwrap();
-            tokio::select! {
-                _ = interrupt.recv() => {
-                    server.shutdown().await.unwrap();
-                }
-            };
-        });
-    }
-
-    #[tokio::test]
-    async fn test_unseal_env() {
-        let rt = tokio::runtime::Runtime::new().unwrap();
-        let _guard = rt.enter();
-        start_ttrpc_server();
-        std::thread::sleep(std::time::Duration::from_secs(2));
-
-        let cc = Some(CDHClient::new().unwrap());
-        let cdh_client = cc
-            .as_ref()
-            .ok_or(anyhow!("get confidential-data-hub client failed"))
-            .unwrap();
-        let sealed_env = String::from("key=sealed.testdata");
-        let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
-        assert_eq!(unsealed_env, String::from("key=unsealed"));
-        let normal_env = String::from("key=testdata");
-        let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
-        assert_eq!(unchanged_env, String::from("key=testdata"));
-
-        rt.shutdown_background();
-        std::thread::sleep(std::time::Duration::from_secs(2));
-    }
-
-    #[tokio::test]
-    async fn test_unseal_file() {
-        let rt = tokio::runtime::Runtime::new().unwrap();
-        let _guard = rt.enter();
-        start_ttrpc_server();
-        std::thread::sleep(std::time::Duration::from_secs(2));
-
-        let cc = Some(CDHClient::new().unwrap());
-        let cdh_client = cc
-            .as_ref()
-            .ok_or(anyhow!("get confidential-data-hub client failed"))
-            .unwrap();
-
-        fs::create_dir_all(SECRETS_DIR).unwrap();
-
-        let sealed_filename = "passwd";
-        let mut sealed_file = File::create(sealed_filename).unwrap();
-        let dir = String::from(".");
-        sealed_file.write_all(b"sealed.passwd").unwrap();
-        cdh_client.unseal_file(&dir).await.unwrap();
-        let unsealed_filename = SECRETS_DIR.to_string() + "/passwd";
-        let mut unsealed_file = fs::File::open(unsealed_filename.clone()).unwrap();
-        let mut contents = String::new();
-        unsealed_file.read_to_string(&mut contents).unwrap();
-        assert_eq!(contents, String::from("unsealed"));
-        fs::remove_file(sealed_filename).unwrap();
-        fs::remove_file(unsealed_filename).unwrap();
-
-        let normal_filename = "passwd";
-        let mut normal_file = File::create(normal_filename).unwrap();
-        normal_file.write_all(b"passwd").unwrap();
-        cdh_client.unseal_file(&dir).await.unwrap();
-        let filename = SECRETS_DIR.to_string() + "/passwd";
-        assert!(!Path::new(&filename).exists());
-        fs::remove_file(normal_filename).unwrap();
-
-        rt.shutdown_background();
-        std::thread::sleep(std::time::Duration::from_secs(2));
-    }
-}

src/agent/src/config.rs

@@ -5,13 +5,11 @@
 use crate::rpc;
 use anyhow::{bail, ensure, Context, Result};
 use serde::Deserialize;
-use std::collections::HashSet;
 use std::env;
 use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
-use url::Url;
 
 use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
@@ -26,15 +24,6 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
-const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
-const REST_API_OPTION: &str = "agent.rest_api";
-const HTTPS_PROXY: &str = "agent.https_proxy";
-const NO_PROXY: &str = "agent.no_proxy";
-const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
-const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
-const IMAGE_POLICY_FILE: &str = "agent.image_policy";
-const IMAGE_REGISTRY_AUTH_FILE: &str = "agent.image_registry_auth";
-const SIMPLE_SIGNING_SIGSTORE_CONFIG: &str = "agent.simple_signing_sigstore_config";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
@@ -62,17 +51,6 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
-#[derive(Debug, Default, Deserialize)]
-pub struct EndpointsConfig {
-    pub allowed: Vec<String>,
-}
-
-#[derive(Debug, Default)]
-pub struct AgentEndpoints {
-    pub allowed: HashSet<String>,
-    pub all_allowed: bool,
-}
-
 #[derive(Debug)]
 pub struct AgentConfig {
     pub debug_console: bool,
@@ -85,18 +63,7 @@ pub struct AgentConfig {
     pub server_addr: String,
     pub unified_cgroup_hierarchy: bool,
     pub tracing: bool,
-    pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
-    pub container_policy_path: String,
-    pub aa_kbc_params: String,
-    pub rest_api: String,
-    pub https_proxy: String,
-    pub no_proxy: String,
-    pub data_integrity: bool,
-    pub enable_signature_verification: bool,
-    pub image_policy_file: String,
-    pub image_registry_auth_file: String,
-    pub simple_signing_sigstore_config: String,
 }
 
 #[derive(Debug, Deserialize)]
@@ -111,17 +78,6 @@ pub struct AgentConfigBuilder {
     pub server_addr: Option<String>,
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
-    pub endpoints: Option<EndpointsConfig>,
-    pub container_policy_path: Option<String>,
-    pub aa_kbc_params: Option<String>,
-    pub rest_api: Option<String>,
-    pub https_proxy: Option<String>,
-    pub no_proxy: Option<String>,
-    pub data_integrity: Option<bool>,
-    pub enable_signature_verification: Option<bool>,
-    pub image_policy_file: Option<String>,
-    pub image_registry_auth_file: Option<String>,
-    pub simple_signing_sigstore_config: Option<String>,
 }
 
 macro_rules! config_override {
@@ -181,18 +137,7 @@ impl Default for AgentConfig {
             server_addr: format!("{}:{}", VSOCK_ADDR, DEFAULT_AGENT_VSOCK_PORT),
             unified_cgroup_hierarchy: false,
             tracing: false,
-            endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
-            container_policy_path: String::from(""),
-            aa_kbc_params: String::from(""),
-            rest_api: String::from(""),
-            https_proxy: String::from(""),
-            no_proxy: String::from(""),
-            data_integrity: false,
-            enable_signature_verification: true,
-            image_policy_file: String::from(""),
-            image_registry_auth_file: String::from(""),
-            simple_signing_sigstore_config: String::from(""),
         }
     }
 }
@@ -221,31 +166,6 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
-        config_override!(agent_config_builder, agent_config, container_policy_path);
-        config_override!(agent_config_builder, agent_config, aa_kbc_params);
-        config_override!(agent_config_builder, agent_config, rest_api);
-        config_override!(agent_config_builder, agent_config, https_proxy);
-        config_override!(agent_config_builder, agent_config, no_proxy);
-        config_override!(agent_config_builder, agent_config, data_integrity);
-        config_override!(
-            agent_config_builder,
-            agent_config,
-            enable_signature_verification
-        );
-        config_override!(agent_config_builder, agent_config, image_policy_file);
-        config_override!(agent_config_builder, agent_config, image_registry_auth_file);
-        config_override!(
-            agent_config_builder,
-            agent_config,
-            simple_signing_sigstore_config
-        );
-
-        // Populate the allowed endpoints hash set, if we got any from the config file.
-        if let Some(endpoints) = agent_config_builder.endpoints {
-            for ep in endpoints.allowed {
-                agent_config.endpoints.allowed.insert(ep);
-            }
-        }
 
         Ok(agent_config)
     }
@@ -268,10 +188,6 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
-
-        let mut using_config_file = false;
-        // Check if there is config file before parsing params that might
-        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -279,15 +195,10 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                config = AgentConfig::from_config_file(&config_file)
-                    .context("AgentConfig from kernel cmdline")
-                    .unwrap();
-                using_config_file = true;
-                break;
+                return AgentConfig::from_config_file(&config_file)
+                    .context("AgentConfig from kernel cmdline");
             }
-        }
 
-        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -347,49 +258,6 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
-
-            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
-            parse_cmdline_param!(param, REST_API_OPTION, config.rest_api, get_string_value);
-            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
-            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
-            parse_cmdline_param!(
-                param,
-                ENABLE_DATA_INTEGRITY,
-                config.data_integrity,
-                get_bool_value
-            );
-
-            parse_cmdline_param!(
-                param,
-                ENABLE_SIGNATURE_VERIFICATION,
-                config.enable_signature_verification,
-                get_bool_value
-            );
-
-            // URI of the image security file
-            parse_cmdline_param!(
-                param,
-                IMAGE_POLICY_FILE,
-                config.image_policy_file,
-                get_string_value
-            );
-
-            // URI of the registry auth file
-            parse_cmdline_param!(
-                param,
-                IMAGE_REGISTRY_AUTH_FILE,
-                config.image_registry_auth_file,
-                get_string_value
-            );
-
-            // URI of the simple signing sigstore file
-            // used when simple signing verification is used
-            parse_cmdline_param!(
-                param,
-                SIMPLE_SIGNING_SIGSTORE_CONFIG,
-                config.simple_signing_sigstore_config,
-                get_string_value
-            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -408,11 +276,6 @@ impl AgentConfig {
             config.tracing = get_bool_value(&name_value)?;
         }
 
-        // We did not get a configuration file: allow all endpoints.
-        if !using_config_file {
-            config.endpoints.all_allowed = true;
-        }
-
         Ok(config)
     }
 
@@ -422,10 +285,6 @@ impl AgentConfig {
             .with_context(|| format!("Failed to read config file {}", file))?;
         AgentConfig::from_str(&config)
     }
-
-    pub fn is_allowed_endpoint(&self, ep: &str) -> bool {
-        self.endpoints.all_allowed || self.endpoints.allowed.contains(ep)
-    }
 }
 
 #[instrument]
@@ -546,12 +405,6 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
-#[instrument]
-fn get_url_value(param: &str) -> Result<String> {
-    let value = get_string_value(param)?;
-    Ok(Url::parse(&value)?.to_string())
-}
-
 #[cfg(test)]
 mod tests {
     use test_utils::assert_result;
@@ -570,11 +423,6 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
-        assert_eq!(config.container_policy_path, "");
-        assert!(config.enable_signature_verification);
-        assert_eq!(config.image_policy_file, "");
-        assert_eq!(config.image_registry_auth_file, "");
-        assert_eq!(config.simple_signing_sigstore_config, "");
     }
 
     #[test]
@@ -593,16 +441,6 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
-            container_policy_path: &'a str,
-            aa_kbc_params: &'a str,
-            rest_api: &'a str,
-            https_proxy: &'a str,
-            no_proxy: &'a str,
-            data_integrity: bool,
-            enable_signature_verification: bool,
-            image_policy_file: &'a str,
-            image_registry_auth_file: &'a str,
-            simple_signing_sigstore_config: &'a str,
         }
 
         impl Default for TestData<'_> {
@@ -618,16 +456,6 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
-                    container_policy_path: "",
-                    aa_kbc_params: "",
-                    rest_api: "",
-                    https_proxy: "",
-                    no_proxy: "",
-                    data_integrity: false,
-                    enable_signature_verification: true,
-                    image_policy_file: "",
-                    image_registry_auth_file: "",
-                    simple_signing_sigstore_config: "",
                 }
             }
         }
@@ -997,141 +825,6 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
-            TestData {
-                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
-                aa_kbc_params: "offline_fs_kbc::null",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
-                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.rest_api=attestation",
-                rest_api: "attestation",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.rest_api=resource",
-                rest_api: "resource",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.rest_api=all",
-                rest_api: "all",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://proxy.url.com:81/",
-                https_proxy: "http://proxy.url.com:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://192.168.1.100:81/",
-                https_proxy: "http://192.168.1.100:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=*.internal.url.com",
-                no_proxy: "*.internal.url.com",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
-                no_proxy: "192.168.1.0/24,172.16.0.0/12",
-                ..Default::default()
-            },
-            TestData {
-                contents: "",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=true",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=false",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=1",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=0",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=false",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=0",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=1",
-                enable_signature_verification: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=foo",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.image_policy=file:///etc/policy.json",
-                image_policy_file: "file:///etc/policy.json",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.image_policy=kbs:///default/security-policy/test",
-                image_policy_file: "kbs:///default/security-policy/test",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.image_policy=kbs://example.kbs.org/default/security-policy/test",
-                image_policy_file: "kbs://example.kbs.org/default/security-policy/test",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.image_registry_auth=file:///etc/auth.json",
-                image_registry_auth_file: "file:///etc/auth.json",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.image_registry_auth=kbs:///default/credential/test",
-                image_registry_auth_file: "kbs:///default/credential/test",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.image_registry_auth=kbs://example.kbs.org/default/credential/test",
-                image_registry_auth_file: "kbs://example.kbs.org/default/credential/test",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.simple_signing_sigstore_config=file:///etc/containers/signature/default.yml",
-                simple_signing_sigstore_config: "file:///etc/containers/signature/default.yml",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.simple_signing_sigstore_config=kbs:///default/sigstore-config/test",
-                simple_signing_sigstore_config: "kbs:///default/sigstore-config/test",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.simple_signing_sigstore_config=kbs://example.kbs.org/default/sigstore-config/test",
-                simple_signing_sigstore_config: "kbs://example.kbs.org/default/sigstore-config/test",
-                ..Default::default()
-            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -1179,32 +872,6 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
-            assert_eq!(
-                d.container_policy_path, config.container_policy_path,
-                "{}",
-                msg
-            );
-            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
-            assert_eq!(d.rest_api, config.rest_api, "{}", msg);
-            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
-            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
-            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
-            assert_eq!(
-                d.enable_signature_verification, config.enable_signature_verification,
-                "{}",
-                msg
-            );
-            assert_eq!(d.image_policy_file, config.image_policy_file, "{}", msg);
-            assert_eq!(
-                d.image_registry_auth_file, config.image_registry_auth_file,
-                "{}",
-                msg
-            );
-            assert_eq!(
-                d.simple_signing_sigstore_config, config.simple_signing_sigstore_config,
-                "{}",
-                msg
-            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1682,74 +1349,15 @@ Caused by:
             r#"
                dev_mode = true
                server_addr = 'vsock://8:2048'
-
-               [endpoints]
-               allowed = ["CreateContainer", "StartContainer"]
               "#,
         )
         .unwrap();
 
-        // Verify that the all_allowed flag is false
-        assert!(!config.endpoints.all_allowed);
-
         // Verify that the override worked
         assert!(config.dev_mode);
         assert_eq!(config.server_addr, "vsock://8:2048");
-        assert_eq!(
-            config.endpoints.allowed,
-            ["CreateContainer".to_string(), "StartContainer".to_string()]
-                .iter()
-                .cloned()
-                .collect()
-        );
 
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
-
-    #[test]
-    fn test_config_from_cmdline_and_config_file() {
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        let agent_config = r#"
-               dev_mode = false
-               server_addr = 'vsock://8:2048'
-
-               [endpoints]
-               allowed = ["CreateContainer", "StartContainer"]
-              "#;
-
-        let config_path = dir.path().join("agent-config.toml");
-        let config_filename = config_path.to_str().expect("failed to get config filename");
-
-        fs::write(config_filename, agent_config).expect("failed to write agen config");
-
-        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
-
-        let cmdline_path = dir.path().join("cmdline");
-        let cmdline_filename = cmdline_path
-            .to_str()
-            .expect("failed to get cmdline filename");
-
-        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
-
-        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
-            .expect("failed to parse command line");
-
-        // Should be overwritten by cmdline
-        assert!(config.dev_mode);
-
-        // Should be from agent config
-        assert_eq!(config.server_addr, "vsock://8:2048");
-
-        // Should be from agent config
-        assert_eq!(
-            config.endpoints.allowed,
-            ["CreateContainer".to_string(), "StartContainer".to_string()]
-                .iter()
-                .cloned()
-                .collect()
-        );
-        assert!(!config.endpoints.all_allowed);
-    }
 }

src/agent/src/device.rs

@@ -651,15 +651,13 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_type), Some(host_major), Some(host_minor)) =
-                (r.r#type.as_ref(), r.major, r.minor)
-            {
-                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
+            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
+                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl(),
                         "update_spec_devices() updating resource";
-                        "type" => &host_type,
+                        "type" => &r.r#type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -971,7 +969,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: Some(String::from("b")),
+        r#type: String::from("b"),
         access: String::from("rw"),
     });
 
@@ -1134,13 +1132,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1233,13 +1231,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: Some("b".to_string()),
+                            r#type: "b".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()

src/agent/src/image_rpc.rs

@@ -1,586 +0,0 @@
-// Copyright (c) 2021 Alibaba Cloud
-// Copyright (c) 2021, 2023 IBM Corporation
-// Copyright (c) 2022 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use std::collections::HashMap;
-use std::env;
-use std::fs;
-use std::path::Path;
-use std::sync::atomic::{AtomicU16, Ordering};
-use std::sync::Arc;
-
-use anyhow::{anyhow, Context, Result};
-use async_trait::async_trait;
-use image_rs::image::ImageClient;
-use protocols::image;
-use tokio::sync::Mutex;
-use ttrpc::{self, error::get_rpc_status as ttrpc_error};
-
-use crate::rpc::{verify_cid, CONTAINER_BASE};
-use crate::AGENT_CONFIG;
-
-// A marker to merge container spec for images pulled inside guest.
-const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
-
-// kata rootfs is readonly, use tmpfs before CC storage is implemented.
-const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
-const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
-const CONFIG_JSON: &str = "config.json";
-
-#[rustfmt::skip]
-lazy_static! {
-    pub static ref IMAGE_SERVICE: Mutex<Option<ImageService>> = Mutex::new(None);
-}
-
-// Convenience function to obtain the scope logger.
-fn sl() -> slog::Logger {
-    slog_scope::logger().new(o!("subsystem" => "cgroups"))
-}
-
-#[derive(Clone)]
-pub struct ImageService {
-    image_client: Arc<Mutex<ImageClient>>,
-    images: Arc<Mutex<HashMap<String, String>>>,
-    container_count: Arc<AtomicU16>,
-}
-
-impl ImageService {
-    pub fn new() -> Self {
-        env::set_var("CC_IMAGE_WORK_DIR", KATA_CC_IMAGE_WORK_DIR);
-
-        let mut image_client = ImageClient::default();
-        if !AGENT_CONFIG.image_policy_file.is_empty() {
-            image_client.config.file_paths.policy_path = AGENT_CONFIG.image_policy_file.clone();
-        }
-        if !AGENT_CONFIG.simple_signing_sigstore_config.is_empty() {
-            image_client.config.file_paths.sigstore_config =
-                AGENT_CONFIG.simple_signing_sigstore_config.clone();
-        }
-        if !AGENT_CONFIG.image_registry_auth_file.is_empty() {
-            image_client.config.file_paths.auth_file =
-                AGENT_CONFIG.image_registry_auth_file.clone();
-        }
-
-        Self {
-            image_client: Arc::new(Mutex::new(image_client)),
-            images: Arc::new(Mutex::new(HashMap::new())),
-            container_count: Arc::new(AtomicU16::new(0)),
-        }
-    }
-
-    /// Get the singleton instance of image service.
-    pub async fn singleton() -> Result<ImageService> {
-        IMAGE_SERVICE
-            .lock()
-            .await
-            .clone()
-            .ok_or_else(|| anyhow!("image service is uninitialized"))
-    }
-
-    // pause image is packaged in rootfs for CC
-    fn unpack_pause_image(cid: &str, target_subpath: &str) -> Result<String> {
-        let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
-        if !cc_pause_bundle.exists() {
-            return Err(anyhow!("Pause image not present in rootfs"));
-        }
-
-        info!(sl(), "use guest pause image cid {:?}", cid);
-        let pause_bundle = Path::new(CONTAINER_BASE).join(cid).join(target_subpath);
-        let pause_rootfs = pause_bundle.join("rootfs");
-        let pause_config = pause_bundle.join(CONFIG_JSON);
-        let pause_binary = pause_rootfs.join("pause");
-        fs::create_dir_all(&pause_rootfs)?;
-        if !pause_config.exists() {
-            fs::copy(
-                cc_pause_bundle.join(CONFIG_JSON),
-                pause_bundle.join(CONFIG_JSON),
-            )?;
-        }
-        if !pause_binary.exists() {
-            fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
-        }
-
-        Ok(pause_rootfs.display().to_string())
-    }
-
-    /// Determines the container id (cid) to use for a given request.
-    ///
-    /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
-    /// In either case, verify that the chosen id is valid.
-    fn cid_from_request(&self, req: &image::PullImageRequest) -> Result<String> {
-        let req_cid = req.container_id();
-        let cid = if !req_cid.is_empty() {
-            req_cid.to_string()
-        } else if let Some(last) = req.image().rsplit('/').next() {
-            // Support multiple containers with same image
-            let index = self.container_count.fetch_add(1, Ordering::Relaxed);
-
-            // ':' not valid for container id
-            format!("{}_{}", last.replace(':', "_"), index)
-        } else {
-            return Err(anyhow!("Invalid image name. {}", req.image()));
-        };
-        verify_cid(&cid)?;
-        Ok(cid)
-    }
-
-    /// Set proxy environment from AGENT_CONFIG
-    fn set_proxy_env_vars() {
-        let https_proxy = &AGENT_CONFIG.https_proxy;
-        if !https_proxy.is_empty() {
-            env::set_var("HTTPS_PROXY", https_proxy);
-        }
-        let no_proxy = &AGENT_CONFIG.no_proxy;
-        if !no_proxy.is_empty() {
-            env::set_var("NO_PROXY", no_proxy);
-        }
-    }
-
-    /// init atestation agent and read config from AGENT_CONFIG
-    async fn get_security_config(&self) -> Result<String> {
-        let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
-        // If the attestation-agent is being used, then enable the authenticated credentials support
-        info!(
-            sl(),
-            "image_client.config.auth set to: {}",
-            !aa_kbc_params.is_empty()
-        );
-        self.image_client.lock().await.config.auth = !aa_kbc_params.is_empty();
-        let decrypt_config = format!("provider:attestation-agent:{}", aa_kbc_params);
-
-        // Read enable signature verification from the agent config and set it in the image_client
-        let enable_signature_verification = &AGENT_CONFIG.enable_signature_verification;
-        info!(
-            sl(),
-            "enable_signature_verification set to: {}", enable_signature_verification
-        );
-        self.image_client.lock().await.config.security_validate = *enable_signature_verification;
-        Ok(decrypt_config)
-    }
-
-    /// Call image-rs to pull and unpack image.
-    async fn common_image_pull(
-        &self,
-        image: &str,
-        bundle_path: &Path,
-        decrypt_config: &str,
-        source_creds: Option<&str>,
-        cid: &str,
-    ) -> Result<()> {
-        let res = self
-            .image_client
-            .lock()
-            .await
-            .pull_image(image, bundle_path, &source_creds, &Some(decrypt_config))
-            .await;
-        match res {
-            Ok(image) => {
-                info!(
-                    sl(),
-                    "pull and unpack image {:?}, cid: {:?}, with image-rs succeed. ", image, cid
-                );
-            }
-            Err(e) => {
-                error!(
-                    sl(),
-                    "pull and unpack image {:?}, cid: {:?}, with image-rs failed with {:?}. ",
-                    image,
-                    cid,
-                    e.to_string()
-                );
-                return Err(e);
-            }
-        };
-        self.add_image(String::from(image), String::from(cid)).await;
-        Ok(())
-    }
-
-    /// Pull image when creating container and return the bundle path with rootfs.
-    pub async fn pull_image_for_container(
-        &self,
-        image: &str,
-        cid: &str,
-        image_metadata: &HashMap<String, String>,
-    ) -> Result<String> {
-        info!(sl(), "image metadata: {:?}", image_metadata);
-        Self::set_proxy_env_vars();
-        let is_sandbox = if let Some(value) = image_metadata.get("io.kubernetes.cri.container-type")
-        {
-            value == "sandbox"
-        } else if let Some(value) = image_metadata.get("io.kubernetes.cri-o.ContainerType") {
-            value == "sandbox"
-        } else {
-            false
-        };
-
-        if is_sandbox {
-            let mount_path = Self::unpack_pause_image(cid, "pause")?;
-            self.add_image(String::from(image), String::from(cid)).await;
-            return Ok(mount_path);
-        }
-        let bundle_path = Path::new(CONTAINER_BASE).join(cid).join("images");
-        fs::create_dir_all(&bundle_path)?;
-        info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
-
-        let decrypt_config = self.get_security_config().await?;
-
-        let source_creds = None; // You need to determine how to obtain this.
-
-        self.common_image_pull(image, &bundle_path, &decrypt_config, source_creds, cid)
-            .await?;
-        Ok(format! {"{}/rootfs",bundle_path.display()})
-    }
-
-    /// Pull image when recieving the PullImageRequest and return the image digest.
-    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
-        Self::set_proxy_env_vars();
-        let cid = self.cid_from_request(req)?;
-        let image = req.image();
-        if cid.starts_with("pause") {
-            Self::unpack_pause_image(&cid, "")?;
-            self.add_image(String::from(image), cid).await;
-            return Ok(image.to_owned());
-        }
-
-        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
-        // with rootfs and config.json will store under CONTAINER_BASE/cid.
-        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
-        fs::create_dir_all(&bundle_path)?;
-
-        let decrypt_config = self.get_security_config().await?;
-        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
-
-        self.common_image_pull(
-            image,
-            &bundle_path,
-            &decrypt_config,
-            source_creds,
-            cid.clone().as_str(),
-        )
-        .await?;
-        Ok(image.to_owned())
-    }
-
-    async fn add_image(&self, image: String, cid: String) {
-        self.images.lock().await.insert(image, cid);
-    }
-
-    // When being passed an image name through a container annotation, merge its
-    // corresponding bundle OCI specification into the passed container creation one.
-    pub async fn merge_bundle_oci(&self, container_oci: &mut oci::Spec) -> Result<()> {
-        if let Some(image_name) = container_oci
-            .annotations
-            .get(&ANNO_K8S_IMAGE_NAME.to_string())
-        {
-            let images = self.images.lock().await;
-            if let Some(container_id) = images.get(image_name) {
-                let image_oci_config_path = Path::new(CONTAINER_BASE)
-                    .join(container_id)
-                    .join(CONFIG_JSON);
-                debug!(
-                    sl(),
-                    "Image bundle config path: {:?}", image_oci_config_path
-                );
-
-                let image_oci =
-                    oci::Spec::load(image_oci_config_path.to_str().ok_or_else(|| {
-                        anyhow!(
-                            "Invalid container image OCI config path {:?}",
-                            image_oci_config_path
-                        )
-                    })?)
-                    .context("load image bundle")?;
-
-                if let Some(container_root) = container_oci.root.as_mut() {
-                    if let Some(image_root) = image_oci.root.as_ref() {
-                        let root_path = Path::new(CONTAINER_BASE)
-                            .join(container_id)
-                            .join(image_root.path.clone());
-                        container_root.path =
-                            String::from(root_path.to_str().ok_or_else(|| {
-                                anyhow!("Invalid container image root path {:?}", root_path)
-                            })?);
-                    }
-                }
-
-                if let Some(container_process) = container_oci.process.as_mut() {
-                    if let Some(image_process) = image_oci.process.as_ref() {
-                        self.merge_oci_process(container_process, image_process);
-                    }
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    // Partially merge an OCI process specification into another one.
-    fn merge_oci_process(&self, target: &mut oci::Process, source: &oci::Process) {
-        if target.args.is_empty() && !source.args.is_empty() {
-            target.args.append(&mut source.args.clone());
-        }
-
-        if target.cwd == "/" && source.cwd != "/" {
-            target.cwd = String::from(&source.cwd);
-        }
-
-        for source_env in &source.env {
-            let variable_name: Vec<&str> = source_env.split('=').collect();
-            if !target.env.iter().any(|i| i.contains(variable_name[0])) {
-                target.env.push(source_env.to_string());
-            }
-        }
-    }
-}
-
-#[async_trait]
-impl protocols::image_ttrpc_async::Image for ImageService {
-    async fn pull_image(
-        &self,
-        _ctx: &ttrpc::r#async::TtrpcContext,
-        req: image::PullImageRequest,
-    ) -> ttrpc::Result<image::PullImageResponse> {
-        match self.pull_image(&req).await {
-            Ok(r) => {
-                let mut resp = image::PullImageResponse::new();
-                resp.image_ref = r;
-                return Ok(resp);
-            }
-            Err(e) => {
-                return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::ImageService;
-    use protocols::image;
-
-    #[tokio::test]
-    async fn test_cid_from_request() {
-        struct Case {
-            cid: &'static str,
-            image: &'static str,
-            result: Option<&'static str>,
-        }
-
-        let cases = [
-            Case {
-                cid: "",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "..",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "..",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "abc/..",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "abc/",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "../abc",
-                result: Some("abc_4"),
-            },
-            Case {
-                cid: "",
-                image: "../9abc",
-                result: Some("9abc_5"),
-            },
-            Case {
-                cid: "some-string.1_2",
-                image: "",
-                result: Some("some-string.1_2"),
-            },
-            Case {
-                cid: "0some-string.1_2",
-                image: "",
-                result: Some("0some-string.1_2"),
-            },
-            Case {
-                cid: "a:b",
-                image: "",
-                result: None,
-            },
-            Case {
-                cid: "",
-                image: "prefix/a:b",
-                result: Some("a_b_6"),
-            },
-            Case {
-                cid: "",
-                image: "/a/b/c/d:e",
-                result: Some("d_e_7"),
-            },
-        ];
-
-        let image_service = ImageService::new();
-        for case in &cases {
-            let mut req = image::PullImageRequest::new();
-            req.set_image(case.image.to_string());
-            req.set_container_id(case.cid.to_string());
-            let ret = image_service.cid_from_request(&req);
-            match (case.result, ret) {
-                (Some(expected), Ok(actual)) => assert_eq!(expected, actual),
-                (None, Err(_)) => (),
-                (None, Ok(r)) => panic!("Expected an error, got {}", r),
-                (Some(expected), Err(e)) => {
-                    panic!("Expected {} but got an error ({})", expected, e)
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn test_merge_cwd() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            container_process_cwd: &'a str,
-            image_process_cwd: &'a str,
-            expected: &'a str,
-        }
-
-        let tests = &[
-            // Image cwd should override blank container cwd
-            // TODO - how can we tell the user didn't specifically set it to `/` vs not setting at all? Is that scenario valid?
-            TestData {
-                container_process_cwd: "/",
-                image_process_cwd: "/imageDir",
-                expected: "/imageDir",
-            },
-            // Container cwd should override image cwd
-            TestData {
-                container_process_cwd: "/containerDir",
-                image_process_cwd: "/imageDir",
-                expected: "/containerDir",
-            },
-            // Container cwd should override blank image cwd
-            TestData {
-                container_process_cwd: "/containerDir",
-                image_process_cwd: "/",
-                expected: "/containerDir",
-            },
-        ];
-
-        let image_service = ImageService::new();
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let mut container_process = oci::Process {
-                cwd: d.container_process_cwd.to_string(),
-                ..Default::default()
-            };
-
-            let image_process = oci::Process {
-                cwd: d.image_process_cwd.to_string(),
-                ..Default::default()
-            };
-
-            image_service.merge_oci_process(&mut container_process, &image_process);
-
-            assert_eq!(d.expected, container_process.cwd, "{}", msg);
-        }
-    }
-
-    #[tokio::test]
-    async fn test_merge_env() {
-        #[derive(Debug)]
-        struct TestData {
-            container_process_env: Vec<String>,
-            image_process_env: Vec<String>,
-            expected: Vec<String>,
-        }
-
-        let tests = &[
-            // Test that the pods environment overrides the images
-            TestData {
-                container_process_env: vec!["ISPRODUCTION=true".to_string()],
-                image_process_env: vec!["ISPRODUCTION=false".to_string()],
-                expected: vec!["ISPRODUCTION=true".to_string()],
-            },
-            // Test that multiple environment variables can be overrided
-            TestData {
-                container_process_env: vec![
-                    "ISPRODUCTION=true".to_string(),
-                    "ISDEVELOPMENT=false".to_string(),
-                ],
-                image_process_env: vec![
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-                expected: vec![
-                    "ISPRODUCTION=true".to_string(),
-                    "ISDEVELOPMENT=false".to_string(),
-                ],
-            },
-            // Test that when none of the variables match do not override them
-            TestData {
-                container_process_env: vec!["ANOTHERENV=TEST".to_string()],
-                image_process_env: vec![
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-                expected: vec![
-                    "ANOTHERENV=TEST".to_string(),
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-            },
-            // Test a mix of both overriding and not
-            TestData {
-                container_process_env: vec![
-                    "ANOTHERENV=TEST".to_string(),
-                    "ISPRODUCTION=true".to_string(),
-                ],
-                image_process_env: vec![
-                    "ISPRODUCTION=false".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-                expected: vec![
-                    "ANOTHERENV=TEST".to_string(),
-                    "ISPRODUCTION=true".to_string(),
-                    "ISDEVELOPMENT=true".to_string(),
-                ],
-            },
-        ];
-
-        let image_service = ImageService::new();
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let mut container_process = oci::Process {
-                env: d.container_process_env.clone(),
-                ..Default::default()
-            };
-
-            let image_process = oci::Process {
-                env: d.image_process_env.clone(),
-                ..Default::default()
-            };
-
-            image_service.merge_oci_process(&mut container_process, &image_process);
-
-            assert_eq!(d.expected, container_process.env, "{}", msg);
-        }
-    }
-}

src/agent/src/main.rs

@@ -22,7 +22,6 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
-use const_format::concatcp;
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
 use nix::unistd::{self, dup, Pid};
@@ -33,12 +32,9 @@ use std::os::unix::fs as unixfs;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
-use std::process::Command;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
-#[cfg(feature = "confidential-data-hub")]
-mod cdh;
 mod config;
 mod console;
 mod device;
@@ -75,10 +71,12 @@ use tokio::{
     task::JoinHandle,
 };
 
-mod image_rpc;
 mod rpc;
 mod tracer;
 
+#[cfg(feature = "agent-policy")]
+mod policy;
+
 cfg_if! {
     if #[cfg(target_arch = "s390x")] {
         mod ap;
@@ -88,27 +86,6 @@ cfg_if! {
 
 const NAME: &str = "kata-agent";
 
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
-const AA_PATH: &str = "/usr/local/bin/attestation-agent";
-const AA_UNIX_SOCKET_DIR: &str = "/run/confidential-containers/attestation-agent/";
-const UNIX_SOCKET_PREFIX: &str = "unix://";
-const AA_KEYPROVIDER_URI: &str =
-    concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "keyprovider.sock");
-const AA_GETRESOURCE_URI: &str =
-    concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "getresource.sock");
-const AA_ATTESTATION_SOCKET: &str = concatcp!(AA_UNIX_SOCKET_DIR, "attestation-agent.sock");
-const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SOCKET);
-
-const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
-
-cfg_if! {
-    if #[cfg(feature = "confidential-data-hub")] {
-        const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub";
-        const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock";
-        const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
-    }
-}
-
 lazy_static! {
     static ref AGENT_CONFIG: AgentConfig =
         // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
@@ -117,6 +94,11 @@ lazy_static! {
         AgentConfig::from_cmdline("/proc/cmdline", env::args().collect()).unwrap();
 }
 
+#[cfg(feature = "agent-policy")]
+lazy_static! {
+    static ref AGENT_POLICY: Mutex<policy::AgentPolicy> = Mutex::new(AgentPolicy::new());
+}
+
 #[derive(Parser)]
 // The default clap version info doesn't match our form, so we need to override it
 #[clap(global_setting(AppSettings::DisableVersionFlag))]
@@ -353,6 +335,19 @@ async fn start_sandbox(
         s.rtnl.handle_localhost().await?;
     }
 
+    // - When init_mode is true, enabling the localhost link during the
+    //   handle_localhost call above is required before starting OPA with the
+    //   initialize_policy call below.
+    // - When init_mode is false, the Policy could be initialized earlier,
+    //   because initialize_policy doesn't start OPA. OPA is started by
+    //   systemd after localhost has been enabled.
+    #[cfg(feature = "agent-policy")]
+    if let Err(e) = initialize_policy(init_mode).await {
+        error!(logger, "Failed to initialize agent policy: {:?}", e);
+        // Continuing execution without a security policy could be dangerous.
+        std::process::abort();
+    }
+
     let sandbox = Arc::new(Mutex::new(s));
 
     let signal_handler_task = tokio::spawn(setup_signal_handler(
@@ -370,12 +365,8 @@ async fn start_sandbox(
     let (tx, rx) = tokio::sync::oneshot::channel();
     sandbox.lock().await.sender = Some(tx);
 
-    if !config.aa_kbc_params.is_empty() {
-        init_attestation_agent(logger, config)?;
-    }
-
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
     server.start().await?;
 
     rx.await?;
@@ -384,110 +375,6 @@ async fn start_sandbox(
     Ok(())
 }
 
-// If we fail to start the AA, ocicrypt won't be able to unwrap keys
-// and container decryption will fail.
-fn init_attestation_agent(logger: &Logger, _config: &AgentConfig) -> Result<()> {
-    let config_path = OCICRYPT_CONFIG_PATH;
-
-    // The image will need to be encrypted using a keyprovider
-    // that has the same name (at least according to the config).
-    let ocicrypt_config = serde_json::json!({
-        "key-providers": {
-            "attestation-agent":{
-                "ttrpc":AA_KEYPROVIDER_URI
-            }
-        }
-    });
-
-    fs::write(config_path, ocicrypt_config.to_string().as_bytes())?;
-
-    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path);
-
-    // The Attestation Agent will run for the duration of the guest.
-    launch_process(
-        logger,
-        AA_PATH,
-        &vec![
-            "--keyprovider_sock",
-            AA_KEYPROVIDER_URI,
-            "--getresource_sock",
-            AA_GETRESOURCE_URI,
-            "--attestation_sock",
-            AA_ATTESTATION_URI,
-        ],
-        AA_ATTESTATION_SOCKET,
-        DEFAULT_LAUNCH_PROCESS_TIMEOUT,
-    )
-    .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?;
-
-    #[cfg(feature = "confidential-data-hub")]
-    {
-        if let Err(e) = launch_process(
-            logger,
-            CDH_PATH,
-            &vec![],
-            CDH_SOCKET,
-            DEFAULT_LAUNCH_PROCESS_TIMEOUT,
-        ) {
-            error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e);
-        } else if !_config.rest_api.is_empty() {
-            if let Err(e) = launch_process(
-                logger,
-                API_SERVER_PATH,
-                &vec!["--features", &_config.rest_api],
-                "",
-                0,
-            ) {
-                error!(logger, "launch_process {} failed: {:?}", API_SERVER_PATH, e);
-            }
-        }
-    }
-
-    Ok(())
-}
-
-fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
-    let p = Path::new(path);
-    let mut attempts = 0;
-    loop {
-        std::thread::sleep(std::time::Duration::from_secs(1));
-        if p.exists() {
-            return Ok(());
-        }
-        if attempts >= timeout_secs {
-            break;
-        }
-        attempts += 1;
-        info!(
-            logger,
-            "waiting for {} to exist (attempts={})", path, attempts
-        );
-    }
-
-    Err(anyhow!("wait for {} to exist timeout.", path))
-}
-
-fn launch_process(
-    logger: &Logger,
-    path: &str,
-    args: &Vec<&str>,
-    unix_socket_path: &str,
-    timeout_secs: i32,
-) -> Result<()> {
-    if !Path::new(path).exists() {
-        return Err(anyhow!("path {} does not exist.", path));
-    }
-    if !unix_socket_path.is_empty() && Path::new(unix_socket_path).exists() {
-        fs::remove_file(unix_socket_path)?;
-    }
-    Command::new(path).args(args).spawn()?;
-    if !unix_socket_path.is_empty() && timeout_secs > 0 {
-        wait_for_path_to_exist(logger, unix_socket_path, timeout_secs)?;
-    }
-
-    Ok(())
-}
-
 // init_agent_as_init will do the initializations such as setting up the rootfs
 // when this agent has been run as the init process.
 fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<()> {
@@ -522,6 +409,18 @@ fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result
     Ok(())
 }
 
+#[cfg(feature = "agent-policy")]
+async fn initialize_policy(init_mode: bool) -> Result<()> {
+    let opa_addr = "localhost:8181";
+    let agent_policy_path = "/agent_policy";
+    let default_agent_policy = "/etc/kata-opa/default-policy.rego";
+    AGENT_POLICY
+        .lock()
+        .await
+        .initialize(init_mode, opa_addr, agent_policy_path, default_agent_policy)
+        .await
+}
+
 // The Rust standard library had suppressed the default SIGPIPE behavior,
 // see https://github.com/rust-lang/rust/pull/13158.
 // Since the parent's signal handler would be inherited by it's child process,
@@ -536,6 +435,9 @@ fn reset_sigpipe() {
 use crate::config::AgentConfig;
 use std::os::unix::io::{FromRawFd, RawFd};
 
+#[cfg(feature = "agent-policy")]
+use crate::policy::AgentPolicy;
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/agent/src/policy.rs

@@ -0,0 +1,267 @@
+// Copyright (c) 2023 Microsoft Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{bail, Result};
+use serde::{Deserialize, Serialize};
+use slog::Drain;
+use tokio::io::AsyncWriteExt;
+use tokio::time::{sleep, Duration};
+
+static EMPTY_JSON_INPUT: &str = "{\"input\":{}}";
+
+static OPA_DATA_PATH: &str = "/data";
+static OPA_POLICIES_PATH: &str = "/policies";
+
+static POLICY_LOG_FILE: &str = "/tmp/policy.txt";
+
+/// Convenience macro to obtain the scope logger
+macro_rules! sl {
+    () => {
+        slog_scope::logger()
+    };
+}
+
+/// Example of HTTP response from OPA: {"result":true}
+#[derive(Debug, Serialize, Deserialize)]
+struct AllowResponse {
+    result: bool,
+}
+
+/// Singleton policy object.
+#[derive(Debug, Default)]
+pub struct AgentPolicy {
+    /// When true policy errors are ignored, for debug purposes.
+    allow_failures: bool,
+
+    /// OPA path used to query if an Agent gRPC request should be allowed.
+    /// The request name (e.g., CreateContainerRequest) must be added to
+    /// this path.
+    query_path: String,
+
+    /// OPA path used to add or delete a rego format Policy.
+    policy_path: String,
+
+    /// Client used to connect a single time to the OPA service and reused
+    /// for all the future communication with OPA.
+    opa_client: Option<reqwest::Client>,
+
+    /// "/tmp/policy.txt" log file for policy activity.
+    log_file: Option<tokio::fs::File>,
+}
+
+impl AgentPolicy {
+    /// Create AgentPolicy object.
+    pub fn new() -> Self {
+        Self {
+            allow_failures: false,
+            ..Default::default()
+        }
+    }
+
+    /// Wait for OPA to start and connect to it.
+    pub async fn initialize(
+        &mut self,
+        launch_opa: bool,
+        opa_addr: &str,
+        policy_name: &str,
+        default_policy: &str,
+    ) -> Result<()> {
+        if sl!().is_enabled(slog::Level::Debug) {
+            self.log_file = Some(
+                tokio::fs::OpenOptions::new()
+                    .write(true)
+                    .truncate(true)
+                    .create(true)
+                    .open(POLICY_LOG_FILE)
+                    .await?,
+            );
+            debug!(sl!(), "policy: log file: {}", POLICY_LOG_FILE);
+        }
+
+        if launch_opa {
+            start_opa(opa_addr)?;
+        }
+
+        let opa_uri = format!("http://{opa_addr}/v1");
+        self.query_path = format!("{opa_uri}{OPA_DATA_PATH}{policy_name}/");
+        self.policy_path = format!("{opa_uri}{OPA_POLICIES_PATH}{policy_name}");
+        let opa_client = reqwest::Client::builder().http1_only().build()?;
+        let policy = tokio::fs::read_to_string(default_policy).await?;
+
+        // This loop is necessary to get the opa_client connected to the
+        // OPA service while that service is starting. Future requests to
+        // OPA are expected to work without retrying, after connecting
+        // successfully for the first time.
+        for i in 0..50 {
+            if i > 0 {
+                sleep(Duration::from_millis(100)).await;
+                debug!(sl!(), "policy initialize: PUT failed, retrying");
+            }
+
+            // Set-up the default policy.
+            if opa_client
+                .put(&self.policy_path)
+                .body(policy.clone())
+                .send()
+                .await
+                .is_ok()
+            {
+                self.opa_client = Some(opa_client);
+
+                // Check if requests causing policy errors should actually
+                // be allowed. That is an insecure configuration but is
+                // useful for allowing insecure pods to start, then connect to
+                // them and inspect Guest logs for the root cause of a failure.
+                //
+                // Note that post_query returns Ok(false) in case
+                // AllowRequestsFailingPolicy was not defined in the policy.
+                self.allow_failures = self
+                    .post_query("AllowRequestsFailingPolicy", EMPTY_JSON_INPUT)
+                    .await?;
+                return Ok(());
+            }
+        }
+        bail!("Failed to connect to OPA")
+    }
+
+    /// Ask OPA to check if an API call should be allowed or not.
+    pub async fn is_allowed_endpoint(&mut self, ep: &str, request: &str) -> bool {
+        let post_input = format!("{{\"input\":{request}}}");
+        self.log_opa_input(ep, &post_input).await;
+        match self.post_query(ep, &post_input).await {
+            Err(e) => {
+                debug!(
+                    sl!(),
+                    "policy: failed to query endpoint {}: {:?}. Returning false.", ep, e
+                );
+                false
+            }
+            Ok(allowed) => allowed,
+        }
+    }
+
+    /// Replace the Policy in OPA.
+    pub async fn set_policy(&mut self, policy: &str) -> Result<()> {
+        if let Some(opa_client) = &mut self.opa_client {
+            // Delete the old rules.
+            opa_client.delete(&self.policy_path).send().await?;
+
+            // Put the new rules.
+            opa_client
+                .put(&self.policy_path)
+                .body(policy.to_string())
+                .send()
+                .await?;
+
+            // Check if requests causing policy errors should actually be allowed.
+            // That is an insecure configuration but is useful for allowing insecure
+            // pods to start, then connect to them and inspect Guest logs for the
+            // root cause of a failure.
+            //
+            // Note that post_query returns Ok(false) in case
+            // AllowRequestsFailingPolicy was not defined in the policy.
+            self.allow_failures = self
+                .post_query("AllowRequestsFailingPolicy", EMPTY_JSON_INPUT)
+                .await?;
+
+            Ok(())
+        } else {
+            bail!("Agent Policy is not initialized")
+        }
+    }
+
+    // Post query to OPA.
+    async fn post_query(&mut self, ep: &str, post_input: &str) -> Result<bool> {
+        debug!(sl!(), "policy check: {ep}");
+
+        if let Some(opa_client) = &mut self.opa_client {
+            let uri = format!("{}{ep}", &self.query_path);
+            let response = opa_client
+                .post(uri)
+                .body(post_input.to_string())
+                .send()
+                .await?;
+
+            if response.status() != http::StatusCode::OK {
+                bail!("policy: POST {} response status {}", ep, response.status());
+            }
+
+            let http_response = response.text().await?;
+            let opa_response: serde_json::Result<AllowResponse> =
+                serde_json::from_str(&http_response);
+
+            match opa_response {
+                Ok(resp) => {
+                    if !resp.result {
+                        if self.allow_failures {
+                            warn!(
+                                sl!(),
+                                "policy: POST {} response <{}>. Ignoring error!", ep, http_response
+                            );
+                            return Ok(true);
+                        } else {
+                            error!(sl!(), "policy: POST {} response <{}>", ep, http_response);
+                        }
+                    }
+                    Ok(resp.result)
+                }
+                Err(_) => {
+                    warn!(
+                        sl!(),
+                        "policy: endpoint {} not found in policy. Returning false.", ep,
+                    );
+                    Ok(false)
+                }
+            }
+        } else {
+            bail!("Agent Policy is not initialized")
+        }
+    }
+
+    async fn log_opa_input(&mut self, ep: &str, input: &str) {
+        if let Some(log_file) = &mut self.log_file {
+            match ep {
+                "StatsContainerRequest" | "ReadStreamRequest" | "SetPolicyRequest" => {
+                    // - StatsContainerRequest and ReadStreamRequest are called
+                    //   relatively often, so we're not logging them, to avoid
+                    //   growing this log file too much.
+                    // - Confidential Containers Policy documents are relatively
+                    //   large, so we're not logging them here, for SetPolicyRequest.
+                    //   The Policy text can be obtained directly from the pod YAML.
+                }
+                _ => {
+                    let log_entry = format!("[\"ep\":\"{ep}\",{input}],\n\n");
+
+                    if let Err(e) = log_file.write_all(log_entry.as_bytes()).await {
+                        warn!(sl!(), "policy: log_opa_input: write_all failed: {}", e);
+                    } else if let Err(e) = log_file.flush().await {
+                        warn!(sl!(), "policy: log_opa_input: flush failed: {}", e);
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn start_opa(opa_addr: &str) -> Result<()> {
+    let bin_dirs = vec!["/bin", "/usr/bin", "/usr/local/bin"];
+    for bin_dir in &bin_dirs {
+        let opa_path = bin_dir.to_string() + "/opa";
+        if std::fs::metadata(&opa_path).is_ok() {
+            // args copied from kata-opa.service.in.
+            std::process::Command::new(&opa_path)
+                .arg("run")
+                .arg("--server")
+                .arg("--disable-telemetry")
+                .arg("--addr")
+                .arg(opa_addr)
+                .arg("--log-level")
+                .arg("info")
+                .spawn()?;
+            return Ok(());
+        }
+    }
+    bail!("OPA binary not found in {:?}", &bin_dirs);
+}

src/agent/src/storage/block_handler.rs

@@ -27,29 +27,19 @@ use crate::{ccw, device::get_virtio_blk_ccw_device_name};
 #[derive(Debug)]
 pub struct VirtioBlkMmioHandler {}
 
-impl VirtioBlkMmioHandler {
-    pub async fn update_device_path(
-        storage: &mut Storage,
-        ctx: &mut StorageContext<'_>,
-    ) -> Result<()> {
-        if !Path::new(&storage.source).exists() {
-            get_virtio_mmio_device_name(ctx.sandbox, &storage.source)
-                .await
-                .context("failed to get mmio device name")?;
-        }
-        Ok(())
-    }
-}
-
 #[async_trait::async_trait]
 impl StorageHandler for VirtioBlkMmioHandler {
     #[instrument]
     async fn create_device(
         &self,
-        mut storage: Storage,
+        storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        Self::update_device_path(&mut storage, ctx).await?;
+        if !Path::new(&storage.source).exists() {
+            get_virtio_mmio_device_name(ctx.sandbox, &storage.source)
+                .await
+                .context("failed to get mmio device name")?;
+        }
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -58,11 +48,14 @@ impl StorageHandler for VirtioBlkMmioHandler {
 #[derive(Debug)]
 pub struct VirtioBlkPciHandler {}
 
-impl VirtioBlkPciHandler {
-    pub async fn update_device_path(
-        storage: &mut Storage,
-        ctx: &mut StorageContext<'_>,
-    ) -> Result<()> {
+#[async_trait::async_trait]
+impl StorageHandler for VirtioBlkPciHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
         // If hot-plugged, get the device node path based on the PCI path
         // otherwise use the virt path provided in Storage Source
         if storage.source.starts_with("/dev") {
@@ -78,19 +71,6 @@ impl VirtioBlkPciHandler {
             storage.source = dev_path;
         }
 
-        Ok(())
-    }
-}
-
-#[async_trait::async_trait]
-impl StorageHandler for VirtioBlkPciHandler {
-    #[instrument]
-    async fn create_device(
-        &self,
-        mut storage: Storage,
-        ctx: &mut StorageContext,
-    ) -> Result<Arc<dyn StorageDevice>> {
-        Self::update_device_path(&mut storage, ctx).await?;
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -99,21 +79,6 @@ impl StorageHandler for VirtioBlkPciHandler {
 #[derive(Debug)]
 pub struct VirtioBlkCcwHandler {}
 
-impl VirtioBlkCcwHandler {
-    pub async fn update_device_path(
-        _storage: &mut Storage,
-        _ctx: &mut StorageContext<'_>,
-    ) -> Result<()> {
-        #[cfg(target_arch = "s390x")]
-        {
-            let ccw_device = ccw::Device::from_str(&_storage.source)?;
-            let dev_path = get_virtio_blk_ccw_device_name(_ctx.sandbox, &ccw_device).await?;
-            _storage.source = dev_path;
-        }
-        Ok(())
-    }
-}
-
 #[async_trait::async_trait]
 impl StorageHandler for VirtioBlkCcwHandler {
     #[cfg(target_arch = "s390x")]
@@ -123,7 +88,9 @@ impl StorageHandler for VirtioBlkCcwHandler {
         mut storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        Self::update_device_path(&mut storage, ctx).await?;
+        let ccw_device = ccw::Device::from_str(&storage.source)?;
+        let dev_path = get_virtio_blk_ccw_device_name(ctx.sandbox, &ccw_device).await?;
+        storage.source = dev_path;
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -142,18 +109,6 @@ impl StorageHandler for VirtioBlkCcwHandler {
 #[derive(Debug)]
 pub struct ScsiHandler {}
 
-impl ScsiHandler {
-    pub async fn update_device_path(
-        storage: &mut Storage,
-        ctx: &mut StorageContext<'_>,
-    ) -> Result<()> {
-        // Retrieve the device path from SCSI address.
-        let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
-        storage.source = dev_path;
-        Ok(())
-    }
-}
-
 #[async_trait::async_trait]
 impl StorageHandler for ScsiHandler {
     #[instrument]
@@ -162,7 +117,10 @@ impl StorageHandler for ScsiHandler {
         mut storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        Self::update_device_path(&mut storage, ctx).await?;
+        // Retrieve the device path from SCSI address.
+        let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
+        storage.source = dev_path;
+
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }
@@ -171,26 +129,17 @@ impl StorageHandler for ScsiHandler {
 #[derive(Debug)]
 pub struct PmemHandler {}
 
-impl PmemHandler {
-    pub async fn update_device_path(
-        storage: &mut Storage,
-        ctx: &mut StorageContext<'_>,
-    ) -> Result<()> {
-        // Retrieve the device for pmem storage
-        wait_for_pmem_device(ctx.sandbox, &storage.source).await?;
-        Ok(())
-    }
-}
-
 #[async_trait::async_trait]
 impl StorageHandler for PmemHandler {
     #[instrument]
     async fn create_device(
         &self,
-        mut storage: Storage,
+        storage: Storage,
         ctx: &mut StorageContext,
     ) -> Result<Arc<dyn StorageDevice>> {
-        Self::update_device_path(&mut storage, ctx).await?;
+        // Retrieve the device for pmem storage
+        wait_for_pmem_device(ctx.sandbox, &storage.source).await?;
+
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)
     }

src/agent/src/storage/dm_verity.rs

@@ -1,165 +0,0 @@
-// Copyright (c) 2023 Alibaba Cloud
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use std::path::Path;
-use std::sync::Arc;
-
-use anyhow::{anyhow, Context, Result};
-use image_rs::verity::{create_dmverity_device, destroy_dmverity_device};
-use kata_sys_util::mount::create_mount_destination;
-use kata_types::mount::{DmVerityInfo, StorageDevice};
-use kata_types::volume::{
-    KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE, KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO,
-    KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM, KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI,
-    KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW, KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO,
-    KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI,
-};
-use protocols::agent::Storage;
-use slog::Logger;
-use tracing::instrument;
-
-use crate::storage::block_handler::{
-    PmemHandler, ScsiHandler, VirtioBlkCcwHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler,
-};
-use crate::storage::{common_storage_handler, StorageContext, StorageHandler};
-
-use super::StorageDeviceGeneric;
-
-#[derive(Debug)]
-pub struct DmVerityHandler {}
-
-impl DmVerityHandler {
-    fn get_dm_verity_info(storage: &Storage) -> Result<DmVerityInfo> {
-        for option in storage.driver_options.iter() {
-            if let Some((key, value)) = option.split_once('=') {
-                if key == KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO {
-                    let verity_info: DmVerityInfo = serde_json::from_str(value)?;
-                    return Ok(verity_info);
-                }
-            }
-        }
-
-        Err(anyhow!("missing DmVerity information for DmVerity volume"))
-    }
-
-    async fn update_source_device(
-        storage: &mut Storage,
-        ctx: &mut StorageContext<'_>,
-    ) -> Result<()> {
-        for option in storage.driver_options.clone() {
-            if let Some((key, value)) = option.split_once('=') {
-                if key == KATA_VOLUME_DMVERITY_OPTION_SOURCE_TYPE {
-                    match value {
-                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_PCI => {
-                            VirtioBlkPciHandler::update_device_path(storage, ctx).await?;
-                        }
-                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_MMIO => {
-                            VirtioBlkMmioHandler::update_device_path(storage, ctx).await?;
-                        }
-                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_VIRTIO_CCW => {
-                            VirtioBlkCcwHandler::update_device_path(storage, ctx).await?;
-                        }
-                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_SCSI => {
-                            ScsiHandler::update_device_path(storage, ctx).await?;
-                        }
-                        KATA_VOLUME_DMVERITY_SOURCE_TYPE_PMEM => {
-                            PmemHandler::update_device_path(storage, ctx).await?;
-                        }
-                        _ => {}
-                    }
-                }
-            }
-        }
-        Ok(())
-    }
-}
-
-#[async_trait::async_trait]
-impl StorageHandler for DmVerityHandler {
-    #[instrument]
-    async fn create_device(
-        &self,
-        mut storage: Storage,
-        ctx: &mut StorageContext,
-    ) -> Result<Arc<dyn StorageDevice>> {
-        Self::update_source_device(&mut storage, ctx).await?;
-        create_mount_destination(&storage.source, &storage.mount_point, "", &storage.fstype)
-            .context("Could not create mountpoint")?;
-
-        let verity_info = Self::get_dm_verity_info(&storage)?;
-        let verity_info = serde_json::to_string(&verity_info)
-            .map_err(|e| anyhow!("failed to serialize dm_verity info, {}", e))?;
-        let verity_device_path = create_dmverity_device(&verity_info, Path::new(storage.source()))
-            .context("create device with dm-verity enabled")?;
-        storage.source = verity_device_path;
-        common_storage_handler(ctx.logger, &storage)?;
-
-        Ok(Arc::new(DmVerityDevice {
-            common: StorageDeviceGeneric::new(storage.mount_point),
-            verity_device_path: storage.source,
-            logger: ctx.logger.clone(),
-        }))
-    }
-}
-
-struct DmVerityDevice {
-    common: StorageDeviceGeneric,
-    verity_device_path: String,
-    logger: Logger,
-}
-
-impl StorageDevice for DmVerityDevice {
-    fn path(&self) -> Option<&str> {
-        self.common.path()
-    }
-
-    fn cleanup(&self) -> Result<()> {
-        self.common.cleanup().context("clean up dm-verity volume")?;
-        let device_path = &self.verity_device_path;
-        debug!(
-            self.logger,
-            "destroy verity device path = {:?}", device_path
-        );
-        destroy_dmverity_device(device_path)?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use kata_types::{mount::DmVerityInfo, volume::KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO};
-    use protocols::agent::Storage;
-
-    use crate::storage::dm_verity::DmVerityHandler;
-
-    #[test]
-    fn test_get_dm_verity_info() {
-        let verity_info = DmVerityInfo {
-            hashtype: "sha256".to_string(),
-            hash: "d86104eee715a1b59b62148641f4ca73edf1be3006c4d481f03f55ac05640570".to_string(),
-            blocknum: 2361,
-            blocksize: 512,
-            hashsize: 4096,
-            offset: 1212416,
-        };
-
-        let verity_info_str = serde_json::to_string(&verity_info);
-        assert!(verity_info_str.is_ok());
-
-        let storage = Storage {
-            driver: KATA_VOLUME_DMVERITY_OPTION_VERITY_INFO.to_string(),
-            driver_options: vec![format!("verity_info={}", verity_info_str.ok().unwrap())],
-            ..Default::default()
-        };
-
-        match DmVerityHandler::get_dm_verity_info(&storage) {
-            Ok(result) => {
-                assert_eq!(verity_info, result);
-            }
-            Err(e) => panic!("err = {}", e),
-        }
-    }
-}

src/agent/src/storage/fs_handler.rs

@@ -10,7 +10,6 @@ use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
 use kata_types::mount::StorageDevice;
-use kata_types::volume::KATA_VOLUME_OVERLAYFS_CREATE_DIR;
 use protocols::agent::Storage;
 use tracing::instrument;
 
@@ -51,15 +50,6 @@ impl StorageHandler for OverlayfsHandler {
                 .options
                 .push(format!("workdir={}", work.to_string_lossy()));
         }
-        let overlay_create_dir_prefix = &(KATA_VOLUME_OVERLAYFS_CREATE_DIR.to_string() + "=");
-        for driver_option in &storage.driver_options {
-            if let Some(dir) = driver_option
-                .as_str()
-                .strip_prefix(overlay_create_dir_prefix)
-            {
-                fs::create_dir_all(dir).context("Failed to create directory")?;
-            }
-        }
 
         let path = common_storage_handler(ctx.logger, &storage)?;
         new_device(path)

src/agent/src/storage/image_pull_handler.rs

@@ -1,102 +0,0 @@
-// Copyright (c) 2023 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-use anyhow::{anyhow, Result};
-use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
-use kata_types::mount::{ImagePullVolume, StorageDevice};
-use protocols::agent::Storage;
-use std::sync::Arc;
-use tracing::instrument;
-
-use crate::image_rpc;
-use crate::storage::{StorageContext, StorageHandler};
-
-use super::{common_storage_handler, new_device};
-
-#[derive(Debug)]
-pub struct ImagePullHandler {}
-
-impl ImagePullHandler {
-    fn get_image_info(storage: &Storage) -> Result<ImagePullVolume> {
-        for option in storage.driver_options.iter() {
-            if let Some((key, value)) = option.split_once('=') {
-                if key == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
-                    let imagepull_volume: ImagePullVolume = serde_json::from_str(value)?;
-                    return Ok(imagepull_volume);
-                }
-            }
-        }
-        Err(anyhow!("missing Image information for ImagePull volume"))
-    }
-}
-
-#[async_trait::async_trait]
-impl StorageHandler for ImagePullHandler {
-    #[instrument]
-    async fn create_device(
-        &self,
-        mut storage: Storage,
-        ctx: &mut StorageContext,
-    ) -> Result<Arc<dyn StorageDevice>> {
-        //Currently the image metadata is not used to pulling image in the guest.
-        let image_pull_volume = Self::get_image_info(&storage)?;
-        debug!(ctx.logger, "image_pull_volume = {:?}", image_pull_volume);
-        let image_name = storage.source();
-        debug!(ctx.logger, "image_name = {:?}", image_name);
-
-        let cid = ctx
-            .cid
-            .clone()
-            .ok_or_else(|| anyhow!("failed to get container id"))?;
-        let image_service = image_rpc::ImageService::singleton().await?;
-        let bundle_path = image_service
-            .pull_image_for_container(image_name, &cid, &image_pull_volume.metadata)
-            .await?;
-
-        storage.source = bundle_path;
-        storage.options = vec!["bind".to_string(), "ro".to_string()];
-
-        common_storage_handler(ctx.logger, &storage)?;
-
-        new_device(storage.mount_point)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::HashMap;
-
-    use kata_types::mount::{ImagePullVolume, KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL};
-    use protocols::agent::Storage;
-
-    use crate::storage::image_pull_handler::ImagePullHandler;
-
-    #[test]
-    fn test_get_image_info() {
-        let mut res = HashMap::new();
-        res.insert("key1".to_string(), "value1".to_string());
-        res.insert("key2".to_string(), "value2".to_string());
-
-        let image_pull = ImagePullVolume {
-            metadata: res.clone(),
-        };
-
-        let image_pull_str = serde_json::to_string(&image_pull);
-        assert!(image_pull_str.is_ok());
-
-        let storage = Storage {
-            driver: KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL.to_string(),
-            driver_options: vec![format!("image_guest_pull={}", image_pull_str.ok().unwrap())],
-            ..Default::default()
-        };
-
-        match ImagePullHandler::get_image_info(&storage) {
-            Ok(image_info) => {
-                assert_eq!(image_info.metadata, res);
-            }
-            Err(e) => panic!("err = {}", e),
-        }
-    }
-}

src/agent/src/storage/mod.rs

@@ -12,11 +12,7 @@ use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
 use kata_sys_util::mount::{create_mount_destination, parse_mount_options};
-use kata_types::mount::{
-    StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG,
-    KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL,
-};
-use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY;
+use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG};
 use nix::unistd::{Gid, Uid};
 use protocols::agent::Storage;
 use protocols::types::FSGroupChangePolicy;
@@ -26,12 +22,9 @@ use tracing::instrument;
 
 use self::bind_watcher_handler::BindWatcherHandler;
 use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, VirtioBlkPciHandler};
-use self::dm_verity::DmVerityHandler;
 use self::ephemeral_handler::EphemeralHandler;
 use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler};
-use self::image_pull_handler::ImagePullHandler;
 use self::local_handler::LocalHandler;
-
 use crate::device::{
     DRIVER_9P_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_EPHEMERAL_TYPE,
     DRIVER_LOCAL_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
@@ -44,10 +37,8 @@ pub use self::ephemeral_handler::update_ephemeral_mounts;
 
 mod bind_watcher_handler;
 mod block_handler;
-mod dm_verity;
 mod ephemeral_handler;
 mod fs_handler;
-mod image_pull_handler;
 mod local_handler;
 
 const RW_MASK: u32 = 0o660;
@@ -154,8 +145,6 @@ lazy_static! {
         manager.add_handler(DRIVER_SCSI_TYPE, Arc::new(ScsiHandler{})).unwrap();
         manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap();
         manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap();
-        manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap();
-        manager.add_handler(KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL, Arc::new(ImagePullHandler{})).unwrap();
         manager
     };
 }

src/dragonball/.gitignore

@@ -1,3 +1,2 @@
 target
-Cargo.lock
 .idea

src/dragonball/Cargo.toml

@@ -58,6 +58,6 @@ virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"]
 virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
-virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"]
+virtio-fs = ["dbs-virtio-devices/virtio-fs-pro", "virtio-queue", "atomic-guest-memory"]
 virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]
 virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"]

src/dragonball/src/api/v1/instance_info.rs

@@ -5,15 +5,6 @@
 
 use serde_derive::{Deserialize, Serialize};
 
-/// This struct represents the strongly typed equivalent of the json body
-/// from confidential container related requests.
-#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
-#[serde(deny_unknown_fields)]
-pub enum ConfidentialVmType {
-    /// Intel Trusted Domain
-    TDX = 2,
-}
-
 /// The microvm state.
 ///
 /// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is
@@ -67,12 +58,10 @@ pub struct InstanceInfo {
     pub tids: Vec<(u8, u32)>,
     /// Last instance downtime
     pub last_instance_downtime: u64,
-    /// confidential vm type
-    pub confidential_vm_type: Option<ConfidentialVmType>,
 }
 
 impl InstanceInfo {
-    /// create instance info object with given id, version, platform type and confidential vm type.
+    /// create instance info object with given id, version, and platform type
     pub fn new(id: String, vmm_version: String) -> Self {
         InstanceInfo {
             id,
@@ -83,14 +72,8 @@ impl InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
-            confidential_vm_type: None,
         }
     }
-
-    /// return true if VM confidential type is TDX
-    pub fn is_tdx_enabled(&self) -> bool {
-        matches!(self.confidential_vm_type, Some(ConfidentialVmType::TDX))
-    }
 }
 
 impl Default for InstanceInfo {
@@ -104,7 +87,6 @@ impl Default for InstanceInfo {
             async_state: AsyncState::Uninitialized,
             tids: Vec::new(),
             last_instance_downtime: 0,
-            confidential_vm_type: None,
         }
     }
 }

src/dragonball/src/api/v1/mod.rs

@@ -12,7 +12,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER
 
 /// Wrapper over the microVM general information.
 mod instance_info;
-pub use self::instance_info::{ConfidentialVmType, InstanceInfo, InstanceState};
+pub use self::instance_info::{InstanceInfo, InstanceState};
 
 /// Wrapper for configuring the memory and CPU of the microVM.
 mod machine_config;

src/dragonball/src/api/v1/vmm_action.rs

@@ -692,6 +692,7 @@ impl VmmService {
             ));
         }
 
+        #[cfg(feature = "dbs-upcall")]
         vm.resize_vcpu(config, None).map_err(|e| {
             if let VcpuResizeError::UpcallServerNotReady = e {
                 return VmmActionError::UpcallServerNotReady;

src/dragonball/src/dbs_virtio_devices/src/fs/device.rs

@@ -475,7 +475,7 @@ impl<AS: GuestAddressSpace> VirtioFs<AS> {
         let (mut rafs, rafs_cfg) = match config.as_ref() {
             Some(cfg) => {
                 let rafs_conf: Arc<ConfigV2> = Arc::new(
-                    serde_json::from_str(cfg).map_err(|e| FsError::BackendFs(e.to_string()))?,
+                    ConfigV2::from_str(cfg).map_err(|e| FsError::BackendFs(e.to_string()))?,
                 );
 
                 (

src/dragonball/src/dbs_virtio_devices/src/vsock/backend/hybrid_stream.rs

@@ -0,0 +1,94 @@
+// Copyright 2023 Ant Group. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::any::Any;
+use std::io::{Error, Read, Write};
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::time::Duration;
+
+use log::error;
+use nix::errno::Errno;
+
+use super::{VsockBackendType, VsockStream};
+
+pub struct HybridStream {
+    pub hybrid_stream: std::fs::File,
+    pub slave_stream: Option<Box<dyn VsockStream>>,
+}
+
+impl AsRawFd for HybridStream {
+    fn as_raw_fd(&self) -> RawFd {
+        self.hybrid_stream.as_raw_fd()
+    }
+}
+
+impl Read for HybridStream {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        self.hybrid_stream.read(buf)
+    }
+}
+
+impl Write for HybridStream {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        // The slave stream was only used to reply the connect result "ok <port>",
+        // thus it was only used once here, and the data would be replied by the
+        // main stream.
+        if let Some(mut stream) = self.slave_stream.take() {
+            stream.write(buf)
+        } else {
+            self.hybrid_stream.write(buf)
+        }
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.hybrid_stream.flush()
+    }
+}
+
+impl VsockStream for HybridStream {
+    fn backend_type(&self) -> VsockBackendType {
+        VsockBackendType::HybridStream
+    }
+
+    fn set_nonblocking(&mut self, nonblocking: bool) -> std::io::Result<()> {
+        let fd = self.hybrid_stream.as_raw_fd();
+        let mut flag = unsafe { libc::fcntl(fd, libc::F_GETFL) };
+
+        if nonblocking {
+            flag = flag | libc::O_NONBLOCK;
+        } else {
+            flag = flag & !libc::O_NONBLOCK;
+        }
+
+        let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flag) };
+
+        if ret < 0 {
+            error!("failed to set fcntl for fd {} with ret {}", fd, ret);
+            return Err(Error::last_os_error());
+        }
+
+        Ok(())
+    }
+
+    fn set_read_timeout(&mut self, _dur: Option<Duration>) -> std::io::Result<()> {
+        error!("unsupported!");
+        Err(Errno::ENOPROTOOPT.into())
+    }
+
+    fn set_write_timeout(&mut self, _dur: Option<Duration>) -> std::io::Result<()> {
+        error!("unsupported!");
+        Err(Errno::ENOPROTOOPT.into())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn recv_data_fd(
+        &self,
+        _bytes: &mut [u8],
+        _fds: &mut [RawFd],
+    ) -> std::io::Result<(usize, usize)> {
+        Err(Errno::ENOPROTOOPT.into())
+    }
+}

src/dragonball/src/dbs_virtio_devices/src/vsock/backend/mod.rs

@@ -9,13 +9,14 @@ use std::io::{Read, Write};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::time::Duration;
 
+mod hybrid_stream;
 mod inner;
 mod tcp;
 mod unix_stream;
 
+pub use self::hybrid_stream::HybridStream;
 pub use self::inner::{VsockInnerBackend, VsockInnerConnector, VsockInnerStream};
 pub use self::tcp::VsockTcpBackend;
-pub use self::unix_stream::HybridUnixStreamBackend;
 pub use self::unix_stream::VsockUnixStreamBackend;
 
 /// The type of vsock backend.
@@ -27,6 +28,8 @@ pub enum VsockBackendType {
     Tcp,
     /// Inner backend
     Inner,
+    /// Fd passed hybrid stream backend
+    HybridStream,
     /// For test purpose
     #[cfg(test)]
     Test,

src/dragonball/src/dbs_virtio_devices/src/vsock/backend/unix_stream.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::any::Any;
-use std::io::{Read, Write};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::os::unix::net::{UnixListener, UnixStream};
 use std::time::Duration;
@@ -13,66 +12,6 @@ use sendfd::RecvWithFd;
 use super::super::{Result, VsockError};
 use super::{VsockBackend, VsockBackendType, VsockStream};
 
-pub struct HybridUnixStreamBackend {
-    pub unix_stream: Box<dyn VsockStream>,
-    pub slave_stream: Option<Box<dyn VsockStream>>,
-}
-
-impl VsockStream for HybridUnixStreamBackend {
-    fn backend_type(&self) -> VsockBackendType {
-        self.unix_stream.backend_type()
-    }
-
-    fn set_nonblocking(&mut self, nonblocking: bool) -> std::io::Result<()> {
-        self.unix_stream.set_nonblocking(nonblocking)
-    }
-
-    fn set_read_timeout(&mut self, dur: Option<Duration>) -> std::io::Result<()> {
-        self.unix_stream.set_read_timeout(dur)
-    }
-
-    fn set_write_timeout(&mut self, dur: Option<Duration>) -> std::io::Result<()> {
-        self.unix_stream.set_write_timeout(dur)
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self.unix_stream.as_any()
-    }
-
-    fn recv_data_fd(&self, bytes: &mut [u8], fds: &mut [RawFd]) -> std::io::Result<(usize, usize)> {
-        self.unix_stream.recv_data_fd(bytes, fds)
-    }
-}
-
-impl AsRawFd for HybridUnixStreamBackend {
-    fn as_raw_fd(&self) -> RawFd {
-        self.unix_stream.as_raw_fd()
-    }
-}
-
-impl Read for HybridUnixStreamBackend {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        self.unix_stream.read(buf)
-    }
-}
-
-impl Write for HybridUnixStreamBackend {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        // The slave stream was only used to reply the connect result "ok <port>",
-        // thus it was only used once here, and the data would be replied by the
-        // main stream.
-        if let Some(mut stream) = self.slave_stream.take() {
-            stream.write(buf)
-        } else {
-            self.unix_stream.write(buf)
-        }
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        self.unix_stream.flush()
-    }
-}
-
 impl VsockStream for UnixStream {
     fn backend_type(&self) -> VsockBackendType {
         VsockBackendType::UnixStream

src/dragonball/src/dbs_virtio_devices/src/vsock/muxer/mod.rs

@@ -60,6 +60,10 @@ pub enum Error {
     #[error("error connecting to a backend: {0}")]
     BackendConnect(#[source] std::io::Error),
 
+    /// Error set nonblock to a backend stream.
+    #[error("error set nonblocking to a backend: {0}")]
+    BackendSetNonBlock(#[source] std::io::Error),
+
     /// Error reading from backend.
     #[error("error reading from backend: {0}")]
     BackendRead(#[source] std::io::Error),

src/dragonball/src/dbs_virtio_devices/src/vsock/muxer/muxer_impl.rs

@@ -36,14 +36,14 @@
 ///       route all these events to their handlers, the muxer uses another
 ///       `HashMap` object, mapping `RawFd`s to `EpollListener`s.
 use std::collections::{HashMap, HashSet};
+use std::fs::File;
 use std::io::Read;
 use std::os::fd::FromRawFd;
 use std::os::unix::io::{AsRawFd, RawFd};
-use std::os::unix::net::UnixStream;
 
 use log::{debug, error, info, trace, warn};
 
-use super::super::backend::{HybridUnixStreamBackend, VsockBackend, VsockBackendType, VsockStream};
+use super::super::backend::{HybridStream, VsockBackend, VsockBackendType, VsockStream};
 
 use super::super::csm::{ConnState, VsockConnection};
 use super::super::defs::uapi;
@@ -480,13 +480,17 @@ impl VsockMuxer {
                         .and_then(|(nfd, local_port, peer_port)| {
                             // Here we should make sure the nfd the sole owner to convert it
                             // into an UnixStream object, otherwise, it could cause memory unsafety.
-                            let nstream = unsafe { UnixStream::from_raw_fd(nfd) };
+                            let nstream = unsafe { File::from_raw_fd(nfd) };
 
-                            let hybridstream = HybridUnixStreamBackend {
-                                unix_stream: Box::new(nstream),
+                            let mut hybridstream = HybridStream {
+                                hybrid_stream: nstream,
                                 slave_stream: Some(stream),
                             };
 
+                            hybridstream
+                                .set_nonblocking(true)
+                                .map_err(Error::BackendSetNonBlock)?;
+
                             self.add_connection(
                                 ConnMapKey {
                                     local_port,

src/dragonball/src/error.rs

@@ -75,10 +75,6 @@ pub enum Error {
     /// Cannot open the VM file descriptor.
     #[error(transparent)]
     Vm(vm::VmError),
-
-    /// confidential vm type Error
-    #[error("confidential-vm-type can only be used in x86_64 now")]
-    ConfidentialVmType,
 }
 
 /// Errors associated with starting the instance.

src/dragonball/src/hypervisor_metrics.rs

@@ -5,11 +5,14 @@
 
 extern crate procfs;
 
-use crate::metric::{IncMetric, METRICS};
-use anyhow::{anyhow, Result};
-use prometheus::{Encoder, IntCounter, IntGaugeVec, Opts, Registry, TextEncoder};
 use std::sync::Mutex;
 
+use anyhow::{anyhow, Result};
+use dbs_utils::metric::IncMetric;
+use prometheus::{Encoder, IntCounter, IntGaugeVec, Opts, Registry, TextEncoder};
+
+use crate::metric::METRICS;
+
 const NAMESPACE_KATA_HYPERVISOR: &str = "kata_hypervisor";
 
 lazy_static! {
@@ -23,7 +26,7 @@ lazy_static! {
     IntCounter::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"scrape_count"), "Hypervisor metrics scrape count.").unwrap();
 
     static ref HYPERVISOR_VCPU: IntGaugeVec =
-    IntGaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"vcpu"), "Hypervisor metrics specific to VCPUs' mode of functioning."), &["item"]).unwrap();
+    IntGaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"vcpu"), "Hypervisor metrics specific to VCPUs' mode of functioning."), &["cpu_id", "item"]).unwrap();
 
     static ref HYPERVISOR_SECCOMP: IntGaugeVec =
     IntGaugeVec::new(Opts::new(format!("{}_{}",NAMESPACE_KATA_HYPERVISOR,"seccomp"), "Hypervisor metrics for the seccomp filtering."), &["item"]).unwrap();
@@ -75,28 +78,33 @@ fn update_hypervisor_metrics() -> Result<()> {
 }
 
 fn set_intgauge_vec_vcpu(icv: &prometheus::IntGaugeVec) {
-    icv.with_label_values(&["exit_io_in"])
-        .set(METRICS.vcpu.exit_io_in.count() as i64);
-    icv.with_label_values(&["exit_io_out"])
-        .set(METRICS.vcpu.exit_io_out.count() as i64);
-    icv.with_label_values(&["exit_mmio_read"])
-        .set(METRICS.vcpu.exit_mmio_read.count() as i64);
-    icv.with_label_values(&["exit_mmio_write"])
-        .set(METRICS.vcpu.exit_mmio_write.count() as i64);
-    icv.with_label_values(&["failures"])
-        .set(METRICS.vcpu.failures.count() as i64);
-    icv.with_label_values(&["filter_cpuid"])
-        .set(METRICS.vcpu.filter_cpuid.count() as i64);
+    let metric_guard = METRICS.read().unwrap();
+    for (cpu_id, metrics) in metric_guard.vcpu.iter() {
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_io_in"])
+            .set(metrics.exit_io_in.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_io_out"])
+            .set(metrics.exit_io_out.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_mmio_read"])
+            .set(metrics.exit_mmio_read.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "exit_mmio_write"])
+            .set(metrics.exit_mmio_write.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "failures"])
+            .set(metrics.failures.count() as i64);
+        icv.with_label_values(&[cpu_id.to_string().as_str(), "filter_cpuid"])
+            .set(metrics.filter_cpuid.count() as i64);
+    }
 }
 
 fn set_intgauge_vec_seccomp(icv: &prometheus::IntGaugeVec) {
+    let metric_guard = METRICS.read().unwrap();
     icv.with_label_values(&["num_faults"])
-        .set(METRICS.seccomp.num_faults.count() as i64);
+        .set(metric_guard.seccomp.num_faults.count() as i64);
 }
 
 fn set_intgauge_vec_signals(icv: &prometheus::IntGaugeVec) {
+    let metric_guard = METRICS.read().unwrap();
     icv.with_label_values(&["sigbus"])
-        .set(METRICS.signals.sigbus.count() as i64);
+        .set(metric_guard.signals.sigbus.count() as i64);
     icv.with_label_values(&["sigsegv"])
-        .set(METRICS.signals.sigsegv.count() as i64);
+        .set(metric_guard.signals.sigsegv.count() as i64);
 }

src/dragonball/src/kvm_context.rs

@@ -10,7 +10,7 @@ use kvm_bindings::KVM_API_VERSION;
 use kvm_ioctls::{Cap, Kvm, VmFd};
 use std::os::unix::io::{FromRawFd, RawFd};
 
-use crate::error::{Error as VmError, Result};
+use crate::error::{Error, Result};
 
 /// Describes a KVM context that gets attached to the micro VM instance.
 /// It gives access to the functionality of the KVM wrapper as long as every required
@@ -29,11 +29,11 @@ impl KvmContext {
             // Safe because we expect kvm_fd to contain a valid fd number when is_some() == true.
             unsafe { Kvm::from_raw_fd(fd) }
         } else {
-            Kvm::new().map_err(VmError::Kvm)?
+            Kvm::new().map_err(Error::Kvm)?
         };
 
         if kvm.get_api_version() != KVM_API_VERSION as i32 {
-            return Err(VmError::KvmApiVersion(kvm.get_api_version()));
+            return Err(Error::KvmApiVersion(kvm.get_api_version()));
         }
 
         Self::check_cap(&kvm, Cap::Irqchip)?;
@@ -44,8 +44,7 @@ impl KvmContext {
         Self::check_cap(&kvm, Cap::SetTssAddr)?;
 
         #[cfg(target_arch = "x86_64")]
-        let supported_msrs =
-            dbs_arch::msr::supported_guest_msrs(&kvm).map_err(VmError::GuestMSRs)?;
+        let supported_msrs = dbs_arch::msr::supported_guest_msrs(&kvm).map_err(Error::GuestMSRs)?;
         let max_memslots = kvm.get_nr_memslots();
 
         Ok(KvmContext {
@@ -68,7 +67,7 @@ impl KvmContext {
 
     /// Create a virtual machine object.
     pub fn create_vm(&self) -> Result<VmFd> {
-        self.kvm.create_vm().map_err(VmError::Kvm)
+        self.kvm.create_vm().map_err(Error::Kvm)
     }
 
     /// Get the max vcpu count supported by kvm
@@ -76,9 +75,9 @@ impl KvmContext {
         self.kvm.get_max_vcpus()
     }
 
-    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), VmError> {
+    fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), Error> {
         if !kvm.check_extension(cap) {
-            return Err(VmError::KvmCap(cap));
+            return Err(Error::KvmCap(cap));
         }
         Ok(())
     }
@@ -92,18 +91,6 @@ mod x86_64 {
     use std::collections::HashSet;
 
     impl KvmContext {
-        /// Create a virtual machine object with specific type.
-        /// vm_type: u64
-        ///      0: legacy vm
-        ///      2: tdx vm
-        pub fn create_vm_with_type(&self, vm_type: u64) -> Result<VmFd> {
-            let fd = self
-                .kvm
-                .create_vm_with_type(vm_type)
-                .map_err(VmError::Kvm)?;
-            Ok(fd)
-        }
-
         /// Get information about supported CPUID of x86 processor.
         pub fn supported_cpuid(
             &self,
@@ -123,7 +110,7 @@ mod x86_64 {
         // It's very sensible to manipulate MSRs, so please be careful to change code below.
         fn build_msrs_list(kvm: &Kvm) -> Result<Msrs> {
             let mut mset: HashSet<u32> = HashSet::new();
-            let supported_msr_list = kvm.get_msr_index_list().map_err(VmError::Kvm)?;
+            let supported_msr_list = kvm.get_msr_index_list().map_err(super::Error::Kvm)?;
             for msr in supported_msr_list.as_slice() {
                 mset.insert(*msr);
             }
@@ -216,7 +203,7 @@ mod x86_64 {
                 })
                 .collect();
 
-            Msrs::from_entries(&msrs).map_err(VmError::Msr)
+            Msrs::from_entries(&msrs).map_err(super::Error::Msr)
         }
     }
 }
@@ -270,20 +257,4 @@ mod tests {
 
         let _ = c.create_vm().unwrap();
     }
-
-    #[test]
-    fn test_create_vm_with_type() {
-        let c = KvmContext::new(None).unwrap();
-        #[cfg(not(target_arch = "aarch64"))]
-        let _ = c.create_vm_with_type(0_u64).unwrap();
-        #[cfg(target_arch = "aarch64")]
-        {
-            /// aarch64 is using ipa_size to create vm
-            let mut ipa_size = 0; // Create using default VM type
-            if c.check_extension(kvm_ioctls::Cap::ArmVmIPASize) {
-                ipa_size = c.kvm.get_host_ipa_limit();
-            }
-            let _ = c.create_vm_with_type(ipa_size as u64).unwrap();
-        }
-    }
 }

src/dragonball/src/metric.rs

@@ -2,15 +2,20 @@
 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
+
 use dbs_utils::metric::SharedIncMetric;
 use lazy_static::lazy_static;
 use serde::Serialize;
 
-pub use dbs_utils::metric::IncMetric;
-
 lazy_static! {
-    /// Static instance used for handling metrics.
-    pub static ref METRICS: DragonballMetrics = DragonballMetrics::default();
+    /// # Static instance used for handling metrics.
+    ///
+    /// Using a big lock over the DragonballMetrics since we have various device metric types
+    /// and the write operation is only used when creating or removing devices, it has a low
+    /// competitive overhead.
+    pub static ref METRICS: RwLock<DragonballMetrics> = RwLock::new(DragonballMetrics::default());
 }
 
 /// Metrics specific to VCPUs' mode of functioning.
@@ -50,9 +55,121 @@ pub struct SignalMetrics {
 #[derive(Default, Serialize)]
 pub struct DragonballMetrics {
     /// Metrics related to a vcpu's functioning.
-    pub vcpu: VcpuMetrics,
+    pub vcpu: HashMap<u32, Arc<VcpuMetrics>>,
     /// Metrics related to seccomp filtering.
     pub seccomp: SeccompMetrics,
     /// Metrics related to signals.
     pub signals: SignalMetrics,
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+    use std::thread;
+
+    use dbs_utils::metric::IncMetric;
+
+    use crate::metric::{VcpuMetrics, METRICS};
+
+    #[test]
+    fn test_read_map() {
+        let metrics = Arc::new(VcpuMetrics::default());
+        let vcpu_id: u32 = u32::MIN;
+        METRICS
+            .write()
+            .unwrap()
+            .vcpu
+            .insert(vcpu_id, metrics.clone());
+        metrics.failures.inc();
+        assert_eq!(
+            METRICS
+                .read()
+                .unwrap()
+                .vcpu
+                .get(&vcpu_id)
+                .unwrap()
+                .failures
+                .count(),
+            1
+        );
+    }
+
+    #[test]
+    fn test_metrics_count() {
+        let metrics = Arc::new(VcpuMetrics::default());
+        let vcpu_id: u32 = 65535;
+        METRICS
+            .write()
+            .unwrap()
+            .vcpu
+            .insert(vcpu_id, metrics.clone());
+
+        let metrics1 = metrics.clone();
+        let thread1 = thread::spawn(move || {
+            for _i in 0..10 {
+                metrics1.exit_io_in.inc();
+            }
+        });
+
+        let metrics2 = metrics.clone();
+        let thread2 = thread::spawn(move || {
+            for _i in 0..10 {
+                metrics2.exit_io_in.inc();
+            }
+        });
+        thread1.join().unwrap();
+        thread2.join().unwrap();
+        assert_eq!(
+            METRICS
+                .read()
+                .unwrap()
+                .vcpu
+                .get(&vcpu_id)
+                .unwrap()
+                .exit_io_in
+                .count(),
+            20
+        );
+    }
+
+    #[test]
+    fn test_rw_lock() {
+        let metrics = Arc::new(VcpuMetrics::default());
+        let vcpu_id: u32 = u32::MAX;
+        METRICS
+            .write()
+            .unwrap()
+            .vcpu
+            .insert(vcpu_id, metrics.clone());
+
+        let write_thread = thread::spawn(move || {
+            for _ in 0..10 {
+                let metrics = Arc::new(VcpuMetrics::default());
+                let vcpu_id: u32 = 128;
+                METRICS
+                    .write()
+                    .unwrap()
+                    .vcpu
+                    .insert(vcpu_id, metrics.clone());
+            }
+        });
+
+        let read_thread = thread::spawn(move || {
+            for _ in 0..10 {
+                assert_eq!(
+                    METRICS
+                        .read()
+                        .unwrap()
+                        .vcpu
+                        .get(&vcpu_id)
+                        .unwrap()
+                        .failures
+                        .count(),
+                    0
+                );
+            }
+        });
+        write_thread.join().unwrap();
+        read_thread.join().unwrap();
+    }
+}

src/dragonball/src/signal_handler.rs

@@ -1,11 +1,12 @@
 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+use dbs_utils::metric::IncMetric;
 use libc::{_exit, c_int, c_void, siginfo_t, SIGBUS, SIGSEGV, SIGSYS};
 use log::error;
 use vmm_sys_util::signal::register_signal_handler;
 
-use crate::metric::{IncMetric, METRICS};
+use crate::metric::METRICS;
 
 // The offset of `si_syscall` (offending syscall identifier) within the siginfo structure
 // expressed as an `(u)int*`.
@@ -51,7 +52,7 @@ extern "C" fn sigsys_handler(num: c_int, info: *mut siginfo_t, _unused: *mut c_v
     let syscall = unsafe { *(info as *const i32).offset(SI_OFF_SYSCALL) as usize };
     // SIGSYS is triggered when bad syscalls are detected. num_faults is only added when SIGSYS is detected
     // so it actually only collects the count for bad syscalls.
-    METRICS.seccomp.num_faults.inc();
+    METRICS.read().unwrap().seccomp.num_faults.inc();
     error!(
         "Shutting down VM after intercepting a bad syscall ({}).",
         syscall
@@ -82,8 +83,8 @@ extern "C" fn sigbus_sigsegv_handler(num: c_int, info: *mut siginfo_t, _unused:
     // Other signals which might do async unsafe things incompatible with the rest of this
     // function are blocked due to the sa_mask used when registering the signal handler.
     match si_signo {
-        SIGBUS => METRICS.signals.sigbus.inc(),
-        SIGSEGV => METRICS.signals.sigsegv.inc(),
+        SIGBUS => METRICS.read().unwrap().signals.sigbus.inc(),
+        SIGSEGV => METRICS.read().unwrap().signals.sigsegv.inc(),
         _ => (),
     }
 
@@ -182,19 +183,19 @@ mod tests {
             .unwrap();
 
             assert!(apply_filter(&TryInto::<BpfProgram>::try_into(filter).unwrap()).is_ok());
-            assert_eq!(METRICS.seccomp.num_faults.count(), 0);
+            assert_eq!(METRICS.read().unwrap().seccomp.num_faults.count(), 0);
 
             // Call the blacklisted `SYS_mkdirat`.
             unsafe { syscall(libc::SYS_mkdirat, "/foo/bar\0") };
 
             // Call SIGBUS signal handler.
-            assert_eq!(METRICS.signals.sigbus.count(), 0);
+            assert_eq!(METRICS.read().unwrap().signals.sigbus.count(), 0);
             unsafe {
                 syscall(libc::SYS_kill, process::id(), SIGBUS);
             }
 
             // Call SIGSEGV signal handler.
-            assert_eq!(METRICS.signals.sigsegv.count(), 0);
+            assert_eq!(METRICS.read().unwrap().signals.sigsegv.count(), 0);
             unsafe {
                 syscall(libc::SYS_kill, process::id(), SIGSEGV);
             }
@@ -211,9 +212,9 @@ mod tests {
         // tests, so we use this as an heuristic to decide if we check the assertion.
         if cpu_count() > 1 {
             // The signal handler should let the program continue during unit tests.
-            assert!(METRICS.seccomp.num_faults.count() >= 1);
+            assert!(METRICS.read().unwrap().seccomp.num_faults.count() >= 1);
         }
-        assert!(METRICS.signals.sigbus.count() >= 1);
-        assert!(METRICS.signals.sigsegv.count() >= 1);
+        assert!(METRICS.read().unwrap().signals.sigbus.count() >= 1);
+        assert!(METRICS.read().unwrap().signals.sigsegv.count() >= 1);
     }
 }

src/dragonball/src/vcpu/aarch64.rs

@@ -10,7 +10,6 @@ use std::ops::Deref;
 use std::sync::mpsc::{channel, Sender};
 use std::sync::Arc;
 
-use crate::IoManagerCached;
 use dbs_arch::{regs, VpmuFeatureLevel};
 use dbs_boot::get_fdt_addr;
 use dbs_utils::time::TimestampUs;
@@ -19,8 +18,10 @@ use vm_memory::{Address, GuestAddress, GuestAddressSpace};
 use vmm_sys_util::eventfd::EventFd;
 
 use crate::address_space_manager::GuestAddressSpaceImpl;
+use crate::metric::VcpuMetrics;
 use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuError, VcpuStateEvent};
 use crate::vcpu::VcpuConfig;
+use crate::IoManagerCached;
 
 #[allow(unused)]
 impl Vcpu {
@@ -67,6 +68,7 @@ impl Vcpu {
             support_immediate_exit,
             mpidr: 0,
             exit_evt,
+            metrics: Arc::new(VcpuMetrics::default()),
         })
     }
 

src/dragonball/src/vcpu/vcpu_impl.rs

@@ -15,6 +15,7 @@ use std::sync::mpsc::{Receiver, Sender, TryRecvError};
 use std::sync::{Arc, Barrier};
 use std::thread;
 
+use dbs_utils::metric::IncMetric;
 use dbs_utils::time::TimestampUs;
 use kvm_bindings::{KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN};
 use kvm_ioctls::{VcpuExit, VcpuFd};
@@ -25,7 +26,7 @@ use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::{register_signal_handler, Killable};
 
 use super::sm::StateMachine;
-use crate::metric::{IncMetric, METRICS};
+use crate::metric::{VcpuMetrics, METRICS};
 use crate::signal_handler::sigrtmin;
 use crate::IoManagerCached;
 
@@ -303,6 +304,9 @@ pub struct Vcpu {
     // Whether kvm used supports immediate_exit flag.
     support_immediate_exit: bool,
 
+    // metrics for a vCPU.
+    metrics: Arc<VcpuMetrics>,
+
     // CPUID information for the x86_64 CPU
     #[cfg(target_arch = "x86_64")]
     cpuid: kvm_bindings::CpuId,
@@ -446,7 +450,7 @@ impl Vcpu {
                     #[cfg(target_arch = "x86_64")]
                     VcpuExit::IoIn(addr, data) => {
                         let _ = self.io_mgr.pio_read(addr, data);
-                        METRICS.vcpu.exit_io_in.inc();
+                        self.metrics.exit_io_in.inc();
                         Ok(VcpuEmulation::Handled)
                     }
                     #[cfg(target_arch = "x86_64")]
@@ -454,17 +458,17 @@ impl Vcpu {
                         if !self.check_io_port_info(addr, data)? {
                             let _ = self.io_mgr.pio_write(addr, data);
                         }
-                        METRICS.vcpu.exit_io_out.inc();
+                        self.metrics.exit_io_out.inc();
                         Ok(VcpuEmulation::Handled)
                     }
                     VcpuExit::MmioRead(addr, data) => {
                         let _ = self.io_mgr.mmio_read(addr, data);
-                        METRICS.vcpu.exit_mmio_read.inc();
+                        self.metrics.exit_mmio_read.inc();
                         Ok(VcpuEmulation::Handled)
                     }
                     VcpuExit::MmioWrite(addr, data) => {
                         let _ = self.io_mgr.mmio_write(addr, data);
-                        METRICS.vcpu.exit_mmio_write.inc();
+                        self.metrics.exit_mmio_write.inc();
                         Ok(VcpuEmulation::Handled)
                     }
                     VcpuExit::Hlt => {
@@ -477,12 +481,12 @@ impl Vcpu {
                     }
                     // Documentation specifies that below kvm exits are considered errors.
                     VcpuExit::FailEntry(reason, cpu) => {
-                        METRICS.vcpu.failures.inc();
+                        self.metrics.failures.inc();
                         error!("Received KVM_EXIT_FAIL_ENTRY signal, reason {reason}, cpu number {cpu}");
                         Err(VcpuError::VcpuUnhandledKvmExit)
                     }
                     VcpuExit::InternalError => {
-                        METRICS.vcpu.failures.inc();
+                        self.metrics.failures.inc();
                         error!("Received KVM_EXIT_INTERNAL_ERROR signal");
                         Err(VcpuError::VcpuUnhandledKvmExit)
                     }
@@ -495,7 +499,7 @@ impl Vcpu {
                             Ok(VcpuEmulation::Stopped)
                         }
                         _ => {
-                            METRICS.vcpu.failures.inc();
+                            self.metrics.failures.inc();
                             error!(
                                 "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}",
                                 event_type, event_flags
@@ -504,7 +508,7 @@ impl Vcpu {
                         }
                     },
                     r => {
-                        METRICS.vcpu.failures.inc();
+                        self.metrics.failures.inc();
                         // TODO: Are we sure we want to finish running a vcpu upon
                         // receiving a vm exit that is not necessarily an error?
                         error!("Unexpected exit reason on vcpu run: {:?}", r);
@@ -523,7 +527,7 @@ impl Vcpu {
                         Ok(VcpuEmulation::Interrupted)
                     }
                     _ => {
-                        METRICS.vcpu.failures.inc();
+                        self.metrics.failures.inc();
                         error!("Failure during vcpu run: {}", e);
                         #[cfg(target_arch = "x86_64")]
                         {
@@ -731,7 +735,7 @@ impl Vcpu {
     fn waiting_exit(&mut self) -> StateMachine<Self> {
         // trigger vmm to stop machine
         if let Err(e) = self.exit_evt.write(1) {
-            METRICS.vcpu.failures.inc();
+            self.metrics.failures.inc();
             error!("Failed signaling vcpu exit event: {}", e);
         }
 
@@ -765,11 +769,17 @@ impl Vcpu {
     pub fn vcpu_fd(&self) -> &VcpuFd {
         self.fd.as_ref()
     }
+
+    pub fn metrics(&self) -> Arc<VcpuMetrics> {
+        self.metrics.clone()
+    }
 }
 
 impl Drop for Vcpu {
     fn drop(&mut self) {
         let _ = self.reset_thread_local_data();
+        let id: u32 = self.id as u32;
+        METRICS.write().unwrap().vcpu.remove(&id);
     }
 }