Compare commits

..

1 Commits

Author SHA1 Message Date
dependabot[bot]
6d5c9de7dc build(deps): bump const_format from 0.2.34 to 0.2.35 in /src/agent
Bumps [const_format](https://github.com/rodrimati1992/const_format_crates) from 0.2.34 to 0.2.35.
- [Release notes](https://github.com/rodrimati1992/const_format_crates/releases)
- [Changelog](https://github.com/rodrimati1992/const_format_crates/blob/master/Changelog.md)
- [Commits](https://github.com/rodrimati1992/const_format_crates/commits)

---
updated-dependencies:
- dependency-name: const_format
  dependency-version: 0.2.35
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-03-11 16:04:24 +00:00
38 changed files with 425 additions and 522 deletions

View File

@@ -168,6 +168,8 @@ jobs:
- rootfs-image-nvidia-gpu-confidential
- rootfs-initrd
- rootfs-initrd-confidential
- rootfs-initrd-nvidia-gpu
- rootfs-initrd-nvidia-gpu-confidential
steps:
- name: Login to Kata Containers quay.io
if: ${{ inputs.push-to-registry == 'yes' }}
@@ -365,6 +367,7 @@ jobs:
matrix:
asset:
- agent-ctl
- csi-kata-directvolume
- genpolicy
- kata-ctl
- kata-manager

View File

@@ -152,6 +152,7 @@ jobs:
- rootfs-image
- rootfs-image-nvidia-gpu
- rootfs-initrd
- rootfs-initrd-nvidia-gpu
steps:
- name: Login to Kata Containers quay.io
if: ${{ inputs.push-to-registry == 'yes' }}

View File

@@ -19,25 +19,23 @@ permissions: {}
jobs:
scan-scheduled:
name: Scan of whole repo
permissions:
actions: read # # Required to upload SARIF file to CodeQL
contents: read # Read commit contents
security-events: write # Require writing security events to upload SARIF file to security tab
if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@8ae4be80636b94886b3c271caad730985ce0611c" # v2.3.3
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@b00f71e051ddddc6e46a193c31c8c0bf283bf9e6" # v2.1.0
with:
scan-args: |-
-r
./
scan-pr:
name: Scan of just PR code
permissions:
actions: read # Required to upload SARIF file to CodeQL
contents: read # Read commit contents
security-events: write # Require writing security events to upload SARIF file to security tab
if: ${{ github.event_name == 'pull_request' }}
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@8ae4be80636b94886b3c271caad730985ce0611c" # v2.3.3
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@b00f71e051ddddc6e46a193c31c8c0bf283bf9e6" # v2.1.0
with:
# Example of specifying custom arguments
scan-args: |-

View File

@@ -110,6 +110,10 @@ jobs:
timeout-minutes: 10
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
- name: Deploy CSI driver
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
- name: Run tests
timeout-minutes: 100
run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -130,6 +134,10 @@ jobs:
[[ "${KATA_HYPERVISOR}" == "qemu-tdx" ]] && echo "ITA_KEY=${GH_ITA_KEY}" >> "${GITHUB_ENV}"
bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
- name: Delete CSI driver
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
# Generate jobs for testing CoCo on non-TEE environments
run-k8s-tests-coco-nontee:
name: run-k8s-tests-coco-nontee
@@ -227,6 +235,10 @@ jobs:
timeout-minutes: 10
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
- name: Deploy CSI driver
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
- name: Run tests
timeout-minutes: 80
run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -245,6 +257,11 @@ jobs:
timeout-minutes: 10
run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
- name: Delete CSI driver
if: always()
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
# Extensive matrix: autogenerated policy tests (nydus + experimental-force-guest-pull) on k0s, k3s, rke2, microk8s with qemu-coco-dev / qemu-coco-dev-runtime-rs
run-k8s-tests-coco-nontee-extensive-matrix:
if: ${{ inputs.extensive-matrix-autogenerated-policy == 'yes' }}
@@ -348,6 +365,10 @@ jobs:
timeout-minutes: 10
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
- name: Deploy CSI driver
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
- name: Run tests
timeout-minutes: 80
run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -366,6 +387,11 @@ jobs:
timeout-minutes: 10
run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
- name: Delete CSI driver
if: always()
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
# Generate jobs for testing CoCo on non-TEE environments with erofs-snapshotter
run-k8s-tests-coco-nontee-with-erofs-snapshotter:
name: run-k8s-tests-coco-nontee-with-erofs-snapshotter
@@ -452,6 +478,10 @@ jobs:
timeout-minutes: 20
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
- name: Deploy CSI driver
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
- name: Run tests
timeout-minutes: 80
run: bash tests/integration/kubernetes/gha-run.sh run-tests
@@ -464,3 +494,8 @@ jobs:
if: always()
timeout-minutes: 15
run: bash tests/integration/kubernetes/gha-run.sh cleanup
- name: Delete CSI driver
if: always()
timeout-minutes: 5
run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver

View File

@@ -1 +1 @@
3.28.0
3.27.0

View File

@@ -1,64 +1,57 @@
# How to do a Kata Containers Release
This document lists the tasks required to create a Kata Release.
## Requirements
- GitHub permissions to run workflows.
## Release Model
## Versioning
Kata Containers follows a rolling release model with monthly snapshots.
New features, bug fixes, and improvements are continuously integrated into
`main`. Each month, a snapshot is tagged as a new `MINOR` release.
The Kata Containers project uses [semantic versioning](http://semver.org/) for all releases.
Semantic versions are comprised of three fields in the form:
### Versioning
```
MAJOR.MINOR.PATCH
```
Releases use the `MAJOR.MINOR.PATCH` scheme. Monthly snapshots increment
`MINOR`; `PATCH` is typically `0`. Major releases are rare (years apart) and
signal significant architectural changes that may require updates to container
managers (Containerd, CRI-O) or other infrastructure. Breaking changes in
`MINOR` releases are avoided where possible, but may occasionally occur as
features are deprecated or removed.
When `MINOR` increases, the new release adds **new features** but *without changing the existing behavior*.
### No Stable Branches
When `MAJOR` increases, the new release adds **new features, bug fixes, or
both** and which **changes the behavior from the previous release** (incompatible with previous releases).
The Kata Containers project does not maintain stable branches (see
[#9064](https://github.com/kata-containers/kata-containers/issues/9064)).
Bug fixes land on `main` and ship in the next monthly snapshot rather than
being backported. Downstream projects that need extended support or compliance
certifications should select a monthly snapshot as their stable base and manage
their own validation and patch backporting from there.
A major release will also likely require a change of the container manager version used,
-for example Containerd or CRI-O. Please refer to the release notes for further details.
**Important** : the Kata Containers project doesn't have stable branches (see
[this issue](https://github.com/kata-containers/kata-containers/issues/9064) for details).
Bug fixes are released as part of `MINOR` or `MAJOR` releases only. `PATCH` is always `0`.
## Release Process
### Bump the `VERSION` and `Chart.yaml` file
When the `kata-containers/kata-containers` repository is ready for a new
release, first create a PR to set the release in the [`VERSION`](./../VERSION)
file and update the `version` and `appVersion` in the
[`Chart.yaml`](./../tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml)
file and have it merged.
When the `kata-containers/kata-containers` repository is ready for a new release,
first create a PR to set the release in the [`VERSION`](./../VERSION) file and update the
`version` and `appVersion` in the
[`Chart.yaml`](./../tools/packaging/kata-deploy/helm-chart/kata-deploy/Chart.yaml) file and
have it merged.
### Lock the `main` branch
In order to prevent any PRs getting merged during the release process, and
slowing the release process down, by impacting the payload caches, we have
recently trialed setting the `main` branch to read only whilst the release
action runs.
In order to prevent any PRs getting merged during the release process, and slowing the release
process down, by impacting the payload caches, we have recently trailed setting the `main`
branch to read only whilst the release action runs.
> [!NOTE]
> Admin permission is needed to complete this task.
### Wait for the `VERSION` bump PR payload publish to complete
To reduce the chance of need to re-run the release workflow, check the [CI |
Publish Kata Containers
payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml)
To reduce the chance of need to re-run the release workflow, check the
[CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml)
once the `VERSION` PR bump has merged to check that the assets build correctly
and are cached, so that the release process can just download these artifacts
rather than needing to build them all, which takes time and can reveal errors in
infra.
rather than needing to build them all, which takes time and can reveal errors in infra.
### Check GitHub Actions
@@ -70,10 +63,11 @@ release artifacts.
> [!NOTE]
> Write permissions to trigger the action.
The action is manually triggered and is responsible for generating a new release
(including a new tag), pushing those to the `kata-containers/kata-containers`
repository. The new release is initially created as a draft. It is promoted to
an official release when the whole workflow has completed successfully.
The action is manually triggered and is responsible for generating a new
release (including a new tag), pushing those to the
`kata-containers/kata-containers` repository. The new release is initially
created as a draft. It is promoted to an official release when the whole
workflow has completed successfully.
Check the [actions status
page](https://github.com/kata-containers/kata-containers/actions) to verify all
@@ -81,13 +75,12 @@ steps in the actions workflow have completed successfully. On success, a static
tarball containing Kata release artifacts will be uploaded to the [Release
page](https://github.com/kata-containers/kata-containers/releases).
If the workflow fails because of some external environmental causes, e.g.
network timeout, simply re-run the failed jobs until they eventually succeed.
If the workflow fails because of some external environmental causes, e.g. network
timeout, simply re-run the failed jobs until they eventually succeed.
If for some reason you need to cancel the workflow or re-run it entirely, go
first to the [Release
page](https://github.com/kata-containers/kata-containers/releases) and delete
the draft release from the previous run.
If for some reason you need to cancel the workflow or re-run it entirely, go first
to the [Release page](https://github.com/kata-containers/kata-containers/releases) and
delete the draft release from the previous run.
### Unlock the `main` branch
@@ -97,8 +90,9 @@ an admin to do it.
### Improve the release notes
Release notes are auto-generated by the GitHub CLI tool used as part of our
release workflow. However, some manual tweaking may still be necessary in order
to highlight the most important features and bug fixes in a specific release.
release workflow. However, some manual tweaking may still be necessary in
order to highlight the most important features and bug fixes in a specific
release.
With this in mind, please, poke @channel on #kata-dev and people who worked on
the release will be able to contribute to that.

4
src/agent/Cargo.lock generated
View File

@@ -754,9 +754,9 @@ dependencies = [
[[package]]
name = "const_format"
version = "0.2.34"
version = "0.2.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd"
checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad"
dependencies = [
"const_format_proc_macros",
]

View File

@@ -27,7 +27,7 @@ regex = "1.10.5"
serial_test = "0.10.0"
url = "2.5.0"
derivative = "2.2.0"
const_format = "0.2.30"
const_format = "0.2.35"
# Async helpers
async-trait = "0.1.50"

View File

@@ -2317,13 +2317,8 @@ async fn cdh_handler_trusted_storage(oci: &mut Spec) -> Result<()> {
for specdev in devices.iter() {
if specdev.path().as_path().to_str() == Some(TRUSTED_IMAGE_STORAGE_DEVICE) {
let dev_major_minor = format!("{}:{}", specdev.major(), specdev.minor());
cdh_secure_mount(
"block-device",
&dev_major_minor,
"luks2",
KATA_IMAGE_WORK_DIR,
)
.await?;
cdh_secure_mount("BlockDevice", &dev_major_minor, "LUKS", KATA_IMAGE_WORK_DIR)
.await?;
break;
}
}
@@ -2354,21 +2349,10 @@ pub(crate) async fn cdh_secure_mount(
let options = std::collections::HashMap::from([
("deviceId".to_string(), device_id.to_string()),
("sourceType".to_string(), "empty".to_string()),
("targetType".to_string(), "fileSystem".to_string()),
("filesystemType".to_string(), "ext4".to_string()),
("mkfsOpts".to_string(), "-E lazy_journal_init".to_string()),
("encryptionType".to_string(), encrypt_type.to_string()),
("encryptType".to_string(), encrypt_type.to_string()),
("dataIntegrity".to_string(), integrity),
]);
std::fs::create_dir_all(mount_point).inspect_err(|e| {
error!(
sl(),
"Failed to create mount point directory {}: {:?}", mount_point, e
);
})?;
confidential_data_hub::secure_mount(device_type, &options, vec![], mount_point).await?;
Ok(())

View File

@@ -59,8 +59,7 @@ async fn handle_block_storage(
.contains(&"encryption_key=ephemeral".to_string());
if has_ephemeral_encryption {
crate::rpc::cdh_secure_mount("block-device", dev_num, "luks2", &storage.mount_point)
.await?;
crate::rpc::cdh_secure_mount("BlockDevice", dev_num, "LUKS", &storage.mount_point).await?;
set_ownership(logger, storage)?;
new_device(storage.mount_point.clone())
} else {

View File

@@ -24,7 +24,9 @@ message SecureMountRequest {
string mount_point = 4;
}
message SecureMountResponse {}
message SecureMountResponse {
string mount_path = 1;
}
message ImagePullRequest {
// - `image_url`: The reference of the image to pull

View File

@@ -65,6 +65,8 @@ INITRDCONFIDENTIALNAME = $(PROJECT_TAG)-initrd-confidential.img
IMAGENAME_NV = $(PROJECT_TAG)-nvidia-gpu.img
IMAGENAME_CONFIDENTIAL_NV = $(PROJECT_TAG)-nvidia-gpu-confidential.img
INITRDNAME_NV = $(PROJECT_TAG)-initrd-nvidia-gpu.img
INITRDNAME_CONFIDENTIAL_NV = $(PROJECT_TAG)-initrd-nvidia-gpu-confidential.img
TARGET = $(BIN_PREFIX)-runtime
RUNTIME_OUTPUT = $(CURDIR)/$(TARGET)
@@ -134,6 +136,8 @@ INITRDCONFIDENTIALPATH := $(PKGDATADIR)/$(INITRDCONFIDENTIALNAME)
IMAGEPATH_NV := $(PKGDATADIR)/$(IMAGENAME_NV)
IMAGEPATH_CONFIDENTIAL_NV := $(PKGDATADIR)/$(IMAGENAME_CONFIDENTIAL_NV)
INITRDPATH_NV := $(PKGDATADIR)/$(INITRDNAME_NV)
INITRDPATH_CONFIDENTIAL_NV := $(PKGDATADIR)/$(INITRDNAME_CONFIDENTIAL_NV)
ROOTFSTYPE_EXT4 := \"ext4\"
ROOTFSTYPE_XFS := \"xfs\"
@@ -479,12 +483,16 @@ ifneq (,$(QEMUCMD))
KERNELPATH_CONFIDENTIAL_NV = $(KERNELDIR)/$(KERNELNAME_CONFIDENTIAL_NV)
DEFAULTVCPUS_NV = 1
DEFAULTMEMORY_NV = 8192
DEFAULTMEMORY_NV = 2048
DEFAULTTIMEOUT_NV = 1200
DEFAULTVFIOPORT_NV = root-port
DEFAULTPCIEROOTPORT_NV = 8
# Disable the devtmpfs mount in guest. NVRC does this, and later kata-agent
# attempts this as well in a non-failing manner. Otherwise, NVRC fails when
# using an image and /dev is already mounted.
KERNELPARAMS_NV = "cgroup_no_v1=all"
KERNELPARAMS_NV += "devtmpfs.mount=0"
KERNELPARAMS_NV += "pci=realloc"
KERNELPARAMS_NV += "pci=nocrs"
KERNELPARAMS_NV += "pci=assign-busses"
@@ -652,6 +660,10 @@ USER_VARS += IMAGENAME_NV
USER_VARS += IMAGENAME_CONFIDENTIAL_NV
USER_VARS += IMAGEPATH_NV
USER_VARS += IMAGEPATH_CONFIDENTIAL_NV
USER_VARS += INITRDNAME_NV
USER_VARS += INITRDNAME_CONFIDENTIAL_NV
USER_VARS += INITRDPATH_NV
USER_VARS += INITRDPATH_CONFIDENTIAL_NV
USER_VARS += KERNELNAME_NV
USER_VARS += KERNELPATH_NV
USER_VARS += KERNELNAME_CONFIDENTIAL_NV

View File

@@ -599,7 +599,7 @@ debug_console_enabled = false
# Agent connection dialing timeout value in seconds
# (default: 90)
dial_timeout = @DEFAULTTIMEOUT_NV@
dial_timeout = 90
[runtime]
# If enabled, the runtime will log additional debug messages to the

View File

@@ -576,7 +576,7 @@ debug_console_enabled = false
# Agent connection dialing timeout value in seconds
# (default: 90)
dial_timeout = @DEFAULTTIMEOUT_NV@
dial_timeout = 90
[runtime]
# If enabled, the runtime will log additional debug messages to the

View File

@@ -578,7 +578,7 @@ debug_console_enabled = false
# Agent connection dialing timeout value in seconds
# (default: 90)
dial_timeout = @DEFAULTTIMEOUT_NV@
dial_timeout = 90
[runtime]
# If enabled, the runtime will log additional debug messages to the

View File

@@ -1,7 +1,7 @@
module github.com/kata-containers/kata-containers/src/runtime
// Keep in sync with version in versions.yaml
go 1.25.8
go 1.25.7
// WARNING: Do NOT use `replace` directives as those break dependabot:
// https://github.com/kata-containers/kata-containers/issues/11020

View File

@@ -72,7 +72,7 @@ func IsPCIeDevice(bdf string) bool {
}
// read from /sys/bus/pci/devices/xxx/property
func GetPCIDeviceProperty(bdf string, property PCISysFsProperty) string {
func getPCIDeviceProperty(bdf string, property PCISysFsProperty) string {
if len(strings.Split(bdf, ":")) == 2 {
bdf = PCIDomain + ":" + bdf
}
@@ -220,9 +220,9 @@ func GetDeviceFromVFIODev(device config.DeviceInfo) ([]*config.VFIODev, error) {
return nil, err
}
vendorID := GetPCIDeviceProperty(deviceBDF, PCISysFsDevicesVendor)
deviceID := GetPCIDeviceProperty(deviceBDF, PCISysFsDevicesDevice)
pciClass := GetPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
vendorID := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesVendor)
deviceID := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesDevice)
pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
i, err := extractIndex(device.HostPath)
if err != nil {
@@ -276,7 +276,7 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
switch vfioDeviceType {
case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
// This is vfio-pci and vfio-mdev specific
pciClass := GetPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
pciClass := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesClass)
// We need to ignore Host or PCI Bridges that are in the same IOMMU group as the
// passed-through devices. One CANNOT pass-through a PCI bridge or Host bridge.
// Class 0x0604 is PCI bridge, 0x0600 is Host bridge
@@ -288,8 +288,8 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
continue
}
// Fetch the PCI Vendor ID and Device ID
vendorID := GetPCIDeviceProperty(deviceBDF, PCISysFsDevicesVendor)
deviceID := GetPCIDeviceProperty(deviceBDF, PCISysFsDevicesDevice)
vendorID := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesVendor)
deviceID := getPCIDeviceProperty(deviceBDF, PCISysFsDevicesDevice)
// Do not directly assign to `vfio` -- need to access field still
vfio = config.VFIODev{

View File

@@ -7,7 +7,6 @@ package virtcontainers
import (
"context"
"errors"
"fmt"
"io"
"os"
@@ -1136,9 +1135,7 @@ func (c *Container) createDevices(ctx context.Context, contConfig *ContainerConf
// If we're hot-plugging this will be a no-op because at this stage
// no devices are attached to the root-port or switch-port
if err := c.annotateContainerWithVFIOMetadata(vfioColdPlugDevices); err != nil {
return err
}
c.annotateContainerWithVFIOMetadata(vfioColdPlugDevices)
return nil
}
@@ -1197,40 +1194,11 @@ func sortContainerVFIODevices(devices []config.DeviceInfo) []config.DeviceInfo {
return vfioDevices
}
// errNoSiblingFound is returned by siblingAnnotation when the VFIO device is
// not of a supported CDI device type, i.e. it has no entry in the cdiDeviceKind
// table (e.g. NVSwitches). Callers should treat this as a non-fatal "device not
// applicable" condition rather than a sibling-matching failure.
var errNoSiblingFound = fmt.Errorf("no suitable sibling found")
// cdiDeviceKey identifies a device type by vendor ID and PCI class prefix.
type cdiDeviceKey struct {
VendorID string
ClassPrefix string
}
// cdiDeviceKind maps known device types to their CDI annotation kind.
var cdiDeviceKind = map[cdiDeviceKey]string{
{VendorID: "0x10de", ClassPrefix: "0x030"}: "nvidia.com/gpu",
}
// cdiKindForDevice returns the CDI kind for a given vendor ID and PCI class,
// or empty string and false if the device is not recognized.
func cdiKindForDevice(vendorID, class string) (string, bool) {
for key, kind := range cdiDeviceKind {
if vendorID == key.VendorID && strings.Contains(class, key.ClassPrefix) {
return kind, true
}
}
return "", false
}
type DeviceRelation struct {
Bus string
Path string
Index int
BDF string
CDIKind string
Bus string
Path string
Index int
BDF string
}
// Depending on the HW we might need to inject metadata into the container
@@ -1255,13 +1223,15 @@ func (c *Container) annotateContainerWithVFIOMetadata(devices interface{}) error
// so lets first iterate over all root-port devices and then
// switch-port devices no special handling for bridge-port (PCI)
for _, dev := range config.PCIeDevicesPerPort["root-port"] {
if kind, ok := cdiKindForDevice(dev.VendorID, dev.Class); ok {
siblings = append(siblings, DeviceRelation{Bus: dev.Bus, Path: dev.HostPath, BDF: dev.BDF, CDIKind: kind})
// For the NV GPU we need special handling let's use only those
if dev.VendorID == "0x10de" && strings.Contains(dev.Class, "0x030") {
siblings = append(siblings, DeviceRelation{Bus: dev.Bus, Path: dev.HostPath, BDF: dev.BDF})
}
}
for _, dev := range config.PCIeDevicesPerPort["switch-port"] {
if kind, ok := cdiKindForDevice(dev.VendorID, dev.Class); ok {
siblings = append(siblings, DeviceRelation{Bus: dev.Bus, Path: dev.HostPath, BDF: dev.BDF, CDIKind: kind})
// For the NV GPU we need special handling let's use only those
if dev.VendorID == "0x10de" && strings.Contains(dev.Class, "0x030") {
siblings = append(siblings, DeviceRelation{Bus: dev.Bus, Path: dev.HostPath, BDF: dev.BDF})
}
}
// We need to sort the VFIO devices by bus to get the correct
@@ -1274,53 +1244,48 @@ func (c *Container) annotateContainerWithVFIOMetadata(devices interface{}) error
siblings[i].Index = i
}
// Collect container paths from either hot-plug or cold-plug devices
var containerPaths []string
if devs, ok := devices.([]ContainerDevice); ok {
for _, dev := range devs {
containerPaths = append(containerPaths, dev.ContainerPath)
}
}
if devs, ok := devices.([]config.DeviceInfo); ok {
for _, dev := range devs {
containerPaths = append(containerPaths, dev.ContainerPath)
// Now that we have the index lets connect the /dev/vfio/<num>
// to the correct index
if devices, ok := devices.([]ContainerDevice); ok {
for _, dev := range devices {
if dev.ContainerPath == "/dev/vfio/vfio" {
c.Logger().Infof("skipping /dev/vfio/vfio for vfio_mode=guest-kernel")
continue
}
err := c.siblingAnnotation(dev.ContainerPath, siblings)
if err != nil {
return err
}
}
}
// Now that we have the index lets connect the /dev/vfio/<num>
// to the correct index
for _, devPath := range containerPaths {
if !strings.HasPrefix(devPath, "/dev/vfio") {
c.Logger().Infof("skipping guest annotations for non-VFIO device %q", devPath)
continue
}
if devPath == "/dev/vfio/vfio" {
c.Logger().Infof("skipping /dev/vfio/vfio for vfio_mode=guest-kernel")
continue
}
if err := c.siblingAnnotation(devPath, siblings); err != nil {
if errors.Is(err, errNoSiblingFound) {
c.Logger().Infof("no CDI annotation for device %s (not a known CDI device type)", devPath)
if devices, ok := devices.([]config.DeviceInfo); ok {
for _, dev := range devices {
if dev.ContainerPath == "/dev/vfio/vfio" {
c.Logger().Infof("skipping /dev/vfio/vfio for vfio_mode=guest-kernel")
continue
}
return err
err := c.siblingAnnotation(dev.ContainerPath, siblings)
if err != nil {
return err
}
}
}
}
return nil
}
// createCDIAnnotation adds a container annotation mapping a VFIO device to a device index.
// createCDIAnnotation adds a container annotation mapping a VFIO device to a GPU index.
//
// devPath is the path to the VFIO device, which can be in the format
// "/dev/vfio/<num>" or "/dev/vfio/devices/vfio<num>". The function extracts
// the device number from the path and creates an annotation with the key
// "cdi.k8s.io/vfio<num>" and the value "<cdiKind>=<index>", where
// <cdiKind> is the CDI device kind (e.g. "nvidia.com/gpu"),
// <num> is the device number and <index> is the provided device index.
// "cdi.k8s.io/vfio<num>" and the value "nvidia.com/gpu=<index>", where
// <num> is the device number and <index> is the provided GPU index.
// The annotation is stored in c.config.CustomSpec.Annotations.
func (c *Container) createCDIAnnotation(devPath string, index int, cdiKind string) {
func (c *Container) createCDIAnnotation(devPath string, index int) {
// We have here either /dev/vfio/<num> or /dev/vfio/devices/vfio<num>
baseName := filepath.Base(devPath)
vfioNum := baseName
@@ -1329,68 +1294,66 @@ func (c *Container) createCDIAnnotation(devPath string, index int, cdiKind strin
vfioNum = strings.TrimPrefix(baseName, "vfio")
}
annoKey := fmt.Sprintf("cdi.k8s.io/vfio%s", vfioNum)
annoValue := fmt.Sprintf("%s=%d", cdiKind, index)
annoValue := fmt.Sprintf("nvidia.com/gpu=%d", index)
if c.config.CustomSpec.Annotations == nil {
c.config.CustomSpec.Annotations = make(map[string]string)
}
c.config.CustomSpec.Annotations[annoKey] = annoValue
c.Logger().Infof("annotated container with %s: %s", annoKey, annoValue)
}
func (c *Container) siblingAnnotation(devPath string, siblings []DeviceRelation) error {
// Resolve the device's BDFs once upfront. This serves two purposes:
// 1. Determine if the device is a known CDI type (if not, skip it)
// 2. Reuse the BDFs for sibling matching without redundant sysfs reads
isKnownCDIDevice := false
var devBDFs []string
if strings.HasPrefix(filepath.Base(devPath), "vfio") {
// IOMMUFD device (/dev/vfio/devices/vfio<NUM>): single device per char dev
major, minor, err := deviceUtils.GetMajorMinorFromDevPath(devPath)
if err != nil {
return err
for _, sibling := range siblings {
if sibling.Path == devPath {
c.createCDIAnnotation(devPath, sibling.Index)
return nil
}
bdf, err := deviceUtils.GetBDFFromVFIODev(major, minor)
if err != nil {
return err
// If the sandbox has cold-plugged an IOMMUFD device and if the
// device-plugins sends us a /dev/vfio/<NUM> device we need to
// check if the IOMMUFD device and the VFIO device are the same
// We have the sibling.BDF we now need to extract the BDF of the
// devPath that is either /dev/vfio/<NUM> or
// /dev/vfio/devices/vfio<NUM>
if strings.HasPrefix(filepath.Base(devPath), "vfio") {
// IOMMUFD device format (/dev/vfio/devices/vfio<NUM>), extract BDF from sysfs
major, minor, err := deviceUtils.GetMajorMinorFromDevPath(devPath)
if err != nil {
return err
}
iommufdBDF, err := deviceUtils.GetBDFFromVFIODev(major, minor)
if err != nil {
return err
}
if sibling.BDF == iommufdBDF {
c.createCDIAnnotation(devPath, sibling.Index)
// exit handling IOMMUFD device
return nil
}
}
devBDFs = []string{bdf}
vendorID := deviceUtils.GetPCIDeviceProperty(bdf, deviceUtils.PCISysFsDevicesVendor)
class := deviceUtils.GetPCIDeviceProperty(bdf, deviceUtils.PCISysFsDevicesClass)
_, isKnownCDIDevice = cdiKindForDevice(vendorID, class)
} else {
// Legacy VFIO group (/dev/vfio/<GROUP>): may contain multiple devices
// Legacy VFIO group device (/dev/vfio/<GROUP_NUM>), extract BDF from sysfs
vfioGroup := filepath.Base(devPath)
iommuDevicesPath := filepath.Join(config.SysIOMMUGroupPath, vfioGroup, "devices")
deviceFiles, err := os.ReadDir(iommuDevicesPath)
if err != nil {
return err
}
vfioBDFs := make([]string, 0)
for _, deviceFile := range deviceFiles {
// Get bdf of device eg 0000:00:1c.0
deviceBDF, _, _, err := deviceUtils.GetVFIODetails(deviceFile.Name(), iommuDevicesPath)
if err != nil {
return err
}
devBDFs = append(devBDFs, deviceBDF)
if !isKnownCDIDevice {
vendorID := deviceUtils.GetPCIDeviceProperty(deviceBDF, deviceUtils.PCISysFsDevicesVendor)
class := deviceUtils.GetPCIDeviceProperty(deviceBDF, deviceUtils.PCISysFsDevicesClass)
if _, ok := cdiKindForDevice(vendorID, class); ok {
isKnownCDIDevice = true
}
}
vfioBDFs = append(vfioBDFs, deviceBDF)
}
}
if !isKnownCDIDevice {
return fmt.Errorf("device %s: %w", devPath, errNoSiblingFound)
}
for _, sibling := range siblings {
if sibling.Path == devPath || slices.Contains(devBDFs, sibling.BDF) {
c.createCDIAnnotation(devPath, sibling.Index, sibling.CDIKind)
if slices.Contains(vfioBDFs, sibling.BDF) {
c.createCDIAnnotation(devPath, sibling.Index)
// exit handling legacy VFIO device
return nil
}
}
return fmt.Errorf("device %s is a known CDI device type but failed to match any sibling by path or BDF", devPath)
return fmt.Errorf("failed to match device %s with any cold-plugged GPU device by path or BDF; no suitable sibling found", devPath)
}
// create creates and starts a container inside a Sandbox. It has to be
@@ -1419,9 +1382,7 @@ func (c *Container) create(ctx context.Context) (err error) {
return
}
if err := c.annotateContainerWithVFIOMetadata(c.devices); err != nil {
return fmt.Errorf("annotating VFIO devices: %w", err)
}
c.annotateContainerWithVFIOMetadata(c.devices)
// Deduce additional system mount info that should be handled by the agent
// inside the VM

View File

@@ -841,6 +841,7 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
// /dev/vfio/devices/vfio0
// (1) Check if we have the new IOMMUFD or old container based VFIO
if strings.HasPrefix(dev.HostPath, pkgDevice.IommufdDevPath) {
q.Logger().Infof("### IOMMUFD Path: %s", dev.HostPath)
vfioDevices, err = drivers.GetDeviceFromVFIODev(dev)
if err != nil {
return fmt.Errorf("Cannot get VFIO device from IOMMUFD with device: %v err: %v", dev, err)

View File

@@ -80,7 +80,7 @@ version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd"
dependencies = [
"getrandom 0.2.15",
"getrandom",
"once_cell",
"version_check",
]
@@ -414,7 +414,7 @@ dependencies = [
"bitflags 2.6.0",
"cexpr",
"clang-sys",
"itertools 0.11.0",
"itertools 0.10.5",
"log",
"prettyplease",
"proc-macro2",
@@ -974,7 +974,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
dependencies = [
"generic-array",
"rand_core 0.6.4",
"rand_core",
"subtle",
"zeroize",
]
@@ -986,7 +986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"rand_core 0.6.4",
"rand_core",
"typenum",
]
@@ -1375,7 +1375,7 @@ checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
dependencies = [
"curve25519-dalek",
"ed25519",
"rand_core 0.6.4",
"rand_core",
"serde",
"sha2 0.10.9",
"subtle",
@@ -1403,7 +1403,7 @@ dependencies = [
"hkdf",
"pem-rfc7468",
"pkcs8",
"rand_core 0.6.4",
"rand_core",
"sec1",
"subtle",
"zeroize",
@@ -1485,7 +1485,7 @@ checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
dependencies = [
"log",
"once_cell",
"rand 0.8.5",
"rand",
]
[[package]]
@@ -1503,7 +1503,7 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449"
dependencies = [
"rand_core 0.6.4",
"rand_core",
"subtle",
]
@@ -1705,20 +1705,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "getrandom"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if 1.0.4",
"js-sys",
"libc",
"r-efi",
"wasip2",
"wasm-bindgen",
]
[[package]]
name = "getset"
version = "0.1.6"
@@ -1769,7 +1755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
dependencies = [
"ff",
"rand_core 0.6.4",
"rand_core",
"subtle",
]
@@ -2105,7 +2091,7 @@ dependencies = [
"qapi",
"qapi-qmp",
"qapi-spec",
"rand 0.8.5",
"rand",
"rust-ini",
"safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"seccompiler",
@@ -2432,11 +2418,10 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.91"
version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
dependencies = [
"once_cell",
"wasm-bindgen",
]
@@ -2475,7 +2460,7 @@ dependencies = [
"oci-spec",
"protobuf",
"protocols",
"rand 0.8.5",
"rand",
"safe-path 0.1.0",
"serde",
"serde_json",
@@ -2500,7 +2485,7 @@ dependencies = [
"nix 0.26.4",
"oci-spec",
"pci-ids",
"rand 0.8.5",
"rand",
"runtime-spec",
"serde",
"serde_json",
@@ -2605,7 +2590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [
"cfg-if 1.0.4",
"windows-targets 0.52.6",
"windows-targets 0.48.0",
]
[[package]]
@@ -2701,12 +2686,6 @@ dependencies = [
"libc",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "matchit"
version = "0.8.4"
@@ -2916,7 +2895,7 @@ dependencies = [
"num-integer",
"num-iter",
"num-traits",
"rand 0.8.5",
"rand",
"smallvec",
"zeroize",
]
@@ -2975,9 +2954,9 @@ checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.22.1",
"chrono",
"getrandom 0.2.15",
"getrandom",
"http 1.1.0",
"rand 0.8.5",
"rand",
"reqwest",
"serde",
"serde_json",
@@ -3121,7 +3100,7 @@ dependencies = [
"oauth2",
"p256",
"p384",
"rand 0.8.5",
"rand",
"rsa",
"serde",
"serde-value",
@@ -3194,7 +3173,7 @@ dependencies = [
"ecdsa",
"elliptic-curve",
"primeorder",
"rand_core 0.6.4",
"rand_core",
"sha2 0.10.9",
]
@@ -3228,7 +3207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
dependencies = [
"base64ct",
"rand_core 0.6.4",
"rand_core",
"subtle",
]
@@ -3351,7 +3330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
"rand 0.8.5",
"rand",
]
[[package]]
@@ -3429,7 +3408,7 @@ checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
"der",
"pkcs5",
"rand_core 0.6.4",
"rand_core",
"spki",
]
@@ -3638,7 +3617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.11.0",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 2.0.87",
@@ -3827,23 +3806,19 @@ dependencies = [
[[package]]
name = "quinn-proto"
version = "0.11.14"
version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
"rand 0.9.2",
"rand",
"ring",
"rustc-hash 2.1.1",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.12",
"thiserror 1.0.40",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
@@ -3868,12 +3843,6 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "radium"
version = "0.7.0"
@@ -3887,18 +3856,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.4",
]
[[package]]
name = "rand"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.5",
"rand_chacha",
"rand_core",
]
[[package]]
@@ -3908,17 +3867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.4",
]
[[package]]
name = "rand_chacha"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
"rand_core 0.9.5",
"rand_core",
]
[[package]]
@@ -3927,16 +3876,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "rand_core"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
"getrandom",
]
[[package]]
@@ -3972,7 +3912,7 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom 0.2.15",
"getrandom",
"redox_syscall 0.2.16",
"thiserror 1.0.40",
]
@@ -4100,7 +4040,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if 1.0.4",
"getrandom 0.2.15",
"getrandom",
"libc",
"untrusted 0.9.0",
"windows-sys 0.52.0",
@@ -4157,7 +4097,7 @@ dependencies = [
"num-traits",
"pkcs1",
"pkcs8",
"rand_core 0.6.4",
"rand_core",
"signature",
"spki",
"subtle",
@@ -4193,7 +4133,7 @@ dependencies = [
"borsh",
"bytes",
"num-traits",
"rand 0.8.5",
"rand",
"rkyv",
"serde",
"serde_json",
@@ -4294,7 +4234,6 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
dependencies = [
"web-time",
"zeroize",
]
@@ -4484,7 +4423,7 @@ dependencies = [
"ed25519",
"ed25519-dalek",
"flate2",
"getrandom 0.2.15",
"getrandom",
"hkdf",
"idea",
"idna",
@@ -4498,8 +4437,8 @@ dependencies = [
"p256",
"p384",
"p521",
"rand 0.8.5",
"rand_core 0.6.4",
"rand",
"rand_core",
"regex",
"regex-syntax",
"ripemd",
@@ -4758,7 +4697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
dependencies = [
"digest 0.10.7",
"rand_core 0.6.4",
"rand_core",
]
[[package]]
@@ -4787,7 +4726,7 @@ dependencies = [
"pem",
"pkcs1",
"pkcs8",
"rand 0.8.5",
"rand",
"regex",
"reqwest",
"rsa",
@@ -5117,7 +5056,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"kata-types",
"rand 0.8.5",
"rand",
]
[[package]]
@@ -5762,25 +5701,29 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasip2"
version = "1.0.2+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.114"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
dependencies = [
"cfg-if 1.0.4",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.87",
"wasm-bindgen-shared",
]
@@ -5798,9 +5741,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.114"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -5808,25 +5751,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.114"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn 2.0.87",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.114"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
dependencies = [
"unicode-ident",
]
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
[[package]]
name = "wasm-streams"
@@ -5851,16 +5791,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.26.6"
@@ -6233,12 +6163,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
[[package]]
name = "writeable"
version = "0.6.1"
@@ -6261,7 +6185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277"
dependencies = [
"curve25519-dalek",
"rand_core 0.6.4",
"rand_core",
"zeroize",
]

View File

@@ -237,60 +237,6 @@ function create_coco_pod_yaml_with_annotations() {
fi
}
# Sealed secrets (signed JWS ES256). Pre-created with guest-components secret CLI; see
# https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md
# Tests provision the signing public key to KBS and use these pre-created sealed secret strings.
#
# To regenerate the signing key and sealed secrets:
# Install required dependencies, clone guest-components repository and change to guest-components/confidential-data-hub
# Create private and public JWK, for example:
# python3 -c "
# from jwcrypto import jwk
# k = jwk.JWK.generate(kty='EC', crv='P-256', alg='ES256', use='sig', kid='sealed-secret-test-key')
# with open('signing-key-private.jwk', 'w') as f:
# f.write(k.export_private())
# with open('signing-key-public.jwk', 'w') as f:
# f.write(k.export_public())
# print('Created signing-key-private.jwk and signing-key-public.jwk')
# "
#
# Build the secret CLI:
# cargo build -p confidential-data-hub --bin secret
#
# Create the sealed secret test secret:
# cargo run -p confidential-data-hub --bin secret -q -- seal \
# --signing-kid "kbs:///default/signing-key/sealed-secret" \
# --signing-jwk-path ./signing-key-private.jwk \
# vault --resource-uri "kbs:///default/sealed-secret/test" --provider kbs
#
# Create the NIM test instruct secret:
# cargo run -p confidential-data-hub --bin secret -q -- seal \
# --signing-kid "kbs:///default/signing-key/sealed-secret" \
# --signing-jwk-path ./signing-key-private.jwk \
# vault --resource-uri "kbs:///default/ngc-api-key/instruct" --provider kbs
#
# Create the NIM test embedqa secret:
# cargo run -p confidential-data-hub --bin secret -q -- seal \
# --signing-kid "kbs:///default/signing-key/sealed-secret" \
# --signing-jwk-path ./signing-key-private.jwk \
# vault --resource-uri "kbs:///default/ngc-api-key/embedqa" --provider kbs
#
# Public JWK (no private key) used to verify the pre-created sealed secrets. Must match the key pair
# that was used to sign SEALED_SECRET_PRECREATED_*.
SEALED_SECRET_SIGNING_PUBLIC_JWK='{"alg":"ES256","crv":"P-256","kid":"sealed-secret-test-key","kty":"EC","use":"sig","x":"4jH376AuwTUCIx65AJ_56D7SZzWf7sGcEA7_Csq21UM","y":"rjdceysnSa5ZfzWOPGCURMUuHndxBAGUu4ISTIVN0yA"}'
# Pre-created sealed secret for k8s-sealed-secret.bats (points to kbs:///default/sealed-secret/test)
export SEALED_SECRET_PRECREATED_TEST="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwicHJvdmlkZXIiOiJrYnMiLCJwcm92aWRlcl9zZXR0aW5ncyI6e30sImFubm90YXRpb25zIjp7fX0.ZI2fTv5ramHqHQa9DKBFD5hlJ_Mjf6cEIcpsNGshpyhEiKklML0abfH600TD7LAFHf53oDIJmEcVsDtJ20UafQ"
# Pre-created sealed secrets for k8s-nvidia-nim.bats (point to kbs:///default/ngc-api-key/instruct and embedqa)
export SEALED_SECRET_PRECREATED_NIM_INSTRUCT="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvbmdjLWFwaS1rZXkvaW5zdHJ1Y3QiLCJwcm92aWRlciI6ImticyIsInByb3ZpZGVyX3NldHRpbmdzIjp7fSwiYW5ub3RhdGlvbnMiOnt9fQ.wpqvVFUaQymqgf54h70shZWDpk2NLW305wALz09YF0GKFBKBQiQB2sRwvn9Jk_rSju3YGLYxPO2Ub8qUbiMCuA"
export SEALED_SECRET_PRECREATED_NIM_EMBEDQA="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvbmdjLWFwaS1rZXkvZW1iZWRxYSIsInByb3ZpZGVyIjoia2JzIiwicHJvdmlkZXJfc2V0dGluZ3MiOnt9LCJhbm5vdGF0aW9ucyI6e319.4C1uqtVXi_qZT8vh_yZ4KpsRdgr2s4hU6ElKj18Hq1DJi_Iji61yuKsS6S1jWdb7drdoKKACvMD6RmCd85SJOQ"
# Provision the signing public key to KBS so CDH can verify the pre-created sealed secrets.
function setup_sealed_secret_signing_public_key() {
kbs_set_resource "default" "signing-key" "sealed-secret" "${SEALED_SECRET_SIGNING_PUBLIC_JWK}"
}
function get_initdata_with_cdh_image_section() {
CDH_IMAGE_SECTION=${1:-""}

View File

@@ -588,6 +588,7 @@ function main() {
install-kata-tools) install_kata_tools "${2:-}" ;;
install-kbs-client) install_kbs_client ;;
get-cluster-credentials) get_cluster_credentials ;;
deploy-csi-driver) return 0 ;;
deploy-kata) deploy_kata ;;
deploy-kata-aks) deploy_kata "aks" ;;
deploy-kata-kcli) deploy_kata "kcli" ;;
@@ -612,6 +613,7 @@ function main() {
cleanup-garm) cleanup "garm" ;;
cleanup-zvsi) cleanup "zvsi" ;;
cleanup-snapshotter) cleanup_snapshotter ;;
delete-csi-driver) return 0 ;;
delete-coco-kbs) delete_coco_kbs ;;
delete-cluster) cleanup "aks" ;;
delete-cluster-kcli) delete_cluster_kcli ;;

View File

@@ -54,8 +54,27 @@ NGC_API_KEY_BASE64=$(
)
export NGC_API_KEY_BASE64
# pre-created signed sealed secrets for TEE pods (from confidential_common.sh)
NGC_API_KEY_SEALED_SECRET_INSTRUCT="${SEALED_SECRET_PRECREATED_NIM_INSTRUCT}"
# Sealed secret format for TEE pods (vault type pointing to KBS resource)
# Format: sealed.<base64url JWS header>.<base64url payload>.<base64url signature>
# IMPORTANT: JWS uses base64url encoding WITHOUT padding (no trailing '=')
# We use tr to convert standard base64 (+/) to base64url (-_) and remove padding (=)
# For vault type, header and signature can be placeholders since the payload
# contains the KBS resource path where the actual secret is stored.
#
# Vault type sealed secret payload for instruct pod:
# {
# "version": "0.1.0",
# "type": "vault",
# "name": "kbs:///default/ngc-api-key/instruct",
# "provider": "kbs",
# "provider_settings": {},
# "annotations": {}
# }
NGC_API_KEY_SEALED_SECRET_INSTRUCT_PAYLOAD=$(
echo -n '{"version":"0.1.0","type":"vault","name":"kbs:///default/ngc-api-key/instruct","provider":"kbs","provider_settings":{},"annotations":{}}' |
base64 -w0 | tr '+/' '-_' | tr -d '='
)
NGC_API_KEY_SEALED_SECRET_INSTRUCT="sealed.fakejwsheader.${NGC_API_KEY_SEALED_SECRET_INSTRUCT_PAYLOAD}.fakesignature"
export NGC_API_KEY_SEALED_SECRET_INSTRUCT
# Base64 encode the sealed secret for use in Kubernetes Secret data field
@@ -63,7 +82,20 @@ export NGC_API_KEY_SEALED_SECRET_INSTRUCT
NGC_API_KEY_SEALED_SECRET_INSTRUCT_BASE64=$(echo -n "${NGC_API_KEY_SEALED_SECRET_INSTRUCT}" | base64 -w0)
export NGC_API_KEY_SEALED_SECRET_INSTRUCT_BASE64
NGC_API_KEY_SEALED_SECRET_EMBEDQA="${SEALED_SECRET_PRECREATED_NIM_EMBEDQA}"
# Vault type sealed secret payload for embedqa pod:
# {
# "version": "0.1.0",
# "type": "vault",
# "name": "kbs:///default/ngc-api-key/embedqa",
# "provider": "kbs",
# "provider_settings": {},
# "annotations": {}
# }
NGC_API_KEY_SEALED_SECRET_EMBEDQA_PAYLOAD=$(
echo -n '{"version":"0.1.0","type":"vault","name":"kbs:///default/ngc-api-key/embedqa","provider":"kbs","provider_settings":{},"annotations":{}}' |
base64 -w0 | tr '+/' '-_' | tr -d '='
)
NGC_API_KEY_SEALED_SECRET_EMBEDQA="sealed.fakejwsheader.${NGC_API_KEY_SEALED_SECRET_EMBEDQA_PAYLOAD}.fakesignature"
export NGC_API_KEY_SEALED_SECRET_EMBEDQA
NGC_API_KEY_SEALED_SECRET_EMBEDQA_BASE64=$(echo -n "${NGC_API_KEY_SEALED_SECRET_EMBEDQA}" | base64 -w0)
@@ -81,6 +113,27 @@ setup_langchain_flow() {
[[ "$(pip show beautifulsoup4 2>/dev/null | awk '/^Version:/{print $2}')" = "4.13.4" ]] || pip install beautifulsoup4==4.13.4
}
# Create Docker config for genpolicy so it can authenticate to nvcr.io when
# pulling image manifests (avoids "UnauthorizedError" from genpolicy's registry pull).
# Genpolicy (src/tools/genpolicy) uses docker_credential::get_credential() in
# src/tools/genpolicy/src/registry.rs build_auth(). The docker_credential crate
# reads config from DOCKER_CONFIG (directory) + "/config.json", so we set
# DOCKER_CONFIG to a directory containing config.json with nvcr.io auth.
setup_genpolicy_registry_auth() {
if [[ -z "${NGC_API_KEY:-}" ]]; then
return
fi
local auth_dir
auth_dir="${BATS_SUITE_TMPDIR}/.docker-genpolicy"
mkdir -p "${auth_dir}"
# Docker config format: auths -> registry -> auth (base64 of "user:password")
echo -n "{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"${NGC_API_KEY}\",\"auth\":\"$(echo -n "\$oauthtoken:${NGC_API_KEY}" | base64 -w0)\"}}}" \
> "${auth_dir}/config.json"
export DOCKER_CONFIG="${auth_dir}"
# REGISTRY_AUTH_FILE (containers-auth.json format) is the same structure for auths
export REGISTRY_AUTH_FILE="${auth_dir}/config.json"
}
# Create initdata TOML file for genpolicy with CDH configuration.
# This file is used by genpolicy via --initdata-path. Genpolicy will add the
# generated policy.rego to it and set it as the cc_init_data annotation.
@@ -190,9 +243,10 @@ setup_file() {
add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
if [ "${TEE}" = "true" ]; then
# So genpolicy can pull nvcr.io image manifests when generating policy (avoids UnauthorizedError).
setup_genpolicy_registry_auth
setup_kbs_credentials
# provision signing public key to KBS so that CDH can verify pre-created, signed secret.
setup_sealed_secret_signing_public_key
# Overwrite the empty default-initdata.toml with our CDH configuration.
# This must happen AFTER create_tmp_policy_settings_dir() copies the empty
# file and BEFORE auto_generate_policy() runs.

View File

@@ -48,13 +48,25 @@ setup() {
"${kernel_params_annotation}" \
"${kernel_params_value}"
# provision signing public key to KBS so that CDH can verify pre-created, signed secret.
setup_sealed_secret_signing_public_key
# Setup k8s secret
kubectl delete secret sealed-secret --ignore-not-found
kubectl delete secret not-sealed-secret --ignore-not-found
kubectl create secret generic sealed-secret --from-literal="secret=${SEALED_SECRET_PRECREATED_TEST}"
# Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault
# sealed.BASE64URL(UTF8(JWS Protected Header)) || '.
# || BASE64URL(JWS Payload) || '.'
# || BASE64URL(JWS Signature)
# test payload:
# {
# "version": "0.1.0",
# "type": "vault",
# "name": "kbs:///default/sealed-secret/test",
# "provider": "kbs",
# "provider_settings": {},
# "annotations": {}
# }
kubectl create secret generic sealed-secret --from-literal='secret=sealed.fakejwsheader.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwicHJvdmlkZXIiOiJrYnMiLCJwcm92aWRlcl9zZXR0aW5ncyI6e30sImFubm90YXRpb25zIjp7fX0.fakesignature'
kubectl create secret generic not-sealed-secret --from-literal='secret=not_sealed_secret'
if ! is_confidential_hardware; then
@@ -67,10 +79,10 @@ setup() {
@test "Cannot Unseal Env Secrets with CDH without key" {
k8s_create_pod "${K8S_TEST_ENV_YAML}"
logs=$(kubectl logs secret-test-pod-cc)
echo "$logs"
grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs"
run grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs"
kubectl logs secret-test-pod-cc
kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
cmd="kubectl logs secret-test-pod-cc | grep -q \"PROTECTED_SECRET = unsealed_secret\""
run $cmd
[ "$status" -eq 1 ]
}
@@ -79,20 +91,18 @@ setup() {
kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret"
k8s_create_pod "${K8S_TEST_ENV_YAML}"
logs=$(kubectl logs secret-test-pod-cc)
echo "$logs"
grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs"
grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs"
kubectl logs secret-test-pod-cc
kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret"
}
@test "Unseal File Secrets with CDH" {
kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret"
k8s_create_pod "${K8S_TEST_FILE_YAML}"
logs=$(kubectl logs secret-test-pod-cc)
echo "$logs"
grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs"
grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs"
kubectl logs secret-test-pod-cc
kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret"
}
teardown() {

View File

@@ -19,8 +19,6 @@ setup() {
mountpoint="/mnt/temp-encrypted"
host_df="$(exec_host "${node}" df -PT -B1 "$(get_kubelet_data_dir)" | tail -n +2)"
info "host_df output:"
info "${host_df}"
host_cap_bytes="$(echo "${host_df}" | awk '{print $3}')"
yaml_file="${pod_config_dir}/pod-trusted-ephemeral-data-storage.yaml"
@@ -38,7 +36,7 @@ setup() {
# With long device names, df adds line breaks by default, so we pass -P to prevent that.
emptydir_df="$(kubectl exec "${pod_name}" -- df -PT -B1 "${mountpoint}" | tail -n +2)"
info "emptydir_df output:"
info "df output:"
info "${emptydir_df}"
dm_device="$(echo "${emptydir_df}" | awk '{print $1}')"
@@ -48,18 +46,17 @@ setup() {
# The output of the cryptsetup command will contain something like this:
#
# /dev/mapper/741ed4bf-3073-49ed-9b7a-d6fa7cce0db1 is active and is in use.
# type: n/a
# cipher: aes-xts-plain
# /dev/mapper/encrypted_disk_N6PxO is active and is in use.
# type: LUKS2
# cipher: aes-xts-plain64
# keysize: 768 bits
# key location: keyring
# integrity: hmac(sha256)
# integrity keysize: 256 bits
# integrity tag size: 32 bytes
# device: /dev/sdd
# device: /dev/vda
# sector size: 4096
# offset: 0 sectors
# size: 300052568 sectors
# size: 2031880 sectors
# mode: read/write
crypt_status="$(kubectl exec "${pod_name}" -- cryptsetup status "${dm_device}")"
info "cryptsetup status output:"
@@ -68,15 +65,16 @@ setup() {
# Check filesystem type and capacity.
[[ "${fs_type}" == "ext4" ]]
# Allow up to 4% metadata overhead.
(( emptydir_cap_bytes >= host_cap_bytes * 96 / 100 ))
# Allow up to 10% metadata overhead.
(( emptydir_avail_bytes >= host_cap_bytes * 90 / 100 ))
# Allow up to 7% LUKS metadata overhead.
(( emptydir_cap_bytes >= host_cap_bytes * 93 / 100 ))
# Allow up to 15% LUKS + ext4 metadata overhead.
(( emptydir_avail_bytes >= host_cap_bytes * 85 / 100 ))
# Check encryption settings.
grep -q "${dm_device} is active and is in use" <<< "${crypt_status}"
grep -Eq "type: +n/a" <<< "${crypt_status}" # The LUKS header is detached.
grep -Eq "cipher: +aes-xts-plain" <<< "${crypt_status}"
grep -Eq "type: +LUKS2" <<< "${crypt_status}"
grep -Eq "cipher: +aes-xts-plain64" <<< "${crypt_status}"
grep -Eq "integrity: +hmac\(sha256\)" <<< "${crypt_status}"
# Check I/O.

View File

@@ -51,27 +51,6 @@ kernel_params = "${new_params}"
EOF
}
# Create Docker config for genpolicy so it can authenticate to nvcr.io when
# pulling image manifests (avoids "UnauthorizedError" from genpolicy's registry pull).
# Genpolicy (src/tools/genpolicy) uses docker_credential::get_credential() in
# src/tools/genpolicy/src/registry.rs build_auth(). The docker_credential crate
# reads config from DOCKER_CONFIG (directory) + "/config.json", so we set
# DOCKER_CONFIG to a directory containing config.json with nvcr.io auth.
setup_genpolicy_registry_auth() {
if [[ -z "${NGC_API_KEY:-}" ]]; then
return
fi
local auth_dir
auth_dir="${kubernetes_dir}/.docker-genpolicy"
mkdir -p "${auth_dir}"
# Docker config format: auths -> registry -> auth (base64 of "user:password")
echo -n "{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"${NGC_API_KEY}\",\"auth\":\"$(echo -n "\$oauthtoken:${NGC_API_KEY}" | base64 -w0)\"}}}" \
> "${auth_dir}/config.json"
export DOCKER_CONFIG="${auth_dir}"
# REGISTRY_AUTH_FILE (containers-auth.json format) is the same structure for auths
export REGISTRY_AUTH_FILE="${auth_dir}/config.json"
}
cleanup() {
true
}
@@ -105,9 +84,6 @@ if [[ "${ENABLE_NVRC_TRACE:-true}" == "true" ]]; then
enable_nvrc_trace
fi
# So genpolicy can pull nvcr.io image manifests when generating policy (avoids UnauthorizedError).
setup_genpolicy_registry_auth
# Use common bats test runner with proper reporting
export BATS_TEST_FAIL_FAST="${K8S_TEST_FAIL_FAST}"
run_bats_tests "${kubernetes_dir}" K8S_TEST_NV

View File

@@ -61,19 +61,12 @@ install_userspace_components() {
eval "${APT_INSTALL}" nvidia-imex nvidia-firmware \
libnvidia-cfg1 libnvidia-gl libnvidia-extra \
libnvidia-decode libnvidia-fbc1 libnvidia-encode \
libnvidia-nscq libnvidia-compute nvidia-settings
libnvidia-nscq
apt-mark hold nvidia-imex nvidia-firmware \
libnvidia-cfg1 libnvidia-gl libnvidia-extra \
libnvidia-decode libnvidia-fbc1 libnvidia-encode \
libnvidia-nscq libnvidia-compute nvidia-settings
# Needed for confidential-data-hub runtime dependencies
eval "${APT_INSTALL}" cryptsetup-bin dmsetup \
libargon2-1 e2fsprogs
apt-mark hold cryptsetup-bin dmsetup libargon2-1 \
e2fsprogs
libnvidia-nscq
}
setup_apt_repositories() {

View File

@@ -151,8 +151,14 @@ chisseled_nvswitch() {
cp -a "${stage_one}"/usr/share/nvidia/nvswitch usr/share/nvidia/.
libdir=usr/lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}/${libdir}"/libnvidia-nscq.so.* lib/"${machine_arch}"-linux-gnu/.
# Logs will be redirected to console(stderr)
# if the specified log file can't be opened or the path is empty.
# LOG_FILE_NAME=/var/log/fabricmanager.log -> setting to empty for stderr -> kmsg
sed -i 's|^LOG_FILE_NAME=.*|LOG_FILE_NAME=|' usr/share/nvidia/nvswitch/fabricmanager.cfg
# NVLINK SubnetManager dependencies
local nvlsm=usr/share/nvidia/nvlsm
mkdir -p "${nvlsm}"
@@ -160,8 +166,6 @@ chisseled_nvswitch() {
cp -a "${stage_one}"/opt/nvidia/nvlsm/lib/libgrpc_mgr.so lib/.
cp -a "${stage_one}"/opt/nvidia/nvlsm/sbin/nvlsm sbin/.
cp -a "${stage_one}/${nvlsm}"/*.conf "${nvlsm}"/.
# Redirect all the logs to syslog instead of logging to file
sed -i 's|^LOG_USE_SYSLOG=.*|LOG_USE_SYSLOG=1|' usr/share/nvidia/nvswitch/fabricmanager.cfg
}
chisseled_dcgm() {
@@ -207,8 +211,9 @@ chisseled_compute() {
cp -aL "${stage_one}/${libdir}"/ld-linux-* "${libdir}"/.
libdir=usr/lib/"${machine_arch}"-linux-gnu
cp -a "${stage_one}/${libdir}"/libnv* lib/"${machine_arch}"-linux-gnu/.
cp -a "${stage_one}/${libdir}"/libnvidia-ml.so.* lib/"${machine_arch}"-linux-gnu/.
cp -a "${stage_one}/${libdir}"/libcuda.so.* lib/"${machine_arch}"-linux-gnu/.
cp -a "${stage_one}/${libdir}"/libnvidia-cfg.so.* lib/"${machine_arch}"-linux-gnu/.
# basic GPU admin tools
cp -a "${stage_one}"/usr/bin/nvidia-persistenced bin/.
@@ -240,8 +245,6 @@ chisseled_init() {
usr/bin etc/modprobe.d etc/ssl/certs
ln -sf ../run var/run
ln -sf ../run var/log
ln -sf ../run var/cache
# Needed for various RUST static builds with LIBC=gnu
libdir=lib/"${machine_arch}"-linux-gnu
@@ -308,44 +311,6 @@ compress_rootfs() {
chmod +x "${libdir}"/ld-linux-*
}
copy_cdh_runtime_deps() {
local libdir="lib/${machine_arch}-linux-gnu"
# Shared libraries required by /usr/local/bin/confidential-data-hub.
# Note: libcryptsetup loads some optional helpers (e.g. libpopt/libssh) only
# when specific features are used. The current CDH path (LUKS2 + mkfs.ext4)
# does not require those optional libs.
cp -a "${stage_one}/${libdir}"/libcryptsetup.so.12* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libuuid.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libdevmapper.so.1.02.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libselinux.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libpcre2-8.so.0* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libudev.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libcap.so.2* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libcrypto.so.3* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libz.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libzstd.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libjson-c.so.5* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libblkid.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libargon2.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libgcc_s.so.1* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libm.so.6* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libc.so.6* "${libdir}/."
# e2fsprogs (mkfs.ext4) runtime libs
cp -a "${stage_one}/${libdir}"/libext2fs.so.2* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libe2p.so.2* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libss.so.2* "${libdir}/."
cp -a "${stage_one}/${libdir}"/libcom_err.so.2* "${libdir}/."
# mkfs.ext4 and dd are used by CDH secure_mount
mkdir -p sbin etc usr/bin bin
cp -a "${stage_one}/sbin/mke2fs" sbin/.
cp -a "${stage_one}/sbin/mkfs.ext4" sbin/.
cp -a "${stage_one}/etc/mke2fs.conf" etc/.
cp -a "${stage_one}/usr/bin/dd" bin/.
}
coco_guest_components() {
if [[ "${type}" != "confidential" ]]; then
return
@@ -368,7 +333,7 @@ coco_guest_components() {
cp -a "${stage_one}/${pause_dir}"/config.json "${pause_dir}/."
cp -a "${stage_one}/${pause_dir}"/rootfs/pause "${pause_dir}/rootfs/."
copy_cdh_runtime_deps
info "TODO: nvidia: luks-encrypt-storage is a bash script, we do not have a shell!"
}
setup_nvidia_gpu_rootfs_stage_two() {

View File

@@ -15,13 +15,13 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: "3.28.0"
version: "3.27.0"
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "3.28.0"
appVersion: "3.27.0"
dependencies:
- name: node-feature-discovery

View File

@@ -96,9 +96,9 @@ scheduling:
"qemu-snp-runtime-rs" (dict "memory" "2048Mi" "cpu" "1.0")
"qemu-tdx" (dict "memory" "2048Mi" "cpu" "1.0")
"qemu-tdx-runtime-rs" (dict "memory" "2048Mi" "cpu" "1.0")
"qemu-nvidia-gpu" (dict "memory" "10240Mi" "cpu" "1.0")
"qemu-nvidia-gpu-snp" (dict "memory" "10240Mi" "cpu" "1.0")
"qemu-nvidia-gpu-tdx" (dict "memory" "10240Mi" "cpu" "1.0")
"qemu-nvidia-gpu" (dict "memory" "4096Mi" "cpu" "1.0")
"qemu-nvidia-gpu-snp" (dict "memory" "20480Mi" "cpu" "1.0")
"qemu-nvidia-gpu-tdx" (dict "memory" "20480Mi" "cpu" "1.0")
"qemu-cca" (dict "memory" "2048Mi" "cpu" "1.0")
"stratovirt" (dict "memory" "130Mi" "cpu" "250m")
"remote" (dict "memory" "120Mi" "cpu" "250m")

View File

@@ -110,6 +110,9 @@ cloud-hypervisor-tarball:
cloud-hypervisor-glibc-tarball:
${MAKE} $@-build
csi-kata-directvolume-tarball: copy-scripts-for-the-tools-build
${MAKE} $@-build
firecracker-tarball:
${MAKE} $@-build
@@ -191,9 +194,15 @@ rootfs-initrd-tarball: agent-tarball
rootfs-image-nvidia-gpu-tarball: agent-tarball busybox-tarball kernel-nvidia-gpu-tarball
${MAKE} $@-build
rootfs-initrd-nvidia-gpu-tarball: agent-tarball busybox-tarball kernel-nvidia-gpu-tarball
${MAKE} $@-build
rootfs-image-nvidia-gpu-confidential-tarball: agent-tarball busybox-tarball pause-image-tarball coco-guest-components-tarball kernel-nvidia-gpu-tarball
${MAKE} $@-build
rootfs-initrd-nvidia-gpu-confidential-tarball: agent-tarball busybox-tarball pause-image-tarball coco-guest-components-tarball kernel-nvidia-gpu-tarball
${MAKE} $@-build
rootfs-cca-confidential-image-tarball: agent-tarball pause-image-tarball coco-guest-components-tarball kernel-cca-confidential-tarball
${MAKE} $@-build

View File

@@ -108,6 +108,7 @@ options:
coco-guest-components
cloud-hypervisor
cloud-hypervisor-glibc
csi-kata-directvolume
firecracker
genpolicy
kata-ctl
@@ -361,7 +362,7 @@ get_latest_kernel_nvidia_artefact_and_builder_image_version() {
}
get_latest_ctk_version() {
echo $(get_from_kata_deps ".externals.nvidia.ctk.version")
echo $(get_from_kata_deps ".assets.kernel.nvidia.ctk.version")
}
#Install guest image
@@ -607,6 +608,16 @@ install_image_nvidia_gpu() {
install_image "nvidia-gpu"
}
# Install NVIDIA GPU initrd
install_initrd_nvidia_gpu() {
export AGENT_POLICY
export MEASURED_ROOTFS="no"
local version=$(get_from_kata_deps .externals.nvidia.driver.version)
EXTRA_PKGS="apt curl ${EXTRA_PKGS}"
NVIDIA_GPU_STACK=${NVIDIA_GPU_STACK:-"driver=${version},compute,dcgm,nvswitch"}
install_initrd "nvidia-gpu"
}
# Instal NVIDIA GPU confidential image
install_image_nvidia_gpu_confidential() {
export CONFIDENTIAL_GUEST="yes"
@@ -618,6 +629,18 @@ install_image_nvidia_gpu_confidential() {
install_image "nvidia-gpu-confidential"
}
# Install NVIDIA GPU confidential initrd
install_initrd_nvidia_gpu_confidential() {
export CONFIDENTIAL_GUEST="yes"
export AGENT_POLICY
export MEASURED_ROOTFS="no"
local version=$(get_from_kata_deps .externals.nvidia.driver.version)
EXTRA_PKGS="apt curl ${EXTRA_PKGS}"
NVIDIA_GPU_STACK=${NVIDIA_GPU_STACK:-"driver=${version},compute,dcgm,nvswitch"}
install_initrd "nvidia-gpu-confidential"
}
install_se_image() {
info "Create IBM SE image configured with AA_KBC=${AA_KBC}"
"${se_image_builder}" --destdir="${destdir}"
@@ -1182,6 +1205,7 @@ install_tools_helper() {
tool_binary=${tool}
[ ${tool} = "agent-ctl" ] && tool_binary="kata-agent-ctl"
[ ${tool} = "csi-kata-directvolume" ] && tool_binary="directvolplugin"
[ ${tool} = "trace-forwarder" ] && tool_binary="kata-trace-forwarder"
local tool_build_dir="src/tools/${tool}"
@@ -1224,6 +1248,7 @@ install_tools_helper() {
info "Install static ${tool_binary}"
mkdir -p "${destdir}/opt/kata/bin/"
[ ${tool} = "csi-kata-directvolume" ] && tool_binary="csi-kata-directvolume"
install -D --mode "${binary_permissions}" "${binary}" "${destdir}/opt/kata/bin/${tool_binary}"
}
@@ -1235,6 +1260,10 @@ install_genpolicy() {
install_tools_helper "genpolicy"
}
install_csi_kata_directvolume() {
install_tools_helper "csi-kata-directvolume"
}
install_kata_ctl() {
install_tools_helper "kata-ctl"
}
@@ -1309,6 +1338,8 @@ handle_build() {
cloud-hypervisor-glibc) install_clh_glibc ;;
csi-kata-directvolume) install_csi_kata_directvolume ;;
firecracker) install_firecracker ;;
genpolicy) install_genpolicy ;;
@@ -1361,8 +1392,12 @@ handle_build() {
rootfs-image-nvidia-gpu) install_image_nvidia_gpu ;;
rootfs-initrd-nvidia-gpu) install_initrd_nvidia_gpu ;;
rootfs-image-nvidia-gpu-confidential) install_image_nvidia_gpu_confidential ;;
rootfs-initrd-nvidia-gpu-confidential) install_initrd_nvidia_gpu_confidential ;;
rootfs-cca-confidential-image) install_image_confidential ;;
rootfs-cca-confidential-initrd) install_initrd_confidential ;;
@@ -1521,6 +1556,7 @@ main() {
agent-ctl
cloud-hypervisor
coco-guest-components
csi-kata-directvolume
firecracker
genpolicy
kata-ctl

View File

@@ -611,7 +611,6 @@ install_kata() {
fi
install --mode 0644 -D ./.config "${install_path}/config-${kernel_version}-${config_version}${suffix}"
install --mode 0644 -D ./System.map "${install_path}/System.map-${kernel_version}-${config_version}${suffix}"
ln -sf "${vmlinuz}" "${install_path}/vmlinuz${suffix}.container"
ln -sf "${vmlinux}" "${install_path}/vmlinux${suffix}.container"

View File

@@ -1 +1 @@
186
185

View File

@@ -25,7 +25,6 @@ RUN apt-get update && \
g++ \
gcc \
git \
libcryptsetup-dev \
libssl-dev \
libtss2-dev \
make \

View File

@@ -34,6 +34,7 @@ build_coco_guest_components_from_source() {
strip "target/${RUST_ARCH}-unknown-linux-${LIBC}/release/api-server-rest"
DESTDIR="${DESTDIR}/usr/local/bin" TEE_PLATFORM=${TEE_PLATFORM} make install
install -D -m0755 "confidential-data-hub/hub/src/storage/scripts/luks-encrypt-storage" "${DESTDIR}/usr/local/bin/luks-encrypt-storage"
install -D -m0644 "confidential-data-hub/hub/src/image/ocicrypt_config.json" "${DESTDIR}/etc/ocicrypt_config.json"
popd
}

View File

@@ -123,6 +123,7 @@ mapping:
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-asset (virtiofsd, test)
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / create-kata-tarball
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-tools-asset (agent-ctl, test)
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-tools-asset (csi-kata-directvolume, test)
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-tools-asset (genpolicy, test)
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-tools-asset (kata-ctl, test)
- Kata Containers CI / kata-containers-ci-on-push / build-kata-static-tarball-amd64 / build-tools-asset (kata-manager, test)

View File

@@ -234,7 +234,7 @@ externals:
nvrc:
# yamllint disable-line rule:line-length
desc: "The NVRC project provides a Rust binary that implements a simple init system for microVMs"
version: "v0.1.3"
version: "v0.1.1"
url: "https://github.com/NVIDIA/nvrc/releases/download/"
nvidia:
@@ -288,18 +288,18 @@ externals:
coco-guest-components:
description: "Provides attested key unwrapping for image decryption"
url: "https://github.com/confidential-containers/guest-components/"
version: "ab95914ac84c32a43102463cc0ae330710af47be"
version: "9aae2eae6a03ab97d6561bbe74f8b99843836bba"
toolchain: "1.90.0"
coco-trustee:
description: "Provides attestation and secret delivery components"
url: "https://github.com/confidential-containers/trustee"
version: "f5cb8fc1b51b652fc24e2d6b8742cf417805352e"
version: "3b2356a52e0d8a58730a1977e235a7e7f2007b5e"
# image / ita_image and image_tag / ita_image_tag must be in sync
image: "ghcr.io/confidential-containers/staged-images/kbs"
image_tag: "f5cb8fc1b51b652fc24e2d6b8742cf417805352e"
image_tag: "3b2356a52e0d8a58730a1977e235a7e7f2007b5e"
ita_image: "ghcr.io/confidential-containers/staged-images/kbs-ita-as"
ita_image_tag: "f5cb8fc1b51b652fc24e2d6b8742cf417805352e-x86_64"
ita_image_tag: "3b2356a52e0d8a58730a1977e235a7e7f2007b5e-x86_64"
toolchain: "1.90.0"
containerd: