From 2a1d3941478c49476509a33c26d498ab1b7a6e90 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Fri, 6 May 2022 15:11:54 -0700 Subject: [PATCH 01/26] runtime: Adding the correct detection of mediated PCIe devices Fixes #4212 Signed-off-by: Zvonko Kaiser --- src/runtime/virtcontainers/device/drivers/utils.go | 5 ++--- src/runtime/virtcontainers/device/drivers/vfio.go | 14 ++++++++++++++ .../virtcontainers/device/drivers/vfio_test.go | 1 + 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/runtime/virtcontainers/device/drivers/utils.go b/src/runtime/virtcontainers/device/drivers/utils.go index ecabed83d3..aa97747219 100644 --- a/src/runtime/virtcontainers/device/drivers/utils.go +++ b/src/runtime/virtcontainers/device/drivers/utils.go @@ -45,9 +45,8 @@ func deviceLogger() *logrus.Entry { return api.DeviceLogger() } -// Identify PCIe device by /sys/bus/pci/slots/xx/max_bus_speed, sample content "8.0 GT/s PCIe" -// The /sys/bus/pci/slots/xx/address contains bdf, sample content "0000:04:00" -// bdf format: bus:slot.function +// Identify PCIe device by reading the size of the PCI config space +// Plain PCI device have 256 bytes of config space where PCIe devices have 4K func isPCIeDevice(bdf string) bool { if len(strings.Split(bdf, ":")) == 2 { bdf = PCIDomain + ":" + bdf diff --git a/src/runtime/virtcontainers/device/drivers/vfio.go b/src/runtime/virtcontainers/device/drivers/vfio.go index 8507564f60..8e36ff46ba 100644 --- a/src/runtime/virtcontainers/device/drivers/vfio.go +++ b/src/runtime/virtcontainers/device/drivers/vfio.go @@ -222,6 +222,7 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) deviceSysfsDev, err = getSysfsDev(sysfsDevStr) + deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev)) default: err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName) } @@ -229,10 +230,23 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS return deviceBDF, deviceSysfsDev, vfioDeviceType, err } +// getMediatedBDF returns the BDF of a VF +// Expected input string format is /sys/devices/pci0000:d7/BDF0/BDF1/.../MDEVBDF/UUID +func getMediatedBDF(deviceSysfsDev string) string { + tokens := strings.SplitN(deviceSysfsDev, "/", -1) + if len(tokens) < 4 { + return "" + } + return tokens[len(tokens)-2] +} + // getBDF returns the BDF of pci device // Expected input string format is []:[][].[] eg. 0000:02:10.0 func getBDF(deviceSysStr string) string { tokens := strings.SplitN(deviceSysStr, ":", 2) + if len(tokens) == 1 { + return "" + } return tokens[1] } diff --git a/src/runtime/virtcontainers/device/drivers/vfio_test.go b/src/runtime/virtcontainers/device/drivers/vfio_test.go index e98e59e47a..05118871f5 100644 --- a/src/runtime/virtcontainers/device/drivers/vfio_test.go +++ b/src/runtime/virtcontainers/device/drivers/vfio_test.go @@ -46,4 +46,5 @@ func TestGetVFIODetails(t *testing.T) { assert.Nil(t, err) } } + } From 44814dce19e88c2426bd9ada2c015ed073946c24 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Tue, 10 May 2022 19:36:25 +0300 Subject: [PATCH 02/26] qemu: treat console kernel params within appendConsole as it is tightly coupled with the appended console device additionally have it tested Signed-off-by: Snir Sheriber --- src/runtime/virtcontainers/qemu.go | 15 ++++++++------- src/runtime/virtcontainers/qemu_amd64.go | 2 -- src/runtime/virtcontainers/qemu_arch_base.go | 6 ++++++ src/runtime/virtcontainers/qemu_arch_base_test.go | 2 ++ src/runtime/virtcontainers/qemu_arm64.go | 2 -- src/runtime/virtcontainers/qemu_ppc64le.go | 2 -- src/runtime/virtcontainers/qemu_s390x.go | 6 +++--- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index a910c580ff..070ace1e71 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -552,13 +552,6 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi return err } - // Breaks hypervisor abstration Has Kata Specific logic - kernel := govmmQemu.Kernel{ - Path: kernelPath, - InitrdPath: initrdPath, - Params: q.kernelParameters(), - } - incoming := q.setupTemplate(&knobs, &memory) // With the current implementations, VM templating will not work with file @@ -630,6 +623,14 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi return err } + // Breaks hypervisor abstraction has Kata Specific logic + kernel := govmmQemu.Kernel{ + Path: kernelPath, + InitrdPath: initrdPath, + // some devices configuration may also change kernel params, make sure this is called afterwards + Params: q.kernelParameters(), + } + qemuConfig := govmmQemu.Config{ Name: fmt.Sprintf("sandbox-%s", q.id), UUID: q.state.UUID, diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 3f52ab756d..773af95e8a 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -56,8 +56,6 @@ var kernelParams = []Param{ {"i8042.noaux", "1"}, {"noreplace-smp", ""}, {"reboot", "k"}, - {"console", "hvc0"}, - {"console", "hvc1"}, {"cryptomgr.notests", ""}, {"net.ifnames", "0"}, {"pci", "lastbus=0"}, diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 62fec60a7d..0b5d505203 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -337,6 +337,12 @@ func (q *qemuArchBase) appendConsole(_ context.Context, devices []govmmQemu.Devi devices = append(devices, console) + consoleKernelParams := []Param{ + {"console", "hvc0"}, + {"console", "hvc1"}, + } + q.kernelParams = append(q.kernelParams, consoleKernelParams...) + return devices, nil } diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index ff20ba4479..2625a8a99d 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -259,6 +259,8 @@ func TestQemuArchBaseAppendConsoles(t *testing.T) { devices, err = qemuArchBase.appendConsole(context.Background(), devices, path) assert.NoError(err) assert.Equal(expectedOut, devices) + assert.Contains(qemuArchBase.kernelParams, Param{"console", "hvc0"}) + assert.Contains(qemuArchBase.kernelParams, Param{"console", "hvc1"}) } func TestQemuArchBaseAppendImage(t *testing.T) { diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index f5af19e219..f3a5576606 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -33,8 +33,6 @@ const qmpMigrationWaitTimeout = 10 * time.Second const defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=host" var kernelParams = []Param{ - {"console", "hvc0"}, - {"console", "hvc1"}, {"iommu.passthrough", "0"}, } diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index e18f2264b5..cfff5329c1 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -41,8 +41,6 @@ const tpmHostPath = "/dev/tpmrm0" var kernelParams = []Param{ {"rcupdate.rcu_expedited", "1"}, {"reboot", "k"}, - {"console", "hvc0"}, - {"console", "hvc1"}, {"cryptomgr.notests", ""}, {"net.ifnames", "0"}, } diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index 32716e7767..210a341766 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -39,9 +39,7 @@ const ( ) // Verify needed parameters -var kernelParams = []Param{ - {"console", "ttysclp0"}, -} +var kernelParams = []Param{} var ccwbridge = types.NewBridge(types.CCW, "", make(map[uint32]string, types.CCWBridgeMaxCapacity), 0) @@ -112,6 +110,8 @@ func (q *qemuS390x) appendConsole(ctx context.Context, devices []govmmQemu.Devic return devices, fmt.Errorf("Failed to append console %v", err) } + q.kernelParams = append(q.kernelParams, Param{"console", "ttysclp0"}) + serial := govmmQemu.SerialDevice{ Driver: virtioSerialCCW, ID: id, From c67b9d2975f6c24a63a9ae3d11bcb1380dfde8b2 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Thu, 12 May 2022 14:26:51 +0300 Subject: [PATCH 03/26] qemu: allow using legacy serial device for the console This allows to get guest early boot logs which are usually missed when virtconsole is used. - It utilizes previous work on the govmm side: https://github.com/kata-containers/govmm/pull/203 - unit test added Fixes: #4237 Signed-off-by: Snir Sheriber --- src/runtime/config/configuration-qemu.toml.in | 3 + .../pkg/katautils/config-settings.go.in | 1 + src/runtime/pkg/katautils/config.go | 3 + src/runtime/virtcontainers/hypervisor.go | 3 + src/runtime/virtcontainers/qemu_amd64.go | 1 + src/runtime/virtcontainers/qemu_arch_base.go | 59 +++++++++++++------ .../virtcontainers/qemu_arch_base_test.go | 28 +++++++++ src/runtime/virtcontainers/qemu_arm64.go | 1 + src/runtime/virtcontainers/qemu_ppc64le.go | 1 + src/runtime/virtcontainers/qemu_s390x.go | 1 + 10 files changed, 82 insertions(+), 19 deletions(-) diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index e0dd09cf6b..09c219545d 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -389,6 +389,9 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@ # be default_memory. #enable_guest_swap = true +# use legacy serial for guest console if available and implemented for architecture. Default false +#use_legacy_serial = true + [factory] # VM templating support. Once enabled, new VMs are created from template # using vm cloning. They will share the same initial kernel, initramfs and diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 09d6b5f30c..2aad22bd85 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -89,6 +89,7 @@ const defaultGuestSwap = false const defaultRootlessHypervisor = false const defaultDisableSeccomp = false const defaultVfioMode = "guest-kernel" +const defaultLegacySerial = false var defaultSGXEPCSize = int64(0) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index ec02b4bc26..2a62c3d6c3 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -146,6 +146,7 @@ type hypervisor struct { Rootless bool `toml:"rootless"` DisableSeccomp bool `toml:"disable_seccomp"` DisableSeLinux bool `toml:"disable_selinux"` + LegacySerial bool `toml:"use_legacy_serial"` } type runtime struct { @@ -775,6 +776,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { ConfidentialGuest: h.ConfidentialGuest, GuestSwap: h.GuestSwap, Rootless: h.Rootless, + LegacySerial: h.LegacySerial, }, nil } @@ -1132,6 +1134,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { GuestSwap: defaultGuestSwap, Rootless: defaultRootlessHypervisor, DisableSeccomp: defaultDisableSeccomp, + LegacySerial: defaultLegacySerial, } } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index c689eae3c9..9de4dc0d0a 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -537,6 +537,9 @@ type HypervisorConfig struct { // Disable selinux from the hypervisor process DisableSeLinux bool + + // Use legacy serial for the guest console + LegacySerial bool } // vcpu mapping from vcpu number to thread number diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 773af95e8a..61c18c68e4 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -122,6 +122,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { disableNvdimm: config.DisableImageNvdimm, dax: true, protection: noneProtection, + legacySerial: config.LegacySerial, }, vmFactory: factory, } diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 0b5d505203..a2873a2160 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -169,6 +169,7 @@ type qemuArchBase struct { vhost bool disableNvdimm bool dax bool + legacySerial bool } const ( @@ -318,29 +319,49 @@ func (q *qemuArchBase) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8 } func (q *qemuArchBase) appendConsole(_ context.Context, devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { - serial := govmmQemu.SerialDevice{ - Driver: govmmQemu.VirtioSerial, - ID: "serial0", - DisableModern: q.nestedRun, - MaxPorts: uint(2), + var serial, console govmmQemu.Device + var consoleKernelParams []Param + + if q.legacySerial { + serial = govmmQemu.LegacySerialDevice{ + Chardev: "charconsole0", + } + + console = govmmQemu.CharDevice{ + Driver: govmmQemu.LegacySerial, + Backend: govmmQemu.Socket, + DeviceID: "console0", + ID: "charconsole0", + Path: path, + } + + consoleKernelParams = []Param{ + {"console", "ttyS0"}, + } + } else { + serial = govmmQemu.SerialDevice{ + Driver: govmmQemu.VirtioSerial, + ID: "serial0", + DisableModern: q.nestedRun, + MaxPorts: uint(2), + } + + console = govmmQemu.CharDevice{ + Driver: govmmQemu.Console, + Backend: govmmQemu.Socket, + DeviceID: "console0", + ID: "charconsole0", + Path: path, + } + + consoleKernelParams = []Param{ + {"console", "hvc0"}, + {"console", "hvc1"}, + } } devices = append(devices, serial) - - console := govmmQemu.CharDevice{ - Driver: govmmQemu.Console, - Backend: govmmQemu.Socket, - DeviceID: "console0", - ID: "charconsole0", - Path: path, - } - devices = append(devices, console) - - consoleKernelParams := []Param{ - {"console", "hvc0"}, - {"console", "hvc1"}, - } q.kernelParams = append(q.kernelParams, consoleKernelParams...) return devices, nil diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 2625a8a99d..57c97f773b 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -263,6 +263,34 @@ func TestQemuArchBaseAppendConsoles(t *testing.T) { assert.Contains(qemuArchBase.kernelParams, Param{"console", "hvc1"}) } +func TestQemuArchBaseAppendConsolesLegacy(t *testing.T) { + var devices []govmmQemu.Device + var err error + assert := assert.New(t) + qemuArchBase := newQemuArchBase() + qemuArchBase.legacySerial = true + + path := filepath.Join(filepath.Join(fs.MockRunStoragePath(), "test"), consoleSocket) + + expectedOut := []govmmQemu.Device{ + govmmQemu.LegacySerialDevice{ + Chardev: "charconsole0", + }, + govmmQemu.CharDevice{ + Driver: govmmQemu.LegacySerial, + Backend: govmmQemu.Socket, + DeviceID: "console0", + ID: "charconsole0", + Path: path, + }, + } + + devices, err = qemuArchBase.appendConsole(context.Background(), devices, path) + assert.NoError(err) + assert.Equal(expectedOut, devices) + assert.Contains(qemuArchBase.kernelParams, Param{"console", "ttyS0"}) +} + func TestQemuArchBaseAppendImage(t *testing.T) { var devices []govmmQemu.Device assert := assert.New(t) diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index f3a5576606..378b89c6d7 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -62,6 +62,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { disableNvdimm: config.DisableImageNvdimm, dax: true, protection: noneProtection, + legacySerial: config.LegacySerial, }, } diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index cfff5329c1..27bac35818 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -74,6 +74,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { kernelParamsDebug: kernelParamsDebug, kernelParams: kernelParams, protection: noneProtection, + legacySerial: config.LegacySerial, }, } diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index 210a341766..f0946698f7 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -66,6 +66,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { kernelParamsNonDebug: kernelParamsNonDebug, kernelParamsDebug: kernelParamsDebug, kernelParams: kernelParams, + legacySerial: false, }, } // Set first bridge type to CCW From 24a2b0f6a2390162c4919590890a653429f7cb81 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 18 May 2022 14:53:17 +0000 Subject: [PATCH 04/26] docs: Remove clear containers reference in README This PR removes the clear containers reference as this is not longer being used and is deprecated at the rootfs builder README. Fixes #4278 Signed-off-by: Gabriela Cervantes --- tools/osbuilder/rootfs-builder/README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/osbuilder/rootfs-builder/README.md b/tools/osbuilder/rootfs-builder/README.md index f82a569ef4..ea3640f4d8 100644 --- a/tools/osbuilder/rootfs-builder/README.md +++ b/tools/osbuilder/rootfs-builder/README.md @@ -82,12 +82,6 @@ must be met: $ docker info | grep 'Default Runtime: runc' ``` - Note: - - This requirement is specific to the Clear Containers runtime. - See [issue](https://github.com/clearcontainers/runtime/issues/828) for - more information. - 3. Export `USE_DOCKER` variable. ``` From f4994e486bc7118cb0fe498cf48f88b3f0300d68 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Wed, 18 May 2022 14:18:31 +0300 Subject: [PATCH 05/26] runtime: allow annotation configuration to use_legacy_serial and update the docs and test Signed-off-by: Snir Sheriber --- docs/how-to/how-to-set-sandbox-config-kata.md | 1 + src/runtime/pkg/oci/utils.go | 6 ++++++ src/runtime/pkg/oci/utils_test.go | 2 ++ src/runtime/virtcontainers/pkg/annotations/annotations.go | 3 +++ 4 files changed, 12 insertions(+) diff --git a/docs/how-to/how-to-set-sandbox-config-kata.md b/docs/how-to/how-to-set-sandbox-config-kata.md index 28d5ea788e..343b7fe2e6 100644 --- a/docs/how-to/how-to-set-sandbox-config-kata.md +++ b/docs/how-to/how-to-set-sandbox-config-kata.md @@ -91,6 +91,7 @@ There are several kinds of Kata configurations and they are listed below. | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path | | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon | | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest | +| `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) | ## Container Options | Key | Value Type | Comments | diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index f72f1147e5..e6e4dcfe6f 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -474,6 +474,12 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return err } + if err := newAnnotationConfiguration(ocispec, vcAnnotations.UseLegacySerial).setBool(func(useLegacySerial bool) { + config.HypervisorConfig.LegacySerial = useLegacySerial + }); err != nil { + return err + } + if err := newAnnotationConfiguration(ocispec, vcAnnotations.PCIeRootPort).setUint(func(pcieRootPort uint64) { config.HypervisorConfig.PCIeRootPort = uint32(pcieRootPort) }); err != nil { diff --git a/src/runtime/pkg/oci/utils_test.go b/src/runtime/pkg/oci/utils_test.go index bfa71801d2..e6158a96eb 100644 --- a/src/runtime/pkg/oci/utils_test.go +++ b/src/runtime/pkg/oci/utils_test.go @@ -670,6 +670,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.PCIeRootPort] = "2" ocispec.Annotations[vcAnnotations.IOMMUPlatform] = "true" ocispec.Annotations[vcAnnotations.SGXEPC] = "64Mi" + ocispec.Annotations[vcAnnotations.UseLegacySerial] = "true" // 10Mbit ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate] = "10000000" ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate] = "10000000" @@ -706,6 +707,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Equal(config.HypervisorConfig.PCIeRootPort, uint32(2)) assert.Equal(config.HypervisorConfig.IOMMUPlatform, true) assert.Equal(config.HypervisorConfig.SGXEPCSize, int64(67108864)) + assert.Equal(config.HypervisorConfig.LegacySerial, true) assert.Equal(config.HypervisorConfig.RxRateLimiterMaxRate, uint64(10000000)) assert.Equal(config.HypervisorConfig.TxRateLimiterMaxRate, uint64(10000000)) diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 5bd9b26db4..6618d0acdf 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -130,6 +130,9 @@ const ( // entropy (/dev/random, /dev/urandom or real hardware RNG device) EntropySource = kataAnnotHypervisorPrefix + "entropy_source" + // UseLegacySerial sets legacy serial device for guest console if available and implemented for architecture + UseLegacySerial = kataAnnotHypervisorPrefix + "use_legacy_serial" + // // CPU Annotations // From 834f93ce8a11bf6fca5500e6a8f5cecf71e4f6f8 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Thu, 19 May 2022 09:47:32 +0300 Subject: [PATCH 06/26] docs: fix annotations example annotation value should always be quoted, regardless to its type Signed-off-by: Snir Sheriber --- docs/how-to/how-to-set-sandbox-config-kata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/how-to-set-sandbox-config-kata.md b/docs/how-to/how-to-set-sandbox-config-kata.md index 343b7fe2e6..9f612831c2 100644 --- a/docs/how-to/how-to-set-sandbox-config-kata.md +++ b/docs/how-to/how-to-set-sandbox-config-kata.md @@ -173,7 +173,7 @@ kind: Pod metadata: name: pod2 annotations: - io.katacontainers.config.runtime.disable_guest_seccomp: false + io.katacontainers.config.runtime.disable_guest_seccomp: "false" spec: runtimeClassName: kata containers: From ce2e521a0f5f08698429daeacdc852b6a12f1679 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca Date: Wed, 18 May 2022 22:13:05 +0200 Subject: [PATCH 07/26] runtime: remove duplicate 'types' import Fallout of 09f7962ff Fixes #4285 Signed-off-by: Rafael Fonseca --- src/runtime/virtcontainers/acrn.go | 3 +- src/runtime/virtcontainers/agent.go | 3 +- src/runtime/virtcontainers/clh.go | 11 ++--- src/runtime/virtcontainers/container.go | 3 +- src/runtime/virtcontainers/kata_agent.go | 5 +- src/runtime/virtcontainers/kata_agent_test.go | 9 ++-- src/runtime/virtcontainers/mock_agent.go | 3 +- src/runtime/virtcontainers/qemu.go | 47 +++++++++---------- src/runtime/virtcontainers/sandbox.go | 23 +++++---- 9 files changed, 49 insertions(+), 58 deletions(-) diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index 6b26792019..5b73439b47 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -28,7 +28,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -543,7 +542,7 @@ func (a *Acrn) updateBlockDevice(drive *config.BlockDrive) error { slot := AcrnBlkdDevSlot[drive.Index] //Explicitly set PCIPath to NULL, so that VirtPath can be used - drive.PCIPath = vcTypes.PciPath{} + drive.PCIPath = types.PciPath{} args := []string{"blkrescan", a.acrnConfig.Name, fmt.Sprintf("%d,%s", slot, drive.File)} diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index 0d6d6b7951..0db3927683 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -13,7 +13,6 @@ import ( pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/net/context" ) @@ -170,7 +169,7 @@ type agent interface { copyFile(ctx context.Context, src, dst string) error // Tell the agent to setup the swapfile in the guest - addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error + addSwap(ctx context.Context, PCIPath types.PciPath) error // markDead tell agent that the guest is dead markDead(ctx context.Context) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index c82d0d42c1..e95e5a7a42 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -32,7 +32,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -632,18 +631,18 @@ func clhDriveIndexToID(i int) string { // and/or kernel enumerates it. They get away with it only because // they don't use bridges, and so the bus is always 0. Under that // assumption convert a clh PciDeviceInfo into a PCI path -func clhPciInfoToPath(pciInfo chclient.PciDeviceInfo) (vcTypes.PciPath, error) { +func clhPciInfoToPath(pciInfo chclient.PciDeviceInfo) (types.PciPath, error) { tokens := strings.Split(pciInfo.Bdf, ":") if len(tokens) != 3 || tokens[0] != "0000" || tokens[1] != "00" { - return vcTypes.PciPath{}, fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) + return types.PciPath{}, fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } tokens = strings.Split(tokens[2], ".") if len(tokens) != 2 || tokens[1] != "0" || len(tokens[0]) != 2 { - return vcTypes.PciPath{}, fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) + return types.PciPath{}, fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } - return vcTypes.PciPathFromString(tokens[0]) + return types.PciPathFromString(tokens[0]) } func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) error { @@ -719,7 +718,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { return fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } - device.GuestPciPath, err = vcTypes.PciPathFromString(tokens[0]) + device.GuestPciPath, err = types.PciPathFromString(tokens[0]) return err } diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 4db6403ffa..d5239a386e 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -23,7 +23,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -405,7 +404,7 @@ func (c *Container) GetPatchedOCISpec() *specs.Spec { // container. func (c *Container) setContainerState(state types.StateString) error { if state == "" { - return vcTypes.ErrNeedState + return types.ErrNeedState } c.Logger().Debugf("Setting container state from %v to %v", c.state.State, state) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 04619efde6..bee94ef9b3 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -32,7 +32,6 @@ import ( vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/gogo/protobuf/proto" @@ -417,7 +416,7 @@ func (k *kataAgent) configure(ctx context.Context, h Hypervisor, id, sharePath s } case types.MockHybridVSock: default: - return vcTypes.ErrInvalidConfigType + return types.ErrInvalidConfigType } // Neither create shared directory nor add 9p device if hypervisor @@ -2147,7 +2146,7 @@ func (k *kataAgent) copyFile(ctx context.Context, src, dst string) error { return nil } -func (k *kataAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error { +func (k *kataAgent) addSwap(ctx context.Context, PCIPath types.PciPath) error { span, ctx := katatrace.Trace(ctx, k.Logger(), "addSwap", kataAgentTracingTags) defer span.End() diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index f4d996736a..c24934b682 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -35,7 +35,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/mock" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) const sysHugepagesDir = "/sys/kernel/mm/hugepages" @@ -45,7 +44,7 @@ var ( testBlockDeviceCtrPath = "testBlockDeviceCtrPath" testDevNo = "testDevNo" testNvdimmID = "testNvdimmID" - testPCIPath, _ = vcTypes.PciPathFromString("04/02") + testPCIPath, _ = types.PciPathFromString("04/02") testSCSIAddr = "testSCSIAddr" testVirtPath = "testVirtPath" ) @@ -372,11 +371,11 @@ func TestHandleBlockVolume(t *testing.T) { vDestination := "/VhostUserBlk/destination" bDestination := "/DeviceBlock/destination" dDestination := "/DeviceDirectBlock/destination" - vPCIPath, err := vcTypes.PciPathFromString("01/02") + vPCIPath, err := types.PciPathFromString("01/02") assert.NoError(t, err) - bPCIPath, err := vcTypes.PciPathFromString("03/04") + bPCIPath, err := types.PciPathFromString("03/04") assert.NoError(t, err) - dPCIPath, err := vcTypes.PciPathFromString("04/05") + dPCIPath, err := types.PciPathFromString("04/05") assert.NoError(t, err) vDev := drivers.NewVhostUserBlkDevice(&config.DeviceInfo{ID: vDevID}) diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index 99df3537b4..43c84b9471 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -13,7 +13,6 @@ import ( pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/net/context" ) @@ -217,7 +216,7 @@ func (n *mockAgent) copyFile(ctx context.Context, src, dst string) error { } // addSwap is the Noop agent setup swap. It does nothing. -func (n *mockAgent) addSwap(ctx context.Context, PCIPath vcTypes.PciPath) error { +func (n *mockAgent) addSwap(ctx context.Context, PCIPath types.PciPath) error { return nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index a910c580ff..4ad365f75c 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -40,7 +40,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -1328,15 +1327,15 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri } }() - bridgeSlot, err := vcTypes.PciSlotFromInt(bridge.Addr) + bridgeSlot, err := types.PciSlotFromInt(bridge.Addr) if err != nil { return err } - devSlot, err := vcTypes.PciSlotFromString(addr) + devSlot, err := types.PciSlotFromString(addr) if err != nil { return err } - drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + drive.PCIPath, err = types.PciPathFromSlots(bridgeSlot, devSlot) if err != nil { return err } @@ -1420,12 +1419,12 @@ func (q *qemu) hotplugAddVhostUserBlkDevice(ctx context.Context, vAttr *config.V return err } - devSlot, err := vcTypes.PciSlotFromString(addr) + devSlot, err := types.PciSlotFromString(addr) if err != nil { return err } - vAttr.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + vAttr.PCIPath, err = types.PciPathFromSlots(bridgeSlot, devSlot) if err != nil { return err } @@ -1445,16 +1444,16 @@ func (q *qemu) hotplugAddVhostUserBlkDevice(ctx context.Context, vAttr *config.V } }() - bridgeSlot, err := vcTypes.PciSlotFromInt(bridge.Addr) + bridgeSlot, err := types.PciSlotFromInt(bridge.Addr) if err != nil { return err } - devSlot, err := vcTypes.PciSlotFromString(addr) + devSlot, err := types.PciSlotFromString(addr) if err != nil { return err } - vAttr.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + vAttr.PCIPath, err = types.PciPathFromSlots(bridgeSlot, devSlot) if err = q.qmpMonitorCh.qmp.ExecutePCIVhostUserDevAdd(q.qmpMonitorCh.ctx, driver, devID, vAttr.DevID, addr, bridge.ID); err != nil { return err @@ -1519,48 +1518,48 @@ func (q *qemu) hotplugVhostUserDevice(ctx context.Context, vAttr *config.VhostUs } // Query QMP to find the PCI slot of a device, given its QOM path or ID -func (q *qemu) qomGetSlot(qomPath string) (vcTypes.PciSlot, error) { +func (q *qemu) qomGetSlot(qomPath string) (types.PciSlot, error) { addr, err := q.qmpMonitorCh.qmp.ExecQomGet(q.qmpMonitorCh.ctx, qomPath, "addr") if err != nil { - return vcTypes.PciSlot{}, err + return types.PciSlot{}, err } addrf, ok := addr.(float64) // XXX going via float makes no real sense, but that's how // JSON works, and we'll get away with it for the small values // we have here if !ok { - return vcTypes.PciSlot{}, fmt.Errorf("addr QOM property of %q is %T not a number", qomPath, addr) + return types.PciSlot{}, fmt.Errorf("addr QOM property of %q is %T not a number", qomPath, addr) } addri := int(addrf) slotNum, funcNum := addri>>3, addri&0x7 if funcNum != 0 { - return vcTypes.PciSlot{}, fmt.Errorf("Unexpected non-zero PCI function (%02x.%1x) on %q", + return types.PciSlot{}, fmt.Errorf("Unexpected non-zero PCI function (%02x.%1x) on %q", slotNum, funcNum, qomPath) } - return vcTypes.PciSlotFromInt(slotNum) + return types.PciSlotFromInt(slotNum) } // Query QMP to find a device's PCI path given its QOM path or ID -func (q *qemu) qomGetPciPath(qemuID string) (vcTypes.PciPath, error) { +func (q *qemu) qomGetPciPath(qemuID string) (types.PciPath, error) { // XXX: For now we assume there's exactly one bridge, since // that's always how we configure qemu from Kata for now. It // would be good to generalize this to different PCI // topologies devSlot, err := q.qomGetSlot(qemuID) if err != nil { - return vcTypes.PciPath{}, err + return types.PciPath{}, err } busq, err := q.qmpMonitorCh.qmp.ExecQomGet(q.qmpMonitorCh.ctx, qemuID, "parent_bus") if err != nil { - return vcTypes.PciPath{}, err + return types.PciPath{}, err } bus, ok := busq.(string) if !ok { - return vcTypes.PciPath{}, fmt.Errorf("parent_bus QOM property of %s is %t not a string", qemuID, busq) + return types.PciPath{}, fmt.Errorf("parent_bus QOM property of %s is %t not a string", qemuID, busq) } // `bus` is the QOM path of the QOM bus object, but we need @@ -1569,16 +1568,16 @@ func (q *qemu) qomGetPciPath(qemuID string) (vcTypes.PciPath, error) { // path component. idx := strings.LastIndex(bus, "/") if idx == -1 { - return vcTypes.PciPath{}, fmt.Errorf("Bus has unexpected QOM path %s", bus) + return types.PciPath{}, fmt.Errorf("Bus has unexpected QOM path %s", bus) } bridge := bus[:idx] bridgeSlot, err := q.qomGetSlot(bridge) if err != nil { - return vcTypes.PciPath{}, err + return types.PciPath{}, err } - return vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + return types.PciPathFromSlots(bridgeSlot, devSlot) } func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op Operation) (err error) { @@ -1739,15 +1738,15 @@ func (q *qemu) hotplugNetDevice(ctx context.Context, endpoint Endpoint, op Opera } }() - bridgeSlot, err := vcTypes.PciSlotFromInt(bridge.Addr) + bridgeSlot, err := types.PciSlotFromInt(bridge.Addr) if err != nil { return err } - devSlot, err := vcTypes.PciSlotFromString(addr) + devSlot, err := types.PciSlotFromString(addr) if err != nil { return err } - pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + pciPath, err := types.PciPathFromSlots(bridgeSlot, devSlot) endpoint.SetPciPath(pciPath) var machine govmmQemu.Machine diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 369c8d8f41..d494d54bab 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -41,7 +41,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cpuset" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -744,18 +743,18 @@ func rwLockSandbox(sandboxID string) (func() error, error) { // sandbox structure, based on a container ID. func (s *Sandbox) findContainer(containerID string) (*Container, error) { if s == nil { - return nil, vcTypes.ErrNeedSandbox + return nil, types.ErrNeedSandbox } if containerID == "" { - return nil, vcTypes.ErrNeedContainerID + return nil, types.ErrNeedContainerID } if c, ok := s.containers[containerID]; ok { return c, nil } - return nil, errors.Wrapf(vcTypes.ErrNoSuchContainer, "Could not find the container %q from the sandbox %q containers list", + return nil, errors.Wrapf(types.ErrNoSuchContainer, "Could not find the container %q from the sandbox %q containers list", containerID, s.id) } @@ -763,15 +762,15 @@ func (s *Sandbox) findContainer(containerID string) (*Container, error) { // sandbox structure, based on a container ID. func (s *Sandbox) removeContainer(containerID string) error { if s == nil { - return vcTypes.ErrNeedSandbox + return types.ErrNeedSandbox } if containerID == "" { - return vcTypes.ErrNeedContainerID + return types.ErrNeedContainerID } if _, ok := s.containers[containerID]; !ok { - return errors.Wrapf(vcTypes.ErrNoSuchContainer, "Could not remove the container %q from the sandbox %q containers list", + return errors.Wrapf(types.ErrNoSuchContainer, "Could not remove the container %q from the sandbox %q containers list", containerID, s.id) } @@ -1417,7 +1416,7 @@ func (s *Sandbox) KillContainer(ctx context.Context, containerID string, signal // DeleteContainer deletes a container from the sandbox func (s *Sandbox) DeleteContainer(ctx context.Context, containerID string) (VCContainer, error) { if containerID == "" { - return nil, vcTypes.ErrNeedContainerID + return nil, types.ErrNeedContainerID } // Fetch the container. @@ -1453,7 +1452,7 @@ func (s *Sandbox) DeleteContainer(ctx context.Context, containerID string) (VCCo // StatusContainer gets the status of a container func (s *Sandbox) StatusContainer(containerID string) (ContainerStatus, error) { if containerID == "" { - return ContainerStatus{}, vcTypes.ErrNeedContainerID + return ContainerStatus{}, types.ErrNeedContainerID } if c, ok := s.containers[containerID]; ok { @@ -1472,7 +1471,7 @@ func (s *Sandbox) StatusContainer(containerID string) (ContainerStatus, error) { }, nil } - return ContainerStatus{}, vcTypes.ErrNoSuchContainer + return ContainerStatus{}, types.ErrNoSuchContainer } // EnterContainer is the virtcontainers container command execution entry point. @@ -1719,7 +1718,7 @@ func (s *Sandbox) Stop(ctx context.Context, force bool) error { // setSandboxState sets the in-memory state of the sandbox. func (s *Sandbox) setSandboxState(state types.StateString) error { if state == "" { - return vcTypes.ErrNeedState + return types.ErrNeedState } // update in-memory state @@ -2324,7 +2323,7 @@ func (s *Sandbox) getSandboxCPUSet() (string, string, error) { func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err error) { virtLog.Info("fetch sandbox") if sandboxID == "" { - return nil, vcTypes.ErrNeedSandboxID + return nil, types.ErrNeedSandboxID } var config SandboxConfig From 7a5ccd12641c5ef63d4c13eb0269380684452771 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca Date: Wed, 18 May 2022 21:55:28 +0200 Subject: [PATCH 08/26] runtime: sync docstrings with function names The functions were renamed but their docstrings were not. Fixes #4006 Signed-off-by: Rafael Fonseca --- src/runtime/virtcontainers/acrn.go | 10 +++++----- src/runtime/virtcontainers/clh.go | 8 ++++---- src/runtime/virtcontainers/fc.go | 12 ++++++------ src/runtime/virtcontainers/qemu.go | 12 ++++++------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index 6b26792019..2d93970c7f 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -426,7 +426,7 @@ func (a *Acrn) CreateVM(ctx context.Context, id string, network Network, hypervi return nil } -// startSandbox will start the Sandbox's VM. +// StartVM will start the Sandbox's VM. func (a *Acrn) StartVM(ctx context.Context, timeoutSecs int) error { span, ctx := katatrace.Trace(ctx, a.Logger(), "StartVM", acrnTracingTags, map[string]string{"sandbox_id": a.id}) defer span.End() @@ -486,7 +486,7 @@ func (a *Acrn) waitVM(ctx context.Context, timeoutSecs int) error { return nil } -// stopSandbox will stop the Sandbox's VM. +// StopVM will stop the Sandbox's VM. func (a *Acrn) StopVM(ctx context.Context, waitOnly bool) (err error) { span, _ := katatrace.Trace(ctx, a.Logger(), "StopVM", acrnTracingTags, map[string]string{"sandbox_id": a.id}) defer span.End() @@ -600,7 +600,7 @@ func (a *Acrn) ResumeVM(ctx context.Context) error { return nil } -// addDevice will add extra devices to acrn command line. +// AddDevice will add extra devices to acrn command line. func (a *Acrn) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error { var err error span, _ := katatrace.Trace(ctx, a.Logger(), "AddDevice", acrnTracingTags, map[string]string{"sandbox_id": a.id}) @@ -633,8 +633,8 @@ func (a *Acrn) AddDevice(ctx context.Context, devInfo interface{}, devType Devic return err } -// getSandboxConsole builds the path of the console where we can read -// logs coming from the sandbox. +// GetVMConsole builds the path of the console where we can read logs coming +// from the sandbox. func (a *Acrn) GetVMConsole(ctx context.Context, id string) (string, string, error) { span, _ := katatrace.Trace(ctx, a.Logger(), "GetVMConsole", acrnTracingTags, map[string]string{"sandbox_id": a.id}) defer span.End() diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index c82d0d42c1..2dd848047e 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -538,7 +538,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net return nil } -// startSandbox will start the VMM and boot the virtual machine for the given sandbox. +// StartVM will start the VMM and boot the virtual machine for the given sandbox. func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { span, _ := katatrace.Trace(ctx, clh.Logger(), "StartVM", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() @@ -592,8 +592,8 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { return nil } -// getSandboxConsole builds the path of the console where we can read -// logs coming from the sandbox. +// GetVMConsole builds the path of the console where we can read logs coming +// from the sandbox. func (clh *cloudHypervisor) GetVMConsole(ctx context.Context, id string) (string, string, error) { clh.Logger().WithField("function", "GetVMConsole").WithField("id", id).Info("Get Sandbox Console") master, slave, err := console.NewPty() @@ -907,7 +907,7 @@ func (clh *cloudHypervisor) ResumeVM(ctx context.Context) error { return nil } -// stopSandbox will stop the Sandbox's VM. +// StopVM will stop the Sandbox's VM. func (clh *cloudHypervisor) StopVM(ctx context.Context, waitOnly bool) (err error) { span, _ := katatrace.Trace(ctx, clh.Logger(), "StopVM", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 384ce15a12..3a89c7fa44 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -200,7 +200,7 @@ func (fc *firecracker) setConfig(config *HypervisorConfig) error { } // CreateVM For firecracker this call only sets the internal structure up. -// The sandbox will be created and started through startSandbox(). +// The sandbox will be created and started through StartVM(). func (fc *firecracker) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error { fc.ctx = ctx @@ -763,7 +763,7 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error { return nil } -// startSandbox will start the hypervisor for the given sandbox. +// StartVM will start the hypervisor for the given sandbox. // In the context of firecracker, this will start the hypervisor, // for configuration, but not yet start the actual virtual machine func (fc *firecracker) StartVM(ctx context.Context, timeout int) error { @@ -881,7 +881,7 @@ func (fc *firecracker) cleanupJail(ctx context.Context) { } } -// stopSandbox will stop the Sandbox's VM. +// StopVM will stop the Sandbox's VM. func (fc *firecracker) StopVM(ctx context.Context, waitOnly bool) (err error) { span, _ := katatrace.Trace(ctx, fc.Logger(), "StopVM", fcTracingTags, map[string]string{"sandbox_id": fc.id}) defer span.End() @@ -1025,7 +1025,7 @@ func (fc *firecracker) fcUpdateBlockDrive(ctx context.Context, path, id string) return nil } -// addDevice will add extra devices to firecracker. Limited to configure before the +// AddDevice will add extra devices to firecracker. Limited to configure before the // virtual machine starts. Devices include drivers and network interfaces only. func (fc *firecracker) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error { span, _ := katatrace.Trace(ctx, fc.Logger(), "AddDevice", fcTracingTags, map[string]string{"sandbox_id": fc.id}) @@ -1128,8 +1128,8 @@ func (fc *firecracker) HotplugRemoveDevice(ctx context.Context, devInfo interfac } } -// getSandboxConsole builds the path of the console where we can read -// logs coming from the sandbox. +// GetVMConsole builds the path of the console where we can read logs coming +// from the sandbox. func (fc *firecracker) GetVMConsole(ctx context.Context, id string) (string, string, error) { master, slave, err := console.NewPty() if err != nil { diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index a910c580ff..495214eb32 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -797,7 +797,7 @@ func (q *qemu) setupVirtioMem(ctx context.Context) error { return err } -// startSandbox will start the Sandbox's VM. +// StartVM will start the Sandbox's VM. func (q *qemu) StartVM(ctx context.Context, timeout int) error { span, ctx := katatrace.Trace(ctx, q.Logger(), "StartVM", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() @@ -969,7 +969,7 @@ func (q *qemu) waitVM(ctx context.Context, timeout int) error { return nil } -// stopSandbox will stop the Sandbox's VM. +// StopVM will stop the Sandbox's VM. func (q *qemu) StopVM(ctx context.Context, waitOnly bool) error { span, _ := katatrace.Trace(ctx, q.Logger(), "StopVM", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() @@ -2039,7 +2039,7 @@ func (q *qemu) ResumeVM(ctx context.Context) error { return q.togglePauseSandbox(ctx, false) } -// addDevice will add extra devices to Qemu command line. +// AddDevice will add extra devices to Qemu command line. func (q *qemu) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error { var err error span, _ := katatrace.Trace(ctx, q.Logger(), "AddDevice", qemuTracingTags) @@ -2098,8 +2098,8 @@ func (q *qemu) AddDevice(ctx context.Context, devInfo interface{}, devType Devic return err } -// getSandboxConsole builds the path of the console where we can read -// logs coming from the sandbox. +// GetVMConsole builds the path of the console where we can read logs coming +// from the sandbox. func (q *qemu) GetVMConsole(ctx context.Context, id string) (string, string, error) { span, _ := katatrace.Trace(ctx, q.Logger(), "GetVMConsole", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() @@ -2172,7 +2172,7 @@ func (q *qemu) Disconnect(ctx context.Context) { q.qmpShutdown() } -// resizeMemory get a request to update the VM memory to reqMemMB +// ResizeMemory gets a request to update the VM memory to reqMemMB // Memory update is managed with two approaches // Add memory to VM: // When memory is required to be added we hotplug memory From f29595318301133bf080f460ba4f18b99c777bca Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Fri, 20 May 2022 14:55:00 -0700 Subject: [PATCH 09/26] runtime: fix incorrect Action function for direct-volume stats The action function expects a function that returns error but the current direct-volume stats Action returns (string, error) which is invalid. This change fixes the format and print out the stats from the command instead. Fixes: #4293 Signed-off-by: Yibo Zhuang --- src/runtime/cmd/kata-runtime/kata-volume.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/runtime/cmd/kata-runtime/kata-volume.go b/src/runtime/cmd/kata-runtime/kata-volume.go index e08e9482fa..a94d35d72e 100644 --- a/src/runtime/cmd/kata-runtime/kata-volume.go +++ b/src/runtime/cmd/kata-runtime/kata-volume.go @@ -7,10 +7,11 @@ package main import ( "encoding/json" + "fmt" "net/url" containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" - "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" "github.com/urfave/cli" @@ -89,12 +90,14 @@ var statsCommand = cli.Command{ Destination: &volumePath, }, }, - Action: func(c *cli.Context) (string, error) { + Action: func(c *cli.Context) error { stats, err := Stats(volumePath) if err != nil { - return "", cli.NewExitError(err.Error(), 1) + return cli.NewExitError(err.Error(), 1) } - return string(stats), nil + + fmt.Println(string(stats)) + return nil }, } From ffdc065b4cfa0a13dc587eae7671ab3ed62ede21 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Fri, 20 May 2022 18:41:51 -0700 Subject: [PATCH 10/26] runtime: direct-volume stats update to use GET parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The go default http mux AFAIK doesn’t support pattern routing so right now client is padding the url for direct-volume stats with a subpath of the volume path and this will always result in 404 not found returned by the shim. This change will update the shim to take the volume path as a GET query parameter instead of a subpath. If the parameter is missing or empty, then return 400 BadRequest to the client. Fixes: #4297 Signed-off-by: Yibo Zhuang --- .../pkg/containerd-shim-v2/shim_management.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/runtime/pkg/containerd-shim-v2/shim_management.go b/src/runtime/pkg/containerd-shim-v2/shim_management.go index b5ad03eed2..e109222507 100644 --- a/src/runtime/pkg/containerd-shim-v2/shim_management.go +++ b/src/runtime/pkg/containerd-shim-v2/shim_management.go @@ -32,6 +32,8 @@ import ( ) const ( + DirectVolumePathKey = "path" + DirectVolumeStatUrl = "/direct-volume/stats" DirectVolumeResizeUrl = "/direct-volume/resize" ) @@ -139,7 +141,16 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily { } func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) { - volumePath, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, DirectVolumeStatUrl)) + val := r.URL.Query().Get(DirectVolumePathKey) + if val == "" { + msg := fmt.Sprintf("Required parameter %s not found", DirectVolumePathKey) + shimMgtLog.Info(msg) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(msg)) + return + } + + volumePath, err := url.PathUnescape(val) if err != nil { shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path") w.WriteHeader(http.StatusInternalServerError) From 4428ceae16f10254b0126ce4a635c6ba7a5cef87 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Fri, 20 May 2022 18:42:47 -0700 Subject: [PATCH 11/26] runtime: direct-volume stats use correct name Today the shim does a translation when doing direct-volume stats where it takes the source and returns the mount path within the guest. The source for a direct-assigned volume is actually the device path on the host and not the publish volume path. This change will perform a lookup of the mount info during direct-volume stats to ensure that the device path is provided to the shim for querying the volume stats. Fixes: #4297 Signed-off-by: Yibo Zhuang --- src/runtime/cmd/kata-runtime/kata-volume.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/runtime/cmd/kata-runtime/kata-volume.go b/src/runtime/cmd/kata-runtime/kata-volume.go index e08e9482fa..bfc026fd44 100644 --- a/src/runtime/cmd/kata-runtime/kata-volume.go +++ b/src/runtime/cmd/kata-runtime/kata-volume.go @@ -7,10 +7,11 @@ package main import ( "encoding/json" + "fmt" "net/url" containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" - "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" "github.com/urfave/cli" @@ -127,8 +128,14 @@ func Stats(volumePath string) ([]byte, error) { if err != nil { return nil, err } - urlSafeDevicePath := url.PathEscape(volumePath) - body, err := shimclient.DoGet(sandboxId, defaultTimeout, containerdshim.DirectVolumeStatUrl+"/"+urlSafeDevicePath) + volumeMountInfo, err := volume.VolumeMountInfo(volumePath) + if err != nil { + return nil, err + } + + urlSafeDevicePath := url.PathEscape(volumeMountInfo.Device) + body, err := shimclient.DoGet(sandboxId, defaultTimeout, + fmt.Sprintf("%s?%s=%s", containerdshim.DirectVolumeStatUrl, containerdshim.DirectVolumePathKey, urlSafeDevicePath)) if err != nil { return nil, err } @@ -141,8 +148,13 @@ func Resize(volumePath string, size uint64) error { if err != nil { return err } + volumeMountInfo, err := volume.VolumeMountInfo(volumePath) + if err != nil { + return err + } + resizeReq := containerdshim.ResizeRequest{ - VolumePath: volumePath, + VolumePath: volumeMountInfo.Device, Size: size, } encoded, err := json.Marshal(resizeReq) From 8e7c5975c6aedcf1a8287f28156221311ff6295f Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Fri, 20 May 2022 18:43:27 -0700 Subject: [PATCH 12/26] agent: fix direct-assigned volume stats The current implementation of walking the disks to match with the requested volume path in agent doesn't work because the volume path provided by the shim to the agent is the mount path within the guest and not the device name. The current logic is trying to match the device name to the volume path which will never match. This change will simplify the get_volume_capacity_stats and get_volume_inode_stats to just call statfs and get the bytes and inodes usage of the volume path directly. Fixes: #4297 Signed-off-by: Yibo Zhuang --- src/agent/src/rpc.rs | 112 ++++++++++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 33 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 4a2e0837ad..ebda9e9451 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -40,13 +40,11 @@ use rustjail::specconv::CreateOpts; use nix::errno::Errno; use nix::mount::MsFlags; -use nix::sys::stat; +use nix::sys::{stat, statfs}; use nix::unistd::{self, Pid}; use rustjail::cgroups::Manager; use rustjail::process::ProcessOperations; -use sysinfo::{DiskExt, System, SystemExt}; - use crate::device::{ add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci, }; @@ -71,7 +69,6 @@ use tracing::instrument; use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ}; use std::fs; -use std::os::unix::fs::MetadataExt; use std::os::unix::prelude::PermissionsExt; use std::process::{Command, Stdio}; use std::time::Duration; @@ -1468,20 +1465,12 @@ fn get_memory_info( fn get_volume_capacity_stats(path: &str) -> Result { let mut usage = VolumeUsage::new(); - let s = System::new(); - for disk in s.disks() { - if let Some(v) = disk.name().to_str() { - if v.to_string().eq(path) { - usage.available = disk.available_space(); - usage.total = disk.total_space(); - usage.used = usage.total - usage.available; - usage.unit = VolumeUsage_Unit::BYTES; // bytes - break; - } - } else { - return Err(anyhow!(nix::Error::EINVAL)); - } - } + let stat = statfs::statfs(path)?; + let block_size = stat.block_size() as u64; + usage.total = stat.blocks() * block_size; + usage.available = stat.blocks_free() * block_size; + usage.used = usage.total - usage.available; + usage.unit = VolumeUsage_Unit::BYTES; Ok(usage) } @@ -1489,20 +1478,11 @@ fn get_volume_capacity_stats(path: &str) -> Result { fn get_volume_inode_stats(path: &str) -> Result { let mut usage = VolumeUsage::new(); - let s = System::new(); - for disk in s.disks() { - if let Some(v) = disk.name().to_str() { - if v.to_string().eq(path) { - let meta = fs::metadata(disk.mount_point())?; - let inode = meta.ino(); - usage.used = inode; - usage.unit = VolumeUsage_Unit::INODES; - break; - } - } else { - return Err(anyhow!(nix::Error::EINVAL)); - } - } + let stat = statfs::statfs(path)?; + usage.total = stat.files(); + usage.available = stat.files_free(); + usage.used = usage.total - usage.available; + usage.unit = VolumeUsage_Unit::INODES; Ok(usage) } @@ -1894,7 +1874,11 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::{assert_result, namespace::Namespace, protocols::agent_ttrpc::AgentService as _}; + use crate::{ + assert_result, namespace::Namespace, protocols::agent_ttrpc::AgentService as _, + skip_if_not_root, + }; + use nix::mount; use oci::{Hook, Hooks, Linux, LinuxNamespace}; use tempfile::{tempdir, TempDir}; use ttrpc::{r#async::TtrpcContext, MessageHeader}; @@ -2769,4 +2753,66 @@ OtherField:other } } } + + #[tokio::test] + async fn test_volume_capacity_stats() { + skip_if_not_root!(); + + // Verify error if path does not exist + assert!(get_volume_capacity_stats("/does-not-exist").is_err()); + + // Create a new tmpfs mount, and verify the initial values + let mount_dir = tempfile::tempdir().unwrap(); + mount::mount( + Some("tmpfs"), + mount_dir.path().to_str().unwrap(), + Some("tmpfs"), + mount::MsFlags::empty(), + None::<&str>, + ) + .unwrap(); + let mut stats = get_volume_capacity_stats(mount_dir.path().to_str().unwrap()).unwrap(); + assert_eq!(stats.used, 0); + assert_ne!(stats.available, 0); + let available = stats.available; + + // Verify that writing a file will result in increased utilization + fs::write(mount_dir.path().join("file.dat"), "foobar").unwrap(); + stats = get_volume_capacity_stats(mount_dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(stats.used, 4 * 1024); + assert_eq!(stats.available, available - 4 * 1024); + } + + #[tokio::test] + async fn test_get_volume_inode_stats() { + skip_if_not_root!(); + + // Verify error if path does not exist + assert!(get_volume_inode_stats("/does-not-exist").is_err()); + + // Create a new tmpfs mount, and verify the initial values + let mount_dir = tempfile::tempdir().unwrap(); + mount::mount( + Some("tmpfs"), + mount_dir.path().to_str().unwrap(), + Some("tmpfs"), + mount::MsFlags::empty(), + None::<&str>, + ) + .unwrap(); + let mut stats = get_volume_inode_stats(mount_dir.path().to_str().unwrap()).unwrap(); + assert_eq!(stats.used, 1); + assert_ne!(stats.available, 0); + let available = stats.available; + + // Verify that creating a directory and writing a file will result in increased utilization + let dir = mount_dir.path().join("foobar"); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.as_path().join("file.dat"), "foobar").unwrap(); + stats = get_volume_inode_stats(mount_dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(stats.used, 3); + assert_eq!(stats.available, available - 2); + } } From dfad5728a70f7986a23de6125ddf1449f0d89823 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 11 Mar 2022 17:13:58 +0530 Subject: [PATCH 13/26] docs: Mention --cni flag while invoking ctr Specify that the `--cni` flag needs to be passed to the `ctr` tool while starting a container in order to have networking enabled for the container. This flag allows containerd to call into the configured network plugin which in turn creates a network interface for the container. Signed-off-by: Archana Shinde --- docs/how-to/containerd-kata.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/how-to/containerd-kata.md b/docs/how-to/containerd-kata.md index 9cd82128dd..5cdac16b98 100644 --- a/docs/how-to/containerd-kata.md +++ b/docs/how-to/containerd-kata.md @@ -312,10 +312,12 @@ To run a container with Kata Containers through the containerd command line, you ```bash $ sudo ctr image pull docker.io/library/busybox:latest -$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh +$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh ``` This launches a BusyBox container named `hello`, and it will be removed by `--rm` after it quits. +The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a +loopback interface is created. ### Launch Pods with `crictl` command line From 34b80382b65dbed61cede79dc4fe2e849d9d17a7 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 11 Mar 2022 17:17:00 +0530 Subject: [PATCH 14/26] docs: Get rid of note related to networking. One may want to use standalone containerd without k8s and still have network enabled for the container. Getting rid of note due to inaccuracy. Signed-off-by: Archana Shinde --- docs/how-to/containerd-kata.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/how-to/containerd-kata.md b/docs/how-to/containerd-kata.md index 5cdac16b98..2077f371a3 100644 --- a/docs/how-to/containerd-kata.md +++ b/docs/how-to/containerd-kata.md @@ -72,7 +72,6 @@ $ command -v containerd ### Install CNI plugins -> **Note:** You do not need to install CNI plugins if you do not want to use containerd with Kubernetes. > If you have installed Kubernetes with `kubeadm`, you might have already installed the CNI plugins. You can manually install CNI plugins as follows: From c95ba63c0cc2237f93fd2c3d36722d1b84edf8d5 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 11 Mar 2022 20:02:58 +0530 Subject: [PATCH 15/26] docs: Remove information related to Kata 1.x Since Kata 2.x does not support runtime cli, remove information related to it. Update the configuration snippet accordingly. Fixes #3870 Signed-off-by: Archana Shinde --- docs/how-to/containerd-kata.md | 99 +++++++--------------------------- 1 file changed, 19 insertions(+), 80 deletions(-) diff --git a/docs/how-to/containerd-kata.md b/docs/how-to/containerd-kata.md index 2077f371a3..6a7449aea8 100644 --- a/docs/how-to/containerd-kata.md +++ b/docs/how-to/containerd-kata.md @@ -130,74 +130,42 @@ For The `RuntimeClass` is suggested. -The following configuration includes three runtime classes: +The following configuration includes two runtime classes: - `plugins.cri.containerd.runtimes.runc`: the runc, and it is the default runtime. - `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/master/runtime/v2#binary-naming)) where the dot-connected string `io.containerd.kata.v2` is translated to `containerd-shim-kata-v2` (i.e. the binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)). -- `plugins.cri.containerd.runtimes.katacli`: the `containerd-shim-runc-v1` calls `kata-runtime`, which is the legacy process. ```toml [plugins.cri.containerd] no_pivot = false [plugins.cri.containerd.runtimes] - [plugins.cri.containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v1" - [plugins.cri.containerd.runtimes.runc.options] - NoPivotRoot = false - NoNewKeyring = false - ShimCgroup = "" - IoUid = 0 - IoGid = 0 - BinaryName = "runc" - Root = "" - CriuPath = "" - SystemdCgroup = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + privileged_without_host_devices = false + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 [plugins.cri.containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" - [plugins.cri.containerd.runtimes.katacli] - runtime_type = "io.containerd.runc.v1" - [plugins.cri.containerd.runtimes.katacli.options] - NoPivotRoot = false - NoNewKeyring = false - ShimCgroup = "" - IoUid = 0 - IoGid = 0 - BinaryName = "/usr/bin/kata-runtime" - Root = "" - CriuPath = "" - SystemdCgroup = false -``` - -From Containerd v1.2.4 and Kata v1.6.0, there is a new runtime option supported, which allows you to specify a specific Kata configuration file as follows: - -```toml - [plugins.cri.containerd.runtimes.kata] - runtime_type = "io.containerd.kata.v2" - privileged_without_host_devices = true - [plugins.cri.containerd.runtimes.kata.options] - ConfigPath = "/etc/kata-containers/config.toml" + privileged_without_host_devices = true + pod_annotations = ["io.katacontainers.*"] + container_annotations = ["io.katacontainers.*"] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] + ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration.toml" ``` `privileged_without_host_devices` tells containerd that a privileged Kata container should not have direct access to all host devices. If unset, containerd will pass all host devices to Kata container, which may cause security issues. +`pod_annotations` is the list of pod annotations passed to both the pod sandbox as well as container through the OCI config. + +`container_annotations` is the list of container annotations passed through to the OCI config of the containers. + This `ConfigPath` option is optional. If you do not specify it, shimv2 first tries to get the configuration file from the environment variable `KATA_CONF_FILE`. If neither are set, shimv2 will use the default Kata configuration file paths (`/etc/kata-containers/configuration.toml` and `/usr/share/defaults/kata-containers/configuration.toml`). -If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0 and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a -shell script with the following: - -```bash -#!/usr/bin/env bash -KATA_CONF_FILE=/etc/kata-containers/firecracker.toml containerd-shim-kata-v2 $@ -``` - -Name it as `/usr/local/bin/containerd-shim-katafc-v2` and reference it in the configuration of containerd: - -```toml - [plugins.cri.containerd.runtimes.kata-firecracker] - runtime_type = "io.containerd.katafc.v2" -``` - #### Kata Containers as the runtime for untrusted workload For cases without `RuntimeClass` support, we can use the legacy annotation method to support using Kata Containers @@ -217,28 +185,8 @@ and then, run an untrusted workload with Kata Containers: runtime_type = "io.containerd.kata.v2" ``` -For the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration: - -```toml - [plugins.cri.containerd] - - # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd. - [plugins.cri.containerd.default_runtime] - # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux - runtime_type = "io.containerd.runtime.v1.linux" - - # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it. - [plugins.cri.containerd.untrusted_workload_runtime] - # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux - runtime_type = "io.containerd.runtime.v1.linux" - - # runtime_engine is the name of the runtime engine used by containerd. - runtime_engine = "/usr/bin/kata-runtime" -``` - You can find more information on the [Containerd config documentation](https://github.com/containerd/cri/blob/master/docs/config.md) - #### Kata Containers as the default runtime If you want to set Kata Containers as the only runtime in the deployment, you can simply configure as follows: @@ -249,15 +197,6 @@ If you want to set Kata Containers as the only runtime in the deployment, you ca runtime_type = "io.containerd.kata.v2" ``` -Alternatively, for the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration: - -```toml - [plugins.cri.containerd] - [plugins.cri.containerd.default_runtime] - runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "/usr/bin/kata-runtime" -``` - ### Configuration for `cri-tools` > **Note:** If you skipped the [Install `cri-tools`](#install-cri-tools) section, you can skip this section too. From d20bc5a4d28314dad787d0f89457bff3ddbc36de Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 18 May 2022 17:38:35 +0800 Subject: [PATCH 16/26] virtiofsd: build rust based virtiofsd from source for non-x86_64 Based on @fidencio's opoinon, On Arm: static build virtiofsd using musl lib; on ppc64 & s390: static build virtiofsd using gnu lib; Fixes: #4258 Signed-off-by: Jianyong Wu --- .../virtiofsd/build-static-virtiofsd.sh | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh b/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh index d792e36f63..b23fa9a8de 100755 --- a/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh +++ b/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh @@ -9,6 +9,8 @@ set -o nounset set -o pipefail ARCH=$(uname -m) +ARCH_LIBC="" +LIBC="" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -19,11 +21,12 @@ virtiofsd_version="${virtiofsd_version:-}" [ -n "$virtiofsd_version" ] || virtiofsd_version=$(get_from_kata_deps "externals.virtiofsd.version") [ -n "$virtiofsd_version" ] || die "failed to get virtiofsd version" -if [ "${ARCH}" != "x86_64" ]; then - info "Only x86_64 binaries are distributed as part of the virtiofsd releases" && exit 1 -fi +[ -d "virtiofsd" ] && rm -r virtiofsd pull_virtiofsd_released_binary() { + if [ "${ARCH}" != "x86_64" ]; then + info "Only x86_64 binaries are distributed as part of the virtiofsd releases" && return 1 + fi info "Download virtiofsd version: ${virtiofsd_version}" virtiofsd_zip=$(get_from_kata_deps "externals.virtiofsd.meta.binary") [ -n "${virtiofsd_zip}" ] || die "failed to get virtiofsd binary URL" @@ -40,4 +43,47 @@ pull_virtiofsd_released_binary() { popd } -pull_virtiofsd_released_binary +init_env() { + case ${ARCH} in + "aarch64") + LIBC="musl" + ;; + "ppc64le") + LIBC="gnu" + ARCH="powerpc64le" + ;; + "s390") + LIBC="gnu" + ;; + "x86_64") + LIBC="musl" + esac + + ARCH_LIBC=${ARCH}-linux-${LIBC} +} + +build_virtiofsd_from_source() { + echo "build viriofsd from source" + init_env + + virtiofsd_url=$(get_from_kata_deps "externals.virtiofsd.url") + + git clone --depth 1 --branch ${virtiofsd_version} ${virtiofsd_url} virtiofsd + pushd virtiofsd + + RUSTFLAGS='-C target-feature=+crt-static -C link-self-contained=yes' + LIBSECCOMP_LINK_TYPE=static + LIBSECCOMP_LIB_PATH=/usr/lib/${ARCH_LIBC} + LIBCAPNG_LINK_TYPE=static + LIBCAPNG_LIB_PATH=/usr/lib/${ARCH_LIBC} + + cargo build --release --target ${ARCH}-unknown-linux-${LIBC} + + binary=$(find ./ -name virtiofsd) + mv -f ${binary} . + chmod +x virtiofsd + + popd +} + +pull_virtiofsd_released_binary || build_virtiofsd_from_source From 10c13d719a7b499b5955c2a3820fcd28b5fa287c Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 18 May 2022 17:42:33 +0800 Subject: [PATCH 17/26] qemu: remove virtiofsd option in qemu config As virtiofsd will be built base on rust, "virtiofsd" option is no longer needed in qemu. Fixes: #4258 Signed-off-by: Jianyong Wu --- .../packaging/scripts/configure-hypervisor.sh | 25 ------------------- .../virtiofsd/build-static-virtiofsd.sh | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/tools/packaging/scripts/configure-hypervisor.sh b/tools/packaging/scripts/configure-hypervisor.sh index 31dc4db25f..6bf4c1107e 100755 --- a/tools/packaging/scripts/configure-hypervisor.sh +++ b/tools/packaging/scripts/configure-hypervisor.sh @@ -317,31 +317,6 @@ generate_qemu_options() { # Don't build the qemu-io, qemu-nbd and qemu-image tools qemu_options+=(size:--disable-tools) - # Kata Containers may be configured to use the virtiofs daemon. - # - # But since QEMU 5.2 the daemon is built as part of the tools set - # (disabled with --disable-tools) thus it needs to be explicitely - # enabled. - # - # From Kata Containers 2.5.0-alpha2 x86_64 has been using the new - # implementation of virtiofs daemon, which is not part of QEMU. - # For the other arches, at least for now, keep building from while - # building QEMU. - # - # IOW, other arches are still using the C version of the virtiofsd. - case "$arch" in - aarch64) - qemu_options+=(functionality:--enable-virtiofsd) - ;; - x86_64) - ;; - ppc64le) - qemu_options+=(functionality:--enable-virtiofsd) - ;; - s390x) - qemu_options+=(functionality:--enable-virtiofsd) - ;; - esac qemu_options+=(functionality:--enable-virtfs) # Don't build linux-user bsd-user diff --git a/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh b/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh index b23fa9a8de..28124fa775 100755 --- a/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh +++ b/tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh @@ -52,7 +52,7 @@ init_env() { LIBC="gnu" ARCH="powerpc64le" ;; - "s390") + "s390x") LIBC="gnu" ;; "x86_64") From 35619b45aa2ba212383d4890210e9298b38cb592 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Fri, 20 May 2022 14:55:51 +0800 Subject: [PATCH 18/26] runk: merge oci-kata-agent into runk Merge two bins into one. Fixes: #4291 Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 13 +------------ src/tools/runk/Makefile | 22 ++-------------------- src/tools/runk/README.md | 8 ++------ src/tools/runk/src/main.rs | 7 +++++++ 4 files changed, 12 insertions(+), 38 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 37ffa091d2..88c058868a 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -78,9 +78,6 @@ const HOME_ENV_KEY: &str = "HOME"; const PIDNS_FD: &str = "PIDNS_FD"; const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD"; -#[cfg(feature = "standard-oci-runtime")] -const OCI_AGENT_BINARY: &str = "oci-kata-agent"; - #[derive(Debug)] pub struct ContainerStatus { pre_status: ContainerState, @@ -951,15 +948,7 @@ impl BaseContainer for LinuxContainer { let _ = unistd::close(pid); }); - cfg_if::cfg_if! { - if #[cfg(feature = "standard-oci-runtime")] { - let exec_path = PathBuf::from(OCI_AGENT_BINARY); - } - else { - let exec_path = std::env::current_exe()?; - } - } - + let exec_path = std::env::current_exe()?; let mut child = std::process::Command::new(exec_path); #[allow(unused_mut)] diff --git a/src/tools/runk/Makefile b/src/tools/runk/Makefile index cfd795fb6c..580aa39a0d 100644 --- a/src/tools/runk/Makefile +++ b/src/tools/runk/Makefile @@ -8,30 +8,16 @@ include ../../../utils.mk TARGET = runk TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) -AGENT_TARGET = oci-kata-agent -AGENT_TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(AGENT_TARGET) -AGENT_SOURCE_PATH = ../../agent - # BINDIR is a directory for installing executable programs BINDIR := /usr/local/bin .DEFAULT_GOAL := default default: build -build: build-agent build-runk - -build-agent: - make -C $(AGENT_SOURCE_PATH) STANDARD_OCI_RUNTIME=yes - -build-runk: +build: @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) -install: install-agent install-runk - -install-agent: - install -D $(AGENT_SOURCE_PATH)/$(AGENT_TARGET_PATH) $(BINDIR)/$(AGENT_TARGET) - -install-runk: +install: install -D $(TARGET_PATH) $(BINDIR)/$(TARGET) clean: @@ -47,11 +33,7 @@ check: standard_rust_check .PHONY: \ build \ - build-agent \ - build-runk \ install \ - install-agent \ - install-runk \ clean \ clippy \ format \ diff --git a/src/tools/runk/README.md b/src/tools/runk/README.md index 41663625a6..9d18e27bce 100644 --- a/src/tools/runk/README.md +++ b/src/tools/runk/README.md @@ -60,18 +60,14 @@ are welcome. Regarding features compared to `runc`, see the `Status of runk` section in the [issue](https://github.com/kata-containers/kata-containers/issues/2784). ## Building - -`runk` uses the modified the `kata-agent` binary, `oci-kata-agent`, which is an agent to be called from `runk`. -Therefore, you also need to build the `oci-kata-agent` to run `runk`. - -You can build both `runk` and `oci-kata-agent` as follows. +You can build `runk` as follows. ```bash $ cd runk $ make ``` -To install `runk` and `oci-kata-agent` into default directory for install executable program (`/usr/local/bin`): +To install `runk` into default directory for install executable program (`/usr/local/bin`): ```bash $ sudo make install diff --git a/src/tools/runk/src/main.rs b/src/tools/runk/src/main.rs index c73a22b093..8f464f4383 100644 --- a/src/tools/runk/src/main.rs +++ b/src/tools/runk/src/main.rs @@ -28,6 +28,7 @@ enum SubCommand { Common(CommonCmd), #[clap(flatten)] Custom(CustomCmd), + Init {}, } // Copy from https://github.com/containers/youki/blob/v0.0.3/crates/liboci-cli/src/lib.rs#L38-L44 @@ -84,6 +85,7 @@ async fn cmd_run(subcmd: SubCommand, root_path: &Path, logger: &Logger) -> Resul SubCommand::Custom(cmd) => match cmd { CustomCmd::Kill(kill) => commands::kill::run(kill, root_path, logger), }, + _ => unreachable!(), } } @@ -113,6 +115,11 @@ fn setup_logger( async fn real_main() -> Result<()> { let cli = Cli::parse(); + if let SubCommand::Init {} = cli.subcmd { + rustjail::container::init_child(); + exit(0); + } + let root_path = if let Some(path) = cli.global.root { path } else { From 122a85e222bcb2a4a95fb874173d6fb2c2b8ac46 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Mon, 23 May 2022 16:55:16 +0800 Subject: [PATCH 19/26] agent: remove bin oci-kata-agent Fixes: #4291 Signed-off-by: Tim Zhang --- src/agent/Cargo.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 220e66287a..77ec06d8ac 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -81,8 +81,3 @@ standard-oci-runtime = ["rustjail/standard-oci-runtime"] [[bin]] name = "kata-agent" path = "src/main.rs" - -[[bin]] -name = "oci-kata-agent" -path = "src/main.rs" -required-features = ["standard-oci-runtime"] From c9b291509d865e51994c75de4b08f8eb9fcab815 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 19 May 2022 14:33:13 +0100 Subject: [PATCH 20/26] snap: Fix whitespace Remove trailing space. Signed-off-by: James O. D. Hunt --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 06f2899b66..68d9369fb8 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -321,7 +321,7 @@ parts: after: [godeps] override-build: | arch=$(uname -m) - if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then + if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then sudo apt-get -y update sudo apt-get -y install ca-certificates curl gnupg lsb-release curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg From 43de5440e5cab6ef337c1943cfc6ebd9d6b6b545 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 19 May 2022 14:34:34 +0100 Subject: [PATCH 21/26] snap: Fix unbound variable error Don't assume `GITHUB_REF` is set. Signed-off-by: James O. D. Hunt --- snap/snapcraft.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 68d9369fb8..be13a7314b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -21,8 +21,8 @@ parts: version="9999" kata_url="https://github.com/kata-containers/kata-containers" - if echo "${GITHUB_REF}" | grep -q -E "^refs/tags"; then - version=$(echo ${GITHUB_REF} | cut -d/ -f3) + if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then + version=$(echo ${GITHUB_REF:-} | cut -d/ -f3) git checkout ${version} fi From d7b4ce049ec56539743c5c41dcdc8a56ca98471a Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 19 May 2022 14:35:35 +0100 Subject: [PATCH 22/26] snap: Remove unused variable Remove the unused `kata_url` variable and use the value in the `website` YAML metadata instead. Signed-off-by: James O. D. Hunt --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index be13a7314b..b5f5815b57 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,4 +1,5 @@ name: kata-containers +website: https://github.com/kata-containers/kata-containers summary: Build lightweight VMs that seamlessly plug into the containers ecosystem description: | Kata Containers is an open source project and community working to build a @@ -19,7 +20,6 @@ parts: - git-extras override-pull: | version="9999" - kata_url="https://github.com/kata-containers/kata-containers" if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then version=$(echo ${GITHUB_REF:-} | cut -d/ -f3) From 27d903b76a76ce7c1b5dcbbafbfbadb8b421d2c3 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 19 May 2022 14:38:01 +0100 Subject: [PATCH 23/26] snap: Put the yq binary in the staging bin directory Rather than putting the `yq` binary in the staging directory itself, put it in the `bin/` sub-directory. Signed-off-by: James O. D. Hunt --- snap/snapcraft.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b5f5815b57..e2dea40f69 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -46,7 +46,8 @@ parts: # put everything in stage cd ${SNAPCRAFT_STAGE} - yq_path="./yq" + mkdir -p "${SNAPCRAFT_STAGE}/bin/" + yq_path="${SNAPCRAFT_STAGE}/bin/yq" yq_pkg="github.com/mikefarah/yq" goos="linux" case "$(uname -m)" in @@ -82,7 +83,7 @@ parts: override-build: | [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler - yq=${SNAPCRAFT_STAGE}/yq + yq=${SNAPCRAFT_STAGE}/bin/yq # set GOPATH export GOPATH=${SNAPCRAFT_STAGE}/gopath @@ -185,7 +186,7 @@ parts: - bison - flex override-build: | - yq=${SNAPCRAFT_STAGE}/yq + yq=${SNAPCRAFT_STAGE}/bin/yq export PATH="${PATH}:${SNAPCRAFT_STAGE}" export GOPATH=${SNAPCRAFT_STAGE}/gopath kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME} @@ -249,7 +250,7 @@ parts: - libselinux1-dev - ninja-build override-build: | - yq=${SNAPCRAFT_STAGE}/yq + yq=${SNAPCRAFT_STAGE}/bin/yq export GOPATH=${SNAPCRAFT_STAGE}/gopath export GO111MODULE="auto" kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME} From 49361749ed3b85c812150b94fd1ae5498bcb20f5 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 23 May 2022 10:34:56 +0100 Subject: [PATCH 24/26] snap: Build and package rust version of virtiofsd Update the snap config file to build the rust version of `virtiofsd` for x86_64, but build QEMU's C version for other platforms. Fixes: #4261. Signed-off-by: James O. D. Hunt --- snap/snapcraft.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index e2dea40f69..94fd2a8c97 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -317,6 +317,41 @@ parts: # Hack: move qemu to / "snap/kata-containers/current/": "./" + virtiofsd: + plugin: nil + after: [godeps] + override-build: | + # Currently, only one platform uses the new rust virtiofsd. The + # others make use of QEMU's C implementation. + # + # See "tools/packaging/scripts/configure-hypervisor.sh". + if [ "$(uname -m)" = 'x86_64' ] + then + echo "INFO: Building rust version of virtiofsd" + else + echo "INFO: Building QEMU's C version of virtiofsd" + # Handled by the 'qemu' part, so nothing more to do here. + exit 0 + fi + + # put everything in stage + cd ${SNAPCRAFT_STAGE} + + export PATH="$PATH:${SNAPCRAFT_STAGE}/bin" + export GOPATH=${SNAPCRAFT_STAGE}/gopath + + kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME} + cd "${kata_dir}" + # Download the rust implementation of virtiofsd + tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh + sudo install \ + --owner='root' \ + --group='root' \ + --mode=0755 \ + -D \ + --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \ + virtiofsd/virtiofsd + cloud-hypervisor: plugin: nil after: [godeps] From fff832874ecb0850bbd17d5b4328eea54da85e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 26 May 2022 08:51:18 +0000 Subject: [PATCH 25/26] clh: Update to v24.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This release has been tracked through the v24.0 project. virtio-iommu specification describes how a device can be attached by default to a bypass domain. This feature is particularly helpful for booting a VM with guest software which doesn't support virtio-iommu but still need to access the device. Now that Cloud Hypervisor supports this feature, it can boot a VM with Rust Hypervisor Firmware or OVMF even if the virtio-block device exposing the disk image is placed behind a virtual IOMMU. Multiple checks have been added to the code to prevent devices with identical identifiers from being created, and therefore avoid unexpected behaviors at boot or whenever a device was hot plugged into the VM. Sparse mmap support has been added to both VFIO and vfio-user devices. This allows the device regions that are not fully mappable to be partially mapped. And the more a device region can be mapped into the guest address space, the fewer VM exits will be generated when this device is accessed. This directly impacts the performance related to this device. A new serial_number option has been added to --platform, allowing a user to set a specific serial number for the platform. This number is exposed to the guest through the SMBIOS. * Fix loading RAW firmware (#4072) * Reject compressed QCOW images (#4055) * Reject virtio-mem resize if device is not activated (#4003) * Fix potential mmap leaks from VFIO/vfio-user MMIO regions (#4069) * Fix algorithm finding HOB memory resources (#3983) * Refactor interrupt handling (#4083) * Load kernel asynchronously (#4022) * Only create ACPI memory manager DSDT when resizable (#4013) Deprecated features will be removed in a subsequent release and users should plan to use alternatives * The mergeable option from the virtio-pmem support has been deprecated (#3968) * The dax option from the virtio-fs support has been deprecated (#3889) Fixes: #4317 Signed-off-by: Fabiano Fidêncio --- .../cloud-hypervisor/client/api/openapi.yaml | 5 +++ .../client/docs/PlatformConfig.md | 26 ++++++++++++++ .../client/model_platform_config.go | 36 +++++++++++++++++++ .../cloud-hypervisor/cloud-hypervisor.yaml | 2 ++ versions.yaml | 2 +- 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index 43b2e9737a..4687c425e8 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -606,6 +606,7 @@ components: - 3 - 3 num_pci_segments: 3 + serial_number: serial_number pmem: - pci_segment: 6 mergeable: false @@ -948,6 +949,7 @@ components: - 3 - 3 num_pci_segments: 3 + serial_number: serial_number pmem: - pci_segment: 6 mergeable: false @@ -1169,6 +1171,7 @@ components: - 3 - 3 num_pci_segments: 3 + serial_number: serial_number properties: num_pci_segments: format: int16 @@ -1178,6 +1181,8 @@ components: format: int16 type: integer type: array + serial_number: + type: string type: object MemoryZoneConfig: example: diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/PlatformConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/PlatformConfig.md index 91adf0d99c..832444c1fe 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/PlatformConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/PlatformConfig.md @@ -6,6 +6,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **NumPciSegments** | Pointer to **int32** | | [optional] **IommuSegments** | Pointer to **[]int32** | | [optional] +**SerialNumber** | Pointer to **string** | | [optional] ## Methods @@ -76,6 +77,31 @@ SetIommuSegments sets IommuSegments field to given value. HasIommuSegments returns a boolean if a field has been set. +### GetSerialNumber + +`func (o *PlatformConfig) GetSerialNumber() string` + +GetSerialNumber returns the SerialNumber field if non-nil, zero value otherwise. + +### GetSerialNumberOk + +`func (o *PlatformConfig) GetSerialNumberOk() (*string, bool)` + +GetSerialNumberOk returns a tuple with the SerialNumber field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetSerialNumber + +`func (o *PlatformConfig) SetSerialNumber(v string)` + +SetSerialNumber sets SerialNumber field to given value. + +### HasSerialNumber + +`func (o *PlatformConfig) HasSerialNumber() bool` + +HasSerialNumber returns a boolean if a field has been set. + [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_platform_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_platform_config.go index e480c8a911..17e8827f84 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_platform_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_platform_config.go @@ -18,6 +18,7 @@ import ( type PlatformConfig struct { NumPciSegments *int32 `json:"num_pci_segments,omitempty"` IommuSegments *[]int32 `json:"iommu_segments,omitempty"` + SerialNumber *string `json:"serial_number,omitempty"` } // NewPlatformConfig instantiates a new PlatformConfig object @@ -101,6 +102,38 @@ func (o *PlatformConfig) SetIommuSegments(v []int32) { o.IommuSegments = &v } +// GetSerialNumber returns the SerialNumber field value if set, zero value otherwise. +func (o *PlatformConfig) GetSerialNumber() string { + if o == nil || o.SerialNumber == nil { + var ret string + return ret + } + return *o.SerialNumber +} + +// GetSerialNumberOk returns a tuple with the SerialNumber field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *PlatformConfig) GetSerialNumberOk() (*string, bool) { + if o == nil || o.SerialNumber == nil { + return nil, false + } + return o.SerialNumber, true +} + +// HasSerialNumber returns a boolean if a field has been set. +func (o *PlatformConfig) HasSerialNumber() bool { + if o != nil && o.SerialNumber != nil { + return true + } + + return false +} + +// SetSerialNumber gets a reference to the given string and assigns it to the SerialNumber field. +func (o *PlatformConfig) SetSerialNumber(v string) { + o.SerialNumber = &v +} + func (o PlatformConfig) MarshalJSON() ([]byte, error) { toSerialize := map[string]interface{}{} if o.NumPciSegments != nil { @@ -109,6 +142,9 @@ func (o PlatformConfig) MarshalJSON() ([]byte, error) { if o.IommuSegments != nil { toSerialize["iommu_segments"] = o.IommuSegments } + if o.SerialNumber != nil { + toSerialize["serial_number"] = o.SerialNumber + } return json.Marshal(toSerialize) } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 8861f1e710..4c0cff139a 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -616,6 +616,8 @@ components: items: type: integer format: int16 + serial_number: + type: string MemoryZoneConfig: required: diff --git a/versions.yaml b/versions.yaml index 4df02678d5..34723fd857 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "v23.1" + version: "v24.0" firecracker: description: "Firecracker micro-VMM" From d862ca059045dc917c9e8015c1fecf26ebe6c228 Mon Sep 17 00:00:00 2001 From: Manabu Sugimoto Date: Sun, 29 May 2022 16:34:38 +0900 Subject: [PATCH 26/26] runk: Handle rootfs path in config.json properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit enables runk to handle `root.path` in `config.json` properly even if the path is specified by a relative path that includes the single (`.`) or the double (`..`) dots. For example, with a bundle at `/to/bundle` and a rootfs directly under `/to/bundle` such as `/to/bundle/{bin,dev,etc,home,...}`, the `root.path` value can be either `/to/bundle` or just `.`. This behavior conforms to OCI runtime spec. Accordingly, a bundle path managed by runk's status file (`status.json`) always is statically stored as a canonical path. Previously, a bundle path has been　got by `oci_state()` of rustjail's API that returns the path as the parent directory path of a rootfs (`root.path`). In case of the kata-agent, this works properly because the kata containers assume that the rootfs path is always `/to/bundle/rootfs`. However in case of standard OCI runtimes, a rootfs can be placed anywhere under a bundle, so the rootfs path doesn't always have to be at a `/to/bundle/rootfs`. Fixes: #4334 Signed-off-by: Manabu Sugimoto --- src/tools/runk/libcontainer/src/builder.rs | 12 ++++++------ src/tools/runk/libcontainer/src/container.rs | 3 ++- src/tools/runk/libcontainer/src/status.rs | 4 +++- src/tools/runk/libcontainer/src/utils.rs | 4 +++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/tools/runk/libcontainer/src/builder.rs b/src/tools/runk/libcontainer/src/builder.rs index 738c639ae2..28bb730461 100644 --- a/src/tools/runk/libcontainer/src/builder.rs +++ b/src/tools/runk/libcontainer/src/builder.rs @@ -37,20 +37,20 @@ impl Container { // If the rootfs path in the spec file is a relative path, // convert it into a canonical path to pass validation of rootfs in the agent. if !&rootfs_path.is_absolute() { - let rootfs_name = rootfs_path - .file_name() - .ok_or_else(|| anyhow!("invalid rootfs name"))?; spec_root.path = bundle_canon - .join(rootfs_name) + .join(rootfs_path) + .canonicalize()? .to_str() .map(|s| s.to_string()) - .ok_or_else(|| anyhow!("failed to convert bundle path"))?; + .ok_or_else(|| { + anyhow!("failed to convert a rootfs path into a canonical path") + })?; } } Ok(ContainerContext { id: self.id, - bundle: self.bundle, + bundle: bundle_canon, state_root: self.root, spec, // TODO: liboci-cli does not support --no-pivot option for create and run command. diff --git a/src/tools/runk/libcontainer/src/container.rs b/src/tools/runk/libcontainer/src/container.rs index d5464e9239..2d8b423178 100644 --- a/src/tools/runk/libcontainer/src/container.rs +++ b/src/tools/runk/libcontainer/src/container.rs @@ -95,6 +95,7 @@ impl ContainerContext { let oci_state = ctr.oci_state()?; let status = Status::new( &self.state_root, + &self.bundle, oci_state, ctr.init_process_start_time, ctr.created, @@ -141,7 +142,7 @@ mod tests { #[test] fn test_get_fifo_path() { - let test_data = PathBuf::from(TEST_BUNDLE_PATH) + let test_data = PathBuf::from(TEST_STATE_ROOT_PATH) .join(TEST_CONTAINER_ID) .join(EXEC_FIFO_FILENAME); let status = create_dummy_status(); diff --git a/src/tools/runk/libcontainer/src/status.rs b/src/tools/runk/libcontainer/src/status.rs index 21ba00cda4..3cd9768b10 100644 --- a/src/tools/runk/libcontainer/src/status.rs +++ b/src/tools/runk/libcontainer/src/status.rs @@ -42,6 +42,7 @@ pub struct Status { impl Status { pub fn new( root: &Path, + bundle: &Path, oci_state: OCIState, process_start_time: u64, created_time: SystemTime, @@ -64,7 +65,7 @@ impl Status { id: oci_state.id, pid: oci_state.pid, root: root.to_path_buf(), - bundle: PathBuf::from(&oci_state.bundle), + bundle: bundle.to_path_buf(), rootfs, process_start_time, created, @@ -209,6 +210,7 @@ mod tests { let oci_state = create_dummy_oci_state(); let created = SystemTime::now(); let status = Status::new( + Path::new(TEST_STATE_ROOT_PATH), Path::new(TEST_BUNDLE_PATH), oci_state.clone(), 1, diff --git a/src/tools/runk/libcontainer/src/utils.rs b/src/tools/runk/libcontainer/src/utils.rs index 5a356d7c2d..dcd9f7f7f1 100644 --- a/src/tools/runk/libcontainer/src/utils.rs +++ b/src/tools/runk/libcontainer/src/utils.rs @@ -45,7 +45,8 @@ pub(crate) mod test_utils { use std::time::SystemTime; pub const TEST_CONTAINER_ID: &str = "test"; - pub const TEST_BUNDLE_PATH: &str = "/test"; + pub const TEST_STATE_ROOT_PATH: &str = "/state"; + pub const TEST_BUNDLE_PATH: &str = "/bundle"; pub const TEST_ANNOTATION: &str = "test"; pub const TEST_CGM_DATA: &str = r#"{ "paths": { @@ -92,6 +93,7 @@ pub(crate) mod test_utils { let oci_state = create_dummy_oci_state(); let created = SystemTime::now(); let status = Status::new( + Path::new(TEST_STATE_ROOT_PATH), Path::new(TEST_BUNDLE_PATH), oci_state.clone(), 1,