diff --git a/docs/how-to/how-to-set-sandbox-config-kata.md b/docs/how-to/how-to-set-sandbox-config-kata.md index 776a7e02f3..14ba8ed898 100644 --- a/docs/how-to/how-to-set-sandbox-config-kata.md +++ b/docs/how-to/how-to-set-sandbox-config-kata.md @@ -28,6 +28,7 @@ There are several kinds of Kata configurations and they are listed below. | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup | | `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process | | `io.katacontainers.config.runtime.create_container_timeout` | `uint64` | the timeout for create a container in `seconds`, default is `60` | +| `io.katacontainers.config.runtime.experimental_force_guest_pull` | `boolean` | forces the runtime to pull the image in the guest VM, default is `false`. This is an experimental feature and might be removed in the future. | ## Agent Options | Key | Value Type | Comments | diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 896d99fd80..909f7a050c 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -175,6 +175,7 @@ DEFMSIZE9P := 8192 DEFVFIOMODE := guest-kernel DEFBINDMOUNTS := [] DEFDANCONF := /run/kata-containers/dans +DEFFORCEGUESTPULL := false SED = sed CLI_DIR = cmd SHIMV2 = containerd-shim-kata-v2 @@ -503,6 +504,7 @@ USER_VARS += KATA_INSTALL_GROUP USER_VARS += KATA_INSTALL_OWNER USER_VARS += KATA_INSTALL_CFG_PERMS USER_VARS += DEFDANCONF +USER_VARS += DEFFORCEGUESTPULL SOURCES := \ $(shell find . 2>&1 | grep -E '.*\.rs$$') \ diff --git a/src/runtime/Makefile b/src/runtime/Makefile index a3a39183bb..e73967c23f 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -280,6 +280,8 @@ DEFCREATECONTAINERTIMEOUT ?= 60 # Default directory of directly attachable network config. DEFDANCONF := /run/kata-containers/dans +DEFFORCEGUESTPULL := false + SED = sed CLI_DIR = cmd @@ -753,6 +755,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_TEE USER_VARS += DEFBINDMOUNTS USER_VARS += DEFCREATECONTAINERTIMEOUT USER_VARS += DEFDANCONF +USER_VARS += DEFFORCEGUESTPULL USER_VARS += DEFVFIOMODE USER_VARS += DEFVFIOMODE_SE USER_VARS += BUILDFLAGS diff --git a/src/runtime/config/configuration-qemu-coco-dev.toml.in b/src/runtime/config/configuration-qemu-coco-dev.toml.in index fdd3ddae2f..2220af8b93 100644 --- a/src/runtime/config/configuration-qemu-coco-dev.toml.in +++ b/src/runtime/config/configuration-qemu-coco-dev.toml.in @@ -713,3 +713,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in index c020e2d7a3..f3c2f5982c 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in @@ -698,3 +698,8 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in index d403fea935..28f37d70cf 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in @@ -682,3 +682,8 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/config/configuration-qemu-se.toml.in b/src/runtime/config/configuration-qemu-se.toml.in index bf2b79ab3d..a1993f47ea 100644 --- a/src/runtime/config/configuration-qemu-se.toml.in +++ b/src/runtime/config/configuration-qemu-se.toml.in @@ -673,3 +673,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in index 97198e4cdc..0804a3ff2b 100644 --- a/src/runtime/config/configuration-qemu-sev.toml.in +++ b/src/runtime/config/configuration-qemu-sev.toml.in @@ -639,3 +639,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index 9427d26d01..3750a5b116 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -691,3 +691,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index 5ec9c628dc..adbad98ef8 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -676,3 +676,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that +# the container image should be pulled in the guest, without using an external snapshotter. +# This is an experimental feature and might be removed in the future. +experimental_force_guest_pull = @DEFFORCEGUESTPULL@ diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 0c1e211c57..064e590a96 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -193,6 +193,7 @@ type runtime struct { DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` CreateContainerTimeout uint64 `toml:"create_container_timeout"` DanConf string `toml:"dan_conf"` + ForceGuestPull bool `toml:"experimental_force_guest_pull"` } type agent struct { @@ -1587,6 +1588,8 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat return "", config, err } + config.ForceGuestPull = tomlConf.Runtime.ForceGuestPull + return resolved, config, nil } diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index 372255a0ec..d49aabd988 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -171,6 +171,9 @@ type RuntimeConfig struct { // Base directory of directly attachable network config DanConfig string + + // ForceGuestPull enforces guest pull independent of snapshotter annotations. + ForceGuestPull bool } // AddKernelParam allows the addition of new kernel parameters to an existing @@ -1000,6 +1003,12 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r return err } + if err := newAnnotationConfiguration(ocispec, vcAnnotations.ForceGuestPull).setBool(func(forceGuestPull bool) { + sbConfig.ForceGuestPull = forceGuestPull + }); err != nil { + return err + } + if err := newAnnotationConfiguration(ocispec, vcAnnotations.EnableVCPUsPinning).setBool(func(enableVCPUsPinning bool) { sbConfig.EnableVCPUsPinning = enableVCPUsPinning }); err != nil { @@ -1145,6 +1154,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st Experimental: runtime.Experimental, CreateContainerTimeout: runtime.CreateContainerTimeout, + + ForceGuestPull: runtime.ForceGuestPull, } if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil { diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index b6d327a158..3599e618c2 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -93,7 +93,7 @@ type FilesystemShare struct { prepared bool } -func NewFilesystemShare(s *Sandbox) (FilesystemSharer, error) { +func NewFilesystemShare(s *Sandbox) (*FilesystemShare, error) { watcher, err := fsnotify.NewWatcher() if err != nil { return nil, fmt.Errorf("Creating watcher returned error %w", err) @@ -594,8 +594,29 @@ func (f *FilesystemShare) shareRootFilesystemWithErofs(ctx context.Context, c *C }, nil } +func forceGuestPull(c *Container) (*SharedFile, error) { + sf := &SharedFile{ + guestPath: filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix), + } + guestPullVolume := &types.KataVirtualVolume{ + VolumeType: types.KataVirtualVolumeImageGuestPullType, + ImagePull: &types.ImagePullVolume{ + Metadata: map[string]string{}, + }, + } + vol, err := handleVirtualVolumeStorageObject(c, "", guestPullVolume) + if err != nil { + return nil, fmt.Errorf("forcing guest pull virtual volume: %w", err) + } + sf.containerStorages = append(sf.containerStorages, vol) + return sf, nil +} + // func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) { func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) { + if f.sandbox.IsGuestPullForced() { + return forceGuestPull(c) + } if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) { return f.shareRootFilesystemWithVirtualVolume(ctx, c) diff --git a/src/runtime/virtcontainers/fs_share_linux_test.go b/src/runtime/virtcontainers/fs_share_linux_test.go index 6b740d1e6e..a6ea52d8ee 100644 --- a/src/runtime/virtcontainers/fs_share_linux_test.go +++ b/src/runtime/virtcontainers/fs_share_linux_test.go @@ -14,6 +14,8 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" ) @@ -96,3 +98,87 @@ func TestSandboxSharedFilesystem(t *testing.T) { err = sandbox.fsShare.Cleanup(sandbox.ctx) assert.NoError(err) } + +func TestShareRootFilesystem(t *testing.T) { + requireNewFilesystemShare := func(sandbox *Sandbox) *FilesystemShare { + fsShare, err := NewFilesystemShare(sandbox) + assert.NoError(t, err) + return fsShare + } + + testCases := map[string]struct { + fsSharer *FilesystemShare + container *Container + wantErr bool + wantSharedFile *SharedFile + }{ + "force guest pull successful": { + fsSharer: requireNewFilesystemShare(&Sandbox{ + config: &SandboxConfig{ + ForceGuestPull: true, + }, + }), + container: &Container{ + id: "container-id-abc", + rootfsSuffix: "test-suffix", + config: &ContainerConfig{ + Annotations: map[string]string{ + "io.kubernetes.cri.image-name": "test-image-name", + }, + CustomSpec: &specs.Spec{ + Annotations: map[string]string{ + "io.kubernetes.cri.container-type": "", + }, + }, + }, + }, + wantSharedFile: &SharedFile{ + containerStorages: []*grpc.Storage{{ + Fstype: "overlay", + Source: "test-image-name", + MountPoint: "/run/kata-containers/container-id-abc/test-suffix", + Driver: "image_guest_pull", + DriverOptions: []string{ + "image_guest_pull={\"metadata\":{\"io.kubernetes.cri.image-name\":\"test-image-name\"}}", + }, + }}, + guestPath: "/run/kata-containers/container-id-abc/test-suffix", + }, + }, + "force guest pull image name missing": { + fsSharer: requireNewFilesystemShare(&Sandbox{ + config: &SandboxConfig{ + ForceGuestPull: true, + }, + }), + container: &Container{ + id: "container-id-abc", + rootfsSuffix: "test-suffix", + config: &ContainerConfig{ + Annotations: map[string]string{}, + CustomSpec: &specs.Spec{ + Annotations: map[string]string{ + "io.kubernetes.cri.container-type": "", + }, + }, + }, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + sharedFile, err := tc.fsSharer.ShareRootFilesystem(context.Background(), tc.container) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + + assert.Equal(tc.wantSharedFile, sharedFile) + }) + } +} diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 353daabdec..03b9e9b70c 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -283,6 +283,9 @@ const ( // CreateContainerTimeout is a sandbox annotaion that sets the create container timeout. CreateContainerTimeout = kataAnnotRuntimePrefix + "create_container_timeout" + + // ForceGuestPull is a sandbox annotation that sets experimental_force_guest_pull. + ForceGuestPull = kataAnnotRuntimePrefix + "experimental_force_guest_pull" ) // Agent related annotations diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 972ba39433..049ab084a2 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -186,6 +186,9 @@ type SandboxConfig struct { // Create container timeout which, if provided, indicates the create container timeout // needed for the workload(s) CreateContainerTimeout uint64 + + // ForceGuestPull enforces guest pull independent of snapshotter annotations. + ForceGuestPull bool } // valid checks that the sandbox configuration is valid. @@ -448,6 +451,14 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re return c.ioStream(processID) } +// IsGuestPullEnforced returns true if guest pull is forced through the sandbox configuration. +func (s *Sandbox) IsGuestPullForced() bool { + if s.config == nil { + return false + } + return s.config.ForceGuestPull +} + func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error { span, _ := katatrace.Trace(ctx, nil, "createAssets", sandboxTracingTags, map[string]string{"sandbox_id": sandboxConfig.ID}) defer span.End()