Merge pull request #11244 from katexochen/p/guest-pull-config

runtime: add option to force guest pull

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -28,6 +28,7 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
 | `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
 | `io.katacontainers.config.runtime.create_container_timeout` | `uint64` | the timeout for create a container in `seconds`, default is `60` |
+| `io.katacontainers.config.runtime.experimental_force_guest_pull` | `boolean` | forces the runtime to pull the image in the guest VM, default is `false`. This is an experimental feature and might be removed in the future. |
 
 ## Agent Options
 | Key | Value Type | Comments |

src/runtime-rs/Makefile

@@ -175,6 +175,7 @@ DEFMSIZE9P := 8192
 DEFVFIOMODE := guest-kernel
 DEFBINDMOUNTS := []
 DEFDANCONF := /run/kata-containers/dans
+DEFFORCEGUESTPULL := false
 SED = sed
 CLI_DIR = cmd
 SHIMV2 = containerd-shim-kata-v2
@@ -503,6 +504,7 @@ USER_VARS += KATA_INSTALL_GROUP
 USER_VARS += KATA_INSTALL_OWNER
 USER_VARS += KATA_INSTALL_CFG_PERMS
 USER_VARS += DEFDANCONF
+USER_VARS += DEFFORCEGUESTPULL
 
 SOURCES := \
   $(shell find . 2>&1 | grep -E '.*\.rs$$') \

src/runtime/Makefile

@@ -280,6 +280,8 @@ DEFCREATECONTAINERTIMEOUT ?= 60
 # Default directory of directly attachable network config.
 DEFDANCONF := /run/kata-containers/dans
 
+DEFFORCEGUESTPULL := false
+
 SED = sed
 
 CLI_DIR = cmd
@@ -753,6 +755,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_TEE
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFCREATECONTAINERTIMEOUT
 USER_VARS += DEFDANCONF
+USER_VARS += DEFFORCEGUESTPULL
 USER_VARS += DEFVFIOMODE
 USER_VARS += DEFVFIOMODE_SE
 USER_VARS += BUILDFLAGS

src/runtime/config/configuration-qemu-coco-dev.toml.in

@@ -713,3 +713,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/config/configuration-qemu-nvidia-gpu-snp.toml.in

@@ -698,3 +698,8 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/config/configuration-qemu-nvidia-gpu-tdx.toml.in

@@ -682,3 +682,8 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/config/configuration-qemu-se.toml.in

@@ -673,3 +673,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/config/configuration-qemu-sev.toml.in

@@ -639,3 +639,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/config/configuration-qemu-snp.toml.in

@@ -691,3 +691,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/config/configuration-qemu-tdx.toml.in

@@ -676,3 +676,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
 # to the hypervisor.
 # (default: /run/kata-containers/dans)
 dan_conf = "@DEFDANCONF@"
+
+# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
+# the container image should be pulled in the guest, without using an external snapshotter.
+# This is an experimental feature and might be removed in the future.
+experimental_force_guest_pull = @DEFFORCEGUESTPULL@

src/runtime/pkg/katautils/config.go

@@ -193,6 +193,7 @@ type runtime struct {
 	DisableGuestEmptyDir      bool     `toml:"disable_guest_empty_dir"`
 	CreateContainerTimeout    uint64   `toml:"create_container_timeout"`
 	DanConf                   string   `toml:"dan_conf"`
+	ForceGuestPull            bool     `toml:"experimental_force_guest_pull"`
 }
 
 type agent struct {
@@ -1587,6 +1588,8 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		return "", config, err
 	}
 
+	config.ForceGuestPull = tomlConf.Runtime.ForceGuestPull
+
 	return resolved, config, nil
 }
 

src/runtime/pkg/oci/utils.go

@@ -171,6 +171,9 @@ type RuntimeConfig struct {
 
 	// Base directory of directly attachable network config
 	DanConfig string
+
+	// ForceGuestPull enforces guest pull independent of snapshotter annotations.
+	ForceGuestPull bool
 }
 
 // AddKernelParam allows the addition of new kernel parameters to an existing
@@ -1000,6 +1003,12 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r
 		return err
 	}
 
+	if err := newAnnotationConfiguration(ocispec, vcAnnotations.ForceGuestPull).setBool(func(forceGuestPull bool) {
+		sbConfig.ForceGuestPull = forceGuestPull
+	}); err != nil {
+		return err
+	}
+
 	if err := newAnnotationConfiguration(ocispec, vcAnnotations.EnableVCPUsPinning).setBool(func(enableVCPUsPinning bool) {
 		sbConfig.EnableVCPUsPinning = enableVCPUsPinning
 	}); err != nil {
@@ -1145,6 +1154,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
 		Experimental: runtime.Experimental,
 
 		CreateContainerTimeout: runtime.CreateContainerTimeout,
+
+		ForceGuestPull: runtime.ForceGuestPull,
 	}
 
 	if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {

src/runtime/virtcontainers/fs_share_linux.go

@@ -93,7 +93,7 @@ type FilesystemShare struct {
 	prepared bool
 }
 
-func NewFilesystemShare(s *Sandbox) (FilesystemSharer, error) {
+func NewFilesystemShare(s *Sandbox) (*FilesystemShare, error) {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		return nil, fmt.Errorf("Creating watcher returned error %w", err)
@@ -594,8 +594,29 @@ func (f *FilesystemShare) shareRootFilesystemWithErofs(ctx context.Context, c *C
 	}, nil
 }
 
+func forceGuestPull(c *Container) (*SharedFile, error) {
+	sf := &SharedFile{
+		guestPath: filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix),
+	}
+	guestPullVolume := &types.KataVirtualVolume{
+		VolumeType: types.KataVirtualVolumeImageGuestPullType,
+		ImagePull: &types.ImagePullVolume{
+			Metadata: map[string]string{},
+		},
+	}
+	vol, err := handleVirtualVolumeStorageObject(c, "", guestPullVolume)
+	if err != nil {
+		return nil, fmt.Errorf("forcing guest pull virtual volume: %w", err)
+	}
+	sf.containerStorages = append(sf.containerStorages, vol)
+	return sf, nil
+}
+
 // func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) {
 func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) {
+	if f.sandbox.IsGuestPullForced() {
+		return forceGuestPull(c)
+	}
 
 	if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) {
 		return f.shareRootFilesystemWithVirtualVolume(ctx, c)

src/runtime/virtcontainers/fs_share_linux_test.go

@@ -14,6 +14,8 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -96,3 +98,87 @@ func TestSandboxSharedFilesystem(t *testing.T) {
 	err = sandbox.fsShare.Cleanup(sandbox.ctx)
 	assert.NoError(err)
 }
+
+func TestShareRootFilesystem(t *testing.T) {
+	requireNewFilesystemShare := func(sandbox *Sandbox) *FilesystemShare {
+		fsShare, err := NewFilesystemShare(sandbox)
+		assert.NoError(t, err)
+		return fsShare
+	}
+
+	testCases := map[string]struct {
+		fsSharer       *FilesystemShare
+		container      *Container
+		wantErr        bool
+		wantSharedFile *SharedFile
+	}{
+		"force guest pull successful": {
+			fsSharer: requireNewFilesystemShare(&Sandbox{
+				config: &SandboxConfig{
+					ForceGuestPull: true,
+				},
+			}),
+			container: &Container{
+				id:           "container-id-abc",
+				rootfsSuffix: "test-suffix",
+				config: &ContainerConfig{
+					Annotations: map[string]string{
+						"io.kubernetes.cri.image-name": "test-image-name",
+					},
+					CustomSpec: &specs.Spec{
+						Annotations: map[string]string{
+							"io.kubernetes.cri.container-type": "",
+						},
+					},
+				},
+			},
+			wantSharedFile: &SharedFile{
+				containerStorages: []*grpc.Storage{{
+					Fstype:     "overlay",
+					Source:     "test-image-name",
+					MountPoint: "/run/kata-containers/container-id-abc/test-suffix",
+					Driver:     "image_guest_pull",
+					DriverOptions: []string{
+						"image_guest_pull={\"metadata\":{\"io.kubernetes.cri.image-name\":\"test-image-name\"}}",
+					},
+				}},
+				guestPath: "/run/kata-containers/container-id-abc/test-suffix",
+			},
+		},
+		"force guest pull image name missing": {
+			fsSharer: requireNewFilesystemShare(&Sandbox{
+				config: &SandboxConfig{
+					ForceGuestPull: true,
+				},
+			}),
+			container: &Container{
+				id:           "container-id-abc",
+				rootfsSuffix: "test-suffix",
+				config: &ContainerConfig{
+					Annotations: map[string]string{},
+					CustomSpec: &specs.Spec{
+						Annotations: map[string]string{
+							"io.kubernetes.cri.container-type": "",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			sharedFile, err := tc.fsSharer.ShareRootFilesystem(context.Background(), tc.container)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			assert.NoError(err)
+
+			assert.Equal(tc.wantSharedFile, sharedFile)
+		})
+	}
+}

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -283,6 +283,9 @@ const (
 
 	// CreateContainerTimeout is a sandbox annotaion that sets the create container timeout.
 	CreateContainerTimeout = kataAnnotRuntimePrefix + "create_container_timeout"
+
+	// ForceGuestPull is a sandbox annotation that sets experimental_force_guest_pull.
+	ForceGuestPull = kataAnnotRuntimePrefix + "experimental_force_guest_pull"
 )
 
 // Agent related annotations

src/runtime/virtcontainers/sandbox.go

@@ -186,6 +186,9 @@ type SandboxConfig struct {
 	// Create container timeout which, if provided, indicates the create container timeout
 	// needed for the workload(s)
 	CreateContainerTimeout uint64
+
+	// ForceGuestPull enforces guest pull independent of snapshotter annotations.
+	ForceGuestPull bool
 }
 
 // valid checks that the sandbox configuration is valid.
@@ -448,6 +451,14 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
 	return c.ioStream(processID)
 }
 
+// IsGuestPullEnforced returns true if guest pull is forced through the sandbox configuration.
+func (s *Sandbox) IsGuestPullForced() bool {
+	if s.config == nil {
+		return false
+	}
+	return s.config.ForceGuestPull
+}
+
 func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
 	span, _ := katatrace.Trace(ctx, nil, "createAssets", sandboxTracingTags, map[string]string{"sandbox_id": sandboxConfig.ID})
 	defer span.End()