runtime: add option to force guest pull

This enables guest pull via config, without the need of any external
snapshotter. When the config enables runtime.experimental_force_guest_pull, instead of
relying on annotations to select the way to share the root FS, we always
use guest pull.

Co-authored-by: Markus Rudy <mr@edgeless.systems>
Signed-off-by: Paul Meyer <katexochen0@gmail.com>

src/runtime/virtcontainers/fs_share_linux.go

@@ -93,7 +93,7 @@ type FilesystemShare struct {
 	prepared bool
 }
 
-func NewFilesystemShare(s *Sandbox) (FilesystemSharer, error) {
+func NewFilesystemShare(s *Sandbox) (*FilesystemShare, error) {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		return nil, fmt.Errorf("Creating watcher returned error %w", err)
@@ -594,8 +594,29 @@ func (f *FilesystemShare) shareRootFilesystemWithErofs(ctx context.Context, c *C
 	}, nil
 }
 
+func forceGuestPull(c *Container) (*SharedFile, error) {
+	sf := &SharedFile{
+		guestPath: filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix),
+	}
+	guestPullVolume := &types.KataVirtualVolume{
+		VolumeType: types.KataVirtualVolumeImageGuestPullType,
+		ImagePull: &types.ImagePullVolume{
+			Metadata: map[string]string{},
+		},
+	}
+	vol, err := handleVirtualVolumeStorageObject(c, "", guestPullVolume)
+	if err != nil {
+		return nil, fmt.Errorf("forcing guest pull virtual volume: %w", err)
+	}
+	sf.containerStorages = append(sf.containerStorages, vol)
+	return sf, nil
+}
+
 // func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) {
 func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) {
+	if f.sandbox.IsGuestPullForced() {
+		return forceGuestPull(c)
+	}
 
 	if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) {
 		return f.shareRootFilesystemWithVirtualVolume(ctx, c)

src/runtime/virtcontainers/fs_share_linux_test.go

@@ -14,6 +14,8 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -96,3 +98,87 @@ func TestSandboxSharedFilesystem(t *testing.T) {
 	err = sandbox.fsShare.Cleanup(sandbox.ctx)
 	assert.NoError(err)
 }
+
+func TestShareRootFilesystem(t *testing.T) {
+	requireNewFilesystemShare := func(sandbox *Sandbox) *FilesystemShare {
+		fsShare, err := NewFilesystemShare(sandbox)
+		assert.NoError(t, err)
+		return fsShare
+	}
+
+	testCases := map[string]struct {
+		fsSharer       *FilesystemShare
+		container      *Container
+		wantErr        bool
+		wantSharedFile *SharedFile
+	}{
+		"force guest pull successful": {
+			fsSharer: requireNewFilesystemShare(&Sandbox{
+				config: &SandboxConfig{
+					ForceGuestPull: true,
+				},
+			}),
+			container: &Container{
+				id:           "container-id-abc",
+				rootfsSuffix: "test-suffix",
+				config: &ContainerConfig{
+					Annotations: map[string]string{
+						"io.kubernetes.cri.image-name": "test-image-name",
+					},
+					CustomSpec: &specs.Spec{
+						Annotations: map[string]string{
+							"io.kubernetes.cri.container-type": "",
+						},
+					},
+				},
+			},
+			wantSharedFile: &SharedFile{
+				containerStorages: []*grpc.Storage{{
+					Fstype:     "overlay",
+					Source:     "test-image-name",
+					MountPoint: "/run/kata-containers/container-id-abc/test-suffix",
+					Driver:     "image_guest_pull",
+					DriverOptions: []string{
+						"image_guest_pull={\"metadata\":{\"io.kubernetes.cri.image-name\":\"test-image-name\"}}",
+					},
+				}},
+				guestPath: "/run/kata-containers/container-id-abc/test-suffix",
+			},
+		},
+		"force guest pull image name missing": {
+			fsSharer: requireNewFilesystemShare(&Sandbox{
+				config: &SandboxConfig{
+					ForceGuestPull: true,
+				},
+			}),
+			container: &Container{
+				id:           "container-id-abc",
+				rootfsSuffix: "test-suffix",
+				config: &ContainerConfig{
+					Annotations: map[string]string{},
+					CustomSpec: &specs.Spec{
+						Annotations: map[string]string{
+							"io.kubernetes.cri.container-type": "",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			sharedFile, err := tc.fsSharer.ShareRootFilesystem(context.Background(), tc.container)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			assert.NoError(err)
+
+			assert.Equal(tc.wantSharedFile, sharedFile)
+		})
+	}
+}

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -283,6 +283,9 @@ const (
 
 	// CreateContainerTimeout is a sandbox annotaion that sets the create container timeout.
 	CreateContainerTimeout = kataAnnotRuntimePrefix + "create_container_timeout"
+
+	// ForceGuestPull is a sandbox annotation that sets experimental_force_guest_pull.
+	ForceGuestPull = kataAnnotRuntimePrefix + "experimental_force_guest_pull"
 )
 
 // Agent related annotations

src/runtime/virtcontainers/sandbox.go

@@ -186,6 +186,9 @@ type SandboxConfig struct {
 	// Create container timeout which, if provided, indicates the create container timeout
 	// needed for the workload(s)
 	CreateContainerTimeout uint64
+
+	// ForceGuestPull enforces guest pull independent of snapshotter annotations.
+	ForceGuestPull bool
 }
 
 // valid checks that the sandbox configuration is valid.
@@ -448,6 +451,14 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
 	return c.ioStream(processID)
 }
 
+// IsGuestPullEnforced returns true if guest pull is forced through the sandbox configuration.
+func (s *Sandbox) IsGuestPullForced() bool {
+	if s.config == nil {
+		return false
+	}
+	return s.config.ForceGuestPull
+}
+
 func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
 	span, _ := katatrace.Trace(ctx, nil, "createAssets", sandboxTracingTags, map[string]string{"sandbox_id": sandboxConfig.ID})
 	defer span.End()