shimv2: support runtime config path via annotation

So that user can have per pod runtime config which is easiler
to manage in some cases.

Fixes: #2332
Signed-off-by: Peng Tao <bergwolf@hyper.sh>

containerd-shim-v2/create.go

@@ -59,7 +59,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
 		}
 
-		_, err := loadRuntimeConfig(s, r)
+		_, err := loadRuntimeConfig(s, r, ociSpec.Annotations)
 		if err != nil {
 			return nil, err
 		}
@@ -147,10 +147,13 @@ func loadSpec(r *taskAPI.CreateTaskRequest) (*specs.Spec, string, error) {
 	return &ociSpec, bundlePath, nil
 }
 
-func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeConfig, error) {
-	var configPath string
-
-	if r.Options != nil {
+// Config override ordering(high to low):
+// 1. podsandbox annotation
+// 2. shimv2 create task option
+// 3. environment
+func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string]string) (*oci.RuntimeConfig, error) {
+	configPath := oci.GetSandboxConfigPath(anno)
+	if configPath == "" && r.Options != nil {
 		v, err := typeurl.UnmarshalAny(r.Options)
 		if err != nil {
 			return nil, err

containerd-shim-v2/create_test.go

@@ -8,19 +8,24 @@ package containerdshim
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 
 	"github.com/containerd/containerd/namespaces"
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+	crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1"
+	"github.com/containerd/typeurl"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 
 	ktu "github.com/kata-containers/runtime/pkg/katatestutils"
 	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
+	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
 )
@@ -369,3 +374,123 @@ func TestCreateContainerConfigFail(t *testing.T) {
 	_, err = s.Create(ctx, req)
 	assert.Error(err)
 }
+
+func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) {
+	if dir == "" {
+		return "", fmt.Errorf("BUG: need directory")
+	}
+
+	if hypervisor == "" {
+		return "", fmt.Errorf("BUG: need hypervisor")
+	}
+
+	hypervisorPath := path.Join(dir, "hypervisor")
+	kernelPath := path.Join(dir, "kernel")
+	kernelParams := "foo=bar xyz"
+	imagePath := path.Join(dir, "image")
+	shimPath := path.Join(dir, "shim")
+	proxyPath := path.Join(dir, "proxy")
+	netmonPath := path.Join(dir, "netmon")
+	logDir := path.Join(dir, "logs")
+	logPath := path.Join(logDir, "runtime.log")
+	machineType := "machineType"
+	disableBlockDevice := true
+	blockDeviceDriver := "virtio-scsi"
+	enableIOThreads := true
+	hotplugVFIOOnRootBus := true
+	disableNewNetNs := false
+	sharedFS := "virtio-9p"
+
+	configFileOptions := ktu.RuntimeConfigOptions{
+		Hypervisor:           "qemu",
+		HypervisorPath:       hypervisorPath,
+		KernelPath:           kernelPath,
+		ImagePath:            imagePath,
+		KernelParams:         kernelParams,
+		MachineType:          machineType,
+		ShimPath:             shimPath,
+		ProxyPath:            proxyPath,
+		NetmonPath:           netmonPath,
+		LogPath:              logPath,
+		DisableBlock:         disableBlockDevice,
+		BlockDeviceDriver:    blockDeviceDriver,
+		EnableIOThreads:      enableIOThreads,
+		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		DisableNewNetNs:      disableNewNetNs,
+		SharedFS:             sharedFS,
+	}
+
+	runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions)
+
+	configPath := path.Join(dir, "runtime.toml")
+	err = ioutil.WriteFile(configPath, []byte(runtimeConfigFileData), os.FileMode(0640))
+	if err != nil {
+		return "", err
+	}
+
+	files := []string{hypervisorPath, kernelPath, imagePath, shimPath, proxyPath}
+
+	for _, file := range files {
+		// create the resource (which must be >0 bytes)
+		err := ioutil.WriteFile(file, []byte("foo"), os.FileMode(0640))
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return configPath, nil
+}
+
+func TestCreateLoadRuntimeConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	config, err := createAllRuntimeConfigFiles(tmpdir, "qemu")
+	assert.NoError(err)
+
+	s := &service{
+		id:  testSandboxID,
+		ctx: context.Background(),
+	}
+	r := &taskAPI.CreateTaskRequest{}
+	anno := make(map[string]string)
+
+	// set all to fake path
+	fakeConfig := "foobar"
+	anno[vcAnnotations.SandboxConfigPathKey] = fakeConfig
+	option := &crioption.Options{ConfigPath: fakeConfig}
+	r.Options, err = typeurl.MarshalAny(option)
+	assert.NoError(err)
+	err = os.Setenv("KATA_CONF_FILE", fakeConfig)
+	assert.NoError(err)
+	defer os.Setenv("KATA_CONF_FILE", "")
+
+	// fake config should fail
+	_, err = loadRuntimeConfig(s, r, anno)
+	assert.Error(err)
+
+	// 1. podsandbox annotation
+	anno[vcAnnotations.SandboxConfigPathKey] = config
+	_, err = loadRuntimeConfig(s, r, anno)
+	assert.NoError(err)
+	anno[vcAnnotations.SandboxConfigPathKey] = ""
+
+	// 2. shimv2 create task option
+	option.ConfigPath = config
+	r.Options, err = typeurl.MarshalAny(option)
+	assert.NoError(err)
+	_, err = loadRuntimeConfig(s, r, anno)
+	assert.NoError(err)
+	option.ConfigPath = ""
+	r.Options, err = typeurl.MarshalAny(option)
+	assert.NoError(err)
+
+	// 3. environment
+	err = os.Setenv("KATA_CONF_FILE", config)
+	assert.NoError(err)
+	_, err = loadRuntimeConfig(s, r, anno)
+	assert.NoError(err)
+}

virtcontainers/pkg/annotations/annotations.go

@@ -19,6 +19,8 @@ const (
 
 	// ContainerTypeKey is the annotation key to fetch container type.
 	ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
+
+	SandboxConfigPathKey = kataAnnotationsPrefix + "config_path"
 )
 
 // Annotations related to Hypervisor configuration

virtcontainers/pkg/oci/utils.go

@@ -309,6 +309,10 @@ func ContainerType(spec specs.Spec) (vc.ContainerType, error) {
 	return vc.PodSandbox, nil
 }
 
+func GetSandboxConfigPath(annotations map[string]string) string {
+	return annotations[vcAnnotations.SandboxConfigPathKey]
+}
+
 // SandboxID determines the sandbox ID related to an OCI configuration. This function
 // is expected to be called only when the container type is "PodContainer".
 func SandboxID(spec specs.Spec) (string, error) {