config: Add config flag "experimental"

Fixes #1226

Add new flag "experimental" for supporting underworking features.
Some features are under developing which are not ready for release,
there're also some features which will break compatibility which is not
suitable to be merged into a kata minor release(x version in x.y.z)

For getting these features above merged earlier for more testing, we can
mark them as "experimental" features, and move them to formal features
when they are ready.

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>

Makefile

@@ -157,6 +157,8 @@ DEFMEMSLOTS := 10
 #Default number of bridges
 DEFBRIDGES := 1
 DEFDISABLEGUESTSECCOMP := true
+#Default experimental features enabled
+DEFAULTEXPFEATURES := []
 
 #Default entropy source
 DEFENTROPYSOURCE := /dev/urandom
@@ -314,6 +316,7 @@ USER_VARS += DEFBRIDGES
 USER_VARS += DEFNETWORKMODEL_FC
 USER_VARS += DEFNETWORKMODEL_QEMU
 USER_VARS += DEFDISABLEGUESTSECCOMP
+USER_VARS += DEFAULTEXPFEATURES
 USER_VARS += DEFDISABLEBLOCK
 USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
 USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
@@ -503,6 +506,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION
 		-e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \
 		-e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \
 		-e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \
+		-e "s|@DEFAULTEXPFEATURES@|$(DEFAULTEXPFEATURES)|g" \
 		-e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \
 		-e "s|@DEFBLOCKSTORAGEDRIVER_FC@|$(DEFBLOCKSTORAGEDRIVER_FC)|g" \
 		-e "s|@DEFBLOCKSTORAGEDRIVER_QEMU@|$(DEFBLOCKSTORAGEDRIVER_QEMU)|g" \

cli/config/configuration-fc.toml.in

@@ -289,3 +289,11 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # If you are using docker, `disable_new_netns` only works with `docker run --net=none`
 # (default: false)
 #disable_new_netns = true
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# They may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# 1. "newstore": new persist storage driver
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

cli/config/configuration-qemu.toml.in

@@ -336,3 +336,11 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # If you are using docker, `disable_new_netns` only works with `docker run --net=none`
 # (default: false)
 #disable_new_netns = true
+
+# Enabled experimental feature list, format: ["a", "b"].
+# Experimental features are features not stable enough for production,
+# They may break compatibility, and are prepared for a big version bump.
+# Supported experimental features:
+# 1. "newstore": new persist storage driver
+# (default: [])
+experimental=@DEFAULTEXPFEATURES@

cli/kata-env.go

@@ -16,6 +16,7 @@ import (
 	"github.com/BurntSushi/toml"
 	"github.com/kata-containers/runtime/pkg/katautils"
 	vc "github.com/kata-containers/runtime/virtcontainers"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	vcUtils "github.com/kata-containers/runtime/virtcontainers/utils"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -69,6 +70,7 @@ type RuntimeInfo struct {
 	Trace               bool
 	DisableGuestSeccomp bool
 	DisableNewNetNs     bool
+	Experimental        []exp.Feature
 	Path                string
 }
 
@@ -181,6 +183,7 @@ func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo {
 		Config:              runtimeConfig,
 		Path:                runtimePath,
 		DisableNewNetNs:     config.DisableNewNetNs,
+		Experimental:        config.Experimental,
 		DisableGuestSeccomp: config.DisableGuestSeccomp,
 	}
 }

pkg/katautils/config.go

@@ -16,6 +16,7 @@ import (
 	"github.com/BurntSushi/toml"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
@@ -122,11 +123,12 @@ type proxy struct {
 }
 
 type runtime struct {
-	Debug               bool   `toml:"enable_debug"`
-	Tracing             bool   `toml:"enable_tracing"`
-	DisableNewNetNs     bool   `toml:"disable_new_netns"`
-	DisableGuestSeccomp bool   `toml:"disable_guest_seccomp"`
-	InterNetworkModel   string `toml:"internetworking_model"`
+	Debug               bool     `toml:"enable_debug"`
+	Tracing             bool     `toml:"enable_tracing"`
+	DisableNewNetNs     bool     `toml:"disable_new_netns"`
+	DisableGuestSeccomp bool     `toml:"disable_guest_seccomp"`
+	Experimental        []string `toml:"experimental"`
+	InterNetworkModel   string   `toml:"internetworking_model"`
 }
 
 type shim struct {
@@ -800,32 +802,15 @@ func initConfig() (config oci.RuntimeConfig, err error) {
 // All paths are resolved fully meaning if this function does not return an
 // error, all paths are valid at the time of the call.
 func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
-	var resolved string
 
 	config, err = initConfig()
 	if err != nil {
 		return "", oci.RuntimeConfig{}, err
 	}
 
-	if configPath == "" {
-		resolved, err = getDefaultConfigFile()
-	} else {
-		resolved, err = ResolvePath(configPath)
-	}
-
+	tomlConf, resolved, err := decodeConfig(configPath)
 	if err != nil {
-		return "", config, fmt.Errorf("Cannot find usable config file (%v)", err)
-	}
-
-	configData, err := ioutil.ReadFile(resolved)
-	if err != nil {
-		return "", config, err
-	}
-
-	var tomlConf tomlConfig
-	_, err = toml.Decode(string(configData), &tomlConf)
-	if err != nil {
-		return "", config, err
+		return "", oci.RuntimeConfig{}, err
 	}
 
 	config.Debug = tomlConf.Runtime.Debug
@@ -872,6 +857,13 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	}
 
 	config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
+	for _, f := range tomlConf.Runtime.Experimental {
+		feature := exp.Feature(f)
+		if !exp.Supported(feature) {
+			return "", config, fmt.Errorf("Unsupported experimental feature %q", f)
+		}
+		config.Experimental = append(config.Experimental, feature)
+	}
 
 	if err := checkConfig(config); err != nil {
 		return "", config, err
@@ -880,6 +872,36 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	return resolved, config, nil
 }
 
+func decodeConfig(configPath string) (tomlConfig, string, error) {
+	var (
+		resolved string
+		tomlConf tomlConfig
+		err      error
+	)
+
+	if configPath == "" {
+		resolved, err = getDefaultConfigFile()
+	} else {
+		resolved, err = ResolvePath(configPath)
+	}
+
+	if err != nil {
+		return tomlConf, "", fmt.Errorf("Cannot find usable config file (%v)", err)
+	}
+
+	configData, err := ioutil.ReadFile(resolved)
+	if err != nil {
+		return tomlConf, resolved, err
+	}
+
+	_, err = toml.Decode(string(configData), &tomlConf)
+	if err != nil {
+		return tomlConf, resolved, err
+	}
+
+	return tomlConf, resolved, nil
+}
+
 // checkConfig checks the validity of the specified config.
 func checkConfig(config oci.RuntimeConfig) error {
 	if err := checkNetNsConfig(config); err != nil {

virtcontainers/experimental/experimental.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2019 Huawei Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package experimental
+
+import (
+	"fmt"
+)
+
+// Feature to be experimental
+type Feature string
+
+var (
+	supportedFeatures = make(map[Feature]struct{})
+)
+
+// Register register a new experimental feature
+func Register(feature Feature) error {
+	if _, ok := supportedFeatures[feature]; ok {
+		return fmt.Errorf("Feature %q had been registered before", feature)
+	}
+	supportedFeatures[feature] = struct{}{}
+	return nil
+}
+
+// Supported check if the feature is supported
+func Supported(feature Feature) bool {
+	_, ok := supportedFeatures[feature]
+	return ok
+}

virtcontainers/experimental/experimental_test.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2019 Huawei Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package experimental
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExperimental(t *testing.T) {
+	f := Feature("mock")
+	assert.False(t, Supported(f))
+
+	err := Register("mock")
+	assert.Nil(t, err)
+
+	err = Register("mock")
+	assert.NotNil(t, err)
+	assert.Equal(t, len(supportedFeatures), 1)
+
+	assert.True(t, Supported(f))
+}

virtcontainers/pkg/oci/utils.go

@@ -23,6 +23,7 @@ import (
 
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -134,6 +135,9 @@ type RuntimeConfig struct {
 
 	//Determines if create a netns for hypervisor process
 	DisableNewNetNs bool
+
+	//Experimental features enabled
+	Experimental []exp.Feature
 }
 
 // AddKernelParam allows the addition of new kernel parameters to an existing
@@ -500,6 +504,8 @@ func SandboxConfig(ocispec CompatOCISpec, runtime RuntimeConfig, bundlePath, cid
 		SystemdCgroup: systemdCgroup,
 
 		DisableGuestSeccomp: runtime.DisableGuestSeccomp,
+
+		Experimental: runtime.Experimental,
 	}
 
 	addAssetAnnotations(ocispec, &sandboxConfig)

virtcontainers/sandbox.go

@@ -25,6 +25,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/runtime/virtcontainers/store"
@@ -100,6 +101,9 @@ type SandboxConfig struct {
 	SystemdCgroup bool
 
 	DisableGuestSeccomp bool
+
+	// Experimental features enabled
+	Experimental []exp.Feature
 }
 
 func (s *Sandbox) trace(name string) (opentracing.Span, context.Context) {
@@ -136,6 +140,12 @@ func (sandboxConfig *SandboxConfig) valid() bool {
 		sandboxConfig.HypervisorType = QemuHypervisor
 	}
 
+	// validate experimental features
+	for _, f := range sandboxConfig.Experimental {
+		if !exp.Supported(f) {
+			return false
+		}
+	}
 	return true
 }
 
@@ -439,6 +449,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 		return nil, err
 	}
 
+	if len(s.config.Experimental) != 0 {
+		s.Logger().WithField("features", s.config.Experimental).Infof("Enable experimental features")
+	}
+
 	// Fetch sandbox network to be able to access it from the sandbox structure.
 	var networkNS NetworkNamespace
 	if err := s.store.Load(store.Network, &networkNS); err == nil {

virtcontainers/sandbox_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	"github.com/kata-containers/runtime/virtcontainers/device/manager"
+	exp "github.com/kata-containers/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -1680,3 +1681,18 @@ func TestSandboxUpdateResources(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestSandboxExperimentalFeature(t *testing.T) {
+	testFeature := exp.Feature("mock")
+	sconfig := SandboxConfig{
+		ID:           testSandboxID,
+		Experimental: []exp.Feature{testFeature},
+	}
+
+	assert.False(t, exp.Supported(testFeature))
+	assert.False(t, sconfig.valid())
+
+	exp.Register(testFeature)
+	assert.True(t, exp.Supported(testFeature))
+	assert.True(t, sconfig.valid())
+}