Merge pull request #48943 from jamiehannaford/kubeadm-feature-gates

Automatic merge from submit-queue

Add --feature-gate flags to kubeadm

**What this PR does / why we need it**:

Adds `--feature-gates` in similar manner to other `cmd` binaries

**Which issue this PR fixes** 

https://github.com/kubernetes/kubeadm/issues/323

**Special notes for your reviewer**:

This results in a lot of probably unnecessary feature flags. I'm guessing a lot of kubeadm users will be confused when they see:

```
Flags:
      --feature-gates mapStringBool   A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
APIResponseCompression=true|false (ALPHA - default=false)
Accelerators=true|false (ALPHA - default=false)
AdvancedAuditing=true|false (ALPHA - default=false)
AllAlpha=true|false (ALPHA - default=false)
AllowExtTrafficLocalEndpoints=true|false (default=true)
AppArmor=true|false (BETA - default=true)
DebugContainers=true|false (ALPHA - default=false)
DynamicKubeletConfig=true|false (ALPHA - default=false)
DynamicVolumeProvisioning=true|false (ALPHA - default=true)
ExperimentalCriticalPodAnnotation=true|false (ALPHA - default=false)
ExperimentalHostUserNamespaceDefaulting=true|false (BETA - default=false)
LocalStorageCapacityIsolation=true|false (ALPHA - default=false)
PersistentLocalVolumes=true|false (ALPHA - default=false)
PodPriority=true|false (ALPHA - default=false)
RotateKubeletClientCertificate=true|false (ALPHA - default=false)
RotateKubeletServerCertificate=true|false (ALPHA - default=false)
StreamingProxyRedirects=true|false (BETA - default=true)
TaintBasedEvictions=true|false (ALPHA - default=false)
  -h, --help                          help for kubeadm
```

However the feature flags used in the core pkg is global, so I don't think it can be overriden. So we have a few options:

1. Allow these flags for kubeadm
2. Refactor feature pkg to allow granular features
3. Roll our own feature gating for kubeadm

/cc @luxas

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -41,6 +41,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.Etcd.DataDir = "foo"
 			obj.ImageRepository = "foo"
 			obj.UnifiedControlPlaneImage = "foo"
+			obj.FeatureFlags = map[string]bool{}
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -56,6 +56,9 @@ type MasterConfiguration struct {
 	ImageRepository string
 	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
 	UnifiedControlPlaneImage string
+
+	// FeatureFlags enabled by the user
+	FeatureFlags map[string]bool
 }
 
 type API struct {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -56,6 +56,9 @@ type MasterConfiguration struct {
 	ImageRepository string `json:"imageRepository"`
 	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
 	UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"`
+
+	// FeatureFlags enabled by the user
+	FeatureFlags map[string]bool `json:"featureFlags"`
 }
 
 type API struct {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go

@@ -140,6 +140,13 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.FeatureFlags != nil {
+		in, out := &in.FeatureFlags, &out.FeatureFlags
+		*out = make(map[string]bool, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 

cmd/kubeadm/app/apis/kubeadm/validation/BUILD

@@ -26,6 +26,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/features:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util/token:go_default_library",
         "//pkg/api/validation:go_default_library",

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	apivalidation "k8s.io/kubernetes/pkg/api/validation"
@@ -66,6 +67,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificates-dir"))...)
 	allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...)
 	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
+	allErrs = append(allErrs, ValidateFeatureFlags(c.FeatureFlags, field.NewPath("feature-flags"))...)
 	return allErrs
 }
 
@@ -281,3 +283,18 @@ func ValidateMixedArguments(flag *pflag.FlagSet) error {
 	}
 	return nil
 }
+
+func ValidateFeatureFlags(featureFlags map[string]bool, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	validFeatures := features.Keys(features.InitFeatureGates)
+
+	// check valid feature names are provided
+	for k := range featureFlags {
+		if !features.Supports(features.InitFeatureGates, k) {
+			allErrs = append(allErrs, field.Invalid(fldPath, featureFlags,
+				fmt.Sprintf("%s is not a valid feature name. Valid features are: %s", k, validFeatures)))
+		}
+	}
+
+	return allErrs
+}

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -323,3 +323,27 @@ func TestValidateMixedArguments(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateFeatureFlags(t *testing.T) {
+	type featureFlag map[string]bool
+	var tests = []struct {
+		featureFlags featureFlag
+		expected     bool
+	}{
+		{featureFlag{"SelfHosting": true}, true},
+		{featureFlag{"SelfHosting": false}, true},
+		{featureFlag{"StoreCertsInSecrets": true}, true},
+		{featureFlag{"StoreCertsInSecrets": false}, true},
+		{featureFlag{"Foo": true}, false},
+	}
+	for _, rt := range tests {
+		actual := ValidateFeatureFlags(rt.featureFlags, nil)
+		if (len(actual) == 0) != rt.expected {
+			t.Errorf(
+				"failed featureFlags:\n\texpected: %t\n\t  actual: %t",
+				rt.expected,
+				(len(actual) == 0),
+			)
+		}
+	}
+}

cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go

@@ -145,6 +145,13 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.FeatureFlags != nil {
+		in, out := &in.FeatureFlags, &out.FeatureFlags
+		*out = make(map[string]bool, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 

cmd/kubeadm/app/cmd/BUILD

@@ -87,6 +87,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//cmd/kubeadm/app/cmd/features:all-srcs",
         "//cmd/kubeadm/app/cmd/phases:all-srcs",
     ],
     tags = ["automanaged"],

cmd/kubeadm/app/cmd/features/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["features.go"],
+    tags = ["automanaged"],
+    deps = ["//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

cmd/kubeadm/app/cmd/features/features.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+)
+
+const (
+	// SelfHosting is beta in v1.8
+	SelfHosting utilfeature.Feature = "SelfHosting"
+
+	// StoreCertsInSecrets is alpha in v1.8
+	StoreCertsInSecrets utilfeature.Feature = "StoreCertsInSecrets"
+)
+
+// FeatureList represents a list of feature gates
+type FeatureList map[utilfeature.Feature]utilfeature.FeatureSpec
+
+// Supports indicates whether a feature name is supported on the given
+// feature set
+func Supports(featureList FeatureList, featureName string) bool {
+	for k := range featureList {
+		if featureName == string(k) {
+			return true
+		}
+	}
+	return false
+}
+
+// Keys returns a slice of feature names for a given feature set
+func Keys(featureList FeatureList) []string {
+	var list []string
+	for k := range featureList {
+		list = append(list, string(k))
+	}
+	return list
+}
+
+// InitFeatureGates are the default feature gates for the init command
+var InitFeatureGates = FeatureList{
+	SelfHosting:         {Default: false, PreRelease: utilfeature.Beta},
+	StoreCertsInSecrets: {Default: false, PreRelease: utilfeature.Alpha},
+}