Merge pull request #40149 from luxas/followup_rbac

Automatic merge from submit-queue (batch tested with PRs 39772, 39831, 39481, 40167, 40149)

Only enable RBAC if the k8s version is above 1.6.0-alpha.0

...and move the defaulting of the api type into a specific file



**What this PR does / why we need it**: RBAC should only be enabled when k8s version is v1.6.0-alpha.0+, where https://github.com/kubernetes/kubernetes/pull/39625 is merged

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

```
$ sudo ./kubeadm init
[kubeadm] WARNING: kubeadm is in alpha, please do not use it for production clusters.
[init] Using Kubernetes version: v1.5.2
[init] Using Authorization mode: AlwaysAllow
[init] A token has not been provided, generating one
[preflight] Running pre-flight checks
[preflight] WARNING: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[preflight] Starting the kubelet service
[certificates] Generated Certificate Authority key and certificate.
[certificates] Generated API Server key and certificate
[certificates] Generated Service Account signing keys
[certificates] Created keys and certificates in "/etc/kubernetes/pki"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[apiclient] Created API client, waiting for the control plane to become ready
[apiclient] All control plane components are healthy after 39.517396 seconds
[apiclient] Waiting for at least one node to register and become ready
[apiclient] First node is ready after 6.521587 seconds
[apiclient] Creating a test deployment
[apiclient] Test deployment succeeded
[token-discovery] Using token: 6c3430:2889e46ce3f11610
[token-discovery] Created the kube-discovery deployment, waiting for it to become ready
[token-discovery] kube-discovery is ready after 3.502025 seconds
[addons] Created essential addon: kube-proxy
[addons] Created essential addon: kube-dns

Your Kubernetes master has initialized successfully!

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
    http://kubernetes.io/docs/admin/addons/

You can now join any number of machines by running the following on each node:

kubeadm join --discovery token://6c3430:2889e46ce3f11610@192.168.200.101:9898
```

```
$ cat fetch-ci-images.sh
#!/bin/bash

CI_COMMIT=${CI_COMMIT:-latest}
ARCH=${ARCH:-amd64}
OLD_REGISTRY=${OLD_REGISTRY:-gcr.io/google_containers}
NEW_REGISTRY=${NEW_REGISTRY:-${OLD_REGISTRY}}

if [[ ${CI_COMMIT} == "latest" ]]; then
	CI_COMMIT=$(curl -sSL https://storage.googleapis.com/kubernetes-release-dev/ci-cross/latest.txt)
fi

CI_COMMIT_TAG=$(echo ${CI_COMMIT} | sed "s/+/-/")

echo "Using commit: ${CI_COMMIT}"
echo "Tagging to registry: ${NEW_REGISTRY}"

images=(
	kube-apiserver
	kube-scheduler
	kube-controller-manager
	kube-proxy
)

for image in ${images[@]}; do
	curl -sSL https://storage.googleapis.com/kubernetes-release-dev/ci-cross/${CI_COMMIT}/bin/linux/${ARCH}/${image}.tar | docker load
	image_tag=$(curl -sSL https://storage.googleapis.com/kubernetes-release-dev/ci-cross/${CI_COMMIT}/bin/linux/${ARCH}/${image}.md5)
	docker tag ${OLD_REGISTRY}/${image}:${image_tag} ${NEW_REGISTRY}/${image}-${ARCH}:${CI_COMMIT_TAG}
	docker rmi ${OLD_REGISTRY}/${image}:${image_tag}
	echo "Image created: ${NEW_REGISTRY}/${image}-${ARCH}:${CI_COMMIT_TAG}"
done

cat <<EOF
Now you can run:
	kubeadm init --use-kubernetes-version ${CI_COMMIT_TAG}
EOF
$ ./fetch-ci-images.sh
Using commit: v1.6.0-alpha.0.3549+5e4625cad72e5b
Tagging to registry: gcr.io/google_containers
c366cffde3c9: Loading layer 1.312 MB/1.312 MB
1911ea24d99d: Loading layer 1.024 kB/1.024 kB
3af1ee6d3b81: Loading layer 132.9 MB/132.9 MB
Untagged: gcr.io/google_containers/kube-apiserver:8562727e1eabe97a95bc2e52992db195
Image created: gcr.io/google_containers/kube-apiserver-amd64:v1.6.0-alpha.0.3549-5e4625cad72e5b
c366cffde3c9: Loading layer 1.312 MB/1.312 MB
1911ea24d99d: Loading layer 1.024 kB/1.024 kB
86bf938b66e2: Loading layer 54.88 MB/54.88 MB
Untagged: gcr.io/google_containers/kube-scheduler:5319acbc670b5050f119e6f6e3c9b7e3
Image created: gcr.io/google_containers/kube-scheduler-amd64:v1.6.0-alpha.0.3549-5e4625cad72e5b
c366cffde3c9: Loading layer 1.312 MB/1.312 MB
1911ea24d99d: Loading layer 1.024 kB/1.024 kB
6743a986e1ed: Loading layer 104.9 MB/104.9 MB
Untagged: gcr.io/google_containers/kube-controller-manager:6698cef6338d549160d58fd8381f3d07
Image created: gcr.io/google_containers/kube-controller-manager-amd64:v1.6.0-alpha.0.3549-5e4625cad72e5b
bf028f50c346: Loading layer 128.9 MB/128.9 MB
a99e621f3f37: Loading layer 1.024 kB/1.024 kB
76d4fdcd51a9: Loading layer 5.076 MB/5.076 MB
ee455b365369: Loading layer 42.52 MB/42.52 MB
Untagged: gcr.io/google_containers/kube-proxy:3914060ffe3225ae29fe88cb960826f9
Image created: gcr.io/google_containers/kube-proxy-amd64:v1.6.0-alpha.0.3549-5e4625cad72e5b
Now you can run:
	kubeadm init --use-kubernetes-version v1.6.0-alpha.0.3549-5e4625cad72e5b
```

```
$ sudo ./kubeadm init --use-kubernetes-version v1.6.0-alpha.0.3549-5e4625cad72e5b
[kubeadm] WARNING: kubeadm is in alpha, please do not use it for production clusters.
[init] Using Kubernetes version: v1.6.0-alpha.0.3549-5e4625cad72e5b
[init] Using Authorization mode: RBAC
[init] A token has not been provided, generating one
[preflight] Running pre-flight checks
[preflight] WARNING: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[preflight] Starting the kubelet service
[certificates] Generated Certificate Authority key and certificate.
[certificates] Generated API Server key and certificate
[certificates] Generated Service Account signing keys
[certificates] Created keys and certificates in "/etc/kubernetes/pki"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[apiclient] Created API client, waiting for the control plane to become ready
[apiclient] All control plane components are healthy after 31.252686 seconds
[apiclient] Waiting for at least one node to register and become ready
[apiclient] First node is ready after 9.064842 seconds
[apiclient] Creating a test deployment
[apiclient] Test deployment succeeded
[apiconfig] Created kubelet-bootstrap RBAC rules
[apiconfig] Created kube-dns RBAC rules
[apiconfig] Created kube-proxy RBAC rules
[token-discovery] Using token: 99b957:ccb1fbc703a0db8a
[token-discovery] Created the kube-discovery deployment, waiting for it to become ready
[token-discovery] kube-discovery is ready after 3.502064 seconds
[addons] Created essential addon: kube-proxy
[addons] Created essential addon: kube-dns

Your Kubernetes master has initialized successfully!

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
    http://kubernetes.io/docs/admin/addons/

You can now join any number of machines by running the following on each node:

kubeadm join --discovery token://99b957:ccb1fbc703a0db8a@192.168.200.101:9898
```

**Release note**:

```release-note
NONE
```
@pires @mikedanese @andrewrynhard

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -19,13 +19,15 @@ package v1alpha1
 import "k8s.io/apimachinery/pkg/runtime"
 
 const (
-	DefaultServiceDNSDomain          = "cluster.local"
+	DefaultServiceDNSDomain  = "cluster.local"
-	DefaultServicesSubnet            = "10.96.0.0/12"
+	DefaultServicesSubnet    = "10.96.0.0/12"
-	DefaultKubernetesVersion         = "stable"
+	DefaultKubernetesVersion = "stable"
-	DefaultKubernetesFallbackVersion = "v1.5.0"
+	// This is only for clusters without internet, were the latest stable version can't be determined
+	DefaultKubernetesFallbackVersion = "v1.5.2"
 	DefaultAPIBindPort               = 6443
 	DefaultDiscoveryBindPort         = 9898
-	DefaultAuthorizationMode         = "RBAC"
+	// TODO: Default this to RBAC when DefaultKubernetesFallbackVersion is v1.6-something
+	DefaultAuthorizationMode = "AlwaysAllow"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {

cmd/kubeadm/app/cmd/BUILD

@@ -12,6 +12,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "cmd.go",
+        "defaults.go",
         "init.go",
         "join.go",
         "reset.go",
@@ -39,6 +40,7 @@ go_library(
         "//pkg/util/flag:go_default_library",
         "//pkg/util/initsystem:go_default_library",
         "//pkg/version:go_default_library",
+        "//vendor:github.com/blang/semver",
         "//vendor:github.com/renstrom/dedent",
         "//vendor:github.com/spf13/cobra",
         "//vendor:k8s.io/apimachinery/pkg/fields",

cmd/kubeadm/app/cmd/defaults.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"fmt"
+	"strconv"
+
+	netutil "k8s.io/apimachinery/pkg/util/net"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+
+	"github.com/blang/semver"
+)
+
+var (
+	// Maximum version when using AllowAll as the default authz mode. Everything above this will use RBAC by default.
+	allowAllMaxVersion = semver.MustParse("1.6.0-alpha.0")
+)
+
+func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
+	// Auto-detect the IP
+	if len(cfg.API.AdvertiseAddresses) == 0 {
+		ip, err := netutil.ChooseHostInterface()
+		if err != nil {
+			return err
+		}
+		cfg.API.AdvertiseAddresses = []string{ip.String()}
+	}
+
+	// Validate version argument
+	ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
+	if err != nil {
+		if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion {
+			return err
+		} else {
+			ver = kubeadmapiext.DefaultKubernetesFallbackVersion
+		}
+	}
+	cfg.KubernetesVersion = ver
+	fmt.Println("[init] Using Kubernetes version:", ver)
+
+	// Omit the "v" in the beginning, otherwise semver will fail
+	// If the version is newer than the specified version, RBAC v1beta1 support is enabled in the apiserver so we can default to RBAC
+	k8sVersion, err := semver.Parse(cfg.KubernetesVersion[1:])
+	if k8sVersion.GT(allowAllMaxVersion) {
+		cfg.AuthorizationMode = "RBAC"
+	}
+
+	fmt.Println("[init] Using Authorization mode:", cfg.AuthorizationMode)
+
+	// Warn about the limitations with the current cloudprovider solution.
+	if cfg.CloudProvider != "" {
+		fmt.Println("[init] WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
+		fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
+	}
+
+	// Validate token if any, otherwise generate
+	if cfg.Discovery.Token != nil {
+		if cfg.Discovery.Token.ID != "" && cfg.Discovery.Token.Secret != "" {
+			fmt.Printf("[init] A token has been provided, validating [%s]\n", kubeadmutil.BearerToken(cfg.Discovery.Token))
+			if valid, err := kubeadmutil.ValidateToken(cfg.Discovery.Token); valid == false {
+				return err
+			}
+		} else {
+			fmt.Println("[init] A token has not been provided, generating one")
+			if err := kubeadmutil.GenerateToken(cfg.Discovery.Token); err != nil {
+				return err
+			}
+		}
+
+		// If there aren't any addresses specified, default to the first advertised address which can be user-provided or the default network interface's IP address
+		if len(cfg.Discovery.Token.Addresses) == 0 {
+			cfg.Discovery.Token.Addresses = []string{cfg.API.AdvertiseAddresses[0] + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)}
+		}
+	}
+
+	return nil
+}

cmd/kubeadm/app/cmd/init.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"io/ioutil"
 	"path"
-	"strconv"
 
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
@@ -38,7 +37,7 @@ import (
 	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 
 	"k8s.io/apimachinery/pkg/runtime"
-	netutil "k8s.io/apimachinery/pkg/util/net"
+
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/api"
@@ -145,13 +144,10 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight
 		}
 	}
 
-	// Auto-detect the IP
+	// Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.)
-	if len(cfg.API.AdvertiseAddresses) == 0 {
+	err := setInitDynamicDefaults(cfg)
-		ip, err := netutil.ChooseHostInterface()
+	if err != nil {
-		if err != nil {
+		return nil, err
-			return nil, err
-		}
-		cfg.API.AdvertiseAddresses = []string{ip.String()}
 	}
 
 	if !skipPreFlight {
@@ -173,25 +169,6 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight
 	// Try to start the kubelet service in case it's inactive
 	preflight.TryStartKubelet()
 
-	// validate version argument
-	ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion)
-	if err != nil {
-		if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion {
-			return nil, err
-		} else {
-			ver = kubeadmapiext.DefaultKubernetesFallbackVersion
-		}
-	}
-	cfg.KubernetesVersion = ver
-	fmt.Println("[init] Using Kubernetes version:", ver)
-	fmt.Println("[init] Using Authorization mode:", cfg.AuthorizationMode)
-
-	// Warn about the limitations with the current cloudprovider solution.
-	if cfg.CloudProvider != "" {
-		fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
-		fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
-	}
-
 	return &Init{cfg: cfg}, nil
 }
 
@@ -202,34 +179,6 @@ func (i *Init) Validate() error {
 // Run executes master node provisioning, including certificates, needed static pod manifests, etc.
 func (i *Init) Run(out io.Writer) error {
 
-	// Validate token if any, otherwise generate
-	if i.cfg.Discovery.Token != nil {
-		if i.cfg.Discovery.Token.ID != "" && i.cfg.Discovery.Token.Secret != "" {
-			fmt.Printf("[token-discovery] A token has been provided, validating [%s]\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token))
-			if valid, err := kubeadmutil.ValidateToken(i.cfg.Discovery.Token); valid == false {
-				return err
-			}
-		} else {
-			fmt.Println("[token-discovery] A token has not been provided, generating one")
-			if err := kubeadmutil.GenerateToken(i.cfg.Discovery.Token); err != nil {
-				return err
-			}
-		}
-
-		// Make sure there is at least one address
-		if len(i.cfg.Discovery.Token.Addresses) == 0 {
-			ip, err := netutil.ChooseHostInterface()
-			if err != nil {
-				return err
-			}
-			i.cfg.Discovery.Token.Addresses = []string{ip.String() + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)}
-		}
-
-		if err := kubemaster.CreateTokenAuthFile(kubeadmutil.BearerToken(i.cfg.Discovery.Token)); err != nil {
-			return err
-		}
-	}
-
 	// PHASE 1: Generate certificates
 	caCert, err := certphase.CreatePKIAssets(i.cfg, kubeadmapi.GlobalEnvParams.HostPKIPath)
 	if err != nil {
@@ -247,6 +196,14 @@ func (i *Init) Run(out io.Writer) error {
 		return err
 	}
 
+	// TODO: It's not great to have an exception for token here, but necessary because the apiserver doesn't handle this properly in the API yet
+	// but relies on files on disk for now, which is daunting.
+	if i.cfg.Discovery.Token != nil {
+		if err := kubemaster.CreateTokenAuthFile(kubeadmutil.BearerToken(i.cfg.Discovery.Token)); err != nil {
+			return err
+		}
+	}
+
 	// Phase 3: Bootstrap the control plane
 	if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil {
 		return err