Merge pull request #40292 from luxas/kubeadm_node_ca

Automatic merge from submit-queue (batch tested with PRs 38445, 40292)

kubeadm: Secure apiserver -> kubelet communication and set storage backend to etcd3

**What this PR does / why we need it**:

Switch storage backend to etcd3
Writes ca.crt down to disk so we can set `--client-ca-file=/etc/kubernetes/ca.crt` for the kubelet.
Adds --kubelet-client-{certificate,key} to the apiserver args and make it allowed to access the kubelets
In some cases the `path` dependency is switched to `filepath`

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*:

fixes https://github.com/kubernetes/kubeadm/issues/118
fixes https://github.com/kubernetes/kubeadm/issues/129

**Special notes for your reviewer**:

This PR is should make it possible to secure the apiserver -> kubelet communication.

**Release note**:

```release-note
NONE
```

@pires @mikedanese @andrewrynhard @liggitt @deads2k

cmd/kubeadm/app/cmd/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
         "//cmd/kubeadm/app/cmd/flags:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/discovery:go_default_library",
         "//cmd/kubeadm/app/master:go_default_library",
         "//cmd/kubeadm/app/node:go_default_library",
@@ -46,6 +47,7 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/fields",
         "//vendor:k8s.io/apimachinery/pkg/runtime",
         "//vendor:k8s.io/apimachinery/pkg/util/net",
+        "//vendor:k8s.io/client-go/pkg/util/cert",
     ],
 )
 

cmd/kubeadm/app/cmd/init.go

@@ -168,12 +168,6 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight
 	// Try to start the kubelet service in case it's inactive
 	preflight.TryStartKubelet()
 
-	// Warn about the limitations with the current cloudprovider solution.
-	if cfg.CloudProvider != "" {
-		fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
-		fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
-	}
-
 	return &Init{cfg: cfg, selfHosted: selfHosted}, nil
 }
 

cmd/kubeadm/app/cmd/join.go

@@ -20,15 +20,17 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"path"
+	"path/filepath"
 
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	certutil "k8s.io/client-go/pkg/util/cert"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
 	kubenode "k8s.io/kubernetes/cmd/kubeadm/app/node"
 	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
@@ -136,10 +138,20 @@ func (j *Join) Run(out io.Writer) error {
 	if err := kubenode.PerformTLSBootstrap(cfg); err != nil {
 		return err
 	}
-	if err := kubeconfigphase.WriteKubeconfigToDisk(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfigphase.KubeletKubeConfigFileName), cfg); err != nil {
+
+	kubeconfigFile := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfigphase.KubeletKubeConfigFileName)
+	if err := kubeconfigphase.WriteKubeconfigToDisk(kubeconfigFile, cfg); err != nil {
 		return err
 	}
 
+	// Write the ca certificate to disk so kubelet can use it for authentication
+	cluster := cfg.Contexts[cfg.CurrentContext].Cluster
+	caCertFile := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.CACertName)
+	err = certutil.WriteCert(caCertFile, cfg.Clusters[cluster].CertificateAuthorityData)
+	if err != nil {
+		return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
+	}
+
 	fmt.Fprintf(out, joinDoneMsgf)
 	return nil
 }

cmd/kubeadm/app/cmd/reset.go

@@ -21,12 +21,13 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -123,7 +124,7 @@ func (r *Reset) Run(out io.Writer) error {
 
 	// Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user
 	// provided external etcd endpoints. In that case, it is his own responsibility to reset etcd
-	etcdManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests/etcd.json")
+	etcdManifestPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests/etcd.json")
 	if _, err := os.Stat(etcdManifestPath); err == nil {
 		dirsToClean = append(dirsToClean, "/var/lib/etcd")
 	} else {
@@ -151,7 +152,7 @@ func drainAndRemoveNode(removeNode bool) error {
 	hostname = strings.ToLower(hostname)
 
 	// TODO: Use the "native" k8s client for this once we're confident the versioned is working
-	kubeConfigPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)
+	kubeConfigPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)
 
 	getNodesCmd := fmt.Sprintf("kubectl --kubeconfig %s get nodes | grep %s", kubeConfigPath, hostname)
 	output, err := exec.Command("sh", "-c", getNodesCmd).Output()
@@ -180,14 +181,14 @@ func drainAndRemoveNode(removeNode bool) error {
 }
 
 // cleanDir removes everything in a directory, but not the directory itself
-func cleanDir(filepath string) error {
+func cleanDir(filePath string) error {
 	// If the directory doesn't even exist there's nothing to do, and we do
 	// not consider this an error
-	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		return nil
 	}
 
-	d, err := os.Open(filepath)
+	d, err := os.Open(filePath)
 	if err != nil {
 		return err
 	}
@@ -197,7 +198,7 @@ func cleanDir(filepath string) error {
 		return err
 	}
 	for _, name := range names {
-		err = os.RemoveAll(path.Join(filepath, name))
+		err = os.RemoveAll(filepath.Join(filePath, name))
 		if err != nil {
 			return err
 		}
@@ -208,7 +209,7 @@ func cleanDir(filepath string) error {
 // resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/.
 func resetConfigDir(configPathDir, pkiPathDir string) {
 	dirsToClean := []string{
-		path.Join(configPathDir, "manifests"),
+		filepath.Join(configPathDir, "manifests"),
 		pkiPathDir,
 	}
 	fmt.Printf("[reset] Deleting contents of config directories: %v\n", dirsToClean)
@@ -220,8 +221,9 @@ func resetConfigDir(configPathDir, pkiPathDir string) {
 	}
 
 	filesToClean := []string{
-		path.Join(configPathDir, kubeconfig.AdminKubeConfigFileName),
+		filepath.Join(configPathDir, kubeconfig.AdminKubeConfigFileName),
-		path.Join(configPathDir, kubeconfig.KubeletKubeConfigFileName),
+		filepath.Join(configPathDir, kubeconfig.KubeletKubeConfigFileName),
+		filepath.Join(configPathDir, kubeadmconstants.CACertName),
 	}
 	fmt.Printf("[reset] Deleting files: %v\n", filesToClean)
 	for _, path := range filesToClean {

cmd/kubeadm/app/master/manifests.go

@@ -322,9 +322,12 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 		"--client-ca-file="+getCertFilePath(kubeadmconstants.CACertName),
 		"--tls-cert-file="+getCertFilePath(kubeadmconstants.APIServerCertName),
 		"--tls-private-key-file="+getCertFilePath(kubeadmconstants.APIServerKeyName),
+		"--kubelet-client-certificate="+getCertFilePath(kubeadmconstants.APIServerCertName),
+		"--kubelet-client-key="+getCertFilePath(kubeadmconstants.APIServerKeyName),
 		"--token-auth-file="+kubeadmapi.GlobalEnvParams.HostPKIPath+"/tokens.csv",
 		fmt.Sprintf("--secure-port=%d", cfg.API.Port),
 		"--allow-privileged",
+		"--storage-backend=etcd3",
 	)
 
 	if cfg.AuthorizationMode != "" {

cmd/kubeadm/app/master/manifests_test.go

@@ -376,9 +376,12 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
 				"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
+				"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
+				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
 				"--allow-privileged",
+				"--storage-backend=etcd3",
 				"--etcd-servers=http://127.0.0.1:2379",
 			},
 		},
@@ -396,9 +399,12 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
 				"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
+				"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
+				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
 				"--allow-privileged",
+				"--storage-backend=etcd3",
 				"--advertise-address=foo",
 				"--etcd-servers=http://127.0.0.1:2379",
 			},
@@ -418,9 +424,12 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
 				"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
+				"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
+				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
 				"--allow-privileged",
+				"--storage-backend=etcd3",
 				"--etcd-servers=http://127.0.0.1:2379",
 				"--etcd-certfile=fiz",
 				"--etcd-keyfile=faz",
@@ -442,9 +451,12 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
 				"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
+				"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
+				"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
 				"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
 				fmt.Sprintf("--secure-port=%d", 123),
 				"--allow-privileged",
+				"--storage-backend=etcd3",
 				"--advertise-address=foo",
 				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
 				"--anonymous-auth=false",

cmd/kubeadm/app/phases/certs/certs.go

@@ -123,7 +123,9 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration, pkiDir string) error {
 		config := certutil.Config{
 			CommonName: "kube-apiserver",
 			AltNames:   altNames,
-			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+			// This makes the apiserver allowed to talk to the kubelets in the cluster
+			Organization: []string{"system:masters"},
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		}
 		apiCert, apiKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
 		if err != nil {

cmd/kubeadm/app/preflight/BUILD

@@ -14,6 +14,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
         "//pkg/api/validation:go_default_library",
         "//pkg/util/initsystem:go_default_library",

cmd/kubeadm/app/preflight/checks.go

@@ -25,10 +25,11 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path"
+	"path/filepath"
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	"k8s.io/kubernetes/pkg/api/validation"
 	"k8s.io/kubernetes/pkg/util/initsystem"
@@ -213,7 +214,7 @@ func (fcc FileContentCheck) Check() (warnings, errors []error) {
 
 }
 
-// InPathCheck checks if the given executable is present in the path.
+// InPathCheck checks if the given executable is present in the path
 type InPathCheck struct {
 	executable string
 	mandatory  bool
@@ -318,7 +319,7 @@ func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error {
 		PortOpenCheck{port: 10251},
 		PortOpenCheck{port: 10252},
 		HTTPProxyCheck{Proto: "https", Host: cfg.API.AdvertiseAddresses[0], Port: int(cfg.API.Port)},
-		DirAvailableCheck{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
+		DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
 		DirAvailableCheck{Path: "/var/lib/kubelet"},
 		FileContentCheck{Path: bridgenf, Content: []byte{'1'}},
 		InPathCheck{executable: "ip", mandatory: true},
@@ -351,9 +352,10 @@ func RunJoinNodeChecks(cfg *kubeadmapi.NodeConfiguration) error {
 		ServiceCheck{Service: "kubelet", CheckIfActive: false},
 		ServiceCheck{Service: "docker", CheckIfActive: true},
 		PortOpenCheck{port: 10250},
-		DirAvailableCheck{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
+		DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
 		DirAvailableCheck{Path: "/var/lib/kubelet"},
-		FileAvailableCheck{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)},
+		FileAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.CACertName)},
+		FileAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)},
 		FileContentCheck{Path: bridgenf, Content: []byte{'1'}},
 		InPathCheck{executable: "ip", mandatory: true},
 		InPathCheck{executable: "iptables", mandatory: true},