mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-01 07:47:56 +00:00
Merge pull request #40292 from luxas/kubeadm_node_ca
Automatic merge from submit-queue (batch tested with PRs 38445, 40292)
kubeadm: Secure apiserver -> kubelet communication and set storage backend to etcd3
**What this PR does / why we need it**:
Switch storage backend to etcd3
Writes ca.crt down to disk so we can set `--client-ca-file=/etc/kubernetes/ca.crt` for the kubelet.
Adds --kubelet-client-{certificate,key} to the apiserver args and make it allowed to access the kubelets
In some cases the `path` dependency is switched to `filepath`
**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*:
fixes https://github.com/kubernetes/kubeadm/issues/118
fixes https://github.com/kubernetes/kubeadm/issues/129
**Special notes for your reviewer**:
This PR is should make it possible to secure the apiserver -> kubelet communication.
**Release note**:
```release-note
NONE
```
@pires @mikedanese @andrewrynhard @liggitt @deads2k
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
071844e35f
@ -25,6 +25,7 @@ go_library(
|
||||
"//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
|
||||
"//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
|
||||
"//cmd/kubeadm/app/cmd/flags:go_default_library",
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/discovery:go_default_library",
|
||||
"//cmd/kubeadm/app/master:go_default_library",
|
||||
"//cmd/kubeadm/app/node:go_default_library",
|
||||
@ -46,6 +47,7 @@ go_library(
|
||||
"//vendor:k8s.io/apimachinery/pkg/fields",
|
||||
"//vendor:k8s.io/apimachinery/pkg/runtime",
|
||||
"//vendor:k8s.io/apimachinery/pkg/util/net",
|
||||
"//vendor:k8s.io/client-go/pkg/util/cert",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -168,12 +168,6 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight
|
||||
// Try to start the kubelet service in case it's inactive
|
||||
preflight.TryStartKubelet()
|
||||
|
||||
// Warn about the limitations with the current cloudprovider solution.
|
||||
if cfg.CloudProvider != "" {
|
||||
fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
|
||||
fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
|
||||
}
|
||||
|
||||
return &Init{cfg: cfg, selfHosted: selfHosted}, nil
|
||||
}
|
||||
|
||||
|
||||
@ -20,15 +20,17 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/renstrom/dedent"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
certutil "k8s.io/client-go/pkg/util/cert"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
|
||||
kubenode "k8s.io/kubernetes/cmd/kubeadm/app/node"
|
||||
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
|
||||
@ -136,10 +138,20 @@ func (j *Join) Run(out io.Writer) error {
|
||||
if err := kubenode.PerformTLSBootstrap(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kubeconfigphase.WriteKubeconfigToDisk(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfigphase.KubeletKubeConfigFileName), cfg); err != nil {
|
||||
|
||||
kubeconfigFile := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfigphase.KubeletKubeConfigFileName)
|
||||
if err := kubeconfigphase.WriteKubeconfigToDisk(kubeconfigFile, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write the ca certificate to disk so kubelet can use it for authentication
|
||||
cluster := cfg.Contexts[cfg.CurrentContext].Cluster
|
||||
caCertFile := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.CACertName)
|
||||
err = certutil.WriteCert(caCertFile, cfg.Clusters[cluster].CertificateAuthorityData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(out, joinDoneMsgf)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -21,12 +21,13 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
@ -123,7 +124,7 @@ func (r *Reset) Run(out io.Writer) error {
|
||||
|
||||
// Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user
|
||||
// provided external etcd endpoints. In that case, it is his own responsibility to reset etcd
|
||||
etcdManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests/etcd.json")
|
||||
etcdManifestPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests/etcd.json")
|
||||
if _, err := os.Stat(etcdManifestPath); err == nil {
|
||||
dirsToClean = append(dirsToClean, "/var/lib/etcd")
|
||||
} else {
|
||||
@ -151,7 +152,7 @@ func drainAndRemoveNode(removeNode bool) error {
|
||||
hostname = strings.ToLower(hostname)
|
||||
|
||||
// TODO: Use the "native" k8s client for this once we're confident the versioned is working
|
||||
kubeConfigPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)
|
||||
kubeConfigPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)
|
||||
|
||||
getNodesCmd := fmt.Sprintf("kubectl --kubeconfig %s get nodes | grep %s", kubeConfigPath, hostname)
|
||||
output, err := exec.Command("sh", "-c", getNodesCmd).Output()
|
||||
@ -180,14 +181,14 @@ func drainAndRemoveNode(removeNode bool) error {
|
||||
}
|
||||
|
||||
// cleanDir removes everything in a directory, but not the directory itself
|
||||
func cleanDir(filepath string) error {
|
||||
func cleanDir(filePath string) error {
|
||||
// If the directory doesn't even exist there's nothing to do, and we do
|
||||
// not consider this an error
|
||||
if _, err := os.Stat(filepath); os.IsNotExist(err) {
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
d, err := os.Open(filepath)
|
||||
d, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -197,7 +198,7 @@ func cleanDir(filepath string) error {
|
||||
return err
|
||||
}
|
||||
for _, name := range names {
|
||||
err = os.RemoveAll(path.Join(filepath, name))
|
||||
err = os.RemoveAll(filepath.Join(filePath, name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -208,7 +209,7 @@ func cleanDir(filepath string) error {
|
||||
// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/.
|
||||
func resetConfigDir(configPathDir, pkiPathDir string) {
|
||||
dirsToClean := []string{
|
||||
path.Join(configPathDir, "manifests"),
|
||||
filepath.Join(configPathDir, "manifests"),
|
||||
pkiPathDir,
|
||||
}
|
||||
fmt.Printf("[reset] Deleting contents of config directories: %v\n", dirsToClean)
|
||||
@ -220,8 +221,9 @@ func resetConfigDir(configPathDir, pkiPathDir string) {
|
||||
}
|
||||
|
||||
filesToClean := []string{
|
||||
path.Join(configPathDir, kubeconfig.AdminKubeConfigFileName),
|
||||
path.Join(configPathDir, kubeconfig.KubeletKubeConfigFileName),
|
||||
filepath.Join(configPathDir, kubeconfig.AdminKubeConfigFileName),
|
||||
filepath.Join(configPathDir, kubeconfig.KubeletKubeConfigFileName),
|
||||
filepath.Join(configPathDir, kubeadmconstants.CACertName),
|
||||
}
|
||||
fmt.Printf("[reset] Deleting files: %v\n", filesToClean)
|
||||
for _, path := range filesToClean {
|
||||
|
||||
@ -322,9 +322,12 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
|
||||
"--client-ca-file="+getCertFilePath(kubeadmconstants.CACertName),
|
||||
"--tls-cert-file="+getCertFilePath(kubeadmconstants.APIServerCertName),
|
||||
"--tls-private-key-file="+getCertFilePath(kubeadmconstants.APIServerKeyName),
|
||||
"--kubelet-client-certificate="+getCertFilePath(kubeadmconstants.APIServerCertName),
|
||||
"--kubelet-client-key="+getCertFilePath(kubeadmconstants.APIServerKeyName),
|
||||
"--token-auth-file="+kubeadmapi.GlobalEnvParams.HostPKIPath+"/tokens.csv",
|
||||
fmt.Sprintf("--secure-port=%d", cfg.API.Port),
|
||||
"--allow-privileged",
|
||||
"--storage-backend=etcd3",
|
||||
)
|
||||
|
||||
if cfg.AuthorizationMode != "" {
|
||||
|
||||
@ -376,9 +376,12 @@ func TestGetAPIServerCommand(t *testing.T) {
|
||||
"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
|
||||
"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
|
||||
fmt.Sprintf("--secure-port=%d", 123),
|
||||
"--allow-privileged",
|
||||
"--storage-backend=etcd3",
|
||||
"--etcd-servers=http://127.0.0.1:2379",
|
||||
},
|
||||
},
|
||||
@ -396,9 +399,12 @@ func TestGetAPIServerCommand(t *testing.T) {
|
||||
"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
|
||||
"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
|
||||
fmt.Sprintf("--secure-port=%d", 123),
|
||||
"--allow-privileged",
|
||||
"--storage-backend=etcd3",
|
||||
"--advertise-address=foo",
|
||||
"--etcd-servers=http://127.0.0.1:2379",
|
||||
},
|
||||
@ -418,9 +424,12 @@ func TestGetAPIServerCommand(t *testing.T) {
|
||||
"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
|
||||
"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
|
||||
fmt.Sprintf("--secure-port=%d", 123),
|
||||
"--allow-privileged",
|
||||
"--storage-backend=etcd3",
|
||||
"--etcd-servers=http://127.0.0.1:2379",
|
||||
"--etcd-certfile=fiz",
|
||||
"--etcd-keyfile=faz",
|
||||
@ -442,9 +451,12 @@ func TestGetAPIServerCommand(t *testing.T) {
|
||||
"--client-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
|
||||
"--tls-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--tls-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--kubelet-client-certificate=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.crt",
|
||||
"--kubelet-client-key=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/apiserver.key",
|
||||
"--token-auth-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/tokens.csv",
|
||||
fmt.Sprintf("--secure-port=%d", 123),
|
||||
"--allow-privileged",
|
||||
"--storage-backend=etcd3",
|
||||
"--advertise-address=foo",
|
||||
"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
|
||||
"--anonymous-auth=false",
|
||||
|
||||
@ -123,7 +123,9 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration, pkiDir string) error {
|
||||
config := certutil.Config{
|
||||
CommonName: "kube-apiserver",
|
||||
AltNames: altNames,
|
||||
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
// This makes the apiserver allowed to talk to the kubelets in the cluster
|
||||
Organization: []string{"system:masters"},
|
||||
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
apiCert, apiKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
|
||||
if err != nil {
|
||||
|
||||
@ -14,6 +14,7 @@ go_library(
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
|
||||
"//pkg/api/validation:go_default_library",
|
||||
"//pkg/util/initsystem:go_default_library",
|
||||
|
||||
@ -25,10 +25,11 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
|
||||
"k8s.io/kubernetes/pkg/api/validation"
|
||||
"k8s.io/kubernetes/pkg/util/initsystem"
|
||||
@ -213,7 +214,7 @@ func (fcc FileContentCheck) Check() (warnings, errors []error) {
|
||||
|
||||
}
|
||||
|
||||
// InPathCheck checks if the given executable is present in the path.
|
||||
// InPathCheck checks if the given executable is present in the path
|
||||
type InPathCheck struct {
|
||||
executable string
|
||||
mandatory bool
|
||||
@ -318,7 +319,7 @@ func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error {
|
||||
PortOpenCheck{port: 10251},
|
||||
PortOpenCheck{port: 10252},
|
||||
HTTPProxyCheck{Proto: "https", Host: cfg.API.AdvertiseAddresses[0], Port: int(cfg.API.Port)},
|
||||
DirAvailableCheck{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
|
||||
DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
|
||||
DirAvailableCheck{Path: "/var/lib/kubelet"},
|
||||
FileContentCheck{Path: bridgenf, Content: []byte{'1'}},
|
||||
InPathCheck{executable: "ip", mandatory: true},
|
||||
@ -351,9 +352,10 @@ func RunJoinNodeChecks(cfg *kubeadmapi.NodeConfiguration) error {
|
||||
ServiceCheck{Service: "kubelet", CheckIfActive: false},
|
||||
ServiceCheck{Service: "docker", CheckIfActive: true},
|
||||
PortOpenCheck{port: 10250},
|
||||
DirAvailableCheck{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
|
||||
DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
|
||||
DirAvailableCheck{Path: "/var/lib/kubelet"},
|
||||
FileAvailableCheck{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)},
|
||||
FileAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.CACertName)},
|
||||
FileAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)},
|
||||
FileContentCheck{Path: bridgenf, Content: []byte{'1'}},
|
||||
InPathCheck{executable: "ip", mandatory: true},
|
||||
InPathCheck{executable: "iptables", mandatory: true},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user