diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 43e989514be..54a994f7b32 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -739,6 +739,24 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan // bootstrapping is enabled or client certificate rotation is enabled. func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) { if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) { + // Rules for client rotation and the handling of kube config files: + // + // 1. If the client provides only a kubeconfig file, we must use that as the initial client + // kubeadm needs the initial data in the kubeconfig to be placed into the cert store + // 2. If the client provides only an initial bootstrap kubeconfig file, we must create a + // kubeconfig file at the target location that points to the cert store, but until + // the file is present the client config will have no certs + // 3. If the client provides both and the kubeconfig is valid, we must ignore the bootstrap + // kubeconfig. + // 4. If the client provides both and the kubeconfig is expired or otherwise invalid, we must + // replace the kubeconfig with a new file that points to the cert dir + // + // The desired configuration for bootstrapping is to use a bootstrap kubeconfig and to have + // the kubeconfig file be managed by this process. For backwards compatibility with kubeadm, + // which provides a high powered kubeconfig on the master with cert/key data, we must + // bootstrap the cert manager with the contents of the initial client config. + + klog.Infof("Client rotation is on, will bootstrap in background") certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory) if err != nil { return nil, nil, err @@ -750,9 +768,8 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) } // the rotating transport will use the cert from the cert manager instead of these files - transportConfig := restclient.CopyConfig(clientConfig) - transportConfig.CertFile = "" - transportConfig.KeyFile = "" + transportConfig := restclient.AnonymousClientConfig(clientConfig) + kubeClientConfigOverrides(s, transportConfig) // we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable // to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper @@ -774,10 +791,16 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) } } - clientConfig, err := createAPIServerClientConfig(s) + clientConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig}, + &clientcmd.ConfigOverrides{}, + ).ClientConfig() if err != nil { return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err) } + + kubeClientConfigOverrides(s, clientConfig) + return clientConfig, nil, nil } @@ -804,12 +827,26 @@ func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, return kubeletcertificate.NewKubeletClientCertificateManager( certDir, nodeName, + + // this preserves backwards compatibility with kubeadm which passes + // a high powered certificate to the kubelet as --kubeconfig and expects + // it to be rotated out immediately + clientConfig.CertData, + clientConfig.KeyData, + clientConfig.CertFile, clientConfig.KeyFile, newClientFn, ) } +func kubeClientConfigOverrides(s *options.KubeletServer, clientConfig *restclient.Config) { + clientConfig.ContentType = s.ContentType + // Override kubeconfig qps/burst settings from flags + clientConfig.QPS = float32(s.KubeAPIQPS) + clientConfig.Burst = int(s.KubeAPIBurst) +} + // getNodeName returns the node name according to the cloud provider // if cloud provider is specified. Otherwise, returns the hostname of the node. func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) { @@ -898,38 +935,6 @@ func InitializeTLS(kf *options.KubeletFlags, kc *kubeletconfiginternal.KubeletCo return tlsOptions, nil } -func kubeconfigClientConfig(s *options.KubeletServer) (*restclient.Config, error) { - return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig}, - &clientcmd.ConfigOverrides{}, - ).ClientConfig() -} - -// createClientConfig creates a client configuration from the command line arguments. -// If --kubeconfig is explicitly set, it will be used. -func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) { - if len(s.BootstrapKubeconfig) > 0 || len(s.KubeConfig) > 0 { - return kubeconfigClientConfig(s) - } - return nil, fmt.Errorf("createClientConfig called in standalone mode") -} - -// createAPIServerClientConfig generates a client.Config from command line flags -// via createClientConfig and then injects chaos into the configuration via addChaosToClientConfig. -func createAPIServerClientConfig(s *options.KubeletServer) (*restclient.Config, error) { - clientConfig, err := createClientConfig(s) - if err != nil { - return nil, err - } - - clientConfig.ContentType = s.ContentType - // Override kubeconfig qps/burst settings from flags - clientConfig.QPS = float32(s.KubeAPIQPS) - clientConfig.Burst = int(s.KubeAPIBurst) - - return clientConfig, nil -} - // RunKubelet is responsible for setting up and running a kubelet. It is used in three different applications: // 1 Integration tests // 2 Kubelet binary diff --git a/cmd/kubelet/app/server_bootstrap_test.go b/cmd/kubelet/app/server_bootstrap_test.go index f6fcd658dd0..c59029fae86 100644 --- a/cmd/kubelet/app/server_bootstrap_test.go +++ b/cmd/kubelet/app/server_bootstrap_test.go @@ -19,10 +19,13 @@ package app import ( "crypto/ecdsa" "crypto/elliptic" - cryptorand "crypto/rand" + "crypto/rand" "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "io/ioutil" + "math/big" "net/http" "net/http/httptest" "os" @@ -53,7 +56,7 @@ func Test_buildClientCertificateManager(t *testing.T) { } defer func() { os.RemoveAll(testDir) }() - serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatal(err) } @@ -132,6 +135,65 @@ func Test_buildClientCertificateManager(t *testing.T) { } } +func Test_buildClientCertificateManager_populateCertDir(t *testing.T) { + testDir, err := ioutil.TempDir("", "kubeletcert") + if err != nil { + t.Fatal(err) + } + defer func() { os.RemoveAll(testDir) }() + + // when no cert is provided, write nothing to disk + config1 := &restclient.Config{ + UserAgent: "FirstClient", + Host: "http://localhost", + } + config2 := &restclient.Config{ + UserAgent: "SecondClient", + Host: "http://localhost", + } + nodeName := types.NodeName("test") + if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil { + t.Fatal(err) + } + fi := getFileInfo(testDir) + if len(fi) != 0 { + t.Fatalf("Unexpected directory contents: %#v", fi) + } + + // an invalid cert should be ignored + config2.CertData = []byte("invalid contents") + config2.KeyData = []byte("invalid contents") + if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err == nil { + t.Fatal("unexpected non error") + } + fi = getFileInfo(testDir) + if len(fi) != 0 { + t.Fatalf("Unexpected directory contents: %#v", fi) + } + + // an expired client certificate should be written to disk, because the cert manager can + // use config1 to refresh it and the cert manager won't return it for clients. + config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour)) + if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil { + t.Fatal(err) + } + fi = getFileInfo(testDir) + if len(fi) != 2 { + t.Fatalf("Unexpected directory contents: %#v", fi) + } + + // a valid, non-expired client certificate should be written to disk + config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-time.Hour), time.Now().Add(24*time.Hour)) + if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil { + t.Fatal(err) + } + fi = getFileInfo(testDir) + if len(fi) != 2 { + t.Fatalf("Unexpected directory contents: %#v", fi) + } + +} + func getFileInfo(dir string) map[string]os.FileInfo { fi := make(map[string]os.FileInfo) filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { @@ -279,3 +341,37 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) { t.Fatalf("unexpected request: %s %s", req.Method, req.URL) } } + +// genClientCert generates an x509 certificate for testing. Certificate and key +// are returned in PEM encoding. +func genClientCert(t *testing.T, from, to time.Time) ([]byte, []byte) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + keyRaw, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + t.Fatal(err) + } + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"Acme Co"}}, + NotBefore: from, + NotAfter: to, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + certRaw, err := x509.CreateCertificate(rand.Reader, cert, cert, key.Public(), key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certRaw}), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyRaw}) +} diff --git a/pkg/kubelet/certificate/bootstrap/bootstrap.go b/pkg/kubelet/certificate/bootstrap/bootstrap.go index 1d9e56dc8ce..46491a45cb2 100644 --- a/pkg/kubelet/certificate/bootstrap/bootstrap.go +++ b/pkg/kubelet/certificate/bootstrap/bootstrap.go @@ -59,7 +59,8 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig if err != nil { return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err) } - return clientConfig, clientConfig, nil + klog.V(2).Infof("No bootstrapping requested, will use kubeconfig") + return clientConfig, restclient.CopyConfig(clientConfig), nil } store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "") @@ -67,7 +68,7 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig return nil, nil, fmt.Errorf("unable to build bootstrap cert store") } - ok, err := verifyBootstrapClientConfig(kubeconfigPath) + ok, err := isClientConfigStillValid(kubeconfigPath) if err != nil { return nil, nil, err } @@ -78,7 +79,8 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig if err != nil { return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err) } - return clientConfig, clientConfig, nil + klog.V(2).Infof("Current kubeconfig file contents are still valid, no bootstrap necessary") + return clientConfig, restclient.CopyConfig(clientConfig), nil } bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath) @@ -93,6 +95,7 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig if err := writeKubeconfigFromBootstrapping(clientConfig, kubeconfigPath, pemPath); err != nil { return nil, nil, err } + klog.V(2).Infof("Use the bootstrap credentials to request a cert, and set kubeconfig to point to the certificate dir") return bootstrapClientConfig, clientConfig, nil } @@ -102,7 +105,7 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig // The certificate and key file are stored in certDir. func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName types.NodeName) error { // Short-circuit if the kubeconfig file exists and is valid. - ok, err := verifyBootstrapClientConfig(kubeconfigPath) + ok, err := isClientConfigStillValid(kubeconfigPath) if err != nil { return err } @@ -219,10 +222,10 @@ func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) { ).ClientConfig() } -// verifyBootstrapClientConfig checks the provided kubeconfig to see if it has a valid +// isClientConfigStillValid checks the provided kubeconfig to see if it has a valid // client certificate. It returns true if the kubeconfig is valid, or an error if bootstrapping // should stop immediately. -func verifyBootstrapClientConfig(kubeconfigPath string) (bool, error) { +func isClientConfigStillValid(kubeconfigPath string) (bool, error) { _, err := os.Stat(kubeconfigPath) if os.IsNotExist(err) { return false, nil diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index db5075c93e6..cf106c84e10 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -147,7 +147,16 @@ func addressesToHostnamesAndIPs(addresses []v1.NodeAddress) (dnsNames []string, // NewKubeletClientCertificateManager sets up a certificate manager without a // client that can be used to sign new certificates (or rotate). If a CSR // client is set later, it may begin rotating/renewing the client cert. -func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certFile string, keyFile string, clientFn certificate.CSRClientFunc) (certificate.Manager, error) { +func NewKubeletClientCertificateManager( + certDirectory string, + nodeName types.NodeName, + bootstrapCertData []byte, + bootstrapKeyData []byte, + certFile string, + keyFile string, + clientFn certificate.CSRClientFunc, +) (certificate.Manager, error) { + certificateStore, err := certificate.NewFileStore( "kubelet-client", certDirectory, @@ -165,7 +174,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.", }, ) - prometheus.MustRegister(certificateExpiration) + prometheus.Register(certificateExpiration) m, err := certificate.NewManager(&certificate.Config{ ClientFn: clientFn, @@ -190,6 +199,14 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod // authenticate itself to the TLS server. certificates.UsageClientAuth, }, + + // For backwards compatibility, the kubelet supports the ability to + // provide a higher privileged certificate as initial data that will + // then be rotated immediately. This code path is used by kubeadm on + // the masters. + BootstrapCertificatePEM: bootstrapCertData, + BootstrapKeyPEM: bootstrapKeyData, + CertificateStore: certificateStore, CertificateExpiration: certificateExpiration, })