From bc63c412b91dcb9bb4b8ee82d9c92fc9effe270c Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sun, 30 Jun 2024 14:50:46 +0000 Subject: [PATCH] kubelet request certificates if at least one IP exist A Kubernetes Node requires to have at minimum one IP address because those are used on the Pods field HostIPs and in some cases, when pods uses hostNetwork: true, as PodIPs. Nodes that use IP addresses as Hostname are interpreted as an IP address, so it is possible that are nodes that don't hane any DNSname. The feature gate AllowDNSOnlyNodeCSR will allow user to opt-in for the old behavior. Change-Id: I094531d87246f1e7a5ef4fe57bd5d9840cb1375d --- pkg/features/kube_features.go | 9 ++ pkg/kubelet/certificate/kubelet.go | 44 ++++--- pkg/kubelet/certificate/kubelet_test.go | 145 ++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 15 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index e898379de5b..af15ec5dcaa 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -45,6 +45,13 @@ const ( // Enable usage of Provision of PVCs from snapshots in other namespaces CrossNamespaceVolumeDataSource featuregate.Feature = "CrossNamespaceVolumeDataSource" + // owner: @aojea + // Deprecated: v1.31 + // + // Allow kubelet to request a certificate without any Node IP available, only + // with DNS names. + AllowDNSOnlyNodeCSR featuregate.Feature = "AllowDNSOnlyNodeCSR" + // owner: @thockin // deprecated: v1.29 // @@ -983,6 +990,8 @@ func init() { var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ CrossNamespaceVolumeDataSource: {Default: false, PreRelease: featuregate.Alpha}, + AllowDNSOnlyNodeCSR: {Default: false, PreRelease: featuregate.Deprecated}, // remove after 1.33 + AllowServiceLBStatusOnNonLB: {Default: false, PreRelease: featuregate.Deprecated}, // remove after 1.29 AnyVolumeDataSource: {Default: true, PreRelease: featuregate.Beta}, // on by default in 1.24 diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index 1e238030ab4..a2d7a03c832 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -32,16 +32,44 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/server/dynamiccertificates" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/certificate" compbasemetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/metrics" netutils "k8s.io/utils/net" ) +func newGetTemplateFn(nodeName types.NodeName, getAddresses func() []v1.NodeAddress) func() *x509.CertificateRequest { + return func() *x509.CertificateRequest { + hostnames, ips := addressesToHostnamesAndIPs(getAddresses()) + // by default, require at least one IP before requesting a serving certificate + hasRequiredAddresses := len(ips) > 0 + + // optionally allow requesting a serving certificate with just a DNS name + if utilfeature.DefaultFeatureGate.Enabled(features.AllowDNSOnlyNodeCSR) { + hasRequiredAddresses = hasRequiredAddresses || len(hostnames) > 0 + } + + // don't return a template if we have no addresses to request for + if !hasRequiredAddresses { + return nil + } + return &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + DNSNames: hostnames, + IPAddresses: ips, + } + } +} + // NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate // or returns an error. func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, getAddresses func() []v1.NodeAddress, certDirectory string) (certificate.Manager, error) { @@ -92,21 +120,7 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg ) legacyregistry.MustRegister(certificateRotationAge) - getTemplate := func() *x509.CertificateRequest { - hostnames, ips := addressesToHostnamesAndIPs(getAddresses()) - // don't return a template if we have no addresses to request for - if len(hostnames) == 0 && len(ips) == 0 { - return nil - } - return &x509.CertificateRequest{ - Subject: pkix.Name{ - CommonName: fmt.Sprintf("system:node:%s", nodeName), - Organization: []string{"system:nodes"}, - }, - DNSNames: hostnames, - IPAddresses: ips, - } - } + getTemplate := newGetTemplateFn(nodeName, getAddresses) m, err := certificate.NewManager(&certificate.Config{ ClientsetFn: clientsetFn, diff --git a/pkg/kubelet/certificate/kubelet_test.go b/pkg/kubelet/certificate/kubelet_test.go index 709bf02c582..5d84e7d5e09 100644 --- a/pkg/kubelet/certificate/kubelet_test.go +++ b/pkg/kubelet/certificate/kubelet_test.go @@ -19,6 +19,8 @@ package certificate import ( "bytes" "context" + "crypto/x509" + "crypto/x509/pkix" "fmt" "net" "os" @@ -28,8 +30,12 @@ import ( "time" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/util/cert" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" netutils "k8s.io/utils/net" ) @@ -261,3 +267,142 @@ func TestKubeletServerCertificateFromFiles(t *testing.T) { }) } } + +func TestNewCertificateManagerConfigGetTemplate(t *testing.T) { + nodeName := "fake-node" + nodeIP := netutils.ParseIPSloppy("192.168.1.1") + tests := []struct { + name string + nodeAddresses []v1.NodeAddress + want *x509.CertificateRequest + featuregate bool + }{ + { + name: "node addresses or hostnames and gate enabled", + featuregate: true, + }, + { + name: "node addresses or hostnames and gate disabled", + featuregate: false, + }, + { + name: "only hostnames and gate enabled", + nodeAddresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + want: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + DNSNames: []string{nodeName}, + }, + featuregate: true, + }, + { + name: "only hostnames and gate disabled", + nodeAddresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + featuregate: false, + }, + { + name: "only IP addresses and gate enabled", + nodeAddresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: nodeIP.String(), + }, + }, + want: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + IPAddresses: []net.IP{nodeIP}, + }, + featuregate: true, + }, + { + name: "only IP addresses and gate disabled", + nodeAddresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: nodeIP.String(), + }, + }, + want: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + IPAddresses: []net.IP{nodeIP}, + }, + featuregate: false, + }, + { + name: "IP addresses and hostnames and gate enabled", + nodeAddresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: nodeName, + }, + { + Type: v1.NodeInternalIP, + Address: nodeIP.String(), + }, + }, + want: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + DNSNames: []string{nodeName}, + IPAddresses: []net.IP{nodeIP}, + }, + featuregate: true, + }, + { + name: "IP addresses and hostnames and gate disabled", + nodeAddresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: nodeName, + }, + { + Type: v1.NodeInternalIP, + Address: nodeIP.String(), + }, + }, + want: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + DNSNames: []string{nodeName}, + IPAddresses: []net.IP{nodeIP}, + }, + featuregate: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AllowDNSOnlyNodeCSR, tt.featuregate) + getAddresses := func() []v1.NodeAddress { + return tt.nodeAddresses + } + getTemplate := newGetTemplateFn(types.NodeName(nodeName), getAddresses) + got := getTemplate() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Wrong certificate, got %v expected %v", got, tt.want) + return + } + }) + } +}