github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

kubelet: allow specifying dual-stack node IPs on bare metal

Browse Source 
Discussion is ongoing about how to best handle dual-stack with clouds
and autodetected IPs, but there is at least agreement that people on
bare metal ought to be able to specify two explicit IPs on dual-stack
hosts, so allow that.
This commit is contained in:
Dan Winship 2020-09-29 20:47:24 -04:00
parent 2680095414
commit 75242fce7a
10 changed files with 136 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kubelet/app/BUILD
View File

@ -124,6 +124,7 @@ go_library(
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/k8s.io/utils/inotify:go_default_library",

2
cmd/kubelet/app/options/options.go
View File

@ -325,7 +325,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname. If --cloud-provider is set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")
fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.")
fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.")
fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+
"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")

28
cmd/kubelet/app/server.go
View File

@ -102,6 +102,7 @@ import (
"k8s.io/kubernetes/pkg/volume/util/hostutil"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/utils/exec"
utilnet "k8s.io/utils/net"
)
const (
@ -1086,6 +1087,27 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
// Setup event recorder if required.
makeEventRecorder(kubeDeps, nodeName)
var nodeIPs []net.IP
if kubeServer.NodeIP != "" {
for _, ip := range strings.Split(kubeServer.NodeIP, ",") {
parsedNodeIP := net.ParseIP(strings.TrimSpace(ip))
if parsedNodeIP == nil {
klog.Warningf("Could not parse --node-ip value %q; ignoring", ip)
} else {
nodeIPs = append(nodeIPs, parsedNodeIP)
}
}
}
if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && len(nodeIPs) > 1 {
return fmt.Errorf("dual-stack --node-ip %q not supported in a single-stack cluster", kubeServer.NodeIP)
} else if len(nodeIPs) > 2 || (len(nodeIPs) == 2 && utilnet.IsIPv6(nodeIPs[0]) == utilnet.IsIPv6(nodeIPs[1])) {
return fmt.Errorf("bad --node-ip %q; must contain either a single IP or a dual-stack pair of IPs", kubeServer.NodeIP)
} else if len(nodeIPs) == 2 && kubeServer.CloudProvider != "" {
return fmt.Errorf("dual-stack --node-ip %q not supported when using a cloud provider", kubeServer.NodeIP)
} else if len(nodeIPs) == 2 && (nodeIPs[0].IsUnspecified() || nodeIPs[1].IsUnspecified()) {
return fmt.Errorf("dual-stack --node-ip %q cannot include '0.0.0.0' or '::'", kubeServer.NodeIP)
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: true,
})
@ -1104,7 +1126,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
hostname,
hostnameOverridden,
nodeName,
kubeServer.NodeIP,
nodeIPs,
kubeServer.ProviderID,
kubeServer.CloudProvider,
kubeServer.CertDirectory,
@ -1178,7 +1200,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostname string,
hostnameOverridden bool,
nodeName types.NodeName,
nodeIP string,
nodeIPs []net.IP,
providerID string,
cloudProvider string,
certDirectory string,
@ -1209,7 +1231,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostname,
hostnameOverridden,
nodeName,
nodeIP,
nodeIPs,
providerID,
cloudProvider,
certDirectory,

11
pkg/kubelet/kubelet.go
View File

@ -334,7 +334,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
hostname string,
hostnameOverridden bool,
nodeName types.NodeName,
nodeIP string,
nodeIPs []net.IP,
providerID string,
cloudProvider string,
certDirectory string,
@ -462,7 +462,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
}
httpClient := &http.Client{}
parsedNodeIP := net.ParseIP(nodeIP)
klet := &Kubelet{
hostname: hostname,
@ -477,7 +476,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
registerNode: registerNode,
registerWithTaints: registerWithTaints,
registerSchedulable: registerSchedulable,
dnsConfigurer: dns.NewConfigurer(kubeDeps.Recorder, nodeRef, parsedNodeIP, clusterDNS, kubeCfg.ClusterDomain, kubeCfg.ResolverConfig),
dnsConfigurer: dns.NewConfigurer(kubeDeps.Recorder, nodeRef, nodeIPs, clusterDNS, kubeCfg.ClusterDomain, kubeCfg.ResolverConfig),
serviceLister: serviceLister,
serviceHasSynced: serviceHasSynced,
nodeLister: nodeLister,
@ -506,7 +505,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
containerManager: kubeDeps.ContainerManager,
containerRuntimeName: containerRuntime,
redirectContainerStreaming: crOptions.RedirectContainerStreaming,
nodeIP: parsedNodeIP,
nodeIPs: nodeIPs,
nodeIPValidator: validateNodeIP,
clock: clock.RealClock{},
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
@ -1042,8 +1041,8 @@ type Kubelet struct {
// oneTimeInitializer is used to initialize modules that are dependent on the runtime to be up.
oneTimeInitializer sync.Once
// If non-nil, use this IP address for the node
nodeIP net.IP
// If set, use this IP address or addresses for the node
nodeIPs []net.IP
// use this function to validate the kubelet nodeIP
nodeIPValidator func(net.IP) error

4
pkg/kubelet/kubelet_network_linux.go
View File

@ -35,9 +35,9 @@ func (kl *Kubelet) initNetworkUtil() {
exec := utilexec.New()
// At this point in startup we don't know the actual node IPs, so we configure dual stack iptables
// rules if the node _might_ be dual-stack, and single-stack based on requested nodeIP otherwise.
// rules if the node _might_ be dual-stack, and single-stack based on requested nodeIPs[0] otherwise.
maybeDualStack := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack)
ipv6Primary := kl.nodeIP != nil && utilnet.IsIPv6(kl.nodeIP)
ipv6Primary := kl.nodeIPs != nil && utilnet.IsIPv6(kl.nodeIPs[0])
var iptClients []utiliptables.Interface
if maybeDualStack || !ipv6Primary {

2
pkg/kubelet/kubelet_node_status.go
View File

@ -587,7 +587,7 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
}
var setters []func(n *v1.Node) error
setters = append(setters,
nodestatus.NodeAddress(kl.nodeIP, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc),
nodestatus.NodeAddress(kl.nodeIPs, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc),
nodestatus.MachineInfo(string(kl.nodeName), kl.maxPods, kl.podsPerCore, kl.GetCachedMachineInfo, kl.containerManager.GetCapacity,
kl.containerManager.GetDevicePluginResourceCapacity, kl.containerManager.GetNodeAllocatableReservation, kl.recordEvent),
nodestatus.VersionInfo(kl.cadvisor.VersionInfo, kl.containerRuntime.Type, kl.containerRuntime.Version),

1
pkg/kubelet/network/dns/BUILD
View File

@ -15,6 +15,7 @@ go_library(
"//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/io:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
)

21
pkg/kubelet/network/dns/dns.go
View File

@ -35,6 +35,7 @@ import (
"k8s.io/klog/v2"
utilio "k8s.io/utils/io"
utilnet "k8s.io/utils/net"
)
var (
@ -58,7 +59,7 @@ const (
type Configurer struct {
recorder record.EventRecorder
nodeRef *v1.ObjectReference
nodeIP net.IP
nodeIPs []net.IP
// If non-nil, use this for container DNS server.
clusterDNS []net.IP
@ -71,11 +72,11 @@ type Configurer struct {
}
// NewConfigurer returns a DNS configurer for launching pods.
func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIP net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer {
func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIPs []net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer {
return &Configurer{
recorder: recorder,
nodeRef: nodeRef,
nodeIP: nodeIP,
nodeIPs: nodeIPs,
clusterDNS: clusterDNS,
ClusterDomain: clusterDomain,
ResolverConfig: resolverConfig,
@ -373,11 +374,15 @@ func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
// local machine". A nameserver setting of localhost is equivalent to
// this documented behavior.
if c.ResolverConfig == "" {
switch {
case c.nodeIP == nil || c.nodeIP.To4() != nil:
dnsConfig.Servers = []string{"127.0.0.1"}
case c.nodeIP.To16() != nil:
dnsConfig.Servers = []string{"::1"}
for _, nodeIP := range c.nodeIPs {
if utilnet.IsIPv6(nodeIP) {
dnsConfig.Servers = append(dnsConfig.Servers, "::1")
} else {
dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1")
}
}
if len(dnsConfig.Servers) == 0 {
dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1")
}
dnsConfig.Searches = []string{"."}
}

23
pkg/kubelet/nodestatus/setters.go
View File

@ -57,7 +57,7 @@ const (
type Setter func(node *v1.Node) error
// NodeAddress returns a Setter that updates address-related information on the node.
func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP
func NodeAddress(nodeIPs []net.IP, // typically Kubelet.nodeIPs
validateNodeIPFunc func(net.IP) error, // typically Kubelet.nodeIPValidator
hostname string, // typically Kubelet.hostname
hostnameOverridden bool, // was the hostname force set?
@ -65,10 +65,19 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP
cloud cloudprovider.Interface, // typically Kubelet.cloud
nodeAddressesFunc func() ([]v1.NodeAddress, error), // typically Kubelet.cloudResourceSyncManager.NodeAddresses
) Setter {
var nodeIP, secondaryNodeIP net.IP
if len(nodeIPs) > 0 {
nodeIP = nodeIPs[0]
}
preferIPv4 := nodeIP == nil || nodeIP.To4() != nil
isPreferredIPFamily := func(ip net.IP) bool { return (ip.To4() != nil) == preferIPv4 }
nodeIPSpecified := nodeIP != nil && !nodeIP.IsUnspecified()
if len(nodeIPs) > 1 {
secondaryNodeIP = nodeIPs[1]
}
secondaryNodeIPSpecified := secondaryNodeIP != nil && !secondaryNodeIP.IsUnspecified()
return func(node *v1.Node) error {
if nodeIPSpecified {
if err := validateNodeIPFunc(nodeIP); err != nil {
@ -76,6 +85,12 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP
}
klog.V(2).Infof("Using node IP: %q", nodeIP.String())
}
if secondaryNodeIPSpecified {
if err := validateNodeIPFunc(secondaryNodeIP); err != nil {
return fmt.Errorf("failed to validate secondaryNodeIP: %v", err)
}
klog.V(2).Infof("Using secondary node IP: %q", secondaryNodeIP.String())
}
if externalCloudProvider {
if nodeIPSpecified {
@ -185,6 +200,12 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP
}
}
node.Status.Addresses = nodeAddresses
} else if nodeIPSpecified && secondaryNodeIPSpecified {
node.Status.Addresses = []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: nodeIP.String()},
{Type: v1.NodeInternalIP, Address: secondaryNodeIP.String()},
{Type: v1.NodeHostName, Address: hostname},
}
} else {
var ipAddr net.IP
var err error

66
pkg/kubelet/nodestatus/setters_test.go
View File

@ -416,7 +416,7 @@ func TestNodeAddress(t *testing.T) {
}
// construct setter
setter := NodeAddress(nodeIP,
setter := NodeAddress([]net.IP{nodeIP},
nodeIPValidator,
hostname,
testCase.hostnameOverride,
@ -439,6 +439,70 @@ func TestNodeAddress(t *testing.T) {
}
}
// We can't test failure or autodetection cases here because the relevant code isn't mockable
func TestNodeAddress_NoCloudProvider(t *testing.T) {
cases := []struct {
name string
nodeIPs []net.IP
expectedAddresses []v1.NodeAddress
}{
{
name: "Single --node-ip",
nodeIPs: []net.IP{net.ParseIP("10.1.1.1")},
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.1"},
{Type: v1.NodeHostName, Address: testKubeletHostname},
},
},
{
name: "Dual --node-ips",
nodeIPs: []net.IP{net.ParseIP("10.1.1.1"), net.ParseIP("fd01::1234")},
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.1.1.1"},
{Type: v1.NodeInternalIP, Address: "fd01::1234"},
{Type: v1.NodeHostName, Address: testKubeletHostname},
},
},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
// testCase setup
existingNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname, Annotations: make(map[string]string)},
Spec: v1.NodeSpec{},
Status: v1.NodeStatus{
Addresses: []v1.NodeAddress{},
},
}
nodeIPValidator := func(nodeIP net.IP) error {
return nil
}
nodeAddressesFunc := func() ([]v1.NodeAddress, error) {
return nil, fmt.Errorf("not reached")
}
// construct setter
setter := NodeAddress(testCase.nodeIPs,
nodeIPValidator,
testKubeletHostname,
false, // hostnameOverridden
false, // externalCloudProvider
nil, // cloud
nodeAddressesFunc)
// call setter on existing node
err := setter(existingNode)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
assert.True(t, apiequality.Semantic.DeepEqual(testCase.expectedAddresses, existingNode.Status.Addresses),
"Diff: %s", diff.ObjectDiff(testCase.expectedAddresses, existingNode.Status.Addresses))
})
}
}
func TestMachineInfo(t *testing.T) {
const nodeName = "test-node"