diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 2e9ff53fff0..0e9526f117c 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -124,6 +124,7 @@ go_library( "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:android": [ "//vendor/k8s.io/utils/inotify:go_default_library", diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index a208b5ac9a0..37cbb7d6087 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -325,7 +325,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname. If --cloud-provider is set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).") - fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.") + fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.") fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+ "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 5f7e3555aab..c7690392b57 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -102,6 +102,7 @@ import ( "k8s.io/kubernetes/pkg/volume/util/hostutil" "k8s.io/kubernetes/pkg/volume/util/subpath" "k8s.io/utils/exec" + utilnet "k8s.io/utils/net" ) const ( @@ -1086,6 +1087,27 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie // Setup event recorder if required. makeEventRecorder(kubeDeps, nodeName) + var nodeIPs []net.IP + if kubeServer.NodeIP != "" { + for _, ip := range strings.Split(kubeServer.NodeIP, ",") { + parsedNodeIP := net.ParseIP(strings.TrimSpace(ip)) + if parsedNodeIP == nil { + klog.Warningf("Could not parse --node-ip value %q; ignoring", ip) + } else { + nodeIPs = append(nodeIPs, parsedNodeIP) + } + } + } + if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && len(nodeIPs) > 1 { + return fmt.Errorf("dual-stack --node-ip %q not supported in a single-stack cluster", kubeServer.NodeIP) + } else if len(nodeIPs) > 2 || (len(nodeIPs) == 2 && utilnet.IsIPv6(nodeIPs[0]) == utilnet.IsIPv6(nodeIPs[1])) { + return fmt.Errorf("bad --node-ip %q; must contain either a single IP or a dual-stack pair of IPs", kubeServer.NodeIP) + } else if len(nodeIPs) == 2 && kubeServer.CloudProvider != "" { + return fmt.Errorf("dual-stack --node-ip %q not supported when using a cloud provider", kubeServer.NodeIP) + } else if len(nodeIPs) == 2 && (nodeIPs[0].IsUnspecified() || nodeIPs[1].IsUnspecified()) { + return fmt.Errorf("dual-stack --node-ip %q cannot include '0.0.0.0' or '::'", kubeServer.NodeIP) + } + capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: true, }) @@ -1104,7 +1126,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie hostname, hostnameOverridden, nodeName, - kubeServer.NodeIP, + nodeIPs, kubeServer.ProviderID, kubeServer.CloudProvider, kubeServer.CertDirectory, @@ -1178,7 +1200,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, hostname string, hostnameOverridden bool, nodeName types.NodeName, - nodeIP string, + nodeIPs []net.IP, providerID string, cloudProvider string, certDirectory string, @@ -1209,7 +1231,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, hostname, hostnameOverridden, nodeName, - nodeIP, + nodeIPs, providerID, cloudProvider, certDirectory, diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6d478de0c85..cc0dd0ee137 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -334,7 +334,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, hostname string, hostnameOverridden bool, nodeName types.NodeName, - nodeIP string, + nodeIPs []net.IP, providerID string, cloudProvider string, certDirectory string, @@ -462,7 +462,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } } httpClient := &http.Client{} - parsedNodeIP := net.ParseIP(nodeIP) klet := &Kubelet{ hostname: hostname, @@ -477,7 +476,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, registerNode: registerNode, registerWithTaints: registerWithTaints, registerSchedulable: registerSchedulable, - dnsConfigurer: dns.NewConfigurer(kubeDeps.Recorder, nodeRef, parsedNodeIP, clusterDNS, kubeCfg.ClusterDomain, kubeCfg.ResolverConfig), + dnsConfigurer: dns.NewConfigurer(kubeDeps.Recorder, nodeRef, nodeIPs, clusterDNS, kubeCfg.ClusterDomain, kubeCfg.ResolverConfig), serviceLister: serviceLister, serviceHasSynced: serviceHasSynced, nodeLister: nodeLister, @@ -506,7 +505,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, containerManager: kubeDeps.ContainerManager, containerRuntimeName: containerRuntime, redirectContainerStreaming: crOptions.RedirectContainerStreaming, - nodeIP: parsedNodeIP, + nodeIPs: nodeIPs, nodeIPValidator: validateNodeIP, clock: clock.RealClock{}, enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach, @@ -1042,8 +1041,8 @@ type Kubelet struct { // oneTimeInitializer is used to initialize modules that are dependent on the runtime to be up. oneTimeInitializer sync.Once - // If non-nil, use this IP address for the node - nodeIP net.IP + // If set, use this IP address or addresses for the node + nodeIPs []net.IP // use this function to validate the kubelet nodeIP nodeIPValidator func(net.IP) error diff --git a/pkg/kubelet/kubelet_network_linux.go b/pkg/kubelet/kubelet_network_linux.go index e7646e590e7..ea630737196 100644 --- a/pkg/kubelet/kubelet_network_linux.go +++ b/pkg/kubelet/kubelet_network_linux.go @@ -35,9 +35,9 @@ func (kl *Kubelet) initNetworkUtil() { exec := utilexec.New() // At this point in startup we don't know the actual node IPs, so we configure dual stack iptables - // rules if the node _might_ be dual-stack, and single-stack based on requested nodeIP otherwise. + // rules if the node _might_ be dual-stack, and single-stack based on requested nodeIPs[0] otherwise. maybeDualStack := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) - ipv6Primary := kl.nodeIP != nil && utilnet.IsIPv6(kl.nodeIP) + ipv6Primary := kl.nodeIPs != nil && utilnet.IsIPv6(kl.nodeIPs[0]) var iptClients []utiliptables.Interface if maybeDualStack || !ipv6Primary { diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index bb4a7e379d3..6b0312ba436 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -587,7 +587,7 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error { } var setters []func(n *v1.Node) error setters = append(setters, - nodestatus.NodeAddress(kl.nodeIP, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc), + nodestatus.NodeAddress(kl.nodeIPs, kl.nodeIPValidator, kl.hostname, kl.hostnameOverridden, kl.externalCloudProvider, kl.cloud, nodeAddressesFunc), nodestatus.MachineInfo(string(kl.nodeName), kl.maxPods, kl.podsPerCore, kl.GetCachedMachineInfo, kl.containerManager.GetCapacity, kl.containerManager.GetDevicePluginResourceCapacity, kl.containerManager.GetNodeAllocatableReservation, kl.recordEvent), nodestatus.VersionInfo(kl.cadvisor.VersionInfo, kl.containerRuntime.Type, kl.containerRuntime.Version), diff --git a/pkg/kubelet/network/dns/BUILD b/pkg/kubelet/network/dns/BUILD index f2d6cd1611c..438d69b76a9 100644 --- a/pkg/kubelet/network/dns/BUILD +++ b/pkg/kubelet/network/dns/BUILD @@ -15,6 +15,7 @@ go_library( "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/utils/io:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) diff --git a/pkg/kubelet/network/dns/dns.go b/pkg/kubelet/network/dns/dns.go index 247036383c8..0999b0618c7 100644 --- a/pkg/kubelet/network/dns/dns.go +++ b/pkg/kubelet/network/dns/dns.go @@ -35,6 +35,7 @@ import ( "k8s.io/klog/v2" utilio "k8s.io/utils/io" + utilnet "k8s.io/utils/net" ) var ( @@ -58,7 +59,7 @@ const ( type Configurer struct { recorder record.EventRecorder nodeRef *v1.ObjectReference - nodeIP net.IP + nodeIPs []net.IP // If non-nil, use this for container DNS server. clusterDNS []net.IP @@ -71,11 +72,11 @@ type Configurer struct { } // NewConfigurer returns a DNS configurer for launching pods. -func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIP net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer { +func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIPs []net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer { return &Configurer{ recorder: recorder, nodeRef: nodeRef, - nodeIP: nodeIP, + nodeIPs: nodeIPs, clusterDNS: clusterDNS, ClusterDomain: clusterDomain, ResolverConfig: resolverConfig, @@ -373,11 +374,15 @@ func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) { // local machine". A nameserver setting of localhost is equivalent to // this documented behavior. if c.ResolverConfig == "" { - switch { - case c.nodeIP == nil || c.nodeIP.To4() != nil: - dnsConfig.Servers = []string{"127.0.0.1"} - case c.nodeIP.To16() != nil: - dnsConfig.Servers = []string{"::1"} + for _, nodeIP := range c.nodeIPs { + if utilnet.IsIPv6(nodeIP) { + dnsConfig.Servers = append(dnsConfig.Servers, "::1") + } else { + dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1") + } + } + if len(dnsConfig.Servers) == 0 { + dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1") } dnsConfig.Searches = []string{"."} } diff --git a/pkg/kubelet/nodestatus/setters.go b/pkg/kubelet/nodestatus/setters.go index b286e7be3e1..ad824d638ee 100644 --- a/pkg/kubelet/nodestatus/setters.go +++ b/pkg/kubelet/nodestatus/setters.go @@ -57,7 +57,7 @@ const ( type Setter func(node *v1.Node) error // NodeAddress returns a Setter that updates address-related information on the node. -func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP +func NodeAddress(nodeIPs []net.IP, // typically Kubelet.nodeIPs validateNodeIPFunc func(net.IP) error, // typically Kubelet.nodeIPValidator hostname string, // typically Kubelet.hostname hostnameOverridden bool, // was the hostname force set? @@ -65,10 +65,19 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP cloud cloudprovider.Interface, // typically Kubelet.cloud nodeAddressesFunc func() ([]v1.NodeAddress, error), // typically Kubelet.cloudResourceSyncManager.NodeAddresses ) Setter { + var nodeIP, secondaryNodeIP net.IP + if len(nodeIPs) > 0 { + nodeIP = nodeIPs[0] + } preferIPv4 := nodeIP == nil || nodeIP.To4() != nil isPreferredIPFamily := func(ip net.IP) bool { return (ip.To4() != nil) == preferIPv4 } nodeIPSpecified := nodeIP != nil && !nodeIP.IsUnspecified() + if len(nodeIPs) > 1 { + secondaryNodeIP = nodeIPs[1] + } + secondaryNodeIPSpecified := secondaryNodeIP != nil && !secondaryNodeIP.IsUnspecified() + return func(node *v1.Node) error { if nodeIPSpecified { if err := validateNodeIPFunc(nodeIP); err != nil { @@ -76,6 +85,12 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP } klog.V(2).Infof("Using node IP: %q", nodeIP.String()) } + if secondaryNodeIPSpecified { + if err := validateNodeIPFunc(secondaryNodeIP); err != nil { + return fmt.Errorf("failed to validate secondaryNodeIP: %v", err) + } + klog.V(2).Infof("Using secondary node IP: %q", secondaryNodeIP.String()) + } if externalCloudProvider { if nodeIPSpecified { @@ -185,6 +200,12 @@ func NodeAddress(nodeIP net.IP, // typically Kubelet.nodeIP } } node.Status.Addresses = nodeAddresses + } else if nodeIPSpecified && secondaryNodeIPSpecified { + node.Status.Addresses = []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: nodeIP.String()}, + {Type: v1.NodeInternalIP, Address: secondaryNodeIP.String()}, + {Type: v1.NodeHostName, Address: hostname}, + } } else { var ipAddr net.IP var err error diff --git a/pkg/kubelet/nodestatus/setters_test.go b/pkg/kubelet/nodestatus/setters_test.go index 66105ce8df1..80d3223904c 100644 --- a/pkg/kubelet/nodestatus/setters_test.go +++ b/pkg/kubelet/nodestatus/setters_test.go @@ -416,7 +416,7 @@ func TestNodeAddress(t *testing.T) { } // construct setter - setter := NodeAddress(nodeIP, + setter := NodeAddress([]net.IP{nodeIP}, nodeIPValidator, hostname, testCase.hostnameOverride, @@ -439,6 +439,70 @@ func TestNodeAddress(t *testing.T) { } } +// We can't test failure or autodetection cases here because the relevant code isn't mockable +func TestNodeAddress_NoCloudProvider(t *testing.T) { + cases := []struct { + name string + nodeIPs []net.IP + expectedAddresses []v1.NodeAddress + }{ + { + name: "Single --node-ip", + nodeIPs: []net.IP{net.ParseIP("10.1.1.1")}, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + }, + { + name: "Dual --node-ips", + nodeIPs: []net.IP{net.ParseIP("10.1.1.1"), net.ParseIP("fd01::1234")}, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fd01::1234"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + }, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + // testCase setup + existingNode := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname, Annotations: make(map[string]string)}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{}, + }, + } + + nodeIPValidator := func(nodeIP net.IP) error { + return nil + } + nodeAddressesFunc := func() ([]v1.NodeAddress, error) { + return nil, fmt.Errorf("not reached") + } + + // construct setter + setter := NodeAddress(testCase.nodeIPs, + nodeIPValidator, + testKubeletHostname, + false, // hostnameOverridden + false, // externalCloudProvider + nil, // cloud + nodeAddressesFunc) + + // call setter on existing node + err := setter(existingNode) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assert.True(t, apiequality.Semantic.DeepEqual(testCase.expectedAddresses, existingNode.Status.Addresses), + "Diff: %s", diff.ObjectDiff(testCase.expectedAddresses, existingNode.Status.Addresses)) + }) + } +} + func TestMachineInfo(t *testing.T) { const nodeName = "test-node"