diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index ebbe2e7aca6..f786a6921b2 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -262,23 +262,23 @@ func (kl *Kubelet) GetPodCgroupRoot() string { return kl.containerManager.GetPodCgroupRoot() } -// GetHostIP returns host IP or nil in case of error. -func (kl *Kubelet) GetHostIP() (net.IP, error) { +// GetHostIPs returns host IPs or nil in case of error. +func (kl *Kubelet) GetHostIPs() ([]net.IP, error) { node, err := kl.GetNode() if err != nil { return nil, fmt.Errorf("cannot get node: %v", err) } - return utilnode.GetNodeHostIP(node) + return utilnode.GetNodeHostIPs(node) } -// getHostIPAnyway attempts to return the host IP from kubelet's nodeInfo, or +// getHostIPsAnyWay attempts to return the host IPs from kubelet's nodeInfo, or // the initialNode. -func (kl *Kubelet) getHostIPAnyWay() (net.IP, error) { +func (kl *Kubelet) getHostIPsAnyWay() ([]net.IP, error) { node, err := kl.getNodeAnyWay() if err != nil { return nil, err } - return utilnode.GetNodeHostIP(node) + return utilnode.GetNodeHostIPs(node) } // GetExtraSupplementalGroupsForPod returns a list of the extra diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 3d83119fc16..ca398da9351 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -811,11 +811,11 @@ func (kl *Kubelet) podFieldSelectorRuntimeValue(fs *v1.ObjectFieldSelector, pod case "spec.serviceAccountName": return pod.Spec.ServiceAccountName, nil case "status.hostIP": - hostIP, err := kl.getHostIPAnyWay() + hostIPs, err := kl.getHostIPsAnyWay() if err != nil { return "", err } - return hostIP.String(), nil + return hostIPs[0].String(), nil case "status.podIP": return podIP, nil case "status.podIPs": @@ -1531,14 +1531,17 @@ func (kl *Kubelet) generateAPIPodStatus(pod *v1.Pod, podStatus *kubecontainer.Po }) if kl.kubeClient != nil { - hostIP, err := kl.getHostIPAnyWay() + hostIPs, err := kl.getHostIPsAnyWay() if err != nil { - klog.V(4).Infof("Cannot get host IP: %v", err) + klog.V(4).Infof("Cannot get host IPs: %v", err) } else { - s.HostIP = hostIP.String() + s.HostIP = hostIPs[0].String() if kubecontainer.IsHostNetworkPod(pod) && s.PodIP == "" { - s.PodIP = hostIP.String() + s.PodIP = hostIPs[0].String() s.PodIPs = []v1.PodIP{{IP: s.PodIP}} + if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && len(hostIPs) == 2 { + s.PodIPs = append(s.PodIPs, v1.PodIP{IP: hostIPs[1].String()}) + } } } } diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 12e941cad2f..6557be994cb 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "sort" "testing" @@ -2489,3 +2490,126 @@ func TestPodResourcesAreReclaimed(t *testing.T) { }) } } + +func TestGenerateAPIPodStatusHostNetworkPodIPs(t *testing.T) { + testcases := []struct { + name string + dualStack bool + nodeAddresses []v1.NodeAddress + criPodIPs []string + podIPs []v1.PodIP + }{ + { + name: "Simple", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + }, + podIPs: []v1.PodIP{ + {IP: "10.0.0.1"}, + }, + }, + { + name: "InternalIP is preferred over ExternalIP", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "192.168.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + }, + podIPs: []v1.PodIP{ + {IP: "10.0.0.1"}, + }, + }, + { + name: "Dual-stack addresses are ignored in single-stack cluster", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "fd01::1234"}, + }, + podIPs: []v1.PodIP{ + {IP: "10.0.0.1"}, + }, + }, + { + name: "Single-stack addresses in dual-stack cluster", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + }, + dualStack: true, + podIPs: []v1.PodIP{ + {IP: "10.0.0.1"}, + }, + }, + { + name: "Multiple single-stack addresses in dual-stack cluster", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.2"}, + {Type: v1.NodeExternalIP, Address: "192.168.0.1"}, + }, + dualStack: true, + podIPs: []v1.PodIP{ + {IP: "10.0.0.1"}, + }, + }, + { + name: "Dual-stack addresses in dual-stack cluster", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "fd01::1234"}, + }, + dualStack: true, + podIPs: []v1.PodIP{ + {IP: "10.0.0.1"}, + {IP: "fd01::1234"}, + }, + }, + { + name: "CRI PodIPs override NodeAddresses", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + {Type: v1.NodeInternalIP, Address: "fd01::1234"}, + }, + dualStack: true, + criPodIPs: []string{"192.168.0.1"}, + podIPs: []v1.PodIP{ + {IP: "192.168.0.1"}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) + defer testKubelet.Cleanup() + kl := testKubelet.kubelet + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.dualStack)() + + kl.nodeLister = testNodeLister{nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: string(kl.nodeName)}, + Status: v1.NodeStatus{ + Addresses: tc.nodeAddresses, + }, + }, + }} + + pod := podWithUIDNameNs("12345", "test-pod", "test-namespace") + pod.Spec.HostNetwork = true + + criStatus := &kubecontainer.PodStatus{ + ID: pod.UID, + Name: pod.Name, + Namespace: pod.Namespace, + IPs: tc.criPodIPs, + } + + status := kl.generateAPIPodStatus(pod, criStatus) + if !reflect.DeepEqual(status.PodIPs, tc.podIPs) { + t.Fatalf("Expected PodIPs %#v, got %#v", tc.podIPs, status.PodIPs) + } + if tc.criPodIPs == nil && status.HostIP != status.PodIPs[0].IP { + t.Fatalf("Expected HostIP %q to equal PodIPs[0].IP %q", status.HostIP, status.PodIPs[0].IP) + } + }) + } +} diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 9b0961f6aa5..c724e8aaee4 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -231,7 +231,11 @@ func (kvh *kubeletVolumeHost) GetHostName() string { } func (kvh *kubeletVolumeHost) GetHostIP() (net.IP, error) { - return kvh.kubelet.GetHostIP() + hostIPs, err := kvh.kubelet.GetHostIPs() + if err != nil { + return nil, err + } + return hostIPs[0], err } func (kvh *kubeletVolumeHost) GetNodeAllocatable() (v1.ResourceList, error) { diff --git a/pkg/util/node/BUILD b/pkg/util/node/BUILD index 96013df5ace..1ebde8e5073 100644 --- a/pkg/util/node/BUILD +++ b/pkg/util/node/BUILD @@ -11,15 +11,18 @@ go_library( srcs = ["node.go"], importpath = "k8s.io/kubernetes/pkg/util/node", deps = [ + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", ], ) @@ -28,8 +31,11 @@ go_test( srcs = ["node_test.go"], embed = [":go_default_library"], deps = [ + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", ], ) diff --git a/pkg/util/node/node.go b/pkg/util/node/node.go index 6ec37485228..f33e8a8960e 100644 --- a/pkg/util/node/node.go +++ b/pkg/util/node/node.go @@ -33,8 +33,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/kubernetes/pkg/features" + utilnet "k8s.io/utils/net" ) const ( @@ -90,22 +93,55 @@ func GetPreferredNodeAddress(node *v1.Node, preferredAddressTypes []v1.NodeAddre return "", &NoMatchError{addresses: node.Status.Addresses} } -// GetNodeHostIP returns the provided node's IP, based on the priority: -// 1. NodeInternalIP -// 2. NodeExternalIP +// GetNodeHostIPs returns the provided node's IP(s); either a single "primary IP" for the +// node in a single-stack cluster, or a dual-stack pair of IPs in a dual-stack cluster +// (for nodes that actually have dual-stack IPs). Among other things, the IPs returned +// from this function are used as the `.status.PodIPs` values for host-network pods on the +// node, and the first IP is used as the `.status.HostIP` for all pods on the node. +func GetNodeHostIPs(node *v1.Node) ([]net.IP, error) { + // Re-sort the addresses with InternalIPs first and then ExternalIPs + allIPs := make([]net.IP, 0, len(node.Status.Addresses)) + for _, addr := range node.Status.Addresses { + if addr.Type == v1.NodeInternalIP { + ip := net.ParseIP(addr.Address) + if ip != nil { + allIPs = append(allIPs, ip) + } + } + } + for _, addr := range node.Status.Addresses { + if addr.Type == v1.NodeExternalIP { + ip := net.ParseIP(addr.Address) + if ip != nil { + allIPs = append(allIPs, ip) + } + } + } + if len(allIPs) == 0 { + return nil, fmt.Errorf("host IP unknown; known addresses: %v", node.Status.Addresses) + } + + nodeIPs := []net.IP{allIPs[0]} + if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { + for _, ip := range allIPs { + if utilnet.IsIPv6(ip) != utilnet.IsIPv6(nodeIPs[0]) { + nodeIPs = append(nodeIPs, ip) + break + } + } + } + + return nodeIPs, nil +} + +// GetNodeHostIP returns the provided node's "primary" IP; see GetNodeHostIPs for more details func GetNodeHostIP(node *v1.Node) (net.IP, error) { - addresses := node.Status.Addresses - addressMap := make(map[v1.NodeAddressType][]v1.NodeAddress) - for i := range addresses { - addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i]) + ips, err := GetNodeHostIPs(node) + if err != nil { + return nil, err } - if addresses, ok := addressMap[v1.NodeInternalIP]; ok { - return net.ParseIP(addresses[0].Address), nil - } - if addresses, ok := addressMap[v1.NodeExternalIP]; ok { - return net.ParseIP(addresses[0].Address), nil - } - return nil, fmt.Errorf("host IP unknown; known addresses: %v", addresses) + // GetNodeHostIPs always returns at least one IP if it didn't return an error + return ips[0], nil } // GetNodeIP returns an IP (as with GetNodeHostIP) for the node with the provided name. diff --git a/pkg/util/node/node_test.go b/pkg/util/node/node_test.go index c02a8679a85..a4e31c19033 100644 --- a/pkg/util/node/node_test.go +++ b/pkg/util/node/node_test.go @@ -17,10 +17,15 @@ limitations under the License. package node import ( + "net" + "reflect" "testing" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/features" ) func TestGetPreferredAddress(t *testing.T) { @@ -89,6 +94,147 @@ func TestGetPreferredAddress(t *testing.T) { } } +func TestGetNodeHostIPs(t *testing.T) { + testcases := []struct { + name string + addresses []v1.NodeAddress + dualStack bool + + expectIPs []net.IP + }{ + { + name: "no addresses", + expectIPs: nil, + }, + { + name: "no InternalIP/ExternalIP", + addresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "example.com"}, + }, + expectIPs: nil, + }, + { + name: "IPv4-only, simple", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + }, + expectIPs: []net.IP{net.ParseIP("1.2.3.4")}, + }, + { + name: "IPv4-only, external-first", + addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + {Type: v1.NodeInternalIP, Address: "1.2.3.4"}, + }, + expectIPs: []net.IP{net.ParseIP("1.2.3.4")}, + }, + { + name: "IPv4-only, no internal", + addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + }, + expectIPs: []net.IP{net.ParseIP("4.3.2.1")}, + }, + { + name: "dual-stack node, single-stack cluster", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + {Type: v1.NodeInternalIP, Address: "a:b::c:d"}, + {Type: v1.NodeExternalIP, Address: "d:c::b:a"}, + }, + expectIPs: []net.IP{net.ParseIP("1.2.3.4")}, + }, + { + name: "dual-stack node, dual-stack cluster", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + {Type: v1.NodeInternalIP, Address: "a:b::c:d"}, + {Type: v1.NodeExternalIP, Address: "d:c::b:a"}, + }, + dualStack: true, + expectIPs: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("a:b::c:d")}, + }, + { + name: "dual-stack node, different order, single-stack cluster", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeInternalIP, Address: "a:b::c:d"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + {Type: v1.NodeExternalIP, Address: "d:c::b:a"}, + }, + expectIPs: []net.IP{net.ParseIP("1.2.3.4")}, + }, + { + name: "dual-stack node, different order, dual-stack cluster", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeInternalIP, Address: "a:b::c:d"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + {Type: v1.NodeExternalIP, Address: "d:c::b:a"}, + }, + dualStack: true, + expectIPs: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("a:b::c:d")}, + }, + { + name: "dual-stack node, IPv6-first, no internal IPv4, single-stack cluster", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "a:b::c:d"}, + {Type: v1.NodeExternalIP, Address: "d:c::b:a"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + }, + expectIPs: []net.IP{net.ParseIP("a:b::c:d")}, + }, + { + name: "dual-stack node, IPv6-first, no internal IPv4, dual-stack cluster", + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "a:b::c:d"}, + {Type: v1.NodeExternalIP, Address: "d:c::b:a"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.1"}, + {Type: v1.NodeExternalIP, Address: "4.3.2.2"}, + }, + dualStack: true, + expectIPs: []net.IP{net.ParseIP("a:b::c:d"), net.ParseIP("4.3.2.1")}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.dualStack)() + node := &v1.Node{ + Status: v1.NodeStatus{Addresses: tc.addresses}, + } + nodeIPs, err := GetNodeHostIPs(node) + nodeIP, err2 := GetNodeHostIP(node) + + if (err == nil && err2 != nil) || (err != nil && err2 == nil) { + t.Errorf("GetNodeHostIPs() returned error=%q but GetNodeHostIP() returned error=%q", err, err2) + } + if err != nil { + if tc.expectIPs != nil { + t.Errorf("expected %v, got error (%v)", tc.expectIPs, err) + } + } else if tc.expectIPs == nil { + t.Errorf("expected error, got %v", nodeIPs) + } else if !reflect.DeepEqual(nodeIPs, tc.expectIPs) { + t.Errorf("expected %v, got %v", tc.expectIPs, nodeIPs) + } else if !nodeIP.Equal(nodeIPs[0]) { + t.Errorf("GetNodeHostIP did not return same primary (%s) as GetNodeHostIPs (%s)", nodeIP.String(), nodeIPs[0].String()) + } + }) + } +} + func TestGetHostname(t *testing.T) { testCases := []struct { hostName string