From 0f1f1711fe82961f45674e1f551db8c7ae8ab545 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Tue, 14 Feb 2023 10:19:47 -0500 Subject: [PATCH 1/6] Move kubelet `--node-ip` parsing to component-helpers The same code should be used to parse the command-line argument and the annotation. Unfortunately, for compatiblity reasons, they have to handle invalid inputs differently... (It doesn't make sense to put this code in cloud-provider, since ParseNodeIPArgument is used for the non-cloud-provider case too.) --- cmd/kubelet/app/server.go | 21 +- .../controllers/node/node_controller.go | 7 +- .../k8s.io/component-helpers/node/util/ips.go | 81 +++++ .../component-helpers/node/util/ips_test.go | 330 ++++++++++++++++++ 4 files changed, 417 insertions(+), 22 deletions(-) create mode 100644 staging/src/k8s.io/component-helpers/node/util/ips.go create mode 100644 staging/src/k8s.io/component-helpers/node/util/ips_test.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 68af6aaba7b..b755d4e5bfc 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1120,24 +1120,9 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie // Setup event recorder if required. makeEventRecorder(kubeDeps, nodeName) - var nodeIPs []net.IP - if kubeServer.NodeIP != "" { - for _, ip := range strings.Split(kubeServer.NodeIP, ",") { - parsedNodeIP := netutils.ParseIPSloppy(strings.TrimSpace(ip)) - if parsedNodeIP == nil { - klog.InfoS("Could not parse --node-ip ignoring", "IP", ip) - } else { - nodeIPs = append(nodeIPs, parsedNodeIP) - } - } - } - - if len(nodeIPs) > 2 || (len(nodeIPs) == 2 && netutils.IsIPv6(nodeIPs[0]) == netutils.IsIPv6(nodeIPs[1])) { - return fmt.Errorf("bad --node-ip %q; must contain either a single IP or a dual-stack pair of IPs", kubeServer.NodeIP) - } else if len(nodeIPs) == 2 && kubeServer.CloudProvider != "" { - return fmt.Errorf("dual-stack --node-ip %q not supported when using a cloud provider", kubeServer.NodeIP) - } else if len(nodeIPs) == 2 && (nodeIPs[0].IsUnspecified() || nodeIPs[1].IsUnspecified()) { - return fmt.Errorf("dual-stack --node-ip %q cannot include '0.0.0.0' or '::'", kubeServer.NodeIP) + nodeIPs, err := nodeutil.ParseNodeIPArgument(kubeServer.NodeIP, kubeServer.CloudProvider) + if err != nil { + return fmt.Errorf("bad --node-ip %q: %v", kubeServer.NodeIP, err) } capabilities.Initialize(capabilities.Capabilities{ diff --git a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go index 878ca1194a6..e77b45e89d6 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go +++ b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go @@ -45,7 +45,6 @@ import ( controllersmetrics "k8s.io/component-base/metrics/prometheus/controllers" nodeutil "k8s.io/component-helpers/node/util" "k8s.io/klog/v2" - netutils "k8s.io/utils/net" ) // labelReconcileInfo lists Node labels to reconcile, and how to reconcile them. @@ -756,9 +755,9 @@ func getNodeProvidedIP(node *v1.Node) (net.IP, error) { return nil, nil } - nodeIP := netutils.ParseIPSloppy(providedIP) - if nodeIP == nil { - return nil, fmt.Errorf("failed to parse node IP %q for node %q", providedIP, node.Name) + nodeIP, err := nodeutil.ParseNodeIPAnnotation(providedIP) + if err != nil { + return nil, fmt.Errorf("failed to parse node IP %q for node %q: %v", providedIP, node.Name, err) } return nodeIP, nil diff --git a/staging/src/k8s.io/component-helpers/node/util/ips.go b/staging/src/k8s.io/component-helpers/node/util/ips.go new file mode 100644 index 00000000000..d0edd8c0532 --- /dev/null +++ b/staging/src/k8s.io/component-helpers/node/util/ips.go @@ -0,0 +1,81 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "strings" + + "k8s.io/klog/v2" + netutils "k8s.io/utils/net" +) + +const ( + cloudProviderNone = "" + cloudProviderExternal = "external" +) + +// parseNodeIP implements ParseNodeIPArgument and ParseNodeIPAnnotation +func parseNodeIP(nodeIP string, allowDual, sloppy bool) ([]net.IP, error) { + var nodeIPs []net.IP + if nodeIP != "" || !sloppy { + for _, ip := range strings.Split(nodeIP, ",") { + if sloppy { + ip = strings.TrimSpace(ip) + } + parsedNodeIP := netutils.ParseIPSloppy(ip) + if parsedNodeIP == nil { + if sloppy { + klog.InfoS("Could not parse node IP. Ignoring", "IP", ip) + } else { + return nil, fmt.Errorf("could not parse %q", ip) + } + } else { + nodeIPs = append(nodeIPs, parsedNodeIP) + } + } + } + + if len(nodeIPs) > 2 || (len(nodeIPs) == 2 && netutils.IsIPv6(nodeIPs[0]) == netutils.IsIPv6(nodeIPs[1])) { + return nil, fmt.Errorf("must contain either a single IP or a dual-stack pair of IPs") + } else if len(nodeIPs) == 2 && !allowDual { + return nil, fmt.Errorf("dual-stack not supported in this configuration") + } else if len(nodeIPs) == 2 && (nodeIPs[0].IsUnspecified() || nodeIPs[1].IsUnspecified()) { + return nil, fmt.Errorf("dual-stack node IP cannot include '0.0.0.0' or '::'") + } + + return nodeIPs, nil +} + +// ParseNodeIPArgument parses kubelet's --node-ip argument. If nodeIP contains invalid +// values, they will be logged and ignored. Dual-stack node IPs are only supported if +// cloudProvider is unset. +func ParseNodeIPArgument(nodeIP, cloudProvider string) ([]net.IP, error) { + return parseNodeIP(nodeIP, cloudProvider == cloudProviderNone, true) +} + +// ParseNodeIPAnnotation parses the `alpha.kubernetes.io/provided-node-ip` annotation, +// which should be a single IP address. Unlike with ParseNodeIPArgument, invalid values +// are considered an error. +func ParseNodeIPAnnotation(nodeIP string) (net.IP, error) { + ips, err := parseNodeIP(nodeIP, false, false) + if err != nil || len(ips) == 0 { + return nil, err + } + return ips[0], nil +} diff --git a/staging/src/k8s.io/component-helpers/node/util/ips_test.go b/staging/src/k8s.io/component-helpers/node/util/ips_test.go new file mode 100644 index 00000000000..250aae91c59 --- /dev/null +++ b/staging/src/k8s.io/component-helpers/node/util/ips_test.go @@ -0,0 +1,330 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "reflect" + "strings" + "testing" + + netutils "k8s.io/utils/net" +) + +func TestParseNodeIPArgument(t *testing.T) { + testCases := []struct { + desc string + in string + out []net.IP + err string + ssErr string + }{ + { + desc: "empty --node-ip", + in: "", + out: nil, + }, + { + desc: "just whitespace (ignored)", + in: " ", + out: nil, + }, + { + desc: "garbage (ignored)", + in: "blah", + out: nil, + }, + { + desc: "single IPv4", + in: "1.2.3.4", + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + }, + }, + { + desc: "single IPv4 with whitespace", + in: " 1.2.3.4 ", + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + }, + }, + { + desc: "single IPv4 non-canonical", + in: "01.2.3.004", + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + }, + }, + { + desc: "single IPv4 invalid (ignored)", + in: "1.2.3", + out: nil, + }, + { + desc: "single IPv4 CIDR (ignored)", + in: "1.2.3.0/24", + out: nil, + }, + { + desc: "single IPv4 unspecified", + in: "0.0.0.0", + out: []net.IP{ + net.IPv4zero, + }, + }, + { + desc: "single IPv4 plus ignored garbage", + in: "1.2.3.4,not-an-IPv6-address", + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + }, + }, + { + desc: "single IPv6", + in: "abcd::ef01", + out: []net.IP{ + netutils.ParseIPSloppy("abcd::ef01"), + }, + }, + { + desc: "single IPv6 non-canonical", + in: "abcd:0abc:00ab:0000:0000::1", + out: []net.IP{ + netutils.ParseIPSloppy("abcd:abc:ab::1"), + }, + }, + { + desc: "simple dual-stack", + in: "1.2.3.4,abcd::ef01", + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + netutils.ParseIPSloppy("abcd::ef01"), + }, + ssErr: "not supported in this configuration", + }, + { + desc: "dual-stack with whitespace", + in: "abcd::ef01 , 1.2.3.4", + out: []net.IP{ + netutils.ParseIPSloppy("abcd::ef01"), + netutils.ParseIPSloppy("1.2.3.4"), + }, + ssErr: "not supported in this configuration", + }, + { + desc: "double IPv4", + in: "1.2.3.4,5.6.7.8", + err: "either a single IP or a dual-stack pair of IPs", + }, + { + desc: "double IPv6", + in: "abcd::1,abcd::2", + err: "either a single IP or a dual-stack pair of IPs", + }, + { + desc: "dual-stack with unspecified", + in: "1.2.3.4,::", + err: "cannot include '0.0.0.0' or '::'", + ssErr: "not supported in this configuration", + }, + { + desc: "dual-stack with unspecified", + in: "0.0.0.0,abcd::1", + err: "cannot include '0.0.0.0' or '::'", + ssErr: "not supported in this configuration", + }, + { + desc: "dual-stack plus ignored garbage", + in: "abcd::ef01 , 1.2.3.4, something else", + out: []net.IP{ + netutils.ParseIPSloppy("abcd::ef01"), + netutils.ParseIPSloppy("1.2.3.4"), + }, + ssErr: "not supported in this configuration", + }, + { + desc: "triple stack!", + in: "1.2.3.4,abcd::1,5.6.7.8", + err: "either a single IP or a dual-stack pair of IPs", + }, + } + + for _, tc := range testCases { + for _, cloudProvider := range []string{cloudProviderNone, cloudProviderExternal, "gce"} { + desc := fmt.Sprintf("%s, cloudProvider=%q", tc.desc, cloudProvider) + t.Run(desc, func(t *testing.T) { + parsed, err := ParseNodeIPArgument(tc.in, cloudProvider) + + expectedOut := tc.out + expectedErr := tc.err + + // Dual-stack is only supported with no cloudProvider + if cloudProvider != "" { + if len(tc.out) == 2 { + expectedOut = nil + } + if tc.ssErr != "" { + expectedErr = tc.ssErr + } + } + + if !reflect.DeepEqual(parsed, expectedOut) { + t.Errorf("expected %#v, got %#v", expectedOut, parsed) + } + if err != nil { + if expectedErr == "" { + t.Errorf("unexpected error %v", err) + } else if !strings.Contains(err.Error(), expectedErr) { + t.Errorf("expected error with %q, got %v", expectedErr, err) + } + } else if expectedErr != "" { + t.Errorf("expected error with %q, got no error", expectedErr) + } + }) + } + } +} + +func TestParseNodeIPAnnotation(t *testing.T) { + testCases := []struct { + desc string + in string + out net.IP + err string + }{ + { + desc: "empty --node-ip", + in: "", + err: "could not parse", + }, + { + desc: "just whitespace", + in: " ", + err: "could not parse", + }, + { + desc: "garbage", + in: "blah", + err: "could not parse", + }, + { + desc: "single IPv4", + in: "1.2.3.4", + out: netutils.ParseIPSloppy("1.2.3.4"), + }, + { + desc: "single IPv4 with whitespace", + in: " 1.2.3.4 ", + err: "could not parse", + }, + { + desc: "single IPv4 non-canonical", + in: "01.2.3.004", + out: netutils.ParseIPSloppy("1.2.3.4"), + }, + { + desc: "single IPv4 invalid", + in: "1.2.3", + err: "could not parse", + }, + { + desc: "single IPv4 CIDR", + in: "1.2.3.0/24", + err: "could not parse", + }, + { + desc: "single IPv4 unspecified", + in: "0.0.0.0", + out: net.IPv4zero, + }, + { + desc: "single IPv4 plus garbage", + in: "1.2.3.4,not-an-IPv6-address", + err: "could not parse", + }, + { + desc: "single IPv6", + in: "abcd::ef01", + out: netutils.ParseIPSloppy("abcd::ef01"), + }, + { + desc: "single IPv6 non-canonical", + in: "abcd:0abc:00ab:0000:0000::1", + out: netutils.ParseIPSloppy("abcd:abc:ab::1"), + }, + { + desc: "simple dual-stack", + in: "1.2.3.4,abcd::ef01", + err: "not supported in this configuration", + }, + { + desc: "dual-stack with whitespace", + in: "abcd::ef01 , 1.2.3.4", + err: "could not parse", + }, + { + desc: "double IPv4", + in: "1.2.3.4,5.6.7.8", + err: "either a single IP or a dual-stack pair of IPs", + }, + { + desc: "double IPv6", + in: "abcd::1,abcd::2", + err: "either a single IP or a dual-stack pair of IPs", + }, + { + desc: "dual-stack with unspecified", + in: "1.2.3.4,::", + err: "not supported in this configuration", + }, + { + desc: "dual-stack with unspecified", + in: "0.0.0.0,abcd::1", + err: "not supported in this configuration", + }, + { + desc: "dual-stack plus garbage", + in: "abcd::ef01 , 1.2.3.4, something else", + err: "could not parse", + }, + { + desc: "triple stack!", + in: "1.2.3.4,abcd::1,5.6.7.8", + err: "either a single IP or a dual-stack pair of IPs", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + parsed, err := ParseNodeIPAnnotation(tc.in) + + if !reflect.DeepEqual(parsed, tc.out) { + t.Errorf("expected %#v, got %#v", tc.out, parsed) + } + if err != nil { + if tc.err == "" { + t.Errorf("unexpected error %v", err) + } else if !strings.Contains(err.Error(), tc.err) { + t.Errorf("expected error with %q, got %v", tc.err, err) + } + } else if tc.err != "" { + t.Errorf("expected error with %q, got no error", tc.err) + } + }) + } +} From 77e0fbe77448d5bde5f9ef8b62a8e6c041fd544e Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 3 Mar 2023 11:29:31 -0500 Subject: [PATCH 2/6] Update ParseNodeIPArgument for cloud dual-stack Add an arg to ParseNodeIPArgument saying whether to allow dual-stack IPs for external cloud providers. Update kubelet for the new API, but always pass "false" for now. --- cmd/kubelet/app/server.go | 2 +- .../k8s.io/component-helpers/node/util/ips.go | 12 ++++++---- .../component-helpers/node/util/ips_test.go | 22 ++++++++++++++----- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index b755d4e5bfc..fd4aa687e3b 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1120,7 +1120,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie // Setup event recorder if required. makeEventRecorder(kubeDeps, nodeName) - nodeIPs, err := nodeutil.ParseNodeIPArgument(kubeServer.NodeIP, kubeServer.CloudProvider) + nodeIPs, err := nodeutil.ParseNodeIPArgument(kubeServer.NodeIP, kubeServer.CloudProvider, false) if err != nil { return fmt.Errorf("bad --node-ip %q: %v", kubeServer.NodeIP, err) } diff --git a/staging/src/k8s.io/component-helpers/node/util/ips.go b/staging/src/k8s.io/component-helpers/node/util/ips.go index d0edd8c0532..60d3097cc88 100644 --- a/staging/src/k8s.io/component-helpers/node/util/ips.go +++ b/staging/src/k8s.io/component-helpers/node/util/ips.go @@ -63,10 +63,14 @@ func parseNodeIP(nodeIP string, allowDual, sloppy bool) ([]net.IP, error) { } // ParseNodeIPArgument parses kubelet's --node-ip argument. If nodeIP contains invalid -// values, they will be logged and ignored. Dual-stack node IPs are only supported if -// cloudProvider is unset. -func ParseNodeIPArgument(nodeIP, cloudProvider string) ([]net.IP, error) { - return parseNodeIP(nodeIP, cloudProvider == cloudProviderNone, true) +// values, they will be logged and ignored. Dual-stack node IPs are allowed if +// cloudProvider is unset, or if it is `"external"` and allowCloudDualStack is true. +func ParseNodeIPArgument(nodeIP, cloudProvider string, allowCloudDualStack bool) ([]net.IP, error) { + var allowDualStack bool + if (cloudProvider == cloudProviderNone) || (cloudProvider == cloudProviderExternal && allowCloudDualStack) { + allowDualStack = true + } + return parseNodeIP(nodeIP, allowDualStack, true) } // ParseNodeIPAnnotation parses the `alpha.kubernetes.io/provided-node-ip` annotation, diff --git a/staging/src/k8s.io/component-helpers/node/util/ips_test.go b/staging/src/k8s.io/component-helpers/node/util/ips_test.go index 250aae91c59..8afb03f5cf2 100644 --- a/staging/src/k8s.io/component-helpers/node/util/ips_test.go +++ b/staging/src/k8s.io/component-helpers/node/util/ips_test.go @@ -164,17 +164,29 @@ func TestParseNodeIPArgument(t *testing.T) { }, } + configurations := []struct { + cloudProvider string + allowCloudDualStack bool + dualStackSupported bool + }{ + {cloudProviderNone, false, true}, + {cloudProviderNone, true, true}, + {cloudProviderExternal, false, false}, + {cloudProviderExternal, true, true}, + {"gce", false, false}, + {"gce", true, false}, + } + for _, tc := range testCases { - for _, cloudProvider := range []string{cloudProviderNone, cloudProviderExternal, "gce"} { - desc := fmt.Sprintf("%s, cloudProvider=%q", tc.desc, cloudProvider) + for _, conf := range configurations { + desc := fmt.Sprintf("%s, cloudProvider=%q, allowCloudDualStack=%v", tc.desc, conf.cloudProvider, conf.allowCloudDualStack) t.Run(desc, func(t *testing.T) { - parsed, err := ParseNodeIPArgument(tc.in, cloudProvider) + parsed, err := ParseNodeIPArgument(tc.in, conf.cloudProvider, conf.allowCloudDualStack) expectedOut := tc.out expectedErr := tc.err - // Dual-stack is only supported with no cloudProvider - if cloudProvider != "" { + if !conf.dualStackSupported { if len(tc.out) == 2 { expectedOut = nil } From 7605163620c9bd2f0e5fc6d525d13ba9aec3d657 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sun, 5 Mar 2023 15:57:51 -0500 Subject: [PATCH 3/6] Split up PreferNodeIP into legacy and non-legacy versions Though not obvious as currently written, PreferNodeIP() has different semantics with legacy and external cloud providers, since one kind of node IP value never gets passed in the external cloud provider case. Split it into two functions to make this clearer (and to prepare for adding new external-cloud-only semantics, and to make it clearer that some of the code can be deleted when legacy cloud providers go away). --- pkg/kubelet/nodestatus/setters.go | 2 +- .../controllers/node/node_controller.go | 4 +- .../cloud-provider/node/helpers/address.go | 29 ++- .../node/helpers/address_test.go | 167 +++++++++++++++++- 4 files changed, 192 insertions(+), 10 deletions(-) diff --git a/pkg/kubelet/nodestatus/setters.go b/pkg/kubelet/nodestatus/setters.go index 687e8212403..e6460f14d9f 100644 --- a/pkg/kubelet/nodestatus/setters.go +++ b/pkg/kubelet/nodestatus/setters.go @@ -131,7 +131,7 @@ func NodeAddress(nodeIPs []net.IP, // typically Kubelet.nodeIPs return err } - nodeAddresses, err := cloudprovidernodeutil.PreferNodeIP(nodeIP, cloudNodeAddresses) + nodeAddresses, err := cloudprovidernodeutil.GetNodeAddressesFromNodeIPLegacy(nodeIP, cloudNodeAddresses) if err != nil { return err } diff --git a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go index e77b45e89d6..19b02b99c1b 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go +++ b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go @@ -391,7 +391,7 @@ func (cnc *CloudNodeController) updateNodeAddress(ctx context.Context, node *v1. } if nodeIP != nil { - nodeAddresses, err = cloudnodeutil.PreferNodeIP(nodeIP, nodeAddresses) + nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(nodeIP, nodeAddresses) if err != nil { klog.Errorf("Failed to update node addresses for node %q: %v", node.Name, err) return @@ -539,7 +539,7 @@ func (cnc *CloudNodeController) getNodeModifiersFromCloudProvider( } if nodeIP != nil { - _, err := cloudnodeutil.PreferNodeIP(nodeIP, instanceMeta.NodeAddresses) + _, err := cloudnodeutil.GetNodeAddressesFromNodeIP(nodeIP, instanceMeta.NodeAddresses) if err != nil { return nil, fmt.Errorf("provided node ip for node %q is not valid: %w", node.Name, err) } diff --git a/staging/src/k8s.io/cloud-provider/node/helpers/address.go b/staging/src/k8s.io/cloud-provider/node/helpers/address.go index 6eb44a90feb..533de163ac8 100644 --- a/staging/src/k8s.io/cloud-provider/node/helpers/address.go +++ b/staging/src/k8s.io/cloud-provider/node/helpers/address.go @@ -41,8 +41,8 @@ func AddToNodeAddresses(addresses *[]v1.NodeAddress, addAddresses ...v1.NodeAddr } } -// PreferNodeIP filters node addresses to prefer a specific node IP or address -// family. +// GetNodeAddressesFromNodeIPLegacy filters node addresses to prefer a specific node IP or +// address family. This function is used only with legacy cloud providers. // // If nodeIP is either '0.0.0.0' or '::' it is taken to represent any address of // that address family: IPv4 or IPv6. i.e. if nodeIP is '0.0.0.0' we will return @@ -55,7 +55,7 @@ func AddToNodeAddresses(addresses *[]v1.NodeAddress, addAddresses ...v1.NodeAddr // - If nodeIP matches an address of a particular type (internal or external), // that will be the *only* address of that type returned. // - All remaining addresses are listed after. -func PreferNodeIP(nodeIP net.IP, cloudNodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { +func GetNodeAddressesFromNodeIPLegacy(nodeIP net.IP, cloudNodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { // If nodeIP is unset, just use the addresses provided by the cloud provider as-is if nodeIP == nil { return cloudNodeAddresses, nil @@ -83,6 +83,29 @@ func PreferNodeIP(nodeIP net.IP, cloudNodeAddresses []v1.NodeAddress) ([]v1.Node return sortedAddresses, nil } + // Otherwise the result is the same as for GetNodeAddressesFromNodeIP + return GetNodeAddressesFromNodeIP(nodeIP, cloudNodeAddresses) +} + +// GetNodeAddressesFromNodeIP filters the provided list of nodeAddresses to +// match the provided nodeIP. This is used for external cloud providers. +// +// It will return node addresses filtered such that: +// - Any address matching nodeIP will be listed first. +// - If nodeIP matches an address of a particular type (internal or external), +// that will be the *only* address of that type returned. +// - All remaining addresses are listed after. +// +// (This does not have the same behavior with `0.0.0.0` and `::` as +// GetNodeAddressesFromNodeIPLegacy, because that case never occurs for external cloud +// providers, because kubelet does not set the `provided-node-ip` annotation in that +// case.) +func GetNodeAddressesFromNodeIP(nodeIP net.IP, cloudNodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { + // If nodeIP is unset, just use the addresses provided by the cloud provider as-is + if nodeIP == nil { + return cloudNodeAddresses, nil + } + // For every address supplied by the cloud provider that matches nodeIP, nodeIP is the enforced node address for // that address Type (like InternalIP and ExternalIP), meaning other addresses of the same Type are discarded. // See #61921 for more information: some cloud providers may supply secondary IPs, so nodeIP serves as a way to diff --git a/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go b/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go index eccb4a75b81..f5d2c5cad02 100644 --- a/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go +++ b/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go @@ -94,7 +94,7 @@ func TestAddToNodeAddresses(t *testing.T) { } } -func TestPreferNodeIP(t *testing.T) { +func TestGetNodeAddressesFromNodeIPLegacy(t *testing.T) { cases := []struct { name string nodeIP net.IP @@ -301,13 +301,172 @@ func TestPreferNodeIP(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - got, err := PreferNodeIP(tt.nodeIP, tt.nodeAddresses) + got, err := GetNodeAddressesFromNodeIPLegacy(tt.nodeIP, tt.nodeAddresses) if (err != nil) != tt.shouldError { - t.Errorf("PreferNodeIP() error = %v, wantErr %v", err, tt.shouldError) + t.Errorf("GetNodeAddressesFromNodeIPLegacy() error = %v, wantErr %v", err, tt.shouldError) return } if !reflect.DeepEqual(got, tt.expectedAddresses) { - t.Errorf("PreferNodeIP() = %v, want %v", got, tt.expectedAddresses) + t.Errorf("GetNodeAddressesFromNodeIPLegacy() = %v, want %v", got, tt.expectedAddresses) + } + }) + } +} + +func TestGetNodeAddressesFromNodeIP(t *testing.T) { + cases := []struct { + name string + nodeIP net.IP + nodeAddresses []v1.NodeAddress + expectedAddresses []v1.NodeAddress + shouldError bool + }{ + { + name: "A single InternalIP", + nodeIP: netutils.ParseIPSloppy("10.1.1.1"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "NodeIP is external", + nodeIP: netutils.ParseIPSloppy("55.55.55.55"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + // Accommodating #45201 and #49202 + name: "InternalIP and ExternalIP are the same", + nodeIP: netutils.ParseIPSloppy("55.55.55.55"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "44.44.44.44"}, + {Type: v1.NodeExternalIP, Address: "44.44.44.44"}, + {Type: v1.NodeInternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "An Internal/ExternalIP, an Internal/ExternalDNS", + nodeIP: netutils.ParseIPSloppy("10.1.1.1"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalDNS, Address: "ip-10-1-1-1.us-west-2.compute.internal"}, + {Type: v1.NodeExternalDNS, Address: "ec2-55-55-55-55.us-west-2.compute.amazonaws.com"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalDNS, Address: "ip-10-1-1-1.us-west-2.compute.internal"}, + {Type: v1.NodeExternalDNS, Address: "ec2-55-55-55-55.us-west-2.compute.amazonaws.com"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "An Internal with multiple internal IPs", + nodeIP: netutils.ParseIPSloppy("10.1.1.1"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "10.2.2.2"}, + {Type: v1.NodeInternalIP, Address: "10.3.3.3"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "An InternalIP that isn't valid: should error", + nodeIP: netutils.ParseIPSloppy("10.2.2.2"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: nil, + shouldError: true, + }, + { + name: "Dual-stack cloud, with nodeIP, different IPv6 formats", + nodeIP: netutils.ParseIPSloppy("2600:1f14:1d4:d101::ba3d"), + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "2600:1f14:1d4:d101:0:0:0:ba3d"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "2600:1f14:1d4:d101:0:0:0:ba3d"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, IPv4 first, no nodeIP", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, IPv6 first, no nodeIP", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got, err := GetNodeAddressesFromNodeIP(tt.nodeIP, tt.nodeAddresses) + if (err != nil) != tt.shouldError { + t.Errorf("GetNodeAddressesFromNodeIP() error = %v, wantErr %v", err, tt.shouldError) + return + } + if !reflect.DeepEqual(got, tt.expectedAddresses) { + t.Errorf("GetNodeAddressesFromNodeIP() = %v, want %v", got, tt.expectedAddresses) } }) } From d6a11b71386832efb8d7315ac347b87eb258b771 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Mon, 6 Mar 2023 09:02:19 -0500 Subject: [PATCH 4/6] Update GetNodeAddressesFromNodeIP to take the unparsed annotation And simplify the callers in node_controller.go to merge the common code. --- .../controllers/node/node_controller.go | 40 +++++------------ .../cloud-provider/node/helpers/address.go | 16 ++++--- .../node/helpers/address_test.go | 44 ++++--------------- .../src/k8s.io/legacy-cloud-providers/go.mod | 1 + 4 files changed, 29 insertions(+), 72 deletions(-) diff --git a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go index 19b02b99c1b..c9aea6ff50c 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go +++ b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "net" "time" v1 "k8s.io/api/core/v1" @@ -384,19 +383,12 @@ func (cnc *CloudNodeController) updateNodeAddress(ctx context.Context, node *v1. } } // If kubelet provided a node IP, prefer it in the node address list - nodeIP, err := getNodeProvidedIP(node) + nodeAddresses, err := updateNodeAddressesFromNodeIP(node, nodeAddresses) if err != nil { - klog.Errorf("Failed to get preferred node IP for node %q: %v", node.Name, err) + klog.Errorf("Failed to update node addresses for node %q: %v", node.Name, err) return } - if nodeIP != nil { - nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(nodeIP, nodeAddresses) - if err != nil { - klog.Errorf("Failed to update node addresses for node %q: %v", node.Name, err) - return - } - } if !nodeAddressesChangeDetected(node.Status.Addresses, nodeAddresses) { return } @@ -533,16 +525,9 @@ func (cnc *CloudNodeController) getNodeModifiersFromCloudProvider( // If kubelet annotated the node with a node IP, ensure that it is valid // and can be applied to the discovered node addresses before removing // the taint on the node. - nodeIP, err := getNodeProvidedIP(node) + _, err := updateNodeAddressesFromNodeIP(node, instanceMeta.NodeAddresses) if err != nil { - return nil, err - } - - if nodeIP != nil { - _, err := cloudnodeutil.GetNodeAddressesFromNodeIP(nodeIP, instanceMeta.NodeAddresses) - if err != nil { - return nil, fmt.Errorf("provided node ip for node %q is not valid: %w", node.Name, err) - } + return nil, fmt.Errorf("provided node ip for node %q is not valid: %w", node.Name, err) } if instanceMeta.InstanceType != "" { @@ -749,18 +734,15 @@ func nodeAddressesChangeDetected(addressSet1, addressSet2 []v1.NodeAddress) bool return false } -func getNodeProvidedIP(node *v1.Node) (net.IP, error) { - providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr] - if !ok { - return nil, nil +func updateNodeAddressesFromNodeIP(node *v1.Node, nodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { + var err error + + providedNodeIP, exists := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr] + if exists { + nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(providedNodeIP, nodeAddresses) } - nodeIP, err := nodeutil.ParseNodeIPAnnotation(providedIP) - if err != nil { - return nil, fmt.Errorf("failed to parse node IP %q for node %q: %v", providedIP, node.Name, err) - } - - return nodeIP, nil + return nodeAddresses, err } // getInstanceTypeByProviderIDOrName will attempt to get the instance type of node using its providerID diff --git a/staging/src/k8s.io/cloud-provider/node/helpers/address.go b/staging/src/k8s.io/cloud-provider/node/helpers/address.go index 533de163ac8..627167799aa 100644 --- a/staging/src/k8s.io/cloud-provider/node/helpers/address.go +++ b/staging/src/k8s.io/cloud-provider/node/helpers/address.go @@ -21,6 +21,7 @@ import ( "net" "k8s.io/api/core/v1" + nodeutil "k8s.io/component-helpers/node/util" netutils "k8s.io/utils/net" ) @@ -84,11 +85,12 @@ func GetNodeAddressesFromNodeIPLegacy(nodeIP net.IP, cloudNodeAddresses []v1.Nod } // Otherwise the result is the same as for GetNodeAddressesFromNodeIP - return GetNodeAddressesFromNodeIP(nodeIP, cloudNodeAddresses) + return GetNodeAddressesFromNodeIP(nodeIP.String(), cloudNodeAddresses) } -// GetNodeAddressesFromNodeIP filters the provided list of nodeAddresses to -// match the provided nodeIP. This is used for external cloud providers. +// GetNodeAddressesFromNodeIP filters the provided list of nodeAddresses to match the +// providedNodeIP from the Node annotation (which is assumed to be non-empty). This is +// used for external cloud providers. // // It will return node addresses filtered such that: // - Any address matching nodeIP will be listed first. @@ -100,10 +102,10 @@ func GetNodeAddressesFromNodeIPLegacy(nodeIP net.IP, cloudNodeAddresses []v1.Nod // GetNodeAddressesFromNodeIPLegacy, because that case never occurs for external cloud // providers, because kubelet does not set the `provided-node-ip` annotation in that // case.) -func GetNodeAddressesFromNodeIP(nodeIP net.IP, cloudNodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { - // If nodeIP is unset, just use the addresses provided by the cloud provider as-is - if nodeIP == nil { - return cloudNodeAddresses, nil +func GetNodeAddressesFromNodeIP(providedNodeIP string, cloudNodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { + nodeIP, err := nodeutil.ParseNodeIPAnnotation(providedNodeIP) + if err != nil { + return nil, fmt.Errorf("failed to parse node IP %q: %v", providedNodeIP, err) } // For every address supplied by the cloud provider that matches nodeIP, nodeIP is the enforced node address for diff --git a/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go b/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go index f5d2c5cad02..6007bedf206 100644 --- a/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go +++ b/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go @@ -316,14 +316,14 @@ func TestGetNodeAddressesFromNodeIPLegacy(t *testing.T) { func TestGetNodeAddressesFromNodeIP(t *testing.T) { cases := []struct { name string - nodeIP net.IP + nodeIP string nodeAddresses []v1.NodeAddress expectedAddresses []v1.NodeAddress shouldError bool }{ { name: "A single InternalIP", - nodeIP: netutils.ParseIPSloppy("10.1.1.1"), + nodeIP: "10.1.1.1", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, {Type: v1.NodeHostName, Address: testKubeletHostname}, @@ -336,7 +336,7 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, { name: "NodeIP is external", - nodeIP: netutils.ParseIPSloppy("55.55.55.55"), + nodeIP: "55.55.55.55", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, @@ -352,7 +352,7 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { { // Accommodating #45201 and #49202 name: "InternalIP and ExternalIP are the same", - nodeIP: netutils.ParseIPSloppy("55.55.55.55"), + nodeIP: "55.55.55.55", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "44.44.44.44"}, {Type: v1.NodeExternalIP, Address: "44.44.44.44"}, @@ -369,7 +369,7 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, { name: "An Internal/ExternalIP, an Internal/ExternalDNS", - nodeIP: netutils.ParseIPSloppy("10.1.1.1"), + nodeIP: "10.1.1.1", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, @@ -388,7 +388,7 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, { name: "An Internal with multiple internal IPs", - nodeIP: netutils.ParseIPSloppy("10.1.1.1"), + nodeIP: "10.1.1.1", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, {Type: v1.NodeInternalIP, Address: "10.2.2.2"}, @@ -405,7 +405,7 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, { name: "An InternalIP that isn't valid: should error", - nodeIP: netutils.ParseIPSloppy("10.2.2.2"), + nodeIP: "10.2.2.2", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, @@ -416,7 +416,7 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, { name: "Dual-stack cloud, with nodeIP, different IPv6 formats", - nodeIP: netutils.ParseIPSloppy("2600:1f14:1d4:d101::ba3d"), + nodeIP: "2600:1f14:1d4:d101::ba3d", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, {Type: v1.NodeInternalIP, Address: "2600:1f14:1d4:d101:0:0:0:ba3d"}, @@ -428,34 +428,6 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, shouldError: false, }, - { - name: "Dual-stack cloud, IPv4 first, no nodeIP", - nodeAddresses: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, - {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, - {Type: v1.NodeHostName, Address: testKubeletHostname}, - }, - expectedAddresses: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, - {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, - {Type: v1.NodeHostName, Address: testKubeletHostname}, - }, - shouldError: false, - }, - { - name: "Dual-stack cloud, IPv6 first, no nodeIP", - nodeAddresses: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, - {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, - {Type: v1.NodeHostName, Address: testKubeletHostname}, - }, - expectedAddresses: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, - {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, - {Type: v1.NodeHostName, Address: testKubeletHostname}, - }, - shouldError: false, - }, } for _, tt := range cases { diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.mod b/staging/src/k8s.io/legacy-cloud-providers/go.mod index 023c21c920c..5db30abca9a 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.mod +++ b/staging/src/k8s.io/legacy-cloud-providers/go.mod @@ -85,6 +85,7 @@ require ( gopkg.in/warnings.v0 v0.1.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-helpers v0.0.0 // indirect k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect From e3aacb4f30f909907e1f0a9a699c08b5f327cc91 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sun, 5 Mar 2023 16:01:33 -0500 Subject: [PATCH 5/6] Update GetNodeAddressesFromNodeIP and ParseNodeIPAnnotation for cloud dual-stack Update callers for the new APIs, but don't change their behavior yet. --- .../controllers/node/node_controller.go | 2 +- .../cloud-provider/node/helpers/address.go | 44 ++++--- .../node/helpers/address_test.go | 122 +++++++++++++++++- .../k8s.io/component-helpers/node/util/ips.go | 11 +- .../component-helpers/node/util/ips_test.go | 92 ++++++++----- 5 files changed, 214 insertions(+), 57 deletions(-) diff --git a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go index c9aea6ff50c..24d45452c44 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go +++ b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go @@ -739,7 +739,7 @@ func updateNodeAddressesFromNodeIP(node *v1.Node, nodeAddresses []v1.NodeAddress providedNodeIP, exists := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr] if exists { - nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(providedNodeIP, nodeAddresses) + nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(providedNodeIP, nodeAddresses, false) } return nodeAddresses, err diff --git a/staging/src/k8s.io/cloud-provider/node/helpers/address.go b/staging/src/k8s.io/cloud-provider/node/helpers/address.go index 627167799aa..41112cdaeb3 100644 --- a/staging/src/k8s.io/cloud-provider/node/helpers/address.go +++ b/staging/src/k8s.io/cloud-provider/node/helpers/address.go @@ -85,7 +85,7 @@ func GetNodeAddressesFromNodeIPLegacy(nodeIP net.IP, cloudNodeAddresses []v1.Nod } // Otherwise the result is the same as for GetNodeAddressesFromNodeIP - return GetNodeAddressesFromNodeIP(nodeIP.String(), cloudNodeAddresses) + return GetNodeAddressesFromNodeIP(nodeIP.String(), cloudNodeAddresses, false) } // GetNodeAddressesFromNodeIP filters the provided list of nodeAddresses to match the @@ -102,32 +102,40 @@ func GetNodeAddressesFromNodeIPLegacy(nodeIP net.IP, cloudNodeAddresses []v1.Nod // GetNodeAddressesFromNodeIPLegacy, because that case never occurs for external cloud // providers, because kubelet does not set the `provided-node-ip` annotation in that // case.) -func GetNodeAddressesFromNodeIP(providedNodeIP string, cloudNodeAddresses []v1.NodeAddress) ([]v1.NodeAddress, error) { - nodeIP, err := nodeutil.ParseNodeIPAnnotation(providedNodeIP) +func GetNodeAddressesFromNodeIP(providedNodeIP string, cloudNodeAddresses []v1.NodeAddress, allowDualStack bool) ([]v1.NodeAddress, error) { + nodeIPs, err := nodeutil.ParseNodeIPAnnotation(providedNodeIP, allowDualStack) if err != nil { return nil, fmt.Errorf("failed to parse node IP %q: %v", providedNodeIP, err) } - // For every address supplied by the cloud provider that matches nodeIP, nodeIP is the enforced node address for - // that address Type (like InternalIP and ExternalIP), meaning other addresses of the same Type are discarded. - // See #61921 for more information: some cloud providers may supply secondary IPs, so nodeIP serves as a way to - // ensure that the correct IPs show up on a Node object. enforcedNodeAddresses := []v1.NodeAddress{} - nodeIPTypes := make(map[v1.NodeAddressType]bool) - for _, nodeAddress := range cloudNodeAddresses { - if netutils.ParseIPSloppy(nodeAddress.Address).Equal(nodeIP) { - enforcedNodeAddresses = append(enforcedNodeAddresses, v1.NodeAddress{Type: nodeAddress.Type, Address: nodeAddress.Address}) - nodeIPTypes[nodeAddress.Type] = true + + for _, nodeIP := range nodeIPs { + // For every address supplied by the cloud provider that matches nodeIP, + // nodeIP is the enforced node address for that address Type (like + // InternalIP and ExternalIP), meaning other addresses of the same Type + // are discarded. See #61921 for more information: some cloud providers + // may supply secondary IPs, so nodeIP serves as a way to ensure that the + // correct IPs show up on a Node object. + + matched := false + for _, nodeAddress := range cloudNodeAddresses { + if netutils.ParseIPSloppy(nodeAddress.Address).Equal(nodeIP) { + enforcedNodeAddresses = append(enforcedNodeAddresses, v1.NodeAddress{Type: nodeAddress.Type, Address: nodeAddress.Address}) + nodeIPTypes[nodeAddress.Type] = true + matched = true + } + } + + // nodeIP must be among the addresses supplied by the cloud provider + if !matched { + return nil, fmt.Errorf("failed to get node address from cloud provider that matches ip: %v", nodeIP) } } - // nodeIP must be among the addresses supplied by the cloud provider - if len(enforcedNodeAddresses) == 0 { - return nil, fmt.Errorf("failed to get node address from cloud provider that matches ip: %v", nodeIP) - } - - // nodeIP was found, now use all other addresses supplied by the cloud provider NOT of the same Type as nodeIP. + // Now use all other addresses supplied by the cloud provider NOT of the same Type + // as any nodeIP. for _, nodeAddress := range cloudNodeAddresses { if !nodeIPTypes[nodeAddress.Type] { enforcedNodeAddresses = append(enforcedNodeAddresses, v1.NodeAddress{Type: nodeAddress.Type, Address: nodeAddress.Address}) diff --git a/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go b/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go index 6007bedf206..cf208df1a28 100644 --- a/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go +++ b/staging/src/k8s.io/cloud-provider/node/helpers/address_test.go @@ -428,11 +428,131 @@ func TestGetNodeAddressesFromNodeIP(t *testing.T) { }, shouldError: false, }, + { + name: "Single-stack cloud, dual-stack request", + nodeIP: "10.1.1.1,fc01:1234::5678", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: true, + }, + { + name: "Dual-stack cloud, IPv4 first, IPv4-primary request", + nodeIP: "10.1.1.1,fc01:1234::5678", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, IPv6 first, IPv4-primary request", + nodeIP: "10.1.1.1,fc01:1234::5678", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, dual-stack request, multiple IPs", + nodeIP: "10.1.1.1,fc01:1234::5678", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "10.1.1.2"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::1234"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + // additional IPs of the same type are removed, as in the + // single-stack case. + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, dual-stack request, extra ExternalIP", + nodeIP: "10.1.1.1,fc01:1234::5678", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::1234"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + // The ExternalIP is preserved, since no ExternalIP was matched + // by --node-ip. + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, dual-stack request, multiple ExternalIPs", + nodeIP: "fc01:1234::5678,10.1.1.1", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeExternalIP, Address: "2001:db1::1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + // The ExternalIPs are preserved, since no ExternalIP was matched + // by --node-ip. + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeExternalIP, Address: "2001:db1::1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, + { + name: "Dual-stack cloud, dual-stack request, mixed InternalIP/ExternalIP match", + nodeIP: "55.55.55.55,fc01:1234::5678", + nodeAddresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.1.1.1"}, + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeExternalIP, Address: "2001:db1::1"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + // Since the IPv4 --node-ip value matched an ExternalIP, that + // filters out the IPv6 ExternalIP. Since the IPv6 --node-ip value + // matched in InternalIP, that filters out the IPv4 InternalIP + // value. + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "55.55.55.55"}, + {Type: v1.NodeInternalIP, Address: "fc01:1234::5678"}, + {Type: v1.NodeHostName, Address: testKubeletHostname}, + }, + shouldError: false, + }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - got, err := GetNodeAddressesFromNodeIP(tt.nodeIP, tt.nodeAddresses) + got, err := GetNodeAddressesFromNodeIP(tt.nodeIP, tt.nodeAddresses, true) if (err != nil) != tt.shouldError { t.Errorf("GetNodeAddressesFromNodeIP() error = %v, wantErr %v", err, tt.shouldError) return diff --git a/staging/src/k8s.io/component-helpers/node/util/ips.go b/staging/src/k8s.io/component-helpers/node/util/ips.go index 60d3097cc88..ff306a3dc3c 100644 --- a/staging/src/k8s.io/component-helpers/node/util/ips.go +++ b/staging/src/k8s.io/component-helpers/node/util/ips.go @@ -74,12 +74,9 @@ func ParseNodeIPArgument(nodeIP, cloudProvider string, allowCloudDualStack bool) } // ParseNodeIPAnnotation parses the `alpha.kubernetes.io/provided-node-ip` annotation, -// which should be a single IP address. Unlike with ParseNodeIPArgument, invalid values +// which can be either a single IP address or (if allowDualStack is true) a +// comma-separated pair of IP addresses. Unlike with ParseNodeIPArgument, invalid values // are considered an error. -func ParseNodeIPAnnotation(nodeIP string) (net.IP, error) { - ips, err := parseNodeIP(nodeIP, false, false) - if err != nil || len(ips) == 0 { - return nil, err - } - return ips[0], nil +func ParseNodeIPAnnotation(nodeIP string, allowDualStack bool) ([]net.IP, error) { + return parseNodeIP(nodeIP, allowDualStack, false) } diff --git a/staging/src/k8s.io/component-helpers/node/util/ips_test.go b/staging/src/k8s.io/component-helpers/node/util/ips_test.go index 8afb03f5cf2..0d3586ef582 100644 --- a/staging/src/k8s.io/component-helpers/node/util/ips_test.go +++ b/staging/src/k8s.io/component-helpers/node/util/ips_test.go @@ -214,10 +214,11 @@ func TestParseNodeIPArgument(t *testing.T) { func TestParseNodeIPAnnotation(t *testing.T) { testCases := []struct { - desc string - in string - out net.IP - err string + desc string + in string + out []net.IP + err string + ssErr string }{ { desc: "empty --node-ip", @@ -237,7 +238,9 @@ func TestParseNodeIPAnnotation(t *testing.T) { { desc: "single IPv4", in: "1.2.3.4", - out: netutils.ParseIPSloppy("1.2.3.4"), + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + }, }, { desc: "single IPv4 with whitespace", @@ -247,7 +250,9 @@ func TestParseNodeIPAnnotation(t *testing.T) { { desc: "single IPv4 non-canonical", in: "01.2.3.004", - out: netutils.ParseIPSloppy("1.2.3.4"), + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + }, }, { desc: "single IPv4 invalid", @@ -262,7 +267,9 @@ func TestParseNodeIPAnnotation(t *testing.T) { { desc: "single IPv4 unspecified", in: "0.0.0.0", - out: net.IPv4zero, + out: []net.IP{ + net.IPv4zero, + }, }, { desc: "single IPv4 plus garbage", @@ -272,17 +279,25 @@ func TestParseNodeIPAnnotation(t *testing.T) { { desc: "single IPv6", in: "abcd::ef01", - out: netutils.ParseIPSloppy("abcd::ef01"), + out: []net.IP{ + netutils.ParseIPSloppy("abcd::ef01"), + }, }, { desc: "single IPv6 non-canonical", in: "abcd:0abc:00ab:0000:0000::1", - out: netutils.ParseIPSloppy("abcd:abc:ab::1"), + out: []net.IP{ + netutils.ParseIPSloppy("abcd:abc:ab::1"), + }, }, { desc: "simple dual-stack", in: "1.2.3.4,abcd::ef01", - err: "not supported in this configuration", + out: []net.IP{ + netutils.ParseIPSloppy("1.2.3.4"), + netutils.ParseIPSloppy("abcd::ef01"), + }, + ssErr: "not supported in this configuration", }, { desc: "dual-stack with whitespace", @@ -300,14 +315,16 @@ func TestParseNodeIPAnnotation(t *testing.T) { err: "either a single IP or a dual-stack pair of IPs", }, { - desc: "dual-stack with unspecified", - in: "1.2.3.4,::", - err: "not supported in this configuration", + desc: "dual-stack with unspecified", + in: "1.2.3.4,::", + err: "cannot include '0.0.0.0' or '::'", + ssErr: "not supported in this configuration", }, { - desc: "dual-stack with unspecified", - in: "0.0.0.0,abcd::1", - err: "not supported in this configuration", + desc: "dual-stack with unspecified", + in: "0.0.0.0,abcd::1", + err: "cannot include '0.0.0.0' or '::'", + ssErr: "not supported in this configuration", }, { desc: "dual-stack plus garbage", @@ -322,21 +339,36 @@ func TestParseNodeIPAnnotation(t *testing.T) { } for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - parsed, err := ParseNodeIPAnnotation(tc.in) + for _, allowDualStack := range []bool{false, true} { + desc := fmt.Sprintf("%s, allowDualStack=%v", tc.desc, allowDualStack) + t.Run(desc, func(t *testing.T) { + parsed, err := ParseNodeIPAnnotation(tc.in, allowDualStack) - if !reflect.DeepEqual(parsed, tc.out) { - t.Errorf("expected %#v, got %#v", tc.out, parsed) - } - if err != nil { - if tc.err == "" { - t.Errorf("unexpected error %v", err) - } else if !strings.Contains(err.Error(), tc.err) { - t.Errorf("expected error with %q, got %v", tc.err, err) + expectedOut := tc.out + expectedErr := tc.err + + if !allowDualStack { + if len(tc.out) == 2 { + expectedOut = nil + } + if tc.ssErr != "" { + expectedErr = tc.ssErr + } } - } else if tc.err != "" { - t.Errorf("expected error with %q, got no error", tc.err) - } - }) + + if !reflect.DeepEqual(parsed, expectedOut) { + t.Errorf("expected %#v, got %#v", expectedOut, parsed) + } + if err != nil { + if expectedErr == "" { + t.Errorf("unexpected error %v", err) + } else if !strings.Contains(err.Error(), expectedErr) { + t.Errorf("expected error with %q, got %v", expectedErr, err) + } + } else if expectedErr != "" { + t.Errorf("expected error with %q, got no error", expectedErr) + } + }) + } } } From 068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 3 Mar 2023 11:53:11 -0500 Subject: [PATCH 6/6] Add CloudDualStackNodeIPs feature gate(s) Add CloudDualStackNodeIPs feature gates for kubelet and cloud-controller-manager and use as appropriate. --- cmd/kubelet/app/server.go | 2 +- pkg/features/kube_features.go | 8 +++++++ .../controllers/node/node_controller.go | 4 +++- .../controllers/node/node_controller_test.go | 1 + .../pkg/features/kube_features.go | 22 +++++++++++++------ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index fd4aa687e3b..4c4fb27b244 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1120,7 +1120,7 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie // Setup event recorder if required. makeEventRecorder(kubeDeps, nodeName) - nodeIPs, err := nodeutil.ParseNodeIPArgument(kubeServer.NodeIP, kubeServer.CloudProvider, false) + nodeIPs, err := nodeutil.ParseNodeIPArgument(kubeServer.NodeIP, kubeServer.CloudProvider, utilfeature.DefaultFeatureGate.Enabled(features.CloudDualStackNodeIPs)) if err != nil { return fmt.Errorf("bad --node-ip %q: %v", kubeServer.NodeIP, err) } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index bdd459988a6..0d08585be67 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -61,6 +61,12 @@ const ( // beta: v1.4 AppArmor featuregate.Feature = "AppArmor" + // owner: @danwinship + // alpha: v1.27 + // + // Enables dual-stack --node-ip in kubelet with external cloud providers + CloudDualStackNodeIPs featuregate.Feature = "CloudDualStackNodeIPs" + // owner: @szuecs // alpha: v1.12 // @@ -926,6 +932,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS AppArmor: {Default: true, PreRelease: featuregate.Beta}, + CloudDualStackNodeIPs: {Default: false, PreRelease: featuregate.Alpha}, + CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, CPUManager: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // GA in 1.26 diff --git a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go index 24d45452c44..ace70dd0aaf 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go +++ b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -43,6 +44,7 @@ import ( cloudnodeutil "k8s.io/cloud-provider/node/helpers" controllersmetrics "k8s.io/component-base/metrics/prometheus/controllers" nodeutil "k8s.io/component-helpers/node/util" + "k8s.io/controller-manager/pkg/features" "k8s.io/klog/v2" ) @@ -739,7 +741,7 @@ func updateNodeAddressesFromNodeIP(node *v1.Node, nodeAddresses []v1.NodeAddress providedNodeIP, exists := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr] if exists { - nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(providedNodeIP, nodeAddresses, false) + nodeAddresses, err = cloudnodeutil.GetNodeAddressesFromNodeIP(providedNodeIP, nodeAddresses, utilfeature.DefaultFeatureGate.Enabled(features.CloudDualStackNodeIPs)) } return nodeAddresses, err diff --git a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller_test.go b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller_test.go index 723756242aa..9bb65f693d1 100644 --- a/staging/src/k8s.io/cloud-provider/controllers/node/node_controller_test.go +++ b/staging/src/k8s.io/cloud-provider/controllers/node/node_controller_test.go @@ -38,6 +38,7 @@ import ( cloudprovider "k8s.io/cloud-provider" cloudproviderapi "k8s.io/cloud-provider/api" fakecloud "k8s.io/cloud-provider/fake" + _ "k8s.io/controller-manager/pkg/features/register" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" diff --git a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go index fad02729503..8864be325e1 100644 --- a/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go +++ b/staging/src/k8s.io/controller-manager/pkg/features/kube_features.go @@ -32,6 +32,19 @@ const ( // of code conflicts because changes are more likely to be scattered // across the file. + // owner: @nckturner + // kep: http://kep.k8s.io/2699 + // alpha: v1.27 + // Enable webhook in cloud controller manager + CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook" + + // owner: @danwinship + // alpha: v1.27 + // + // Enables dual-stack values in the + // `alpha.kubernetes.io/provided-node-ip` annotation + CloudDualStackNodeIPs featuregate.Feature = "CloudDualStackNodeIPs" + // owner: @alexanderConstantinescu // kep: http://kep.k8s.io/3458 // beta: v1.27 @@ -39,12 +52,6 @@ const ( // Enables less load balancer re-configurations by the service controller // (KCCM) as an effect of changing node state. StableLoadBalancerNodeSet featuregate.Feature = "StableLoadBalancerNodeSet" - - // owner: @nckturner - // kep: http://kep.k8s.io/2699 - // alpha: v1.27 - // Enable webhook in cloud controller manager - CloudControllerManagerWebhook featuregate.Feature = "CloudControllerManagerWebhook" ) func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.MutableFeatureGate) error { @@ -54,6 +61,7 @@ func SetupCurrentKubernetesSpecificFeatureGates(featuregates featuregate.Mutable // cloudPublicFeatureGates consists of cloud-specific feature keys. // To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here. var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - StableLoadBalancerNodeSet: {Default: true, PreRelease: featuregate.Beta}, CloudControllerManagerWebhook: {Default: false, PreRelease: featuregate.Alpha}, + CloudDualStackNodeIPs: {Default: false, PreRelease: featuregate.Alpha}, + StableLoadBalancerNodeSet: {Default: true, PreRelease: featuregate.Beta}, }