Add --nodeport-addresses primary

The behavior when you specify no --nodeport-addresses value in a
dual-stack cluster is terrible and we can't fix it, for
backward-compatibility reasons. Actually, the behavior when you
specify no --nodeport-addresses value in a single-stack cluster isn't
exactly awesome either...

Allow specifying `--nodeport-addresses primary` to get the
previously-nftables-backend-specific behavior of listening on only the
node's primary IP or IPs.
This commit is contained in:
Dan Winship 2024-02-02 09:43:39 -05:00
parent 8de0fc09aa
commit 0b599aa8e3
8 changed files with 49 additions and 9 deletions

View File

@ -197,7 +197,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
"This parameter is ignored if a config file is specified by --config.")
fs.StringSliceVar(&o.config.NodePortAddresses, "nodeport-addresses", o.config.NodePortAddresses,
"A list of CIDR ranges that contain valid node IPs. If set, connections to NodePort services will only be accepted on node IPs in one of the indicated ranges. If unset, NodePort connections will be accepted on all local IPs. This parameter is ignored if a config file is specified by --config.")
"A list of CIDR ranges that contain valid node IPs, or alternatively, the single string 'primary'. If set to a list of CIDRs, connections to NodePort services will only be accepted on node IPs in one of the indicated ranges. If set to 'primary', NodePort services will only be accepted on the node's primary IP(s) according to the Node object. If unset, NodePort connections will be accepted on all local IPs. This parameter is ignored if a config file is specified by --config.")
fs.Int32Var(o.config.OOMScoreAdj, "oom-score-adj", ptr.Deref(o.config.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]. This parameter is ignored if a config file is specified by --config.")
fs.Int32Var(o.config.Conntrack.MaxPerCore, "conntrack-max-per-core", *o.config.Conntrack.MaxPerCore,
@ -631,6 +631,17 @@ func newProxyServer(logger klog.Logger, config *kubeproxyconfig.KubeProxyConfigu
rawNodeIPs := getNodeIPs(logger, s.Client, s.Hostname)
s.PrimaryIPFamily, s.NodeIPs = detectNodeIPs(logger, rawNodeIPs, config.BindAddress)
if len(config.NodePortAddresses) == 1 && config.NodePortAddresses[0] == kubeproxyconfig.NodePortAddressesPrimary {
var nodePortAddresses []string
if nodeIP := s.NodeIPs[v1.IPv4Protocol]; nodeIP != nil && !nodeIP.IsLoopback() {
nodePortAddresses = append(nodePortAddresses, fmt.Sprintf("%s/32", nodeIP.String()))
}
if nodeIP := s.NodeIPs[v1.IPv6Protocol]; nodeIP != nil && !nodeIP.IsLoopback() {
nodePortAddresses = append(nodePortAddresses, fmt.Sprintf("%s/128", nodeIP.String()))
}
config.NodePortAddresses = nodePortAddresses
}
s.Broadcaster = events.NewBroadcaster(&events.EventSinkImpl{Interface: s.Client.EventsV1()})
s.Recorder = s.Broadcaster.NewRecorder(proxyconfigscheme.Scheme, "kube-proxy")

View File

@ -70,6 +70,10 @@ func (o *Options) platformApplyDefaults(config *proxyconfigapi.KubeProxyConfigur
config.Mode = proxyconfigapi.ProxyModeIPTables
}
if config.Mode == proxyconfigapi.ProxyModeNFTables && len(config.NodePortAddresses) == 0 {
config.NodePortAddresses = []string{proxyconfigapi.NodePortAddressesPrimary}
}
if config.DetectLocalMode == "" {
o.logger.V(4).Info("Defaulting detect-local-mode", "localModeClusterCIDR", string(proxyconfigapi.LocalModeClusterCIDR))
config.DetectLocalMode = proxyconfigapi.LocalModeClusterCIDR

View File

@ -59048,7 +59048,7 @@ func schema_k8sio_kube_proxy_config_v1alpha1_KubeProxyConfiguration(ref common.R
},
"nodePortAddresses": {
SchemaProps: spec.SchemaProps{
Description: "nodePortAddresses is a list of CIDR ranges that contain valid node IPs. If set, connections to NodePort services will only be accepted on node IPs in one of the indicated ranges. If unset, NodePort connections will be accepted on all local IPs.",
Description: "nodePortAddresses is a list of CIDR ranges that contain valid node IPs, or alternatively, the single string 'primary'. If set to a list of CIDRs, connections to NodePort services will only be accepted on node IPs in one of the indicated ranges. If set to 'primary', NodePort services will only be accepted on the node's primary IPv4 and/or IPv6 address according to the Node object. If unset, NodePort connections will be accepted on all local IPs.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{

View File

@ -224,10 +224,12 @@ type KubeProxyConfiguration struct {
// used.)
ClusterCIDR string
// nodePortAddresses is a list of CIDR ranges that contain valid node IPs. If set,
// nodePortAddresses is a list of CIDR ranges that contain valid node IPs, or
// alternatively, the single string 'primary'. If set to a list of CIDRs,
// connections to NodePort services will only be accepted on node IPs in one of
// the indicated ranges. If unset, NodePort connections will be accepted on all
// local IPs.
// the indicated ranges. If set to 'primary', NodePort services will only be
// accepted on the node's primary IPv4 and/or IPv6 address according to the Node
// object. If unset, NodePort connections will be accepted on all local IPs.
NodePortAddresses []string
// oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within
@ -303,3 +305,7 @@ func (m *LocalMode) String() string {
func (m *LocalMode) Type() string {
return "LocalMode"
}
// NodePortAddressesPrimary is a special value for NodePortAddresses indicating that it
// should only use the primary node IPs.
const NodePortAddressesPrimary string = "primary"

View File

@ -297,6 +297,13 @@ func validateKubeProxyNodePortAddress(nodePortAddresses []string, fldPath *field
allErrs := field.ErrorList{}
for i := range nodePortAddresses {
if nodePortAddresses[i] == kubeproxyconfig.NodePortAddressesPrimary {
if i != 0 || len(nodePortAddresses) != 1 {
allErrs = append(allErrs, field.Invalid(fldPath.Index(i), nodePortAddresses[i], "can't use both 'primary' and CIDRs"))
}
break
}
if _, _, err := netutils.ParseCIDRSloppy(nodePortAddresses[i]); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Index(i), nodePortAddresses[i], "must be a valid CIDR"))
}

View File

@ -975,6 +975,7 @@ func TestValidateKubeProxyNodePortAddress(t *testing.T) {
{[]string{"10.20.0.0/16", "100.200.0.0/16"}},
{[]string{"10.0.0.0/8"}},
{[]string{"2001:db8::/32"}},
{[]string{kubeproxyconfig.NodePortAddressesPrimary}},
}
for _, successCase := range successCases {
@ -1012,6 +1013,14 @@ func TestValidateKubeProxyNodePortAddress(t *testing.T) {
addresses: []string{"::1/128", "2001:db8::/32", "2001:db8:xyz/64"},
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("NodePortAddresses[2]"), "2001:db8:xyz/64", "must be a valid CIDR")},
},
"invalid primary/CIDR mix 1": {
addresses: []string{"primary", "127.0.0.1/32"},
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("NodePortAddresses[0]"), "primary", "can't use both 'primary' and CIDRs")},
},
"invalid primary/CIDR mix 2": {
addresses: []string{"127.0.0.1/32", "primary"},
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("NodePortAddresses[1]"), "primary", "can't use both 'primary' and CIDRs")},
},
}
for _, testCase := range testCases {

View File

@ -86,6 +86,7 @@ func NewFakeProxier(ipFamily v1.IPFamily) (*knftables.Fake, *Proxier) {
serviceCIDRs = "fd00:10:96::/112"
}
detectLocal, _ := proxyutiliptables.NewDetectLocalByCIDR(podCIDR)
nodePortAddresses := []string{fmt.Sprintf("%s/32", testNodeIP), fmt.Sprintf("%s/128", testNodeIPv6)}
networkInterfacer := proxyutiltest.NewFakeNetwork()
itf := net.Interface{Index: 0, MTU: 0, Name: "lo", HardwareAddr: nil, Flags: 0}
@ -125,7 +126,7 @@ func NewFakeProxier(ipFamily v1.IPFamily) (*knftables.Fake, *Proxier) {
hostname: testHostname,
serviceHealthServer: healthcheck.NewFakeServiceHealthServer(),
nodeIP: nodeIP,
nodePortAddresses: proxyutil.NewNodePortAddresses(ipFamily, nil, nodeIP),
nodePortAddresses: proxyutil.NewNodePortAddresses(ipFamily, nodePortAddresses, nodeIP),
networkInterfacer: networkInterfacer,
staleChains: make(map[string]time.Time),
serviceCIDRs: serviceCIDRs,

View File

@ -224,10 +224,12 @@ type KubeProxyConfiguration struct {
// used.)
ClusterCIDR string `json:"clusterCIDR"`
// nodePortAddresses is a list of CIDR ranges that contain valid node IPs. If set,
// nodePortAddresses is a list of CIDR ranges that contain valid node IPs, or
// alternatively, the single string 'primary'. If set to a list of CIDRs,
// connections to NodePort services will only be accepted on node IPs in one of
// the indicated ranges. If unset, NodePort connections will be accepted on all
// local IPs.
// the indicated ranges. If set to 'primary', NodePort services will only be
// accepted on the node's primary IPv4 and/or IPv6 address according to the Node
// object. If unset, NodePort connections will be accepted on all local IPs.
NodePortAddresses []string `json:"nodePortAddresses"`
// oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within