Merge pull request #35497 from liggitt/node-address

Automatic merge from submit-queue

Allow apiserver to choose preferred kubelet address type

Follow up to #33718 to stay compatible with clusters using DNS names for master->node communications. Adds the `--kubelet-preferred-address-types` apiserver flag for clusters that prefer a different node address type.

```release-note
The apiserver can now select which type of kubelet-reported address to use for master->node communications, using the --kubelet-preferred-address-types flag.
```

cmd/kube-apiserver/app/options/BUILD

@@ -15,6 +15,7 @@ go_library(
     srcs = ["options.go"],
     tags = ["automanaged"],
     deps = [
+        "//pkg/api:go_default_library",
         "//pkg/api/validation:go_default_library",
         "//pkg/genericapiserver/options:go_default_library",
         "//pkg/kubelet/client:go_default_library",

cmd/kube-apiserver/app/options/options.go

@@ -20,6 +20,7 @@ package options
 import (
 	"time"
 
+	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/validation"
 	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
@@ -50,6 +51,12 @@ func NewServerRunOptions() *ServerRunOptions {
 		EventTTL:                1 * time.Hour,
 		KubeletConfig: kubeletclient.KubeletClientConfig{
 			Port: ports.KubeletPort,
+			PreferredAddressTypes: []string{
+				string(api.NodeHostName),
+				string(api.NodeInternalIP),
+				string(api.NodeExternalIP),
+				string(api.NodeLegacyHostIP),
+			},
 			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
@@ -102,6 +109,9 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps,
 		"Use https for kubelet connections.")
 
+	fs.StringSliceVar(&s.KubeletConfig.PreferredAddressTypes, "kubelet-preferred-address-types", s.KubeletConfig.PreferredAddressTypes,
+		"List of the preferred NodeAddressTypes to use for kubelet connections.")
+
 	fs.UintVar(&s.KubeletConfig.Port, "kubelet-port", s.KubeletConfig.Port,
 		"DEPRECATED: kubelet port.")
 	fs.MarkDeprecated("kubelet-port", "kubelet-port is deprecated and will be removed.")

hack/verify-flags/known-flags.txt

@@ -312,6 +312,7 @@ kubelet-kubeconfig
 kubelet-network-plugin
 kubelet-pod-infra-container-image
 kubelet-port
+kubelet-preferred-address-types
 kubelet-read-only-port
 kubelet-root-dir
 kubelet-sync-frequency

pkg/kubelet/client/kubelet_client.go

@@ -35,6 +35,9 @@ type KubeletClientConfig struct {
 	Port        uint
 	EnableHttps bool
 
+	// PreferredAddressTypes - used to select an address from Node.NodeStatus.Addresses
+	PreferredAddressTypes []string
+
 	// TLSClientConfig contains settings to enable transport layer security
 	restclient.TLSClientConfig
 
@@ -119,6 +122,8 @@ type NodeConnectionInfoGetter struct {
 	defaultPort int
 	// transport is the transport to use to send a request to all kubelets
 	transport http.RoundTripper
+	// preferredAddressTypes specifies the preferred order to use to find a node address
+	preferredAddressTypes []api.NodeAddressType
 }
 
 func NewNodeConnectionInfoGetter(nodes NodeGetter, config KubeletClientConfig) (ConnectionInfoGetter, error) {
@@ -132,11 +137,18 @@ func NewNodeConnectionInfoGetter(nodes NodeGetter, config KubeletClientConfig) (
 		return nil, err
 	}
 
+	types := []api.NodeAddressType{}
+	for _, t := range config.PreferredAddressTypes {
+		types = append(types, api.NodeAddressType(t))
+	}
+
 	return &NodeConnectionInfoGetter{
 		nodes:       nodes,
 		scheme:      scheme,
 		defaultPort: int(config.Port),
 		transport:   transport,
+
+		preferredAddressTypes: types,
 	}, nil
 }
 
@@ -147,11 +159,10 @@ func (k *NodeConnectionInfoGetter) GetConnectionInfo(ctx api.Context, nodeName t
 	}
 
 	// Find a kubelet-reported address, using preferred address type
-	hostIP, err := nodeutil.GetNodeHostIP(node)
+	host, err := nodeutil.GetPreferredNodeAddress(node, k.preferredAddressTypes)
 	if err != nil {
 		return nil, err
 	}
-	host := hostIP.String()
 
 	// Use the kubelet-reported port, if present
 	port := int(node.Status.DaemonEndpoints.KubeletEndpoint.Port)

pkg/util/node/BUILD

@@ -22,3 +22,14 @@ go_library(
         "//vendor:github.com/golang/glog",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["node_test.go"],
+    library = "go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/unversioned:go_default_library",
+    ],
+)

pkg/util/node/node.go

@@ -43,6 +43,26 @@ func GetHostname(hostnameOverride string) string {
 	return strings.ToLower(strings.TrimSpace(hostname))
 }
 
+// GetPreferredNodeAddress returns the address of the provided node, using the provided preference order.
+// If none of the preferred address types are found, an error is returned.
+func GetPreferredNodeAddress(node *api.Node, preferredAddressTypes []api.NodeAddressType) (string, error) {
+	for _, addressType := range preferredAddressTypes {
+		for _, address := range node.Status.Addresses {
+			if address.Type == addressType {
+				return address.Address, nil
+			}
+		}
+		// If hostname was requested and no Hostname address was registered...
+		if addressType == api.NodeHostName {
+			// ...fall back to the kubernetes.io/hostname label for compatibility with kubelets before 1.5
+			if hostname, ok := node.Labels[unversioned.LabelHostname]; ok && len(hostname) > 0 {
+				return hostname, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("no preferred addresses found; known addresses: %v", node.Status.Addresses)
+}
+
 // GetNodeHostIP returns the provided node's IP, based on the priority:
 // 1. NodeInternalIP
 // 2. NodeExternalIP

pkg/util/node/node_test.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/unversioned"
+)
+
+func TestGetPreferredAddress(t *testing.T) {
+	testcases := map[string]struct {
+		Labels      map[string]string
+		Addresses   []api.NodeAddress
+		Preferences []api.NodeAddressType
+
+		ExpectErr     string
+		ExpectAddress string
+	}{
+		"no addresses": {
+			ExpectErr: "no preferred addresses found; known addresses: []",
+		},
+		"missing address": {
+			Addresses: []api.NodeAddress{
+				{Type: api.NodeInternalIP, Address: "1.2.3.4"},
+			},
+			Preferences: []api.NodeAddressType{api.NodeHostName},
+			ExpectErr:   "no preferred addresses found; known addresses: [{InternalIP 1.2.3.4}]",
+		},
+		"found address": {
+			Addresses: []api.NodeAddress{
+				{Type: api.NodeInternalIP, Address: "1.2.3.4"},
+				{Type: api.NodeExternalIP, Address: "1.2.3.5"},
+				{Type: api.NodeExternalIP, Address: "1.2.3.7"},
+			},
+			Preferences:   []api.NodeAddressType{api.NodeHostName, api.NodeExternalIP},
+			ExpectAddress: "1.2.3.5",
+		},
+		"found hostname address": {
+			Labels: map[string]string{unversioned.LabelHostname: "label-hostname"},
+			Addresses: []api.NodeAddress{
+				{Type: api.NodeExternalIP, Address: "1.2.3.5"},
+				{Type: api.NodeHostName, Address: "status-hostname"},
+			},
+			Preferences:   []api.NodeAddressType{api.NodeHostName, api.NodeExternalIP},
+			ExpectAddress: "status-hostname",
+		},
+		"found label address": {
+			Labels: map[string]string{unversioned.LabelHostname: "label-hostname"},
+			Addresses: []api.NodeAddress{
+				{Type: api.NodeExternalIP, Address: "1.2.3.5"},
+			},
+			Preferences:   []api.NodeAddressType{api.NodeHostName, api.NodeExternalIP},
+			ExpectAddress: "label-hostname",
+		},
+	}
+
+	for k, tc := range testcases {
+		node := &api.Node{
+			ObjectMeta: api.ObjectMeta{Labels: tc.Labels},
+			Status:     api.NodeStatus{Addresses: tc.Addresses},
+		}
+		address, err := GetPreferredNodeAddress(node, tc.Preferences)
+		errString := ""
+		if err != nil {
+			errString = err.Error()
+		}
+		if errString != tc.ExpectErr {
+			t.Errorf("%s: expected err=%q, got %q", k, tc.ExpectErr, errString)
+		}
+		if address != tc.ExpectAddress {
+			t.Errorf("%s: expected address=%q, got %q", k, tc.ExpectAddress, address)
+		}
+	}
+}

test/test_owners.csv

@@ -778,6 +778,7 @@ k8s.io/kubernetes/pkg/util/limitwriter,deads2k,1
 k8s.io/kubernetes/pkg/util/mount,xiang90,1
 k8s.io/kubernetes/pkg/util/net,spxtr,1
 k8s.io/kubernetes/pkg/util/net/sets,jdef,1
+k8s.io/kubernetes/pkg/util/node,liggitt,0
 k8s.io/kubernetes/pkg/util/oom,vishh,0
 k8s.io/kubernetes/pkg/util/parsers,derekwaynecarr,1
 k8s.io/kubernetes/pkg/util/procfs,roberthbailey,1