From 1a7f7c5399193cfb26ea2cfc3b709df0104e85a5 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Thu, 6 Oct 2016 14:52:32 -0400 Subject: [PATCH] Allow apiserver to choose preferred kubelet address type --- cmd/kube-apiserver/app/options/BUILD | 1 + cmd/kube-apiserver/app/options/options.go | 12 ++- hack/verify-flags/known-flags.txt | 1 + pkg/kubelet/client/kubelet_client.go | 15 +++- pkg/util/node/BUILD | 11 +++ pkg/util/node/node.go | 20 +++++ pkg/util/node/node_test.go | 90 +++++++++++++++++++++++ test/test_owners.csv | 1 + 8 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 pkg/util/node/node_test.go diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD index b80aa4432d2..547889f4946 100644 --- a/cmd/kube-apiserver/app/options/BUILD +++ b/cmd/kube-apiserver/app/options/BUILD @@ -15,6 +15,7 @@ go_library( srcs = ["options.go"], tags = ["automanaged"], deps = [ + "//pkg/api:go_default_library", "//pkg/api/validation:go_default_library", "//pkg/genericapiserver/options:go_default_library", "//pkg/kubelet/client:go_default_library", diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index 5a763950745..56afa61ed91 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -20,6 +20,7 @@ package options import ( "time" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/validation" genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" @@ -49,7 +50,13 @@ func NewServerRunOptions() *ServerRunOptions { GenericServerRunOptions: genericoptions.NewServerRunOptions().WithEtcdOptions(), EventTTL: 1 * time.Hour, KubeletConfig: kubeletclient.KubeletClientConfig{ - Port: ports.KubeletPort, + Port: ports.KubeletPort, + PreferredAddressTypes: []string{ + string(api.NodeHostName), + string(api.NodeInternalIP), + string(api.NodeExternalIP), + string(api.NodeLegacyHostIP), + }, EnableHttps: true, HTTPTimeout: time.Duration(5) * time.Second, }, @@ -102,6 +109,9 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps, "Use https for kubelet connections.") + fs.StringSliceVar(&s.KubeletConfig.PreferredAddressTypes, "kubelet-preferred-address-types", s.KubeletConfig.PreferredAddressTypes, + "List of the preferred NodeAddressTypes to use for kubelet connections.") + fs.UintVar(&s.KubeletConfig.Port, "kubelet-port", s.KubeletConfig.Port, "DEPRECATED: kubelet port.") fs.MarkDeprecated("kubelet-port", "kubelet-port is deprecated and will be removed.") diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index e73dec1d75b..f3c1211e9da 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -312,6 +312,7 @@ kubelet-kubeconfig kubelet-network-plugin kubelet-pod-infra-container-image kubelet-port +kubelet-preferred-address-types kubelet-read-only-port kubelet-root-dir kubelet-sync-frequency diff --git a/pkg/kubelet/client/kubelet_client.go b/pkg/kubelet/client/kubelet_client.go index f4744b2d7da..aa9644b6c96 100644 --- a/pkg/kubelet/client/kubelet_client.go +++ b/pkg/kubelet/client/kubelet_client.go @@ -35,6 +35,9 @@ type KubeletClientConfig struct { Port uint EnableHttps bool + // PreferredAddressTypes - used to select an address from Node.NodeStatus.Addresses + PreferredAddressTypes []string + // TLSClientConfig contains settings to enable transport layer security restclient.TLSClientConfig @@ -119,6 +122,8 @@ type NodeConnectionInfoGetter struct { defaultPort int // transport is the transport to use to send a request to all kubelets transport http.RoundTripper + // preferredAddressTypes specifies the preferred order to use to find a node address + preferredAddressTypes []api.NodeAddressType } func NewNodeConnectionInfoGetter(nodes NodeGetter, config KubeletClientConfig) (ConnectionInfoGetter, error) { @@ -132,11 +137,18 @@ func NewNodeConnectionInfoGetter(nodes NodeGetter, config KubeletClientConfig) ( return nil, err } + types := []api.NodeAddressType{} + for _, t := range config.PreferredAddressTypes { + types = append(types, api.NodeAddressType(t)) + } + return &NodeConnectionInfoGetter{ nodes: nodes, scheme: scheme, defaultPort: int(config.Port), transport: transport, + + preferredAddressTypes: types, }, nil } @@ -147,11 +159,10 @@ func (k *NodeConnectionInfoGetter) GetConnectionInfo(ctx api.Context, nodeName t } // Find a kubelet-reported address, using preferred address type - hostIP, err := nodeutil.GetNodeHostIP(node) + host, err := nodeutil.GetPreferredNodeAddress(node, k.preferredAddressTypes) if err != nil { return nil, err } - host := hostIP.String() // Use the kubelet-reported port, if present port := int(node.Status.DaemonEndpoints.KubeletEndpoint.Port) diff --git a/pkg/util/node/BUILD b/pkg/util/node/BUILD index 1e0e8b1122c..25db78b5264 100644 --- a/pkg/util/node/BUILD +++ b/pkg/util/node/BUILD @@ -22,3 +22,14 @@ go_library( "//vendor:github.com/golang/glog", ], ) + +go_test( + name = "go_default_test", + srcs = ["node_test.go"], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/unversioned:go_default_library", + ], +) diff --git a/pkg/util/node/node.go b/pkg/util/node/node.go index 34bfbb8bb14..0bb4fb574ef 100644 --- a/pkg/util/node/node.go +++ b/pkg/util/node/node.go @@ -43,6 +43,26 @@ func GetHostname(hostnameOverride string) string { return strings.ToLower(strings.TrimSpace(hostname)) } +// GetPreferredNodeAddress returns the address of the provided node, using the provided preference order. +// If none of the preferred address types are found, an error is returned. +func GetPreferredNodeAddress(node *api.Node, preferredAddressTypes []api.NodeAddressType) (string, error) { + for _, addressType := range preferredAddressTypes { + for _, address := range node.Status.Addresses { + if address.Type == addressType { + return address.Address, nil + } + } + // If hostname was requested and no Hostname address was registered... + if addressType == api.NodeHostName { + // ...fall back to the kubernetes.io/hostname label for compatibility with kubelets before 1.5 + if hostname, ok := node.Labels[unversioned.LabelHostname]; ok && len(hostname) > 0 { + return hostname, nil + } + } + } + return "", fmt.Errorf("no preferred addresses found; known addresses: %v", node.Status.Addresses) +} + // GetNodeHostIP returns the provided node's IP, based on the priority: // 1. NodeInternalIP // 2. NodeExternalIP diff --git a/pkg/util/node/node_test.go b/pkg/util/node/node_test.go new file mode 100644 index 00000000000..957bf09352e --- /dev/null +++ b/pkg/util/node/node_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" +) + +func TestGetPreferredAddress(t *testing.T) { + testcases := map[string]struct { + Labels map[string]string + Addresses []api.NodeAddress + Preferences []api.NodeAddressType + + ExpectErr string + ExpectAddress string + }{ + "no addresses": { + ExpectErr: "no preferred addresses found; known addresses: []", + }, + "missing address": { + Addresses: []api.NodeAddress{ + {Type: api.NodeInternalIP, Address: "1.2.3.4"}, + }, + Preferences: []api.NodeAddressType{api.NodeHostName}, + ExpectErr: "no preferred addresses found; known addresses: [{InternalIP 1.2.3.4}]", + }, + "found address": { + Addresses: []api.NodeAddress{ + {Type: api.NodeInternalIP, Address: "1.2.3.4"}, + {Type: api.NodeExternalIP, Address: "1.2.3.5"}, + {Type: api.NodeExternalIP, Address: "1.2.3.7"}, + }, + Preferences: []api.NodeAddressType{api.NodeHostName, api.NodeExternalIP}, + ExpectAddress: "1.2.3.5", + }, + "found hostname address": { + Labels: map[string]string{unversioned.LabelHostname: "label-hostname"}, + Addresses: []api.NodeAddress{ + {Type: api.NodeExternalIP, Address: "1.2.3.5"}, + {Type: api.NodeHostName, Address: "status-hostname"}, + }, + Preferences: []api.NodeAddressType{api.NodeHostName, api.NodeExternalIP}, + ExpectAddress: "status-hostname", + }, + "found label address": { + Labels: map[string]string{unversioned.LabelHostname: "label-hostname"}, + Addresses: []api.NodeAddress{ + {Type: api.NodeExternalIP, Address: "1.2.3.5"}, + }, + Preferences: []api.NodeAddressType{api.NodeHostName, api.NodeExternalIP}, + ExpectAddress: "label-hostname", + }, + } + + for k, tc := range testcases { + node := &api.Node{ + ObjectMeta: api.ObjectMeta{Labels: tc.Labels}, + Status: api.NodeStatus{Addresses: tc.Addresses}, + } + address, err := GetPreferredNodeAddress(node, tc.Preferences) + errString := "" + if err != nil { + errString = err.Error() + } + if errString != tc.ExpectErr { + t.Errorf("%s: expected err=%q, got %q", k, tc.ExpectErr, errString) + } + if address != tc.ExpectAddress { + t.Errorf("%s: expected address=%q, got %q", k, tc.ExpectAddress, address) + } + } +} diff --git a/test/test_owners.csv b/test/test_owners.csv index abaa63625b1..1536ae6beb0 100644 --- a/test/test_owners.csv +++ b/test/test_owners.csv @@ -778,6 +778,7 @@ k8s.io/kubernetes/pkg/util/limitwriter,deads2k,1 k8s.io/kubernetes/pkg/util/mount,xiang90,1 k8s.io/kubernetes/pkg/util/net,spxtr,1 k8s.io/kubernetes/pkg/util/net/sets,jdef,1 +k8s.io/kubernetes/pkg/util/node,liggitt,0 k8s.io/kubernetes/pkg/util/oom,vishh,0 k8s.io/kubernetes/pkg/util/parsers,derekwaynecarr,1 k8s.io/kubernetes/pkg/util/procfs,roberthbailey,1