Merge pull request #67397 from neolit123/bind-address

Automatic merge from submit-queue (batch tested with PRs 67397, 68019). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

kubeadm: fix offline and air-gapped support

**What this PR does / why we need it**:

1.

Change the error output of getAllDefaultRoutes() so that it includes
information on which files were probed for the IP routing tables
even if such files are obvious.

Introduce a new error type which can be used to figure out of this
error is exactly of the "no routes" type.

2.

If netutil.ChooseBindAddress() fails looking up IP route tables
it will fail with an error in which case the kubeadm config
code will hard stop.

This scenario is possible if the Linux user intentionally disables
the WiFi from the distribution settings. In such a case the distro
could empty files such files as /proc/net/route and ChooseBindAddress()
will return an error.

For improved offline support, don't error on such scenarios but instead
show a warning. This is done by using the NoRoutesError type.
Also default the address to 0.0.0.0.

While doing that, prevent some commands like `init`, `join` and also
phases like `controlplane` and `certs` from using such an invalid
address.

3.

If there is no internet, label versions fail and this breaks
air-gapped setups unless the users pass an explicit version.

To work around that:
- Remain using 'release/stable-x.xx' as the default version.
- On timeout or any error different from status 404 return error
- On status 404 fallback to using the version of the client via
kubeadmVersion()

Add unit tests for kubeadmVersion().

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
refs kubernetes/kubeadm#1041

**Special notes for your reviewer**:
1st and second commits fix offline support.
3rd commit fixes air-gabbed support (as discussed in the linked issue)

the api-machinery change is only fmt.Errorf() related.

**Release note**:

```release-note
kubeadm: fix air-gapped support and also allow some kubeadm commands to work without an available networking interface
```

/cc @kubernetes/sig-cluster-lifecycle-pr-reviews 
/cc @kubernetes/sig-api-machinery-pr-reviews 
/assign @kad
/assign @xiangpengzhao 
/area UX
/area kubeadm
/kind bug

cmd/kubeadm/app/cmd/init.go

@@ -231,6 +231,9 @@ func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha3.InitConfiguration,
 	if err != nil {
 		return nil, err
 	}
+	if err := configutil.VerifyAPIServerBindAddress(cfg.APIEndpoint.AdvertiseAddress); err != nil {
+		return nil, err
+	}
 
 	glog.V(1).Infof("[init] validating feature gates")
 	if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil {

cmd/kubeadm/app/cmd/join.go

@@ -277,6 +277,9 @@ func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha3.JoinC
 	if err != nil {
 		return nil, err
 	}
+	if err := configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress); err != nil {
+		return nil, err
+	}
 
 	fmt.Println("[preflight] running pre-flight checks")
 

cmd/kubeadm/app/cmd/phases/certs.go

@@ -223,6 +223,8 @@ func makeCommandForCert(cert *certsphase.KubeadmCert, caCert *certsphase.Kubeadm
 	certCmd.Run = func(cmd *cobra.Command, args []string) {
 		internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg)
 		kubeadmutil.CheckErr(err)
+		err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress)
+		kubeadmutil.CheckErr(err)
 
 		err = certsphase.CreateCertAndKeyFilesWithCA(cert, caCert, internalcfg)
 		kubeadmutil.CheckErr(err)

cmd/kubeadm/app/cmd/phases/controlplane.go

@@ -188,6 +188,8 @@ func runCmdControlPlane(cmdFunc func(outDir string, cfg *kubeadmapi.InitConfigur
 		// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
 		internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg)
 		kubeadmutil.CheckErr(err)
+		err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress)
+		kubeadmutil.CheckErr(err)
 
 		if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil {
 			kubeadmutil.CheckErr(err)

cmd/kubeadm/app/cmd/upgrade/BUILD

@@ -36,7 +36,6 @@ go_library(
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/client-go/discovery/fake:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -25,7 +25,6 @@ import (
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 
-	netutil "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -134,7 +133,7 @@ func NewCmdUpgradeControlPlane() *cobra.Command {
 			flags.nodeName = nodeName
 
 			if flags.advertiseAddress == "" {
-				ip, err := netutil.ChooseBindAddress(nil)
+				ip, err := configutil.ChooseAPIServerBindAddress(nil)
 				if err != nil {
 					kubeadmutil.CheckErr(err)
 					return

cmd/kubeadm/app/constants/constants.go

@@ -311,6 +311,9 @@ const (
 	// YAMLDocumentSeparator is the separator for YAML documents
 	// TODO: Find a better place for this constant
 	YAMLDocumentSeparator = "---\n"
+
+	// DefaultAPIServerBindAddress is the default bind address for the API Server
+	DefaultAPIServerBindAddress = "0.0.0.0"
 )
 
 var (

cmd/kubeadm/app/util/BUILD

@@ -24,6 +24,8 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//pkg/util/version:go_default_library",
+        "//pkg/version:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
@@ -33,6 +35,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )

cmd/kubeadm/app/util/config/common.go

@@ -19,11 +19,13 @@ package config
 import (
 	"fmt"
 	"io/ioutil"
+	"net"
 	"strings"
 
 	"github.com/golang/glog"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	netutil "k8s.io/apimachinery/pkg/util/net"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
@@ -144,3 +146,34 @@ func LowercaseSANs(sans []string) {
 		}
 	}
 }
+
+// VerifyAPIServerBindAddress can be used to verify if a bind address for the API Server is 0.0.0.0,
+// in which case this address is not valid and should not be used.
+func VerifyAPIServerBindAddress(address string) error {
+	ip := net.ParseIP(address)
+	if ip == nil {
+		return fmt.Errorf("cannot parse IP address: %s", address)
+	}
+	if !ip.IsGlobalUnicast() {
+		return fmt.Errorf("cannot use %q as the bind address for the API Server", address)
+	}
+	return nil
+}
+
+// ChooseAPIServerBindAddress is a wrapper for netutil.ChooseBindAddress that also handles
+// the case where no default routes were found and an IP for the API server could not be obatained.
+func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) {
+	ip, err := netutil.ChooseBindAddress(bindAddress)
+	if err != nil {
+		if netutil.IsNoRoutesError(err) {
+			glog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress)
+			defaultIP := net.ParseIP(constants.DefaultAPIServerBindAddress)
+			if defaultIP == nil {
+				return nil, fmt.Errorf("cannot parse default IP address: %s", constants.DefaultAPIServerBindAddress)
+			}
+			return defaultIP, nil
+		}
+		return nil, err
+	}
+	return ip, nil
+}

cmd/kubeadm/app/util/config/common_test.go

@@ -223,3 +223,53 @@ func TestLowercaseSANs(t *testing.T) {
 		})
 	}
 }
+
+func TestVerifyAPIServerBindAddress(t *testing.T) {
+	tests := []struct {
+		name          string
+		address       string
+		expectedError bool
+	}{
+		{
+			name:    "valid address: IPV4",
+			address: "192.168.0.1",
+		},
+		{
+			name:    "valid address: IPV6",
+			address: "2001:db8:85a3::8a2e:370:7334",
+		},
+		{
+			name:          "invalid address: not a global unicast 0.0.0.0",
+			address:       "0.0.0.0",
+			expectedError: true,
+		},
+		{
+			name:          "invalid address: not a global unicast 127.0.0.1",
+			address:       "127.0.0.1",
+			expectedError: true,
+		},
+		{
+			name:          "invalid address: not a global unicast ::",
+			address:       "::",
+			expectedError: true,
+		},
+		{
+			name:          "invalid address: cannot parse IPV4",
+			address:       "255.255.255.255.255",
+			expectedError: true,
+		},
+		{
+			name:          "invalid address: cannot parse IPV6",
+			address:       "2a00:800::2a00:800:10102a00",
+			expectedError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := VerifyAPIServerBindAddress(test.address); (err != nil) != test.expectedError {
+				t.Errorf("expected error: %v, got %v, error: %v", test.expectedError, (err != nil), err)
+			}
+		})
+	}
+}

cmd/kubeadm/app/util/config/masterconfig.go

@@ -29,7 +29,6 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	netutil "k8s.io/apimachinery/pkg/util/net"
 	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
@@ -107,9 +106,9 @@ func SetAPIEndpointDynamicDefaults(cfg *kubeadmapi.APIEndpoint) error {
 	if addressIP == nil && cfg.AdvertiseAddress != "" {
 		return fmt.Errorf("couldn't use \"%s\" as \"apiserver-advertise-address\", must be ipv4 or ipv6 address", cfg.AdvertiseAddress)
 	}
-	// Choose the right address for the API Server to advertise. If the advertise address is localhost or 0.0.0.0, the default interface's IP address is used
-	// This is the same logic as the API Server uses
-	ip, err := netutil.ChooseBindAddress(addressIP)
+	// This is the same logic as the API Server uses, except that if no interface is found the address is set to 0.0.0.0, which is invalid and cannot be used
+	// for bootstrapping a cluster.
+	ip, err := ChooseAPIServerBindAddress(addressIP)
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/util/version.go

@@ -17,6 +17,7 @@ limitations under the License.
 package util
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -24,7 +25,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/golang/glog"
 	netutil "k8s.io/apimachinery/pkg/util/net"
+	versionutil "k8s.io/kubernetes/pkg/util/version"
+	pkgversion "k8s.io/kubernetes/pkg/version"
 )
 
 const (
@@ -72,11 +76,22 @@ func KubernetesReleaseVersion(version string) (string, error) {
 		return ver, nil
 	}
 
+	// kubeReleaseLabelRegex matches labels such as: latest, latest-1, latest-1.10
 	if kubeReleaseLabelRegex.MatchString(versionLabel) {
 		url := fmt.Sprintf("%s/%s.txt", bucketURL, versionLabel)
 		body, err := fetchFromURL(url, getReleaseVersionTimeout)
 		if err != nil {
-			return "", err
+			// If the network operaton was successful but the server did not reply with StatusOK
+			if body != "" {
+				return "", err
+			}
+			// Handle air-gapped environments by falling back to the client version.
+			glog.Infof("could not fetch a Kubernetes version from the internet: %v", err)
+			body, err = kubeadmVersion(pkgversion.Get().String())
+			if err != nil {
+				return "", err
+			}
+			glog.Infof("falling back to the local client version: %s", body)
 		}
 		// Re-validate received version and return.
 		return KubernetesReleaseVersion(body)
@@ -138,18 +153,54 @@ func splitVersion(version string) (string, string, error) {
 
 // Internal helper: return content of URL
 func fetchFromURL(url string, timeout time.Duration) (string, error) {
+	glog.V(2).Infof("fetching Kubernetes version from URL: %s", url)
 	client := &http.Client{Timeout: timeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})}
 	resp, err := client.Get(url)
 	if err != nil {
 		return "", fmt.Errorf("unable to get URL %q: %s", url, err.Error())
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("unable to fetch file. URL: %q Status: %v", url, resp.Status)
-	}
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf("unable to read content of URL %q: %s", url, err.Error())
 	}
-	return strings.TrimSpace(string(body)), nil
+	bodyString := strings.TrimSpace(string(body))
+
+	if resp.StatusCode != http.StatusOK {
+		msg := fmt.Sprintf("unable to fetch file. URL: %q, status: %v", url, resp.Status)
+		return bodyString, errors.New(msg)
+	}
+	return bodyString, nil
+}
+
+// kubeadmVersion returns the version of the client without metadata.
+func kubeadmVersion(info string) (string, error) {
+	v, err := versionutil.ParseSemantic(info)
+	if err != nil {
+		return "", fmt.Errorf("kubeadm version error: %v", err)
+	}
+	// There is no utility in versionutil to get the version without the metadata,
+	// so this needs some manual formatting.
+	// Discard offsets after a release label and keep the labels down to e.g. `alpha.0` instead of
+	// including the offset e.g. `alpha.0.206`. This is done to comply with GCR image tags.
+	pre := v.PreRelease()
+	patch := v.Patch()
+	if len(pre) > 0 {
+		if patch > 0 {
+			// If the patch version is more than zero, decrement it and remove the label.
+			// this is done to comply with the latest stable patch release.
+			patch = patch - 1
+			pre = ""
+		} else {
+			split := strings.Split(pre, ".")
+			if len(split) > 2 {
+				pre = split[0] + "." + split[1] // Exclude the third element
+			} else if len(split) < 2 {
+				pre = split[0] + ".0" // Append .0 to a partial label
+			}
+			pre = "-" + pre
+		}
+	}
+	vStr := fmt.Sprintf("v%d.%d.%d%s", v.Major(), v.Minor(), patch, pre)
+	return vStr, nil
 }

cmd/kubeadm/app/util/version_test.go

@@ -290,3 +290,93 @@ func TestNormalizedBuildVersionVersion(t *testing.T) {
 		}
 	}
 }
+
+func TestKubeadmVersion(t *testing.T) {
+	type T struct {
+		name         string
+		input        string
+		output       string
+		outputError  bool
+		parsingError bool
+	}
+	cases := []T{
+		{
+			name:   "valid version with label and metadata",
+			input:  "v1.8.0-alpha.2.1231+afabd012389d53a",
+			output: "v1.8.0-alpha.2",
+		},
+		{
+			name:   "valid version with label and extra metadata",
+			input:  "v1.8.0-alpha.2.1231+afabd012389d53a.extra",
+			output: "v1.8.0-alpha.2",
+		},
+		{
+			name:   "valid patch version with label and extra metadata",
+			input:  "v1.11.3-beta.0.38+135cc4c1f47994",
+			output: "v1.11.2",
+		},
+		{
+			name:   "valid version with label extra",
+			input:  "v1.8.0-alpha.2.1231",
+			output: "v1.8.0-alpha.2",
+		},
+		{
+			name:   "valid patch version with label",
+			input:  "v1.9.11-beta.0",
+			output: "v1.9.10",
+		},
+		{
+			name:   "handle version with partial label",
+			input:  "v1.8.0-alpha",
+			output: "v1.8.0-alpha.0",
+		},
+		{
+			name:   "handle version missing 'v'",
+			input:  "1.11.0",
+			output: "v1.11.0",
+		},
+		{
+			name:   "valid version without label and metadata",
+			input:  "v1.8.0",
+			output: "v1.8.0",
+		},
+		{
+			name:   "valid patch version without label and metadata",
+			input:  "v1.8.2",
+			output: "v1.8.2",
+		},
+		{
+			name:         "invalid version",
+			input:        "foo",
+			parsingError: true,
+		},
+		{
+			name:         "invalid version with stray dash",
+			input:        "v1.9.11-",
+			parsingError: true,
+		},
+		{
+			name:         "invalid version without patch release",
+			input:        "v1.9",
+			parsingError: true,
+		},
+		{
+			name:        "invalid version with label and metadata",
+			input:       "v1.8.0-alpha.2.1231+afabd012389d53a",
+			output:      "v1.8.0-alpha.3",
+			outputError: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			output, err := kubeadmVersion(tc.input)
+			if (err != nil) != tc.parsingError {
+				t.Fatalf("expected error: %v, got: %v", tc.parsingError, err != nil)
+			}
+			if (output != tc.output) != tc.outputError {
+				t.Fatalf("expected output: %s, got: %s, for input: %s", tc.output, output, tc.input)
+			}
+		})
+	}
+}

staging/src/k8s.io/apimachinery/pkg/util/net/interface.go

@@ -53,6 +53,28 @@ type RouteFile struct {
 	parse func(input io.Reader) ([]Route, error)
 }
 
+// noRoutesError can be returned by ChooseBindAddress() in case of no routes
+type noRoutesError struct {
+	message string
+}
+
+func (e noRoutesError) Error() string {
+	return e.message
+}
+
+// IsNoRoutesError checks if an error is of type noRoutesError
+func IsNoRoutesError(err error) bool {
+	if err == nil {
+		return false
+	}
+	switch err.(type) {
+	case noRoutesError:
+		return true
+	default:
+		return false
+	}
+}
+
 var (
 	v4File = RouteFile{name: ipv4RouteFile, parse: getIPv4DefaultRoutes}
 	v6File = RouteFile{name: ipv6RouteFile, parse: getIPv6DefaultRoutes}
@@ -347,7 +369,9 @@ func getAllDefaultRoutes() ([]Route, error) {
 	v6Routes, _ := v6File.extract()
 	routes = append(routes, v6Routes...)
 	if len(routes) == 0 {
-		return nil, fmt.Errorf("No default routes.")
+		return nil, noRoutesError{
+			message: fmt.Sprintf("no default routes found in %q or %q", v4File.name, v6File.name),
+		}
 	}
 	return routes, nil
 }

staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go

@@ -669,7 +669,7 @@ func TestGetAllDefaultRoutes(t *testing.T) {
 		expected   []Route
 		errStrFrag string
 	}{
-		{"no routes", noInternetConnection, v6noDefaultRoutes, 0, nil, "No default routes"},
+		{"no routes", noInternetConnection, v6noDefaultRoutes, 0, nil, "no default routes"},
 		{"only v4 route", gatewayfirst, v6noDefaultRoutes, 1, routeV4, ""},
 		{"only v6 route", noInternetConnection, v6gatewayfirst, 1, routeV6, ""},
 		{"v4 and v6 routes", gatewayfirst, v6gatewayfirst, 2, bothRoutes, ""},