From 36970b370003ba00f9f0ad5c12ff2cad27387a93 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Tue, 14 Aug 2018 16:51:55 +0300 Subject: [PATCH 1/3] pkg/util/net: use a more descriptive error in getAllDefaultRoutes() Change the error output of getAllDefaultRoutes() so that it includes information on which files were probed for the IP routing tables even if such files are obvious. Introduce a new error type which can be used to figure out of this error is exactly of the "no routes" type. --- .../apimachinery/pkg/util/net/interface.go | 26 ++++++++++++++++++- .../pkg/util/net/interface_test.go | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go index 42816bd7059..0ab9b36080b 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface.go @@ -53,6 +53,28 @@ type RouteFile struct { parse func(input io.Reader) ([]Route, error) } +// noRoutesError can be returned by ChooseBindAddress() in case of no routes +type noRoutesError struct { + message string +} + +func (e noRoutesError) Error() string { + return e.message +} + +// IsNoRoutesError checks if an error is of type noRoutesError +func IsNoRoutesError(err error) bool { + if err == nil { + return false + } + switch err.(type) { + case noRoutesError: + return true + default: + return false + } +} + var ( v4File = RouteFile{name: ipv4RouteFile, parse: getIPv4DefaultRoutes} v6File = RouteFile{name: ipv6RouteFile, parse: getIPv6DefaultRoutes} @@ -347,7 +369,9 @@ func getAllDefaultRoutes() ([]Route, error) { v6Routes, _ := v6File.extract() routes = append(routes, v6Routes...) if len(routes) == 0 { - return nil, fmt.Errorf("No default routes.") + return nil, noRoutesError{ + message: fmt.Sprintf("no default routes found in %q or %q", v4File.name, v6File.name), + } } return routes, nil } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go index 4799d43aea0..d652f479d2c 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/net/interface_test.go @@ -669,7 +669,7 @@ func TestGetAllDefaultRoutes(t *testing.T) { expected []Route errStrFrag string }{ - {"no routes", noInternetConnection, v6noDefaultRoutes, 0, nil, "No default routes"}, + {"no routes", noInternetConnection, v6noDefaultRoutes, 0, nil, "no default routes"}, {"only v4 route", gatewayfirst, v6noDefaultRoutes, 1, routeV4, ""}, {"only v6 route", noInternetConnection, v6gatewayfirst, 1, routeV6, ""}, {"v4 and v6 routes", gatewayfirst, v6gatewayfirst, 2, bothRoutes, ""}, From 682b1b3d45497fcb2f47b4f0507e40166b7dfe62 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Tue, 14 Aug 2018 17:07:26 +0300 Subject: [PATCH 2/3] kubeadm: fix the air-gapped and offline support issues 1) Do not fail in case a bind address cannot be obtained If netutil.ChooseBindAddress() fails looking up IP route tables it will fail with an error in which case the kubeadm config code will hard stop. This scenario is possible if the Linux user intentionally disables the WiFi from the distribution settings. In such a case the distro could empty files such files as /proc/net/route and ChooseBindAddress() will return an error. For improved offline support, don't error on such scenarios but instead show a warning. This is done by using the NoRoutesError type. Also default the address to 0.0.0.0. While doing that, prevent some commands like `init`, `join` and also phases like `controlplane` and `certs` from using such an invalid address. Add unit tests for the new function for address verification. 2) Fallback to local client version If there is no internet, label versions fail and this breaks air-gapped setups unless the users pass an explicit version. To work around that: - Remain using 'release/stable-x.xx' as the default version. - On timeout or any error different from status 404 return error - On status 404 fallback to using the version of the client via kubeadmVersion() Add unit tests for kubeadmVersion(). Co-authored-by: Alexander Kanevskiy --- cmd/kubeadm/app/cmd/init.go | 3 + cmd/kubeadm/app/cmd/join.go | 3 + cmd/kubeadm/app/cmd/phases/certs.go | 2 + cmd/kubeadm/app/cmd/phases/controlplane.go | 2 + cmd/kubeadm/app/cmd/upgrade/node.go | 3 +- cmd/kubeadm/app/constants/constants.go | 3 + cmd/kubeadm/app/util/config/common.go | 33 ++++++++ cmd/kubeadm/app/util/config/common_test.go | 50 ++++++++++++ cmd/kubeadm/app/util/config/masterconfig.go | 7 +- cmd/kubeadm/app/util/version.go | 61 ++++++++++++-- cmd/kubeadm/app/util/version_test.go | 90 +++++++++++++++++++++ 11 files changed, 246 insertions(+), 11 deletions(-) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index c22b2587ff2..5624226c939 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -231,6 +231,9 @@ func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha3.InitConfiguration, if err != nil { return nil, err } + if err := configutil.VerifyAPIServerBindAddress(cfg.APIEndpoint.AdvertiseAddress); err != nil { + return nil, err + } glog.V(1).Infof("[init] validating feature gates") if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index f5ea1e7f9b8..f17ebe5e743 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -277,6 +277,9 @@ func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha3.JoinC if err != nil { return nil, err } + if err := configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress); err != nil { + return nil, err + } fmt.Println("[preflight] running pre-flight checks") diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index 94f0adf362a..d71bf219c4d 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -222,6 +222,8 @@ func makeCommandForCert(cert *certsphase.KubeadmCert, caCert *certsphase.Kubeadm certCmd.Run = func(cmd *cobra.Command, args []string) { internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) kubeadmutil.CheckErr(err) + err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress) + kubeadmutil.CheckErr(err) err = certsphase.CreateCertAndKeyFilesWithCA(cert, caCert, internalcfg) kubeadmutil.CheckErr(err) diff --git a/cmd/kubeadm/app/cmd/phases/controlplane.go b/cmd/kubeadm/app/cmd/phases/controlplane.go index 9585b3f7732..24b0ebcf92d 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane.go @@ -188,6 +188,8 @@ func runCmdControlPlane(cmdFunc func(outDir string, cfg *kubeadmapi.InitConfigur // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) kubeadmutil.CheckErr(err) + err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress) + kubeadmutil.CheckErr(err) if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil { kubeadmutil.CheckErr(err) diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 8758c7736cb..aa47106e74d 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -25,7 +25,6 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" - netutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -134,7 +133,7 @@ func NewCmdUpgradeControlPlane() *cobra.Command { flags.nodeName = nodeName if flags.advertiseAddress == "" { - ip, err := netutil.ChooseBindAddress(nil) + ip, err := configutil.ChooseAPIServerBindAddress(nil) if err != nil { kubeadmutil.CheckErr(err) return diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 7dc5f472df0..f18f7d0417c 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -311,6 +311,9 @@ const ( // YAMLDocumentSeparator is the separator for YAML documents // TODO: Find a better place for this constant YAMLDocumentSeparator = "---\n" + + // DefaultAPIServerBindAddress is the default bind address for the API Server + DefaultAPIServerBindAddress = "0.0.0.0" ) var ( diff --git a/cmd/kubeadm/app/util/config/common.go b/cmd/kubeadm/app/util/config/common.go index 067449d4ce9..3573f2e9789 100644 --- a/cmd/kubeadm/app/util/config/common.go +++ b/cmd/kubeadm/app/util/config/common.go @@ -19,11 +19,13 @@ package config import ( "fmt" "io/ioutil" + "net" "strings" "github.com/golang/glog" "k8s.io/apimachinery/pkg/runtime" + netutil "k8s.io/apimachinery/pkg/util/net" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" @@ -144,3 +146,34 @@ func LowercaseSANs(sans []string) { } } } + +// VerifyAPIServerBindAddress can be used to verify if a bind address for the API Server is 0.0.0.0, +// in which case this address is not valid and should not be used. +func VerifyAPIServerBindAddress(address string) error { + ip := net.ParseIP(address) + if ip == nil { + return fmt.Errorf("cannot parse IP address: %s", address) + } + if !ip.IsGlobalUnicast() { + return fmt.Errorf("cannot use %q as the bind address for the API Server", address) + } + return nil +} + +// ChooseAPIServerBindAddress is a wrapper for netutil.ChooseBindAddress that also handles +// the case where no default routes were found and an IP for the API server could not be obatained. +func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) { + ip, err := netutil.ChooseBindAddress(bindAddress) + if err != nil { + if netutil.IsNoRoutesError(err) { + glog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress) + defaultIP := net.ParseIP(constants.DefaultAPIServerBindAddress) + if defaultIP == nil { + return nil, fmt.Errorf("cannot parse default IP address: %s", constants.DefaultAPIServerBindAddress) + } + return defaultIP, nil + } + return nil, err + } + return ip, nil +} diff --git a/cmd/kubeadm/app/util/config/common_test.go b/cmd/kubeadm/app/util/config/common_test.go index ed1f4359a29..c945a1a9dd6 100644 --- a/cmd/kubeadm/app/util/config/common_test.go +++ b/cmd/kubeadm/app/util/config/common_test.go @@ -223,3 +223,53 @@ func TestLowercaseSANs(t *testing.T) { }) } } + +func TestVerifyAPIServerBindAddress(t *testing.T) { + tests := []struct { + name string + address string + expectedError bool + }{ + { + name: "valid address: IPV4", + address: "192.168.0.1", + }, + { + name: "valid address: IPV6", + address: "2001:db8:85a3::8a2e:370:7334", + }, + { + name: "invalid address: not a global unicast 0.0.0.0", + address: "0.0.0.0", + expectedError: true, + }, + { + name: "invalid address: not a global unicast 127.0.0.1", + address: "127.0.0.1", + expectedError: true, + }, + { + name: "invalid address: not a global unicast ::", + address: "::", + expectedError: true, + }, + { + name: "invalid address: cannot parse IPV4", + address: "255.255.255.255.255", + expectedError: true, + }, + { + name: "invalid address: cannot parse IPV6", + address: "2a00:800::2a00:800:10102a00", + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := VerifyAPIServerBindAddress(test.address); (err != nil) != test.expectedError { + t.Errorf("expected error: %v, got %v, error: %v", test.expectedError, (err != nil), err) + } + }) + } +} diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index d3fde8264c9..7d446dfe216 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -29,7 +29,6 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - netutil "k8s.io/apimachinery/pkg/util/net" bootstraputil "k8s.io/client-go/tools/bootstrap/token/util" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" @@ -107,9 +106,9 @@ func SetAPIEndpointDynamicDefaults(cfg *kubeadmapi.APIEndpoint) error { if addressIP == nil && cfg.AdvertiseAddress != "" { return fmt.Errorf("couldn't use \"%s\" as \"apiserver-advertise-address\", must be ipv4 or ipv6 address", cfg.AdvertiseAddress) } - // Choose the right address for the API Server to advertise. If the advertise address is localhost or 0.0.0.0, the default interface's IP address is used - // This is the same logic as the API Server uses - ip, err := netutil.ChooseBindAddress(addressIP) + // This is the same logic as the API Server uses, except that if no interface is found the address is set to 0.0.0.0, which is invalid and cannot be used + // for bootstrapping a cluster. + ip, err := ChooseAPIServerBindAddress(addressIP) if err != nil { return err } diff --git a/cmd/kubeadm/app/util/version.go b/cmd/kubeadm/app/util/version.go index 9c7795d576b..15cc900e992 100644 --- a/cmd/kubeadm/app/util/version.go +++ b/cmd/kubeadm/app/util/version.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "errors" "fmt" "io/ioutil" "net/http" @@ -24,7 +25,10 @@ import ( "strings" "time" + "github.com/golang/glog" netutil "k8s.io/apimachinery/pkg/util/net" + versionutil "k8s.io/kubernetes/pkg/util/version" + pkgversion "k8s.io/kubernetes/pkg/version" ) const ( @@ -72,11 +76,22 @@ func KubernetesReleaseVersion(version string) (string, error) { return ver, nil } + // kubeReleaseLabelRegex matches labels such as: latest, latest-1, latest-1.10 if kubeReleaseLabelRegex.MatchString(versionLabel) { url := fmt.Sprintf("%s/%s.txt", bucketURL, versionLabel) body, err := fetchFromURL(url, getReleaseVersionTimeout) if err != nil { - return "", err + // If the network operaton was successful but the server did not reply with StatusOK + if body != "" { + return "", err + } + // Handle air-gapped environments by falling back to the client version. + glog.Infof("could not fetch a Kubernetes version from the internet: %v", err) + body, err = kubeadmVersion(pkgversion.Get().String()) + if err != nil { + return "", err + } + glog.Infof("falling back to the local client version: %s", body) } // Re-validate received version and return. return KubernetesReleaseVersion(body) @@ -138,18 +153,54 @@ func splitVersion(version string) (string, string, error) { // Internal helper: return content of URL func fetchFromURL(url string, timeout time.Duration) (string, error) { + glog.V(2).Infof("fetching Kubernetes version from URL: %s", url) client := &http.Client{Timeout: timeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})} resp, err := client.Get(url) if err != nil { return "", fmt.Errorf("unable to get URL %q: %s", url, err.Error()) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("unable to fetch file. URL: %q Status: %v", url, resp.Status) - } body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("unable to read content of URL %q: %s", url, err.Error()) } - return strings.TrimSpace(string(body)), nil + bodyString := strings.TrimSpace(string(body)) + + if resp.StatusCode != http.StatusOK { + msg := fmt.Sprintf("unable to fetch file. URL: %q, status: %v", url, resp.Status) + return bodyString, errors.New(msg) + } + return bodyString, nil +} + +// kubeadmVersion returns the version of the client without metadata. +func kubeadmVersion(info string) (string, error) { + v, err := versionutil.ParseSemantic(info) + if err != nil { + return "", fmt.Errorf("kubeadm version error: %v", err) + } + // There is no utility in versionutil to get the version without the metadata, + // so this needs some manual formatting. + // Discard offsets after a release label and keep the labels down to e.g. `alpha.0` instead of + // including the offset e.g. `alpha.0.206`. This is done to comply with GCR image tags. + pre := v.PreRelease() + patch := v.Patch() + if len(pre) > 0 { + if patch > 0 { + // If the patch version is more than zero, decrement it and remove the label. + // this is done to comply with the latest stable patch release. + patch = patch - 1 + pre = "" + } else { + split := strings.Split(pre, ".") + if len(split) > 2 { + pre = split[0] + "." + split[1] // Exclude the third element + } else if len(split) < 2 { + pre = split[0] + ".0" // Append .0 to a partial label + } + pre = "-" + pre + } + } + vStr := fmt.Sprintf("v%d.%d.%d%s", v.Major(), v.Minor(), patch, pre) + return vStr, nil } diff --git a/cmd/kubeadm/app/util/version_test.go b/cmd/kubeadm/app/util/version_test.go index 017f25936a3..1027edd7e6f 100644 --- a/cmd/kubeadm/app/util/version_test.go +++ b/cmd/kubeadm/app/util/version_test.go @@ -290,3 +290,93 @@ func TestNormalizedBuildVersionVersion(t *testing.T) { } } } + +func TestKubeadmVersion(t *testing.T) { + type T struct { + name string + input string + output string + outputError bool + parsingError bool + } + cases := []T{ + { + name: "valid version with label and metadata", + input: "v1.8.0-alpha.2.1231+afabd012389d53a", + output: "v1.8.0-alpha.2", + }, + { + name: "valid version with label and extra metadata", + input: "v1.8.0-alpha.2.1231+afabd012389d53a.extra", + output: "v1.8.0-alpha.2", + }, + { + name: "valid patch version with label and extra metadata", + input: "v1.11.3-beta.0.38+135cc4c1f47994", + output: "v1.11.2", + }, + { + name: "valid version with label extra", + input: "v1.8.0-alpha.2.1231", + output: "v1.8.0-alpha.2", + }, + { + name: "valid patch version with label", + input: "v1.9.11-beta.0", + output: "v1.9.10", + }, + { + name: "handle version with partial label", + input: "v1.8.0-alpha", + output: "v1.8.0-alpha.0", + }, + { + name: "handle version missing 'v'", + input: "1.11.0", + output: "v1.11.0", + }, + { + name: "valid version without label and metadata", + input: "v1.8.0", + output: "v1.8.0", + }, + { + name: "valid patch version without label and metadata", + input: "v1.8.2", + output: "v1.8.2", + }, + { + name: "invalid version", + input: "foo", + parsingError: true, + }, + { + name: "invalid version with stray dash", + input: "v1.9.11-", + parsingError: true, + }, + { + name: "invalid version without patch release", + input: "v1.9", + parsingError: true, + }, + { + name: "invalid version with label and metadata", + input: "v1.8.0-alpha.2.1231+afabd012389d53a", + output: "v1.8.0-alpha.3", + outputError: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + output, err := kubeadmVersion(tc.input) + if (err != nil) != tc.parsingError { + t.Fatalf("expected error: %v, got: %v", tc.parsingError, err != nil) + } + if (output != tc.output) != tc.outputError { + t.Fatalf("expected output: %s, got: %s, for input: %s", tc.output, output, tc.input) + } + }) + } +} From 90df4b4adddeab08b814d469d875672fcccab695 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Tue, 14 Aug 2018 17:07:26 +0300 Subject: [PATCH 3/3] kubeadm: update auto-generated BUILD files --- cmd/kubeadm/app/cmd/upgrade/BUILD | 1 - cmd/kubeadm/app/util/BUILD | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD index b38438a0da0..73a1684fe16 100644 --- a/cmd/kubeadm/app/cmd/upgrade/BUILD +++ b/cmd/kubeadm/app/cmd/upgrade/BUILD @@ -36,7 +36,6 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/discovery/fake:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 2ce74082df5..4dc0c068f85 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -24,6 +24,8 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//pkg/util/version:go_default_library", + "//pkg/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", @@ -33,6 +35,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], )