diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index 9636a85c9e3..69509a2243a 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -95,6 +95,7 @@ go_test( "gce_healthchecks_test.go", "gce_loadbalancer_external_test.go", "gce_test.go", + "gce_util_test.go", "metrics_test.go", ], importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/gce", diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 833ff56d500..b515c8ff67a 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -17,6 +17,7 @@ limitations under the License. package gce import ( + "context" "fmt" "io" "net/http" @@ -443,24 +444,26 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) { } else if config.SubnetworkName != "" { subnetURL = gceSubnetworkURL(config.ApiEndpoint, netProjID, config.Region, config.SubnetworkName) } else { - // Attempt to determine the subnetwork in case it's an automatic network. - // Legacy networks will not have a subnetwork, so subnetworkURL should remain empty. + // Determine the type of network and attempt to discover the correct subnet for AUTO mode. + // Gracefully fail because kubelet calls CreateGCECloud without any config, and minions + // lack the proper credentials for API calls. if networkName := lastComponent(networkURL); networkName != "" { - if n, err := getNetwork(service, netProjID, networkName); err != nil { - // Gracefully fail because kubelet calls CreateGCECloud without any config, and API calls will fail coming from minions. - glog.Warningf("Could not retrieve network %q in attempt to determine if legacy network or see list of subnets, err %v", networkURL, err) + var n *compute.Network + if n, err = getNetwork(service, netProjID, networkName); err != nil { + glog.Warningf("Could not retrieve network %q; err: %v", networkName, err) } else { - // Legacy networks have a non-empty IPv4Range - if len(n.IPv4Range) > 0 { - glog.Infof("Determined network %q is type legacy", networkURL) + switch typeOfNetwork(n) { + case netTypeLegacy: + glog.Infof("Network %q is type legacy - no subnetwork", networkName) isLegacyNetwork = true - } else { - // Try to find the subnet in the list of subnets - subnetURL = findSubnetForRegion(n.Subnetworks, config.Region) - if len(subnetURL) > 0 { - glog.Infof("Using subnet %q within network %q & region %q because none was specified.", subnetURL, n.Name, config.Region) + case netTypeCustom: + glog.Warningf("Network %q is type custom - cannot auto select a subnetwork", networkName) + case netTypeAuto: + subnetURL, err = determineSubnetURL(service, netProjID, networkName, config.Region) + if err != nil { + glog.Warningf("Could not determine subnetwork for network %q and region %v; err: %v", networkName, config.Region, err) } else { - glog.Warningf("Could not find any subnet in region %q within list %v.", config.Region, n.Subnetworks) + glog.Infof("Auto selecting subnetwork %q", subnetURL) } } } @@ -507,6 +510,30 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) { return gce, nil } +// determineSubnetURL queries for all subnetworks in a region for a given network and returns +// the URL of the subnetwork which exists in the auto-subnet range. +func determineSubnetURL(service *compute.Service, networkProjectID, networkName, region string) (string, error) { + subnets, err := listSubnetworksOfNetwork(service, networkProjectID, networkName, region) + if err != nil { + return "", err + } + + autoSubnets, err := subnetsInCIDR(subnets, autoSubnetIPRange) + if err != nil { + return "", err + } + + if len(autoSubnets) == 0 { + return "", fmt.Errorf("no subnet exists in auto CIDR") + } + + if len(autoSubnets) > 1 { + return "", fmt.Errorf("multiple subnetworks in the same region exist in auto CIDR") + } + + return autoSubnets[0].SelfLink, nil +} + func tryConvertToProjectNames(configProject, configNetworkProject string, service *compute.Service) (projID, netProjID string) { projID = configProject if isProjectNumber(projID) { @@ -734,6 +761,16 @@ func getNetwork(svc *compute.Service, networkProjectID, networkID string) (*comp return svc.Networks.Get(networkProjectID, networkID).Do() } +// listSubnetworksOfNetwork returns a list of subnetworks for a particular region of a network. +func listSubnetworksOfNetwork(svc *compute.Service, networkProjectID, networkID, region string) ([]*compute.Subnetwork, error) { + var subnets []*compute.Subnetwork + err := svc.Subnetworks.List(networkProjectID, region).Filter(fmt.Sprintf("network eq .*/%v$", networkID)).Pages(context.Background(), func(res *compute.SubnetworkList) error { + subnets = append(subnets, res.Items...) + return nil + }) + return subnets, err +} + // getProjectID returns the project's string ID given a project number or string func getProjectID(svc *compute.Service, projectNumberOrID string) (string, error) { proj, err := svc.Projects.Get(projectNumberOrID).Do() diff --git a/pkg/cloudprovider/providers/gce/gce_util.go b/pkg/cloudprovider/providers/gce/gce_util.go index 484a67e0b37..fb70d1dd053 100644 --- a/pkg/cloudprovider/providers/gce/gce_util.go +++ b/pkg/cloudprovider/providers/gce/gce_util.go @@ -19,6 +19,7 @@ package gce import ( "errors" "fmt" + "net" "net/http" "regexp" "strings" @@ -40,6 +41,13 @@ type gceInstance struct { Type string } +var ( + autoSubnetIPRange = &net.IPNet{ + IP: net.ParseIP("10.128.0.0"), + Mask: net.CIDRMask(9, 32), + } +) + var providerIdRE = regexp.MustCompile(`^` + ProviderName + `://([^/]+)/([^/]+)/([^/]+)$`) func getProjectAndZone() (string, string, error) { @@ -211,3 +219,58 @@ func handleAlphaNetworkTierGetError(err error) (string, error) { // Can't get the network tier, just return an error. return "", err } + +// containsCIDR returns true if outer contains inner. +func containsCIDR(outer, inner *net.IPNet) bool { + return outer.Contains(firstIPInRange(inner)) && outer.Contains(lastIPInRange(inner)) +} + +// firstIPInRange returns the first IP in a given IP range. +func firstIPInRange(ipNet *net.IPNet) net.IP { + return ipNet.IP.Mask(ipNet.Mask) +} + +// lastIPInRange returns the last IP in a given IP range. +func lastIPInRange(cidr *net.IPNet) net.IP { + ip := append([]byte{}, cidr.IP...) + for i, b := range cidr.Mask { + ip[i] |= ^b + } + return ip +} + +// subnetsInCIDR takes a list of subnets for a single region and +// returns subnets which exists in the specified CIDR range. +func subnetsInCIDR(subnets []*compute.Subnetwork, cidr *net.IPNet) ([]*compute.Subnetwork, error) { + var res []*compute.Subnetwork + for _, subnet := range subnets { + _, subnetRange, err := net.ParseCIDR(subnet.IpCidrRange) + if err != nil { + return nil, fmt.Errorf("unable to parse CIDR %q for subnet %q: %v", subnet.IpCidrRange, subnet.Name, err) + } + if containsCIDR(cidr, subnetRange) { + res = append(res, subnet) + } + } + return res, nil +} + +type netType string + +const ( + netTypeLegacy netType = "LEGACY" + netTypeAuto netType = "AUTO" + netTypeCustom netType = "CUSTOM" +) + +func typeOfNetwork(network *compute.Network) netType { + if network.IPv4Range != "" { + return netTypeLegacy + } + + if network.AutoCreateSubnetworks { + return netTypeAuto + } + + return netTypeCustom +} diff --git a/pkg/cloudprovider/providers/gce/gce_util_test.go b/pkg/cloudprovider/providers/gce/gce_util_test.go new file mode 100644 index 00000000000..f0bd4379b00 --- /dev/null +++ b/pkg/cloudprovider/providers/gce/gce_util_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "net" + "reflect" + "testing" + + compute "google.golang.org/api/compute/v1" +) + +func TestLastIPInRange(t *testing.T) { + for _, tc := range []struct { + cidr string + want string + }{ + {"10.1.2.3/32", "10.1.2.3"}, + {"10.1.2.0/31", "10.1.2.1"}, + {"10.1.0.0/30", "10.1.0.3"}, + {"10.0.0.0/29", "10.0.0.7"}, + {"::0/128", "::"}, + {"::0/127", "::1"}, + {"::0/126", "::3"}, + {"::0/120", "::ff"}, + } { + _, c, err := net.ParseCIDR(tc.cidr) + if err != nil { + t.Errorf("net.ParseCIDR(%v) = _, %v, %v; want nil", tc.cidr, c, err) + continue + } + + if lastIP := lastIPInRange(c); lastIP.String() != tc.want { + t.Errorf("LastIPInRange(%v) = %v; want %v", tc.cidr, lastIP, tc.want) + } + } +} + +func TestSubnetsInCIDR(t *testing.T) { + subnets := []*compute.Subnetwork{ + { + Name: "A", + IpCidrRange: "10.0.0.0/20", + }, + { + Name: "B", + IpCidrRange: "10.0.16.0/20", + }, + { + Name: "C", + IpCidrRange: "10.132.0.0/20", + }, + { + Name: "D", + IpCidrRange: "10.0.32.0/20", + }, + { + Name: "E", + IpCidrRange: "10.134.0.0/20", + }, + } + expectedNames := []string{"C", "E"} + + gotSubs, err := subnetsInCIDR(subnets, autoSubnetIPRange) + if err != nil { + t.Errorf("autoSubnetInList() = _, %v", err) + } + + var gotNames []string + for _, v := range gotSubs { + gotNames = append(gotNames, v.Name) + } + if !reflect.DeepEqual(gotNames, expectedNames) { + t.Errorf("autoSubnetInList() = %v, expected: %v", gotNames, expectedNames) + } +}