diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 5fb27ab59e6..bf50ed15555 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -20,7 +20,7 @@ # The intent is to allow experimentation/advanced functionality before we # are ready to commit to supporting it. # Experimental functionality: -# KUBE_SHARE_MASTER=true +# KUBE_USE_EXISTING_MASTER=true # Detect and reuse an existing master; useful if you want to # create more nodes, perhaps with a different instance type or in # a different subnet/AZ @@ -808,8 +808,8 @@ function kube-up { # HTTPS to the master is allowed (for API access) authorize-security-group-ingress "${MASTER_SG_ID}" "--protocol tcp --port 443 --cidr 0.0.0.0/0" - # KUBE_SHARE_MASTER is used to add minions to an existing master - if [[ "${KUBE_SHARE_MASTER:-}" == "true" ]]; then + # KUBE_USE_EXISTING_MASTER is used to add minions to an existing master + if [[ "${KUBE_USE_EXISTING_MASTER:-}" == "true" ]]; then # Detect existing master detect-master diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index ab68a5ff18a..326844eb4c7 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -586,22 +586,42 @@ grains: - kubernetes-master cloud: gce EOF - if ! [[ -z "${PROJECT_ID:-}" ]] && ! [[ -z "${TOKEN_URL:-}" ]] && ! [[ -z "${TOKEN_BODY:-}" ]] && ! [[ -z "${NODE_NETWORK:-}" ]] ; then - cat </etc/gce.conf + + cat </etc/gce.conf [global] +EOF + CLOUD_CONFIG='' # Set to non-empty path if we are using the gce.conf file + + if ! [[ -z "${PROJECT_ID:-}" ]] && ! [[ -z "${TOKEN_URL:-}" ]] && ! [[ -z "${TOKEN_BODY:-}" ]] && ! [[ -z "${NODE_NETWORK:-}" ]] ; then + cat <>/etc/gce.conf token-url = ${TOKEN_URL} token-body = ${TOKEN_BODY} project-id = ${PROJECT_ID} network-name = ${NODE_NETWORK} EOF + CLOUD_CONFIG=/etc/gce.conf EXTERNAL_IP=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip") cat <>/etc/salt/minion.d/grains.conf - cloud_config: /etc/gce.conf advertise_address: '${EXTERNAL_IP}' proxy_ssh_user: '${PROXY_SSH_USER}' EOF fi + if [[ -n "${MULTIZONE:-}" ]]; then + cat <>/etc/gce.conf +multizone = ${MULTIZONE} +EOF + CLOUD_CONFIG=/etc/gce.conf + fi + + if [[ -n ${CLOUD_CONFIG:-} ]]; then + cat <>/etc/salt/minion.d/grains.conf + cloud_config: ${CLOUD_CONFIG} +EOF + else + rm -f /etc/gce.conf + fi + # If the kubelet on the master is enabled, give it the same CIDR range # as a generic node. if [[ ! -z "${KUBELET_APISERVER:-}" ]] && [[ ! -z "${KUBELET_CERT:-}" ]] && [[ ! -z "${KUBELET_KEY:-}" ]]; then diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f57f953e598..be69e01389f 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -575,6 +575,22 @@ function kube-up { find-release-tars upload-server-tars + if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then + create-nodes + create-autoscaler + else + check-existing + create-network + create-master + create-nodes-firewall + create-nodes-template + create-nodes + create-autoscaler + check-cluster + fi +} + +function check-existing() { local running_in_terminal=false # May be false if tty is not allocated (for example with ssh -T). if [ -t 1 ]; then @@ -595,7 +611,9 @@ function kube-up { fi fi fi +} +function create-network() { if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then echo "Creating new network: ${NETWORK}" # The network needs to be created synchronously or we have a race. The @@ -618,7 +636,9 @@ function kube-up { --source-ranges "0.0.0.0/0" \ --allow "tcp:22" & fi +} +function create-master() { echo "Starting master and configuring firewalls" gcloud compute firewall-rules create "${MASTER_NAME}-https" \ --project "${PROJECT}" \ @@ -663,7 +683,9 @@ function kube-up { create-certs "${MASTER_RESERVED_IP}" create-master-instance "${MASTER_RESERVED_IP}" & +} +function create-nodes-firewall() { # Create a single firewall rule for all minions. create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" & @@ -676,7 +698,9 @@ function kube-up { kube::util::wait-for-jobs || { echo -e "${color_red}${fail} commands failed.${color_norm}" >&2 } +} +function create-nodes-template() { echo "Creating minions." # TODO(zmerlynn): Refactor setting scope flags. @@ -690,8 +714,12 @@ function kube-up { write-node-env local template_name="${NODE_INSTANCE_PREFIX}-template" - + create-node-instance-template $template_name +} + +function create-nodes() { + local template_name="${NODE_INSTANCE_PREFIX}-template" local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-500} @@ -731,10 +759,9 @@ function kube-up { "${NODE_INSTANCE_PREFIX}-group" \ --zone "${ZONE}" \ --project "${PROJECT}" || true; +} - detect-node-names - detect-master - +function create-autoscaler() { # Create autoscaler for nodes if requested if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then METRICS="" @@ -764,6 +791,11 @@ function kube-up { gcloud compute instance-groups managed set-autoscaling "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}" \ --min-num-replicas "${last_min_instances}" --max-num-replicas "${last_max_instances}" ${METRICS} || true fi +} + +function check-cluster() { + detect-node-names + detect-master echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization." echo @@ -845,7 +877,7 @@ function kube-down { fi # Get the name of the managed instance group template before we delete the - # managed instange group. (The name of the managed instnace group template may + # managed instance group. (The name of the managed instance group template may # change during a cluster upgrade.) local template=$(get-template "${PROJECT}" "${ZONE}" "${NODE_INSTANCE_PREFIX}-group") @@ -1379,6 +1411,7 @@ OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-}) E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-}) KUBE_IMAGE_TAG: $(yaml-quote ${KUBE_IMAGE_TAG:-}) KUBE_DOCKER_REGISTRY: $(yaml-quote ${KUBE_DOCKER_REGISTRY:-}) +MULTIZONE: $(yaml-quote ${MULTIZONE:-}) EOF if [ -n "${KUBELET_PORT:-}" ]; then cat >>$file < len(longest_tag) { - longest_tag = tag - } + + for zone, hostNames := range hostNamesByZone { + listCall := gce.service.Instances.List(gce.projectID, zone) + + // Add the filter for hosts + listCall = listCall.Filter("name eq (" + strings.Join(hostNames, "|") + ")") + + // Add the fields we want + listCall = listCall.Fields("items(name,tags)") + + res, err := listCall.Do() + if err != nil { + return nil, err } - if len(longest_tag) > 0 { - tags.Insert(longest_tag) - } else if len(tags) > 0 { - return nil, fmt.Errorf("Some, but not all, instances have prefix tags (%s is missing)", instance.Name) + + for _, instance := range res.Items { + longest_tag := "" + for _, tag := range instance.Tags.Items { + if strings.HasPrefix(instance.Name, tag) && len(tag) > len(longest_tag) { + longest_tag = tag + } + } + if len(longest_tag) > 0 { + tags.Insert(longest_tag) + } else if len(tags) > 0 { + return nil, fmt.Errorf("Some, but not all, instances have prefix tags (%s is missing)", instance.Name) + } } } @@ -818,7 +879,12 @@ func (gce *GCECloud) createOrPromoteStaticIP(name, region, existingIP string) (i } // UpdateLoadBalancer is an implementation of LoadBalancer.UpdateLoadBalancer. -func (gce *GCECloud) UpdateLoadBalancer(name, region string, hosts []string) error { +func (gce *GCECloud) UpdateLoadBalancer(name, region string, hostNames []string) error { + hosts, err := gce.getInstancesByNames(hostNames) + if err != nil { + return err + } + pool, err := gce.service.TargetPools.Get(gce.projectID, region, name).Do() if err != nil { return err @@ -831,7 +897,7 @@ func (gce *GCECloud) UpdateLoadBalancer(name, region string, hosts []string) err var toAdd []*compute.InstanceReference var toRemove []*compute.InstanceReference for _, host := range hosts { - link := makeComparableHostPath(gce.zone, host) + link := host.makeComparableHostPath() if !existing.Has(link) { toAdd = append(toAdd, &compute.InstanceReference{Instance: link}) } @@ -1216,52 +1282,52 @@ func (gce *GCECloud) ListHttpHealthChecks() (*compute.HttpHealthCheckList, error // InstanceGroup Management // CreateInstanceGroup creates an instance group with the given instances. It is the callers responsibility to add named ports. -func (gce *GCECloud) CreateInstanceGroup(name string) (*compute.InstanceGroup, error) { +func (gce *GCECloud) CreateInstanceGroup(name string, zone string) (*compute.InstanceGroup, error) { op, err := gce.service.InstanceGroups.Insert( - gce.projectID, gce.zone, &compute.InstanceGroup{Name: name}).Do() + gce.projectID, zone, &compute.InstanceGroup{Name: name}).Do() if err != nil { return nil, err } - if err = gce.waitForZoneOp(op); err != nil { + if err = gce.waitForZoneOp(op, zone); err != nil { return nil, err } - return gce.GetInstanceGroup(name) + return gce.GetInstanceGroup(name, zone) } // DeleteInstanceGroup deletes an instance group. -func (gce *GCECloud) DeleteInstanceGroup(name string) error { +func (gce *GCECloud) DeleteInstanceGroup(name string, zone string) error { op, err := gce.service.InstanceGroups.Delete( - gce.projectID, gce.zone, name).Do() + gce.projectID, zone, name).Do() if err != nil { return err } - return gce.waitForZoneOp(op) + return gce.waitForZoneOp(op, zone) } // ListInstanceGroups lists all InstanceGroups in the project and zone. -func (gce *GCECloud) ListInstanceGroups() (*compute.InstanceGroupList, error) { - return gce.service.InstanceGroups.List(gce.projectID, gce.zone).Do() +func (gce *GCECloud) ListInstanceGroups(zone string) (*compute.InstanceGroupList, error) { + return gce.service.InstanceGroups.List(gce.projectID, zone).Do() } -// ListInstancesInInstanceGroup lists all the instances in a given istance group and state. -func (gce *GCECloud) ListInstancesInInstanceGroup(name string, state string) (*compute.InstanceGroupsListInstances, error) { +// ListInstancesInInstanceGroup lists all the instances in a given instance group and state. +func (gce *GCECloud) ListInstancesInInstanceGroup(name string, zone string, state string) (*compute.InstanceGroupsListInstances, error) { return gce.service.InstanceGroups.ListInstances( - gce.projectID, gce.zone, name, + gce.projectID, zone, name, &compute.InstanceGroupsListInstancesRequest{InstanceState: state}).Do() } // AddInstancesToInstanceGroup adds the given instances to the given instance group. -func (gce *GCECloud) AddInstancesToInstanceGroup(name string, instanceNames []string) error { +func (gce *GCECloud) AddInstancesToInstanceGroup(name string, zone string, instanceNames []string) error { if len(instanceNames) == 0 { return nil } // Adding the same instance twice will result in a 4xx error instances := []*compute.InstanceReference{} for _, ins := range instanceNames { - instances = append(instances, &compute.InstanceReference{Instance: makeHostURL(gce.projectID, gce.zone, ins)}) + instances = append(instances, &compute.InstanceReference{Instance: makeHostURL(gce.projectID, zone, ins)}) } op, err := gce.service.InstanceGroups.AddInstances( - gce.projectID, gce.zone, name, + gce.projectID, zone, name, &compute.InstanceGroupsAddInstancesRequest{ Instances: instances, }).Do() @@ -1269,21 +1335,21 @@ func (gce *GCECloud) AddInstancesToInstanceGroup(name string, instanceNames []st if err != nil { return err } - return gce.waitForZoneOp(op) + return gce.waitForZoneOp(op, zone) } // RemoveInstancesFromInstanceGroup removes the given instances from the instance group. -func (gce *GCECloud) RemoveInstancesFromInstanceGroup(name string, instanceNames []string) error { +func (gce *GCECloud) RemoveInstancesFromInstanceGroup(name string, zone string, instanceNames []string) error { if len(instanceNames) == 0 { return nil } instances := []*compute.InstanceReference{} for _, ins := range instanceNames { - instanceLink := makeHostURL(gce.projectID, gce.zone, ins) + instanceLink := makeHostURL(gce.projectID, zone, ins) instances = append(instances, &compute.InstanceReference{Instance: instanceLink}) } op, err := gce.service.InstanceGroups.RemoveInstances( - gce.projectID, gce.zone, name, + gce.projectID, zone, name, &compute.InstanceGroupsRemoveInstancesRequest{ Instances: instances, }).Do() @@ -1294,7 +1360,7 @@ func (gce *GCECloud) RemoveInstancesFromInstanceGroup(name string, instanceNames } return err } - return gce.waitForZoneOp(op) + return gce.waitForZoneOp(op, zone) } // AddPortToInstanceGroup adds a port to the given instance group. @@ -1309,21 +1375,21 @@ func (gce *GCECloud) AddPortToInstanceGroup(ig *compute.InstanceGroup, port int6 namedPort := compute.NamedPort{Name: fmt.Sprintf("port%v", port), Port: port} ig.NamedPorts = append(ig.NamedPorts, &namedPort) op, err := gce.service.InstanceGroups.SetNamedPorts( - gce.projectID, gce.zone, ig.Name, + gce.projectID, ig.Zone, ig.Name, &compute.InstanceGroupsSetNamedPortsRequest{ NamedPorts: ig.NamedPorts}).Do() if err != nil { return nil, err } - if err = gce.waitForZoneOp(op); err != nil { + if err = gce.waitForZoneOp(op, ig.Zone); err != nil { return nil, err } return &namedPort, nil } // GetInstanceGroup returns an instance group by name. -func (gce *GCECloud) GetInstanceGroup(name string) (*compute.InstanceGroup, error) { - return gce.service.InstanceGroups.Get(gce.projectID, gce.zone, name).Do() +func (gce *GCECloud) GetInstanceGroup(name string, zone string) (*compute.InstanceGroup, error) { + return gce.service.InstanceGroups.Get(gce.projectID, zone, name).Do() } // Take a GCE instance 'hostname' and break it down to something that can be fed @@ -1337,20 +1403,6 @@ func canonicalizeInstanceName(name string) string { return name } -// Return the instances matching the relevant name. -func (gce *GCECloud) getInstanceByName(name string) (*compute.Instance, error) { - name = canonicalizeInstanceName(name) - res, err := gce.service.Instances.Get(gce.projectID, gce.zone, name).Do() - if err != nil { - glog.Errorf("Failed to retrieve TargetInstance resource for instance: %s", name) - if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code == http.StatusNotFound { - return nil, cloudprovider.InstanceNotFound - } - return nil, err - } - return res, nil -} - // Implementation of Instances.CurrentNodeName func (gce *GCECloud) CurrentNodeName(hostname string) (string, error) { return hostname, nil @@ -1446,27 +1498,34 @@ func (gce *GCECloud) ExternalID(instance string) (string, error) { if err != nil { return "", err } - return strconv.FormatUint(inst.Id, 10), nil + return strconv.FormatUint(inst.ID, 10), nil } // InstanceID returns the cloud provider ID of the specified instance. -func (gce *GCECloud) InstanceID(instance string) (string, error) { - return gce.projectID + "/" + gce.zone + "/" + canonicalizeInstanceName(instance), nil +func (gce *GCECloud) InstanceID(instanceName string) (string, error) { + instance, err := gce.getInstanceByName(instanceName) + if err != nil { + return "", err + } + return gce.projectID + "/" + instance.Zone + "/" + instance.Name, nil } // List is an implementation of Instances.List. func (gce *GCECloud) List(filter string) ([]string, error) { - listCall := gce.service.Instances.List(gce.projectID, gce.zone) - if len(filter) > 0 { - listCall = listCall.Filter("name eq " + filter) - } - res, err := listCall.Do() - if err != nil { - return nil, err - } var instances []string - for _, instance := range res.Items { - instances = append(instances, instance.Name) + // TODO: Parallelize, although O(zones) so not too bad (N <= 3 typically) + for _, zone := range gce.managedZones { + listCall := gce.service.Instances.List(gce.projectID, zone) + if len(filter) > 0 { + listCall = listCall.Filter("name eq " + filter) + } + res, err := listCall.Do() + if err != nil { + return nil, err + } + for _, instance := range res.Items { + instances = append(instances, instance.Name) + } } return instances, nil } @@ -1524,11 +1583,14 @@ func gceNetworkURL(project, network string) string { func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error { routeName := truncateClusterName(clusterName) + "-" + nameHint - instanceName := canonicalizeInstanceName(route.TargetInstance) + targetInstance, err := gce.getInstanceByName(route.TargetInstance) + if err != nil { + return err + } insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{ Name: routeName, DestRange: route.DestinationCIDR, - NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", gce.zone, instanceName), + NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name), Network: gce.networkURL, Priority: 1000, Description: k8sNodeRouteTag, @@ -1548,40 +1610,46 @@ func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) } func (gce *GCECloud) GetZone() (cloudprovider.Zone, error) { - region, err := getGceRegion(gce.zone) - if err != nil { - return cloudprovider.Zone{}, err - } return cloudprovider.Zone{ - FailureDomain: gce.zone, - Region: region, + FailureDomain: gce.localZone, + Region: gce.region, }, nil } -func (gce *GCECloud) CreateDisk(name string, sizeGb int64) error { +// Create a new Persistent Disk, with the specified name & size, in the specified zone. +func (gce *GCECloud) CreateDisk(name string, zone string, sizeGb int64) error { diskToCreate := &compute.Disk{ Name: name, SizeGb: sizeGb, } - createOp, err := gce.service.Disks.Insert(gce.projectID, gce.zone, diskToCreate).Do() + createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Do() if err != nil { return err } - return gce.waitForZoneOp(createOp) + return gce.waitForZoneOp(createOp, zone) } func (gce *GCECloud) DeleteDisk(diskToDelete string) error { - deleteOp, err := gce.service.Disks.Delete(gce.projectID, gce.zone, diskToDelete).Do() + disk, err := gce.getDiskByNameUnknownZone(diskToDelete) if err != nil { return err } - return gce.waitForZoneOp(deleteOp) + deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Do() + if err != nil { + return err + } + + return gce.waitForZoneOp(deleteOp, disk.Zone) } func (gce *GCECloud) AttachDisk(diskName, instanceID string, readOnly bool) error { - disk, err := gce.getDisk(diskName) + instance, err := gce.getInstanceByName(instanceID) + if err != nil { + return fmt.Errorf("error getting instance %q", instanceID) + } + disk, err := gce.getDiskByName(diskName, instance.Zone) if err != nil { return err } @@ -1591,25 +1659,30 @@ func (gce *GCECloud) AttachDisk(diskName, instanceID string, readOnly bool) erro } attachedDisk := gce.convertDiskToAttachedDisk(disk, readWrite) - attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, gce.zone, instanceID, attachedDisk).Do() + attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instanceID, attachedDisk).Do() if err != nil { return err } - return gce.waitForZoneOp(attachOp) + return gce.waitForZoneOp(attachOp, disk.Zone) } func (gce *GCECloud) DetachDisk(devicePath, instanceID string) error { - detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, gce.zone, instanceID, devicePath).Do() + inst, err := gce.getInstanceByName(instanceID) + if err != nil { + return fmt.Errorf("error getting instance %q", instanceID) + } + + detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Do() if err != nil { return err } - return gce.waitForZoneOp(detachOp) + return gce.waitForZoneOp(detachOp, inst.Zone) } func (gce *GCECloud) DiskIsAttached(diskName, instanceID string) (bool, error) { - instance, err := gce.service.Instances.Get(gce.projectID, gce.zone, instanceID).Do() + instance, err := gce.getInstanceByName(instanceID) if err != nil { return false, err } @@ -1624,15 +1697,62 @@ func (gce *GCECloud) DiskIsAttached(diskName, instanceID string) (bool, error) { return false, nil } -func (gce *GCECloud) getDisk(diskName string) (*compute.Disk, error) { - return gce.service.Disks.Get(gce.projectID, gce.zone, diskName).Do() +// Returns a gceDisk for the disk, if it is found in the specified zone. +// If not found, returns (nil, nil) +func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, error) { + disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Do() + if err == nil { + d := &gceDisk{ + Zone: lastComponent(disk.Zone), + Name: disk.Name, + Kind: disk.Kind, + } + return d, nil + } + if !isHTTPErrorCode(err, http.StatusNotFound) { + return nil, err + } + return nil, nil } -// getGceRegion returns region of the gce zone. Zone names +// Like findDiskByName, but returns an error if the disk is not found +func (gce *GCECloud) getDiskByName(diskName string, zone string) (*gceDisk, error) { + disk, err := gce.findDiskByName(diskName, zone) + if disk == nil && err == nil { + return nil, fmt.Errorf("GCE persistent disk not found: %q", diskName) + } + return disk, err +} + +// Scans all managed zones to return the GCE PD +// Prefer getDiskByName, if the zone can be established +func (gce *GCECloud) getDiskByNameUnknownZone(diskName string) (*gceDisk, error) { + // Note: this is the gotcha right now with GCE PD support: + // disk names are not unique per-region. + // (I can create two volumes with name "myvol" in e.g. us-central1-b & us-central1-f) + // For now, this is simply undefined behvaiour. + // + // In future, we will have to require users to qualify their disk + // "us-central1-a/mydisk". We could do this for them as part of + // admission control, but that might be a little weird (values changing + // on create) + for _, zone := range gce.managedZones { + disk, err := gce.findDiskByName(diskName, zone) + if err != nil { + return nil, err + } + if disk != nil { + return disk, nil + } + } + return nil, fmt.Errorf("GCE persistent disk not found: %q", diskName) +} + +// GetGCERegion returns region of the gce zone. Zone names // are of the form: ${region-name}-${ix}. // For example "us-central1-b" has a region of "us-central1". // So we look for the last '-' and trim to just before that. -func getGceRegion(zone string) (string, error) { +func GetGCERegion(zone string) (string, error) { ix := strings.LastIndex(zone, "-") if ix == -1 { return "", fmt.Errorf("unexpected zone: %s", zone) @@ -1641,18 +1761,18 @@ func getGceRegion(zone string) (string, error) { } // Converts a Disk resource to an AttachedDisk resource. -func (gce *GCECloud) convertDiskToAttachedDisk(disk *compute.Disk, readWrite string) *compute.AttachedDisk { +func (gce *GCECloud) convertDiskToAttachedDisk(disk *gceDisk, readWrite string) *compute.AttachedDisk { return &compute.AttachedDisk{ DeviceName: disk.Name, Kind: disk.Kind, Mode: readWrite, - Source: "https://" + path.Join("www.googleapis.com/compute/v1/projects/", gce.projectID, "zones", gce.zone, "disks", disk.Name), + Source: "https://" + path.Join("www.googleapis.com/compute/v1/projects/", gce.projectID, "zones", disk.Zone, "disks", disk.Name), Type: "PERSISTENT", } } -func (gce *GCECloud) ListClusters() ([]string, error) { - list, err := gce.containerService.Projects.Zones.Clusters.List(gce.projectID, gce.zone).Do() +func (gce *GCECloud) listClustersInZone(zone string) ([]string, error) { + list, err := gce.containerService.Projects.Zones.Clusters.List(gce.projectID, zone).Do() if err != nil { return nil, err } @@ -1663,6 +1783,129 @@ func (gce *GCECloud) ListClusters() ([]string, error) { return result, nil } +func (gce *GCECloud) ListClusters() ([]string, error) { + allClusters := []string{} + + for _, zone := range gce.managedZones { + clusters, err := gce.listClustersInZone(zone) + if err != nil { + return nil, err + } + // TODO: Scoping? Do we need to qualify the cluster name? + allClusters = append(allClusters, clusters...) + } + + return allClusters, nil +} + func (gce *GCECloud) Master(clusterName string) (string, error) { return "k8s-" + clusterName + "-master.internal", nil } + +type gceInstance struct { + Zone string + Name string + ID uint64 + Disks []*compute.AttachedDisk +} + +type gceDisk struct { + Zone string + Name string + Kind string +} + +func (gce *GCECloud) getInstancesByNames(names []string) ([]*gceInstance, error) { + instances := make(map[string]*gceInstance) + + for _, name := range names { + name = canonicalizeInstanceName(name) + instances[name] = nil + } + + for _, zone := range gce.managedZones { + var remaining []string + for name, instance := range instances { + if instance == nil { + remaining = append(remaining, name) + } + } + + if len(remaining) == 0 { + break + } + + listCall := gce.service.Instances.List(gce.projectID, zone) + + // Add the filter for hosts + listCall = listCall.Filter("name eq (" + strings.Join(remaining, "|") + ")") + + listCall = listCall.Fields("items(name,id,disks)") + + res, err := listCall.Do() + if err != nil { + return nil, err + } + + for _, i := range res.Items { + name := i.Name + instance := &gceInstance{ + Zone: zone, + Name: name, + ID: i.Id, + Disks: i.Disks, + } + instances[name] = instance + } + } + + instanceArray := make([]*gceInstance, len(names)) + for i, name := range names { + instance := instances[name] + if instance == nil { + return nil, fmt.Errorf("failed to retrieve instance: %q", name) + } + instanceArray[i] = instances[name] + } + + return instanceArray, nil +} + +func (gce *GCECloud) getInstanceByName(name string) (*gceInstance, error) { + // Avoid changing behaviour when not managing multiple zones + if len(gce.managedZones) == 1 { + name = canonicalizeInstanceName(name) + zone := gce.managedZones[0] + res, err := gce.service.Instances.Get(gce.projectID, zone, name).Do() + if err != nil { + if !isHTTPErrorCode(err, http.StatusNotFound) { + return nil, err + } + } + return &gceInstance{ + Zone: lastComponent(res.Zone), + Name: res.Name, + ID: res.Id, + Disks: res.Disks, + }, nil + } + + instances, err := gce.getInstancesByNames([]string{name}) + if err != nil { + return nil, err + } + if len(instances) != 1 || instances[0] == nil { + return nil, fmt.Errorf("unexpected return value from getInstancesByNames: %v", instances) + } + return instances[0], nil +} + +// Returns the last component of a URL, i.e. anything after the last slash +// If there is no slash, returns the whole string +func lastComponent(s string) string { + lastSlash := strings.LastIndex(s, "/") + if lastSlash != -1 { + s = s[lastSlash+1:] + } + return s +} diff --git a/pkg/cloudprovider/providers/gce/gce_test.go b/pkg/cloudprovider/providers/gce/gce_test.go index a614b3065f6..f1633d8890d 100644 --- a/pkg/cloudprovider/providers/gce/gce_test.go +++ b/pkg/cloudprovider/providers/gce/gce_test.go @@ -22,8 +22,17 @@ import ( ) func TestGetRegion(t *testing.T) { + zoneName := "us-central1-b" + regionName, err := GetGCERegion(zoneName) + if err != nil { + t.Fatalf("unexpected error from GetGCERegion: %v", err) + } + if regionName != "us-central1" { + t.Errorf("Unexpected region from GetGCERegion: %s", regionName) + } gce := &GCECloud{ - zone: "us-central1-b", + localZone: zoneName, + region: regionName, } zones, ok := gce.Zones() if !ok { @@ -91,7 +100,11 @@ func TestComparingHostURLs(t *testing.T) { for _, test := range tests { link1 := hostURLToComparablePath(test.host1) - link2 := makeComparableHostPath(test.zone, test.name) + testInstance := &gceInstance{ + Name: canonicalizeInstanceName(test.name), + Zone: test.zone, + } + link2 := testInstance.makeComparableHostPath() if test.expectEqual && link1 != link2 { t.Errorf("expected link1 and link2 to be equal, got %s and %s", link1, link2) } else if !test.expectEqual && link1 == link2 { diff --git a/pkg/volume/gce_pd/gce_util.go b/pkg/volume/gce_pd/gce_util.go index 3f184f78914..fddf7312359 100644 --- a/pkg/volume/gce_pd/gce_util.go +++ b/pkg/volume/gce_pd/gce_util.go @@ -137,7 +137,16 @@ func (gceutil *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner) (volum requestBytes := c.options.Capacity.Value() // GCE works with gigabytes, convert to GiB with rounding up requestGB := volume.RoundUpSize(requestBytes, 1024*1024*1024) - err = cloud.CreateDisk(name, int64(requestGB)) + + // The disk will be created in the zone in which this code is currently running + // TODO: We should support auto-provisioning volumes in multiple/specified zones + zone, err := cloud.GetZone() + if err != nil { + glog.V(2).Infof("error getting zone information from GCE: %v", err) + return "", 0, err + } + + err = cloud.CreateDisk(name, zone.FailureDomain, int64(requestGB)) if err != nil { glog.V(2).Infof("Error creating GCE PD volume: %v", err) return "", 0, err diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 802b01097b4..6fc3488ef36 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -119,7 +119,13 @@ func TestE2E(t *testing.T) { Logf("Using service account %q as token source.", cloudConfig.ServiceAccount) tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount) } - cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, testContext.CloudConfig.Zone, "" /* networkUrl */, tokenSource, false /* useMetadataServer */) + zone := testContext.CloudConfig.Zone + region, err := gcecloud.GetGCERegion(zone) + if err != nil { + glog.Fatalf("error parsing GCE region from zone %q: %v", zone, err) + } + managedZones := []string{zone} // Only single-zone for now + cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */) if err != nil { glog.Fatal("Error building GCE provider: ", err) } diff --git a/test/e2e/pd.go b/test/e2e/pd.go index 22ba5fcb9fc..bb491afe272 100644 --- a/test/e2e/pd.go +++ b/test/e2e/pd.go @@ -314,7 +314,7 @@ func createPD() (string, error) { return "", err } - err = gceCloud.CreateDisk(pdName, 10 /* sizeGb */) + err = gceCloud.CreateDisk(pdName, testContext.CloudConfig.Zone, 10 /* sizeGb */) if err != nil { return "", err }