diff --git a/pkg/api/types.go b/pkg/api/types.go index 28acced55c2..f898afdbf63 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -781,6 +781,8 @@ type NodeSpec struct { // PodCIDR represents the pod IP range assigned to the node // Note: assigning IP ranges to nodes might need to be revisited when we support migratable IPs. PodCIDR string `json:"cidr,omitempty"` + // External ID of the node assigned by some machine database (e.g. a cloud provider) + ExternalID string `json:"externalID,omitempty"` } // NodeStatus is information about the current status of a node. diff --git a/pkg/api/v1beta1/conversion.go b/pkg/api/v1beta1/conversion.go index 7126d61ee9b..b1496605ba4 100644 --- a/pkg/api/v1beta1/conversion.go +++ b/pkg/api/v1beta1/conversion.go @@ -699,6 +699,7 @@ func init() { out.HostIP = in.Status.HostIP out.PodCIDR = in.Spec.PodCIDR + out.ExternalID = in.Spec.ExternalID return s.Convert(&in.Spec.Capacity, &out.NodeResources.Capacity, 0) }, func(in *Minion, out *newer.Node, s conversion.Scope) error { @@ -720,6 +721,7 @@ func init() { out.Status.HostIP = in.HostIP out.Spec.PodCIDR = in.PodCIDR + out.Spec.ExternalID = in.ExternalID return s.Convert(&in.NodeResources.Capacity, &out.Spec.Capacity, 0) }, func(in *newer.LimitRange, out *LimitRange, s conversion.Scope) error { diff --git a/pkg/api/v1beta1/types.go b/pkg/api/v1beta1/types.go index 9e2bd555a68..9354847994a 100644 --- a/pkg/api/v1beta1/types.go +++ b/pkg/api/v1beta1/types.go @@ -690,6 +690,8 @@ type Minion struct { Status NodeStatus `json:"status,omitempty" description:"current status of node"` // Labels for the node Labels map[string]string `json:"labels,omitempty" description:"map of string keys and values that can be used to organize and categorize minions; labels of a minion assigned by the scheduler must match the scheduled pod's nodeSelector"` + // External ID of the node + ExternalID string `json:"externalID,omitempty" description:"external id of the node assigned by some machine database (e.g. a cloud provider)"` } // MinionList is a list of minions. diff --git a/pkg/api/v1beta2/conversion.go b/pkg/api/v1beta2/conversion.go index c1c5e7603b8..a0ac51de0d0 100644 --- a/pkg/api/v1beta2/conversion.go +++ b/pkg/api/v1beta2/conversion.go @@ -619,6 +619,7 @@ func init() { out.HostIP = in.Status.HostIP out.PodCIDR = in.Spec.PodCIDR + out.ExternalID = in.Spec.ExternalID return s.Convert(&in.Spec.Capacity, &out.NodeResources.Capacity, 0) }, func(in *Minion, out *newer.Node, s conversion.Scope) error { @@ -640,6 +641,7 @@ func init() { out.Status.HostIP = in.HostIP out.Spec.PodCIDR = in.PodCIDR + out.Spec.ExternalID = in.ExternalID return s.Convert(&in.NodeResources.Capacity, &out.Spec.Capacity, 0) }, func(in *newer.LimitRange, out *LimitRange, s conversion.Scope) error { diff --git a/pkg/api/v1beta2/types.go b/pkg/api/v1beta2/types.go index 6787cc6c32e..8d51040071b 100644 --- a/pkg/api/v1beta2/types.go +++ b/pkg/api/v1beta2/types.go @@ -654,6 +654,8 @@ type Minion struct { Status NodeStatus `json:"status,omitempty" description:"current status of node"` // Labels for the node Labels map[string]string `json:"labels,omitempty" description:"map of string keys and values that can be used to organize and categorize minions; labels of a minion assigned by the scheduler must match the scheduled pod's nodeSelector"` + // External ID of the node + ExternalID string `json:"externalID,omitempty" description:"external id of the node assigned by some machine database (e.g. a cloud provider)"` } // MinionList is a list of minions. diff --git a/pkg/api/v1beta3/types.go b/pkg/api/v1beta3/types.go index 2258946e6ba..e10c45e058e 100644 --- a/pkg/api/v1beta3/types.go +++ b/pkg/api/v1beta3/types.go @@ -814,6 +814,8 @@ type NodeSpec struct { Capacity ResourceList `json:"capacity,omitempty"` // PodCIDR represents the pod IP range assigned to the node PodCIDR string `json:"cidr,omitempty"` + // External ID of the node assigned by some machine database (e.g. a cloud provider) + ExternalID string `json:"externalID,omitempty"` } // NodeStatus is information about the current status of a node. diff --git a/pkg/cloudprovider/aws/aws.go b/pkg/cloudprovider/aws/aws.go index fda91ee960e..41e82078651 100644 --- a/pkg/cloudprovider/aws/aws.go +++ b/pkg/cloudprovider/aws/aws.go @@ -122,6 +122,28 @@ func (aws *AWSCloud) Zones() (cloudprovider.Zones, bool) { // IPAddress is an implementation of Instances.IPAddress. func (aws *AWSCloud) IPAddress(name string) (net.IP, error) { + inst, err := aws.getInstancesByDnsName(name) + if err != nil { + return nil, err + } + ip := net.ParseIP(inst.PrivateIpAddress) + if ip == nil { + return nil, fmt.Errorf("invalid network IP: %s", inst.PrivateIpAddress) + } + return ip, nil +} + +// ExternalID returns the cloud provider ID of the specified instance. +func (aws *AWSCloud) ExternalID(name string) (string, error) { + inst, err := aws.getInstancesByDnsName(name) + if err != nil { + return "", err + } + return inst.InstanceId, nil +} + +// Return the instances matching the relevant private dns name. +func (aws *AWSCloud) getInstancesByDnsName(name string) (*ec2.Instance, error) { f := ec2.NewFilter() f.Add("private-dns-name", name) @@ -142,12 +164,7 @@ func (aws *AWSCloud) IPAddress(name string) (net.IP, error) { return nil, fmt.Errorf("multiple instances found for host: %s", name) } - ipAddress := resp.Reservations[0].Instances[0].PrivateIpAddress - ip := net.ParseIP(ipAddress) - if ip == nil { - return nil, fmt.Errorf("invalid network IP: %s", ipAddress) - } - return ip, nil + return &resp.Reservations[0].Instances[0], nil } // Return a list of instances matching regex string. diff --git a/pkg/cloudprovider/cloud.go b/pkg/cloudprovider/cloud.go index 5ba83617940..7960c00c57e 100644 --- a/pkg/cloudprovider/cloud.go +++ b/pkg/cloudprovider/cloud.go @@ -59,6 +59,8 @@ type TCPLoadBalancer interface { type Instances interface { // IPAddress returns an IP address of the specified instance. IPAddress(name string) (net.IP, error) + // ExternalID returns the cloud provider ID of the specified instance. + ExternalID(name string) (string, error) // List lists instances that match 'filter' which is a regular expression which must match the entire instance name (fqdn) List(filter string) ([]string, error) // GetNodeResources gets the resources for a particular node diff --git a/pkg/cloudprovider/controller/nodecontroller.go b/pkg/cloudprovider/controller/nodecontroller.go index 180016ee44f..6771a3ee15d 100644 --- a/pkg/cloudprovider/controller/nodecontroller.go +++ b/pkg/cloudprovider/controller/nodecontroller.go @@ -231,6 +231,12 @@ func (s *NodeController) PopulateIPs(nodes *api.NodeList) (*api.NodeList, error) } else { node.Status.HostIP = hostIP.String() } + instanceID, err := instances.ExternalID(node.Name) + if err != nil { + glog.Errorf("error getting instance id for %s: %v", node.Name, err) + } else { + node.Spec.ExternalID = instanceID + } } } else { for i := range nodes.Items { diff --git a/pkg/cloudprovider/fake/fake.go b/pkg/cloudprovider/fake/fake.go index 4efc6d82d78..b4bf3b37c9c 100644 --- a/pkg/cloudprovider/fake/fake.go +++ b/pkg/cloudprovider/fake/fake.go @@ -30,6 +30,7 @@ type FakeCloud struct { Err error Calls []string IP net.IP + ExtID string Machines []string NodeResources *api.NodeResources ClusterList []string @@ -110,6 +111,13 @@ func (f *FakeCloud) IPAddress(instance string) (net.IP, error) { return f.IP, f.Err } +// ExternalID is a test-spy implementation of Instances.ExternalID. +// It adds an entry "external-id" into the internal method call record. +func (f *FakeCloud) ExternalID(instance string) (string, error) { + f.addCall("external-id") + return f.ExtID, f.Err +} + // List is a test-spy implementation of Instances.List. // It adds an entry "list" into the internal method call record. func (f *FakeCloud) List(filter string) ([]string, error) { diff --git a/pkg/cloudprovider/gce/gce.go b/pkg/cloudprovider/gce/gce.go index 9210b7f0320..e26bf2b44ba 100644 --- a/pkg/cloudprovider/gce/gce.go +++ b/pkg/cloudprovider/gce/gce.go @@ -299,21 +299,39 @@ func canonicalizeInstanceName(name string) string { return name } -// IPAddress is an implementation of Instances.IPAddress. -func (gce *GCECloud) IPAddress(instance string) (net.IP, error) { - instance = canonicalizeInstanceName(instance) - res, err := gce.service.Instances.Get(gce.projectID, gce.zone, instance).Do() +// Return the instances matching the relevant name. +func (gce *GCECloud) getInstanceByName(name string) (*compute.Instance, error) { + name = canonicalizeInstanceName(name) + res, err := gce.service.Instances.Get(gce.projectID, gce.zone, name).Do() if err != nil { - glog.Errorf("Failed to retrieve TargetInstance resource for instance:%s", instance) + glog.Errorf("Failed to retrieve TargetInstance resource for instance:%s", name) return nil, err } - ip := net.ParseIP(res.NetworkInterfaces[0].AccessConfigs[0].NatIP) + return res, nil +} + +// IPAddress is an implementation of Instances.IPAddress. +func (gce *GCECloud) IPAddress(instance string) (net.IP, error) { + inst, err := gce.getInstanceByName(instance) + if err != nil { + return nil, err + } + ip := net.ParseIP(inst.NetworkInterfaces[0].AccessConfigs[0].NatIP) if ip == nil { - return nil, fmt.Errorf("invalid network IP: %s", res.NetworkInterfaces[0].AccessConfigs[0].NatIP) + return nil, fmt.Errorf("invalid network IP: %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP) } return ip, nil } +// ExternalID returns the cloud provider ID of the specified instance. +func (gce *GCECloud) ExternalID(instance string) (string, error) { + inst, err := gce.getInstanceByName(instance) + if err != nil { + return "", err + } + return string(inst.Id), nil +} + // fqdnSuffix is hacky function to compute the delta between hostame and hostname -f. func fqdnSuffix() (string, error) { fullHostname, err := exec.Command("hostname", "-f").Output() diff --git a/pkg/cloudprovider/openstack/openstack.go b/pkg/cloudprovider/openstack/openstack.go index 4e5947d6cca..409495c75d6 100644 --- a/pkg/cloudprovider/openstack/openstack.go +++ b/pkg/cloudprovider/openstack/openstack.go @@ -312,6 +312,15 @@ func (i *Instances) IPAddress(name string) (net.IP, error) { return net.ParseIP(ip), err } +// ExternalID returns the cloud provider ID of the specified instance. +func (i *Instances) ExternalID(name string) (string, error) { + srv, err := getServerByName(i.compute, name) + if err != nil { + return "", err + } + return srv.ID, nil +} + func (i *Instances) GetNodeResources(name string) (*api.NodeResources, error) { glog.V(2).Infof("GetNodeResources(%v) called", name) diff --git a/pkg/cloudprovider/ovirt/ovirt.go b/pkg/cloudprovider/ovirt/ovirt.go index 187da1eddfe..c9cdae6e665 100644 --- a/pkg/cloudprovider/ovirt/ovirt.go +++ b/pkg/cloudprovider/ovirt/ovirt.go @@ -25,6 +25,7 @@ import ( "net/http" "net/url" "path" + "sort" "strings" "code.google.com/p/gcfg" @@ -32,6 +33,14 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" ) +type OVirtInstance struct { + UUID string + Name string + IPAddress string +} + +type OVirtInstanceMap map[string]OVirtInstance + type OVirtCloud struct { VmsRequest *url.URL HostsRequest *url.URL @@ -48,9 +57,16 @@ type OVirtApiConfig struct { } } +type XmlVmAddress struct { + Address string `xml:"address,attr"` +} + type XmlVmInfo struct { - Hostname string `xml:"guest_info>fqdn"` - State string `xml:"status>state"` + UUID string `xml:"id,attr"` + Name string `xml:"name"` + Hostname string `xml:"guest_info>fqdn"` + Addresses []XmlVmAddress `xml:"guest_info>ips>ip"` + State string `xml:"status>state"` } type XmlVmsList struct { @@ -115,16 +131,40 @@ func (v *OVirtCloud) Zones() (cloudprovider.Zones, bool) { } // IPAddress returns the address of a particular machine instance -func (v *OVirtCloud) IPAddress(instance string) (net.IP, error) { - // since the instance now is the IP in the ovirt env, this is trivial no-op - ip, err := net.LookupIP(instance) - if err != nil || len(ip) < 1 { - return nil, fmt.Errorf("cannot find ip address for: %s", instance) +func (v *OVirtCloud) IPAddress(name string) (net.IP, error) { + instance, err := v.fetchInstance(name) + if err != nil { + return nil, err } - return ip[0], nil + + var address net.IP + + if instance.IPAddress != "" { + address = net.ParseIP(instance.IPAddress) + if address == nil { + return nil, fmt.Errorf("couldn't parse address: %s", instance.IPAddress) + } + } else { + resolved, err := net.LookupIP(name) + if err != nil || len(resolved) < 1 { + return nil, fmt.Errorf("couldn't lookup address: %s", name) + } + address = resolved[0] + } + + return address, nil } -func getInstancesFromXml(body io.Reader) ([]string, error) { +// ExternalID returns the cloud provider ID of the specified instance. +func (v *OVirtCloud) ExternalID(name string) (string, error) { + instance, err := v.fetchInstance(name) + if err != nil { + return "", err + } + return instance.UUID, nil +} + +func getInstancesFromXml(body io.Reader) (OVirtInstanceMap, error) { if body == nil { return nil, fmt.Errorf("ovirt rest-api response body is missing") } @@ -140,20 +180,28 @@ func getInstancesFromXml(body io.Reader) ([]string, error) { return nil, err } - var instances []string + instances := make(OVirtInstanceMap) for _, vm := range vmlist.Vm { // Always return only vms that are up and running if vm.Hostname != "" && strings.ToLower(vm.State) == "up" { - instances = append(instances, vm.Hostname) + address := "" + if len(vm.Addresses) > 0 { + address = vm.Addresses[0].Address + } + + instances[vm.Hostname] = OVirtInstance{ + UUID: vm.UUID, + Name: vm.Name, + IPAddress: address, + } } } return instances, nil } -// List enumerates the set of minions instances known by the cloud provider -func (v *OVirtCloud) List(filter string) ([]string, error) { +func (v *OVirtCloud) fetchAllInstances() (OVirtInstanceMap, error) { response, err := http.Get(v.VmsRequest.String()) if err != nil { return nil, err @@ -164,6 +212,41 @@ func (v *OVirtCloud) List(filter string) ([]string, error) { return getInstancesFromXml(response.Body) } +func (v *OVirtCloud) fetchInstance(name string) (*OVirtInstance, error) { + allInstances, err := v.fetchAllInstances() + if err != nil { + return nil, err + } + + instance, found := allInstances[name] + if !found { + return nil, fmt.Errorf("cannot find instance: %s", name) + } + + return &instance, nil +} + +func (m *OVirtInstanceMap) ListSortedNames() []string { + var names []string + + for k := range *m { + names = append(names, k) + } + + sort.Strings(names) + + return names +} + +// List enumerates the set of minions instances known by the cloud provider +func (v *OVirtCloud) List(filter string) ([]string, error) { + instances, err := v.fetchAllInstances() + if err != nil { + return nil, err + } + return instances.ListSortedNames(), nil +} + func (v *OVirtCloud) GetNodeResources(name string) (*api.NodeResources, error) { return nil, nil } diff --git a/pkg/cloudprovider/ovirt/ovirt_test.go b/pkg/cloudprovider/ovirt/ovirt_test.go index 6f79cf01703..afeb45c4a9a 100644 --- a/pkg/cloudprovider/ovirt/ovirt_test.go +++ b/pkg/cloudprovider/ovirt/ovirt_test.go @@ -118,7 +118,9 @@ func TestOVirtCloudXmlParsing(t *testing.T) { if len(instances4) != 2 { t.Fatalf("Unexpected number of instance(s): %d", len(instances4)) } - if instances4[0] != "host1" || instances4[1] != "host3" { + + names := instances4.ListSortedNames() + if names[0] != "host1" || names[1] != "host3" { t.Fatalf("Unexpected instance(s): %s", instances4) } } diff --git a/pkg/cloudprovider/rackspace/rackspace.go b/pkg/cloudprovider/rackspace/rackspace.go index 5ab564c9ee2..95d3ad67e02 100644 --- a/pkg/cloudprovider/rackspace/rackspace.go +++ b/pkg/cloudprovider/rackspace/rackspace.go @@ -363,6 +363,11 @@ func (i *Instances) IPAddress(name string) (net.IP, error) { return net.ParseIP(ip), err } +// ExternalID returns the cloud provider ID of the specified instance. +func (i *Instances) ExternalID(name string) (string, error) { + return "", fmt.Errorf("unimplemented") +} + func (i *Instances) GetNodeResources(name string) (*api.NodeResources, error) { glog.V(2).Infof("GetNodeResources(%v) called", name) diff --git a/pkg/cloudprovider/vagrant/vagrant.go b/pkg/cloudprovider/vagrant/vagrant.go index 3a7c6b4c179..adb11e498ff 100644 --- a/pkg/cloudprovider/vagrant/vagrant.go +++ b/pkg/cloudprovider/vagrant/vagrant.go @@ -99,8 +99,8 @@ func (v *VagrantCloud) Zones() (cloudprovider.Zones, bool) { return nil, false } -// IPAddress returns the address of a particular machine instance. -func (v *VagrantCloud) IPAddress(instance string) (net.IP, error) { +// getInstanceByAddress retuns +func (v *VagrantCloud) getInstanceByAddress(address string) (*SaltMinion, error) { token, err := v.saltLogin() if err != nil { return nil, err @@ -112,11 +112,31 @@ func (v *VagrantCloud) IPAddress(instance string) (net.IP, error) { filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool") for _, minion := range filteredMinions { // Due to vagrant not running with a dedicated DNS setup, we return the IP address of a minion as its hostname at this time - if minion.IP == instance { - return net.ParseIP(minion.IP), nil + if minion.IP == address { + return &minion, nil } } - return nil, fmt.Errorf("unable to find IP address for instance: %s", instance) + return nil, fmt.Errorf("unable to find instance for address: %s", address) +} + +// IPAddress returns the address of a particular machine instance. +func (v *VagrantCloud) IPAddress(instance string) (net.IP, error) { + // Due to vagrant not running with a dedicated DNS setup, we return the IP address of a minion as its hostname at this time + minion, err := v.getInstanceByAddress(instance) + if err != nil { + return nil, err + } + return net.ParseIP(minion.IP), nil +} + +// ExternalID returns the cloud provider ID of the specified instance. +func (v *VagrantCloud) ExternalID(instance string) (string, error) { + // Due to vagrant not running with a dedicated DNS setup, we return the IP address of a minion as its hostname at this time + minion, err := v.getInstanceByAddress(instance) + if err != nil { + return "", err + } + return minion.IP, nil } // saltMinionsByRole filters a list of minions that have a matching role.