From a68c783dc8909a8b673bc8e4cbd045797bba98e8 Mon Sep 17 00:00:00 2001 From: wlan0 Date: Mon, 6 Mar 2017 13:33:26 -0800 Subject: [PATCH] Use ProviderID to address nodes in the cloudprovider The cloudprovider is being refactored out of kubernetes core. This is being done by moving all the cloud-specific calls from kube-apiserver, kubelet and kube-controller-manager into a separately maintained binary(by vendors) called cloud-controller-manager. The Kubelet relies on the cloudprovider to detect information about the node that it is running on. Some of the cloudproviders worked by querying local information to obtain this information. In the new world of things, local information cannot be relied on, since cloud-controller-manager will not run on every node. Only one active instance of it will be run in the cluster. Today, all calls to the cloudprovider are based on the nodename. Nodenames are unqiue within the kubernetes cluster, but generally not unique within the cloud. This model of addressing nodes by nodename will not work in the future because local services cannot be queried to uniquely identify a node in the cloud. Therefore, I propose that we perform all cloudprovider calls based on ProviderID. This ID is a unique identifier for identifying a node on an external database (such as the instanceID in aws cloud). --- pkg/cloudprovider/cloud.go | 8 ++++++++ pkg/cloudprovider/providers/aws/aws.go | 14 ++++++++++++++ .../providers/azure/azure_instances.go | 15 +++++++++++++++ pkg/cloudprovider/providers/fake/fake.go | 13 +++++++++++++ pkg/cloudprovider/providers/gce/gce.go | 16 ++++++++++++++++ pkg/cloudprovider/providers/mesos/mesos.go | 14 ++++++++++++++ .../providers/openstack/openstack_instances.go | 14 ++++++++++++++ pkg/cloudprovider/providers/ovirt/ovirt.go | 14 ++++++++++++++ pkg/cloudprovider/providers/photon/photon.go | 14 ++++++++++++++ .../providers/rackspace/rackspace.go | 14 ++++++++++++++ pkg/cloudprovider/providers/vsphere/vsphere.go | 14 ++++++++++++++ pkg/volume/cinder/attacher_test.go | 8 ++++++++ 12 files changed, 158 insertions(+) diff --git a/pkg/cloudprovider/cloud.go b/pkg/cloudprovider/cloud.go index de83e9bd1b6..2810ab3019f 100644 --- a/pkg/cloudprovider/cloud.go +++ b/pkg/cloudprovider/cloud.go @@ -112,6 +112,12 @@ type Instances interface { // returns the address of the calling instance. We should do a rename to // make this clearer. NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) + // NodeAddressesByProviderID returns the addresses of the specified instance. + // The instance is specified using the providerID of the node. The + // ProviderID is a unique identifier of the node. This will not be called + // from the node whose nodeaddresses are being queried. i.e. local metadata + // services cannot be used in this method to obtain nodeaddresses + NodeAddressesByProviderID(providerId string) ([]v1.NodeAddress, error) // ExternalID returns the cloud provider ID of the node with the specified NodeName. // Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound) ExternalID(nodeName types.NodeName) (string, error) @@ -119,6 +125,8 @@ type Instances interface { InstanceID(nodeName types.NodeName) (string, error) // InstanceType returns the type of the specified instance. InstanceType(name types.NodeName) (string, error) + // InstanceTypeByProviderID returns the type of the specified instance. + InstanceTypeByProviderID(providerID string) (string, error) // AddSSHKeyToAllInstances adds an SSH public key as a legal identity for all instances // expected format for the key is standard ssh-keygen format: AddSSHKeyToAllInstances(user string, keyData []byte) error diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index bc6890f3572..64a111da81d 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -976,6 +976,13 @@ func (c *Cloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) { return addresses, nil } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (c *Cloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + // ExternalID returns the cloud provider ID of the node with the specified nodeName (deprecated). func (c *Cloud) ExternalID(nodeName types.NodeName) (string, error) { if c.selfAWSInstance.nodeName == nodeName { @@ -1008,6 +1015,13 @@ func (c *Cloud) InstanceID(nodeName types.NodeName) (string, error) { return "/" + orEmpty(inst.Placement.AvailabilityZone) + "/" + orEmpty(inst.InstanceId), nil } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (c *Cloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + // InstanceType returns the type of the node with the specified nodeName. func (c *Cloud) InstanceType(nodeName types.NodeName) (string, error) { if c.selfAWSInstance.nodeName == nodeName { diff --git a/pkg/cloudprovider/providers/azure/azure_instances.go b/pkg/cloudprovider/providers/azure/azure_instances.go index 7ae81f6da0d..cf948ff9dba 100644 --- a/pkg/cloudprovider/providers/azure/azure_instances.go +++ b/pkg/cloudprovider/providers/azure/azure_instances.go @@ -17,6 +17,7 @@ limitations under the License. package azure import ( + "errors" "fmt" "regexp" @@ -40,6 +41,13 @@ func (az *Cloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) { }, nil } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (az *Cloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + // ExternalID returns the cloud provider ID of the specified instance (deprecated). func (az *Cloud) ExternalID(name types.NodeName) (string, error) { return az.InstanceID(name) @@ -57,6 +65,13 @@ func (az *Cloud) InstanceID(name types.NodeName) (string, error) { return *machine.ID, nil } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (az *Cloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + // InstanceType returns the type of the specified instance. // Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound) // (Implementer Note): This is used by kubelet. Kubelet will label the node. Real log from kubelet: diff --git a/pkg/cloudprovider/providers/fake/fake.go b/pkg/cloudprovider/providers/fake/fake.go index 2c7951f5b91..ad0e620bba5 100644 --- a/pkg/cloudprovider/providers/fake/fake.go +++ b/pkg/cloudprovider/providers/fake/fake.go @@ -185,6 +185,13 @@ func (f *FakeCloud) NodeAddresses(instance types.NodeName) ([]v1.NodeAddress, er return f.Addresses, f.Err } +// NodeAddressesByProviderID is a test-spy implementation of Instances.NodeAddressesByProviderID. +// It adds an entry "node-addresses-by-provider-id" into the internal method call record. +func (f *FakeCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + f.addCall("node-addresses-by-provider-id") + return f.Addresses, f.Err +} + // ExternalID is a test-spy implementation of Instances.ExternalID. // It adds an entry "external-id" into the internal method call record. // It returns an external id to the mapped instance name, if not found, it will return "ext-{instance}" @@ -205,6 +212,12 @@ func (f *FakeCloud) InstanceType(instance types.NodeName) (string, error) { return f.InstanceTypes[instance], nil } +// InstanceTypeByProviderID returns the type of the specified instance. +func (f *FakeCloud) InstanceTypeByProviderID(providerID string) (string, error) { + f.addCall("instance-type-by-provider-id") + return f.InstanceTypes[types.NodeName(providerID)], nil +} + // List is a test-spy implementation of Instances.List. // It adds an entry "list" into the internal method call record. func (f *FakeCloud) List(filter string) ([]types.NodeName, error) { diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 92e22442456..f9253856dac 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -17,6 +17,7 @@ limitations under the License. package gce import ( + "errors" "fmt" "io" "regexp" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/flowcontrol" + "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/cloudprovider" "github.com/golang/glog" @@ -340,3 +342,17 @@ func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string } return zones, nil } + +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (gce *GCECloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (gce *GCECloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} diff --git a/pkg/cloudprovider/providers/mesos/mesos.go b/pkg/cloudprovider/providers/mesos/mesos.go index e5f187a9dff..3dbe53014ad 100644 --- a/pkg/cloudprovider/providers/mesos/mesos.go +++ b/pkg/cloudprovider/providers/mesos/mesos.go @@ -226,6 +226,13 @@ func (c *MesosCloud) InstanceID(nodeName types.NodeName) (string, error) { return "", nil } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (c *MesosCloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + // InstanceType returns the type of the instance with the specified nodeName. func (c *MesosCloud) InstanceType(nodeName types.NodeName) (string, error) { return "", nil @@ -296,3 +303,10 @@ func (c *MesosCloud) NodeAddresses(nodeName types.NodeName) ([]v1.NodeAddress, e {Type: v1.NodeExternalIP, Address: ip.String()}, }, nil } + +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (c *MesosCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/pkg/cloudprovider/providers/openstack/openstack_instances.go index ce22459e006..cb64ed55a79 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_instances.go +++ b/pkg/cloudprovider/providers/openstack/openstack_instances.go @@ -138,6 +138,13 @@ func (i *Instances) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) return addrs, nil } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + // ExternalID returns the cloud provider ID of the specified instance (deprecated). func (i *Instances) ExternalID(name types.NodeName) (string, error) { srv, err := getServerByName(i.compute, name) @@ -166,6 +173,13 @@ func (i *Instances) InstanceID(name types.NodeName) (string, error) { return "/" + srv.ID, nil } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + // InstanceType returns the type of the specified instance. func (i *Instances) InstanceType(name types.NodeName) (string, error) { return "", nil diff --git a/pkg/cloudprovider/providers/ovirt/ovirt.go b/pkg/cloudprovider/providers/ovirt/ovirt.go index a742329f71f..ee3f7b43662 100644 --- a/pkg/cloudprovider/providers/ovirt/ovirt.go +++ b/pkg/cloudprovider/providers/ovirt/ovirt.go @@ -180,6 +180,13 @@ func (v *OVirtCloud) NodeAddresses(nodeName types.NodeName) ([]v1.NodeAddress, e }, nil } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (v *OVirtCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + // mapNodeNameToInstanceName maps from a k8s NodeName to an ovirt instance name (the hostname) // This is a simple string cast func mapNodeNameToInstanceName(nodeName types.NodeName) string { @@ -208,6 +215,13 @@ func (v *OVirtCloud) InstanceID(nodeName types.NodeName) (string, error) { return "/" + instance.UUID, err } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (v *OVirtCloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + // InstanceType returns the type of the specified instance. func (v *OVirtCloud) InstanceType(name types.NodeName) (string, error) { return "", nil diff --git a/pkg/cloudprovider/providers/photon/photon.go b/pkg/cloudprovider/providers/photon/photon.go index 6f6743236f2..ad300b61a88 100644 --- a/pkg/cloudprovider/providers/photon/photon.go +++ b/pkg/cloudprovider/providers/photon/photon.go @@ -420,6 +420,13 @@ func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, return nodeAddrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name) } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (pc *PCCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + func (pc *PCCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error { return errors.New("unimplemented") } @@ -483,6 +490,13 @@ func (pc *PCCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) { } } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (pc *PCCloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + func (pc *PCCloud) InstanceType(nodeName k8stypes.NodeName) (string, error) { return "", nil } diff --git a/pkg/cloudprovider/providers/rackspace/rackspace.go b/pkg/cloudprovider/providers/rackspace/rackspace.go index 5939ba3a2e5..532c04dbd1a 100644 --- a/pkg/cloudprovider/providers/rackspace/rackspace.go +++ b/pkg/cloudprovider/providers/rackspace/rackspace.go @@ -387,6 +387,13 @@ func (i *Instances) NodeAddresses(nodeName types.NodeName) ([]v1.NodeAddress, er }, nil } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + // mapNodeNameToServerName maps from a k8s NodeName to a rackspace Server Name // This is a simple string cast. func mapNodeNameToServerName(nodeName types.NodeName) string { @@ -420,6 +427,13 @@ func (i *Instances) InstanceType(name types.NodeName) (string, error) { return "", nil } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error { return errors.New("unimplemented") } diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index 2eb5455286d..309bb7a0d61 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -525,6 +525,13 @@ func (i *Instances) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, return addrs, nil } +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("unimplemented") +} + func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error { return errors.New("unimplemented") } @@ -609,6 +616,13 @@ func (i *Instances) InstanceID(nodeName k8stypes.NodeName) (string, error) { return "", cloudprovider.InstanceNotFound } +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + func (i *Instances) InstanceType(name k8stypes.NodeName) (string, error) { return "", nil } diff --git a/pkg/volume/cinder/attacher_test.go b/pkg/volume/cinder/attacher_test.go index d45b99784d2..1b7386e1075 100644 --- a/pkg/volume/cinder/attacher_test.go +++ b/pkg/volume/cinder/attacher_test.go @@ -565,6 +565,10 @@ func (instances *instances) NodeAddresses(name types.NodeName) ([]v1.NodeAddress return []v1.NodeAddress{}, errors.New("Not implemented") } +func (instances *instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{}, errors.New("Not implemented") +} + func (instances *instances) ExternalID(name types.NodeName) (string, error) { return "", errors.New("Not implemented") } @@ -577,6 +581,10 @@ func (instances *instances) InstanceType(name types.NodeName) (string, error) { return "", errors.New("Not implemented") } +func (instances *instances) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("Not implemented") +} + func (instances *instances) List(filter string) ([]types.NodeName, error) { return []types.NodeName{}, errors.New("Not implemented") }