Update Photon Controller cloud provider for authentication support

1. Update the configuration file for Photon Controller cloud provider
2. Only master nodes can communicate with Photon Controller endpoint
3. Enable support for authentication-enabled Photon Controller endpoint
4. Update NodeAddresses function for query from local node

pkg/cloudprovider/providers/photon/photon.go

@@ -24,10 +24,13 @@ limitations under the License.
 package photon
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
 	"log"
+	"net"
+	"os"
 	"os/exec"
 	"strings"
 
@@ -42,12 +45,10 @@ import (
 const (
 	ProviderName = "photon"
 	DiskSpecKind = "persistent-disk"
+	MAC_OUI_VC   = "00:50:56"
+	MAC_OUI_ESX  = "00:0c:29"
 )
 
-// Global variable pointing to photon client
-var photonClient *photon.Client
-var logger *log.Logger = nil
-
 // overrideIP indicates if the hostname is overriden by IP address, such as when
 // running multi-node kubernetes using docker. In this case the user should set
 // overrideIP = true in cloud config file. Default value is false.
@@ -66,22 +67,23 @@ type PCCloud struct {
 	// in the environment per current Photon Controller deployment methodology.
 	projID string
 	cloudprovider.Zone
+	photonClient *photon.Client
+	logger       *log.Logger
 }
 
 type PCConfig struct {
 	Global struct {
 		// the Photon Controller endpoint IP address
 		CloudTarget string `gcfg:"target"`
-		// when the Photon Controller authentication is enabled, set to true;
-		// otherwise, set to false.
-		IgnoreCertificate bool `gcfg:"ignoreCertificate"`
-		// Photon Controller tenant name
-		Tenant string `gcfg:"tenant"`
 		// Photon Controller project name
 		Project string `gcfg:"project"`
 		// when kubelet is started with '--hostname-override=${IP_ADDRESS}', set to true;
 		// otherwise, set to false.
 		OverrideIP bool `gcfg:"overrideIP"`
+		// VM ID for this node
+		VMID string `gcfg:"vmID"`
+		// Authentication enabled or not
+		AuthEnabled bool `gcfg:"authentication"`
 	}
 }
 
@@ -140,10 +142,16 @@ func init() {
 }
 
 // Retrieve the Photon VM ID from the Photon Controller endpoint based on the node name
-func getVMIDbyNodename(project string, nodeName string) (string, error) {
+func getVMIDbyNodename(pc *PCCloud, nodeName string) (string, error) {
-	vmList, err := photonClient.Projects.GetVMs(project, nil)
+	photonClient, err := getPhotonClient(pc)
 	if err != nil {
-		glog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", project, nodeName, err)
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err)
+		return "", err
+	}
+
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", pc.projID, nodeName, err)
 		return "", err
 	}
 
@@ -157,10 +165,16 @@ func getVMIDbyNodename(project string, nodeName string) (string, error) {
 }
 
 // Retrieve the Photon VM ID from the Photon Controller endpoint based on the IP address
-func getVMIDbyIP(project string, IPAddress string) (string, error) {
+func getVMIDbyIP(pc *PCCloud, IPAddress string) (string, error) {
-	vmList, err := photonClient.Projects.GetVMs(project, nil)
+	photonClient, err := getPhotonClient(pc)
 	if err != nil {
-		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", project, err)
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err)
+		return "", err
+	}
+
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", pc.projID, err)
 		return "", err
 	}
 
@@ -191,59 +205,67 @@ func getVMIDbyIP(project string, IPAddress string) (string, error) {
 	return "", fmt.Errorf("No matching VM is found with IP %s", IPAddress)
 }
 
-// Retrieve the the Photon project ID from the Photon Controller endpoint based on the project name
+func getPhotonClient(pc *PCCloud) (*photon.Client, error) {
-func getProjIDbyName(tenantName, projName string) (string, error) {
+	var err error
-	tenants, err := photonClient.Tenants.GetAll()
+	if len(pc.cfg.Global.CloudTarget) == 0 {
-	if err != nil {
-		glog.Errorf("Photon Cloud Provider: GetAll tenants failed with error [%v].", err)
-		return "", err
-	}
-
-	for _, tenant := range tenants.Items {
-		if tenant.Name == tenantName {
-			projects, err := photonClient.Tenants.GetProjects(tenant.ID, nil)
-			if err != nil {
-				glog.Errorf("Photon Cloud Provider: Failed to GetProjects for tenant %s. error [%v]", tenantName, err)
-				return "", err
-			}
-
-			for _, project := range projects.Items {
-				if project.Name == projName {
-					return project.ID, nil
-				}
-			}
-		}
-	}
-
-	return "", fmt.Errorf("No matching tenant/project name is found with %s/%s", tenantName, projName)
-}
-
-func newPCCloud(cfg PCConfig) (*PCCloud, error) {
-	if len(cfg.Global.CloudTarget) == 0 {
 		return nil, fmt.Errorf("Photon Controller endpoint was not specified.")
 	}
 
-	// Currently we support Photon Controller endpoint with authentication disabled.
 	options := &photon.ClientOptions{
-		IgnoreCertificate: cfg.Global.IgnoreCertificate,
+		IgnoreCertificate: true,
 	}
 
-	photonClient = photon.NewClient(cfg.Global.CloudTarget, options, logger)
+	pc.photonClient = photon.NewClient(pc.cfg.Global.CloudTarget, options, pc.logger)
-	status, err := photonClient.Status.Get()
+	if pc.cfg.Global.AuthEnabled == true {
+		// work around before metadata is available
+		file, err := os.Open("/etc/kubernetes/pc_login_info")
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: Authentication is enabled but found no username/password at /etc/kubernetes/pc_login_info. Error[%v]", err)
+			return nil, err
+		}
+		defer file.Close()
+		scanner := bufio.NewScanner(file)
+		if !scanner.Scan() {
+			glog.Errorf("Photon Cloud Provider: Empty username inside /etc/kubernetes/pc_login_info.")
+			return nil, fmt.Errorf("Failed to create authentication enabled client with invalid username")
+		}
+		username := scanner.Text()
+		if !scanner.Scan() {
+			glog.Errorf("Photon Cloud Provider: Empty password set inside /etc/kubernetes/pc_login_info.")
+			return nil, fmt.Errorf("Failed to create authentication enabled client with invalid password")
+		}
+		password := scanner.Text()
+
+		token_options, err := pc.photonClient.Auth.GetTokensByPassword(username, password)
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: failed to get tokens by password")
+			return nil, err
+		}
+
+		options = &photon.ClientOptions{
+			IgnoreCertificate: true,
+			TokenOptions: &photon.TokenOptions{
+				AccessToken: token_options.AccessToken,
+			},
+		}
+		pc.photonClient = photon.NewClient(pc.cfg.Global.CloudTarget, options, pc.logger)
+	}
+
+	status, err := pc.photonClient.Status.Get()
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: new client creation failed. Error[%v]", err)
 		return nil, err
 	}
 	glog.V(2).Info("Photon Cloud Provider: Status of the new photon controller client: %v", status)
 
-	// Get Photon Controller project ID for future use
+	return pc.photonClient, nil
-	projID, err := getProjIDbyName(cfg.Global.Tenant, cfg.Global.Project)
+}
-	if err != nil {
-		glog.Errorf("Photon Cloud Provider: getProjIDbyName failed when creating new Photon Controller client. Error[%v]", err)
-		return nil, err
-	}
 
-	// Get local hostname for localInstanceID
+func newPCCloud(cfg PCConfig) (*PCCloud, error) {
+	projID := cfg.Global.Project
+	vmID := cfg.Global.VMID
+
+	// Get local hostname
 	cmd := exec.Command("bash", "-c", `echo $HOSTNAME`)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -255,11 +277,6 @@ func newPCCloud(cfg PCConfig) (*PCCloud, error) {
 		return nil, fmt.Errorf("unable to retrieve hostname for Instance ID")
 	}
 	hostname := strings.TrimRight(string(out), "\n")
-	vmID, err := getVMIDbyNodename(projID, hostname)
-	if err != nil {
-		glog.Errorf("Photon Cloud Provider: getVMIDbyNodename failed when creating new Photon Controller client. Error[%v]", err)
-		return nil, err
-	}
 
 	pc := PCCloud{
 		cfg:              &cfg,
@@ -286,26 +303,68 @@ func (pc *PCCloud) List(filter string) ([]k8stypes.NodeName, error) {
 
 // NodeAddresses is an implementation of Instances.NodeAddresses.
 func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, error) {
-	addrs := []v1.NodeAddress{}
+	nodeAddrs := []v1.NodeAddress{}
 	name := string(nodeName)
 
-	var vmID string
-	var err error
 	if name == pc.localK8sHostname {
-		vmID = pc.localInstanceID
+		ifaces, err := net.Interfaces()
-	} else {
-		vmID, err = getInstanceID(name, pc.projID)
 		if err != nil {
-			glog.Errorf("Photon Cloud Provider: getInstanceID failed for NodeAddresses. Error[%v]", err)
+			glog.Errorf("Photon Cloud Provider: net.Interfaces() failed for NodeAddresses. Error[%v]", err)
-			return addrs, err
+			return nodeAddrs, err
 		}
+
+		for _, i := range ifaces {
+			addrs, err := i.Addrs()
+			if err != nil {
+				glog.Warningf("Photon Cloud Provider: Failed to extract addresses for NodeAddresses. Error[%v]", err)
+			} else {
+				for _, addr := range addrs {
+					if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+						if ipnet.IP.To4() != nil {
+							// Filter external IP by MAC address OUIs from vCenter and from ESX
+							if strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_VC) ||
+								strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_ESX) {
+								v1.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeExternalIP,
+										Address: ipnet.IP.String(),
+									},
+								)
+							} else {
+								v1.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeInternalIP,
+										Address: ipnet.IP.String(),
+									},
+								)
+							}
+						}
+					}
+				}
+			}
+		}
+		return nodeAddrs, nil
+	}
+
+	// Inquiring IP addresses from photon controller endpoint only for a node other than this node.
+	// This is assumed to be done by master only.
+	vmID, err := getInstanceID(pc, name)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: getInstanceID failed for NodeAddresses. Error[%v]", err)
+		return nodeAddrs, err
+	}
+
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for NodeAddresses, error: [%v]", err)
+		return nodeAddrs, err
 	}
 
 	// Retrieve the Photon VM's IP addresses from the Photon Controller endpoint based on the VM ID
 	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. Error[%v]", pc.projID, err)
-		return addrs, err
+		return nodeAddrs, err
 	}
 
 	for _, vm := range vmList.Items {
@@ -313,23 +372,37 @@ func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress,
 			task, err := photonClient.VMs.GetNetworks(vm.ID)
 			if err != nil {
 				glog.Errorf("Photon Cloud Provider: GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err)
-				return addrs, err
+				return nodeAddrs, err
 			} else {
 				task, err = photonClient.Tasks.Wait(task.ID)
 				if err != nil {
 					glog.Errorf("Photon Cloud Provider: Wait task for GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err)
-					return addrs, err
+					return nodeAddrs, err
 				} else {
 					networkConnections := task.ResourceProperties.(map[string]interface{})
 					networks := networkConnections["networkConnections"].([]interface{})
 					for _, nt := range networks {
+						ipAddr := "-"
+						macAddr := "-"
 						network := nt.(map[string]interface{})
 						if val, ok := network["ipAddress"]; ok && val != nil {
-							ipAddr := val.(string)
+							ipAddr = val.(string)
-							if ipAddr != "-" {
+						}
-								v1.AddToNodeAddresses(&addrs,
+						if val, ok := network["macAddress"]; ok && val != nil {
+							macAddr = val.(string)
+						}
+						if ipAddr != "-" {
+							if strings.HasPrefix(macAddr, MAC_OUI_VC) ||
+								strings.HasPrefix(macAddr, MAC_OUI_ESX) {
+								v1.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeExternalIP,
+										Address: ipAddr,
+									},
+								)
+							} else {
+								v1.AddToNodeAddresses(&nodeAddrs,
 									v1.NodeAddress{
-										// TODO: figure out the type of the IP
 										Type:    v1.NodeInternalIP,
 										Address: ipAddr,
 									},
@@ -337,14 +410,14 @@ func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress,
 							}
 						}
 					}
-					return addrs, nil
+					return nodeAddrs, nil
 				}
 			}
 		}
 	}
 
 	glog.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
-	return addrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
+	return nodeAddrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
 }
 
 func (pc *PCCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
@@ -356,14 +429,14 @@ func (pc *PCCloud) CurrentNodeName(hostname string) (k8stypes.NodeName, error) {
 	return k8stypes.NodeName(hostname), nil
 }
 
-func getInstanceID(name string, projID string) (string, error) {
+func getInstanceID(pc *PCCloud, name string) (string, error) {
 	var vmID string
 	var err error
 
 	if overrideIP == true {
-		vmID, err = getVMIDbyIP(projID, name)
+		vmID, err = getVMIDbyIP(pc, name)
 	} else {
-		vmID, err = getVMIDbyNodename(projID, name)
+		vmID, err = getVMIDbyNodename(pc, name)
 	}
 	if err != nil {
 		return "", err
@@ -382,7 +455,8 @@ func (pc *PCCloud) ExternalID(nodeName k8stypes.NodeName) (string, error) {
 	if name == pc.localK8sHostname {
 		return pc.localInstanceID, nil
 	} else {
-		ID, err := getInstanceID(name, pc.projID)
+		// We assume only master need to get InstanceID of a node other than itself
+		ID, err := getInstanceID(pc, name)
 		if err != nil {
 			glog.Errorf("Photon Cloud Provider: getInstanceID failed for ExternalID. Error[%v]", err)
 			return ID, err
@@ -398,7 +472,8 @@ func (pc *PCCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) {
 	if name == pc.localK8sHostname {
 		return pc.localInstanceID, nil
 	} else {
-		ID, err := getInstanceID(name, pc.projID)
+		// We assume only master need to get InstanceID of a node other than itself
+		ID, err := getInstanceID(pc, name)
 		if err != nil {
 			glog.Errorf("Photon Cloud Provider: getInstanceID failed for InstanceID. Error[%v]", err)
 			return ID, err
@@ -447,6 +522,12 @@ func (pc *PCCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []st
 
 // Attaches given virtual disk volume to the compute running kubelet.
 func (pc *PCCloud) AttachDisk(pdID string, nodeName k8stypes.NodeName) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for AttachDisk, error: [%v]", err)
+		return err
+	}
+
 	operation := &photon.VmDiskOperation{
 		DiskID: pdID,
 	}
@@ -474,6 +555,12 @@ func (pc *PCCloud) AttachDisk(pdID string, nodeName k8stypes.NodeName) error {
 
 // Detaches given virtual disk volume from the compute running kubelet.
 func (pc *PCCloud) DetachDisk(pdID string, nodeName k8stypes.NodeName) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DetachDisk, error: [%v]", err)
+		return err
+	}
+
 	operation := &photon.VmDiskOperation{
 		DiskID: pdID,
 	}
@@ -501,6 +588,12 @@ func (pc *PCCloud) DetachDisk(pdID string, nodeName k8stypes.NodeName) error {
 
 // DiskIsAttached returns if disk is attached to the VM using controllers supported by the plugin.
 func (pc *PCCloud) DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool, error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DiskIsAttached, error: [%v]", err)
+		return false, err
+	}
+
 	disk, err := photonClient.Disks.Get(pdID)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: Failed to Get disk with pdID %s. Error[%v]", pdID, err)
@@ -525,6 +618,12 @@ func (pc *PCCloud) DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool
 // DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin.
 func (pc *PCCloud) DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error) {
 	attached := make(map[string]bool)
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DisksAreAttached, error: [%v]", err)
+		return attached, err
+	}
+
 	for _, pdID := range pdIDs {
 		attached[pdID] = false
 	}
@@ -553,6 +652,12 @@ func (pc *PCCloud) DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName)
 
 // Create a volume of given size (in GB).
 func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for CreateDisk, error: [%v]", err)
+		return "", err
+	}
+
 	diskSpec := photon.DiskCreateSpec{}
 	diskSpec.Name = volumeOptions.Name
 	diskSpec.Flavor = volumeOptions.Flavor
@@ -576,6 +681,12 @@ func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err er
 
 // Deletes a volume given volume name.
 func (pc *PCCloud) DeleteDisk(pdID string) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DeleteDisk, error: [%v]", err)
+		return err
+	}
+
 	task, err := photonClient.Disks.Delete(pdID)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: Failed to DeleteDisk. Error[%v]", err)

pkg/cloudprovider/providers/photon/photon_test.go

@@ -29,21 +29,21 @@ import (
 )
 
 func configFromEnv() (TestVM string, TestFlavor string, cfg PCConfig, ok bool) {
-	var IgnoreCertificate bool
+	var AuthEnabled bool
 	var OverrideIP bool
 	var err error
 	cfg.Global.CloudTarget = os.Getenv("PHOTON_TARGET")
-	cfg.Global.Tenant = os.Getenv("PHOTON_TENANT")
 	cfg.Global.Project = os.Getenv("PHOTON_PROJECT")
-	if os.Getenv("PHOTON_IGNORE_CERTIFICATE") != "" {
+	cfg.Global.VMID = os.Getenv("PHOTON_VMID")
-		IgnoreCertificate, err = strconv.ParseBool(os.Getenv("PHOTON_IGNORE_CERTIFICATE"))
+	if os.Getenv("PHOTON_AUTH_ENABLED") != "" {
+		AuthEnabled, err = strconv.ParseBool(os.Getenv("PHOTON_AUTH_ENABLED"))
 	} else {
-		IgnoreCertificate = false
+		AuthEnabled = false
 	}
 	if err != nil {
 		log.Fatal(err)
 	}
-	cfg.Global.IgnoreCertificate = IgnoreCertificate
+	cfg.Global.AuthEnabled = AuthEnabled
 	if os.Getenv("PHOTON_OVERRIDE_IP") != "" {
 		OverrideIP, err = strconv.ParseBool(os.Getenv("PHOTON_OVERRIDE_IP"))
 	} else {
@@ -65,8 +65,8 @@ func configFromEnv() (TestVM string, TestFlavor string, cfg PCConfig, ok bool) {
 	}
 
 	ok = (cfg.Global.CloudTarget != "" &&
-		cfg.Global.Tenant != "" &&
 		cfg.Global.Project != "" &&
+		cfg.Global.VMID != "" &&
 		TestVM != "")
 
 	return
@@ -81,10 +81,10 @@ func TestReadConfig(t *testing.T) {
 	cfg, err := readConfig(strings.NewReader(`
 [Global]
 target = 0.0.0.0
-ignoreCertificate = true
-tenant = tenant
 project = project
-overrideIP = false
+overrideIP = true
+vmID = vmid
+authentication = false
 `))
 	if err != nil {
 		t.Fatalf("Should succeed when a valid config is provided: %s", err)
@@ -94,13 +94,13 @@ overrideIP = false
 		t.Errorf("incorrect photon target ip: %s", cfg.Global.CloudTarget)
 	}
 
-	if cfg.Global.Tenant != "tenant" {
-		t.Errorf("incorrect tenant: %s", cfg.Global.Tenant)
-	}
-
 	if cfg.Global.Project != "project" {
 		t.Errorf("incorrect project: %s", cfg.Global.Project)
 	}
+
+	if cfg.Global.VMID != "vmid" {
+		t.Errorf("incorrect vmid: %s", cfg.Global.VMID)
+	}
 }
 
 func TestNewPCCloud(t *testing.T) {