Merge pull request #63902 from vmware/vcp_secrets

Automatic merge from submit-queue (batch tested with PRs 63969, 63902, 63689, 63973, 63978). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adds a mechanism in vSphere Cloud Provider to get credentials from Kubernetes secrets

**What this PR does / why we need it**:
Currently, vCenter credentials are stored in plain text in vsphere.conf. This PR adds a mechanism in vSphere Cloud Provider to get vCenter credentials from Kubernetes secrets.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:
Internally review here: https://github.com/vmware/kubernetes/pull/484
**Workflow:**
1. Create vsphere.conf file with ```secret-name``` and ```secret-namespace```.
	```
	[Global]
	insecure-flag = 1
	secret-name = "vcconf"
	secret-namespace = "kube-system"

	[VirtualCenter "10.160.45.119"]
	port = 443
	datacenters = k8s-dc-1

	[Workspace]
	server = 10.160.45.119
	datacenter = k8s-dc-1
	default-datastore = sharedVMFS-0
	folder = Discovered virtual machine
	```
2. Launch Kubernetes cluster with vSphere Cloud Provider Configured.
3. Create secret with vCenter credentials.
	a. Create base64 encoding for username and password:
	username:
	```	
		> echo -n 'admin' | base64
		YWRtaW4= 
	```
	password:
	```
		> echo -n 'vsphere' | base64
		dnNwaGVyZQ==
	```

	b. kubectl create -f vccredentials.yaml
	```
		#vccredentials.yaml
		apiVersion: v1
		kind: Secret
		metadata:
			name: vcconf
		type: Opaque
		data:
			10.192.44.199.username: YWRtaW4=
			10.192.44.199.password: dnNwaGVyZQ==
	```
4. vSphere Cloud Provider can be used now.

**Note:**
Secrets info can be provided with both (old and new) vSphere Cloud provider configuration formats.


**Tests Done:**
- [x] vSphere Cloud Provider unit test.
- [x] Volume lifecyle with Username and Password in vsphere.conf (for backward compability)
- [x] Volume lifecyle with secrets information in vsphere.conf.
- [x] Update secrets workflow

**Release note**:

```release-note
Adds a mechanism in vSphere Cloud Provider to get credentials from Kubernetes secrets
```

pkg/cloudprovider/providers/vsphere/BUILD

@@ -9,6 +9,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "credentialmanager.go",
         "nodemanager.go",
         "vsphere.go",
         "vsphere_util.go",
@@ -26,25 +27,36 @@ go_library(
         "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library",
         "//vendor/gopkg.in/gcfg.v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["vsphere_test.go"],
+    srcs = [
+        "credentialmanager_test.go",
+        "vsphere_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//pkg/controller:go_default_library",
         "//vendor/github.com/vmware/govmomi/lookup/simulator:go_default_library",
         "//vendor/github.com/vmware/govmomi/simulator:go_default_library",
         "//vendor/github.com/vmware/govmomi/simulator/vpx:go_default_library",
         "//vendor/github.com/vmware/govmomi/sts/simulator:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//vendor/k8s.io/client-go/informers:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
     ],
 )
 

pkg/cloudprovider/providers/vsphere/credentialmanager.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"errors"
+	"fmt"
+	"github.com/golang/glog"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/listers/core/v1"
+	"net/http"
+	"strings"
+	"sync"
+)
+
+// Error Messages
+const (
+	CredentialsNotFoundErrMsg = "Credentials not found"
+	CredentialMissingErrMsg   = "Username/Password is missing"
+	UnknownSecretKeyErrMsg    = "Unknown secret key"
+)
+
+// Error constants
+var (
+	ErrCredentialsNotFound = errors.New(CredentialsNotFoundErrMsg)
+	ErrCredentialMissing   = errors.New(CredentialMissingErrMsg)
+	ErrUnknownSecretKey    = errors.New(UnknownSecretKeyErrMsg)
+)
+
+type SecretCache struct {
+	cacheLock     sync.Mutex
+	VirtualCenter map[string]*Credential
+	Secret        *corev1.Secret
+}
+
+type Credential struct {
+	User     string `gcfg:"user"`
+	Password string `gcfg:"password"`
+}
+
+type SecretCredentialManager struct {
+	SecretName      string
+	SecretNamespace string
+	SecretLister    v1.SecretLister
+	Cache           *SecretCache
+}
+
+// GetCredential returns credentials for the given vCenter Server.
+// GetCredential returns error if Secret is not added.
+// GetCredential return error is the secret doesn't contain any credentials.
+func (secretCredentialManager *SecretCredentialManager) GetCredential(server string) (*Credential, error) {
+	err := secretCredentialManager.updateCredentialsMap()
+	if err != nil {
+		statusErr, ok := err.(*apierrors.StatusError)
+		if (ok && statusErr.ErrStatus.Code != http.StatusNotFound) || !ok {
+			return nil, err
+		}
+		// Handle secrets deletion by finding credentials from cache
+		glog.Warningf("secret %q not found in namespace %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace)
+	}
+
+	credential, found := secretCredentialManager.Cache.GetCredential(server)
+	if !found {
+		glog.Errorf("credentials not found for server %q", server)
+		return nil, ErrCredentialsNotFound
+	}
+	return &credential, nil
+}
+
+func (secretCredentialManager *SecretCredentialManager) updateCredentialsMap() error {
+	if secretCredentialManager.SecretLister == nil {
+		return fmt.Errorf("SecretLister is not initialized")
+	}
+	secret, err := secretCredentialManager.SecretLister.Secrets(secretCredentialManager.SecretNamespace).Get(secretCredentialManager.SecretName)
+	if err != nil {
+		glog.Errorf("Cannot get secret %s in namespace %s. error: %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace, err)
+		return err
+	}
+	cacheSecret := secretCredentialManager.Cache.GetSecret()
+	if cacheSecret != nil &&
+		cacheSecret.GetResourceVersion() == secret.GetResourceVersion() {
+		glog.V(4).Infof("VCP SecretCredentialManager: Secret %q will not be updated in cache. Since, secrets have same resource version %q", secretCredentialManager.SecretName, cacheSecret.GetResourceVersion())
+		return nil
+	}
+	secretCredentialManager.Cache.UpdateSecret(secret)
+	return secretCredentialManager.Cache.parseSecret()
+}
+
+func (cache *SecretCache) GetSecret() *corev1.Secret {
+	cache.cacheLock.Lock()
+	defer cache.cacheLock.Unlock()
+	return cache.Secret
+}
+
+func (cache *SecretCache) UpdateSecret(secret *corev1.Secret) {
+	cache.cacheLock.Lock()
+	defer cache.cacheLock.Unlock()
+	cache.Secret = secret
+}
+
+func (cache *SecretCache) GetCredential(server string) (Credential, bool) {
+	cache.cacheLock.Lock()
+	defer cache.cacheLock.Unlock()
+	credential, found := cache.VirtualCenter[server]
+	if !found {
+		return Credential{}, found
+	}
+	return *credential, found
+}
+
+func (cache *SecretCache) parseSecret() error {
+	cache.cacheLock.Lock()
+	defer cache.cacheLock.Unlock()
+	return parseConfig(cache.Secret.Data, cache.VirtualCenter)
+}
+
+// parseConfig returns vCenter ip/fdqn mapping to its credentials viz. Username and Password.
+func parseConfig(data map[string][]byte, config map[string]*Credential) error {
+	if len(data) == 0 {
+		return ErrCredentialMissing
+	}
+	for credentialKey, credentialValue := range data {
+		credentialKey = strings.ToLower(credentialKey)
+		vcServer := ""
+		if strings.HasSuffix(credentialKey, "password") {
+			vcServer = strings.Split(credentialKey, ".password")[0]
+			if _, ok := config[vcServer]; !ok {
+				config[vcServer] = &Credential{}
+			}
+			config[vcServer].Password = string(credentialValue)
+		} else if strings.HasSuffix(credentialKey, "username") {
+			vcServer = strings.Split(credentialKey, ".username")[0]
+			if _, ok := config[vcServer]; !ok {
+				config[vcServer] = &Credential{}
+			}
+			config[vcServer].User = string(credentialValue)
+		} else {
+			glog.Errorf("Unknown secret key %s", credentialKey)
+			return ErrUnknownSecretKey
+		}
+	}
+	for vcServer, credential := range config {
+		if credential.User == "" || credential.Password == "" {
+			glog.Errorf("Username/Password is missing for server %s", vcServer)
+			return ErrCredentialMissing
+		}
+	}
+	return nil
+}

pkg/cloudprovider/providers/vsphere/credentialmanager_test.go

@@ -0,0 +1,338 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"reflect"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+func TestSecretCredentialManager_GetCredential(t *testing.T) {
+	var (
+		userKey             = "username"
+		passwordKey         = "password"
+		testUser            = "user"
+		testPassword        = "password"
+		testServer          = "0.0.0.0"
+		testServer2         = "0.0.1.1"
+		testUserServer2     = "user1"
+		testPasswordServer2 = "password1"
+		testIncorrectServer = "1.1.1.1"
+	)
+	var (
+		secretName      = "vsconf"
+		secretNamespace = "kube-system"
+	)
+	var (
+		addSecretOp      = "ADD_SECRET_OP"
+		getCredentialsOp = "GET_CREDENTIAL_OP"
+		deleteSecretOp   = "DELETE_SECRET_OP"
+	)
+	type GetCredentialsTest struct {
+		server   string
+		username string
+		password string
+		err      error
+	}
+	type OpSecretTest struct {
+		secret *corev1.Secret
+	}
+	type testEnv struct {
+		testName       string
+		ops            []string
+		expectedValues []interface{}
+	}
+
+	client := &fake.Clientset{}
+	metaObj := metav1.ObjectMeta{
+		Name:      secretName,
+		Namespace: secretNamespace,
+	}
+
+	defaultSecret := &corev1.Secret{
+		ObjectMeta: metaObj,
+		Data: map[string][]byte{
+			testServer + "." + userKey:     []byte(testUser),
+			testServer + "." + passwordKey: []byte(testPassword),
+		},
+	}
+
+	multiVCSecret := &corev1.Secret{
+		ObjectMeta: metaObj,
+		Data: map[string][]byte{
+			testServer + "." + userKey:      []byte(testUser),
+			testServer + "." + passwordKey:  []byte(testPassword),
+			testServer2 + "." + userKey:     []byte(testUserServer2),
+			testServer2 + "." + passwordKey: []byte(testPasswordServer2),
+		},
+	}
+
+	emptySecret := &corev1.Secret{
+		ObjectMeta: metaObj,
+		Data:       map[string][]byte{},
+	}
+
+	tests := []testEnv{
+		{
+			testName: "Deleting secret should give the credentials from cache",
+			ops:      []string{addSecretOp, getCredentialsOp, deleteSecretOp, getCredentialsOp},
+			expectedValues: []interface{}{
+				OpSecretTest{
+					secret: defaultSecret,
+				},
+				GetCredentialsTest{
+					username: testUser,
+					password: testPassword,
+					server:   testServer,
+				},
+				OpSecretTest{
+					secret: defaultSecret,
+				},
+				GetCredentialsTest{
+					username: testUser,
+					password: testPassword,
+					server:   testServer,
+				},
+			},
+		},
+		{
+			testName: "Add secret and get credentials",
+			ops:      []string{addSecretOp, getCredentialsOp},
+			expectedValues: []interface{}{
+				OpSecretTest{
+					secret: defaultSecret,
+				},
+				GetCredentialsTest{
+					username: testUser,
+					password: testPassword,
+					server:   testServer,
+				},
+			},
+		},
+		{
+			testName: "Getcredentials should fail by not adding at secret at first time",
+			ops:      []string{getCredentialsOp},
+			expectedValues: []interface{}{
+				GetCredentialsTest{
+					username: testUser,
+					password: testPassword,
+					server:   testServer,
+					err:      ErrCredentialsNotFound,
+				},
+			},
+		},
+		{
+			testName: "GetCredential should fail to get credentials from empty secrets",
+			ops:      []string{addSecretOp, getCredentialsOp},
+			expectedValues: []interface{}{
+				OpSecretTest{
+					secret: emptySecret,
+				},
+				GetCredentialsTest{
+					server: testServer,
+					err:    ErrCredentialMissing,
+				},
+			},
+		},
+		{
+			testName: "GetCredential should fail to get credentials for invalid server",
+			ops:      []string{addSecretOp, getCredentialsOp},
+			expectedValues: []interface{}{
+				OpSecretTest{
+					secret: defaultSecret,
+				},
+				GetCredentialsTest{
+					server: testIncorrectServer,
+					err:    ErrCredentialsNotFound,
+				},
+			},
+		},
+		{
+			testName: "GetCredential for multi-vc",
+			ops:      []string{addSecretOp, getCredentialsOp},
+			expectedValues: []interface{}{
+				OpSecretTest{
+					secret: multiVCSecret,
+				},
+				GetCredentialsTest{
+					server:   testServer2,
+					username: testUserServer2,
+					password: testPasswordServer2,
+				},
+			},
+		},
+	}
+
+	informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
+	secretInformer := informerFactory.Core().V1().Secrets()
+	secretCredentialManager := &SecretCredentialManager{
+		SecretName:      secretName,
+		SecretNamespace: secretNamespace,
+		SecretLister:    secretInformer.Lister(),
+		Cache: &SecretCache{
+			VirtualCenter: make(map[string]*Credential),
+		},
+	}
+	cleanupSecretCredentialManager := func() {
+		secretCredentialManager.Cache.Secret = nil
+		for key := range secretCredentialManager.Cache.VirtualCenter {
+			delete(secretCredentialManager.Cache.VirtualCenter, key)
+		}
+		secrets, err := secretCredentialManager.SecretLister.List(labels.Everything())
+		if err != nil {
+			t.Fatal("Failed to get all secrets from sharedInformer. error: ", err)
+		}
+		for _, secret := range secrets {
+			secretInformer.Informer().GetIndexer().Delete(secret)
+		}
+	}
+
+	for _, test := range tests {
+		t.Logf("Executing Testcase: %s", test.testName)
+		for ntest, op := range test.ops {
+			switch op {
+			case addSecretOp:
+				expected := test.expectedValues[ntest].(OpSecretTest)
+				t.Logf("Adding secret: %s", expected.secret)
+				err := secretInformer.Informer().GetIndexer().Add(expected.secret)
+				if err != nil {
+					t.Fatalf("Failed to add secret to internal cache: %v", err)
+				}
+			case getCredentialsOp:
+				expected := test.expectedValues[ntest].(GetCredentialsTest)
+				credential, err := secretCredentialManager.GetCredential(expected.server)
+				t.Logf("Retrieving credentials for server %s", expected.server)
+				if err != expected.err {
+					t.Fatalf("Fail to get credentials with error: %v", err)
+				}
+				if expected.err == nil {
+					if expected.username != credential.User ||
+						expected.password != credential.Password {
+						t.Fatalf("Received credentials %v "+
+							"are different than actual credential user:%s password:%s", credential, expected.username,
+							expected.password)
+					}
+				}
+			case deleteSecretOp:
+				expected := test.expectedValues[ntest].(OpSecretTest)
+				t.Logf("Deleting secret: %s", expected.secret)
+				err := secretInformer.Informer().GetIndexer().Delete(expected.secret)
+				if err != nil {
+					t.Fatalf("Failed to delete secret to internal cache: %v", err)
+				}
+			}
+		}
+		cleanupSecretCredentialManager()
+	}
+}
+
+func TestParseSecretConfig(t *testing.T) {
+	var (
+		testUsername = "Admin"
+		testPassword = "Password"
+		testIP       = "10.20.30.40"
+	)
+	var testcases = []struct {
+		testName      string
+		data          map[string][]byte
+		config        map[string]*Credential
+		expectedError error
+	}{
+		{
+			testName: "Valid username and password",
+			data: map[string][]byte{
+				"10.20.30.40.username": []byte(testUsername),
+				"10.20.30.40.password": []byte(testPassword),
+			},
+			config: map[string]*Credential{
+				testIP: {
+					User:     testUsername,
+					Password: testPassword,
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			testName: "Invalid username key with valid password key",
+			data: map[string][]byte{
+				"10.20.30.40.usernam":  []byte(testUsername),
+				"10.20.30.40.password": []byte(testPassword),
+			},
+			config:        nil,
+			expectedError: ErrUnknownSecretKey,
+		},
+		{
+			testName: "Missing username",
+			data: map[string][]byte{
+				"10.20.30.40.password": []byte(testPassword),
+			},
+			config: map[string]*Credential{
+				testIP: {
+					Password: testPassword,
+				},
+			},
+			expectedError: ErrCredentialMissing,
+		},
+		{
+			testName: "Missing password",
+			data: map[string][]byte{
+				"10.20.30.40.username": []byte(testUsername),
+			},
+			config: map[string]*Credential{
+				testIP: {
+					User: testUsername,
+				},
+			},
+			expectedError: ErrCredentialMissing,
+		},
+		{
+			testName: "IP with unknown key",
+			data: map[string][]byte{
+				"10.20.30.40": []byte(testUsername),
+			},
+			config:        nil,
+			expectedError: ErrUnknownSecretKey,
+		},
+	}
+
+	resultConfig := make(map[string]*Credential)
+	cleanupResultConfig := func(config map[string]*Credential) {
+		for k := range config {
+			delete(config, k)
+		}
+	}
+
+	for _, testcase := range testcases {
+		err := parseConfig(testcase.data, resultConfig)
+		t.Logf("Executing Testcase: %s", testcase.testName)
+		if err != testcase.expectedError {
+			t.Fatalf("Parsing Secret failed for data %+v: %s", testcase.data, err)
+		}
+		if testcase.config != nil && !reflect.DeepEqual(testcase.config, resultConfig) {
+			t.Fatalf("Parsing Secret failed for data %+v expected config %+v and actual config %+v",
+				testcase.data, resultConfig, testcase.config)
+		}
+		cleanupResultConfig(resultConfig)
+	}
+}

pkg/cloudprovider/providers/vsphere/nodemanager.go

@@ -45,10 +45,13 @@ type NodeManager struct {
 	nodeInfoMap map[string]*NodeInfo
 	// Maps node name to node structure
 	registeredNodes map[string]*v1.Node
+	//CredentialsManager
+	credentialManager *SecretCredentialManager
 
 	// Mutexes
-	registeredNodesLock sync.RWMutex
+	registeredNodesLock   sync.RWMutex
-	nodeInfoLock        sync.RWMutex
+	nodeInfoLock          sync.RWMutex
+	credentialManagerLock sync.Mutex
 }
 
 type NodeDetails struct {
@@ -119,7 +122,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			err := vsi.conn.Connect(ctx)
+			err := nm.vcConnect(ctx, vsi)
 			if err != nil {
 				glog.V(4).Info("Discovering node error vc:", err)
 				setGlobalErr(err)
@@ -297,30 +300,17 @@ func (nm *NodeManager) GetNodeInfo(nodeName k8stypes.NodeName) (NodeInfo, error)
 //
 // This method is a getter but it can cause side-effect of updating NodeInfo objects.
 func (nm *NodeManager) GetNodeDetails() ([]NodeDetails, error) {
-	nm.nodeInfoLock.RLock()
+	nm.registeredNodesLock.Lock()
-	defer nm.nodeInfoLock.RUnlock()
+	defer nm.registeredNodesLock.Unlock()
 	var nodeDetails []NodeDetails
-	vsphereSessionRefreshMap := make(map[string]bool)
 
-	for nodeName, nodeInfo := range nm.nodeInfoMap {
+	for nodeName, nodeObj := range nm.registeredNodes {
-		var n *NodeInfo
+		nodeInfo, err := nm.GetNodeInfoWithNodeObject(nodeObj)
-		var err error
-		if vsphereSessionRefreshMap[nodeInfo.vcServer] {
-			// vSphere connection already refreshed. Just refresh VM and Datacenter.
-			glog.V(4).Infof("Renewing NodeInfo %+v for node %q. No new connection needed.", nodeInfo, nodeName)
-			n, err = nm.renewNodeInfo(nodeInfo, false)
-		} else {
-			// Refresh vSphere connection, VM and Datacenter.
-			glog.V(4).Infof("Renewing NodeInfo %+v for node %q with new vSphere connection.", nodeInfo, nodeName)
-			n, err = nm.renewNodeInfo(nodeInfo, true)
-			vsphereSessionRefreshMap[nodeInfo.vcServer] = true
-		}
 		if err != nil {
 			return nil, err
 		}
-		nm.nodeInfoMap[nodeName] = n
 		glog.V(4).Infof("Updated NodeInfo %q for node %q.", nodeInfo, nodeName)
-		nodeDetails = append(nodeDetails, NodeDetails{nodeName, n.vm, n.vmUUID})
+		nodeDetails = append(nodeDetails, NodeDetails{nodeName, nodeInfo.vm, nodeInfo.vmUUID})
 	}
 	return nodeDetails, nil
 }
@@ -355,7 +345,7 @@ func (nm *NodeManager) renewNodeInfo(nodeInfo *NodeInfo, reconnect bool) (*NodeI
 		return nil, err
 	}
 	if reconnect {
-		err := vsphereInstance.conn.Connect(ctx)
+		err := nm.vcConnect(ctx, vsphereInstance)
 		if err != nil {
 			return nil, err
 		}
@@ -370,3 +360,82 @@ func (nodeInfo *NodeInfo) VM() *vclib.VirtualMachine {
 	}
 	return nodeInfo.vm
 }
+
+// vcConnect connects to vCenter with existing credentials
+// If credentials are invalid:
+// 		1. It will fetch credentials from credentialManager
+//      2. Update the credentials
+//		3. Connects again to vCenter with fetched credentials
+func (nm *NodeManager) vcConnect(ctx context.Context, vsphereInstance *VSphereInstance) error {
+	err := vsphereInstance.conn.Connect(ctx)
+	if err == nil {
+		return nil
+	}
+
+	credentialManager := nm.CredentialManager()
+	if !vclib.IsInvalidCredentialsError(err) || credentialManager == nil {
+		glog.Errorf("Cannot connect to vCenter with err: %v", err)
+		return err
+	}
+
+	glog.V(4).Infof("Invalid credentials. Cannot connect to server %q. "+
+		"Fetching credentials from secrets.", vsphereInstance.conn.Hostname)
+
+	// Get latest credentials from SecretCredentialManager
+	credentials, err := credentialManager.GetCredential(vsphereInstance.conn.Hostname)
+	if err != nil {
+		glog.Errorf("Failed to get credentials from Secret Credential Manager with err: %v", err)
+		return err
+	}
+	vsphereInstance.conn.UpdateCredentials(credentials.User, credentials.Password)
+	return vsphereInstance.conn.Connect(ctx)
+}
+
+// GetNodeInfoWithNodeObject returns a NodeInfo which datacenter, vm and vc server ip address.
+// This method returns an error if it is unable find node VCs and DCs listed in vSphere.conf
+// NodeInfo returned may not be updated to reflect current VM location.
+//
+// This method is a getter but it can cause side-effect of updating NodeInfo object.
+func (nm *NodeManager) GetNodeInfoWithNodeObject(node *v1.Node) (NodeInfo, error) {
+	nodeName := node.Name
+	getNodeInfo := func(nodeName string) *NodeInfo {
+		nm.nodeInfoLock.RLock()
+		nodeInfo := nm.nodeInfoMap[nodeName]
+		nm.nodeInfoLock.RUnlock()
+		return nodeInfo
+	}
+	nodeInfo := getNodeInfo(nodeName)
+	var err error
+	if nodeInfo == nil {
+		// Rediscover node if no NodeInfo found.
+		glog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", nodeName)
+		err = nm.DiscoverNode(node)
+		if err != nil {
+			glog.Errorf("Error %q node info for node %q not found", err, nodeName)
+			return NodeInfo{}, err
+		}
+		nodeInfo = getNodeInfo(nodeName)
+	} else {
+		// Renew the found NodeInfo to avoid stale vSphere connection.
+		glog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, nodeName)
+		nodeInfo, err = nm.renewNodeInfo(nodeInfo, true)
+		if err != nil {
+			glog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, nodeName)
+			return NodeInfo{}, err
+		}
+		nm.addNodeInfo(nodeName, nodeInfo)
+	}
+	return *nodeInfo, nil
+}
+
+func (nm *NodeManager) CredentialManager() *SecretCredentialManager {
+	nm.credentialManagerLock.Lock()
+	defer nm.credentialManagerLock.Unlock()
+	return nm.credentialManager
+}
+
+func (nm *NodeManager) UpdateCredentialManager(credentialManager *SecretCredentialManager) {
+	nm.credentialManagerLock.Lock()
+	defer nm.credentialManagerLock.Unlock()
+	nm.credentialManager = credentialManager
+}

pkg/cloudprovider/providers/vsphere/vclib/connection.go

@@ -40,6 +40,7 @@ type VSphereConnection struct {
 	Port              string
 	Insecure          bool
 	RoundTripperCount uint
+	credentialsLock   sync.Mutex
 }
 
 var (
@@ -85,6 +86,8 @@ func (connection *VSphereConnection) Connect(ctx context.Context) error {
 // otherwise calls SessionManager.Login with user and password.
 func (connection *VSphereConnection) login(ctx context.Context, client *vim25.Client) error {
 	m := session.NewManager(client)
+	connection.credentialsLock.Lock()
+	defer connection.credentialsLock.Unlock()
 
 	// TODO: Add separate fields for certificate and private-key.
 	// For now we can leave the config structs and validation as-is and
@@ -163,3 +166,12 @@ func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Clie
 	client.RoundTripper = vim25.Retry(client.RoundTripper, vim25.TemporaryNetworkError(int(connection.RoundTripperCount)))
 	return client, nil
 }
+
+// UpdateCredentials updates username and password.
+// Note: Updated username and password will be used when there is no session active
+func (connection *VSphereConnection) UpdateCredentials(username string, password string) {
+	connection.credentialsLock.Lock()
+	defer connection.credentialsLock.Unlock()
+	connection.Username = username
+	connection.Password = password
+}

pkg/cloudprovider/providers/vsphere/vclib/utils.go

@@ -172,6 +172,15 @@ func IsManagedObjectNotFoundError(err error) bool {
 	return isManagedObjectNotFoundError
 }
 
+// IsInvalidCredentialsError returns true if error is of type InvalidLogin
+func IsInvalidCredentialsError(err error) bool {
+	isInvalidCredentialsError := false
+	if soap.IsSoapFault(err) {
+		_, isInvalidCredentialsError = soap.ToSoapFault(err).VimFault().(types.InvalidLogin)
+	}
+	return isInvalidCredentialsError
+}
+
 // VerifyVolumePathsForVM verifies if the volume paths (volPaths) are attached to VM.
 func VerifyVolumePathsForVM(vmMo mo.VirtualMachine, volPaths []string, nodeName string, nodeVolumeMap map[string]map[string]bool) {
 	// Verify if the volume paths are present on the VM backing virtual disk devices

pkg/cloudprovider/providers/vsphere/vsphere.go

@@ -61,6 +61,18 @@ var datastoreFolderIDMap = make(map[string]map[string]string)
 var cleanUpRoutineInitLock sync.Mutex
 var cleanUpDummyVMLock sync.RWMutex
 
+// Error Messages
+const (
+	MissingUsernameErrMsg = "Username is missing"
+	MissingPasswordErrMsg = "Password is missing"
+)
+
+// Error constants
+var (
+	ErrUsernameMissing = errors.New(MissingUsernameErrMsg)
+	ErrPasswordMissing = errors.New(MissingPasswordErrMsg)
+)
+
 // VSphere is an implementation of cloud provider Interface for VSphere.
 type VSphere struct {
 	cfg      *VSphereConfig
@@ -68,8 +80,9 @@ type VSphere struct {
 	// Maps the VSphere IP address to VSphereInstance
 	vsphereInstanceMap map[string]*VSphereInstance
 	// Responsible for managing discovery of k8s node, their location etc.
-	nodeManager *NodeManager
+	nodeManager          *NodeManager
-	vmUUID      string
+	vmUUID               string
+	isSecretInfoProvided bool
 }
 
 // Represents a vSphere instance where one or more kubernetes nodes are running.
@@ -131,6 +144,10 @@ type VSphereConfig struct {
 		// Combining the WorkingDir and VMName can form a unique InstanceID.
 		// When vm-name is set, no username/password is required on worker nodes.
 		VMName string `gcfg:"vm-name"`
+		// Name of the secret were vCenter credentials are present.
+		SecretName string `gcfg:"secret-name"`
+		// Secret Namespace where secret will be present that has vCenter credentials.
+		SecretNamespace string `gcfg:"secret-namespace"`
 	}
 
 	VirtualCenter map[string]*VirtualCenterConfig
@@ -217,6 +234,18 @@ func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory)
 		return
 	}
 
+	if vs.isSecretInfoProvided {
+		secretCredentialManager := &SecretCredentialManager{
+			SecretName:      vs.cfg.Global.SecretName,
+			SecretNamespace: vs.cfg.Global.SecretNamespace,
+			SecretLister:    informerFactory.Core().V1().Secrets().Lister(),
+			Cache: &SecretCache{
+				VirtualCenter: make(map[string]*Credential),
+			},
+		}
+		vs.nodeManager.UpdateCredentialManager(secretCredentialManager)
+	}
+
 	// Only on controller node it is required to register listeners.
 	// Register callbacks for node updates
 	glog.V(4).Infof("Setting up node informers for vSphere Cloud Provider")
@@ -226,6 +255,7 @@ func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory)
 		DeleteFunc: vs.NodeDeleted,
 	})
 	glog.V(4).Infof("Node informers in vSphere cloud provider initialized")
+
 }
 
 // Creates new worker node interface and returns
@@ -247,19 +277,40 @@ func newWorkerNode() (*VSphere, error) {
 
 func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance, error) {
 	vsphereInstanceMap := make(map[string]*VSphereInstance)
+	isSecretInfoProvided := true
+
+	if cfg.Global.SecretName == "" || cfg.Global.SecretNamespace == "" {
+		glog.Warningf("SecretName and/or SecretNamespace is not provided. " +
+			"VCP will use username and password from config file")
+		isSecretInfoProvided = false
+	}
+
+	if isSecretInfoProvided {
+		if cfg.Global.User != "" {
+			glog.Warning("Global.User and Secret info provided. VCP will use secret to get credentials")
+			cfg.Global.User = ""
+		}
+		if cfg.Global.Password != "" {
+			glog.Warning("Global.Password and Secret info provided. VCP will use secret to get credentials")
+			cfg.Global.Password = ""
+		}
+	}
 
 	// Check if the vsphere.conf is in old format. In this
 	// format the cfg.VirtualCenter will be nil or empty.
 	if cfg.VirtualCenter == nil || len(cfg.VirtualCenter) == 0 {
 		glog.V(4).Infof("Config is not per virtual center and is in old format.")
-		if cfg.Global.User == "" {
+		if !isSecretInfoProvided {
-			glog.Error("Global.User is empty!")
+			if cfg.Global.User == "" {
-			return nil, errors.New("Global.User is empty!")
+				glog.Error("Global.User is empty!")
-		}
+				return nil, ErrUsernameMissing
-		if cfg.Global.Password == "" {
+			}
-			glog.Error("Global.Password is empty!")
+			if cfg.Global.Password == "" {
-			return nil, errors.New("Global.Password is empty!")
+				glog.Error("Global.Password is empty!")
+				return nil, ErrPasswordMissing
+			}
 		}
+
 		if cfg.Global.WorkingDir == "" {
 			glog.Error("Global.WorkingDir is empty!")
 			return nil, errors.New("Global.WorkingDir is empty!")
@@ -285,6 +336,8 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance
 			RoundTripperCount: cfg.Global.RoundTripperCount,
 		}
 
+		// Note: If secrets info is provided username and password will be populated
+		// once secret is created.
 		vSphereConn := vclib.VSphereConnection{
 			Username:          vcConfig.User,
 			Password:          vcConfig.Password,
@@ -305,31 +358,44 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance
 			glog.Error(msg)
 			return nil, errors.New(msg)
 		}
+
 		for vcServer, vcConfig := range cfg.VirtualCenter {
 			glog.V(4).Infof("Initializing vc server %s", vcServer)
 			if vcServer == "" {
 				glog.Error("vsphere.conf does not have the VirtualCenter IP address specified")
 				return nil, errors.New("vsphere.conf does not have the VirtualCenter IP address specified")
 			}
-			if vcConfig.User == "" {
+
-				vcConfig.User = cfg.Global.User
+			if !isSecretInfoProvided {
-			}
+				if vcConfig.User == "" {
-			if vcConfig.Password == "" {
+					vcConfig.User = cfg.Global.User
-				vcConfig.Password = cfg.Global.Password
+					if vcConfig.User == "" {
-			}
+						glog.Errorf("vcConfig.User is empty for vc %s!", vcServer)
-			if vcConfig.User == "" {
+						return nil, ErrUsernameMissing
-				msg := fmt.Sprintf("vcConfig.User is empty for vc %s!", vcServer)
+					}
-				glog.Error(msg)
+				}
-				return nil, errors.New(msg)
+				if vcConfig.Password == "" {
-			}
+					vcConfig.Password = cfg.Global.Password
-			if vcConfig.Password == "" {
+					if vcConfig.Password == "" {
-				msg := fmt.Sprintf("vcConfig.Password is empty for vc %s!", vcServer)
+						glog.Errorf("vcConfig.Password is empty for vc %s!", vcServer)
-				glog.Error(msg)
+						return nil, ErrPasswordMissing
-				return nil, errors.New(msg)
+					}
+				}
+			} else {
+				if vcConfig.User != "" {
+					glog.Warningf("vcConfig.User for server %s and Secret info provided. VCP will use secret to get credentials", vcServer)
+					vcConfig.User = ""
+				}
+				if vcConfig.Password != "" {
+					glog.Warningf("vcConfig.Password for server %s and Secret info provided. VCP will use secret to get credentials", vcServer)
+					vcConfig.Password = ""
+				}
 			}
+
 			if vcConfig.VCenterPort == "" {
 				vcConfig.VCenterPort = cfg.Global.VCenterPort
 			}
+
 			if vcConfig.Datacenters == "" {
 				if cfg.Global.Datacenters != "" {
 					vcConfig.Datacenters = cfg.Global.Datacenters
@@ -342,6 +408,8 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance
 				vcConfig.RoundTripperCount = cfg.Global.RoundTripperCount
 			}
 
+			// Note: If secrets info is provided username and password will be populated
+			// once secret is created.
 			vSphereConn := vclib.VSphereConnection{
 				Username:          vcConfig.User,
 				Password:          vcConfig.Password,
@@ -365,7 +433,30 @@ var getVMUUID = GetVMUUID
 
 // Creates new Controller node interface and returns
 func newControllerNode(cfg VSphereConfig) (*VSphere, error) {
-	var err error
+	vs, err := buildVSphereFromConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	vs.hostName, err = os.Hostname()
+	if err != nil {
+		glog.Errorf("Failed to get hostname. err: %+v", err)
+		return nil, err
+	}
+	vs.vmUUID, err = getVMUUID()
+	if err != nil {
+		glog.Errorf("Failed to get uuid. err: %+v", err)
+		return nil, err
+	}
+	runtime.SetFinalizer(vs, logout)
+	return vs, nil
+}
+
+// Initializes vSphere from vSphere CloudProvider Configuration
+func buildVSphereFromConfig(cfg VSphereConfig) (*VSphere, error) {
+	isSecretInfoProvided := false
+	if cfg.Global.SecretName != "" && cfg.Global.SecretNamespace != "" {
+		isSecretInfoProvided = true
+	}
 
 	if cfg.Disk.SCSIControllerType == "" {
 		cfg.Disk.SCSIControllerType = vclib.PVSCSIControllerType
@@ -394,20 +485,9 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) {
 			nodeInfoMap:        make(map[string]*NodeInfo),
 			registeredNodes:    make(map[string]*v1.Node),
 		},
-		cfg: &cfg,
+		isSecretInfoProvided: isSecretInfoProvided,
+		cfg:                  &cfg,
 	}
-
-	vs.hostName, err = os.Hostname()
-	if err != nil {
-		glog.Errorf("Failed to get hostname. err: %+v", err)
-		return nil, err
-	}
-	vs.vmUUID, err = getVMUUID()
-	if err != nil {
-		glog.Errorf("Failed to get uuid. err: %+v", err)
-		return nil, err
-	}
-	runtime.SetFinalizer(&vs, logout)
 	return &vs, nil
 }
 
@@ -480,7 +560,7 @@ func (vs *VSphere) getVSphereInstanceForServer(vcServer string, ctx context.Cont
 		return nil, errors.New(fmt.Sprintf("Cannot find node %q in vsphere configuration map", vcServer))
 	}
 	// Ensure client is logged in and session is valid
-	err := vsphereIns.conn.Connect(ctx)
+	err := vs.nodeManager.vcConnect(ctx, vsphereIns)
 	if err != nil {
 		glog.Errorf("failed connecting to vcServer %q with error %+v", vcServer, err)
 		return nil, err
@@ -519,7 +599,7 @@ func (vs *VSphere) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName
 		return nil, err
 	}
 	// Ensure client is logged in and session is valid
-	err = vsi.conn.Connect(ctx)
+	err = vs.nodeManager.vcConnect(ctx, vsi)
 	if err != nil {
 		return nil, err
 	}
@@ -634,7 +714,7 @@ func (vs *VSphere) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) (
 			return "", err
 		}
 		// Ensure client is logged in and session is valid
-		err = vsi.conn.Connect(ctx)
+		err = vs.nodeManager.vcConnect(ctx, vsi)
 		if err != nil {
 			return "", err
 		}
@@ -725,7 +805,7 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyName string, nodeN
 			return "", err
 		}
 		// Ensure client is logged in and session is valid
-		err = vsi.conn.Connect(ctx)
+		err = vs.nodeManager.vcConnect(ctx, vsi)
 		if err != nil {
 			return "", err
 		}
@@ -792,7 +872,7 @@ func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error
 			return err
 		}
 		// Ensure client is logged in and session is valid
-		err = vsi.conn.Connect(ctx)
+		err = vs.nodeManager.vcConnect(ctx, vsi)
 		if err != nil {
 			return err
 		}
@@ -847,7 +927,7 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b
 			return false, err
 		}
 		// Ensure client is logged in and session is valid
-		err = vsi.conn.Connect(ctx)
+		err = vs.nodeManager.vcConnect(ctx, vsi)
 		if err != nil {
 			return false, err
 		}

pkg/cloudprovider/providers/vsphere/vsphere_test.go

@@ -352,3 +352,221 @@ func TestVolumes(t *testing.T) {
 	// 	t.Fatalf("Cannot delete VMDK volume %s: %v", volPath, err)
 	// }
 }
+
+func TestSecretVSphereConfig(t *testing.T) {
+	var vs *VSphere
+	var (
+		username = "user"
+		password = "password"
+	)
+	var testcases = []struct {
+		testName                 string
+		conf                     string
+		expectedIsSecretProvided bool
+		expectedUsername         string
+		expectedPassword         string
+		expectedError            error
+	}{
+		{
+			testName: "Username and password with old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			user = user
+			password = password
+			datacenter = us-west
+			working-dir = kubernetes
+			`,
+			expectedUsername: username,
+			expectedPassword: password,
+			expectedError:    nil,
+		},
+		{
+			testName: "SecretName and SecretNamespace in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			datacenter = us-west
+			secret-name = "vccreds"
+			secret-namespace = "kube-system"
+			working-dir = kubernetes
+			`,
+			expectedIsSecretProvided: true,
+			expectedError:            nil,
+		},
+		{
+			testName: "SecretName and SecretNamespace with Username and Password in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			user = user
+			password = password
+			datacenter = us-west
+			secret-name = "vccreds"
+			secret-namespace = "kube-system"
+			working-dir = kubernetes
+			`,
+			expectedIsSecretProvided: true,
+			expectedError:            nil,
+		},
+		{
+			testName: "SecretName and SecretNamespace with Username missing in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			password = password
+			datacenter = us-west
+			secret-name = "vccreds"
+			secret-namespace = "kube-system"
+			working-dir = kubernetes
+			`,
+			expectedIsSecretProvided: true,
+			expectedError:            nil,
+		},
+		{
+			testName: "SecretNamespace missing with Username and Password in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			user = user
+			password = password
+			datacenter = us-west
+			secret-name = "vccreds"
+			working-dir = kubernetes
+			`,
+			expectedUsername: username,
+			expectedPassword: password,
+			expectedError:    nil,
+		},
+		{
+			testName: "SecretNamespace and Username missing in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			password = password
+			datacenter = us-west
+			secret-name = "vccreds"
+			working-dir = kubernetes
+			`,
+			expectedError: ErrUsernameMissing,
+		},
+		{
+			testName: "SecretNamespace and Password missing in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			user = user
+			datacenter = us-west
+			secret-name = "vccreds"
+			working-dir = kubernetes
+			`,
+			expectedError: ErrPasswordMissing,
+		},
+		{
+			testName: "SecretNamespace, Username and Password missing in old configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			datacenter = us-west
+			secret-name = "vccreds"
+			working-dir = kubernetes
+			`,
+			expectedError: ErrUsernameMissing,
+		},
+		{
+			testName: "Username and password with new configuration but username and password in global section",
+			conf: `[Global]
+			user = user
+			password = password
+			datacenter = us-west
+			[VirtualCenter "0.0.0.0"]
+			[Workspace]
+			server = 0.0.0.0
+			datacenter = us-west
+			folder = kubernetes
+			`,
+			expectedUsername: username,
+			expectedPassword: password,
+			expectedError:    nil,
+		},
+		{
+			testName: "Username and password with new configuration, username and password in virtualcenter section",
+			conf: `[Global]
+			server = 0.0.0.0
+			port = 443
+			insecure-flag = true
+			datacenter = us-west
+			[VirtualCenter "0.0.0.0"]
+			user = user
+			password = password
+			[Workspace]
+			server = 0.0.0.0
+			datacenter = us-west
+			folder = kubernetes
+			`,
+			expectedUsername: username,
+			expectedPassword: password,
+			expectedError:    nil,
+		},
+		{
+			testName: "SecretName and SecretNamespace with new configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			secret-name = "vccreds"
+			secret-namespace = "kube-system"
+			datacenter = us-west
+			[VirtualCenter "0.0.0.0"]
+			[Workspace]
+			server = 0.0.0.0
+			datacenter = us-west
+			folder = kubernetes
+			`,
+			expectedIsSecretProvided: true,
+			expectedError:            nil,
+		},
+		{
+			testName: "SecretName and SecretNamespace with Username missing in new configuration",
+			conf: `[Global]
+			server = 0.0.0.0
+			port = 443
+			insecure-flag = true
+			datacenter = us-west
+			secret-name = "vccreds"
+			secret-namespace = "kube-system"
+			[VirtualCenter "0.0.0.0"]
+			password = password
+			[Workspace]
+			server = 0.0.0.0
+			datacenter = us-west
+			folder = kubernetes
+			`,
+			expectedIsSecretProvided: true,
+			expectedError:            nil,
+		},
+	}
+
+	for _, testcase := range testcases {
+		t.Logf("Executing Testcase: %s", testcase.testName)
+		cfg, err := readConfig(strings.NewReader(testcase.conf))
+		if err != nil {
+			t.Fatalf("Should succeed when a valid config is provided: %s", err)
+		}
+		vs, err = buildVSphereFromConfig(cfg)
+		if err != testcase.expectedError {
+			t.Fatalf("Should succeed when a valid config is provided: %s", err)
+		}
+		if err != nil {
+			continue
+		}
+		if vs.isSecretInfoProvided != testcase.expectedIsSecretProvided {
+			t.Fatalf("SecretName and SecretNamespace was expected in config %s. error: %s",
+				testcase.conf, err)
+		}
+		if !testcase.expectedIsSecretProvided {
+			for _, vsInstance := range vs.vsphereInstanceMap {
+				if vsInstance.conn.Username != testcase.expectedUsername {
+					t.Fatalf("Expected username %s doesn't match actual username %s in config %s. error: %s",
+						testcase.expectedUsername, vsInstance.conn.Username, testcase.conf, err)
+				}
+				if vsInstance.conn.Password != testcase.expectedPassword {
+					t.Fatalf("Expected password %s doesn't match actual password %s in config %s. error: %s",
+						testcase.expectedPassword, vsInstance.conn.Password, testcase.conf, err)
+				}
+
+			}
+		}
+
+	}
+}