From e21385b9ba22583ed8e2dacf5cdc4f53f8ec89f7 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Tue, 6 Apr 2021 19:50:33 +0200 Subject: [PATCH 1/6] Disable zones obtaining attempts for legacy vSphere cloud provider if secret provided and no CredentialsManager was set up. Partially solves #75175. Kubelet does not stucking on startup. --- .../legacy-cloud-providers/vsphere/vsphere.go | 7 ++- .../vsphere/vsphere_test.go | 62 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index 82c4895b1a5..fefc0ec6a18 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -896,7 +896,12 @@ func (vs *VSphere) LoadBalancer() (cloudprovider.LoadBalancer, bool) { } func (vs *VSphere) isZoneEnabled() bool { - return vs.cfg != nil && vs.cfg.Labels.Zone != "" && vs.cfg.Labels.Region != "" + isEnabled := vs.cfg != nil && vs.cfg.Labels.Zone != "" && vs.cfg.Labels.Region != "" + if isEnabled && vs.isSecretInfoProvided && vs.nodeManager.credentialManager == nil { + klog.V(1).Info("Zones can not be populated now due to credentials in Secret, skip.") + return false + } + return isEnabled } // Zones returns an implementation of Zones for vSphere. diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere_test.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere_test.go index 64db1cef399..c00d19de153 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere_test.go @@ -463,6 +463,68 @@ func TestZonesNoConfig(t *testing.T) { } } +func TestZonesWithCredsInSecret(t *testing.T) { + noSecretCfg, err := readConfig(strings.NewReader(` +[Global] +user = "vsphere-creds" +password = "kube-system" +insecure-flag = "1" +[Workspace] +server = "vcenter.example.com" +datacenter = "LAB" +default-datastore = "datastore" +folder = "/LAB/vm/lab-gxjfk" +[VirtualCenter "vcenter.example.com"] +datacenters = "LAB" +[Labels] +region = "kube-region" +zone = "kube-zone" +`)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + vsphere, err := buildVSphereFromConfig(noSecretCfg) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + _, ok := vsphere.Zones() + if !ok { + t.Fatalf("Zones should return true with plain text credentials") + } + + // Return false in case if secret provided but no informers (no NodeManager.credentialManager basically) set up. + // Such situation happens during kubelet startup process, when InitialNode creates. + // See https://github.com/kubernetes/kubernetes/issues/75175 + // and https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet_node_status.go#L418 + withSecretCfg, err := readConfig(strings.NewReader(` +[Global] +secret-name = "vsphere-creds" +secret-namespace = "kube-system" +insecure-flag = "1" +[Workspace] +server = "vcenter.example.com" +datacenter = "LAB" +default-datastore = "datastore_big" +folder = "/LAB/vm/lab-gxjfk" +[VirtualCenter "vcenter.example.com"] +datacenters = "LAB" +[Labels] +region = "kube-region" +zone = "kube-zone" +`)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + vsphere, err = buildVSphereFromConfig(withSecretCfg) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + _, ok = vsphere.Zones() + if ok { + t.Fatalf("Zones should return false with plain credentials in secret") + } +} + func TestZones(t *testing.T) { // Any context will do ctx := context.Background() From bae7a214f65b57634ecf8ddaf0c8f979ca6ea73a Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Tue, 6 Apr 2021 19:50:33 +0200 Subject: [PATCH 2/6] Set zones for node within kcm if creds stored in secret --- .../legacy-cloud-providers/vsphere/vsphere.go | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index fefc0ec6a18..d0ccb410425 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net" "net/url" @@ -47,6 +48,7 @@ import ( k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" cloudprovider "k8s.io/cloud-provider" nodehelpers "k8s.io/cloud-provider/node/helpers" @@ -62,6 +64,7 @@ import ( // VSphere Cloud Provider constants const ( ProviderName = "vsphere" + providerIDPrefix = "vsphere://" VolDir = "kubevols" RoundTripperDefaultCount = 3 DummyVMPrefixName = "vsphere-k8s" @@ -95,8 +98,9 @@ var _ cloudprovider.PVLabeler = (*VSphere)(nil) // VSphere is an implementation of cloud provider Interface for VSphere. type VSphere struct { - cfg *VSphereConfig - hostName string + cfg *VSphereConfig + kubeClient clientset.Interface + hostName string // Maps the VSphere IP address to VSphereInstance vsphereInstanceMap map[string]*VSphereInstance vsphereVolumeMap *VsphereVolumeMap @@ -268,6 +272,7 @@ func init() { // Initialize passes a Kubernetes clientBuilder interface to the cloud provider func (vs *VSphere) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { + vs.kubeClient = clientBuilder.ClientOrDie("vsphere-legacy-cloud-provider") } // Initialize Node Informers @@ -1530,6 +1535,37 @@ func (vs *VSphere) NodeAdded(obj interface{}) { if err := vs.nodeManager.RegisterNode(node); err != nil { klog.Errorf("failed to add node %+v: %v", node, err) } + vs.reconcileZonesForNode(node) +} + +func (vs *VSphere) reconcileZonesForNode(node *v1.Node) { + nodeZone := node.ObjectMeta.Labels[v1.LabelTopologyZone] + nodeRegion := node.ObjectMeta.Labels[v1.LabelTopologyRegion] + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if vs.isSecretInfoProvided && vs.isZoneEnabled() { + zone, err := vs.GetZoneByProviderID(ctx, node.Spec.ProviderID) + if err != nil { + klog.Warningf("Can not get Zones from vCenter: %v", err) + } + + if zone.FailureDomain != nodeZone || zone.Region != nodeRegion { + updatedNode := node.DeepCopy() + labels := updatedNode.ObjectMeta.Labels + if labels == nil { + labels = make(map[string]string) + } + labels[v1.LabelTopologyZone] = zone.FailureDomain + labels[v1.LabelTopologyRegion] = zone.Region + + updatedNode, err = vs.kubeClient.CoreV1().Nodes().Update(ctx, updatedNode, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("vSphere cloud provider can not update node with zones info: %v", err) + } else { + klog.V(3).Infof("Node %s updated with zone and region lables", node.Name) + } + } + } } // Notification handler when node is removed from k8s cluster. @@ -1716,8 +1752,89 @@ func (vs *VSphere) GetZoneByNodeName(ctx context.Context, nodeName k8stypes.Node return cloudprovider.Zone{}, cloudprovider.NotImplemented } +//TODO (dmoiseev) refactor, commonize with GetZone func (vs *VSphere) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { - return cloudprovider.Zone{}, cloudprovider.NotImplemented + zone := cloudprovider.Zone{} + vmUUID := strings.Replace(providerID, providerIDPrefix, "", 1) + + vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) + if err != nil { + klog.Errorf("Cannot connect to vsphere. Get zone for vm %s error", vmUUID) + return cloudprovider.Zone{}, err + } + dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) + if err != nil { + klog.Errorf("Cannot connect to datacenter. Get zone for vm %s error", vmUUID) + return cloudprovider.Zone{}, err + } + vmHost, err := dc.GetHostByVMUUID(ctx, vmUUID) + if err != nil { + klog.Errorf("Cannot find VM runtime host. Get zone for vm %s error", vmUUID) + return cloudprovider.Zone{}, err + } + + pc := vsi.conn.Client.ServiceContent.PropertyCollector + err = withTagsClient(ctx, vsi.conn, func(c *rest.Client) error { + client := tags.NewManager(c) + // example result: ["Folder", "Datacenter", "Cluster", "Host"] + objects, err := mo.Ancestors(ctx, vsi.conn.Client, pc, *vmHost) + if err != nil { + return err + } + + // search the hierarchy, example order: ["Host", "Cluster", "Datacenter", "Folder"] + for i := range objects { + obj := objects[len(objects)-1-i] + tags, err := client.ListAttachedTags(ctx, obj) + if err != nil { + klog.Errorf("Cannot list attached tags. Get zone for vm %s: %s", vmUUID, err) + return err + } + for _, value := range tags { + tag, err := client.GetTag(ctx, value) + if err != nil { + klog.Errorf("Get tag %s: %s", value, err) + return err + } + category, err := client.GetCategory(ctx, tag.CategoryID) + if err != nil { + klog.Errorf("Get category %s error", value) + return err + } + + found := func() { + klog.Errorf("Found %q tag (%s) for %s attached to %s", category.Name, tag.Name, vmUUID, obj.Reference()) + } + switch { + case category.Name == vs.cfg.Labels.Zone: + zone.FailureDomain = tag.Name + found() + case category.Name == vs.cfg.Labels.Region: + zone.Region = tag.Name + found() + } + + if zone.FailureDomain != "" && zone.Region != "" { + return nil + } + } + } + + if zone.Region == "" { + return fmt.Errorf("vSphere region category %q does not match any tags for vm %s", vs.cfg.Labels.Region, vmUUID) + } + if zone.FailureDomain == "" { + return fmt.Errorf("vSphere zone category %q does not match any tags for vm %s", vs.cfg.Labels.Zone, vmUUID) + } + + return nil + }) + if err != nil { + klog.Errorf("Get zone for vm %s: %s", vmUUID, err) + return cloudprovider.Zone{}, err + } + + return zone, nil } // GetLabelsForVolume implements the PVLabeler interface for VSphere From 89b3887aeaee424d59d1027b3616494a22c0bf56 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Mon, 12 Apr 2021 13:32:25 +0200 Subject: [PATCH 3/6] Commonize GetZone with GetZoneByProviderID, retries for node update in case of conflicts --- .../legacy-cloud-providers/vsphere/vsphere.go | 162 ++++++------------ 1 file changed, 56 insertions(+), 106 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index d0ccb410425..c29c289c569 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "io" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net" @@ -65,6 +66,7 @@ import ( const ( ProviderName = "vsphere" providerIDPrefix = "vsphere://" + updateNodeRetryCount = 3 VolDir = "kubevols" RoundTripperDefaultCount = 3 DummyVMPrefixName = "vsphere-k8s" @@ -1535,10 +1537,10 @@ func (vs *VSphere) NodeAdded(obj interface{}) { if err := vs.nodeManager.RegisterNode(node); err != nil { klog.Errorf("failed to add node %+v: %v", node, err) } - vs.reconcileZonesForNode(node) + vs.setNodeZoneLabels(node) } -func (vs *VSphere) reconcileZonesForNode(node *v1.Node) { +func (vs *VSphere) setNodeZoneLabels(node *v1.Node) { nodeZone := node.ObjectMeta.Labels[v1.LabelTopologyZone] nodeRegion := node.ObjectMeta.Labels[v1.LabelTopologyRegion] ctx, cancel := context.WithCancel(context.Background()) @@ -1558,16 +1560,30 @@ func (vs *VSphere) reconcileZonesForNode(node *v1.Node) { labels[v1.LabelTopologyZone] = zone.FailureDomain labels[v1.LabelTopologyRegion] = zone.Region - updatedNode, err = vs.kubeClient.CoreV1().Nodes().Update(ctx, updatedNode, metav1.UpdateOptions{}) + err = tryUpdateNode(ctx, vs.kubeClient, updatedNode) if err != nil { klog.Errorf("vSphere cloud provider can not update node with zones info: %v", err) } else { - klog.V(3).Infof("Node %s updated with zone and region lables", node.Name) + klog.V(4).Infof("Node %s updated with zone and region labels", updatedNode.Name) } } } } +func tryUpdateNode(ctx context.Context, client clientset.Interface, updatedNode *v1.Node) error { + for i := 0; i < updateNodeRetryCount; i++ { + _, err := client.CoreV1().Nodes().Update(ctx, updatedNode, metav1.UpdateOptions{}) + if err != nil { + if !apierrors.IsConflict(err) { + return fmt.Errorf("vSphere cloud provider can not update node with zones info: %v", err) + } + } else { + return nil + } + } + return fmt.Errorf("update node exceeds retry count") +} + // Notification handler when node is removed from k8s cluster. func (vs *VSphere) NodeDeleted(obj interface{}) { node, ok := obj.(*v1.Node) @@ -1661,14 +1677,9 @@ func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f return f(c) } -// GetZone implements Zones.GetZone -func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { - nodeName, err := vs.CurrentNodeName(ctx, vs.hostName) - if err != nil { - klog.Errorf("Cannot get node name.") - return cloudprovider.Zone{}, err - } +func (vs *VSphere) getZoneByVmUUIDAndNodeName(ctx context.Context, vmUUID string, nodeName k8stypes.NodeName) (cloudprovider.Zone, error) { zone := cloudprovider.Zone{} + vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) if err != nil { klog.Errorf("Cannot connect to vsphere. Get zone for node %s error", nodeName) @@ -1679,7 +1690,7 @@ func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { klog.Errorf("Cannot connect to datacenter. Get zone for node %s error", nodeName) return cloudprovider.Zone{}, err } - vmHost, err := dc.GetHostByVMUUID(ctx, vs.vmUUID) + vmHost, err := dc.GetHostByVMUUID(ctx, vmUUID) if err != nil { klog.Errorf("Cannot find VM runtime host. Get zone for node %s error", nodeName) return cloudprovider.Zone{}, err @@ -1697,100 +1708,12 @@ func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { // search the hierarchy, example order: ["Host", "Cluster", "Datacenter", "Folder"] for i := range objects { obj := objects[len(objects)-1-i] - tags, err := client.ListAttachedTags(ctx, obj) + attachedTags, err := client.ListAttachedTags(ctx, obj) if err != nil { klog.Errorf("Cannot list attached tags. Get zone for node %s: %s", nodeName, err) return err } - for _, value := range tags { - tag, err := client.GetTag(ctx, value) - if err != nil { - klog.Errorf("Get tag %s: %s", value, err) - return err - } - category, err := client.GetCategory(ctx, tag.CategoryID) - if err != nil { - klog.Errorf("Get category %s error", value) - return err - } - - found := func() { - klog.Errorf("Found %q tag (%s) for %s attached to %s", category.Name, tag.Name, vs.vmUUID, obj.Reference()) - } - switch { - case category.Name == vs.cfg.Labels.Zone: - zone.FailureDomain = tag.Name - found() - case category.Name == vs.cfg.Labels.Region: - zone.Region = tag.Name - found() - } - - if zone.FailureDomain != "" && zone.Region != "" { - return nil - } - } - } - - if zone.Region == "" { - return fmt.Errorf("vSphere region category %q does not match any tags for node %s [%s]", vs.cfg.Labels.Region, nodeName, vs.vmUUID) - } - if zone.FailureDomain == "" { - return fmt.Errorf("vSphere zone category %q does not match any tags for node %s [%s]", vs.cfg.Labels.Zone, nodeName, vs.vmUUID) - } - - return nil - }) - if err != nil { - klog.Errorf("Get zone for node %s: %s", nodeName, err) - return cloudprovider.Zone{}, err - } - return zone, nil -} - -func (vs *VSphere) GetZoneByNodeName(ctx context.Context, nodeName k8stypes.NodeName) (cloudprovider.Zone, error) { - return cloudprovider.Zone{}, cloudprovider.NotImplemented -} - -//TODO (dmoiseev) refactor, commonize with GetZone -func (vs *VSphere) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { - zone := cloudprovider.Zone{} - vmUUID := strings.Replace(providerID, providerIDPrefix, "", 1) - - vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) - if err != nil { - klog.Errorf("Cannot connect to vsphere. Get zone for vm %s error", vmUUID) - return cloudprovider.Zone{}, err - } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) - if err != nil { - klog.Errorf("Cannot connect to datacenter. Get zone for vm %s error", vmUUID) - return cloudprovider.Zone{}, err - } - vmHost, err := dc.GetHostByVMUUID(ctx, vmUUID) - if err != nil { - klog.Errorf("Cannot find VM runtime host. Get zone for vm %s error", vmUUID) - return cloudprovider.Zone{}, err - } - - pc := vsi.conn.Client.ServiceContent.PropertyCollector - err = withTagsClient(ctx, vsi.conn, func(c *rest.Client) error { - client := tags.NewManager(c) - // example result: ["Folder", "Datacenter", "Cluster", "Host"] - objects, err := mo.Ancestors(ctx, vsi.conn.Client, pc, *vmHost) - if err != nil { - return err - } - - // search the hierarchy, example order: ["Host", "Cluster", "Datacenter", "Folder"] - for i := range objects { - obj := objects[len(objects)-1-i] - tags, err := client.ListAttachedTags(ctx, obj) - if err != nil { - klog.Errorf("Cannot list attached tags. Get zone for vm %s: %s", vmUUID, err) - return err - } - for _, value := range tags { + for _, value := range attachedTags { tag, err := client.GetTag(ctx, value) if err != nil { klog.Errorf("Get tag %s: %s", value, err) @@ -1821,22 +1744,49 @@ func (vs *VSphere) GetZoneByProviderID(ctx context.Context, providerID string) ( } if zone.Region == "" { - return fmt.Errorf("vSphere region category %q does not match any tags for vm %s", vs.cfg.Labels.Region, vmUUID) + return fmt.Errorf("vSphere region category %q does not match any tags for node %s [%s]", vs.cfg.Labels.Region, nodeName, vmUUID) } if zone.FailureDomain == "" { - return fmt.Errorf("vSphere zone category %q does not match any tags for vm %s", vs.cfg.Labels.Zone, vmUUID) + return fmt.Errorf("vSphere zone category %q does not match any tags for node %s [%s]", vs.cfg.Labels.Zone, nodeName, vmUUID) } return nil }) if err != nil { - klog.Errorf("Get zone for vm %s: %s", vmUUID, err) + klog.Errorf("Get zone for node %s: %s", nodeName, err) return cloudprovider.Zone{}, err } - return zone, nil } +// GetZone implements Zones.GetZone +func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) { + nodeName, err := vs.CurrentNodeName(ctx, vs.hostName) + if err != nil { + klog.Errorf("Cannot get node name.") + return cloudprovider.Zone{}, err + } + return vs.getZoneByVmUUIDAndNodeName(ctx, vs.vmUUID, nodeName) +} + +func (vs *VSphere) GetZoneByNodeName(ctx context.Context, nodeName k8stypes.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, cloudprovider.NotImplemented +} + +func (vs *VSphere) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { + var nodeName k8stypes.NodeName + vmUUID := strings.Replace(providerID, providerIDPrefix, "", 1) + + for nName, nInfo := range vs.nodeManager.nodeInfoMap { + if nInfo.vmUUID == vmUUID { + nodeName = convertToK8sType(nName) + break + } + } + + return vs.getZoneByVmUUIDAndNodeName(ctx, vmUUID, nodeName) +} + // GetLabelsForVolume implements the PVLabeler interface for VSphere // since this interface is used by the PV label admission controller. func (vs *VSphere) GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error) { From 273099bd522eb0a057e828fe0a497c781125db91 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Mon, 12 Apr 2021 13:49:59 +0200 Subject: [PATCH 4/6] reordered imports --- staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index c29c289c569..7dcb1477162 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -24,8 +24,6 @@ import ( "errors" "fmt" "io" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net" "net/url" @@ -46,6 +44,8 @@ import ( "github.com/vmware/govmomi/vim25/mo" vmwaretypes "github.com/vmware/govmomi/vim25/types" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" From bfdbfa2ab6f1710077fe1c44b5fa3a21949b7c39 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Thu, 15 Apr 2021 12:20:10 +0200 Subject: [PATCH 5/6] add description for elaborate changes in isZonesEnabled method --- staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index 7dcb1477162..de052b7f964 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -904,6 +904,10 @@ func (vs *VSphere) LoadBalancer() (cloudprovider.LoadBalancer, bool) { func (vs *VSphere) isZoneEnabled() bool { isEnabled := vs.cfg != nil && vs.cfg.Labels.Zone != "" && vs.cfg.Labels.Region != "" + // Return false within kubelet in case of credentials stored in secret. + // Otherwise kubelet will not be able to obtain zone labels from vSphere and create initial node + // due to no credentials at this step. + // See https://github.com/kubernetes/kubernetes/blob/b960f7a0e04687c17e0b0801e17e7cab89f273cc/pkg/kubelet/kubelet_node_status.go#L384-L386 if isEnabled && vs.isSecretInfoProvided && vs.nodeManager.credentialManager == nil { klog.V(1).Info("Zones can not be populated now due to credentials in Secret, skip.") return false From cd1d530a1a3a35db33356203b6f48125430acfd6 Mon Sep 17 00:00:00 2001 From: Denis Moiseev <1239415+lobziik@users.noreply.github.com> Date: Tue, 25 Jan 2022 14:37:58 +0100 Subject: [PATCH 6/6] Add zone/region labels sync function In the case of `vsphere-legacy-cloud-provider` client has insufficient permissions to update Node after its addition, this handler will attempt to populate Nodes topology labels later. --- .../legacy-cloud-providers/vsphere/vsphere.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index de052b7f964..756c3f6c999 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/staging/src/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -67,6 +67,7 @@ const ( ProviderName = "vsphere" providerIDPrefix = "vsphere://" updateNodeRetryCount = 3 + zoneLabelsResyncPeriod = 5 * time.Minute VolDir = "kubevols" RoundTripperDefaultCount = 3 DummyVMPrefixName = "vsphere-k8s" @@ -312,6 +313,11 @@ func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory) AddFunc: vs.NodeAdded, DeleteFunc: vs.NodeDeleted, }) + // Register sync function for node zone/region labels + nodeInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{UpdateFunc: vs.syncNodeZoneLabels}, + zoneLabelsResyncPeriod, + ) klog.V(4).Infof("Node informers in vSphere cloud provider initialized") } @@ -1544,6 +1550,31 @@ func (vs *VSphere) NodeAdded(obj interface{}) { vs.setNodeZoneLabels(node) } +// Node zone labels sync function, intended to be called periodically within kube-controller-manager. +func (vs *VSphere) syncNodeZoneLabels(_ interface{}, newObj interface{}) { + node, ok := newObj.(*v1.Node) + if node == nil || !ok { + klog.Warningf("NodeUpdated: unrecognized object %+v", newObj) + return + } + + // Populate zone and region labels if needed. + // This logic engages only if credentials provided via secret. + // Returns early if topology labels are already presented. + // https://github.com/kubernetes/kubernetes/issues/75175 + if vs.isSecretInfoProvided && vs.isZoneEnabled() { + labels := node.GetLabels() + _, zoneOk := labels[v1.LabelTopologyZone] + _, regionOk := labels[v1.LabelTopologyRegion] + if zoneOk && regionOk { + klog.V(6).Infof("Node topology labels are already populated") + return + } + klog.V(4).Infof("Topology labels was not found, trying to populate for node %s", node.Name) + vs.setNodeZoneLabels(node) + } +} + func (vs *VSphere) setNodeZoneLabels(node *v1.Node) { nodeZone := node.ObjectMeta.Labels[v1.LabelTopologyZone] nodeRegion := node.ObjectMeta.Labels[v1.LabelTopologyRegion]