Allow setting Azure cloud provider from Kubernetes secrets instread of local configure files

This commit is contained in:
Pengfei Ni 2019-05-20 22:10:06 +08:00
parent a07b027261
commit 8f0e05fb6f
5 changed files with 256 additions and 67 deletions

View File

@ -14,6 +14,7 @@ go_library(
"azure_blobDiskController.go",
"azure_cache.go",
"azure_client.go",
"azure_config.go",
"azure_controller_common.go",
"azure_controller_standard.go",
"azure_controller_vmss.go",
@ -39,6 +40,7 @@ go_library(
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",

View File

@ -36,25 +36,25 @@ var (
// AzureAuthConfig holds auth related part of cloud config
type AzureAuthConfig struct {
// The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13
Cloud string `json:"cloud" yaml:"cloud"`
Cloud string `json:"cloud,omitempty" yaml:"cloud,omitempty"`
// The AAD Tenant ID for the Subscription that the cluster is deployed in
TenantID string `json:"tenantId" yaml:"tenantId"`
TenantID string `json:"tenantId,omitempty" yaml:"tenantId,omitempty"`
// The ClientID for an AAD application with RBAC access to talk to Azure RM APIs
AADClientID string `json:"aadClientId" yaml:"aadClientId"`
AADClientID string `json:"aadClientId,omitempty" yaml:"aadClientId,omitempty"`
// The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs
AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
AADClientSecret string `json:"aadClientSecret,omitempty" yaml:"aadClientSecret,omitempty"`
// The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs
AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
AADClientCertPath string `json:"aadClientCertPath,omitempty" yaml:"aadClientCertPath,omitempty"`
// The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs
AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"`
AADClientCertPassword string `json:"aadClientCertPassword,omitempty" yaml:"aadClientCertPassword,omitempty"`
// Use managed service identity for the virtual machine to access Azure ARM APIs
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"`
UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty" yaml:"useManagedIdentityExtension,omitempty"`
// UserAssignedIdentityID contains the Client ID of the user assigned MSI which is assigned to the underlying VMs. If empty the user assigned identity is not used.
// More details of the user assigned identity can be found at: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/overview
// For the user assigned identity specified here to be used, the UseManagedIdentityExtension has to be set to true.
UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"`
UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"`
// The ID of the Azure Subscription that the cluster is deployed in
SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
SubscriptionID string `json:"subscriptionId,omitempty" yaml:"subscriptionId,omitempty"`
}
// GetServicePrincipalToken creates a new service principal token based on the configuration

View File

@ -36,13 +36,13 @@ import (
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/flowcontrol"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/klog"
"k8s.io/legacy-cloud-providers/azure/auth"
"sigs.k8s.io/yaml"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"k8s.io/klog"
"sigs.k8s.io/yaml"
)
const (
@ -68,6 +68,10 @@ const (
externalResourceGroupLabel = "kubernetes.azure.com/resource-group"
managedByAzureLabel = "kubernetes.azure.com/managed"
// the prefix of secret for Azure cloud provider. The secret should include
// base64-encoded cloud config data with key 'cloud-config'.
azureSecretNamePrefix = "azure-cloud-provider"
)
var (
@ -83,78 +87,83 @@ type Config struct {
auth.AzureAuthConfig
// The name of the resource group that the cluster is deployed in
ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"`
ResourceGroup string `json:"resourceGroup,omitempty" yaml:"resourceGroup,omitempty"`
// The location of the resource group that the cluster is deployed in
Location string `json:"location" yaml:"location"`
Location string `json:"location,omitempty" yaml:"location,omitempty"`
// The name of the VNet that the cluster is deployed in
VnetName string `json:"vnetName" yaml:"vnetName"`
VnetName string `json:"vnetName,omitempty" yaml:"vnetName,omitempty"`
// The name of the resource group that the Vnet is deployed in
VnetResourceGroup string `json:"vnetResourceGroup" yaml:"vnetResourceGroup"`
VnetResourceGroup string `json:"vnetResourceGroup,omitempty" yaml:"vnetResourceGroup,omitempty"`
// The name of the subnet that the cluster is deployed in
SubnetName string `json:"subnetName" yaml:"subnetName"`
SubnetName string `json:"subnetName,omitempty" yaml:"subnetName,omitempty"`
// The name of the security group attached to the cluster's subnet
SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"`
SecurityGroupName string `json:"securityGroupName,omitempty" yaml:"securityGroupName,omitempty"`
// (Optional in 1.6) The name of the route table attached to the subnet that the cluster is deployed in
RouteTableName string `json:"routeTableName" yaml:"routeTableName"`
RouteTableName string `json:"routeTableName,omitempty" yaml:"routeTableName,omitempty"`
// The name of the resource group that the RouteTable is deployed in
RouteTableResourceGroup string `json:"routeTableResourceGroup" yaml:"routeTableResourceGroup"`
RouteTableResourceGroup string `json:"routeTableResourceGroup,omitempty" yaml:"routeTableResourceGroup,omitempty"`
// (Optional) The name of the availability set that should be used as the load balancer backend
// If this is set, the Azure cloudprovider will only add nodes from that availability set to the load
// balancer backend pool. If this is not set, and multiple agent pools (availability sets) are used, then
// the cloudprovider will try to add all nodes to a single backend pool which is forbidden.
// In other words, if you use multiple agent pools (availability sets), you MUST set this field.
PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName" yaml:"primaryAvailabilitySetName"`
PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName,omitempty" yaml:"primaryAvailabilitySetName,omitempty"`
// The type of azure nodes. Candidate values are: vmss and standard.
// If not set, it will be default to standard.
VMType string `json:"vmType" yaml:"vmType"`
VMType string `json:"vmType,omitempty" yaml:"vmType,omitempty"`
// The name of the scale set that should be used as the load balancer backend.
// If this is set, the Azure cloudprovider will only add nodes from that scale set to the load
// balancer backend pool. If this is not set, and multiple agent pools (scale sets) are used, then
// the cloudprovider will try to add all nodes to a single backend pool which is forbidden.
// In other words, if you use multiple agent pools (scale sets), you MUST set this field.
PrimaryScaleSetName string `json:"primaryScaleSetName" yaml:"primaryScaleSetName"`
PrimaryScaleSetName string `json:"primaryScaleSetName,omitempty" yaml:"primaryScaleSetName,omitempty"`
// Enable exponential backoff to manage resource request retries
CloudProviderBackoff bool `json:"cloudProviderBackoff" yaml:"cloudProviderBackoff"`
CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty" yaml:"cloudProviderBackoff,omitempty"`
// Backoff retry limit
CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries" yaml:"cloudProviderBackoffRetries"`
CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty" yaml:"cloudProviderBackoffRetries,omitempty"`
// Backoff exponent
CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent" yaml:"cloudProviderBackoffExponent"`
CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty" yaml:"cloudProviderBackoffExponent,omitempty"`
// Backoff duration
CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration" yaml:"cloudProviderBackoffDuration"`
CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty" yaml:"cloudProviderBackoffDuration,omitempty"`
// Backoff jitter
CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter" yaml:"cloudProviderBackoffJitter"`
CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty" yaml:"cloudProviderBackoffJitter,omitempty"`
// Backoff mode, options are v2 and default.
// * default means two-layer backoff retrying, one in the cloud provider and the other in the Azure SDK.
// * v2 means only backoff in the Azure SDK is used. In such mode, CloudProviderBackoffDuration and
// CloudProviderBackoffJitter are omitted.
// "default" will be used if not specified.
CloudProviderBackoffMode string `json:"cloudProviderBackoffMode" yaml:"cloudProviderBackoffMode"`
CloudProviderBackoffMode string `json:"cloudProviderBackoffMode,omitempty" yaml:"cloudProviderBackoffMode,omitempty"`
// Enable rate limiting
CloudProviderRateLimit bool `json:"cloudProviderRateLimit" yaml:"cloudProviderRateLimit"`
CloudProviderRateLimit bool `json:"cloudProviderRateLimit,omitempty" yaml:"cloudProviderRateLimit,omitempty"`
// Rate limit QPS (Read)
CloudProviderRateLimitQPS float32 `json:"cloudProviderRateLimitQPS" yaml:"cloudProviderRateLimitQPS"`
CloudProviderRateLimitQPS float32 `json:"cloudProviderRateLimitQPS,omitempty" yaml:"cloudProviderRateLimitQPS,omitempty"`
// Rate limit Bucket Size
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"`
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty" yaml:"cloudProviderRateLimitBucket,omitempty"`
// Rate limit QPS (Write)
CloudProviderRateLimitQPSWrite float32 `json:"cloudProviderRateLimitQPSWrite" yaml:"cloudProviderRateLimitQPSWrite"`
CloudProviderRateLimitQPSWrite float32 `json:"cloudProviderRateLimitQPSWrite,omitempty" yaml:"cloudProviderRateLimitQPSWrite,omitempty"`
// Rate limit Bucket Size
CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite" yaml:"cloudProviderRateLimitBucketWrite"`
CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty" yaml:"cloudProviderRateLimitBucketWrite,omitempty"`
// Use instance metadata service where possible
UseInstanceMetadata bool `json:"useInstanceMetadata" yaml:"useInstanceMetadata"`
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty" yaml:"useInstanceMetadata,omitempty"`
// Sku of Load Balancer and Public IP. Candidate values are: basic and standard.
// If not set, it will be default to basic.
LoadBalancerSku string `json:"loadBalancerSku" yaml:"loadBalancerSku"`
LoadBalancerSku string `json:"loadBalancerSku,omitempty" yaml:"loadBalancerSku,omitempty"`
// ExcludeMasterFromStandardLB excludes master nodes from standard load balancer.
// If not set, it will be default to true.
ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB" yaml:"excludeMasterFromStandardLB"`
ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty" yaml:"excludeMasterFromStandardLB,omitempty"`
// DisableOutboundSNAT disables the outbound SNAT for public load balancer rules.
// It should only be set when loadBalancerSku is standard. If not set, it will be default to false.
DisableOutboundSNAT *bool `json:"disableOutboundSNAT" yaml:"disableOutboundSNAT"`
DisableOutboundSNAT *bool `json:"disableOutboundSNAT,omitempty" yaml:"disableOutboundSNAT,omitempty"`
// Maximum allowed LoadBalancer Rule Count is the limit enforced by Azure Load balancer
MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount" yaml:"maximumLoadBalancerRuleCount"`
MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty" yaml:"maximumLoadBalancerRuleCount,omitempty"`
// The configure type for Azure cloud provider secret.
ConfigType secretConfigureType `json:"configType,omitempty" yaml:"configType,omitempty"`
// The override type for Azure cloud provider secret.
OverrideType secretOverrideType `json:"overrideType,omitempty" yaml:"overrideType,omitempty"`
}
var _ cloudprovider.Interface = (*Cloud)(nil)
@ -233,6 +242,28 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
return nil, err
}
az := &Cloud{
nodeZones: map[string]sets.String{},
nodeResourceGroups: map[string]string{},
unmanagedNodes: sets.NewString(),
routeCIDRs: map[string]string{},
}
err = az.initializeCloudFromConfig(config, false)
if err != nil {
return nil, err
}
return az, nil
}
// initializeCloudFromConfig initializes the Cloud from config.
func (az *Cloud) initializeCloudFromConfig(config *Config, fromSecret bool) error {
// cloud-config not set, return nil so that it would be initialized from secret.
if config == nil {
klog.Warning("cloud-config is not provided, Azure cloud provider would be initialized from secret")
return nil
}
if config.RouteTableResourceGroup == "" {
config.RouteTableResourceGroup = config.ResourceGroup
}
@ -244,19 +275,42 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
env, err := auth.ParseAzureEnvironment(config.Cloud)
if err != nil {
return nil, err
return err
}
servicePrincipalToken, err := auth.GetServicePrincipalToken(&config.AzureAuthConfig, env)
if err == auth.ErrorNoAuth {
if !config.UseInstanceMetadata {
// No credentials provided, useInstanceMetadata should be enabled.
return nil, fmt.Errorf("useInstanceMetadata must be enabled without Azure credentials")
runingAsKubelet, err := isRunningAsKubelet()
if err != nil {
return err
}
if runingAsKubelet {
// No credentials provided, useInstanceMetadata should be enabled for Kubelet.
if !config.UseInstanceMetadata {
return fmt.Errorf("useInstanceMetadata must be enabled without Azure credentials")
}
} else {
// Credentials are required for controller-manager for lazy initialization from secret.
if fromSecret {
err := fmt.Errorf("No credentials provided for Azure cloud provider")
klog.Fatalf("%v", err)
return err
}
// Credentials are required if override type is "no".
if az.Config.OverrideType == secretOverrideTypeNo {
return fmt.Errorf("no credentials provided for Azure cloud provider")
}
// Controller manager could be initialized from secret.
klog.V(2).Infof("No credentials provided, lazy initialize from secret %s", getConfigSecretName(az.Config.ConfigType))
return nil
}
klog.V(2).Infof("Azure cloud provider is starting without credentials")
} else if err != nil {
return nil, err
return err
}
// operationPollRateLimiter.Accept() is a no-op if rate limits are configured off.
@ -351,28 +405,22 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
}
} else {
if config.DisableOutboundSNAT != nil && *config.DisableOutboundSNAT {
return nil, fmt.Errorf("disableOutboundSNAT should only set when loadBalancerSku is standard")
return fmt.Errorf("disableOutboundSNAT should only set when loadBalancerSku is standard")
}
}
az := Cloud{
Config: *config,
Environment: *env,
nodeZones: map[string]sets.String{},
nodeResourceGroups: map[string]string{},
unmanagedNodes: sets.NewString(),
routeCIDRs: map[string]string{},
resourceRequestBackoff: resourceRequestBackoff,
}
az.Config = *config
az.Environment = *env
az.resourceRequestBackoff = resourceRequestBackoff
az.metadata, err = NewInstanceMetadataService(metadataURL)
if err != nil {
return nil, err
return err
}
// No credentials provided, InstanceMetadataService would be used for getting Azure resources.
// Note that this only applies to Kubelet, controller-manager should configure credentials for managing Azure resources.
if servicePrincipalToken == nil {
return &az, nil
return nil
}
// Initialize Azure clients.
@ -407,52 +455,53 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
}
if strings.EqualFold(vmTypeVMSS, az.Config.VMType) {
az.vmSet, err = newScaleSet(&az)
az.vmSet, err = newScaleSet(az)
if err != nil {
return nil, err
return err
}
} else {
az.vmSet = newAvailabilitySet(&az)
az.vmSet = newAvailabilitySet(az)
}
az.vmCache, err = az.newVMCache()
if err != nil {
return nil, err
return err
}
az.lbCache, err = az.newLBCache()
if err != nil {
return nil, err
return err
}
az.nsgCache, err = az.newNSGCache()
if err != nil {
return nil, err
return err
}
az.rtCache, err = az.newRouteTableCache()
if err != nil {
return nil, err
return err
}
if err := initDiskControllers(&az); err != nil {
return nil, err
if err := initDiskControllers(az); err != nil {
return err
}
return &az, nil
return nil
}
// parseConfig returns a parsed configuration for an Azure cloudprovider config file
func parseConfig(configReader io.Reader) (*Config, error) {
var config Config
if configReader == nil {
return &config, nil
return nil, nil
}
configContents, err := ioutil.ReadAll(configReader)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(configContents, &config)
if err != nil {
return nil, err
@ -470,6 +519,7 @@ func (az *Cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder,
az.eventBroadcaster = record.NewBroadcaster()
az.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: az.kubeClient.CoreV1().Events("")})
az.eventRecorder = az.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "azure-cloud-provider"})
az.initializeCloudFromSecret()
}
// LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise.

View File

@ -0,0 +1,122 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog"
"sigs.k8s.io/yaml"
)
const (
secretNamespace = "kube-system"
secretCloudConfigKey = "cloud-config"
)
// The configure type for Azure cloud provider secret. Supported values are:
// * all : configure applied for components (kubelet and controller-manager). This is the default value.
// * node : configure applied for nodes (kubelet).
// * control-plane : configure applied for control plane components (controller-manager).
//
// For different configure types, the secret name would also be different:
// * all : secret name would be azure-cloud-provider.
// * node : secret name would azure-cloud-provider-node.
// * control-plane : secret name would be azure-cloud-provider-control-plane.
type secretConfigureType string
const (
secretConfigureAll secretConfigureType = "all"
secretConfigureNode secretConfigureType = "node"
secretConfigureControlPlane secretConfigureType = "control-plane"
)
// The override type for Azure cloud provider secret. Supported values are:
// * no : The values from secret won't override any configures from local cloud-config file.
// * must : The values from secret would override all configures from local cloud-config file.
// * can : The values from secret would override only configurations that are explicitly set in the secret. This is the default value.
type secretOverrideType string
const (
secretOverrideTypeNo secretOverrideType = "no"
secretOverrideTypeMust secretOverrideType = "must"
secretOverrideTypeCan secretOverrideType = "can"
)
func (az *Cloud) initializeCloudFromSecret() {
config, err := az.getConfigFromSecret()
if err != nil {
klog.Warningf("Failed to get cloud-config from secret: %v, skip initializing from secret", err)
return
}
if config == nil {
// Skip re-initialization if the config is not override.
return
}
if err := az.initializeCloudFromConfig(config, true); err != nil {
klog.Errorf("Failed to initialize Azure cloud provider: %v", err)
}
}
func (az *Cloud) getConfigFromSecret() (*Config, error) {
// No override, return nil.
if az.Config.OverrideType == secretOverrideTypeNo {
return nil, nil
}
secretName := getConfigSecretName(az.Config.ConfigType)
secret, err := az.kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("Failed to get secret %s: %v", secretName, err)
}
cloudConfigData, ok := secret.Data[secretCloudConfigKey]
if !ok {
return nil, fmt.Errorf("cloud-config is not set in the secret (%s)", secretName)
}
config := Config{}
if az.Config.OverrideType == "" || az.Config.OverrideType == secretOverrideTypeCan {
// "can" override, set default value to existing config.
config = az.Config
}
err = yaml.Unmarshal(cloudConfigData, &config)
if err != nil {
return nil, fmt.Errorf("Failed to parse Azure cloud-config: %v", err)
}
return &config, nil
}
func getConfigSecretName(configType secretConfigureType) string {
switch configType {
case secretConfigureAll:
return azureSecretNamePrefix
case secretConfigureNode:
return fmt.Sprintf("%s-node", azureSecretNamePrefix)
case secretConfigureControlPlane:
return fmt.Sprintf("%s-control-plane", azureSecretNamePrefix)
default:
// default secret name is azure-cloud-provider.
return azureSecretNamePrefix
}
}

View File

@ -19,6 +19,7 @@ package azure
import (
"fmt"
"net/http"
"path/filepath"
"regexp"
"strings"
"time"
@ -26,6 +27,7 @@ import (
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-07-01/network"
"github.com/Azure/go-autorest/autorest"
"github.com/kardianos/osext"
"k8s.io/apimachinery/pkg/types"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/klog"
@ -362,3 +364,16 @@ func isBackendPoolOnSameLB(newBackendPoolID string, existingBackendPools []strin
return true, "", nil
}
func isRunningAsKubelet() (bool, error) {
exe, err := osext.Executable()
if err != nil {
return false, fmt.Errorf("cloud not find the service executable: %v", err)
}
if strings.Contains(filepath.Base(exe), "kubelet") {
return true, nil
}
return false, nil
}