Merge pull request #449 from galal-hussein/azure_provider

Add azure cloud provider

README.md

@@ -295,6 +295,50 @@ services:
 
 Note that RKE only supports connecting to TLS enabled etcd setup, user can enable multiple endpoints in the `external_urls` field. RKE will not accept having external urls and nodes with `etcd` role at the same time, user should only specify either etcd role for servers or external etcd but not both.
 
+## Cloud Providers
+
+Starting from v0.1.3 rke supports cloud providers.
+
+### AWS Cloud Provider
+
+To enable AWS cloud provider, you can set the following in the cluster configuration file:
+```
+cloud_provider:
+  name: aws
+```
+
+AWS cloud provider has to be enabled on ec2 instances with the right IAM role.
+
+### Azure Cloud provider
+
+Azure cloud provider can be enabled by passing `azure` as the cloud provider name and set of options to the configuration file:
+```
+cloud_provider:
+  name: azure
+  cloud_config:
+    aadClientId: xxxxxxxxxxxx
+    aadClientSecret: xxxxxxxxxxx
+    location: westus
+    resourceGroup: rke-rg
+    subnetName: rke-subnet
+    subscriptionId: xxxxxxxxxxx
+    vnetName: rke-vnet
+    tenantId: xxxxxxxxxx
+    securityGroupName: rke-nsg
+```
+
+You also have to make sure that the Azure node name must match the kubernetes node name, you can do that by changing the value of hostname_override in the config file:
+```
+nodes:
+  - address: x.x.x.x
+    hostname_override: azure-rke1
+    user: ubuntu
+    role:
+    - controlplane
+    - etcd
+    - worker
+```
+
 ## Operating Systems Notes
 
 ### Atomic OS

cluster/cloud-provider.go

@@ -0,0 +1,97 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/rancher/rke/docker"
+	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/log"
+	"github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	CloudConfigDeployer    = "cloud-config-deployer"
+	CloudConfigServiceName = "cloud"
+	CloudConfigPath        = "/etc/kubernetes/cloud-config.json"
+	CloudConfigEnv         = "RKE_CLOUD_CONFIG"
+)
+
+func deployCloudProviderConfig(ctx context.Context, uniqueHosts []*hosts.Host, cloudProvider v3.CloudProvider, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
+	cloudConfig, err := getCloudConfigFile(ctx, cloudProvider)
+	if err != nil {
+		return err
+	}
+	for _, host := range uniqueHosts {
+		log.Infof(ctx, "[%s] Deploying cloud config file to node [%s]", CloudConfigServiceName, host.Address)
+		if err := doDeployConfigFile(ctx, host, cloudConfig, alpineImage, prsMap); err != nil {
+			return fmt.Errorf("Failed to deploy cloud config file on node [%s]: %v", host.Address, err)
+		}
+	}
+	return nil
+}
+
+func getCloudConfigFile(ctx context.Context, cloudProvider v3.CloudProvider) (string, error) {
+	if len(cloudProvider.CloudConfig) == 0 {
+		return "", nil
+	}
+	tmpMap := make(map[string]interface{})
+	for key, value := range cloudProvider.CloudConfig {
+		tmpBool, err := strconv.ParseBool(value)
+		if err == nil {
+			tmpMap[key] = tmpBool
+			continue
+		}
+		tmpInt, err := strconv.ParseInt(value, 10, 64)
+		if err == nil {
+			tmpMap[key] = tmpInt
+			continue
+		}
+		tmpFloat, err := strconv.ParseFloat(value, 64)
+		if err == nil {
+			tmpMap[key] = tmpFloat
+			continue
+		}
+		tmpMap[key] = value
+	}
+	jsonString, err := json.MarshalIndent(tmpMap, "", "\n")
+	if err != nil {
+		return "", err
+	}
+	return string(jsonString), nil
+}
+
+func doDeployConfigFile(ctx context.Context, host *hosts.Host, cloudConfig, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
+	// remove existing container. Only way it's still here is if previous deployment failed
+	if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil {
+		return err
+	}
+	containerEnv := []string{CloudConfigEnv + "=" + cloudConfig}
+	imageCfg := &container.Config{
+		Image: alpineImage,
+		Cmd: []string{
+			"sh",
+			"-c",
+			fmt.Sprintf("if [ ! -f %s ]; then echo -e \"$%s\" > %s;fi", CloudConfigPath, CloudConfigEnv, CloudConfigPath),
+		},
+		Env: containerEnv,
+	}
+	hostCfg := &container.HostConfig{
+		Binds: []string{
+			"/etc/kubernetes:/etc/kubernetes",
+		},
+		Privileged: true,
+	}
+	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, CloudConfigDeployer, host.Address, CloudConfigServiceName, prsMap); err != nil {
+		return err
+	}
+	if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil {
+		return err
+	}
+	logrus.Debugf("[%s] Successfully started cloud config deployer container on node [%s]", CloudConfigServiceName, host.Address)
+	return nil
+}

cluster/hosts.go

@@ -120,6 +120,12 @@ func (c *Cluster) SetUpHosts(ctx context.Context) error {
 			return err
 		}
 		log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes")
+		if c.CloudProvider.Name != "" {
+			if err := deployCloudProviderConfig(ctx, hosts, c.CloudProvider, c.SystemImages.Alpine, c.PrivateRegistriesMap); err != nil {
+				return err
+			}
+			log.Infof(ctx, "[%s] Successfully deployed kubernetes cloud config to Cluster nodes", CloudConfigServiceName)
+		}
 	}
 	return nil
 }

cluster/plan.go

@@ -97,7 +97,9 @@ func (c *Cluster) BuildKubeAPIProcess() v3.Process {
 		"kubelet-client-key":              pki.GetKeyPath(pki.KubeAPICertName),
 		"service-account-key-file":        pki.GetKeyPath(pki.KubeAPICertName),
 	}
-
+	if len(c.CloudProvider.Name) > 0 {
+		CommandArgs["cloud-config"] = CloudConfigPath
+	}
 	args := []string{
 		"--etcd-cafile=" + etcdCAClientCert,
 		"--etcd-certfile=" + etcdClientCert,
@@ -174,7 +176,9 @@ func (c *Cluster) BuildKubeControllerProcess() v3.Process {
 		"service-account-private-key-file": pki.GetKeyPath(pki.KubeAPICertName),
 		"root-ca-file":                     pki.GetCertPath(pki.CACertName),
 	}
-
+	if len(c.CloudProvider.Name) > 0 {
+		CommandArgs["cloud-config"] = CloudConfigPath
+	}
 	args := []string{}
 	if c.Authorization.Mode == services.RBACAuthorizationMode {
 		args = append(args, "--use-service-account-credentials=true")
@@ -249,6 +253,9 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host) v3.Process {
 	if host.Address != host.InternalAddress {
 		CommandArgs["node-ip"] = host.InternalAddress
 	}
+	if len(c.CloudProvider.Name) > 0 {
+		CommandArgs["cloud-config"] = CloudConfigPath
+	}
 	VolumesFrom := []string{
 		services.SidekickContainerName,
 	}

hosts/hosts.go

@@ -37,7 +37,7 @@ type Host struct {
 
 const (
 	ToCleanEtcdDir       = "/var/lib/etcd"
-	ToCleanSSLDir        = "/etc/kubernetes/ssl"
+	ToCleanSSLDir        = "/etc/kubernetes"
 	ToCleanCNIConf       = "/etc/cni"
 	ToCleanCNIBin        = "/opt/cni"
 	ToCleanCNILib        = "/var/lib/cni"

vendor.conf

@@ -24,4 +24,4 @@ github.com/coreos/go-semver              e214231b295a8ea9479f11b70b35d5acf3556d9
 github.com/ugorji/go/codec               ccfe18359b55b97855cee1d3f74e5efbda4869dc
 
 github.com/rancher/norman                151aa66e3e99de7e0d195e2d5ca96b1f95544555
-github.com/rancher/types                 c8a2bab012799603994eba13f59098c368350b27
+github.com/rancher/types                 1e2d576b838b7e5bf71644e5bb488348262960e3

vendor/github.com/rancher/types/apis/management.cattle.io/v3/alerting_types.go

@@ -81,13 +81,12 @@ type TargetPod struct {
 }
 
 type TargetEvent struct {
-	Type         string `json:"type,omitempty" norman:"required,options=Normal|Warning,default=Warning"`
+	EventType    string `json:"eventType,omitempty" norman:"required,options=Normal|Warning,default=Warning"`
 	ResourceKind string `json:"resourceKind,omitempty" norman:"required,options=Pod|Node|Deployment|Statefulset|Daemonset"`
 }
 
 type TargetWorkload struct {
 	WorkloadID          string            `json:"workloadId,omitempty"`
-	Type                string            `json:"type,omitempty" norman:"required,options=deployment|statefulset|daemonset,default=deployment"`
 	Selector            map[string]string `json:"selector,omitempty"`
 	AvailablePercentage int               `json:"availablePercentage,omitempty" norman:"required,min=1,max=100,default=70"`
 }
@@ -157,3 +156,8 @@ type WebhookConfig struct {
 
 type NotifierStatus struct {
 }
+
+type AlertSystemImages struct {
+	AlertManager       string `json:"alertManager,omitempty"`
+	AlertManagerHelper string `json:"alertManagerHelper,omitempty"`
+}

vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go

@@ -20,6 +20,31 @@ var (
 		"v1.9.5-rancher1-1": v19SystemImages,
 	}
 
+	// ToolsSystemImages default images for alert, pipeline, logging
+	ToolsSystemImages = struct {
+		AlertSystemImages    AlertSystemImages
+		PipelineSystemImages PipelineSystemImages
+		LoggingSystemImages  LoggingSystemImages
+	}{
+		AlertSystemImages: AlertSystemImages{
+			AlertManager:       "prom/alertmanager:v0.11.0",
+			AlertManagerHelper: "rancher/alertmanager-helper:v0.0.2",
+		},
+		PipelineSystemImages: PipelineSystemImages{
+			Jenkins:       "jenkins/jenkins:lts",
+			JenkinsJnlp:   "jenkins/jnlp-slave:3.10-1-alpine",
+			AlpineGit:     "alpine/git",
+			PluginsDocker: "plugins/docker",
+		},
+		LoggingSystemImages: LoggingSystemImages{
+			Fluentd:       "rancher/fluentd:v0.1.4",
+			FluentdHelper: "rancher/fluentd-helper:v0.1.1",
+			Elaticsearch:  "rancher/docker-elasticsearch-kubernetes:5.6.2",
+			Kibana:        "kibana:5.6.4",
+			Busybox:       "busybox",
+		},
+	}
+
 	// v18 system images defaults
 	v18SystemImages = RKESystemImages{
 		Etcd:                      "rancher/coreos-etcd:v3.0.17",
@@ -27,7 +52,7 @@ var (
 		Alpine:                    "alpine:latest",
 		NginxProxy:                "rancher/rke-nginx-proxy:v0.1.1",
 		CertDownloader:            "rancher/rke-cert-deployer:v0.1.1",
-		KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.0",
+		KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.1",
 		KubeDNS:                   "rancher/k8s-dns-kube-dns-amd64:1.14.5",
 		DNSmasq:                   "rancher/k8s-dns-dnsmasq-nanny-amd64:1.14.5",
 		KubeDNSSidecar:            "rancher/k8s-dns-sidecar-amd64:1.14.5",
@@ -54,7 +79,7 @@ var (
 		Alpine:                    "alpine:latest",
 		NginxProxy:                "rancher/rke-nginx-proxy:v0.1.1",
 		CertDownloader:            "rancher/rke-cert-deployer:v0.1.1",
-		KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.0",
+		KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.1",
 		KubeDNS:                   "rancher/k8s-dns-kube-dns-amd64:1.14.7",
 		DNSmasq:                   "rancher/k8s-dns-dnsmasq-nanny-amd64:1.14.7",
 		KubeDNSSidecar:            "rancher/k8s-dns-sidecar-amd64:1.14.7",

vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go

@@ -119,3 +119,11 @@ type SyslogConfig struct {
 	Program  string `json:"program,omitempty"`
 	Protocol string `json:"protocol,omitempty" norman:"default=udp,type=enum,options=udp|tcp"`
 }
+
+type LoggingSystemImages struct {
+	Fluentd       string `json:"fluentd,omitempty"`
+	FluentdHelper string `json:"fluentdHelper,omitempty"`
+	Elaticsearch  string `json:"elaticsearch,omitempty"`
+	Kibana        string `json:"kibana,omitempty"`
+	Busybox       string `json:"busybox,omitempty"`
+}

vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go

@@ -220,14 +220,15 @@ type Condition struct {
 }
 
 type NodeDriverSpec struct {
-	DisplayName string `json:"displayName"`
+	DisplayName      string   `json:"displayName"`
-	Description string `json:"description"`
+	Description      string   `json:"description"`
-	URL         string `json:"url" norman:"required"`
+	URL              string   `json:"url" norman:"required"`
-	ExternalID  string `json:"externalId"`
+	ExternalID       string   `json:"externalId"`
-	Builtin     bool   `json:"builtin"`
+	Builtin          bool     `json:"builtin"`
-	Active      bool   `json:"active"`
+	Active           bool     `json:"active"`
-	Checksum    string `json:"checksum"`
+	Checksum         string   `json:"checksum"`
-	UIURL       string `json:"uiUrl"`
+	UIURL            string   `json:"uiUrl"`
+	WhitelistDomains []string `json:"whitelistDomains,omitempty"`
 }
 
 type PublicEndpoint struct {

vendor/github.com/rancher/types/apis/management.cattle.io/v3/pipeline_types.go

@@ -235,3 +235,10 @@ type AuthUserInput struct {
 	RedirectURL    string `json:"redirectUrl,omitempty" norman:"type=string"`
 	Code           string `json:"code,omitempty" norman:"type=string,required"`
 }
+
+type PipelineSystemImages struct {
+	Jenkins       string `json:"jenkins,omitempty"`
+	JenkinsJnlp   string `json:"jenkinsJnlp,omitempty"`
+	AlpineGit     string `json:"alpineGit,omitempty"`
+	PluginsDocker string `json:"pluginsDocker,omitempty"`
+}

vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go

@@ -39,6 +39,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			in.(*AlertStatus).DeepCopyInto(out.(*AlertStatus))
 			return nil
 		}, InType: reflect.TypeOf(&AlertStatus{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*AlertSystemImages).DeepCopyInto(out.(*AlertSystemImages))
+			return nil
+		}, InType: reflect.TypeOf(&AlertSystemImages{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*AuthAppInput).DeepCopyInto(out.(*AuthAppInput))
 			return nil
@@ -391,6 +395,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			in.(*LoggingStatus).DeepCopyInto(out.(*LoggingStatus))
 			return nil
 		}, InType: reflect.TypeOf(&LoggingStatus{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*LoggingSystemImages).DeepCopyInto(out.(*LoggingSystemImages))
+			return nil
+		}, InType: reflect.TypeOf(&LoggingSystemImages{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*NetworkConfig).DeepCopyInto(out.(*NetworkConfig))
 			return nil
@@ -539,6 +547,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			in.(*PipelineStatus).DeepCopyInto(out.(*PipelineStatus))
 			return nil
 		}, InType: reflect.TypeOf(&PipelineStatus{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*PipelineSystemImages).DeepCopyInto(out.(*PipelineSystemImages))
+			return nil
+		}, InType: reflect.TypeOf(&PipelineSystemImages{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*PodSecurityPolicyTemplate).DeepCopyInto(out.(*PodSecurityPolicyTemplate))
 			return nil
@@ -977,6 +989,22 @@ func (in *AlertStatus) DeepCopy() *AlertStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AlertSystemImages) DeepCopyInto(out *AlertSystemImages) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertSystemImages.
+func (in *AlertSystemImages) DeepCopy() *AlertSystemImages {
+	if in == nil {
+		return nil
+	}
+	out := new(AlertSystemImages)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AuthAppInput) DeepCopyInto(out *AuthAppInput) {
 	*out = *in
@@ -3299,6 +3327,22 @@ func (in *LoggingStatus) DeepCopy() *LoggingStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LoggingSystemImages) DeepCopyInto(out *LoggingSystemImages) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoggingSystemImages.
+func (in *LoggingSystemImages) DeepCopy() *LoggingSystemImages {
+	if in == nil {
+		return nil
+	}
+	out := new(LoggingSystemImages)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NetworkConfig) DeepCopyInto(out *NetworkConfig) {
 	*out = *in
@@ -3420,7 +3464,7 @@ func (in *NodeDriver) DeepCopyInto(out *NodeDriver) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
 	return
 }
@@ -3481,6 +3525,11 @@ func (in *NodeDriverList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NodeDriverSpec) DeepCopyInto(out *NodeDriverSpec) {
 	*out = *in
+	if in.WhitelistDomains != nil {
+		in, out := &in.WhitelistDomains, &out.WhitelistDomains
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -4399,6 +4448,22 @@ func (in *PipelineStatus) DeepCopy() *PipelineStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PipelineSystemImages) DeepCopyInto(out *PipelineSystemImages) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PipelineSystemImages.
+func (in *PipelineSystemImages) DeepCopy() *PipelineSystemImages {
+	if in == nil {
+		return nil
+	}
+	out := new(PipelineSystemImages)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PodSecurityPolicyTemplate) DeepCopyInto(out *PodSecurityPolicyTemplate) {
 	*out = *in

vendor/github.com/rancher/types/vendor.conf

@@ -5,4 +5,4 @@ k8s.io/kubernetes                             v1.8.3
 bitbucket.org/ww/goautoneg                    a547fc61f48d567d5b4ec6f8aee5573d8efce11d https://github.com/rancher/goautoneg.git
 golang.org/x/sync                             fd80eb99c8f653c847d294a001bdf2a3a6f768f5
 
-github.com/rancher/norman                     a978cad0e8751968fec4371f9ab6df6d446a389b
+github.com/rancher/norman                     41c044bb256b54652f7fae51fde7c45cb6c4ccb9