diff --git a/README.md b/README.md index 0d14995d..5fae7797 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,50 @@ services: Note that RKE only supports connecting to TLS enabled etcd setup, user can enable multiple endpoints in the `external_urls` field. RKE will not accept having external urls and nodes with `etcd` role at the same time, user should only specify either etcd role for servers or external etcd but not both. +## Cloud Providers + +Starting from v0.1.3 rke supports cloud providers. + +### AWS Cloud Provider + +To enable AWS cloud provider, you can set the following in the cluster configuration file: +``` +cloud_provider: + name: aws +``` + +AWS cloud provider has to be enabled on ec2 instances with the right IAM role. + +### Azure Cloud provider + +Azure cloud provider can be enabled by passing `azure` as the cloud provider name and set of options to the configuration file: +``` +cloud_provider: + name: azure + cloud_config: + aadClientId: xxxxxxxxxxxx + aadClientSecret: xxxxxxxxxxx + location: westus + resourceGroup: rke-rg + subnetName: rke-subnet + subscriptionId: xxxxxxxxxxx + vnetName: rke-vnet + tenantId: xxxxxxxxxx + securityGroupName: rke-nsg +``` + +You also have to make sure that the Azure node name must match the kubernetes node name, you can do that by changing the value of hostname_override in the config file: +``` +nodes: + - address: x.x.x.x + hostname_override: azure-rke1 + user: ubuntu + role: + - controlplane + - etcd + - worker +``` + ## Operating Systems Notes ### Atomic OS diff --git a/cluster/cloud-provider.go b/cluster/cloud-provider.go new file mode 100644 index 00000000..fc8b75ec --- /dev/null +++ b/cluster/cloud-provider.go @@ -0,0 +1,97 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/docker/docker/api/types/container" + "github.com/rancher/rke/docker" + "github.com/rancher/rke/hosts" + "github.com/rancher/rke/log" + "github.com/rancher/types/apis/management.cattle.io/v3" + "github.com/sirupsen/logrus" +) + +const ( + CloudConfigDeployer = "cloud-config-deployer" + CloudConfigServiceName = "cloud" + CloudConfigPath = "/etc/kubernetes/cloud-config.json" + CloudConfigEnv = "RKE_CLOUD_CONFIG" +) + +func deployCloudProviderConfig(ctx context.Context, uniqueHosts []*hosts.Host, cloudProvider v3.CloudProvider, alpineImage string, prsMap map[string]v3.PrivateRegistry) error { + cloudConfig, err := getCloudConfigFile(ctx, cloudProvider) + if err != nil { + return err + } + for _, host := range uniqueHosts { + log.Infof(ctx, "[%s] Deploying cloud config file to node [%s]", CloudConfigServiceName, host.Address) + if err := doDeployConfigFile(ctx, host, cloudConfig, alpineImage, prsMap); err != nil { + return fmt.Errorf("Failed to deploy cloud config file on node [%s]: %v", host.Address, err) + } + } + return nil +} + +func getCloudConfigFile(ctx context.Context, cloudProvider v3.CloudProvider) (string, error) { + if len(cloudProvider.CloudConfig) == 0 { + return "", nil + } + tmpMap := make(map[string]interface{}) + for key, value := range cloudProvider.CloudConfig { + tmpBool, err := strconv.ParseBool(value) + if err == nil { + tmpMap[key] = tmpBool + continue + } + tmpInt, err := strconv.ParseInt(value, 10, 64) + if err == nil { + tmpMap[key] = tmpInt + continue + } + tmpFloat, err := strconv.ParseFloat(value, 64) + if err == nil { + tmpMap[key] = tmpFloat + continue + } + tmpMap[key] = value + } + jsonString, err := json.MarshalIndent(tmpMap, "", "\n") + if err != nil { + return "", err + } + return string(jsonString), nil +} + +func doDeployConfigFile(ctx context.Context, host *hosts.Host, cloudConfig, alpineImage string, prsMap map[string]v3.PrivateRegistry) error { + // remove existing container. Only way it's still here is if previous deployment failed + if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil { + return err + } + containerEnv := []string{CloudConfigEnv + "=" + cloudConfig} + imageCfg := &container.Config{ + Image: alpineImage, + Cmd: []string{ + "sh", + "-c", + fmt.Sprintf("if [ ! -f %s ]; then echo -e \"$%s\" > %s;fi", CloudConfigPath, CloudConfigEnv, CloudConfigPath), + }, + Env: containerEnv, + } + hostCfg := &container.HostConfig{ + Binds: []string{ + "/etc/kubernetes:/etc/kubernetes", + }, + Privileged: true, + } + if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, CloudConfigDeployer, host.Address, CloudConfigServiceName, prsMap); err != nil { + return err + } + if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil { + return err + } + logrus.Debugf("[%s] Successfully started cloud config deployer container on node [%s]", CloudConfigServiceName, host.Address) + return nil +} diff --git a/cluster/hosts.go b/cluster/hosts.go index a1a3b90b..032758cc 100644 --- a/cluster/hosts.go +++ b/cluster/hosts.go @@ -120,6 +120,12 @@ func (c *Cluster) SetUpHosts(ctx context.Context) error { return err } log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes") + if c.CloudProvider.Name != "" { + if err := deployCloudProviderConfig(ctx, hosts, c.CloudProvider, c.SystemImages.Alpine, c.PrivateRegistriesMap); err != nil { + return err + } + log.Infof(ctx, "[%s] Successfully deployed kubernetes cloud config to Cluster nodes", CloudConfigServiceName) + } } return nil } diff --git a/cluster/plan.go b/cluster/plan.go index dabda3e3..7ea71dfe 100644 --- a/cluster/plan.go +++ b/cluster/plan.go @@ -97,7 +97,9 @@ func (c *Cluster) BuildKubeAPIProcess() v3.Process { "kubelet-client-key": pki.GetKeyPath(pki.KubeAPICertName), "service-account-key-file": pki.GetKeyPath(pki.KubeAPICertName), } - + if len(c.CloudProvider.Name) > 0 { + CommandArgs["cloud-config"] = CloudConfigPath + } args := []string{ "--etcd-cafile=" + etcdCAClientCert, "--etcd-certfile=" + etcdClientCert, @@ -174,7 +176,9 @@ func (c *Cluster) BuildKubeControllerProcess() v3.Process { "service-account-private-key-file": pki.GetKeyPath(pki.KubeAPICertName), "root-ca-file": pki.GetCertPath(pki.CACertName), } - + if len(c.CloudProvider.Name) > 0 { + CommandArgs["cloud-config"] = CloudConfigPath + } args := []string{} if c.Authorization.Mode == services.RBACAuthorizationMode { args = append(args, "--use-service-account-credentials=true") @@ -249,6 +253,9 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host) v3.Process { if host.Address != host.InternalAddress { CommandArgs["node-ip"] = host.InternalAddress } + if len(c.CloudProvider.Name) > 0 { + CommandArgs["cloud-config"] = CloudConfigPath + } VolumesFrom := []string{ services.SidekickContainerName, } diff --git a/hosts/hosts.go b/hosts/hosts.go index fcb9da5e..ed60dc9c 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -37,7 +37,7 @@ type Host struct { const ( ToCleanEtcdDir = "/var/lib/etcd" - ToCleanSSLDir = "/etc/kubernetes/ssl" + ToCleanSSLDir = "/etc/kubernetes" ToCleanCNIConf = "/etc/cni" ToCleanCNIBin = "/opt/cni" ToCleanCNILib = "/var/lib/cni" diff --git a/vendor.conf b/vendor.conf index d462ad3e..fb6c496e 100644 --- a/vendor.conf +++ b/vendor.conf @@ -24,4 +24,4 @@ github.com/coreos/go-semver e214231b295a8ea9479f11b70b35d5acf3556d9 github.com/ugorji/go/codec ccfe18359b55b97855cee1d3f74e5efbda4869dc github.com/rancher/norman 151aa66e3e99de7e0d195e2d5ca96b1f95544555 -github.com/rancher/types c8a2bab012799603994eba13f59098c368350b27 +github.com/rancher/types 1e2d576b838b7e5bf71644e5bb488348262960e3 diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/alerting_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/alerting_types.go index f96aa39e..b61bf761 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/alerting_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/alerting_types.go @@ -81,13 +81,12 @@ type TargetPod struct { } type TargetEvent struct { - Type string `json:"type,omitempty" norman:"required,options=Normal|Warning,default=Warning"` + EventType string `json:"eventType,omitempty" norman:"required,options=Normal|Warning,default=Warning"` ResourceKind string `json:"resourceKind,omitempty" norman:"required,options=Pod|Node|Deployment|Statefulset|Daemonset"` } type TargetWorkload struct { WorkloadID string `json:"workloadId,omitempty"` - Type string `json:"type,omitempty" norman:"required,options=deployment|statefulset|daemonset,default=deployment"` Selector map[string]string `json:"selector,omitempty"` AvailablePercentage int `json:"availablePercentage,omitempty" norman:"required,min=1,max=100,default=70"` } @@ -157,3 +156,8 @@ type WebhookConfig struct { type NotifierStatus struct { } + +type AlertSystemImages struct { + AlertManager string `json:"alertManager,omitempty"` + AlertManagerHelper string `json:"alertManagerHelper,omitempty"` +} diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go index 0033998c..a19239ef 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/k8s_defaults.go @@ -20,6 +20,31 @@ var ( "v1.9.5-rancher1-1": v19SystemImages, } + // ToolsSystemImages default images for alert, pipeline, logging + ToolsSystemImages = struct { + AlertSystemImages AlertSystemImages + PipelineSystemImages PipelineSystemImages + LoggingSystemImages LoggingSystemImages + }{ + AlertSystemImages: AlertSystemImages{ + AlertManager: "prom/alertmanager:v0.11.0", + AlertManagerHelper: "rancher/alertmanager-helper:v0.0.2", + }, + PipelineSystemImages: PipelineSystemImages{ + Jenkins: "jenkins/jenkins:lts", + JenkinsJnlp: "jenkins/jnlp-slave:3.10-1-alpine", + AlpineGit: "alpine/git", + PluginsDocker: "plugins/docker", + }, + LoggingSystemImages: LoggingSystemImages{ + Fluentd: "rancher/fluentd:v0.1.4", + FluentdHelper: "rancher/fluentd-helper:v0.1.1", + Elaticsearch: "rancher/docker-elasticsearch-kubernetes:5.6.2", + Kibana: "kibana:5.6.4", + Busybox: "busybox", + }, + } + // v18 system images defaults v18SystemImages = RKESystemImages{ Etcd: "rancher/coreos-etcd:v3.0.17", @@ -27,7 +52,7 @@ var ( Alpine: "alpine:latest", NginxProxy: "rancher/rke-nginx-proxy:v0.1.1", CertDownloader: "rancher/rke-cert-deployer:v0.1.1", - KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.0", + KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.1", KubeDNS: "rancher/k8s-dns-kube-dns-amd64:1.14.5", DNSmasq: "rancher/k8s-dns-dnsmasq-nanny-amd64:1.14.5", KubeDNSSidecar: "rancher/k8s-dns-sidecar-amd64:1.14.5", @@ -54,7 +79,7 @@ var ( Alpine: "alpine:latest", NginxProxy: "rancher/rke-nginx-proxy:v0.1.1", CertDownloader: "rancher/rke-cert-deployer:v0.1.1", - KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.0", + KubernetesServicesSidecar: "rancher/rke-service-sidekick:v0.1.1", KubeDNS: "rancher/k8s-dns-kube-dns-amd64:1.14.7", DNSmasq: "rancher/k8s-dns-dnsmasq-nanny-amd64:1.14.7", KubeDNSSidecar: "rancher/k8s-dns-sidecar-amd64:1.14.7", diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go index aba12e30..0ae2bbbf 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/logging_types.go @@ -119,3 +119,11 @@ type SyslogConfig struct { Program string `json:"program,omitempty"` Protocol string `json:"protocol,omitempty" norman:"default=udp,type=enum,options=udp|tcp"` } + +type LoggingSystemImages struct { + Fluentd string `json:"fluentd,omitempty"` + FluentdHelper string `json:"fluentdHelper,omitempty"` + Elaticsearch string `json:"elaticsearch,omitempty"` + Kibana string `json:"kibana,omitempty"` + Busybox string `json:"busybox,omitempty"` +} diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go index f459bbd4..2cb31879 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/machine_types.go @@ -220,14 +220,15 @@ type Condition struct { } type NodeDriverSpec struct { - DisplayName string `json:"displayName"` - Description string `json:"description"` - URL string `json:"url" norman:"required"` - ExternalID string `json:"externalId"` - Builtin bool `json:"builtin"` - Active bool `json:"active"` - Checksum string `json:"checksum"` - UIURL string `json:"uiUrl"` + DisplayName string `json:"displayName"` + Description string `json:"description"` + URL string `json:"url" norman:"required"` + ExternalID string `json:"externalId"` + Builtin bool `json:"builtin"` + Active bool `json:"active"` + Checksum string `json:"checksum"` + UIURL string `json:"uiUrl"` + WhitelistDomains []string `json:"whitelistDomains,omitempty"` } type PublicEndpoint struct { diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/pipeline_types.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/pipeline_types.go index 67f2c4d9..a151e67f 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/pipeline_types.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/pipeline_types.go @@ -235,3 +235,10 @@ type AuthUserInput struct { RedirectURL string `json:"redirectUrl,omitempty" norman:"type=string"` Code string `json:"code,omitempty" norman:"type=string,required"` } + +type PipelineSystemImages struct { + Jenkins string `json:"jenkins,omitempty"` + JenkinsJnlp string `json:"jenkinsJnlp,omitempty"` + AlpineGit string `json:"alpineGit,omitempty"` + PluginsDocker string `json:"pluginsDocker,omitempty"` +} diff --git a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go index 5e0acc0c..3c230d92 100644 --- a/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go +++ b/vendor/github.com/rancher/types/apis/management.cattle.io/v3/zz_generated_deepcopy.go @@ -39,6 +39,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*AlertStatus).DeepCopyInto(out.(*AlertStatus)) return nil }, InType: reflect.TypeOf(&AlertStatus{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*AlertSystemImages).DeepCopyInto(out.(*AlertSystemImages)) + return nil + }, InType: reflect.TypeOf(&AlertSystemImages{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*AuthAppInput).DeepCopyInto(out.(*AuthAppInput)) return nil @@ -391,6 +395,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*LoggingStatus).DeepCopyInto(out.(*LoggingStatus)) return nil }, InType: reflect.TypeOf(&LoggingStatus{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*LoggingSystemImages).DeepCopyInto(out.(*LoggingSystemImages)) + return nil + }, InType: reflect.TypeOf(&LoggingSystemImages{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*NetworkConfig).DeepCopyInto(out.(*NetworkConfig)) return nil @@ -539,6 +547,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*PipelineStatus).DeepCopyInto(out.(*PipelineStatus)) return nil }, InType: reflect.TypeOf(&PipelineStatus{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*PipelineSystemImages).DeepCopyInto(out.(*PipelineSystemImages)) + return nil + }, InType: reflect.TypeOf(&PipelineSystemImages{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*PodSecurityPolicyTemplate).DeepCopyInto(out.(*PodSecurityPolicyTemplate)) return nil @@ -977,6 +989,22 @@ func (in *AlertStatus) DeepCopy() *AlertStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AlertSystemImages) DeepCopyInto(out *AlertSystemImages) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertSystemImages. +func (in *AlertSystemImages) DeepCopy() *AlertSystemImages { + if in == nil { + return nil + } + out := new(AlertSystemImages) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AuthAppInput) DeepCopyInto(out *AuthAppInput) { *out = *in @@ -3299,6 +3327,22 @@ func (in *LoggingStatus) DeepCopy() *LoggingStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoggingSystemImages) DeepCopyInto(out *LoggingSystemImages) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoggingSystemImages. +func (in *LoggingSystemImages) DeepCopy() *LoggingSystemImages { + if in == nil { + return nil + } + out := new(LoggingSystemImages) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkConfig) DeepCopyInto(out *NetworkConfig) { *out = *in @@ -3420,7 +3464,7 @@ func (in *NodeDriver) DeepCopyInto(out *NodeDriver) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) return } @@ -3481,6 +3525,11 @@ func (in *NodeDriverList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeDriverSpec) DeepCopyInto(out *NodeDriverSpec) { *out = *in + if in.WhitelistDomains != nil { + in, out := &in.WhitelistDomains, &out.WhitelistDomains + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -4399,6 +4448,22 @@ func (in *PipelineStatus) DeepCopy() *PipelineStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PipelineSystemImages) DeepCopyInto(out *PipelineSystemImages) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PipelineSystemImages. +func (in *PipelineSystemImages) DeepCopy() *PipelineSystemImages { + if in == nil { + return nil + } + out := new(PipelineSystemImages) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodSecurityPolicyTemplate) DeepCopyInto(out *PodSecurityPolicyTemplate) { *out = *in diff --git a/vendor/github.com/rancher/types/vendor.conf b/vendor/github.com/rancher/types/vendor.conf index 67b4fbb9..4366f6fc 100644 --- a/vendor/github.com/rancher/types/vendor.conf +++ b/vendor/github.com/rancher/types/vendor.conf @@ -5,4 +5,4 @@ k8s.io/kubernetes v1.8.3 bitbucket.org/ww/goautoneg a547fc61f48d567d5b4ec6f8aee5573d8efce11d https://github.com/rancher/goautoneg.git golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 -github.com/rancher/norman a978cad0e8751968fec4371f9ab6df6d446a389b +github.com/rancher/norman 41c044bb256b54652f7fae51fde7c45cb6c4ccb9