From 5ae161e0862ec3911f0aac1b8f40967ae0131d25 Mon Sep 17 00:00:00 2001 From: Elson O Rodriguez Date: Wed, 13 Apr 2016 16:40:38 -0700 Subject: [PATCH] Adding cloudprovider support for Openstack kube-up provider. Making the assumption that the person running kube-up has their Openstack environment setup, those same variables are being passed into heat, and then into openstack.conf. The salt codebase was modified to add openstack as well. --- .../fragments/configure-salt.yaml | 11 +++++- .../kubernetes-heat/fragments/run-salt.sh | 3 -- .../kubernetes-heat/kubecluster.yaml | 35 +++++++++++++++++++ .../openstack/kubernetes-heat/kubeminion.yaml | 30 ++++++++++++++++ cluster/openstack/util.sh | 5 +++ cluster/saltbase/salt/kube-apiserver/init.sls | 2 +- .../kube-apiserver/kube-apiserver.manifest | 8 +++-- .../kube-controller-manager.manifest | 6 +++- .../salt/kube-proxy/kube-proxy.manifest | 2 +- cluster/saltbase/salt/kubelet/default | 11 ++++-- cluster/saltbase/salt/top.sls | 6 +++- 11 files changed, 106 insertions(+), 13 deletions(-) diff --git a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml index aa20df28fb8..f137e6c73a6 100644 --- a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml @@ -16,13 +16,22 @@ write_files: network_mode: openvswitch networkInterfaceName: eth0 api_servers: $MASTER_IP - cloud: vagrant # It's not vagrant, but required to install Docker + cloud: openstack + cloud_config: /srv/kubernetes/openstack.conf roles: - $role runtime_config: "" docker_opts: "" master_extra_sans: "DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:kubernetes-master" keep_host_etcd: true + - path: /srv/kubernetes/openstack.conf + content: | + [Global] + auth-url=$OS_AUTH_URL + username=$OS_USERNAME + password=$OS_PASSWORD + region=$OS_REGION_NAME + tenant-id=$OS_TENANT_ID - path: /srv/salt-overlay/pillar/cluster-params.sls content: | service_cluster_ip_range: 10.246.0.0/16 diff --git a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh index 9f92e58d22d..65f018f7653 100644 --- a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh +++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh @@ -43,9 +43,6 @@ fi # Salt server runs at locahost echo "127.0.0.1 salt" >> /etc/hosts -# Currently heat template tells a lie that the target is Vagrant. If Vagrant cloud provider is enabled, "Unable to construct api.Node object for kubelet" error will occur. -sed -e 's/{{cloud_provider}}//' -i /srv/salt/kubelet/default - # Run salt-call # salt-call wants to start docker daemon but is unable to. # See . diff --git a/cluster/openstack/kubernetes-heat/kubecluster.yaml b/cluster/openstack/kubernetes-heat/kubecluster.yaml index af385cd44a0..47497606ed3 100644 --- a/cluster/openstack/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml @@ -90,6 +90,31 @@ parameters: timeout for the Wait Conditions default: 6000 + os_auth_url: + type: string + description: OpenStack Auth URL + default: false + + os_username: + type: string + description: OpenStack Username + default: false + + os_password: + type: string + description: OpenStack Password + default: false + + os_region_name: + type: string + description: OpenStack Region Name + default: false + + os_tenant_id: + type: string + description: OpenStack Tenant ID + default: false + enable_proxy: type: string description: Whether or not to enable proxy settings @@ -259,6 +284,11 @@ resources: template: {get_file: fragments/configure-salt.yaml} params: "$MASTER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} + "$OS_AUTH_URL": {get_param: os_auth_url} + "$OS_USERNAME": {get_param: os_username} + "$OS_PASSWORD": {get_param: os_password} + "$OS_REGION_NAME": {get_param: os_region_name} + "$OS_TENANT_ID": {get_param: os_tenant_id} "$role": "kubernetes-master" run_salt: @@ -354,6 +384,11 @@ resources: cluster_name: {get_param: "OS::stack_name"} secgroup_base: {get_resource: secgroup_base} secgroup_node: {get_resource: secgroup_node} + os_auth_url: {get_param: os_auth_url} + os_username: {get_param: os_username} + os_password: {get_param: os_password} + os_region_name: {get_param: os_region_name} + os_tenant_id: {get_param: os_tenant_id} enable_proxy: {get_param: enable_proxy } ftp_proxy: {get_param: ftp_proxy } http_proxy: {get_param: http_proxy } diff --git a/cluster/openstack/kubernetes-heat/kubeminion.yaml b/cluster/openstack/kubernetes-heat/kubeminion.yaml index 83888ce0334..0e34e9b51b4 100644 --- a/cluster/openstack/kubernetes-heat/kubeminion.yaml +++ b/cluster/openstack/kubernetes-heat/kubeminion.yaml @@ -41,6 +41,31 @@ parameters: type: string description: Token used by kube-proxy + os_auth_url: + type: string + description: OpenStack Auth URL + default: false + + os_username: + type: string + description: OpenStack Username + default: false + + os_password: + type: string + description: OpenStack Password + default: false + + os_region_name: + type: string + description: OpenStack Region Name + default: false + + os_tenant_id: + type: string + description: OpenStack Tenant ID + default: false + enable_proxy: type: string description: Whether or not to enable proxy settings @@ -172,6 +197,11 @@ resources: template: {get_file: fragments/configure-salt.yaml} params: "$MASTER_IP": {get_param: kube_master_ip} + "$OS_AUTH_URL": {get_param: os_auth_url} + "$OS_USERNAME": {get_param: os_username} + "$OS_PASSWORD": {get_param: os_password} + "$OS_REGION_NAME": {get_param: os_region_name} + "$OS_TENANT_ID": {get_param: os_tenant_id} "$role": "kubernetes-pool" run_salt: diff --git a/cluster/openstack/util.sh b/cluster/openstack/util.sh index b1a102cff0c..dd8014fd2c1 100644 --- a/cluster/openstack/util.sh +++ b/cluster/openstack/util.sh @@ -200,6 +200,11 @@ function run-heat-script() { -P dns_nameserver=${DNS_SERVER} \ -P kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \ -P kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \ + -P os_auth_url=${OS_AUTH_URL} \ + -P os_username=${OS_USERNAME} \ + -P os_password=${OS_PASSWORD} \ + -P os_region_name=${OS_REGION_NAME} \ + -P os_tenant_id=${OS_TENANT_ID} \ -P enable_proxy=${ENABLE_PROXY} \ -P ftp_proxy="${FTP_PROXY}" \ -P http_proxy="${HTTP_PROXY}" \ diff --git a/cluster/saltbase/salt/kube-apiserver/init.sls b/cluster/saltbase/salt/kube-apiserver/init.sls index fff84648877..2329437dbd1 100644 --- a/cluster/saltbase/salt/kube-apiserver/init.sls +++ b/cluster/saltbase/salt/kube-apiserver/init.sls @@ -1,4 +1,4 @@ -{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] %} +{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack] %} # TODO: generate and distribute tokens on other cloud providers. /srv/kubernetes/known_tokens.csv: file.managed: diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index a8ac7239e38..5dd4eaf8e22 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -18,6 +18,10 @@ {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} + {% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} + {% endif -%} + {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} @@ -58,7 +62,7 @@ {% set client_ca_file = "" -%} {% set secure_port = "6443" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set secure_port = "443" -%} {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} @@ -72,7 +76,7 @@ {% set basic_auth_file = "" -%} {% set authz_mode = "" -%} {% set abac_policy_file = "" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere'] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack'] %} {% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%} {% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%} {% set authz_mode = " --authorization-mode=ABAC" -%} diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index cbf650d2565..e7fc51803bf 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -37,6 +37,10 @@ {% endif -%} {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%} + {% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} + {% endif -%} + {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} @@ -46,7 +50,7 @@ {% set root_ca_file = "" -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index c5f8f2d8094..e49491b67c7 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -5,7 +5,7 @@ {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} {% set api_servers = "--master=https://" + ips[0][0] -%} {% endif -%} -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack' ] %} {% set api_servers_with_port = api_servers -%} {% else -%} {% set api_servers_with_port = api_servers + ":6443" -%} diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 5462c761a03..255faeb59df 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -16,7 +16,7 @@ {% endif -%} # TODO: remove nginx for other cloud providers. -{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %} {% set api_servers_with_port = api_servers -%} {% else -%} {% set api_servers_with_port = api_servers + ":6443" -%} @@ -28,7 +28,7 @@ {% set reconcile_cidr_args = "" -%} {% if grains['roles'][0] == 'kubernetes-master' -%} - {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller'] -%} + {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] -%} # Unless given a specific directive, disable registration for the kubelet # running on the master. @@ -52,6 +52,11 @@ {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} +{% set cloud_config = "" -%} +{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} +{% endif -%} + {% set config = "--config=/etc/kubernetes/manifests" -%} {% set manifest_url = "" -%} @@ -192,4 +197,4 @@ {% endif -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}" diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 4b84cefccf5..2bb99b7b084 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -58,6 +58,10 @@ base: - kube-controller-manager - kube-scheduler - supervisor +{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack' ] %} + - nginx +{% endif %} + - cadvisor - kube-client-tools - kube-master-addons - kube-admission-controls @@ -72,7 +76,7 @@ base: - logrotate {% endif %} - kube-addons -{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller' ] %} +{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller', 'openstack'] %} - docker - kubelet {% endif %}