diff --git a/build/common.sh b/build/common.sh index 9aafdbdea5e..af87a0d3cc9 100644 --- a/build/common.sh +++ b/build/common.sh @@ -551,6 +551,14 @@ function kube::release::package_salt_tarball() { cp -R "${KUBE_ROOT}/cluster/saltbase" "${release_stage}/" + # TODO(#3579): This is a temporary hack. It gathers up the yaml, + # yaml.in files in cluster/addons (minus any demos) and overlays + # them into kube-addons, where we expect them. (This pipeline is a + # fancy copy, stripping anything but the files we don't want.) + local objects + objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . -name \*.yaml -or -name \*.yaml.in | grep -v demo) + tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/saltbase/salt/kube-addons" + local package_name="${RELEASE_DIR}/kubernetes-salt.tar.gz" kube::release::create_tarball "${package_name}" "${release_stage}/.." } diff --git a/cluster/addons/dns/README.md b/cluster/addons/dns/README.md index cf2c14630a7..512bedd7570 100644 --- a/cluster/addons/dns/README.md +++ b/cluster/addons/dns/README.md @@ -1,6 +1,6 @@ # DNS in Kubernetes -This directory holds an example of how to run -[SkyDNS](https://github.com/skynetservices/skydns) in a Kubernetes cluster. +[SkyDNS](https://github.com/skynetservices/skydns) can be configured +to automatically run in a Kubernetes cluster. ## What things get DNS names? The only objects to which we are assigning DNS names are Services. Every @@ -18,23 +18,14 @@ Of course, giving services a name is just half of the problem - DNS names need a domain also. This implementation uses the variable `DNS_DOMAIN` (see below). You can configure your docker daemon with the flag `--dns-search`. -## How do I run it? -The first thing you have to do is substitute the variables into the -configuration. You can then feed the result into `kubectl`. +## How do I configure it? +The following environment variables are used at cluster startup to create the SkyDNS pods and configure the kubelets. If you need to, you can reconfigure your provider as necessary (e.g. `cluster/gce/config-default.sh`): ```shell -DNS_SERVER_IP=10.0.0.10 -DNS_DOMAIN=kubernetes.local -DNS_REPLICAS=2 - -sed -e "s/{DNS_DOMAIN}/$DNS_DOMAIN/g" \ - -e "s/{DNS_REPLICAS}/$DNS_REPLICAS/g" \ - ./cluster/addons/dns/skydns-rc.yaml.in \ - | ./cluster/kubectl.sh create -f - - -sed -e "s/{DNS_SERVER_IP}/$DNS_SERVER_IP/g" \ - ./cluster/addons/dns/skydns-svc.yaml.in \ - | ./cluster/kubectl.sh create -f - +ENABLE_CLUSTER_DNS=true +DNS_SERVER_IP="10.0.0.10" +DNS_DOMAIN="kubernetes.local" +DNS_REPLICAS=1 ``` ## How does it work? diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in index 1043de7d997..15c6d50623d 100644 --- a/cluster/addons/dns/skydns-rc.yaml.in +++ b/cluster/addons/dns/skydns-rc.yaml.in @@ -5,7 +5,7 @@ namespace: default labels: k8s-app: skydns desiredState: - replicas: {DNS_REPLICAS} + replicas: {{ pillar['dns_replicas'] }} replicaSelector: k8s-app: skydns podTemplate: @@ -28,7 +28,7 @@ desiredState: image: kubernetes/kube2sky:1.0 command: [ # entrypoint = "/kube2sky", - "-domain={DNS_DOMAIN}", + "-domain={{ pillar['dns_domain'] }}", ] - name: skydns image: kubernetes/skydns:2014-12-23-001 @@ -36,7 +36,7 @@ desiredState: # entrypoint = "/skydns", "-machines=http://localhost:4001", "-addr=0.0.0.0:53", - "-domain={DNS_DOMAIN}.", + "-domain={{ pillar['dns_domain'] }}.", ] ports: - name: dns diff --git a/cluster/addons/dns/skydns-svc.yaml.in b/cluster/addons/dns/skydns-svc.yaml.in index 3898a638bf7..f94d414d5f3 100644 --- a/cluster/addons/dns/skydns-svc.yaml.in +++ b/cluster/addons/dns/skydns-svc.yaml.in @@ -4,7 +4,7 @@ id: skydns namespace: default protocol: UDP port: 53 -portalIP: {DNS_SERVER_IP} +portalIP: {{ pillar['dns_server'] }} containerPort: 53 labels: k8s-app: skydns diff --git a/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in b/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in index 87ab48d30fd..ebad3bab2ff 100644 --- a/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in +++ b/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in @@ -2,7 +2,7 @@ apiVersion: v1beta1 kind: ReplicationController id: elasticsearch-logging-controller desiredState: - replicas: {ELASTICSEARCH_LOGGING_REPLICAS} + replicas: {{ pillar['elasticsearch_replicas'] }} replicaSelector: name: elasticsearch-logging podTemplate: diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 96e7b4ae1a9..e2b0a6e727e 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -106,14 +106,14 @@ function ensure-temp-dir { fi } -function setup-monitoring { +function setup-monitoring-firewall { if [[ "${ENABLE_CLUSTER_MONITORING:-false}" == "true" ]]; then # TODO: Implement this. echo "Monitoring not currently supported on AWS" fi } -function teardown-monitoring { +function teardown-monitoring-firewall { if [[ "${ENABLE_CLUSTER_MONITORING:-false}" == "true" ]]; then # TODO: Implement this. echo "Monitoring not currently supported on AWS" @@ -296,10 +296,14 @@ function kube-up { echo "readonly AWS_ZONE='${ZONE}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly PORTAL_NET='${PORTAL_NET}'" + echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'" echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'" echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" + echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'" echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'" + echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'" echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh" @@ -498,10 +502,10 @@ function kube-down { $AWS_CMD delete-vpc --vpc-id $vpc_id > $LOG } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" -} \ No newline at end of file +} diff --git a/cluster/azure/util.sh b/cluster/azure/util.sh index e8c9a215064..d8675bbabb4 100644 --- a/cluster/azure/util.sh +++ b/cluster/azure/util.sh @@ -558,18 +558,18 @@ function restart-kube-proxy { } # Setup monitoring using heapster and InfluxDB -function setup-monitoring { +function setup-monitoring-firewall { echo "not implemented" >/dev/null } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "not implemented" >/dev/null } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } diff --git a/cluster/gce/templates/create-dynamic-salt-files.sh b/cluster/gce/templates/create-dynamic-salt-files.sh index 0f894423d73..5440da94331 100644 --- a/cluster/gce/templates/create-dynamic-salt-files.sh +++ b/cluster/gce/templates/create-dynamic-salt-files.sh @@ -22,10 +22,14 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")' portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")' +enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' +enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")' enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' +elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' +dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' EOF diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 391e8ca542a..35fc22e72da 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -393,10 +393,14 @@ function kube-up { echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly PORTAL_NET='${PORTAL_NET}'" + echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'" echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'" echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" + echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'" echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'" + echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'" echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" @@ -731,106 +735,70 @@ function restart-kube-proxy { ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart" } -# Setup monitoring using heapster and InfluxDB -function setup-monitoring { - if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then - echo "Setting up cluster monitoring using Heapster." +# Setup monitoring firewalls using heapster and InfluxDB +function setup-monitoring-firewall { + if [[ "${ENABLE_CLUSTER_MONITORING}" != "true" ]]; then + return + fi - detect-project - if ! gcloud compute firewall-rules --project "${PROJECT}" describe monitoring-heapster &> /dev/null; then - if ! gcloud compute firewall-rules create monitoring-heapster \ - --project "${PROJECT}" \ - --target-tags="${MINION_TAG}" \ - --network="${NETWORK}" \ - --allow tcp:80 tcp:8083 tcp:8086; then - echo -e "${color_red}Failed to set up firewall for monitoring ${color_norm}" && false - fi - fi + echo "Setting up firewalls to Heapster based cluster monitoring." - local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - local grafana_host="" - if "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/" &> /dev/null; then - # wait for pods to be scheduled on a node. - echo "waiting for monitoring pods to be scheduled." - for i in `seq 1 10`; do - grafana_host=$("${kubectl}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.hostIP}}:{{end}} | sed s/://g) - if [[ $grafana_host != *"<"* ]]; then + detect-project + gcloud compute firewall-rules create "${INSTANCE_PREFIX}-monitoring-heapster" --project "${PROJECT}" \ + --allow tcp:80 tcp:8083 tcp:8086 --target-tags="${MINION_TAG}" --network="${NETWORK}" + + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + local grafana_host="" + echo "waiting for monitoring pods to be scheduled." + for i in `seq 1 10`; do + grafana_host=$("${kubectl}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.hostIP}}:{{end}} | sed s/://g) + if [[ ${grafana_host} != *"<"* ]]; then break - fi - sleep 10 - done - if [[ $grafana_host != *"<"* ]]; then - echo - echo -e "${color_green}Grafana dashboard will be available at ${color_yellow}http://$grafana_host${color_green}. Wait for the monitoring dashboard to be online.${color_norm}" - echo - else - echo -e "${color_red}monitoring pods failed to be scheduled.${color_norm}" - fi - else - echo -e "${color_red}Failed to Setup Monitoring ${color_norm}" - teardown-monitoring fi + sleep 10 + done + if [[ ${grafana_host} != *"<"* ]]; then + echo + echo -e "${color_green}Grafana dashboard will be available at ${color_yellow}http://${grafana_host}${color_green}. Wait for the monitoring dashboard to be online.${color_norm}" + echo + else + echo -e "${color_red}Monitoring pods failed to be scheduled!${color_norm}" fi } -function teardown-monitoring { - if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then - detect-project - - local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - local kubecfg="${KUBE_ROOT}/cluster/kubecfg.sh" - "${kubecfg}" resize monitoring-influxGrafanaController 0 &> /dev/null || true - "${kubecfg}" resize monitoring-heapsterController 0 &> /dev/null || true - "${kubectl}" delete -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/" &> /dev/null || true - if gcloud compute firewall-rules describe --project "${PROJECT}" monitoring-heapster &> /dev/null; then - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - monitoring-heapster &> /dev/null || true - fi +function teardown-monitoring-firewall { + if [[ "${ENABLE_CLUSTER_MONITORING}" != "true" ]]; then + return fi + + detect-project + gcloud compute firewall-rules delete -q "${INSTANCE_PREFIX}-monitoring-heapster" --project "${PROJECT}" || true } -function setup-logging { +function setup-logging-firewall { # If logging with Fluentd to Elasticsearch is enabled then create pods # and services for Elasticsearch (for ingesting logs) and Kibana (for # viewing logs). - if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]] && \ - [[ "${LOGGING_DESTINATION-}" == "elasticsearch" ]] && \ - [[ "${ENABLE_CLUSTER_LOGGING-}" == "true" ]]; then - local -r kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - if sed -e "s/{ELASTICSEARCH_LOGGING_REPLICAS}/${ELASTICSEARCH_LOGGING_REPLICAS}/g" \ - "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in | \ - "${kubectl}" create -f - &> /dev/null && \ - "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/es-service.yaml &> /dev/null && \ - "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml &> /dev/null && \ - "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/kibana-service.yaml &> /dev/null; then - gcloud compute firewall-rules create fluentd-elasticsearch-logging --project "${PROJECT}" \ - --allow tcp:5601 tcp:9200 tcp:9300 --target-tags "${INSTANCE_PREFIX}"-minion || true - local -r region="${ZONE::-2}" - local -r es_ip=$(gcloud compute forwarding-rules --project "${PROJECT}" describe --region "${region}" elasticsearch-logging | grep IPAddress | awk '{print $2}') - local -r kibana_ip=$(gcloud compute forwarding-rules --project "${PROJECT}" describe --region "${region}" kibana-logging | grep IPAddress | awk '{print $2}') - echo - echo -e "${color_green}Cluster logs are ingested into Elasticsearch running at ${color_yellow}http://${es_ip}:9200" - echo -e "${color_green}Kibana logging dashboard will be available at ${color_yellow}http://${kibana_ip}:5601${color_norm}" - echo - else - echo -e "${color_red}Failed to launch Elasticsearch and Kibana pods and services for logging.${color_norm}" - fi + if [[ "${ENABLE_NODE_LOGGING-}" != "true" ]] || \ + [[ "${LOGGING_DESTINATION-}" != "elasticsearch" ]] || \ + [[ "${ENABLE_CLUSTER_LOGGING-}" != "true" ]]; then + return fi + + detect-project + gcloud compute firewall-rules create "${INSTANCE_PREFIX}-fluentd-elasticsearch-logging" --project "${PROJECT}" \ + --allow tcp:5601 tcp:9200 tcp:9300 --target-tags "${MINION_TAG}" --network="${NETWORK}" } -function teardown-logging { - if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]] && \ - [[ "${LOGGING_DESTINATION-}" == "elasticsearch" ]] && \ - [[ "${ENABLE_CLUSTER_LOGGING-}" == "true" ]]; then - local -r kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - "${kubectl}" delete replicationController elasticsearch-logging-controller &> /dev/null || true - "${kubectl}" delete service elasticsearch-logging &> /dev/null || true - "${kubectl}" delete replicationController kibana-logging-controller &> /dev/null || true - "${kubectl}" delete service kibana-logging &> /dev/null || true - gcloud compute firewall-rules delete -q fluentd-elasticsearch-logging --project "${PROJECT}" || true +function teardown-logging-firewall { + if [[ "${ENABLE_NODE_LOGGING-}" != "true" ]] || \ + [[ "${LOGGING_DESTINATION-}" != "elasticsearch" ]] || \ + [[ "${ENABLE_CLUSTER_LOGGING-}" != "true" ]]; then + return fi + + detect-project + gcloud compute firewall-rules delete -q "${INSTANCE_PREFIX}-fluentd-elasticsearch-logging" --project "${PROJECT}" || true } # Perform preparations required to run e2e tests diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index 2b92efed561..aa2388afee4 100644 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -115,8 +115,8 @@ function kube-up() { } # Called during cluster/kube-up.sh -function setup-monitoring() { - echo "... in setup-monitoring()" >&2 +function setup-monitoring-firewall() { + echo "... in setup-monitoring-firewall()" >&2 # TODO(mbforbes): This isn't currently supported in GKE. } @@ -239,8 +239,8 @@ function test-teardown() { } # Tears down monitoring. -function teardown-monitoring() { - echo "... in teardown-monitoring()" >&2 +function teardown-monitoring-firewall() { + echo "... in teardown-monitoring-firewall()" >&2 # TODO(mbforbes): This isn't currently supported in GKE. } @@ -257,10 +257,10 @@ function kube-down() { --zone="${ZONE}" "${CLUSTER_NAME}" } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } diff --git a/cluster/kube-down.sh b/cluster/kube-down.sh index 866a5caaf2e..cb431f13697 100755 --- a/cluster/kube-down.sh +++ b/cluster/kube-down.sh @@ -27,8 +27,8 @@ source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" echo "Bringing down cluster using provider: $KUBERNETES_PROVIDER" verify-prereqs -teardown-monitoring -teardown-logging +teardown-monitoring-firewall +teardown-logging-firewall kube-down diff --git a/cluster/kube-up.sh b/cluster/kube-up.sh index 247287cdeff..37d59d9f59d 100755 --- a/cluster/kube-up.sh +++ b/cluster/kube-up.sh @@ -39,22 +39,10 @@ kube-up echo "... calling validate-cluster" >&2 "${KUBE_ROOT}/cluster/validate-cluster.sh" -echo "... calling setup-monitoring" >&2 -setup-monitoring +echo "... calling setup-monitoring-firewall" >&2 +setup-monitoring-firewall -if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then - echo "... setting up cluster DNS" - sed -e "s/{DNS_DOMAIN}/$DNS_DOMAIN/g" \ - -e "s/{DNS_REPLICAS}/$DNS_REPLICAS/g" \ - "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.in" \ - | "${KUBE_ROOT}/cluster/kubectl.sh" create -f - - - sed -e "s/{DNS_SERVER_IP}/$DNS_SERVER_IP/g" \ - "${KUBE_ROOT}/cluster/addons/dns/skydns-svc.yaml.in" \ - | "${KUBE_ROOT}/cluster/kubectl.sh" create -f - -fi - -echo "... calling setup-logging" >&2 -setup-logging +echo "... calling setup-logging-firewall" >&2 +setup-logging-firewall echo "Done" >&2 diff --git a/cluster/rackspace/util.sh b/cluster/rackspace/util.sh index f8c4552fc0d..bc1637d6194 100644 --- a/cluster/rackspace/util.sh +++ b/cluster/rackspace/util.sh @@ -342,19 +342,19 @@ kube-up() { echo } -function setup-monitoring { +function setup-monitoring-firewall { echo "TODO" } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "TODO" } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls new file mode 100644 index 00000000000..a073f43afa4 --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -0,0 +1,79 @@ +{% if pillar.get('enable_cluster_monitoring', '').lower() == 'true' %} +/etc/kubernetes/addons/cluster-monitoring: + file.recurse: + - source: salt://kube-addons/cluster-monitoring + - include_pat: E@^.+\.yaml$ + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %} +/etc/kubernetes/addons/dns/skydns-svc.yaml: + file.managed: + - source: salt://kube-addons/dns/skydns-svc.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True + +/etc/kubernetes/addons/dns/skydns-rc.yaml: + file.managed: + - source: salt://kube-addons/dns/skydns-rc.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True +{% endif %} + +{% if pillar.get('enable_node_logging', '').lower() == 'true' + and pillar.get('logging_destination').lower() == 'elasticsearch' + and pillar.get('enable_cluster_logging', '').lower() == 'true' %} +/etc/kubernetes/addons/fluentd-elasticsearch: + file.recurse: + - source: salt://kube-addons/fluentd-elasticsearch + - include_pat: E@^.+\.yaml$ + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 + +/etc/kubernetes/addons/fluentd-elasticsearch/es-controller.yaml: + file.managed: + - source: salt://kube-addons/fluentd-elasticsearch/es-controller.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True +{% endif %} + +{% if grains['os_family'] == 'RedHat' %} + +/usr/lib/systemd/system/kube-addons.service: + file.managed: + - source: salt://kube-addons/kube-addons.service + - user: root + - group: root + +/usr/lib/systemd/scripts/kube-addons: + file.managed: + - source: salt://kube-addons/initd + - user: root + - group: root + - mode: 755 + +{% else %} + +/etc/init.d/kube-addons: + file.managed: + - source: salt://kube-addons/initd + - user: root + - group: root + - mode: 755 + +{% endif %} + +kube-addons: + service.running: + - enable: True diff --git a/cluster/saltbase/salt/kube-addons/initd b/cluster/saltbase/salt/kube-addons/initd new file mode 100644 index 00000000000..8cc1d422b5d --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/initd @@ -0,0 +1,123 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kube-addons +# Required-Start: $local_fs $network $syslog kube-apiserver +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Kubernetes Addon Object Manager +# Description: +# Enforces installation of Kubernetes Addon Objects +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Kubernetes Addon Object Manager" +NAME=kube-addons +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +KUBECTL=/usr/local/bin/kubectl + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +function addon_manager_async() { + # The business logic for whether a given object should be created + # was already enforced by salt, and /etc/kubernetes/addons is the + # managed result is of that. Start everything below that directory. + echo "== Kubernetes addon manager started at $(date -Is) ==" + for obj in $(find /etc/kubernetes/addons -name \*.yaml); do + ${KUBECTL} create -f ${obj} & + echo "++ addon ${obj} started in pid $! ++" + done + noerrors="true" + for pid in $(jobs -p); do + wait ${pid} || noerrors="false" + echo "++ pid ${pid} complete ++" + done + if [ ${noerrors} == "true" ]; then + echo "== Kubernetes addon manager completed successfully at $(date -Is) ==" + else + echo "== Kubernetes addon manager completed with errors at $(date -Is) ==" + fi + + # We stay around so that status checks by salt make it look like + # the service is good. (We could do this is other ways, but this + # is simple.) + sleep infinity +} + +# +# Function that starts the daemon/service +# +do_start() +{ + addon_manager_async >${DAEMON_LOG_FILE} 2>&1 & + echo $! > ${PIDFILE} + disown +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + kill $(cat ${PIDFILE}) + rm ${PIDFILE} + return +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + if [ ! -e ${PIDFILE} ]; then + exit 1 + fi + pid=$(cat ${PIDFILE}) + # Checks that ${pid} is running AND is us. + ps --no-headers ${pid} | grep ${SCRIPTNAME} > /dev/null || exit $? + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/cluster/saltbase/salt/kube-addons/kube-addons.service b/cluster/saltbase/salt/kube-addons/kube-addons.service new file mode 100644 index 00000000000..6942e87587b --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/kube-addons.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kubernetes Addon Object Manager +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +ExecStart=/usr/lib/systemd/scripts/kube-addons start + +[Install] +WantedBy=multi-user.target diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index e4ec1660a70..f58c6cec153 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -36,6 +36,7 @@ base: - nginx - kube-client-tools - logrotate + - kube-addons {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} - openvpn {% endif %} diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index c7f5a6fab9e..22df0ebf07d 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -82,10 +82,14 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")' cert_ip: '$(echo "$MASTER_IP" | sed -e "s/'/''/g")' + enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' + enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")' enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' + elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' + dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' EOF diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh index ca72fa3b690..4e8e54829cc 100644 --- a/cluster/vagrant/util.sh +++ b/cluster/vagrant/util.sh @@ -287,11 +287,11 @@ function restart-kube-proxy { ssh-to-node "$1" "sudo systemctl restart kube-proxy" } -function setup-monitoring { +function setup-monitoring-firewall { echo "TODO" 1>&2 } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "TODO" 1>&2 } @@ -300,10 +300,10 @@ function prepare-e2e() { echo "Vagrant doesn't need special preparations for e2e tests" 1>&2 } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" -} \ No newline at end of file +} diff --git a/cluster/vsphere/templates/create-dynamic-salt-files.sh b/cluster/vsphere/templates/create-dynamic-salt-files.sh index f9a01c41a9b..cb483ad1340 100755 --- a/cluster/vsphere/templates/create-dynamic-salt-files.sh +++ b/cluster/vsphere/templates/create-dynamic-salt-files.sh @@ -22,10 +22,14 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: $NODE_INSTANCE_PREFIX portal_net: $PORTAL_NET +enable_cluster_monitoring: $ENABLE_CLUSTER_MONITORING enable_node_monitoring: $ENABLE_NODE_MONITORING +enable_cluster_logging: $ENABLE_CLUSTER_LOGGING enable_node_logging: $ENABLE_NODE_LOGGING logging_destination: $LOGGING_DESTINATION +elasticsearch_replicas: $ELASTICSEARCH_LOGGING_REPLICAS enable_cluster_dns: $ENABLE_CLUSTER_DNS +dns_replicas: $DNS_REPLICAS dns_server: $DNS_SERVER_IP dns_domain: $DNS_DOMAIN EOF diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh index 03d279fa6b3..7469e0ac02d 100755 --- a/cluster/vsphere/util.sh +++ b/cluster/vsphere/util.sh @@ -478,18 +478,18 @@ function test-teardown { echo "TODO" } -function setup-monitoring { +function setup-monitoring-firewall { echo "TODO" } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "TODO" } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } \ No newline at end of file