diff --git a/cluster/azure/templates/salt-master.sh b/cluster/azure/templates/salt-master.sh index 21fd555363b..d9da3e6b07d 100644 --- a/cluster/azure/templates/salt-master.sh +++ b/cluster/azure/templates/salt-master.sh @@ -41,6 +41,14 @@ EOF mkdir -p /srv/salt/nginx echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd +mkdir -p /etc/openvpn +umask=$(umask) +umask 0066 +echo "$CA_CRT" > /etc/openvpn/ca.crt +echo "$SERVER_CRT" > /etc/openvpn/server.crt +echo "$SERVER_KEY" > /etc/openvpn/server.key +umask $umask + # Install Salt # # We specify -X to avoid a race condition that can cause minion failure to diff --git a/cluster/azure/templates/salt-minion.sh b/cluster/azure/templates/salt-minion.sh index 6792ecd3410..e60152ef7e6 100644 --- a/cluster/azure/templates/salt-minion.sh +++ b/cluster/azure/templates/salt-minion.sh @@ -14,6 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +mkdir -p /etc/openvpn +umask=$(umask) +umask 0066 +echo "$CA_CRT" > /etc/openvpn/ca.crt +echo "$CLIENT_CRT" > /etc/openvpn/client.crt +echo "$CLIENT_KEY" > /etc/openvpn/client.key +umask $umask + # Prepopulate the name of the Master mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf @@ -22,6 +30,10 @@ echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf # echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion hostnamef=$(hostname -f) +sudo apt-get install ipcalc +netmask=$(ipcalc $MINION_IP_RANGE | grep Netmask | awk '{ print $2 }') +network=$(ipcalc $MINION_IP_RANGE | grep Address | awk '{ print $2 }') +cbrstring="$network $netmask" # Our minions will have a pool role to distinguish them from the master. cat </etc/salt/minion.d/grains.conf @@ -31,6 +43,7 @@ grains: cbr-cidr: $MINION_IP_RANGE cloud: azure hostnamef: $hostnamef + cbr-string: $cbrstring EOF # Install Salt diff --git a/cluster/azure/util.sh b/cluster/azure/util.sh index 5302200fb41..290dadd3f32 100644 --- a/cluster/azure/util.sh +++ b/cluster/azure/util.sh @@ -70,12 +70,47 @@ function kube-up { ${KUBE_TEMP}/htpasswd $user $passwd HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) + # Generate openvpn certs + echo 01 > ${KUBE_TEMP}/ca.srl + openssl genrsa -out ${KUBE_TEMP}/ca.key + openssl req -new -x509 -days 1095 \ + -key ${KUBE_TEMP}/ca.key \ + -out ${KUBE_TEMP}/ca.crt \ + -subj "/CN=openvpn-ca" + openssl genrsa -out ${KUBE_TEMP}/server.key + openssl req -new \ + -key ${KUBE_TEMP}/server.key \ + -out ${KUBE_TEMP}/server.csr \ + -subj "/CN=server" + openssl x509 -req -days 1095 \ + -in ${KUBE_TEMP}/server.csr \ + -CA ${KUBE_TEMP}/ca.crt \ + -CAkey ${KUBE_TEMP}/ca.key \ + -CAserial ${KUBE_TEMP}/ca.srl \ + -out ${KUBE_TEMP}/server.crt + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + openssl genrsa -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.key + openssl req -new \ + -key ${KUBE_TEMP}/${MINION_NAMES[$i]}.key \ + -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \ + -subj "/CN=${MINION_NAMES[$i]}" + openssl x509 -req -days 1095 \ + -in ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \ + -CA ${KUBE_TEMP}/ca.crt \ + -CAkey ${KUBE_TEMP}/ca.key \ + -CAserial ${KUBE_TEMP}/ca.srl \ + -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt + done + # Build up start up script for master ( echo "#!/bin/bash" echo "MASTER_NAME=${MASTER_NAME}" echo "MASTER_RELEASE_TAR=${FULL_URL}" echo "MASTER_HTPASSWD='${HTPASSWD}'" + echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\"" + echo "SERVER_CRT=\"$(cat ${KUBE_TEMP}/server.crt)\"" + echo "SERVER_KEY=\"$(cat ${KUBE_TEMP}/server.key)\"" grep -v "^#" $SCRIPT_DIR/azure/templates/download-release.sh grep -v "^#" $SCRIPT_DIR/azure/templates/salt-master.sh ) > ${KUBE_TEMP}/master-start.sh @@ -87,11 +122,8 @@ function kube-up { fi if [ ! -f $AZ_SSH_CERT ]; then - openssl req -new -key $AZ_SSH_KEY -out ${KUBE_TEMP}/temp.csr \ - -subj "/C=US/ST=WA/L=Redmond/O=Azure-CLI/CN=Azure" - openssl req -x509 -key $AZ_SSH_KEY -in ${KUBE_TEMP}/temp.csr \ - -out $AZ_SSH_CERT -days 1095 - rm ${KUBE_TEMP}/temp.csr + openssl req -new -x509 -days 1095 -key $AZ_SSH_KEY -out $AZ_SSH_CERT \ + -subj "/CN=azure-ssh-key" fi if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then @@ -118,6 +150,9 @@ function kube-up { echo "#!/bin/bash" echo "MASTER_NAME=${MASTER_NAME}" echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" + echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\"" + echo "CLIENT_CRT=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt)\"" + echo "CLIENT_KEY=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.key)\"" grep -v "^#" $SCRIPT_DIR/azure/templates/salt-minion.sh ) > ${KUBE_TEMP}/minion-start-${i}.sh diff --git a/cluster/saltbase/salt/openvpn-client/client.conf b/cluster/saltbase/salt/openvpn-client/client.conf new file mode 100644 index 00000000000..a6207624474 --- /dev/null +++ b/cluster/saltbase/salt/openvpn-client/client.conf @@ -0,0 +1,53 @@ +# Specify that we are a client and that we +# will be pulling certain config file directives +# from the server. +client + +# Use the same setting as you are using on +# the server. +# On most systems, the VPN will not function +# unless you partially or fully disable +# the firewall for the TUN/TAP interface. +dev tun + +# Are we connecting to a TCP or +# UDP server? Use the same setting as +# on the server. +proto udp + +# The hostname/IP and port of the server. +# You can have multiple remote entries +# to load balance between the servers. +remote {{ salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').keys()[0] }} 1194 + +# Keep trying indefinitely to resolve the +# host name of the OpenVPN server. Very useful +# on machines which are not permanently connected +# to the internet such as laptops. +resolv-retry infinite + +# Most clients don't need to bind to +# a specific local port number. +nobind + +# Try to preserve some state across restarts. +persist-key +persist-tun + +# SSL/TLS parms. +# See the server config file for more +# description. It's best to use +# a separate .crt/.key file pair +# for each client. A single ca +# file can be used for all clients. +ca /etc/openvpn/ca.crt +cert /etc/openvpn/client.crt +key /etc/openvpn/client.key + +# Enable compression on the VPN link. +# Don't enable this unless it is also +# enabled in the server config file. +comp-lzo + +# Set log file verbosity. +verb 3 diff --git a/cluster/saltbase/salt/openvpn-client/init.sls b/cluster/saltbase/salt/openvpn-client/init.sls new file mode 100644 index 00000000000..c0dbc04b06a --- /dev/null +++ b/cluster/saltbase/salt/openvpn-client/init.sls @@ -0,0 +1,16 @@ +/etc/openvpn/client.conf: + file.managed: + - source: salt://openvpn-client/client.conf + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: True + +openvpn: + pkg: + - latest + service.running: + - enable: True + - watch: + - file: /etc/openvpn/client.conf diff --git a/cluster/saltbase/salt/openvpn/init.sls b/cluster/saltbase/salt/openvpn/init.sls new file mode 100644 index 00000000000..585238ccf4a --- /dev/null +++ b/cluster/saltbase/salt/openvpn/init.sls @@ -0,0 +1,31 @@ +/etc/openvpn/server.conf: + file.managed: + - source: salt://openvpn/server.conf + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: True + +{% for (minion, grains) in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').items() %} +/etc/openvpn/ccd/{{ minion }}: + file.managed: + - contents: "iroute {{ grains['cbr-string'] }}\n" + - user: root + - group: root + - mode: 644 + - makedirs: True +{% endfor %} + +openssl dhparam -out /etc/openvpn/dh1024.pem 1024: + cmd.run: + - creates: /etc/openvpn/dh1024.pem + - unless: file /etc/openvpn/dh1024.pem + +openvpn: + pkg: + - latest + service.running: + - enable: True + - watch: + - file: /etc/openvpn/server.conf diff --git a/cluster/saltbase/salt/openvpn/server.conf b/cluster/saltbase/salt/openvpn/server.conf new file mode 100644 index 00000000000..64ae567de86 --- /dev/null +++ b/cluster/saltbase/salt/openvpn/server.conf @@ -0,0 +1,123 @@ +# Which TCP/UDP port should OpenVPN listen on? +# If you want to run multiple OpenVPN instances +# on the same machine, use a different port +# number for each one. You will need to +# open up this port on your firewall. +port 1194 + +# TCP or UDP server? +proto udp + +# "dev tun" will create a routed IP tunnel, +# "dev tap" will create an ethernet tunnel. +# Use "dev tap0" if you are ethernet bridging +# and have precreated a tap0 virtual interface +# and bridged it with your ethernet interface. +# If you want to control access policies +# over the VPN, you must create firewall +# rules for the the TUN/TAP interface. +# On non-Windows systems, you can give +# an explicit unit number, such as tun0. +# On Windows, use "dev-node" for this. +# On most systems, the VPN will not function +# unless you partially or fully disable +# the firewall for the TUN/TAP interface. +dev tun + +# SSL/TLS root certificate (ca), certificate +# (cert), and private key (key). Each client +# and the server must have their own cert and +# key file. The server and all clients will +# use the same ca file. +# +# See the "easy-rsa" directory for a series +# of scripts for generating RSA certificates +# and private keys. Remember to use +# a unique Common Name for the server +# and each of the client certificates. +# +# Any X509 key management system can be used. +# OpenVPN can also use a PKCS #12 formatted key file +# (see "pkcs12" directive in man page). +ca /etc/openvpn/ca.crt +cert /etc/openvpn/server.crt +key /etc/openvpn/server.key # This file should be kept secret + +# Diffie hellman parameters. +# Generate your own with: +# openssl dhparam -out dh1024.pem 1024 +# Substitute 2048 for 1024 if you are using +# 2048 bit keys. +dh /etc/openvpn/dh1024.pem + +# Configure server mode and supply a VPN subnet +# for OpenVPN to draw client addresses from. +# The server will take 10.8.0.1 for itself, +# the rest will be made available to clients. +# Each client will be able to reach the server +# on 10.8.0.1. Comment this line out if you are +# ethernet bridging. See the man page for more info. +server 10.8.0.0 255.255.255.0 + +# Maintain a record of client <-> virtual IP address +# associations in this file. If OpenVPN goes down or +# is restarted, reconnecting clients can be assigned +# the same virtual IP address from the pool that was +# previously assigned. +ifconfig-pool-persist ipp.txt + +# To assign specific IP addresses to specific +# clients or if a connecting client has a private +# subnet behind it that should also have VPN access, +# use the subdirectory "ccd" for client-specific +# configuration files (see man page for more info). + +client-config-dir /etc/openvpn/ccd + +{% for minion in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values() %} +push "route {{ minion['cbr-string'] }}" +route {{ minion['cbr-string'] }} +{% endfor %} + +# Uncomment this directive to allow different +# clients to be able to "see" each other. +# By default, clients will only see the server. +# To force clients to only see the server, you +# will also need to appropriately firewall the +# server's TUN/TAP interface. +client-to-client + +# The keepalive directive causes ping-like +# messages to be sent back and forth over +# the link so that each side knows when +# the other side has gone down. +# Ping every 10 seconds, assume that remote +# peer is down if no ping received during +# a 120 second time period. +keepalive 10 120 + +# Enable compression on the VPN link. +# If you enable it here, you must also +# enable it in the client config file. +comp-lzo + +# The persist options will try to avoid +# accessing certain resources on restart +# that may no longer be accessible because +# of the privilege downgrade. +persist-key +persist-tun + +# Output a short status file showing +# current connections, truncated +# and rewritten every minute. +status openvpn-status.log + +# Set the appropriate level of log +# file verbosity. +# +# 0 is silent, except for fatal errors +# 4 is reasonable for general usage +# 5 and 6 can help to debug connection problems +# 9 is extremely verbose +verb 3 diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 1a7e038713e..eff1a260574 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -10,6 +10,9 @@ base: - kube-proxy - cadvisor - nsinit +{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} + - openvpn-client +{% endif %} 'roles:kubernetes-master': - match: grain @@ -19,6 +22,9 @@ base: - controller-manager - scheduler - nginx +{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} + - openvpn +{% endif %} 'roles:kubernetes-pool-vsphere': - match: grain