diff --git a/cluster/azure/templates/salt-master.sh b/cluster/azure/templates/salt-master.sh index 21fd555363b..d9da3e6b07d 100644 --- a/cluster/azure/templates/salt-master.sh +++ b/cluster/azure/templates/salt-master.sh @@ -41,6 +41,14 @@ EOF mkdir -p /srv/salt/nginx echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd +mkdir -p /etc/openvpn +umask=$(umask) +umask 0066 +echo "$CA_CRT" > /etc/openvpn/ca.crt +echo "$SERVER_CRT" > /etc/openvpn/server.crt +echo "$SERVER_KEY" > /etc/openvpn/server.key +umask $umask + # Install Salt # # We specify -X to avoid a race condition that can cause minion failure to diff --git a/cluster/azure/templates/salt-minion.sh b/cluster/azure/templates/salt-minion.sh index 6792ecd3410..e60152ef7e6 100644 --- a/cluster/azure/templates/salt-minion.sh +++ b/cluster/azure/templates/salt-minion.sh @@ -14,6 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +mkdir -p /etc/openvpn +umask=$(umask) +umask 0066 +echo "$CA_CRT" > /etc/openvpn/ca.crt +echo "$CLIENT_CRT" > /etc/openvpn/client.crt +echo "$CLIENT_KEY" > /etc/openvpn/client.key +umask $umask + # Prepopulate the name of the Master mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf @@ -22,6 +30,10 @@ echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf # echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion hostnamef=$(hostname -f) +sudo apt-get install ipcalc +netmask=$(ipcalc $MINION_IP_RANGE | grep Netmask | awk '{ print $2 }') +network=$(ipcalc $MINION_IP_RANGE | grep Address | awk '{ print $2 }') +cbrstring="$network $netmask" # Our minions will have a pool role to distinguish them from the master. cat </etc/salt/minion.d/grains.conf @@ -31,6 +43,7 @@ grains: cbr-cidr: $MINION_IP_RANGE cloud: azure hostnamef: $hostnamef + cbr-string: $cbrstring EOF # Install Salt diff --git a/cluster/azure/util.sh b/cluster/azure/util.sh index 5302200fb41..290dadd3f32 100644 --- a/cluster/azure/util.sh +++ b/cluster/azure/util.sh @@ -70,12 +70,47 @@ function kube-up { ${KUBE_TEMP}/htpasswd $user $passwd HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) + # Generate openvpn certs + echo 01 > ${KUBE_TEMP}/ca.srl + openssl genrsa -out ${KUBE_TEMP}/ca.key + openssl req -new -x509 -days 1095 \ + -key ${KUBE_TEMP}/ca.key \ + -out ${KUBE_TEMP}/ca.crt \ + -subj "/CN=openvpn-ca" + openssl genrsa -out ${KUBE_TEMP}/server.key + openssl req -new \ + -key ${KUBE_TEMP}/server.key \ + -out ${KUBE_TEMP}/server.csr \ + -subj "/CN=server" + openssl x509 -req -days 1095 \ + -in ${KUBE_TEMP}/server.csr \ + -CA ${KUBE_TEMP}/ca.crt \ + -CAkey ${KUBE_TEMP}/ca.key \ + -CAserial ${KUBE_TEMP}/ca.srl \ + -out ${KUBE_TEMP}/server.crt + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + openssl genrsa -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.key + openssl req -new \ + -key ${KUBE_TEMP}/${MINION_NAMES[$i]}.key \ + -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \ + -subj "/CN=${MINION_NAMES[$i]}" + openssl x509 -req -days 1095 \ + -in ${KUBE_TEMP}/${MINION_NAMES[$i]}.csr \ + -CA ${KUBE_TEMP}/ca.crt \ + -CAkey ${KUBE_TEMP}/ca.key \ + -CAserial ${KUBE_TEMP}/ca.srl \ + -out ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt + done + # Build up start up script for master ( echo "#!/bin/bash" echo "MASTER_NAME=${MASTER_NAME}" echo "MASTER_RELEASE_TAR=${FULL_URL}" echo "MASTER_HTPASSWD='${HTPASSWD}'" + echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\"" + echo "SERVER_CRT=\"$(cat ${KUBE_TEMP}/server.crt)\"" + echo "SERVER_KEY=\"$(cat ${KUBE_TEMP}/server.key)\"" grep -v "^#" $SCRIPT_DIR/azure/templates/download-release.sh grep -v "^#" $SCRIPT_DIR/azure/templates/salt-master.sh ) > ${KUBE_TEMP}/master-start.sh @@ -87,11 +122,8 @@ function kube-up { fi if [ ! -f $AZ_SSH_CERT ]; then - openssl req -new -key $AZ_SSH_KEY -out ${KUBE_TEMP}/temp.csr \ - -subj "/C=US/ST=WA/L=Redmond/O=Azure-CLI/CN=Azure" - openssl req -x509 -key $AZ_SSH_KEY -in ${KUBE_TEMP}/temp.csr \ - -out $AZ_SSH_CERT -days 1095 - rm ${KUBE_TEMP}/temp.csr + openssl req -new -x509 -days 1095 -key $AZ_SSH_KEY -out $AZ_SSH_CERT \ + -subj "/CN=azure-ssh-key" fi if [ -z "$(azure network vnet show $AZ_VNET 2>/dev/null | grep data)" ]; then @@ -118,6 +150,9 @@ function kube-up { echo "#!/bin/bash" echo "MASTER_NAME=${MASTER_NAME}" echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" + echo "CA_CRT=\"$(cat ${KUBE_TEMP}/ca.crt)\"" + echo "CLIENT_CRT=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.crt)\"" + echo "CLIENT_KEY=\"$(cat ${KUBE_TEMP}/${MINION_NAMES[$i]}.key)\"" grep -v "^#" $SCRIPT_DIR/azure/templates/salt-minion.sh ) > ${KUBE_TEMP}/minion-start-${i}.sh diff --git a/cluster/saltbase/salt/openvpn-client/client.conf b/cluster/saltbase/salt/openvpn-client/client.conf new file mode 100644 index 00000000000..a8f07c815ce --- /dev/null +++ b/cluster/saltbase/salt/openvpn-client/client.conf @@ -0,0 +1,123 @@ +############################################## +# Sample client-side OpenVPN 2.0 config file # +# for connecting to multi-client server. # +# # +# This configuration can be used by multiple # +# clients, however each client should have # +# its own cert and key files. # +# # +# On Windows, you might want to rename this # +# file so it has a .ovpn extension # +############################################## + +# Specify that we are a client and that we +# will be pulling certain config file directives +# from the server. +client + +# Use the same setting as you are using on +# the server. +# On most systems, the VPN will not function +# unless you partially or fully disable +# the firewall for the TUN/TAP interface. +;dev tap +dev tun + +# Windows needs the TAP-Win32 adapter name +# from the Network Connections panel +# if you have more than one. On XP SP2, +# you may need to disable the firewall +# for the TAP adapter. +;dev-node MyTap + +# Are we connecting to a TCP or +# UDP server? Use the same setting as +# on the server. +;proto tcp +proto udp + +# The hostname/IP and port of the server. +# You can have multiple remote entries +# to load balance between the servers. +remote {{ salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').keys()[0] }} 1194 +;remote my-server-2 1194 + +# Choose a random host from the remote +# list for load-balancing. Otherwise +# try hosts in the order specified. +;remote-random + +# Keep trying indefinitely to resolve the +# host name of the OpenVPN server. Very useful +# on machines which are not permanently connected +# to the internet such as laptops. +resolv-retry infinite + +# Most clients don't need to bind to +# a specific local port number. +nobind + +# Downgrade privileges after initialization (non-Windows only) +;user nobody +;group nogroup + +# Try to preserve some state across restarts. +persist-key +persist-tun + +# If you are connecting through an +# HTTP proxy to reach the actual OpenVPN +# server, put the proxy server/IP and +# port number here. See the man page +# if your proxy server requires +# authentication. +;http-proxy-retry # retry on connection failures +;http-proxy [proxy server] [proxy port #] + +# Wireless networks often produce a lot +# of duplicate packets. Set this flag +# to silence duplicate packet warnings. +;mute-replay-warnings + +# SSL/TLS parms. +# See the server config file for more +# description. It's best to use +# a separate .crt/.key file pair +# for each client. A single ca +# file can be used for all clients. +ca /etc/openvpn/ca.crt +cert /etc/openvpn/client.crt +key /etc/openvpn/client.key + +# Verify server certificate by checking +# that the certicate has the nsCertType +# field set to "server". This is an +# important precaution to protect against +# a potential attack discussed here: +# http://openvpn.net/howto.html#mitm +# +# To use this feature, you will need to generate +# your server certificates with the nsCertType +# field set to "server". The build-key-server +# script in the easy-rsa folder will do this. +;ns-cert-type server + +# If a tls-auth key is used on the server +# then every client must also have the key. +;tls-auth ta.key 1 + +# Select a cryptographic cipher. +# If the cipher option is used on the server +# then you must also specify it here. +;cipher x + +# Enable compression on the VPN link. +# Don't enable this unless it is also +# enabled in the server config file. +comp-lzo + +# Set log file verbosity. +verb 3 + +# Silence repeating messages +;mute 20 diff --git a/cluster/saltbase/salt/openvpn-client/init.sls b/cluster/saltbase/salt/openvpn-client/init.sls new file mode 100644 index 00000000000..c0dbc04b06a --- /dev/null +++ b/cluster/saltbase/salt/openvpn-client/init.sls @@ -0,0 +1,16 @@ +/etc/openvpn/client.conf: + file.managed: + - source: salt://openvpn-client/client.conf + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: True + +openvpn: + pkg: + - latest + service.running: + - enable: True + - watch: + - file: /etc/openvpn/client.conf diff --git a/cluster/saltbase/salt/openvpn/init.sls b/cluster/saltbase/salt/openvpn/init.sls new file mode 100644 index 00000000000..585238ccf4a --- /dev/null +++ b/cluster/saltbase/salt/openvpn/init.sls @@ -0,0 +1,31 @@ +/etc/openvpn/server.conf: + file.managed: + - source: salt://openvpn/server.conf + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: True + +{% for (minion, grains) in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').items() %} +/etc/openvpn/ccd/{{ minion }}: + file.managed: + - contents: "iroute {{ grains['cbr-string'] }}\n" + - user: root + - group: root + - mode: 644 + - makedirs: True +{% endfor %} + +openssl dhparam -out /etc/openvpn/dh1024.pem 1024: + cmd.run: + - creates: /etc/openvpn/dh1024.pem + - unless: file /etc/openvpn/dh1024.pem + +openvpn: + pkg: + - latest + service.running: + - enable: True + - watch: + - file: /etc/openvpn/server.conf diff --git a/cluster/saltbase/salt/openvpn/server.conf b/cluster/saltbase/salt/openvpn/server.conf new file mode 100644 index 00000000000..31b78cf4665 --- /dev/null +++ b/cluster/saltbase/salt/openvpn/server.conf @@ -0,0 +1,304 @@ +################################################# +# Sample OpenVPN 2.0 config file for # +# multi-client server. # +# # +# This file is for the server side # +# of a many-clients <-> one-server # +# OpenVPN configuration. # +# # +# OpenVPN also supports # +# single-machine <-> single-machine # +# configurations (See the Examples page # +# on the web site for more info). # +# # +# This config should work on Windows # +# or Linux/BSD systems. Remember on # +# Windows to quote pathnames and use # +# double backslashes, e.g.: # +# "C:\\Program Files\\OpenVPN\\config\\foo.key" # +# # +# Comments are preceded with '#' or ';' # +################################################# + +# Which local IP address should OpenVPN +# listen on? (optional) +;local a.b.c.d + +# Which TCP/UDP port should OpenVPN listen on? +# If you want to run multiple OpenVPN instances +# on the same machine, use a different port +# number for each one. You will need to +# open up this port on your firewall. +port 1194 + +# TCP or UDP server? +;proto tcp +proto udp + +# "dev tun" will create a routed IP tunnel, +# "dev tap" will create an ethernet tunnel. +# Use "dev tap0" if you are ethernet bridging +# and have precreated a tap0 virtual interface +# and bridged it with your ethernet interface. +# If you want to control access policies +# over the VPN, you must create firewall +# rules for the the TUN/TAP interface. +# On non-Windows systems, you can give +# an explicit unit number, such as tun0. +# On Windows, use "dev-node" for this. +# On most systems, the VPN will not function +# unless you partially or fully disable +# the firewall for the TUN/TAP interface. +;dev tap +dev tun + +# Windows needs the TAP-Win32 adapter name +# from the Network Connections panel if you +# have more than one. On XP SP2 or higher, +# you may need to selectively disable the +# Windows firewall for the TAP adapter. +# Non-Windows systems usually don't need this. +;dev-node MyTap + +# SSL/TLS root certificate (ca), certificate +# (cert), and private key (key). Each client +# and the server must have their own cert and +# key file. The server and all clients will +# use the same ca file. +# +# See the "easy-rsa" directory for a series +# of scripts for generating RSA certificates +# and private keys. Remember to use +# a unique Common Name for the server +# and each of the client certificates. +# +# Any X509 key management system can be used. +# OpenVPN can also use a PKCS #12 formatted key file +# (see "pkcs12" directive in man page). +ca /etc/openvpn/ca.crt +cert /etc/openvpn/server.crt +key /etc/openvpn/server.key # This file should be kept secret + +# Diffie hellman parameters. +# Generate your own with: +# openssl dhparam -out dh1024.pem 1024 +# Substitute 2048 for 1024 if you are using +# 2048 bit keys. +dh /etc/openvpn/dh1024.pem + +# Configure server mode and supply a VPN subnet +# for OpenVPN to draw client addresses from. +# The server will take 10.8.0.1 for itself, +# the rest will be made available to clients. +# Each client will be able to reach the server +# on 10.8.0.1. Comment this line out if you are +# ethernet bridging. See the man page for more info. +server 10.8.0.0 255.255.255.0 + +# Maintain a record of client <-> virtual IP address +# associations in this file. If OpenVPN goes down or +# is restarted, reconnecting clients can be assigned +# the same virtual IP address from the pool that was +# previously assigned. +ifconfig-pool-persist ipp.txt + +# Configure server mode for ethernet bridging. +# You must first use your OS's bridging capability +# to bridge the TAP interface with the ethernet +# NIC interface. Then you must manually set the +# IP/netmask on the bridge interface, here we +# assume 10.8.0.4/255.255.255.0. Finally we +# must set aside an IP range in this subnet +# (start=10.8.0.50 end=10.8.0.100) to allocate +# to connecting clients. Leave this line commented +# out unless you are ethernet bridging. +;server-bridge 10.8.0.4 255.255.255.0 10.8.0.50 10.8.0.100 + +# Configure server mode for ethernet bridging +# using a DHCP-proxy, where clients talk +# to the OpenVPN server-side DHCP server +# to receive their IP address allocation +# and DNS server addresses. You must first use +# your OS's bridging capability to bridge the TAP +# interface with the ethernet NIC interface. +# Note: this mode only works on clients (such as +# Windows), where the client-side TAP adapter is +# bound to a DHCP client. +;server-bridge + +# Push routes to the client to allow it +# to reach other private subnets behind +# the server. Remember that these +# private subnets will also need +# to know to route the OpenVPN client +# address pool (10.8.0.0/255.255.255.0) +# back to the OpenVPN server. +;push "route 192.168.10.0 255.255.255.0" +;push "route 192.168.20.0 255.255.255.0" + +# To assign specific IP addresses to specific +# clients or if a connecting client has a private +# subnet behind it that should also have VPN access, +# use the subdirectory "ccd" for client-specific +# configuration files (see man page for more info). + +# EXAMPLE: Suppose the client +# having the certificate common name "Thelonious" +# also has a small subnet behind his connecting +# machine, such as 192.168.40.128/255.255.255.248. +# First, uncomment out these lines: +client-config-dir /etc/openvpn/ccd + +{% for minion in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values() %} +push "route {{ minion['cbr-string'] }}" +route {{ minion['cbr-string'] }} +{% endfor %} + +# Then create a file ccd/Thelonious with this line: +# iroute 192.168.40.128 255.255.255.248 +# This will allow Thelonious' private subnet to +# access the VPN. This example will only work +# if you are routing, not bridging, i.e. you are +# using "dev tun" and "server" directives. + +# EXAMPLE: Suppose you want to give +# Thelonious a fixed VPN IP address of 10.9.0.1. +# First uncomment out these lines: +;client-config-dir ccd +;route 10.9.0.0 255.255.255.252 +# Then add this line to ccd/Thelonious: +# ifconfig-push 10.9.0.1 10.9.0.2 + +# Suppose that you want to enable different +# firewall access policies for different groups +# of clients. There are two methods: +# (1) Run multiple OpenVPN daemons, one for each +# group, and firewall the TUN/TAP interface +# for each group/daemon appropriately. +# (2) (Advanced) Create a script to dynamically +# modify the firewall in response to access +# from different clients. See man +# page for more info on learn-address script. +;learn-address ./script + +# If enabled, this directive will configure +# all clients to redirect their default +# network gateway through the VPN, causing +# all IP traffic such as web browsing and +# and DNS lookups to go through the VPN +# (The OpenVPN server machine may need to NAT +# or bridge the TUN/TAP interface to the internet +# in order for this to work properly). +;push "redirect-gateway def1 bypass-dhcp" + +# Certain Windows-specific network settings +# can be pushed to clients, such as DNS +# or WINS server addresses. CAVEAT: +# http://openvpn.net/faq.html#dhcpcaveats +# The addresses below refer to the public +# DNS servers provided by opendns.com. +;push "dhcp-option DNS 208.67.222.222" +;push "dhcp-option DNS 208.67.220.220" + +# Uncomment this directive to allow different +# clients to be able to "see" each other. +# By default, clients will only see the server. +# To force clients to only see the server, you +# will also need to appropriately firewall the +# server's TUN/TAP interface. +client-to-client + +# Uncomment this directive if multiple clients +# might connect with the same certificate/key +# files or common names. This is recommended +# only for testing purposes. For production use, +# each client should have its own certificate/key +# pair. +# +# IF YOU HAVE NOT GENERATED INDIVIDUAL +# CERTIFICATE/KEY PAIRS FOR EACH CLIENT, +# EACH HAVING ITS OWN UNIQUE "COMMON NAME", +# UNCOMMENT THIS LINE OUT. +;duplicate-cn + +# The keepalive directive causes ping-like +# messages to be sent back and forth over +# the link so that each side knows when +# the other side has gone down. +# Ping every 10 seconds, assume that remote +# peer is down if no ping received during +# a 120 second time period. +keepalive 10 120 + +# For extra security beyond that provided +# by SSL/TLS, create an "HMAC firewall" +# to help block DoS attacks and UDP port flooding. +# +# Generate with: +# openvpn --genkey --secret ta.key +# +# The server and each client must have +# a copy of this key. +# The second parameter should be '0' +# on the server and '1' on the clients. +;tls-auth ta.key 0 # This file is secret + +# Select a cryptographic cipher. +# This config item must be copied to +# the client config file as well. +;cipher BF-CBC # Blowfish (default) +;cipher AES-128-CBC # AES +;cipher DES-EDE3-CBC # Triple-DES + +# Enable compression on the VPN link. +# If you enable it here, you must also +# enable it in the client config file. +comp-lzo + +# The maximum number of concurrently connected +# clients we want to allow. +;max-clients 100 + +# It's a good idea to reduce the OpenVPN +# daemon's privileges after initialization. +# +# You can uncomment this out on +# non-Windows systems. +;user nobody +;group nogroup + +# The persist options will try to avoid +# accessing certain resources on restart +# that may no longer be accessible because +# of the privilege downgrade. +persist-key +persist-tun + +# Output a short status file showing +# current connections, truncated +# and rewritten every minute. +status openvpn-status.log + +# By default, log messages will go to the syslog (or +# on Windows, if running as a service, they will go to +# the "\Program Files\OpenVPN\log" directory). +# Use log or log-append to override this default. +# "log" will truncate the log file on OpenVPN startup, +# while "log-append" will append to it. Use one +# or the other (but not both). +;log openvpn.log +;log-append openvpn.log + +# Set the appropriate level of log +# file verbosity. +# +# 0 is silent, except for fatal errors +# 4 is reasonable for general usage +# 5 and 6 can help to debug connection problems +# 9 is extremely verbose +verb 3 + +# Silence repeating messages. At most 20 +# sequential messages of the same message +# category will be output to the log. +;mute 20 diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 1a7e038713e..eff1a260574 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -10,6 +10,9 @@ base: - kube-proxy - cadvisor - nsinit +{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} + - openvpn-client +{% endif %} 'roles:kubernetes-master': - match: grain @@ -19,6 +22,9 @@ base: - controller-manager - scheduler - nginx +{% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} + - openvpn +{% endif %} 'roles:kubernetes-pool-vsphere': - match: grain