diff --git a/cluster/juju/charms/trusty/kubernetes-master/config.yaml b/cluster/juju/charms/trusty/kubernetes-master/config.yaml index 3041f7f07b2..d34463ce368 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/config.yaml +++ b/cluster/juju/charms/trusty/kubernetes-master/config.yaml @@ -1,9 +1,33 @@ options: version: type: string - default: "v0.15.0" + default: "v1.0.0" description: | The kubernetes release to use in this charm. The binary files are compiled from the source identified by this tag in github. Using the value of "source" will use the master kubernetes branch when compiling the binaries. + username: + type: string + default: "admin" + description: | + The initial user for the kubernetes basic authentication file. + password: + type: string + default: "" + description: | + The password for the kubernetes basic authentication. If this value is + empty, a password will be generated at random for the username. + apiserver-cert: + type: string + default: "" + description: | + The ssl certificate to use for tls communication to the Kubernetes api + server. If this value is empty a self signed certificate and key will + be generated. + apiserver-key: + type: string + default: "" + description: | + The private key to use for tls communication to the Kubernetes api + server. If this value is empty a key and certificate will be generated. diff --git a/cluster/juju/charms/trusty/kubernetes-master/files/apiserver.upstart.tmpl b/cluster/juju/charms/trusty/kubernetes-master/files/apiserver.upstart.tmpl index 28d9799978c..40d9849eb0d 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/files/apiserver.upstart.tmpl +++ b/cluster/juju/charms/trusty/kubernetes-master/files/apiserver.upstart.tmpl @@ -8,13 +8,12 @@ limit nofile 20000 20000 kill timeout 30 # wait 30s between SIGTERM and SIGKILL. exec /usr/local/bin/apiserver \ - --address=%(api_bind_address)s \ + --basic-auth-file=/srv/kubernetes/basic-auth.csv \ + --bind-address=%(api_private_address)s \ --etcd-servers=%(etcd_servers)s \ + --insecure-bind-address=%(api_private_address)s \ --logtostderr=true \ - --service-cluster-ip-range=10.244.240.0/20 - - - - - - + --secure-port=6443 \ + --service-cluster-ip-range=10.244.240.0/20 \ + --tls-cert-file=/srv/kubernetes/apiserver.crt \ + --tls-private-key-file=/srv/kubernetes/apiserver.key diff --git a/cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl b/cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl index 0cf183b2b49..8279e391133 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl +++ b/cluster/juju/charms/trusty/kubernetes-master/files/controller-manager.upstart.tmpl @@ -10,11 +10,4 @@ kill timeout 30 # wait 30s between SIGTERM and SIGKILL. exec /usr/local/bin/controller-manager \ --address=%(bind_address)s \ --logtostderr=true \ - --master=%(api_server_address)s - - - - - - - + --master=%(api_http_uri)s diff --git a/cluster/juju/charms/trusty/kubernetes-master/files/distribution.conf.tmpl b/cluster/juju/charms/trusty/kubernetes-master/files/distribution.conf.tmpl index 5a279cab874..6ae91f14468 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/files/distribution.conf.tmpl +++ b/cluster/juju/charms/trusty/kubernetes-master/files/distribution.conf.tmpl @@ -1,5 +1,9 @@ +# This file configures ngnix to serve Kubernetes binaries using http. +# The charms find the location path from the api relation to the charm. server { - listen %(api_bind_address)s:80; + listen 80 default_server; + root %(alias)s; + location %(web_uri)s { alias %(alias)s; } diff --git a/cluster/juju/charms/trusty/kubernetes-master/files/nginx.conf.tmpl b/cluster/juju/charms/trusty/kubernetes-master/files/nginx.conf.tmpl index 1101c0c62da..9c9f31d2891 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/files/nginx.conf.tmpl +++ b/cluster/juju/charms/trusty/kubernetes-master/files/nginx.conf.tmpl @@ -1,39 +1,27 @@ -# HTTP/HTTPS server -# +# Proxy HTTPS from the public address to the kube-apiserver running at 6443. server { - listen 80; - server_name localhost; + listen 443; + server_name localhost; - root html; - index index.html index.htm; + root html; + index index.html index.htm; -# ssl on; -# ssl_certificate /usr/share/nginx/server.cert; -# ssl_certificate_key /usr/share/nginx/server.key; + ssl on; + ssl_certificate /srv/kubernetes/apiserver.crt; + ssl_certificate_key /srv/kubernetes/apiserver.key; + ssl_session_timeout 5m; -# ssl_session_timeout 5m; -# ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2; -# ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; -# ssl_prefer_server_ciphers on; + # don't use SSLv3 because of POODLE + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; + ssl_prefer_server_ciphers on; - location / { -# auth_basic "Restricted"; -# auth_basic_user_file /usr/share/nginx/htpasswd; - - # Proxy settings - # disable buffering so that watch works - proxy_buffering off; - proxy_pass %(api_server_address)s; - proxy_connect_timeout 159s; - proxy_send_timeout 600s; - proxy_read_timeout 600s; - - # Disable retry - proxy_next_upstream off; - - # Support web sockets - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } + location / { + proxy_buffering off; + proxy_pass %(api_https_uri)s; + proxy_connect_timeout 159s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + proxy_redirect off; + } } diff --git a/cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl b/cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl index f1bae362d5f..4500dccac4c 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl +++ b/cluster/juju/charms/trusty/kubernetes-master/files/scheduler.upstart.tmpl @@ -10,11 +10,4 @@ kill timeout 30 # wait 30s between SIGTERM and SIGKILL. exec /usr/local/bin/scheduler \ --address=%(bind_address)s \ --logtostderr=true \ - --master=%(api_server_address)s - - - - - - - + --master=%(api_http_uri)s diff --git a/cluster/juju/charms/trusty/kubernetes-master/hooks/hooks.py b/cluster/juju/charms/trusty/kubernetes-master/hooks/hooks.py index 838420dccc9..97ac46d6800 100755 --- a/cluster/juju/charms/trusty/kubernetes-master/hooks/hooks.py +++ b/cluster/juju/charms/trusty/kubernetes-master/hooks/hooks.py @@ -23,8 +23,9 @@ import socket import subprocess import sys from charmhelpers.core import hookenv, host +from charmhelpers.contrib import ssl from kubernetes_installer import KubernetesInstaller -from path import path +from path import Path hooks = hookenv.Hooks() @@ -55,10 +56,14 @@ def config_changed(): create kubernetes binary files. """ hookenv.log('Starting config-changed') - charm_dir = path(hookenv.charm_dir()) + charm_dir = Path(hookenv.charm_dir()) config = hookenv.config() # Get the version of kubernetes to install. version = config['version'] + username = config['username'] + password = config['password'] + certificate = config['apiserver-cert'] + key = config['apiserver-key'] if version == 'master': # The 'master' branch of kuberentes is used when master is configured. @@ -70,32 +75,59 @@ def config_changed(): # Create a branch to a tag to get the release version. branch = 'tags/{0}'.format(version) - # Get the package architecture, rather than arch from the kernel (uname -m). + cert_file = '/srv/kubernetes/apiserver.crt' + key_file = '/srv/kubernetes/apiserver.key' + # When the cert or key changes we need to restart the apiserver. + if config.changed('apiserver-cert') or config.changed('apiserver-key'): + hookenv.log('Certificate or key has changed.') + if not certificate or not key: + generate_cert(key=key_file, cert=cert_file) + else: + hookenv.log('Writing new certificate and key to server.') + with open(key_file, 'w') as file: + file.write(key) + with open(cert_file, 'w') as file: + file.write(certificate) + # Restart apiserver as the certificate or key has changed. + if host.service_running('apiserver'): + host.service_restart('apiserver') + # Reload nginx because it proxies https to apiserver. + if host.service_running('nginx'): + host.service_reload('nginx') + + if config.changed('username') or config.changed('password'): + hookenv.log('Username or password changed, creating authentication.') + basic_auth(config['username'], config['username'], config['password']) + if host.service_running('apiserver'): + host.service_restart('apiserver') + + # Get package architecture, rather than arch from the kernel (uname -m). arch = subprocess.check_output(['dpkg', '--print-architecture']).strip() if not branch: output_path = charm_dir / 'files/output' - installer = KubernetesInstaller(arch, version, output_path) + kube_installer = KubernetesInstaller(arch, version, output_path) else: # Build the kuberentes binaries from source on the units. - kubernetes_dir = path('/opt/kubernetes') + kubernetes_dir = Path('/opt/kubernetes') # Construct the path to the binaries using the arch. output_path = kubernetes_dir / '_output/local/bin/linux' / arch - installer = KubernetesInstaller(arch, version, output_path) + kube_installer = KubernetesInstaller(arch, version, output_path) if not kubernetes_dir.exists(): - print('The source directory {0} does not exist'.format(kubernetes_dir)) - print('Was the kubernetes code cloned during install?') + message = 'The kubernetes source directory {0} does not exist. ' \ + 'Was the kubernetes repository cloned during the install?' + print(message.format(kubernetes_dir)) exit(1) # Change to the kubernetes directory (git repository). with kubernetes_dir: - # Create a command to get the current branch. git_branch = 'git branch | grep "\*" | cut -d" " -f2' - current_branch = subprocess.check_output(git_branch, shell=True).strip() + current_branch = subprocess.check_output(git_branch, shell=True) + current_branch = current_branch.strip() print('Current branch: ', current_branch) # Create the path to a file to indicate if the build was broken. broken_build = charm_dir / '.broken_build' @@ -104,12 +136,12 @@ def config_changed(): print('Last build failed: ', last_build_failed) # Rebuild if current version is different or last build failed. if current_branch != version or last_build_failed: - installer.build(branch) + kube_installer.build(branch) if not output_path.isdir(): broken_build.touch() # Create the symoblic links to the right directories. - installer.install() + kube_installer.install() relation_changed() @@ -123,10 +155,10 @@ def relation_changed(): # Check required keys for k in ('etcd_servers',): if not template_data.get(k): - print "Missing data for", k, template_data + print 'Missing data for', k, template_data return - print "Running with\n", template_data + print 'Running with\n', template_data # Render and restart as needed for n in ('apiserver', 'controller-manager', 'scheduler'): @@ -157,7 +189,7 @@ def network_relation_changed(): def notify_minions(): - print("Notify minions.") + print('Notify minions.') config = hookenv.config() for r in hookenv.relation_ids('minions-api'): hookenv.relation_set( @@ -165,7 +197,49 @@ def notify_minions(): hostname=hookenv.unit_private_ip(), port=8080, version=config['version']) - print("Notified minions of version " + config['version']) + print('Notified minions of version ' + config['version']) + + +def basic_auth(name, id, pwd=None, file='/srv/kubernetes/basic-auth.csv'): + """ + Create a basic authentication file for kubernetes. The file is a csv file + with 3 columns: password, user name, user id. From the Kubernetes docs: + The basic auth credentials last indefinitely, and the password cannot be + changed without restarting apiserver. + """ + if not pwd: + import random + import string + alphanumeric = string.ascii_letters + string.digits + pwd = ''.join(random.choice(alphanumeric) for _ in range(16)) + lines = [] + auth_file = Path(file) + if auth_file.isfile(): + lines = auth_file.lines() + for line in lines: + target = ',{0},{1}'.format(name, id) + if target in line: + lines.remove(line) + auth_line = '{0},{1},{2}'.format(pwd, name, id) + lines.append(auth_line) + auth_file.write_lines(lines) + + +def generate_cert(common_name=None, + key='/srv/kubernetes/apiserver.key', + cert='/srv/kubernetes/apiserver.crt'): + """ + Create the certificate and key for the Kubernetes tls enablement. + """ + hookenv.log('Generating new self signed certificate and key', 'INFO') + if not common_name: + common_name = hookenv.unit_get('public-address') + if os.path.isfile(key) or os.path.isfile(cert): + hookenv.log('Overwriting the existing certificate or key', 'WARNING') + hookenv.log('Generating certificate for {0}'.format(common_name), 'INFO') + # Generate the self signed certificate with the public address as CN. + # https://pythonhosted.org/charmhelpers/api/charmhelpers.contrib.ssl.html + ssl.generate_selfsigned(key, cert, cn=common_name) def get_template_data(): @@ -173,18 +247,21 @@ def get_template_data(): config = hookenv.config() version = config['version'] template_data = {} - template_data['etcd_servers'] = ",".join([ - "http://%s:%s" % (s[0], s[1]) for s in sorted( + template_data['etcd_servers'] = ','.join([ + 'http://%s:%s' % (s[0], s[1]) for s in sorted( get_rel_hosts('etcd', rels, ('hostname', 'port')))]) - template_data['minions'] = ",".join(get_rel_hosts('minions-api', rels)) + template_data['minions'] = ','.join(get_rel_hosts('minions-api', rels)) + private_ip = hookenv.unit_private_ip() + public_ip = hookenv.unit_public_ip() + template_data['api_public_address'] = _bind_addr(public_ip) + template_data['api_private_address'] = _bind_addr(private_ip) + template_data['bind_address'] = '127.0.0.1' + template_data['api_http_uri'] = 'http://%s:%s' % (private_ip, 8080) + template_data['api_https_uri'] = 'https://%s:%s' % (private_ip, 6443) - template_data['api_bind_address'] = _bind_addr(hookenv.unit_private_ip()) - template_data['bind_address'] = "127.0.0.1" - template_data['api_server_address'] = "http://%s:%s" % ( - hookenv.unit_private_ip(), 8080) arch = subprocess.check_output(['dpkg', '--print-architecture']).strip() - template_data['web_uri'] = "/kubernetes/%s/local/bin/linux/%s/" % (version, + template_data['web_uri'] = '/kubernetes/%s/local/bin/linux/%s/' % (version, arch) if version == 'local': template_data['alias'] = hookenv.charm_dir() + '/files/output/' @@ -201,7 +278,7 @@ def _bind_addr(addr): try: return socket.gethostbyname(addr) except socket.error: - raise ValueError("Could not resolve private address") + raise ValueError('Could not resolve address %s' % addr) def _encode(d): @@ -223,7 +300,7 @@ def get_rel_hosts(rel_name, rels, keys=('private-address',)): return hosts -def render_file(name, data, src_suffix="upstart.tmpl", tgt_path=None): +def render_file(name, data, src_suffix='upstart.tmpl', tgt_path=None): tmpl_path = os.path.join( os.environ.get('CHARM_DIR'), 'files', '%s.%s' % (name, src_suffix)) @@ -244,5 +321,6 @@ def render_file(name, data, src_suffix="upstart.tmpl", tgt_path=None): fh.write(rendered) return True + if __name__ == '__main__': hooks.execute(sys.argv) diff --git a/cluster/juju/charms/trusty/kubernetes-master/hooks/install.py b/cluster/juju/charms/trusty/kubernetes-master/hooks/install.py index 96af8b471e2..364d05f94f6 100755 --- a/cluster/juju/charms/trusty/kubernetes-master/hooks/install.py +++ b/cluster/juju/charms/trusty/kubernetes-master/hooks/install.py @@ -18,10 +18,10 @@ import setup setup.pre_install() import subprocess -from charmhelpers.core import hookenv from charmhelpers import fetch +from charmhelpers.core import hookenv from charmhelpers.fetch import archiveurl -from path import path +from path import Path def install(): @@ -30,21 +30,28 @@ def install(): download_go() hookenv.log('Adding kubernetes and go to the path') - + address = hookenv.unit_private_ip() strings = [ 'export GOROOT=/usr/local/go\n', 'export PATH=$PATH:$GOROOT/bin\n', - 'export KUBE_MASTER_IP=0.0.0.0\n', - 'export KUBERNETES_MASTER=http://$KUBE_MASTER_IP\n', + 'export KUBERNETES_MASTER=http://{0}:8080\n'.format(address), ] update_rc_files(strings) hookenv.log('Downloading kubernetes code') clone_repository() + # Create the directory to store the keys and auth files. + srv = Path('/srv/kubernetes') + if not srv.isdir(): + srv.makedirs_p() + hookenv.open_port(8080) + hookenv.open_port(6443) + hookenv.open_port(443) hookenv.log('Install complete') + def download_go(): """ Kubernetes charm strives to support upstream. Part of this is installing a @@ -59,12 +66,12 @@ def download_go(): def clone_repository(): """ - Clone the upstream repository into /opt/kubernetes for deployment compilation - of kubernetes. Subsequently used during upgrades. + Clone the upstream repository into /opt/kubernetes for deployment + compilation of kubernetes. Subsequently used during upgrades. """ repository = 'https://github.com/kubernetes/kubernetes.git' - kubernetes_directory = path('/opt/kubernetes') + kubernetes_directory = Path('/opt/kubernetes') # Since we can not clone twice, check for the directory and remove it. if kubernetes_directory.isdir(): kubernetes_directory.rmtree_p() @@ -75,7 +82,6 @@ def clone_repository(): print(output) - def install_packages(): """ Install required packages to build the k8s source, and syndicate between @@ -83,17 +89,21 @@ def install_packages(): """ hookenv.log('Installing Debian packages') # Create the list of packages to install. - apt_packages = ['build-essential', 'git', 'make', 'nginx', 'python-pip'] + apt_packages = ['apache2-utils', + 'build-essential', + 'git', + 'make', + 'nginx', + 'python-pip', ] fetch.apt_install(fetch.filter_installed_packages(apt_packages)) - def update_rc_files(strings): """ Preseed the bash environment for ubuntu and root with K8's env vars to make interfacing with the api easier. (see: kubectrl docs) """ - rc_files = [path('/home/ubuntu/.bashrc'), path('/root/.bashrc')] + rc_files = [Path('/home/ubuntu/.bashrc'), Path('/root/.bashrc')] for rc_file in rc_files: lines = rc_file.lines() for string in strings: @@ -102,6 +112,5 @@ def update_rc_files(strings): rc_file.write_lines(lines) - if __name__ == "__main__": install() diff --git a/cluster/juju/charms/trusty/kubernetes-master/hooks/kubernetes_installer.py b/cluster/juju/charms/trusty/kubernetes-master/hooks/kubernetes_installer.py index 777699140a4..6c7cbc9b42b 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/hooks/kubernetes_installer.py +++ b/cluster/juju/charms/trusty/kubernetes-master/hooks/kubernetes_installer.py @@ -17,7 +17,7 @@ import os import shlex import subprocess -from path import path +from path import Path def run(command, shell=False): @@ -46,7 +46,7 @@ class KubernetesInstaller(): 'kubelet': 'kubelet'} self.arch = arch self.version = version - self.output_dir = path(output_dir) + self.output_dir = Path(output_dir) def build(self, branch): """ Build kubernetes from a github repository using the Makefile. """ @@ -88,7 +88,7 @@ class KubernetesInstaller(): print(make_what) rc = subprocess.call(shlex.split(make_what), env=go_env) - def install(self, install_dir=path('/usr/local/bin')): + def install(self, install_dir=Path('/usr/local/bin')): """ Install kubernetes binary files from the output directory. """ if not install_dir.isdir(): diff --git a/cluster/juju/charms/trusty/kubernetes-master/hooks/setup.py b/cluster/juju/charms/trusty/kubernetes-master/hooks/setup.py index 84b757ce3ad..6afdd49bf5f 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/hooks/setup.py +++ b/cluster/juju/charms/trusty/kubernetes-master/hooks/setup.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def pre_install(): """ Do any setup required before the install hook. diff --git a/cluster/juju/charms/trusty/kubernetes-master/unit_tests/kubernetes_installer_test.py b/cluster/juju/charms/trusty/kubernetes-master/unit_tests/kubernetes_installer_test.py index 020ec401051..f3c8ed8c769 100644 --- a/cluster/juju/charms/trusty/kubernetes-master/unit_tests/kubernetes_installer_test.py +++ b/cluster/juju/charms/trusty/kubernetes-master/unit_tests/kubernetes_installer_test.py @@ -15,7 +15,6 @@ # limitations under the License. from mock import patch -from path import path from path import Path import pytest import subprocess @@ -38,7 +37,7 @@ def test_run(): assert output assert 'kubernetes_installer.py' in output - invalid_directory = path('/not/a/real/directory') + invalid_directory = Path('/not/a/real/directory') assert not invalid_directory.exists() invalid_command = 'ls {0}'.format(invalid_directory) with pytest.raises(subprocess.CalledProcessError) as error: @@ -67,13 +66,13 @@ class TestKubernetesInstaller(): assert 'kubelet' in ki.aliases assert ki.arch == 'i386' assert ki.version == '3.0.1' - assert ki.output_dir == path('/tmp/does_not_exist') + assert ki.output_dir == Path('/tmp/does_not_exist') @patch('kubernetes_installer.run') @patch('kubernetes_installer.subprocess.call') def test_build(self, cmock, rmock): """ Test the build method with master and non-master branches. """ - directory = path('/tmp/kubernetes_installer_test/build') + directory = Path('/tmp/kubernetes_installer_test/build') ki = self.makeone('amd64', 'v99.00.11', directory) assert not directory.exists(), 'The %s directory exists!' % directory # Call the build method with "master" branch. @@ -94,7 +93,7 @@ class TestKubernetesInstaller(): def test_install(self): """ Test the install method that it creates the correct links. """ - directory = path('/tmp/kubernetes_installer_test/install') + directory = Path('/tmp/kubernetes_installer_test/install') ki = self.makeone('ppc64le', '1.2.3', directory) assert not directory.exists(), 'The %s directory exits!' % directory directory.makedirs_p() diff --git a/cluster/juju/charms/trusty/kubernetes/hooks/hooks.py b/cluster/juju/charms/trusty/kubernetes/hooks/hooks.py index c47d73baa3a..1f749014094 100755 --- a/cluster/juju/charms/trusty/kubernetes/hooks/hooks.py +++ b/cluster/juju/charms/trusty/kubernetes/hooks/hooks.py @@ -17,10 +17,7 @@ """ The main hook file that is called by Juju. """ -import json -import httplib import os -import time import socket import subprocess import sys @@ -28,7 +25,7 @@ import urlparse from charmhelpers.core import hookenv, host from kubernetes_installer import KubernetesInstaller -from path import path +from path import Path from lib.registrator import Registrator @@ -43,10 +40,10 @@ def api_relation_changed(): from the kubernetes-master charm and installs it locally on this machine. """ hookenv.log('Starting api-relation-changed') - charm_dir = path(hookenv.charm_dir()) + charm_dir = Path(hookenv.charm_dir()) # Get the package architecture, rather than the from the kernel (uname -m). arch = subprocess.check_output(['dpkg', '--print-architecture']).strip() - kubernetes_bin_dir = path('/opt/kubernetes/bin') + kubernetes_bin_dir = Path('/opt/kubernetes/bin') # Get the version of kubernetes to install. version = subprocess.check_output(['relation-get', 'version']).strip() print('Relation version: ', version) @@ -228,6 +225,7 @@ def register_machine(apiserver, retry=False): # for now this is OK pass + def setup_kubernetes_group(): output = subprocess.check_output(['groups', 'kubernetes']) diff --git a/cluster/juju/charms/trusty/kubernetes/hooks/install b/cluster/juju/charms/trusty/kubernetes/hooks/install index 32c3251eb4a..ee5565702ae 100755 --- a/cluster/juju/charms/trusty/kubernetes/hooks/install +++ b/cluster/juju/charms/trusty/kubernetes/hooks/install @@ -13,20 +13,21 @@ apt-get install -q -y \ python-pip \ wget -pip install path.py +pip install -r $CHARM_DIR/python_requirements.txt # Create the necessary kubernetes group. -groupadd kubernetes -useradd -d /var/lib/kubernetes \ - -g kubernetes \ - -s /sbin/nologin \ - --system \ - kubernetes +groupadd --force kubernetes + +if grep -q "^kubernetes:" /etc/passwd; then + echo "The kubernetes user already exists!" +else + # Create the user when kubernetes does not exist. + useradd -d /var/lib/kubernetes \ + -g kubernetes \ + -s /sbin/nologin \ + --system \ + kubernetes +fi install -d -m 0744 -o kubernetes -g kubernetes /var/lib/kubernetes install -d -m 0744 -o kubernetes -g kubernetes /etc/kubernetes/manifests - -# wait for the world, depends on where we installed it from distro -#sudo service docker.io stop -# or upstream archive -#sudo service docker stop diff --git a/cluster/juju/charms/trusty/kubernetes/hooks/kubernetes_installer.py b/cluster/juju/charms/trusty/kubernetes/hooks/kubernetes_installer.py index b7db6458446..4da4313dbd2 100644 --- a/cluster/juju/charms/trusty/kubernetes/hooks/kubernetes_installer.py +++ b/cluster/juju/charms/trusty/kubernetes/hooks/kubernetes_installer.py @@ -15,7 +15,7 @@ # limitations under the License. import subprocess -from path import path +from path import Path class KubernetesInstaller(): @@ -49,7 +49,7 @@ class KubernetesInstaller(): print(output) destination.chmod(0o755) - def install(self, install_dir=path('/usr/local/bin')): + def install(self, install_dir=Path('/usr/local/bin')): """ Create links to the binary files to the install directory. """ if not install_dir.isdir(): diff --git a/cluster/juju/charms/trusty/kubernetes/python_requirements.txt b/cluster/juju/charms/trusty/kubernetes/python_requirements.txt new file mode 100644 index 00000000000..09375e79b58 --- /dev/null +++ b/cluster/juju/charms/trusty/kubernetes/python_requirements.txt @@ -0,0 +1 @@ +path.py diff --git a/cluster/juju/config-default.sh b/cluster/juju/config-default.sh new file mode 100644 index 00000000000..0bdc6180b64 --- /dev/null +++ b/cluster/juju/config-default.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/cluster/juju/config-test.sh b/cluster/juju/config-test.sh new file mode 100644 index 00000000000..7f7ad16b1f0 --- /dev/null +++ b/cluster/juju/config-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +NUM_MINIONS=${NUM_MINIONS:-2} diff --git a/cluster/juju/kube-system-ns.yaml b/cluster/juju/kube-system-ns.yaml new file mode 100644 index 00000000000..986f4b48221 --- /dev/null +++ b/cluster/juju/kube-system-ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system diff --git a/cluster/juju/util.sh b/cluster/juju/util.sh index 9407b44004f..2ed2dd70aba 100755 --- a/cluster/juju/util.sh +++ b/cluster/juju/util.sh @@ -18,29 +18,68 @@ set -o errexit set -o nounset set -o pipefail +#set -o xtrace UTIL_SCRIPT=$(readlink -m "${BASH_SOURCE}") JUJU_PATH=$(dirname ${UTIL_SCRIPT}) +KUBE_ROOT=$(readlink -m ${JUJU_PATH}/../../) +# Use the config file specified in $KUBE_CONFIG_FILE, or config-default.sh. +source "${JUJU_PATH}/${KUBE_CONFIG_FILE-config-default.sh}" source ${JUJU_PATH}/prereqs/ubuntu-juju.sh export JUJU_REPOSITORY=${JUJU_PATH}/charms #KUBE_BUNDLE_URL='https://raw.githubusercontent.com/whitmo/bundle-kubernetes/master/bundles.yaml' KUBE_BUNDLE_PATH=${JUJU_PATH}/bundles/local.yaml -function verify-prereqs() { - gather_installation_reqs -} - +# Build the binaries on the local system and copy the binaries to the Juju charm. function build-local() { + local targets=( + cmd/kube-proxy \ + cmd/kube-apiserver \ + cmd/kube-controller-manager \ + cmd/kubelet \ + plugin/cmd/kube-scheduler \ + cmd/kubectl \ + test/e2e/e2e.test \ + ) # Make a clean environment to avoid compiler errors. make clean # Build the binaries locally that are used in the charms. - make all WHAT="cmd/kube-apiserver cmd/kubectl cmd/kube-controller-manager plugin/cmd/kube-scheduler cmd/kubelet cmd/kube-proxy" - OUTPUT_DIR=_output/local/bin/linux/amd64 + make all WHAT="${targets[*]}" + local OUTPUT_DIR=_output/local/bin/linux/amd64 mkdir -p cluster/juju/charms/trusty/kubernetes-master/files/output - # Copy the binary output to the charm directory. + # Copy the binaries from the output directory to the charm directory. cp -v $OUTPUT_DIR/* cluster/juju/charms/trusty/kubernetes-master/files/output } +function detect-master() { + local kubestatus + # Capturing a newline, and my awk-fu was weak - pipe through tr -d + kubestatus=$(juju status --format=oneline kubernetes-master | grep kubernetes-master/0 | awk '{print $3}' | tr -d "\n") + export KUBE_MASTER_IP=${kubestatus} + export KUBE_SERVER=http://${KUBE_MASTER_IP}:8080 +} + +function detect-minions() { + # Run the Juju command that gets the minion private IP addresses. + local ipoutput + ipoutput=$(juju run --service kubernetes "unit-get private-address" --format=json) + # [ + # {"MachineId":"2","Stdout":"192.168.122.188\n","UnitId":"kubernetes/0"}, + # {"MachineId":"3","Stdout":"192.168.122.166\n","UnitId":"kubernetes/1"} + # ] + + # Strip out the IP addresses + export KUBE_MINION_IP_ADDRESSES=($(${JUJU_PATH}/return-node-ips.py "${ipoutput}")) + # echo "Kubernetes minions: " ${KUBE_MINION_IP_ADDRESSES[@]} 1>&2 + export NUM_MINIONS=${#KUBE_MINION_IP_ADDRESSES[@]} +} + +function get-password() { + export KUBE_USER=admin + # Get the password from the basic-auth.csv file on kubernetes-master. + export KUBE_PASSWORD=$(juju run --unit kubernetes-master/0 "cat /srv/kubernetes/basic-auth.csv" | grep ${KUBE_USER} | cut -d, -f1) +} + function kube-up() { build-local if [[ -d "~/.juju/current-env" ]]; then @@ -51,61 +90,45 @@ function kube-up() { # The juju-deployer command will deploy the bundle and can be run # multiple times to continue deploying the parts that fail. juju deployer -c ${KUBE_BUNDLE_PATH} + + source "${KUBE_ROOT}/cluster/common.sh" + get-password + # Sleep due to juju bug http://pad.lv/1432759 sleep-status detect-master detect-minions - export KUBE_MASTER_IP="${KUBE_MASTER_IP}:8080" + local prefix=$RANDOM + export KUBE_CERT="/tmp/${prefix}-kubecfg.crt" + export KUBE_KEY="/tmp/${prefix}-kubecfg.key" + export CA_CERT="/tmp/${prefix}-kubecfg.ca" export CONTEXT="juju" + + # Copy the cert and key to this machine. + ( + umask 077 + juju scp kubernetes-master/0:/srv/kubernetes/apiserver.crt ${KUBE_CERT} + juju run --unit kubernetes-master/0 'chmod 644 /srv/kubernetes/apiserver.key' + juju scp kubernetes-master/0:/srv/kubernetes/apiserver.key ${KUBE_KEY} + juju run --unit kubernetes-master/0 'chmod 600 /srv/kubernetes/apiserver.key' + cp ${KUBE_CERT} ${CA_CERT} + + create-kubeconfig + ) } function kube-down() { + local force="${1-}" # Remove the binary files from the charm directory. rm -rf cluster/juju/charms/trusty/kubernetes-master/files/output/ local jujuenv jujuenv=$(cat ~/.juju/current-environment) - juju destroy-environment $jujuenv + juju destroy-environment ${jujuenv} ${force} || true } -function detect-master() { - local kubestatus - # Capturing a newline, and my awk-fu was weak - pipe through tr -d - kubestatus=$(juju status --format=oneline kubernetes-master | grep kubernetes-master/0 | awk '{print $3}' | tr -d "\n") - export KUBE_MASTER_IP=${kubestatus} - export KUBE_MASTER=${KUBE_MASTER_IP} - export KUBERNETES_MASTER=http://${KUBE_MASTER}:8080 - echo "Kubernetes master: " ${KUBERNETES_MASTER} -} - -function detect-minions() { - # Run the Juju command that gets the minion private IP addresses. - local ipoutput - ipoutput=$(juju run --service kubernetes "unit-get private-address" --format=json) - echo $ipoutput - # Strip out the IP addresses - # - # Example Output: - #- MachineId: "10" - # Stdout: | - # 10.197.55.232 - # UnitId: kubernetes/0 - # - MachineId: "11" - # Stdout: | - # 10.202.146.124 - # UnitId: kubernetes/1 - export KUBE_MINION_IP_ADDRESSES=($(${JUJU_PATH}/return-node-ips.py "${ipoutput}")) - echo "Kubernetes minions: " ${KUBE_MINION_IP_ADDRESSES[@]} - export NUM_MINIONS=${#KUBE_MINION_IP_ADDRESSES[@]} - export MINION_NAMES=$KUBE_MINION_IP_ADDRESSES -} - -function setup-logging-firewall() { - echo "TODO: setup logging and firewall rules" -} - -function teardown-logging-firewall() { - echo "TODO: teardown logging and firewall rules" +function prepare-e2e() { + echo "prepare-e2e() The Juju provider does not need any preperations for e2e." 1>&2 } function sleep-status() { @@ -115,7 +138,7 @@ function sleep-status() { i=0 maxtime=900 jujustatus='' - echo "Waiting up to 15 minutes to allow the cluster to come online... wait for it..." + echo "Waiting up to 15 minutes to allow the cluster to come online... wait for it..." 1>&2 jujustatus=$(juju status kubernetes-master --format=oneline) if [[ $jujustatus == *"started"* ]]; @@ -132,6 +155,28 @@ function sleep-status() { # sleep because we cannot get the status back of where the minions are in the deploy phase # thanks to a generic "started" state and our service not actually coming online until the # minions have received the binary from the master distribution hub during relations - echo "Sleeping an additional minute to allow the cluster to settle" + echo "Sleeping an additional minute to allow the cluster to settle" 1>&2 sleep 60 } + +# Execute prior to running tests to build a release if required for environment. +function test-build-release { + echo "test-build-release() " 1>&2 +} + +# Execute prior to running tests to initialize required structure. This is +# called from hack/e2e.go only when running -up (it is run after kube-up). +function test-setup { + echo "test-setup() " 1>&2 +} + +# Execute after running tests to perform any required clean-up. This is called +# from hack/e2e.go +function test-teardown() { + kube-down "-y" +} + +# Verify the prerequisites are statisfied before running. +function verify-prereqs() { + gather_installation_reqs +}