From 4408b82a908473fa885c9ad5e2a421637c9230bc Mon Sep 17 00:00:00 2001 From: Charles Butler Date: Fri, 10 Feb 2017 07:53:07 -0600 Subject: [PATCH] Fixes for #41019 This branch sync's the crypto keys, and flat-files used for auth with all the masters when scaling the kubernetes-master units. This should fix the mis-matched crypto keys seen when rebooting units after first deploy. --- .../juju/layers/kubernetes-master/layer.yaml | 1 + .../reactive/kubernetes_master.py | 88 +++++++++++++++---- 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml index 92d9544c11f..7625119a011 100644 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ b/cluster/juju/layers/kubernetes-master/layer.yaml @@ -2,6 +2,7 @@ repo: https://github.com/kubernetes/kubernetes.git includes: - 'layer:basic' - 'layer:tls-client' + - 'layer:leadership' - 'layer:debug' - 'interface:etcd' - 'interface:http' diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 50b7203ebf6..86c69e2ec59 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -21,6 +21,8 @@ import socket import string import json +import charms.leadership + from shlex import split from subprocess import call from subprocess import check_call @@ -140,34 +142,92 @@ def configure_cni(cni): cni.set_config(is_master=True, kubeconfig_path='') +@when('leadership.is_leader') @when('kubernetes-master.components.installed') @when_not('authentication.setup') -def setup_authentication(): +def setup_leader_authentication(): '''Setup basic authentication and token access for the cluster.''' api_opts = FlagManager('kube-apiserver') controller_opts = FlagManager('kube-controller-manager') - api_opts.add('--basic-auth-file', '/srv/kubernetes/basic_auth.csv') - api_opts.add('--token-auth-file', '/srv/kubernetes/known_tokens.csv') + service_key = '/etc/kubernetes/serviceaccount.key' + basic_auth = '/srv/kubernetes/basic_auth.csv' + known_tokens = '/srv/kubernetes/known_tokens.csv' + + api_opts.add('--basic-auth-file', basic_auth) + api_opts.add('--token-auth-file', known_tokens) api_opts.add('--service-cluster-ip-range', service_cidr()) hookenv.status_set('maintenance', 'Rendering authentication templates.') - htaccess = '/srv/kubernetes/basic_auth.csv' - if not os.path.isfile(htaccess): + if not os.path.isfile(basic_auth): setup_basic_auth('admin', 'admin', 'admin') - known_tokens = '/srv/kubernetes/known_tokens.csv' if not os.path.isfile(known_tokens): setup_tokens(None, 'admin', 'admin') setup_tokens(None, 'kubelet', 'kubelet') setup_tokens(None, 'kube_proxy', 'kube_proxy') # Generate the default service account token key os.makedirs('/etc/kubernetes', exist_ok=True) - cmd = ['openssl', 'genrsa', '-out', '/etc/kubernetes/serviceaccount.key', + + cmd = ['openssl', 'genrsa', '-out', service_key, '2048'] check_call(cmd) - api_opts.add('--service-account-key-file', - '/etc/kubernetes/serviceaccount.key') - controller_opts.add('--service-account-private-key-file', - '/etc/kubernetes/serviceaccount.key') + api_opts.add('--service-account-key-file', service_key) + controller_opts.add('--service-account-private-key-file', service_key) + + # read service account key for syndication + leader_data = {} + for f in [known_tokens, basic_auth, service_key]: + with open(f, 'r') as fp: + leader_data[f] = fp.read() + + # this is slightly opaque, but we are sending file contents under its file + # path as a key. + # eg: + # {'/etc/kubernetes/serviceaccount.key': 'RSA:2471731...'} + charms.leadership.leader_set(leader_data) + + set_state('authentication.setup') + + +@when_not('leadership.is_leader') +@when('kubernetes-master.components.installed') +@when_not('authentication.setup') +def setup_non_leader_authentication(): + api_opts = FlagManager('kube-apiserver') + controller_opts = FlagManager('kube-controller-manager') + + service_key = '/etc/kubernetes/serviceaccount.key' + basic_auth = '/srv/kubernetes/basic_auth.csv' + known_tokens = '/srv/kubernetes/known_tokens.csv' + + # This races with other codepaths, and seems to require being created first + # This block may be extracted later, but for now seems to work as intended + os.makedirs('/etc/kubernetes', exist_ok=True) + os.makedirs('/srv/kubernetes', exist_ok=True) + + hookenv.status_set('maintenance', 'Rendering authentication templates.') + + # Set an array for looping logic + keys = [service_key, basic_auth, known_tokens] + for k in keys: + # If the path does not exist, assume we need it + if not os.path.exists(k): + # Fetch data from leadership broadcast + contents = charms.leadership.leader_get(k) + # Default to logging the warning and wait for leader data to be set + if contents is None: + msg = "Waiting on leaders crypto keys." + hookenv.status_set('waiting', msg) + hookenv.log('Missing content for file {}'.format(k)) + return + # Write out the file and move on to the next item + with open(k, 'w+') as fp: + fp.write(contents) + + api_opts.add('--basic-auth-file', basic_auth) + api_opts.add('--token-auth-file', known_tokens) + api_opts.add('--service-cluster-ip-range', service_cidr()) + api_opts.add('--service-account-key-file', service_key) + controller_opts.add('--service-account-private-key-file', service_key) set_state('authentication.setup') @@ -185,9 +245,7 @@ def idle_status(): if not all_kube_system_pods_running(): hookenv.status_set('waiting', 'Waiting for kube-system pods to start') elif hookenv.config('service-cidr') != service_cidr(): - msg = "Cannot change {0} to {1}".format(service_cidr(), - hookenv.config('service-cidr')) - hookenv.log(msg, level=hookenv.WARN) + msg = 'WARN: cannot change service-cidr, still using ' + service_cidr() hookenv.status_set('active', msg) else: hookenv.status_set('active', 'Kubernetes master running.') @@ -305,7 +363,7 @@ def start_kube_dns(): context = { 'arch': arch(), - # The dictionary named pillar is a construct of the k8s template files. + # The dictionary named 'pillar' is a construct of the k8s template file 'pillar': { 'dns_server': get_dns_ip(), 'dns_replicas': 1,