From d697b10885e8defe6a7bb81eb3da0174008ce0d0 Mon Sep 17 00:00:00 2001 From: George Kraft Date: Fri, 10 Feb 2017 10:10:43 -0600 Subject: [PATCH 1/2] Fix UpdateAddonsTactic to use local repo, add KUBE_VERSION option --- .../tactics/update_addons.py | 89 ++++++++++++------- 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/tactics/update_addons.py b/cluster/juju/layers/kubernetes-master/tactics/update_addons.py index 09e36688aca..4f1814a1e39 100755 --- a/cluster/juju/layers/kubernetes-master/tactics/update_addons.py +++ b/cluster/juju/layers/kubernetes-master/tactics/update_addons.py @@ -45,31 +45,46 @@ def clean_addon_dir(addon_dir): os.makedirs(addon_dir) +def run_with_logging(command): + """ Run a command with controlled logging """ + log.debug("Running: %s" % command) + process = subprocess.Popen(command, stderr=subprocess.PIPE) + stderr = process.communicate()[1].rstrip() + process.wait() + if process.returncode != 0: + log.error(stderr) + raise Exception("%s: exit code %d" % (command, process.returncode)) + log.debug(stderr) + + @contextmanager def kubernetes_repo(): - """ Shallow clone kubernetes repo and clean up when we are done """ - repo = "https://github.com/kubernetes/kubernetes.git" - path = tempfile.mkdtemp(prefix="kubernetes") - try: - log.info("Cloning " + repo) - cmd = ["git", "clone", "--depth", "1", repo, path] - process = subprocess.Popen(cmd, stderr=subprocess.PIPE) - stderr = process.communicate()[1].rstrip() - process.wait() - if process.returncode != 0: - log.error(stderr) - raise Exception("clone failed: exit code %d" % process.returncode) - log.debug(stderr) - yield path - finally: - shutil.rmtree(path) + """ Yield a kubernetes repo to copy addons from. + + If KUBE_VERSION is set, this will clone the local repo and checkout the + corresponding branch. Otherwise, the local branch will be used. """ + repo = os.path.abspath("../../../..") + if "KUBE_VERSION" in os.environ: + branch = os.environ["KUBE_VERSION"] + log.info("Cloning %s with branch %s" % (repo, branch)) + path = tempfile.mkdtemp(prefix="kubernetes") + try: + cmd = ["git", "clone", repo, path, "-b", branch] + run_with_logging(cmd) + yield path + finally: + shutil.rmtree(path) + else: + log.info("Using local repo " + repo) + yield repo -def add_addon(source, dest): - """ Add an addon manifest from the given source. +def add_addon(repo, source, dest): + """ Add an addon manifest from the given repo and source. Any occurrences of 'amd64' are replaced with '{{ arch }}' so the charm can fill it in during deployment. """ + source = os.path.join(repo, "cluster/addons", source) if os.path.isdir(dest): dest = os.path.join(dest, os.path.basename(source)) log.debug("Copying: %s -> %s" % (source, dest)) @@ -86,20 +101,26 @@ def update_addons(dest): with kubernetes_repo() as repo: log.info("Copying addons to charm") clean_addon_dir(dest) - add_addon(repo + "/cluster/addons/dashboard/dashboard-controller.yaml", - dest) - add_addon(repo + "/cluster/addons/dashboard/dashboard-service.yaml", - dest) - add_addon(repo + "/cluster/addons/dns/kubedns-controller.yaml.in", - dest + "/kubedns-controller.yaml") - add_addon(repo + "/cluster/addons/dns/kubedns-svc.yaml.in", - dest + "/kubedns-svc.yaml") - influxdb = "/cluster/addons/cluster-monitoring/influxdb" - add_addon(repo + influxdb + "/grafana-service.yaml", dest) - add_addon(repo + influxdb + "/heapster-controller.yaml", dest) - add_addon(repo + influxdb + "/heapster-service.yaml", dest) - add_addon(repo + influxdb + "/influxdb-grafana-controller.yaml", dest) - add_addon(repo + influxdb + "/influxdb-service.yaml", dest) + add_addon(repo, "dashboard/dashboard-controller.yaml", dest) + add_addon(repo, "dashboard/dashboard-service.yaml", dest) + try: + add_addon(repo, "dns/kubedns-controller.yaml.in", + dest + "/kubedns-controller.yaml") + add_addon(repo, "dns/kubedns-svc.yaml.in", + dest + "/kubedns-svc.yaml") + except IOError as e: + # fall back to the older filenames + log.debug(e) + add_addon(repo, "dns/skydns-rc.yaml.in", + dest + "/kubedns-controller.yaml") + add_addon(repo, "dns/skydns-svc.yaml.in", + dest + "/kubedns-svc.yaml") + influxdb = "cluster-monitoring/influxdb" + add_addon(repo, influxdb + "/grafana-service.yaml", dest) + add_addon(repo, influxdb + "/heapster-controller.yaml", dest) + add_addon(repo, influxdb + "/heapster-service.yaml", dest) + add_addon(repo, influxdb + "/influxdb-grafana-controller.yaml", dest) + add_addon(repo, influxdb + "/influxdb-service.yaml", dest) # Entry points @@ -151,8 +172,8 @@ def parse_args(): def main(): """ Update addons into the layer's templates/addons folder """ parse_args() - dest = os.path.abspath(os.path.join(os.path.dirname(__file__), - "../templates/addons")) + os.chdir(os.path.join(os.path.dirname(__file__), "..")) + dest = "templates/addons" update_addons(dest) From 4408b82a908473fa885c9ad5e2a421637c9230bc Mon Sep 17 00:00:00 2001 From: Charles Butler Date: Fri, 10 Feb 2017 07:53:07 -0600 Subject: [PATCH 2/2] Fixes for #41019 This branch sync's the crypto keys, and flat-files used for auth with all the masters when scaling the kubernetes-master units. This should fix the mis-matched crypto keys seen when rebooting units after first deploy. --- .../juju/layers/kubernetes-master/layer.yaml | 1 + .../reactive/kubernetes_master.py | 88 +++++++++++++++---- 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml index 92d9544c11f..7625119a011 100644 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ b/cluster/juju/layers/kubernetes-master/layer.yaml @@ -2,6 +2,7 @@ repo: https://github.com/kubernetes/kubernetes.git includes: - 'layer:basic' - 'layer:tls-client' + - 'layer:leadership' - 'layer:debug' - 'interface:etcd' - 'interface:http' diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 50b7203ebf6..86c69e2ec59 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -21,6 +21,8 @@ import socket import string import json +import charms.leadership + from shlex import split from subprocess import call from subprocess import check_call @@ -140,34 +142,92 @@ def configure_cni(cni): cni.set_config(is_master=True, kubeconfig_path='') +@when('leadership.is_leader') @when('kubernetes-master.components.installed') @when_not('authentication.setup') -def setup_authentication(): +def setup_leader_authentication(): '''Setup basic authentication and token access for the cluster.''' api_opts = FlagManager('kube-apiserver') controller_opts = FlagManager('kube-controller-manager') - api_opts.add('--basic-auth-file', '/srv/kubernetes/basic_auth.csv') - api_opts.add('--token-auth-file', '/srv/kubernetes/known_tokens.csv') + service_key = '/etc/kubernetes/serviceaccount.key' + basic_auth = '/srv/kubernetes/basic_auth.csv' + known_tokens = '/srv/kubernetes/known_tokens.csv' + + api_opts.add('--basic-auth-file', basic_auth) + api_opts.add('--token-auth-file', known_tokens) api_opts.add('--service-cluster-ip-range', service_cidr()) hookenv.status_set('maintenance', 'Rendering authentication templates.') - htaccess = '/srv/kubernetes/basic_auth.csv' - if not os.path.isfile(htaccess): + if not os.path.isfile(basic_auth): setup_basic_auth('admin', 'admin', 'admin') - known_tokens = '/srv/kubernetes/known_tokens.csv' if not os.path.isfile(known_tokens): setup_tokens(None, 'admin', 'admin') setup_tokens(None, 'kubelet', 'kubelet') setup_tokens(None, 'kube_proxy', 'kube_proxy') # Generate the default service account token key os.makedirs('/etc/kubernetes', exist_ok=True) - cmd = ['openssl', 'genrsa', '-out', '/etc/kubernetes/serviceaccount.key', + + cmd = ['openssl', 'genrsa', '-out', service_key, '2048'] check_call(cmd) - api_opts.add('--service-account-key-file', - '/etc/kubernetes/serviceaccount.key') - controller_opts.add('--service-account-private-key-file', - '/etc/kubernetes/serviceaccount.key') + api_opts.add('--service-account-key-file', service_key) + controller_opts.add('--service-account-private-key-file', service_key) + + # read service account key for syndication + leader_data = {} + for f in [known_tokens, basic_auth, service_key]: + with open(f, 'r') as fp: + leader_data[f] = fp.read() + + # this is slightly opaque, but we are sending file contents under its file + # path as a key. + # eg: + # {'/etc/kubernetes/serviceaccount.key': 'RSA:2471731...'} + charms.leadership.leader_set(leader_data) + + set_state('authentication.setup') + + +@when_not('leadership.is_leader') +@when('kubernetes-master.components.installed') +@when_not('authentication.setup') +def setup_non_leader_authentication(): + api_opts = FlagManager('kube-apiserver') + controller_opts = FlagManager('kube-controller-manager') + + service_key = '/etc/kubernetes/serviceaccount.key' + basic_auth = '/srv/kubernetes/basic_auth.csv' + known_tokens = '/srv/kubernetes/known_tokens.csv' + + # This races with other codepaths, and seems to require being created first + # This block may be extracted later, but for now seems to work as intended + os.makedirs('/etc/kubernetes', exist_ok=True) + os.makedirs('/srv/kubernetes', exist_ok=True) + + hookenv.status_set('maintenance', 'Rendering authentication templates.') + + # Set an array for looping logic + keys = [service_key, basic_auth, known_tokens] + for k in keys: + # If the path does not exist, assume we need it + if not os.path.exists(k): + # Fetch data from leadership broadcast + contents = charms.leadership.leader_get(k) + # Default to logging the warning and wait for leader data to be set + if contents is None: + msg = "Waiting on leaders crypto keys." + hookenv.status_set('waiting', msg) + hookenv.log('Missing content for file {}'.format(k)) + return + # Write out the file and move on to the next item + with open(k, 'w+') as fp: + fp.write(contents) + + api_opts.add('--basic-auth-file', basic_auth) + api_opts.add('--token-auth-file', known_tokens) + api_opts.add('--service-cluster-ip-range', service_cidr()) + api_opts.add('--service-account-key-file', service_key) + controller_opts.add('--service-account-private-key-file', service_key) set_state('authentication.setup') @@ -185,9 +245,7 @@ def idle_status(): if not all_kube_system_pods_running(): hookenv.status_set('waiting', 'Waiting for kube-system pods to start') elif hookenv.config('service-cidr') != service_cidr(): - msg = "Cannot change {0} to {1}".format(service_cidr(), - hookenv.config('service-cidr')) - hookenv.log(msg, level=hookenv.WARN) + msg = 'WARN: cannot change service-cidr, still using ' + service_cidr() hookenv.status_set('active', msg) else: hookenv.status_set('active', 'Kubernetes master running.') @@ -305,7 +363,7 @@ def start_kube_dns(): context = { 'arch': arch(), - # The dictionary named pillar is a construct of the k8s template files. + # The dictionary named 'pillar' is a construct of the k8s template file 'pillar': { 'dns_server': get_dns_ip(), 'dns_replicas': 1,